upd

app.py

@@ -1,6 +1,17 @@
 import gradio as gr
 from gradio_image_annotation import image_annotator
+from diffusers import StableDiffusionPipeline
 import os
+import torch
+
+
+# Load model
+pipe = StableDiffusionPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5",
+    torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
+).to("cuda" if torch.cuda.is_available() else "cpu")
+pipe.safety_checker = None
+
 example_annotation = {
     "image": os.path.join(os.path.dirname(__file__), "background.png"),
     "boxes": [],
@@ -12,23 +23,34 @@ def get_boxes_json(annotations):
     width = image.shape[1]
     height = image.shape[0]
     boxes = annotations["boxes"]
+    prompt_final = [[]]
     for box in boxes:
         box["xmin"] = box["xmin"] / width
         box["xmax"] = box["xmax"] / width
         box["ymin"] = box["ymin"] / height
         box["ymax"] = box["ymax"] / height
-    return annotations["boxes"]
+        prompt_final[0].append(box["label"])
+    # import pdb; pdb.set_trace()
+    prompt = ", ".join(prompt_final[0])
+    image = pipe(prompt).images[0]
+    return image
+    # return annotations["boxes"]
 
 with gr.Blocks() as demo:
-    with gr.Tab("DreamRenderer", id="DreamRenderer"):
-        annotator = image_annotator(example_annotation,
-                                    height=512,
-                                    width=512
-                                    )
-
-        button_get = gr.Button("Get bounding boxes")
-        json_boxes = gr.JSON()
-        button_get.click(get_boxes_json, annotator, json_boxes)
+   with gr.Tab("DreamRenderer", id="DreamRenderer"):
+        with gr.Row():
+            with gr.Column(scale=1):
+                annotator = image_annotator(
+                    example_annotation,
+                    height=512,
+                    width=512
+                )
+            with gr.Column(scale=1):
+                generated_image = gr.Image(label="Generated Image", height=512, width=512)
+
+        button_get = gr.Button("Generation")
+        button_get.click(get_boxes_json, inputs=annotator, outputs=generated_image)
+
 
 if __name__ == "__main__":
     demo.launch()