AP123 multimodalart HF staff commited on
Commit
4984c7e
β€’
1 Parent(s): 4783dcc

high quality (#17)

Browse files

- High quality (0f59bfafcd98b4b4a93790a81be7de43d068135f)
- Update app.py (09d2a3966a11c5d694bde9833a8a514a0e291a14)


Co-authored-by: ApolinΓ‘rio from multimodal AI art <[email protected]>

Files changed (4) hide show
  1. app.py +75 -23
  2. checkers_mid.jpg +0 -0
  3. funky.jpeg +0 -0
  4. ultra_checkers.png +0 -0
app.py CHANGED
@@ -8,6 +8,8 @@ from diffusers import (
8
  StableDiffusionControlNetPipeline,
9
  ControlNetModel,
10
  StableDiffusionLatentUpscalePipeline,
 
 
11
  DPMSolverMultistepScheduler, # <-- Added import
12
  EulerDiscreteScheduler # <-- Added import
13
  )
@@ -18,17 +20,22 @@ from illusion_style import css
18
  BASE_MODEL = "SG161222/Realistic_Vision_V5.1_noVAE"
19
 
20
  # Initialize both pipelines
21
- vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse")
22
  #init_pipe = DiffusionPipeline.from_pretrained("SG161222/Realistic_Vision_V5.1_noVAE", torch_dtype=torch.float16)
23
- controlnet = ControlNetModel.from_pretrained("monster-labs/control_v1p_sd15_qrcode_monster")#, torch_dtype=torch.float16)
24
  main_pipe = StableDiffusionControlNetPipeline.from_pretrained(
25
  BASE_MODEL,
26
  controlnet=controlnet,
27
  vae=vae,
28
  safety_checker=None,
29
- #torch_dtype=torch.float16,
30
  ).to("cuda")
 
 
 
31
  #model_id = "stabilityai/sd-x2-latent-upscaler"
 
 
32
  #upscaler = StableDiffusionLatentUpscalePipeline.from_pretrained(model_id, torch_dtype=torch.float16)
33
  #upscaler.to("cuda")
34
 
@@ -55,6 +62,31 @@ def center_crop_resize(img, output_size=(512, 512)):
55
 
56
  return img
57
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  # Inference function
59
  def inference(
60
  control_image: Image.Image,
@@ -62,6 +94,9 @@ def inference(
62
  negative_prompt: str,
63
  guidance_scale: float = 8.0,
64
  controlnet_conditioning_scale: float = 1,
 
 
 
65
  seed: int = -1,
66
  sampler = "DPM++ Karras SDE",
67
  progress = gr.Progress(track_tqdm=True)
@@ -73,65 +108,82 @@ def inference(
73
  #init_image = init_pipe(prompt).images[0]
74
 
75
  # Rest of your existing code
76
- control_image = center_crop_resize(control_image)
77
  main_pipe.scheduler = SAMPLER_MAP[sampler](main_pipe.scheduler.config)
78
  generator = torch.manual_seed(seed) if seed != -1 else torch.Generator()
79
 
80
  out = main_pipe(
81
  prompt=prompt,
82
  negative_prompt=negative_prompt,
83
- image=control_image,
84
- #control_image=control_image,
85
  guidance_scale=float(guidance_scale),
86
  controlnet_conditioning_scale=float(controlnet_conditioning_scale),
87
  generator=generator,
88
- #strength=strength,
89
- num_inference_steps=30,
90
- #output_type="latent"
91
- ).images[0]
92
-
93
- return out, gr.update(visible=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94
 
95
  with gr.Blocks(css=css) as app:
96
  gr.Markdown(
97
  '''
98
- <center><h1>Illusion Diffusion πŸŒ€</h1></span>
99
- <span font-size:16px;">Generate stunning illusion artwork with Stable Diffusion</span>
100
  </center>
101
 
102
  A space by AP [Follow me on Twitter](https://twitter.com/angrypenguinPNG)
103
 
104
  This project works by using [Monster Labs QR Control Net](https://huggingface.co/monster-labs/control_v1p_sd15_qrcode_monster).
105
- Given a prompt and your pattern, we use a QR code conditioned controlnet to create a stunning illusion! Credit to: MrUgleh (https://twitter.com/MrUgleh) for discovering the workflow :)
106
-
107
  '''
108
  )
109
 
110
  with gr.Row():
111
  with gr.Column():
112
  control_image = gr.Image(label="Input Illusion", type="pil", elem_id="control_image")
113
- controlnet_conditioning_scale = gr.Slider(minimum=0.0, maximum=5.0, step=0.01, value=0.8, label="Illusion strength", info="ControlNet conditioning scale", elem_id="illusion_strength")
114
- gr.Examples(examples=["checkers.png", "pattern.png", "spiral.jpeg"], inputs=control_image)
115
  prompt = gr.Textbox(label="Prompt", elem_id="prompt")
116
  negative_prompt = gr.Textbox(label="Negative Prompt", value="low quality", elem_id="negative_prompt")
117
  with gr.Accordion(label="Advanced Options", open=False):
118
- #strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, value=0.9, label="Strength")
119
  guidance_scale = gr.Slider(minimum=0.0, maximum=50.0, step=0.25, value=7.5, label="Guidance Scale")
120
  sampler = gr.Dropdown(choices=list(SAMPLER_MAP.keys()), value="Euler")
121
- seed = gr.Slider(minimum=-1, maximum=9999999999, step=1, value=2313123, label="Seed", randomize=True)
 
 
 
122
  run_btn = gr.Button("Run")
123
  with gr.Column():
124
- result_image = gr.Image(label="Illusion Diffusion Output", elem_id="output")
125
  with gr.Group(elem_id="share-btn-container", visible=False) as share_group:
126
  community_icon = gr.HTML(community_icon_html)
127
  loading_icon = gr.HTML(loading_icon_html)
128
  share_button = gr.Button("Share to community", elem_id="share-btn")
129
 
130
  history = show_gallery_history()
131
-
132
  run_btn.click(
133
  inference,
134
- inputs=[control_image, prompt, negative_prompt, guidance_scale, controlnet_conditioning_scale, seed, sampler],
135
  outputs=[result_image, share_group]
136
  ).then(
137
  fn=fetch_gallery_history, inputs=[prompt, result_image], outputs=history, queue=False
 
8
  StableDiffusionControlNetPipeline,
9
  ControlNetModel,
10
  StableDiffusionLatentUpscalePipeline,
11
+ StableDiffusionImg2ImgPipeline,
12
+ StableDiffusionControlNetImg2ImgPipeline,
13
  DPMSolverMultistepScheduler, # <-- Added import
14
  EulerDiscreteScheduler # <-- Added import
15
  )
 
20
  BASE_MODEL = "SG161222/Realistic_Vision_V5.1_noVAE"
21
 
22
  # Initialize both pipelines
23
+ vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse", torch_dtype=torch.float16)
24
  #init_pipe = DiffusionPipeline.from_pretrained("SG161222/Realistic_Vision_V5.1_noVAE", torch_dtype=torch.float16)
25
+ controlnet = ControlNetModel.from_pretrained("monster-labs/control_v1p_sd15_qrcode_monster", torch_dtype=torch.float16)#, torch_dtype=torch.float16)
26
  main_pipe = StableDiffusionControlNetPipeline.from_pretrained(
27
  BASE_MODEL,
28
  controlnet=controlnet,
29
  vae=vae,
30
  safety_checker=None,
31
+ torch_dtype=torch.float16,
32
  ).to("cuda")
33
+ #main_pipe.unet = torch.compile(main_pipe.unet, mode="reduce-overhead", fullgraph=True)
34
+ #main_pipe.unet.to(memory_format=torch.channels_last)
35
+ #main_pipe.unet = torch.compile(main_pipe.unet, mode="reduce-overhead", fullgraph=True)
36
  #model_id = "stabilityai/sd-x2-latent-upscaler"
37
+ image_pipe = StableDiffusionControlNetImg2ImgPipeline.from_pretrained(BASE_MODEL, unet=main_pipe.unet, vae=vae, controlnet=controlnet, safety_checker=None, torch_dtype=torch.float16).to("cuda")
38
+ #image_pipe.unet = torch.compile(image_pipe.unet, mode="reduce-overhead", fullgraph=True)
39
  #upscaler = StableDiffusionLatentUpscalePipeline.from_pretrained(model_id, torch_dtype=torch.float16)
40
  #upscaler.to("cuda")
41
 
 
62
 
63
  return img
64
 
65
+ def common_upscale(samples, width, height, upscale_method, crop=False):
66
+ if crop == "center":
67
+ old_width = samples.shape[3]
68
+ old_height = samples.shape[2]
69
+ old_aspect = old_width / old_height
70
+ new_aspect = width / height
71
+ x = 0
72
+ y = 0
73
+ if old_aspect > new_aspect:
74
+ x = round((old_width - old_width * (new_aspect / old_aspect)) / 2)
75
+ elif old_aspect < new_aspect:
76
+ y = round((old_height - old_height * (old_aspect / new_aspect)) / 2)
77
+ s = samples[:,:,y:old_height-y,x:old_width-x]
78
+ else:
79
+ s = samples
80
+
81
+ return torch.nn.functional.interpolate(s, size=(height, width), mode=upscale_method)
82
+
83
+ def upscale(samples, upscale_method, scale_by):
84
+ #s = samples.copy()
85
+ width = round(samples["images"].shape[3] * scale_by)
86
+ height = round(samples["images"].shape[2] * scale_by)
87
+ s = common_upscale(samples["images"], width, height, upscale_method, "disabled")
88
+ return (s)
89
+
90
  # Inference function
91
  def inference(
92
  control_image: Image.Image,
 
94
  negative_prompt: str,
95
  guidance_scale: float = 8.0,
96
  controlnet_conditioning_scale: float = 1,
97
+ control_guidance_start: float = 1,
98
+ control_guidance_end: float = 1,
99
+ upscaler_strength: float = 0.5,
100
  seed: int = -1,
101
  sampler = "DPM++ Karras SDE",
102
  progress = gr.Progress(track_tqdm=True)
 
108
  #init_image = init_pipe(prompt).images[0]
109
 
110
  # Rest of your existing code
111
+ control_image_small = center_crop_resize(control_image)
112
  main_pipe.scheduler = SAMPLER_MAP[sampler](main_pipe.scheduler.config)
113
  generator = torch.manual_seed(seed) if seed != -1 else torch.Generator()
114
 
115
  out = main_pipe(
116
  prompt=prompt,
117
  negative_prompt=negative_prompt,
118
+ image=control_image_small,
 
119
  guidance_scale=float(guidance_scale),
120
  controlnet_conditioning_scale=float(controlnet_conditioning_scale),
121
  generator=generator,
122
+ control_guidance_start=float(control_guidance_start),
123
+ control_guidance_end=float(control_guidance_end),
124
+ num_inference_steps=15,
125
+ output_type="latent"
126
+ )
127
+ control_image_large = center_crop_resize(control_image, (1024, 1024))
128
+ upscaled_latents = upscale(out, "nearest-exact", 2)
129
+ out_image = image_pipe(
130
+ prompt=prompt,
131
+ negative_prompt=negative_prompt,
132
+ control_image=control_image_large,
133
+ image=upscaled_latents,
134
+ guidance_scale=float(guidance_scale),
135
+ generator=generator,
136
+ num_inference_steps=20,
137
+ strength=upscaler_strength,
138
+ control_guidance_start=float(control_guidance_start),
139
+ control_guidance_end=float(control_guidance_end),
140
+ controlnet_conditioning_scale=float(controlnet_conditioning_scale)
141
+ )
142
+ return out_image["images"][0], gr.update(visible=True)
143
+
144
+ #return out
145
 
146
  with gr.Blocks(css=css) as app:
147
  gr.Markdown(
148
  '''
149
+ <center><h1>Illusion Diffusion HQ πŸŒ€</h1></span>
150
+ <span font-size:16px;">Generate stunning high quality illusion artwork with Stable Diffusion</span>
151
  </center>
152
 
153
  A space by AP [Follow me on Twitter](https://twitter.com/angrypenguinPNG)
154
 
155
  This project works by using [Monster Labs QR Control Net](https://huggingface.co/monster-labs/control_v1p_sd15_qrcode_monster).
156
+ Given a prompt and your pattern, we use a QR code conditioned controlnet to create a stunning illusion! Credit to: [MrUgleh](https://twitter.com/MrUgleh) for discovering the workflow :)
 
157
  '''
158
  )
159
 
160
  with gr.Row():
161
  with gr.Column():
162
  control_image = gr.Image(label="Input Illusion", type="pil", elem_id="control_image")
163
+ controlnet_conditioning_scale = gr.Slider(minimum=0.0, maximum=5.0, step=0.01, value=0.8, label="Illusion strength", elem_id="illusion_strength", info="ControlNet conditioning scale")
164
+ gr.Examples(examples=["checkers.png", "checkers_mid.jpg", "pattern.png", "ultra_checkers.png", "spiral.jpeg", "funky.jpeg" ], inputs=control_image)
165
  prompt = gr.Textbox(label="Prompt", elem_id="prompt")
166
  negative_prompt = gr.Textbox(label="Negative Prompt", value="low quality", elem_id="negative_prompt")
167
  with gr.Accordion(label="Advanced Options", open=False):
 
168
  guidance_scale = gr.Slider(minimum=0.0, maximum=50.0, step=0.25, value=7.5, label="Guidance Scale")
169
  sampler = gr.Dropdown(choices=list(SAMPLER_MAP.keys()), value="Euler")
170
+ control_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=0, label="Start of ControlNet")
171
+ control_end = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=1, label="End of ControlNet")
172
+ strength = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, value=1, label="Strength of the upscaler")
173
+ seed = gr.Slider(minimum=-1, maximum=9999999999, step=1, value=-1, label="Seed", info="-1 means random seed", randomize=True)
174
  run_btn = gr.Button("Run")
175
  with gr.Column():
176
+ result_image = gr.Image(label="Illusion Diffusion Output", interactive=False, elem_id="output")
177
  with gr.Group(elem_id="share-btn-container", visible=False) as share_group:
178
  community_icon = gr.HTML(community_icon_html)
179
  loading_icon = gr.HTML(loading_icon_html)
180
  share_button = gr.Button("Share to community", elem_id="share-btn")
181
 
182
  history = show_gallery_history()
183
+
184
  run_btn.click(
185
  inference,
186
+ inputs=[control_image, prompt, negative_prompt, guidance_scale, controlnet_conditioning_scale, control_start, control_end, strength, seed, sampler],
187
  outputs=[result_image, share_group]
188
  ).then(
189
  fn=fetch_gallery_history, inputs=[prompt, result_image], outputs=history, queue=False
checkers_mid.jpg ADDED
funky.jpeg ADDED
ultra_checkers.png ADDED