|
from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL |
|
from diffusers.utils import load_image |
|
from PIL import Image |
|
import torch |
|
import numpy as np |
|
import cv2 |
|
import gradio as gr |
|
|
|
controlnet_conditioning_scale = 0.5 |
|
|
|
controlnet = ControlNetModel.from_pretrained( |
|
"diffusers/controlnet-canny-sdxl-1.0", |
|
torch_dtype=torch.float16 |
|
) |
|
|
|
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16) |
|
pipe = StableDiffusionXLControlNetPipeline.from_pretrained( |
|
"mann-e/Mann-E_Dreams", |
|
controlnet=controlnet, |
|
vae=vae, |
|
torch_dtype=torch.float16, |
|
) |
|
pipe.enable_model_cpu_offload() |
|
|
|
low_threshold = 100 |
|
high_threshold = 200 |
|
|
|
def get_canny_filter(image): |
|
|
|
if not isinstance(image, np.ndarray): |
|
image = np.array(image) |
|
|
|
image = cv2.Canny(image, low_threshold, high_threshold) |
|
image = image[:, :, None] |
|
image = np.concatenate([image, image, image], axis=2) |
|
canny_image = Image.fromarray(image) |
|
return canny_image |
|
|
|
def process(input_image, prompt): |
|
canny_image = get_canny_filter(input_image) |
|
images = pipe( |
|
prompt,image=canny_image, controlnet_conditioning_scale=controlnet_conditioning_scale, |
|
).images |
|
|
|
return [canny_image,images[0]] |
|
|
|
block = gr.Blocks().queue() |
|
|
|
with block: |
|
gr.Markdown("## ControlNet SDXL Canny") |
|
gr.HTML(''' |
|
<p style="margin-bottom: 10px; font-size: 94%"> |
|
This is a demo for ControlNet Mann-E Dreams (SDXL based), which is a neural network structure to control Stable Diffusion XL model by adding extra condition such as canny edge detection. |
|
</p> |
|
''') |
|
gr.HTML("<p>You can duplicate this Space to run it privately without a queue and load additional checkpoints. : <a style='display:inline-block' href='https://huggingface.co/spaces/RamAnanth1/controlnet-sdxl-canny?duplicate=true'><img src='https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14' alt='Duplicate Space'></a> </p>") |
|
with gr.Row(): |
|
with gr.Column(): |
|
input_image = gr.Image(source='upload', type="numpy") |
|
prompt = gr.Textbox(label="Prompt") |
|
run_button = gr.Button(label="Run") |
|
|
|
|
|
with gr.Column(): |
|
result_gallery = gr.Gallery(label='Output', show_label=False, elem_id="gallery").style(grid_cols=2, height='auto') |
|
ips = [input_image, prompt] |
|
run_button.click(fn=process, inputs=ips, outputs=[result_gallery]) |
|
|
|
block.launch(debug = True, show_error=True) |
