coreml-community/ControlNet-v1-1-Annotators-cpu
15
1import gradio as gr2import cv23import numpy as np4 5from annotator.util import resize_image, HWC36 7DESCRIPTION = '# ControlNet v1.1 Annotators (that runs on cpu only)'8DESCRIPTION += '\n<p>This app generates Control Image for Core ML Stable Diffusion apps such as Mochi Diffusion.</p>'9DESCRIPTION += '\n<p>HEIC image is not converted. Please use PNG or JPG image.</p>'10 11 12model_canny = None13 14 15def canny(img, res, l, h):16 img = resize_image(HWC3(img), res)17 global model_canny18 if model_canny is None:19 from annotator.canny import CannyDetector20 model_canny = CannyDetector()21 result = model_canny(img, l, h)22 return [result]23 24 25model_hed = None26 27 28def hed(img, res):29 img = resize_image(HWC3(img), res)30 global model_hed31 if model_hed is None:32 from annotator.hed import HEDdetector33 model_hed = HEDdetector()34 result = model_hed(img)35 return [result]36 37 38model_pidi = None39 40 41def pidi(img, res):42 img = resize_image(HWC3(img), res)43 global model_pidi44 if model_pidi is None:45 from annotator.pidinet import PidiNetDetector46 model_pidi = PidiNetDetector()47 result = model_pidi(img)48 return [result]49 50 51model_mlsd = None52 53 54def mlsd(img, res, thr_v, thr_d):55 img = resize_image(HWC3(img), res)56 global model_mlsd57 if model_mlsd is None:58 from annotator.mlsd import MLSDdetector59 model_mlsd = MLSDdetector()60 result = model_mlsd(img, thr_v, thr_d)61 return [result]62 63 64model_midas = None65 66 67def midas(img, res):68 img = resize_image(HWC3(img), res)69 global model_midas70 if model_midas is None:71 from annotator.midas import MidasDetector72 model_midas = MidasDetector()73 result = model_midas(img)74 return [result]75 76 77model_zoe = None78 79 80def zoe(img, res):81 img = resize_image(HWC3(img), res)82 global model_zoe83 if model_zoe is None:84 from annotator.zoe import ZoeDetector85 model_zoe = ZoeDetector()86 result = model_zoe(img)87 return [result]88 89 90model_normalbae = None91 92 93def normalbae(img, res):94 img = resize_image(HWC3(img), res)95 global model_normalbae96 if model_normalbae is None:97 from annotator.normalbae import NormalBaeDetector98 model_normalbae = NormalBaeDetector()99 result = model_normalbae(img)100 return [result]101 102 103model_dwpose = None104 105def dwpose(img, res):106 img = resize_image(HWC3(img), res)107 global model_dwpose108 if model_dwpose is None:109 from annotator.dwpose import DWposeDetector110 model_dwpose = DWposeDetector()111 result = model_dwpose(img)112 return [result]113 114 115model_openpose = None116 117 118def openpose(img, res, hand_and_face):119 img = resize_image(HWC3(img), res)120 global model_openpose121 if model_openpose is None:122 from annotator.openpose import OpenposeDetector123 model_openpose = OpenposeDetector()124 result = model_openpose(img, hand_and_face)125 return [result]126 127 128model_uniformer = None129 130 131#def uniformer(img, res):132# img = resize_image(HWC3(img), res)133# global model_uniformer134# if model_uniformer is None:135# from annotator.uniformer import UniformerDetector136# model_uniformer = UniformerDetector()137# result = model_uniformer(img)138# return [result]139 140 141model_lineart_anime = None142 143 144def lineart_anime(img, res, invert=True):145 img = resize_image(HWC3(img), res)146 global model_lineart_anime147 if model_lineart_anime is None:148 from annotator.lineart_anime import LineartAnimeDetector149 model_lineart_anime = LineartAnimeDetector()150# result = model_lineart_anime(img)151 if (invert):152 result = cv2.bitwise_not(model_lineart_anime(img))153 else:154 result = model_lineart_anime(img)155 return [result]156 157 158model_lineart = None159 160 161def lineart(img, res, coarse=False, invert=True):162 img = resize_image(HWC3(img), res)163 global model_lineart164 if model_lineart is None:165 from annotator.lineart import LineartDetector166 model_lineart = LineartDetector()167# result = model_lineart(img, coarse)168 if (invert):169 result = cv2.bitwise_not(model_lineart(img, coarse))170 else:171 result = model_lineart(img, coarse) 172 return [result]173 174 175model_oneformer_coco = None176 177 178def oneformer_coco(img, res):179 img = resize_image(HWC3(img), res)180 global model_oneformer_coco181 if model_oneformer_coco is None:182 from annotator.oneformer import OneformerCOCODetector183 model_oneformer_coco = OneformerCOCODetector()184 result = model_oneformer_coco(img)185 return [result]186 187 188model_oneformer_ade20k = None189 190 191def oneformer_ade20k(img, res):192 img = resize_image(HWC3(img), res)193 global model_oneformer_ade20k194 if model_oneformer_ade20k is None:195 from annotator.oneformer import OneformerADE20kDetector196 model_oneformer_ade20k = OneformerADE20kDetector()197 result = model_oneformer_ade20k(img)198 return [result]199 200 201model_content_shuffler = None202 203 204def content_shuffler(img, res):205 img = resize_image(HWC3(img), res)206 global model_content_shuffler207 if model_content_shuffler is None:208 from annotator.shuffle import ContentShuffleDetector209 model_content_shuffler = ContentShuffleDetector()210 result = model_content_shuffler(img)211 return [result]212 213 214model_color_shuffler = None215 216 217def color_shuffler(img, res):218 img = resize_image(HWC3(img), res)219 global model_color_shuffler220 if model_color_shuffler is None:221 from annotator.shuffle import ColorShuffleDetector222 model_color_shuffler = ColorShuffleDetector()223 result = model_color_shuffler(img)224 return [result]225 226model_inpaint = None227 228 229def inpaint(image, invert):230# color = HWC3(image["image"])231 color = HWC3(image["background"])232 if(invert):233# alpha = image["mask"][:, :, 0:1]234 alpha = image["layers"][0][:, :, 3:]235 else:236# alpha = 255 - image["mask"][:, :, 0:1]237 alpha = 255 - image["layers"][0][:, :, 3:]238 result = np.concatenate([color, alpha], axis=2)239 return [result]240 241block = gr.Blocks().queue()242with block:243 gr.Markdown(DESCRIPTION)244 with gr.Row():245 gr.Markdown("## Canny Edge")246 with gr.Row():247 with gr.Column():248# input_image = gr.Image(source='upload', type="numpy")249 input_image = gr.Image(label="Input Image", type="numpy", height=512)250 low_threshold = gr.Slider(label="low_threshold", minimum=1, maximum=255, value=100, step=1)251 high_threshold = gr.Slider(label="high_threshold", minimum=1, maximum=255, value=200, step=1)252 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)253 run_button = gr.Button("Run")254# run_button = gr.Button(label="Run")255 with gr.Column():256# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")257 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")258 run_button.click(fn=canny, inputs=[input_image, resolution, low_threshold, high_threshold], outputs=[gallery])259 260 gr.Markdown("<hr>")261 with gr.Row():262 gr.Markdown("## HED Edge "SoftEdge"")263 with gr.Row():264 with gr.Column():265# input_image = gr.Image(source='upload', type="numpy")266 input_image = gr.Image(label="Input Image", type="numpy", height=512)267 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)268 run_button = gr.Button("Run")269# run_button = gr.Button(label="Run")270 with gr.Column():271# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")272 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")273 run_button.click(fn=hed, inputs=[input_image, resolution], outputs=[gallery])274 275 gr.Markdown("<hr>")276 with gr.Row():277 gr.Markdown("## Pidi Edge "SoftEdge"")278 with gr.Row():279 with gr.Column():280# input_image = gr.Image(source='upload', type="numpy")281 input_image = gr.Image(label="Input Image", type="numpy", height=512)282 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)283 run_button = gr.Button("Run")284# run_button = gr.Button(label="Run")285 with gr.Column():286# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")287 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")288 run_button.click(fn=pidi, inputs=[input_image, resolution], outputs=[gallery])289 290 gr.Markdown("<hr>")291 with gr.Row():292 gr.Markdown("## MLSD Edge")293 with gr.Row():294 with gr.Column():295# input_image = gr.Image(source='upload', type="numpy")296 input_image = gr.Image(label="Input Image", type="numpy", height=512)297 value_threshold = gr.Slider(label="value_threshold", minimum=0.01, maximum=2.0, value=0.1, step=0.01)298 distance_threshold = gr.Slider(label="distance_threshold", minimum=0.01, maximum=20.0, value=0.1, step=0.01)299 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=384, step=64)300 run_button = gr.Button("Run")301# run_button = gr.Button(label="Run")302 with gr.Column():303# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")304 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")305 run_button.click(fn=mlsd, inputs=[input_image, resolution, value_threshold, distance_threshold], outputs=[gallery])306 307 gr.Markdown("<hr>")308 with gr.Row():309 gr.Markdown("## MIDAS Depth")310 with gr.Row():311 with gr.Column():312# input_image = gr.Image(source='upload', type="numpy")313 input_image = gr.Image(label="Input Image", type="numpy", height=512)314 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=384, step=64)315 run_button = gr.Button("Run")316# run_button = gr.Button(label="Run")317 with gr.Column():318# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")319 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")320 run_button.click(fn=midas, inputs=[input_image, resolution], outputs=[gallery])321 322 323 gr.Markdown("<hr>")324 with gr.Row():325 gr.Markdown("## Zoe Depth")326 with gr.Row():327 with gr.Column():328# input_image = gr.Image(source='upload', type="numpy")329 input_image = gr.Image(label="Input Image", type="numpy", height=512)330 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)331 run_button = gr.Button("Run")332# run_button = gr.Button(label="Run")333 with gr.Column():334# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")335 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")336 run_button.click(fn=zoe, inputs=[input_image, resolution], outputs=[gallery])337 338 gr.Markdown("<hr>")339 with gr.Row():340 gr.Markdown("## Normal Bae")341 with gr.Row():342 with gr.Column():343# input_image = gr.Image(source='upload', type="numpy")344 input_image = gr.Image(label="Input Image", type="numpy", height=512)345 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)346 run_button = gr.Button("Run")347# run_button = gr.Button(label="Run")348 with gr.Column():349# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")350 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")351 run_button.click(fn=normalbae, inputs=[input_image, resolution], outputs=[gallery])352 353 gr.Markdown("<hr>")354 with gr.Row():355 gr.Markdown("## DWPose")356 with gr.Row():357 with gr.Column():358# input_image = gr.Image(source='upload', type="numpy")359 input_image = gr.Image(label="Input Image", type="numpy", height=512)360 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)361 run_button = gr.Button("Run")362# run_button = gr.Button(label="Run")363 with gr.Column():364# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")365 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")366 run_button.click(fn=dwpose, inputs=[input_image, resolution], outputs=[gallery])367 368 gr.Markdown("<hr>")369 with gr.Row():370 gr.Markdown("## Openpose")371 with gr.Row():372 with gr.Column():373# input_image = gr.Image(source='upload', type="numpy")374 input_image = gr.Image(label="Input Image", type="numpy", height=512)375 hand_and_face = gr.Checkbox(label='Hand and Face', value=False)376 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)377 run_button = gr.Button("Run")378# run_button = gr.Button(label="Run")379 with gr.Column():380# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")381 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")382 run_button.click(fn=openpose, inputs=[input_image, resolution, hand_and_face], outputs=[gallery])383 384 gr.Markdown("<hr>")385 with gr.Row():386 gr.Markdown("## Lineart Anime \n<p>Check Invert to use with Mochi Diffusion.")387 with gr.Row():388 with gr.Column():389# input_image = gr.Image(source='upload', type="numpy")390 input_image = gr.Image(label="Input Image", type="numpy", height=512)391 invert = gr.Checkbox(label='Invert', value=True)392 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)393 run_button = gr.Button("Run")394# run_button = gr.Button(label="Run")395 with gr.Column():396# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")397 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")398 run_button.click(fn=lineart_anime, inputs=[input_image, resolution, invert], outputs=[gallery])399 400 gr.Markdown("<hr>")401 with gr.Row():402 gr.Markdown("## Lineart \n<p>Check Invert to use with Mochi Diffusion. Inverted image can also be created here for use with ControlNet Scribble.")403 with gr.Row():404 with gr.Column():405# input_image = gr.Image(source='upload', type="numpy")406 input_image = gr.Image(label="Input Image", type="numpy", height=512)407 coarse = gr.Checkbox(label='Using coarse model', value=False)408 invert = gr.Checkbox(label='Invert', value=True)409 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)410 run_button = gr.Button("Run")411# run_button = gr.Button(label="Run")412 with gr.Column():413# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")414 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")415 run_button.click(fn=lineart, inputs=[input_image, resolution, coarse, invert], outputs=[gallery])416 417 gr.Markdown("<hr>")418 with gr.Row():419 gr.Markdown("## InPaint")420 with gr.Row():421 with gr.Column():422# input_image = gr.Image(source='upload', type="numpy", tool="sketch", height=512)423 input_image = gr.ImageMask(sources="upload", type="numpy", height="auto")424 invert = gr.Checkbox(label='Invert Mask', value=False)425 run_button = gr.Button("Run")426# run_button = gr.Button(label="Run")427 with gr.Column():428# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")429 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")430 run_button.click(fn=inpaint, inputs=[input_image, invert], outputs=[gallery])431 432# with gr.Row():433# gr.Markdown("## Uniformer Segmentation")434# with gr.Row():435# with gr.Column():436# input_image = gr.Image(source='upload', type="numpy")437# resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)438# run_button = gr.Button(label="Run")439# with gr.Column():440# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")441# run_button.click(fn=uniformer, inputs=[input_image, resolution], outputs=[gallery])442 443 gr.Markdown("<hr>")444 with gr.Row():445 gr.Markdown("## Oneformer COCO Segmentation")446 with gr.Row():447 with gr.Column():448# input_image = gr.Image(source='upload', type="numpy")449 input_image = gr.Image(label="Input Image", type="numpy", height=512)450 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)451 run_button = gr.Button("Run")452# run_button = gr.Button(label="Run")453 with gr.Column():454# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")455 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")456 run_button.click(fn=oneformer_coco, inputs=[input_image, resolution], outputs=[gallery])457 458 gr.Markdown("<hr>")459 with gr.Row():460 gr.Markdown("## Oneformer ADE20K Segmentation")461 with gr.Row():462 with gr.Column():463# input_image = gr.Image(source='upload', type="numpy")464 input_image = gr.Image(label="Input Image", type="numpy", height=512)465 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=640, step=64)466 run_button = gr.Button("Run")467# run_button = gr.Button(label="Run")468 with gr.Column():469# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")470 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")471 run_button.click(fn=oneformer_ade20k, inputs=[input_image, resolution], outputs=[gallery])472 473 gr.Markdown("<hr>")474 with gr.Row():475 gr.Markdown("## Content Shuffle")476 with gr.Row():477 with gr.Column():478# input_image = gr.Image(source='upload', type="numpy")479 input_image = gr.Image(label="Input Image", type="numpy", height=512)480 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)481 run_button = gr.Button("Run")482# run_button = gr.Button(label="Run")483 with gr.Column():484# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")485 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")486 run_button.click(fn=content_shuffler, inputs=[input_image, resolution], outputs=[gallery])487 488 gr.Markdown("<hr>")489 with gr.Row():490 gr.Markdown("## Color Shuffle")491 with gr.Row():492 with gr.Column():493# input_image = gr.Image(source='upload', type="numpy")494 input_image = gr.Image(label="Input Image", type="numpy", height=512)495 resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64)496 run_button = gr.Button("Run")497# run_button = gr.Button(label="Run")498 with gr.Column():499# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto")500 gallery = gr.Gallery(label="Generated images", show_label=False, height="auto")501 run_button.click(fn=color_shuffler, inputs=[input_image, resolution], outputs=[gallery])502 503 504block.launch(server_name='0.0.0.0')505