leafShen/CodeFormer

1"""2This file is used for deploying hugging face demo:3https://huggingface.co/spaces/sczhou/CodeFormer4"""5 6import sys7sys.path.append('CodeFormer')8import os9import cv210import torch11import torch.nn.functional as F12import gradio as gr13 14from torchvision.transforms.functional import normalize15 16from basicsr.utils import imwrite, img2tensor, tensor2img17from basicsr.utils.download_util import load_file_from_url18from facelib.utils.face_restoration_helper import FaceRestoreHelper19from facelib.utils.misc import is_gray20from basicsr.archs.rrdbnet_arch import RRDBNet21from basicsr.utils.realesrgan_utils import RealESRGANer22 23from basicsr.utils.registry import ARCH_REGISTRY24 25 26os.system("pip freeze")27 28pretrain_model_url = {29    'codeformer': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth',30    'detection': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/detection_Resnet50_Final.pth',31    'parsing': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/parsing_parsenet.pth',32    'realesrgan': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/RealESRGAN_x2plus.pth'33}34# download weights35if not os.path.exists('CodeFormer/weights/CodeFormer/codeformer.pth'):36    load_file_from_url(url=pretrain_model_url['codeformer'], model_dir='CodeFormer/weights/CodeFormer', progress=True, file_name=None)37if not os.path.exists('CodeFormer/weights/facelib/detection_Resnet50_Final.pth'):38    load_file_from_url(url=pretrain_model_url['detection'], model_dir='CodeFormer/weights/facelib', progress=True, file_name=None)39if not os.path.exists('CodeFormer/weights/facelib/parsing_parsenet.pth'):40    load_file_from_url(url=pretrain_model_url['parsing'], model_dir='CodeFormer/weights/facelib', progress=True, file_name=None)41if not os.path.exists('CodeFormer/weights/realesrgan/RealESRGAN_x2plus.pth'):42    load_file_from_url(url=pretrain_model_url['realesrgan'], model_dir='CodeFormer/weights/realesrgan', progress=True, file_name=None)43 44# download images45torch.hub.download_url_to_file(46    'https://replicate.com/api/models/sczhou/codeformer/files/fa3fe3d1-76b0-4ca8-ac0d-0a925cb0ff54/06.png',47    '01.png')48torch.hub.download_url_to_file(49    'https://replicate.com/api/models/sczhou/codeformer/files/a1daba8e-af14-4b00-86a4-69cec9619b53/04.jpg',50    '02.jpg')51torch.hub.download_url_to_file(52    'https://replicate.com/api/models/sczhou/codeformer/files/542d64f9-1712-4de7-85f7-3863009a7c3d/03.jpg',53    '03.jpg')54torch.hub.download_url_to_file(55    'https://replicate.com/api/models/sczhou/codeformer/files/a11098b0-a18a-4c02-a19a-9a7045d68426/010.jpg',56    '04.jpg')57torch.hub.download_url_to_file(58    'https://replicate.com/api/models/sczhou/codeformer/files/7cf19c2c-e0cf-4712-9af8-cf5bdbb8d0ee/012.jpg',59    '05.jpg')60 61def imread(img_path):62    img = cv2.imread(img_path)63    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)64    return img65 66# set enhancer with RealESRGAN67def set_realesrgan():68    half = True if torch.cuda.is_available() else False69    model = RRDBNet(70        num_in_ch=3,71        num_out_ch=3,72        num_feat=64,73        num_block=23,74        num_grow_ch=32,75        scale=2,76    )77    upsampler = RealESRGANer(78        scale=2,79        model_path="CodeFormer/weights/realesrgan/RealESRGAN_x2plus.pth",80        model=model,81        tile=400,82        tile_pad=40,83        pre_pad=0,84        half=half,85    )86    return upsampler87 88upsampler = set_realesrgan()89device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')90codeformer_net = ARCH_REGISTRY.get("CodeFormer")(91    dim_embd=512,92    codebook_size=1024,93    n_head=8,94    n_layers=9,95    connect_list=["32", "64", "128", "256"],96).to(device)97ckpt_path = "CodeFormer/weights/CodeFormer/codeformer.pth"98checkpoint = torch.load(ckpt_path)["params_ema"]99codeformer_net.load_state_dict(checkpoint)100codeformer_net.eval()101 102os.makedirs('output', exist_ok=True)103 104def inference(image, background_enhance, face_upsample, upscale, codeformer_fidelity):105    """Run a single prediction on the model"""106    try: # global try107        # take the default setting for the demo108        has_aligned = False109        only_center_face = False110        draw_box = False111        detection_model = "retinaface_resnet50"112        print('Inp:', image, background_enhance, face_upsample, upscale, codeformer_fidelity)113 114        background_enhance = background_enhance if background_enhance is not None else True115        face_upsample = face_upsample if face_upsample is not None else True116        upscale = upscale if (upscale is not None and upscale > 0) else 2117 118        img = cv2.imread(str(image), cv2.IMREAD_COLOR)119        print('\timage size:', img.shape)120 121        upscale = int(upscale) # convert type to int122        if upscale > 4: # avoid memory exceeded due to too large upscale123            upscale = 4 124        if upscale > 2 and max(img.shape[:2])>1000: # avoid memory exceeded due to too large img resolution125            upscale = 2 126        if max(img.shape[:2]) > 1500: # avoid memory exceeded due to too large img resolution127            upscale = 1128            background_enhance = False129            face_upsample = False130 131        face_helper = FaceRestoreHelper(132            upscale,133            face_size=512,134            crop_ratio=(1, 1),135            det_model=detection_model,136            save_ext="png",137            use_parse=True,138            device=device,139        )140        bg_upsampler = upsampler if background_enhance else None141        face_upsampler = upsampler if face_upsample else None142 143        if has_aligned:144            # the input faces are already cropped and aligned145            img = cv2.resize(img, (512, 512), interpolation=cv2.INTER_LINEAR)146            face_helper.is_gray = is_gray(img, threshold=5)147            if face_helper.is_gray:148                print('\tgrayscale input: True')149            face_helper.cropped_faces = [img]150        else:151            face_helper.read_image(img)152            # get face landmarks for each face153            num_det_faces = face_helper.get_face_landmarks_5(154            only_center_face=only_center_face, resize=640, eye_dist_threshold=5155            )156            print(f'\tdetect {num_det_faces} faces')157            # align and warp each face158            face_helper.align_warp_face()159 160        # face restoration for each cropped face161        for idx, cropped_face in enumerate(face_helper.cropped_faces):162            # prepare data163            cropped_face_t = img2tensor(164                cropped_face / 255.0, bgr2rgb=True, float32=True165            )166            normalize(cropped_face_t, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True)167            cropped_face_t = cropped_face_t.unsqueeze(0).to(device)168 169            try:170                with torch.no_grad():171                    output = codeformer_net(172                        cropped_face_t, w=codeformer_fidelity, adain=True173                    )[0]174                    restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1))175                del output176                torch.cuda.empty_cache()177            except RuntimeError as error:178                print(f"Failed inference for CodeFormer: {error}")179                restored_face = tensor2img(180                    cropped_face_t, rgb2bgr=True, min_max=(-1, 1)181                )182 183            restored_face = restored_face.astype("uint8")184            face_helper.add_restored_face(restored_face)185 186        # paste_back187        if not has_aligned:188            # upsample the background189            if bg_upsampler is not None:190                # Now only support RealESRGAN for upsampling background191                bg_img = bg_upsampler.enhance(img, outscale=upscale)[0]192            else:193                bg_img = None194            face_helper.get_inverse_affine(None)195            # paste each restored face to the input image196            if face_upsample and face_upsampler is not None:197                restored_img = face_helper.paste_faces_to_input_image(198                    upsample_img=bg_img,199                    draw_box=draw_box,200                    face_upsampler=face_upsampler,201                )202            else:203                restored_img = face_helper.paste_faces_to_input_image(204                    upsample_img=bg_img, draw_box=draw_box205                )206 207        # save restored img208        save_path = f'output/out.png'209        imwrite(restored_img, str(save_path))210 211        restored_img = cv2.cvtColor(restored_img, cv2.COLOR_BGR2RGB)212        return restored_img213    except Exception as error:214        print('Global exception', error)215        return None, None216 217 218title = "CodeFormer: Robust Face Restoration and Enhancement Network"219 220description = r"""<center><img src='https://user-images.githubusercontent.com/14334509/189166076-94bb2cac-4f4e-40fb-a69f-66709e3d98f5.png' alt='CodeFormer logo'></center>221<br>222<b>Official Gradio demo</b> for <a href='https://github.com/sczhou/CodeFormer' target='_blank'><b>Towards Robust Blind Face Restoration with Codebook Lookup Transformer (NeurIPS 2022)</b></a><br>223🔥 CodeFormer is a robust face restoration algorithm for old photos or AI-generated faces.<br>224🤗 Try CodeFormer for improved stable-diffusion generation!<br>225"""226 227article = r"""228If CodeFormer is helpful, please help to ⭐ the <a href='https://github.com/sczhou/CodeFormer' target='_blank'>Github Repo</a>. Thanks! 229[![GitHub Stars](https://img.shields.io/github/stars/sczhou/CodeFormer?style=social)](https://github.com/sczhou/CodeFormer)230 231---232 233📝 **Citation**234 235If our work is useful for your research, please consider citing:236```bibtex237@inproceedings{zhou2022codeformer,238    author = {Zhou, Shangchen and Chan, Kelvin C.K. and Li, Chongyi and Loy, Chen Change},239    title = {Towards Robust Blind Face Restoration with Codebook Lookup TransFormer},240    booktitle = {NeurIPS},241    year = {2022}242}243```244 245📋 **License**246 247This project is licensed under <a rel="license" href="https://github.com/sczhou/CodeFormer/blob/master/LICENSE">S-Lab License 1.0</a>. 248Redistribution and use for non-commercial purposes should follow this license.249 250📧 **Contact**251 252If you have any questions, please feel free to reach me out at <b>shangchenzhou@gmail.com</b>.253 254🤗 **Find Me:**255<style type="text/css">256td {257    padding-right: 0px !important;258}259</style>260 261<table>262<tr>263    <td><a href="https://github.com/sczhou"><img style="margin:-0.8em 0 2em 0" src="https://img.shields.io/github/followers/sczhou?style=social" alt="Github Follow"></a></td>264    <td><a href="https://twitter.com/ShangchenZhou"><img style="margin:-0.8em 0 2em 0" src="https://img.shields.io/twitter/follow/ShangchenZhou?label=%40ShangchenZhou&style=social" alt="Twitter Follow"></a></td>265</tr>266</table>267 268<center><img src='https://api.infinitescript.com/badgen/count?name=sczhou/CodeFormer&ltext=Visitors&color=6dc9aa' alt='visitors'></center>269"""270 271demo = gr.Interface(272    inference, [273        gr.Image(type="filepath", label="Input"),274        gr.Checkbox(value=True, label="Background_Enhance"),275        gr.Checkbox(value=True, label="Face_Upsample"),276        gr.Number(value=2, label="Rescaling_Factor (up to 4)"),277        gr.Slider(0, 1, value=0.5, step=0.01, label='Codeformer_Fidelity (0 for better quality, 1 for better identity)')278    ], [279        gr.Image(type="numpy", label="Output").style(height='auto')280    ],281    title=title,282    description=description,283    article=article,       284    examples=[285        ['01.png', True, True, 2, 0.7],286        ['02.jpg', True, True, 2, 0.7],287        ['03.jpg', True, True, 2, 0.7],288        ['04.jpg', True, True, 2, 0.1],289        ['05.jpg', True, True, 2, 0.1]290      ])291 292DEBUG = os.getenv('DEBUG') == '1'293demo.queue(api_open=False, concurrency_count=2, max_size=10)294demo.launch(debug=DEBUG)