MinhQuangIntercom/tryon

1import numpy as np2import cv23from PIL import Image, ImageDraw4 5label_map = {6    "background": 0,7    "hat": 1,8    "hair": 2,9    "sunglasses": 3,10    "upper_clothes": 4,11    "skirt": 5,12    "pants": 6,13    "dress": 7,14    "belt": 8,15    "left_shoe": 9,16    "right_shoe": 10,17    "head": 11,18    "left_leg": 12,19    "right_leg": 13,20    "left_arm": 14,21    "right_arm": 15,22    "bag": 16,23    "scarf": 17,24}25 26def extend_arm_mask(wrist, elbow, scale):27  wrist = elbow + scale * (wrist - elbow)28  return wrist29 30def hole_fill(img):31    img = np.pad(img[1:-1, 1:-1], pad_width = 1, mode = 'constant', constant_values=0)32    img_copy = img.copy()33    mask = np.zeros((img.shape[0] + 2, img.shape[1] + 2), dtype=np.uint8)34 35    cv2.floodFill(img, mask, (0, 0), 255)36    img_inverse = cv2.bitwise_not(img)37    dst = cv2.bitwise_or(img_copy, img_inverse)38    return dst39 40def refine_mask(mask):41    contours, hierarchy = cv2.findContours(mask.astype(np.uint8),42                                           cv2.RETR_CCOMP, cv2.CHAIN_APPROX_TC89_L1)43    area = []44    for j in range(len(contours)):45        a_d = cv2.contourArea(contours[j], True)46        area.append(abs(a_d))47    refine_mask = np.zeros_like(mask).astype(np.uint8)48    if len(area) != 0:49        i = area.index(max(area))50        cv2.drawContours(refine_mask, contours, i, color=255, thickness=-1)51 52    return refine_mask53 54def get_mask_location(model_type, category, model_parse: Image.Image, keypoint: dict, width=384,height=512):55    im_parse = model_parse.resize((width, height), Image.NEAREST)56    parse_array = np.array(im_parse)57 58    if model_type == 'hd':59        arm_width = 6060    elif model_type == 'dc':61        arm_width = 4562    else:63        raise ValueError("model_type must be \'hd\' or \'dc\'!")64 65    parse_head = (parse_array == 1).astype(np.float32) + \66                 (parse_array == 3).astype(np.float32) + \67                 (parse_array == 11).astype(np.float32)68 69    parser_mask_fixed = (parse_array == label_map["left_shoe"]).astype(np.float32) + \70                        (parse_array == label_map["right_shoe"]).astype(np.float32) + \71                        (parse_array == label_map["hat"]).astype(np.float32) + \72                        (parse_array == label_map["sunglasses"]).astype(np.float32) + \73                        (parse_array == label_map["bag"]).astype(np.float32)74 75    parser_mask_changeable = (parse_array == label_map["background"]).astype(np.float32)76 77    arms_left = (parse_array == 14).astype(np.float32)78    arms_right = (parse_array == 15).astype(np.float32)79 80    if category == 'dresses':81        parse_mask = (parse_array == 7).astype(np.float32) + \82                     (parse_array == 4).astype(np.float32) + \83                     (parse_array == 5).astype(np.float32) + \84                     (parse_array == 6).astype(np.float32)85 86        parser_mask_changeable += np.logical_and(parse_array, np.logical_not(parser_mask_fixed))87 88    elif category == 'upper_body':89        parse_mask = (parse_array == 4).astype(np.float32) + (parse_array == 7).astype(np.float32)90        parser_mask_fixed_lower_cloth = (parse_array == label_map["skirt"]).astype(np.float32) + \91                                        (parse_array == label_map["pants"]).astype(np.float32)92        parser_mask_fixed += parser_mask_fixed_lower_cloth93        parser_mask_changeable += np.logical_and(parse_array, np.logical_not(parser_mask_fixed))94    elif category == 'lower_body':95        parse_mask = (parse_array == 6).astype(np.float32) + \96                     (parse_array == 12).astype(np.float32) + \97                     (parse_array == 13).astype(np.float32) + \98                     (parse_array == 5).astype(np.float32)99        parser_mask_fixed += (parse_array == label_map["upper_clothes"]).astype(np.float32) + \100                             (parse_array == 14).astype(np.float32) + \101                             (parse_array == 15).astype(np.float32)102        parser_mask_changeable += np.logical_and(parse_array, np.logical_not(parser_mask_fixed))103    else:104        raise NotImplementedError105 106    # Load pose points107    pose_data = keypoint["pose_keypoints_2d"]108    pose_data = np.array(pose_data)109    pose_data = pose_data.reshape((-1, 2))110 111    im_arms_left = Image.new('L', (width, height))112    im_arms_right = Image.new('L', (width, height))113    arms_draw_left = ImageDraw.Draw(im_arms_left)114    arms_draw_right = ImageDraw.Draw(im_arms_right)115    if category == 'dresses' or category == 'upper_body':116        shoulder_right = np.multiply(tuple(pose_data[2][:2]), height / 512.0)117        shoulder_left = np.multiply(tuple(pose_data[5][:2]), height / 512.0)118        elbow_right = np.multiply(tuple(pose_data[3][:2]), height / 512.0)119        elbow_left = np.multiply(tuple(pose_data[6][:2]), height / 512.0)120        wrist_right = np.multiply(tuple(pose_data[4][:2]), height / 512.0)121        wrist_left = np.multiply(tuple(pose_data[7][:2]), height / 512.0)122        ARM_LINE_WIDTH = int(arm_width / 512 * height)123        size_left = [shoulder_left[0] - ARM_LINE_WIDTH // 2, shoulder_left[1] - ARM_LINE_WIDTH // 2, shoulder_left[0] + ARM_LINE_WIDTH // 2, shoulder_left[1] + ARM_LINE_WIDTH // 2]124        size_right = [shoulder_right[0] - ARM_LINE_WIDTH // 2, shoulder_right[1] - ARM_LINE_WIDTH // 2, shoulder_right[0] + ARM_LINE_WIDTH // 2,125                      shoulder_right[1] + ARM_LINE_WIDTH // 2]126        127 128        if wrist_right[0] <= 1. and wrist_right[1] <= 1.:129            im_arms_right = arms_right130        else:131            wrist_right = extend_arm_mask(wrist_right, elbow_right, 1.2)132            arms_draw_right.line(np.concatenate((shoulder_right, elbow_right, wrist_right)).astype(np.uint16).tolist(), 'white', ARM_LINE_WIDTH, 'curve')133            arms_draw_right.arc(size_right, 0, 360, 'white', ARM_LINE_WIDTH // 2)134 135        if wrist_left[0] <= 1. and wrist_left[1] <= 1.:136            im_arms_left = arms_left137        else:138            wrist_left = extend_arm_mask(wrist_left, elbow_left, 1.2)139            arms_draw_left.line(np.concatenate((wrist_left, elbow_left, shoulder_left)).astype(np.uint16).tolist(), 'white', ARM_LINE_WIDTH, 'curve')140            arms_draw_left.arc(size_left, 0, 360, 'white', ARM_LINE_WIDTH // 2)141 142        hands_left = np.logical_and(np.logical_not(im_arms_left), arms_left)143        hands_right = np.logical_and(np.logical_not(im_arms_right), arms_right)144        parser_mask_fixed += hands_left + hands_right145 146    parser_mask_fixed = np.logical_or(parser_mask_fixed, parse_head)147    parse_mask = cv2.dilate(parse_mask, np.ones((5, 5), np.uint16), iterations=5)148    if category == 'dresses' or category == 'upper_body':149        neck_mask = (parse_array == 18).astype(np.float32)150        neck_mask = cv2.dilate(neck_mask, np.ones((5, 5), np.uint16), iterations=1)151        neck_mask = np.logical_and(neck_mask, np.logical_not(parse_head))152        parse_mask = np.logical_or(parse_mask, neck_mask)153        arm_mask = cv2.dilate(np.logical_or(im_arms_left, im_arms_right).astype('float32'), np.ones((5, 5), np.uint16), iterations=4)154        parse_mask += np.logical_or(parse_mask, arm_mask)155 156    parse_mask = np.logical_and(parser_mask_changeable, np.logical_not(parse_mask))157 158    parse_mask_total = np.logical_or(parse_mask, parser_mask_fixed)159    inpaint_mask = 1 - parse_mask_total160    img = np.where(inpaint_mask, 255, 0)161    dst = hole_fill(img.astype(np.uint8))162    dst = refine_mask(dst)163    inpaint_mask = dst / 255 * 1164    mask = Image.fromarray(inpaint_mask.astype(np.uint8) * 255)165    mask_gray = Image.fromarray(inpaint_mask.astype(np.uint8) * 127)166 167    return mask, mask_gray168