RabbitRUI/ruispace
0
1import os2import cv23import time4import glob5import argparse6import scipy7import numpy as np8from PIL import Image9from tqdm import tqdm10from itertools import cycle11 12from torch.multiprocessing import Pool, Process, set_start_method13 14 15"""16brief: face alignment with FFHQ method (https://github.com/NVlabs/ffhq-dataset)17author: lzhbrian (https://lzhbrian.me)18date: 2020.1.519note: code is heavily borrowed from 20 https://github.com/NVlabs/ffhq-dataset21 http://dlib.net/face_landmark_detection.py.html22requirements:23 apt install cmake24 conda install Pillow numpy scipy25 pip install dlib26 # download face landmark model from: 27 # http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz228"""29 30import numpy as np31from PIL import Image32import dlib33 34 35class Croper:36 def __init__(self, path_of_lm):37 # download model from: http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz238 self.predictor = dlib.shape_predictor(path_of_lm)39 40 def get_landmark(self, img_np):41 """get landmark with dlib42 :return: np.array shape=(68, 2)43 """44 detector = dlib.get_frontal_face_detector()45 dets = detector(img_np, 1)46 # print("Number of faces detected: {}".format(len(dets)))47 # for k, d in enumerate(dets):48 if len(dets) == 0:49 return None50 d = dets[0]51 # Get the landmarks/parts for the face in box d.52 shape = self.predictor(img_np, d)53 # print("Part 0: {}, Part 1: {} ...".format(shape.part(0), shape.part(1)))54 t = list(shape.parts())55 a = []56 for tt in t:57 a.append([tt.x, tt.y])58 lm = np.array(a)59 # lm is a shape=(68,2) np.array60 return lm61 62 def align_face(self, img, lm, output_size=1024):63 """64 :param filepath: str65 :return: PIL Image66 """67 lm_chin = lm[0: 17] # left-right68 lm_eyebrow_left = lm[17: 22] # left-right69 lm_eyebrow_right = lm[22: 27] # left-right70 lm_nose = lm[27: 31] # top-down71 lm_nostrils = lm[31: 36] # top-down72 lm_eye_left = lm[36: 42] # left-clockwise73 lm_eye_right = lm[42: 48] # left-clockwise74 lm_mouth_outer = lm[48: 60] # left-clockwise75 lm_mouth_inner = lm[60: 68] # left-clockwise76 77 # Calculate auxiliary vectors.78 eye_left = np.mean(lm_eye_left, axis=0)79 eye_right = np.mean(lm_eye_right, axis=0)80 eye_avg = (eye_left + eye_right) * 0.581 eye_to_eye = eye_right - eye_left82 mouth_left = lm_mouth_outer[0]83 mouth_right = lm_mouth_outer[6]84 mouth_avg = (mouth_left + mouth_right) * 0.585 eye_to_mouth = mouth_avg - eye_avg86 87 # Choose oriented crop rectangle.88 x = eye_to_eye - np.flipud(eye_to_mouth) * [-1, 1] # 双眼差与双嘴差相加89 x /= np.hypot(*x) # hypot函数计算直角三角形的斜边长，用斜边长对三角形两条直边做归一化90 x *= max(np.hypot(*eye_to_eye) * 2.0, np.hypot(*eye_to_mouth) * 1.8) # 双眼差和眼嘴差，选较大的作为基准尺度91 y = np.flipud(x) * [-1, 1]92 c = eye_avg + eye_to_mouth * 0.193 quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y]) # 定义四边形，以面部基准位置为中心上下左右平移得到四个顶点94 qsize = np.hypot(*x) * 2 # 定义四边形的大小（边长），为基准尺度的2倍95 96 # Shrink.97 # 如果计算出的四边形太大了，就按比例缩小它98 shrink = int(np.floor(qsize / output_size * 0.5))99 if shrink > 1:100 rsize = (int(np.rint(float(img.size[0]) / shrink)), int(np.rint(float(img.size[1]) / shrink)))101 img = img.resize(rsize, Image.ANTIALIAS)102 quad /= shrink103 qsize /= shrink104 105 # Crop.106 border = max(int(np.rint(qsize * 0.1)), 3)107 crop = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))),108 int(np.ceil(max(quad[:, 1]))))109 crop = (max(crop[0] - border, 0), max(crop[1] - border, 0), min(crop[2] + border, img.size[0]),110 min(crop[3] + border, img.size[1]))111 if crop[2] - crop[0] < img.size[0] or crop[3] - crop[1] < img.size[1]:112 # img = img.crop(crop)113 quad -= crop[0:2]114 115 # Pad.116 pad = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))),117 int(np.ceil(max(quad[:, 1]))))118 pad = (max(-pad[0] + border, 0), max(-pad[1] + border, 0), max(pad[2] - img.size[0] + border, 0),119 max(pad[3] - img.size[1] + border, 0))120 # if enable_padding and max(pad) > border - 4:121 # pad = np.maximum(pad, int(np.rint(qsize * 0.3)))122 # img = np.pad(np.float32(img), ((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)), 'reflect')123 # h, w, _ = img.shape124 # y, x, _ = np.ogrid[:h, :w, :1]125 # mask = np.maximum(1.0 - np.minimum(np.float32(x) / pad[0], np.float32(w - 1 - x) / pad[2]),126 # 1.0 - np.minimum(np.float32(y) / pad[1], np.float32(h - 1 - y) / pad[3]))127 # blur = qsize * 0.02128 # img += (scipy.ndimage.gaussian_filter(img, [blur, blur, 0]) - img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0)129 # img += (np.median(img, axis=(0, 1)) - img) * np.clip(mask, 0.0, 1.0)130 # img = Image.fromarray(np.uint8(np.clip(np.rint(img), 0, 255)), 'RGB')131 # quad += pad[:2]132 133 # Transform.134 quad = (quad + 0.5).flatten()135 lx = max(min(quad[0], quad[2]), 0)136 ly = max(min(quad[1], quad[7]), 0)137 rx = min(max(quad[4], quad[6]), img.size[0])138 ry = min(max(quad[3], quad[5]), img.size[0])139 # img = img.transform((transform_size, transform_size), Image.QUAD, (quad + 0.5).flatten(),140 # Image.BILINEAR)141 # if output_size < transform_size:142 # img = img.resize((output_size, output_size), Image.ANTIALIAS)143 144 # Save aligned image.145 return crop, [lx, ly, rx, ry]146 147 # def crop(self, img_np_list):148 # for _i in range(len(img_np_list)):149 # img_np = img_np_list[_i]150 # lm = self.get_landmark(img_np)151 # if lm is None:152 # return None153 # crop, quad = self.align_face(img=Image.fromarray(img_np), lm=lm, output_size=512)154 # clx, cly, crx, cry = crop155 # lx, ly, rx, ry = quad156 # lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry)157 158 # _inp = img_np_list[_i]159 # _inp = _inp[cly:cry, clx:crx]160 # _inp = _inp[ly:ry, lx:rx]161 # img_np_list[_i] = _inp162 # return img_np_list163 164 def crop(self, img_np_list, still=False, xsize=512): # first frame for all video165 img_np = img_np_list[0]166 lm = self.get_landmark(img_np)167 if lm is None:168 return None169 crop, quad = self.align_face(img=Image.fromarray(img_np), lm=lm, output_size=xsize)170 clx, cly, crx, cry = crop171 lx, ly, rx, ry = quad172 lx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry)173 for _i in range(len(img_np_list)):174 _inp = img_np_list[_i]175 _inp = _inp[cly:cry, clx:crx]176 # cv2.imwrite('test1.jpg', _inp)177 if not still:178 _inp = _inp[ly:ry, lx:rx]179 # cv2.imwrite('test2.jpg', _inp)180 img_np_list[_i] = _inp181 return img_np_list, crop, quad182 183 184def read_video(filename, uplimit=100):185 frames = []186 cap = cv2.VideoCapture(filename)187 cnt = 0188 while cap.isOpened():189 ret, frame = cap.read()190 if ret:191 frame = cv2.resize(frame, (512, 512))192 frames.append(frame)193 else:194 break195 cnt += 1196 if cnt >= uplimit:197 break198 cap.release()199 assert len(frames) > 0, f'{filename}: video with no frames!'200 return frames201 202 203def create_video(video_name, frames, fps=25, video_format='.mp4', resize_ratio=1):204 # video_name = os.path.dirname(image_folder) + video_format205 # img_list = glob.glob1(image_folder, 'frame*')206 # img_list.sort()207 # frame = cv2.imread(os.path.join(image_folder, img_list[0]))208 # frame = cv2.resize(frame, (0, 0), fx=resize_ratio, fy=resize_ratio)209 # height, width, layers = frames[0].shape210 height, width, layers = 512, 512, 3211 if video_format == '.mp4':212 fourcc = cv2.VideoWriter_fourcc(*'mp4v')213 elif video_format == '.avi':214 fourcc = cv2.VideoWriter_fourcc(*'XVID')215 video = cv2.VideoWriter(video_name, fourcc, fps, (width, height))216 for _frame in frames:217 _frame = cv2.resize(_frame, (height, width), interpolation=cv2.INTER_LINEAR)218 video.write(_frame)219 220def create_images(video_name, frames):221 height, width, layers = 512, 512, 3222 images_dir = video_name.split('.')[0]223 os.makedirs(images_dir, exist_ok=True)224 for i, _frame in enumerate(frames):225 _frame = cv2.resize(_frame, (height, width), interpolation=cv2.INTER_LINEAR)226 _frame_path = os.path.join(images_dir, str(i)+'.jpg')227 cv2.imwrite(_frame_path, _frame)228 229def run(data):230 filename, opt, device = data231 os.environ['CUDA_VISIBLE_DEVICES'] = device232 croper = Croper()233 234 frames = read_video(filename, uplimit=opt.uplimit)235 name = filename.split('/')[-1] # .split('.')[0]236 name = os.path.join(opt.output_dir, name)237 238 frames = croper.crop(frames)239 if frames is None:240 print(f'{name}: detect no face. should removed')241 return242 # create_video(name, frames)243 create_images(name, frames)244 245 246def get_data_path(video_dir):247 eg_video_files = ['/apdcephfs/share_1290939/quincheng/datasets/HDTF/backup_fps25/WDA_KatieHill_000.mp4']248 # filenames = list()249 # VIDEO_EXTENSIONS_LOWERCASE = {'mp4'}250 # VIDEO_EXTENSIONS = VIDEO_EXTENSIONS_LOWERCASE.union({f.upper() for f in VIDEO_EXTENSIONS_LOWERCASE})251 # extensions = VIDEO_EXTENSIONS252 # for ext in extensions:253 # filenames = sorted(glob.glob(f'{opt.input_dir}/**/*.{ext}'))254 # print('Total number of videos:', len(filenames))255 return eg_video_files256 257 258def get_wra_data_path(video_dir):259 if opt.option == 'video':260 videos_path = sorted(glob.glob(f'{video_dir}/*.mp4'))261 elif opt.option == 'image':262 videos_path = sorted(glob.glob(f'{video_dir}/*/'))263 else:264 raise NotImplementedError265 print('Example videos: ', videos_path[:2])266 return videos_path267 268 269if __name__ == '__main__':270 set_start_method('spawn')271 parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)272 parser.add_argument('--input_dir', type=str, help='the folder of the input files')273 parser.add_argument('--output_dir', type=str, help='the folder of the output files')274 parser.add_argument('--device_ids', type=str, default='0,1')275 parser.add_argument('--workers', type=int, default=8)276 parser.add_argument('--uplimit', type=int, default=500)277 parser.add_argument('--option', type=str, default='video')278 279 root = '/apdcephfs/share_1290939/quincheng/datasets/HDTF'280 cmd = f'--input_dir {root}/backup_fps25_first20s_sync/ ' \281 f'--output_dir {root}/crop512_stylegan_firstframe_sync/ ' \282 '--device_ids 0 ' \283 '--workers 8 ' \284 '--option video ' \285 '--uplimit 500 '286 opt = parser.parse_args(cmd.split())287 # filenames = get_data_path(opt.input_dir)288 filenames = get_wra_data_path(opt.input_dir)289 os.makedirs(opt.output_dir, exist_ok=True)290 print(f'Video numbers: {len(filenames)}')291 pool = Pool(opt.workers)292 args_list = cycle([opt])293 device_ids = opt.device_ids.split(",")294 device_ids = cycle(device_ids)295 for data in tqdm(pool.imap_unordered(run, zip(filenames, args_list, device_ids))):296 None