multimodalart/EchoMimic-zero
8
1from skimage.transform import estimate_transform, AffineTransform2import numpy as np3from IPython import embed4import mediapipe as mp5import copy6 7mp_face_mesh = mp.solutions.face_mesh8 9 10FACEMESH_LEFT_EYE = [i for i in mp_face_mesh.FACEMESH_LEFT_EYE] 11FACEMESH_RIGHT_EYE = [i for i in mp_face_mesh.FACEMESH_RIGHT_EYE]12FACEMESH_LEFT_EYEBROW = [i for i in mp_face_mesh.FACEMESH_LEFT_EYEBROW]13FACEMESH_RIGHT_EYEBROW = [i for i in mp_face_mesh.FACEMESH_RIGHT_EYEBROW]14 15# copy from draw_utils16FACEMESH_LIPS_OUTER_BOTTOM_LEFT = [(61,146),(146,91),(91,181),(181,84),(84,17)]17FACEMESH_LIPS_OUTER_BOTTOM_RIGHT = [(17,314),(314,405),(405,321),(321,375),(375,291)]18FACEMESH_LIPS_INNER_BOTTOM_LEFT = [(78,95),(95,88),(88,178),(178,87),(87,14)]19FACEMESH_LIPS_INNER_BOTTOM_RIGHT = [(14,317),(317,402),(402,318),(318,324),(324,308)]20FACEMESH_LIPS_OUTER_TOP_LEFT = [(61,185),(185,40),(40,39),(39,37),(37,0)]21FACEMESH_LIPS_OUTER_TOP_RIGHT = [(0,267),(267,269),(269,270),(270,409),(409,291)]22FACEMESH_LIPS_INNER_TOP_LEFT = [(78,191),(191,80),(80,81),(81,82),(82,13)]23FACEMESH_LIPS_INNER_TOP_RIGHT = [(13,312),(312,311),(311,310),(310,415),(415,308)]24FACEMESH_MOUSE = \25 FACEMESH_LIPS_OUTER_BOTTOM_LEFT + \26 FACEMESH_LIPS_OUTER_BOTTOM_RIGHT + \27 FACEMESH_LIPS_INNER_BOTTOM_LEFT + \28 FACEMESH_LIPS_INNER_BOTTOM_RIGHT + \29 FACEMESH_LIPS_OUTER_TOP_LEFT + \30 FACEMESH_LIPS_OUTER_TOP_RIGHT + \31 FACEMESH_LIPS_INNER_TOP_LEFT + \32 FACEMESH_LIPS_INNER_TOP_RIGHT33 34LANDMARK_IDXES_DICT = {35 "left_eye" : sorted(list(set([j for i in FACEMESH_LEFT_EYE for j in i])) + [473]),36 "right_eye" : sorted(list(set([j for i in FACEMESH_RIGHT_EYE for j in i])) + [468]),37 "mouse" : sorted(list(set([j for i in FACEMESH_MOUSE for j in i]))),38 "nose" : sorted(list(set([1,4,5,274,275,281,44,45,51,220,440]))),39 "left_eyebow" : sorted(list(set([j for i in FACEMESH_LEFT_EYEBROW for j in i]))),40 "right_eyebow" : sorted(list(set([j for i in FACEMESH_RIGHT_EYEBROW for j in i]))),41}42 43def create_perspective_matrix(aspect_ratio):44 kDegreesToRadians = np.pi / 180.45 near = 146 far = 1000047 perspective_matrix = np.zeros(16, dtype=np.float32)48 49 # Standard perspective projection matrix calculations.50 f = 1.0 / np.tan(kDegreesToRadians * 63 / 2.)51 52 denom = 1.0 / (near - far)53 perspective_matrix[0] = f / aspect_ratio54 perspective_matrix[5] = f55 perspective_matrix[10] = (near + far) * denom56 perspective_matrix[11] = -1.57 perspective_matrix[14] = 1. * far * near * denom58 59 # If the environment's origin point location is in the top left corner,60 # then skip additional flip along Y-axis is required to render correctly.61 62 perspective_matrix[5] *= -1.63 return perspective_matrix64 65 66def project_points_with_trans(points_3d, transformation_matrix, image_shape):67 P = create_perspective_matrix(image_shape[1] / image_shape[0]).reshape(4, 4).T68 L, N, _ = points_3d.shape69 projected_points = np.zeros((L, N, 2))70 #embed()71 for i in range(L):72 points_3d_frame = points_3d[i]73 ones = np.ones((points_3d_frame.shape[0], 1))74 points_3d_homogeneous = np.hstack([points_3d_frame, ones]) 75 transformed_points = points_3d_homogeneous @ transformation_matrix[i].T @ P76 projected_points_frame = transformed_points[:, :2] / transformed_points[:, 3, np.newaxis] # -1 ~ 177 projected_points_frame[:, 0] = (projected_points_frame[:, 0] + 1) * 0.5 * image_shape[1] 78 projected_points_frame[:, 1] = (projected_points_frame[:, 1] + 1) * 0.5 * image_shape[0]79 projected_points[i] = projected_points_frame80 return projected_points81 82def project_vertices_from_ref2tgt(ref_lmks3d, tgt_trans_mat):83 #eye_point_idxes84 projected_vertices = project_points_with_trans(ref_lmks3d[np.newaxis, ...], tgt_trans_mat[np.newaxis, ...], [512, 512])[0]85 return projected_vertices86 87 88def old_motion_sync_old(sequence_driver_det, reference_det):89 assert type(sequence_driver_det) is list90 assert type(sequence_driver_det[0]) is type(reference_det) 91 92 lmks3d_mean = sum([i["lmks3d"] for i in sequence_driver_det]) / len(sequence_driver_det)93 overall_transform = estimate_transform('affine', lmks3d_mean, reference_det["lmks3d"])94 95 eye_idxes_all = LANDMARK_IDXES_DICT["left_eye"] + LANDMARK_IDXES_DICT["right_eye"]96 for det_id in range(len(sequence_driver_det)):97 trans = estimate_transform('affine', sequence_driver_det[det_id]["lmks"][eye_idxes_all], sequence_driver_det[det_id]["lmks3d"][eye_idxes_all])98 sequence_driver_det[det_id]["lmks3d"] = np.vstack([99 sequence_driver_det[det_id]["lmks3d"],100 trans(sequence_driver_det[det_id]["lmks"][-10:])101 ])102 103 trans_mats = [] 104 for det in sequence_driver_det: 105 trans_mats.append(det["trans_mat"] @ np.linalg.inv(sequence_driver_det[0]["trans_mat"]))106 107 trans_mats_smooth = []108 smooth_margin = 2109 for tm_itx in range(len(trans_mats)):110 smooth_idxes = [i for i in range(tm_itx - smooth_margin, tm_itx + smooth_margin + 1) if i >= 0 and i < len(trans_mats)]111 tm = sum([trans_mats[i] for i in smooth_idxes]) / len(smooth_idxes)112 trans_mats_smooth.append(tm)113 114 lmks3d_smooth = []115 smooth_margin = 1116 for det_itx in range(len(sequence_driver_det)):117 smooth_idxes = [i for i in range(det_itx - smooth_margin, det_itx + smooth_margin + 1) if i >= 0 and i < len(sequence_driver_det)]118 lmks3d_smooth.append(sum([sequence_driver_det[i]["lmks3d"] for i in smooth_idxes]) / len(smooth_idxes))119 120 for det_itx, lmks3d in enumerate(lmks3d_smooth):121 sequence_driver_det[det_itx]["lmks3d"] = lmks3d122 123 projected_vertices_list = []124 for det_itx in range(len(sequence_driver_det)):125 aligned_3d = overall_transform(sequence_driver_det[det_itx]["lmks3d"])126 tmat = reference_det["trans_mat"] @ trans_mats_smooth[det_itx]127 projected_vertices = project_vertices_from_ref2tgt(aligned_3d, tmat)128 projected_vertices_list.append(projected_vertices)129 130 # note : use normed=False after motion_sync, when draw(ing)_landmarks131 # kps_image = vis.draw_landmarks((512, 512), projected_vertices, normed=False) 132 return projected_vertices_list133 134 135def motion_sync(sequence_driver_det, reference_det, per_landmark_align=True):136 assert type(sequence_driver_det) is list137 assert type(sequence_driver_det[0]) is type(reference_det) 138 139 eye_idxes_all = [i for i in sorted(list(set(LANDMARK_IDXES_DICT["left_eye"] + LANDMARK_IDXES_DICT["right_eye"]))) if i < len(reference_det["lmks3d"])]140 for det_id in range(len(sequence_driver_det)):141 trans_iris = estimate_transform('affine', sequence_driver_det[det_id]["lmks"][eye_idxes_all], sequence_driver_det[det_id]["lmks3d"][eye_idxes_all])142 sequence_driver_det[det_id]["lmks3d"] = np.vstack([143 sequence_driver_det[det_id]["lmks3d"],144 trans_iris(sequence_driver_det[det_id]["lmks"][-10:])145 ])146 147 trans_iris = estimate_transform('affine', reference_det["lmks"][eye_idxes_all], reference_det["lmks3d"][eye_idxes_all])148 reference_det["lmks3d"] = np.vstack([149 reference_det["lmks3d"],150 trans_iris(reference_det["lmks"][-10:])151 ])152 153 lmks3d_mean = sum([i["lmks3d"] for i in sequence_driver_det]) / len(sequence_driver_det)154 155 landmark_trans_dict = {}156 for landmark_name, landmark_idxes in LANDMARK_IDXES_DICT.items():157 rf_lm = reference_det["lmks3d"][landmark_idxes]158 dr_lm = lmks3d_mean[landmark_idxes]159 landmark_trans_dict[landmark_name] = estimate_transform('affine', dr_lm, rf_lm)160 161 #embed()162 overall_transform = estimate_transform('affine', lmks3d_mean, reference_det["lmks3d"])163 #embed()164 #lmks3d_mean = sum([i["lmks3d"] for i in sequence_driver_det]) / len(sequence_driver_det)165 #overall_transform = estimate_transform('affine', lmks3d_mean, reference_det["lmks3d"])166 167 #driver_start_center = sequence_driver_det[0]["lmks3d"].mean(axis=0)168 #reference_center = reference_det["lmks3d"].mean(axis=0)169 #driver_start_size = ((sequence_driver_det[0]["lmks3d"] - driver_start_center)**2).sum()**(0.5)170 #reference_size = ((reference_det["lmks3d"] - reference_center)**2).sum()**(0.5)171 172 #reference_det_lmks3d_rescale = (reference_det["lmks3d"] - reference_center) / reference_size * driver_start_size + driver_start_center173 #reference_transform_back = estimate_transform('affine', reference_det_lmks3d_rescale, reference_det["lmks3d"])174 175 #driver_lmks3d_mean = sum([i["lmks3d"] for i in sequence_driver_det]) / len(sequence_driver_det)176 #facial_transform = estimate_transform('affine', driver_lmks3d_mean, reference_det_lmks3d_rescale)177 178 179 #for det_id in range(len(sequence_driver_det)):180 # trans = estimate_transform('affine', sequence_driver_det[det_id]["lmks"][:-10], sequence_driver_det[det_id]["lmks3d"])181 # sequence_driver_det[det_id]["lmks3d"] = trans(sequence_driver_det[det_id]["lmks"])182 183 trans_mats = [] 184 for det in sequence_driver_det: 185 trans_mats.append(det["trans_mat"] @ np.linalg.inv(sequence_driver_det[0]["trans_mat"]))186 187 trans_mats_smooth = []188 smooth_margin = 2189 for tm_itx in range(len(trans_mats)):190 smooth_idxes = [i for i in range(tm_itx - smooth_margin, tm_itx + smooth_margin + 1) if i >= 0 and i < len(trans_mats)]191 tm = sum([trans_mats[i] for i in smooth_idxes]) / len(smooth_idxes)192 trans_mats_smooth.append(tm)193 194 lmks3d_smooth = []195 smooth_margin = 1196 for det_itx in range(len(sequence_driver_det)):197 smooth_idxes = [i for i in range(det_itx - smooth_margin, det_itx + smooth_margin + 1) if i >= 0 and i < len(sequence_driver_det)]198 lmks3d_smooth.append(sum([sequence_driver_det[i]["lmks3d"] for i in smooth_idxes]) / len(smooth_idxes))199 200 for det_itx, lmks3d in enumerate(lmks3d_smooth):201 sequence_driver_det[det_itx]["lmks3d"] = lmks3d202 203 projected_vertices_list = []204 for det_itx in range(len(sequence_driver_det)):205 #aligned_3d = overall_transform(sequence_driver_det[det_itx]["lmks3d"])206 aligned_3d = copy.deepcopy(sequence_driver_det[det_itx]["lmks3d"])207 if per_landmark_align:208 for landmark_name, landmark_idxes in LANDMARK_IDXES_DICT.items():209 dr_lm = sequence_driver_det[det_itx]["lmks3d"][landmark_idxes]210 lm_trans = landmark_trans_dict[landmark_name]211 aligned_3d[landmark_idxes] = lm_trans(dr_lm)212 213 #aligned_3d = lmks3d_mean214 tmat = trans_mats_smooth[det_itx] @ reference_det["trans_mat"]215 projected_vertices = project_vertices_from_ref2tgt(aligned_3d, tmat)216 projected_vertices_list.append(projected_vertices)217 218 continue219 220 trans_ref_aligned_to_driver = (sequence_driver_det[det_itx]["trans_mat"]) @ np.linalg.inv(reference_det["trans_mat"])221 ref_aligned_to_driver = AffineTransform(trans_ref_aligned_to_driver)(reference_det["lmks3d"])222 det["trans_mat"] @ np.linalg.inv(sequence_driver_det[0]["trans_mat"])223 aligned_3d = sequence_driver_det[det_itx]["lmks3d"]224 225 226 #facial_transform(sequence_driver_det[det_itx]["lmks3d"])227 #tmat = reference_det["trans_mat"] @ trans_mats_smooth[det_itx]228 229 tmat = sequence_driver_det[det_itx]["trans_mat"] @ trans_mats_smooth[det_itx] #@ reference_transform_back.params230 projected_vertices = project_vertices_from_ref2tgt(aligned_3d, tmat)231 #embed()232 #projected_vertices = reference_transform_back(projected_vertices)233 projected_vertices_list.append(projected_vertices)234 235 # note : use normed=False after motion_sync, when draw(ing)_landmarks236 # kps_image = vis.draw_landmarks((512, 512), projected_vertices, normed=False) 237 return projected_vertices_list238 239