Abs6187/ISL_Sign_Language_Translation
0
1"""
2ISL Sign Language Translation - TechMatrix Solvers Initiative
3Core ISL Processing and Translation Models
4
5Developed by: TechMatrix Solvers Team
6- Abhay Gupta (Team Lead)
7- Kripanshu Gupta (Backend Developer)
8- Dipanshu Patel (UI/UX Designer)
9- Bhumika Patel (Deployment & Female Presenter)
10
11Institution: Shri Ram Group of Institutions
12"""
13
14import keras
15import numpy as np
16import cv2
17import torch
18try:
19 from scipy.ndimage.filters import gaussian_filter
20except ImportError:
21 from scipy.ndimage import gaussian_filter
22import math
23import os
24from skimage.measure import label
25import pose_utils as utils
26
27# Simple TorchModuleWrapper replacement for compatibility
28class TorchModuleWrapper:
29 """
30 Simple wrapper to make PyTorch models compatible with Keras-style usage
31 """
32 def __init__(self, torch_model):
33 self.torch_model = torch_model
34 self.trainable = False
35
36 def __call__(self, x):
37 """Forward pass through the PyTorch model"""
38 return self.torch_model(x)
39
40 def eval(self):
41 """Set model to evaluation mode"""
42 if hasattr(self.torch_model, 'eval'):
43 self.torch_model.eval()
44
45 def train(self, mode=True):
46 """Set model to train mode"""
47 if hasattr(self.torch_model, 'train'):
48 self.torch_model.train(mode)
49
50
51class ISLPoseEstimator(keras.Model):
52 """
53 ISL Pose Estimation Model combining body and hand pose detection
54 Developed by TechMatrix Solvers for accurate sign language recognition
55 """
56
57 def __init__(self, pytorch_body_model, pytorch_hand_model):
58 super().__init__()
59 self.pytorch_body_wrapper = TorchModuleWrapper(pytorch_body_model)
60 self.pytorch_body_wrapper.trainable = False
61 self.pytorch_hand_wrapper = TorchModuleWrapper(pytorch_hand_model)
62 self.pytorch_hand_wrapper.trainable = False
63 self.num_body_joints = 26
64 self.num_body_pafs = 52
65
66 def call(self, input_image):
67 """
68 Process input image and extract pose information
69
70 Args:
71 input_image: Input image tensor
72
73 Returns:
74 tuple: (body_candidates, body_subset, hand_peaks)
75 """
76 candidate, subset = self.extract_body_pose(input_image.cpu().numpy())
77 hand_regions = utils.detect_hand_regions(candidate, subset, input_image.cpu().numpy())
78
79 all_hand_keypoints = []
80 for x, y, w, is_left in hand_regions:
81 hand_peaks = self.extract_hand_pose(input_image.cpu().numpy()[y:y+w, x:x+w, :])
82 hand_peaks[:, 0] = np.where(hand_peaks[:, 0] == 0, hand_peaks[:, 0], hand_peaks[:, 0] + x)
83 hand_peaks[:, 1] = np.where(hand_peaks[:, 1] == 0, hand_peaks[:, 1], hand_peaks[:, 1] + y)
84 all_hand_keypoints.append(hand_peaks)
85
86 return candidate, subset, all_hand_keypoints
87
88 def extract_body_pose(self, input_image):
89 """
90 Extract body pose keypoints from input image
91
92 Args:
93 input_image: Input image array
94
95 Returns:
96 tuple: (candidates, subset) containing pose information
97 """
98 model_type = 'body25'
99 scale_factors = [0.5]
100 box_size = 368
101 stride = 8
102 padding_value = 128
103 threshold_1 = 0.1
104 threshold_2 = 0.05
105
106 # Calculate scale multipliers
107 multiplier = [x * box_size / input_image.shape[0] for x in scale_factors]
108 heatmap_average = np.zeros((input_image.shape[0], input_image.shape[1], self.num_body_joints))
109 paf_average = np.zeros((input_image.shape[0], input_image.shape[1], self.num_body_pafs))
110
111 for m in range(len(multiplier)):
112 scale = multiplier[m]
113 test_image = cv2.resize(input_image, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
114 padded_image, pad = utils.pad_image_corner(test_image, stride, padding_value)
115
116 # Prepare image tensor
117 image_tensor = np.transpose(np.float32(padded_image[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
118 image_tensor = np.ascontiguousarray(image_tensor)
119
120 # Convert to PyTorch tensor
121 data = torch.from_numpy(image_tensor).float()
122 if torch.cuda.is_available():
123 data = data.cuda()
124
125 with torch.no_grad():
126 stage6_L1, stage6_L2 = self.pytorch_body_wrapper(data)
127
128 stage6_L1 = stage6_L1.cpu().numpy()
129 stage6_L2 = stage6_L2.cpu().numpy()
130
131 # Process heatmaps
132 heatmap = np.transpose(np.squeeze(stage6_L2), (1, 2, 0))
133 heatmap = cv2.resize(heatmap, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
134 heatmap = heatmap[:padded_image.shape[0] - pad[2], :padded_image.shape[1] - pad[3], :]
135 heatmap = cv2.resize(heatmap, (input_image.shape[1], input_image.shape[0]), interpolation=cv2.INTER_CUBIC)
136
137 # Process PAFs (Part Affinity Fields)
138 paf = np.transpose(np.squeeze(stage6_L1), (1, 2, 0))
139 paf = cv2.resize(paf, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
140 paf = paf[:padded_image.shape[0] - pad[2], :padded_image.shape[1] - pad[3], :]
141 paf = cv2.resize(paf, (input_image.shape[1], input_image.shape[0]), interpolation=cv2.INTER_CUBIC)
142
143 heatmap_average += heatmap / len(multiplier)
144 paf_average += paf / len(multiplier)
145
146 # Extract peaks from heatmaps
147 all_peaks = []
148 peak_counter = 0
149
150 for part in range(self.num_body_joints - 1):
151 original_map = heatmap_average[:, :, part]
152 smoothed_heatmap = gaussian_filter(original_map, sigma=3)
153
154 # Find local maxima
155 left_map = np.zeros(smoothed_heatmap.shape)
156 left_map[1:, :] = smoothed_heatmap[:-1, :]
157 right_map = np.zeros(smoothed_heatmap.shape)
158 right_map[:-1, :] = smoothed_heatmap[1:, :]
159 up_map = np.zeros(smoothed_heatmap.shape)
160 up_map[:, 1:] = smoothed_heatmap[:, :-1]
161 down_map = np.zeros(smoothed_heatmap.shape)
162 down_map[:, :-1] = smoothed_heatmap[:, 1:]
163
164 peaks_binary = np.logical_and.reduce(
165 (smoothed_heatmap >= left_map, smoothed_heatmap >= right_map,
166 smoothed_heatmap >= up_map, smoothed_heatmap >= down_map,
167 smoothed_heatmap > threshold_1)
168 )
169
170 peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0]))
171 peaks_with_score = [x + (original_map[x[1], x[0]],) for x in peaks]
172 peak_id = range(peak_counter, peak_counter + len(peaks))
173 peaks_with_score_and_id = [peaks_with_score[i] + (peak_id[i],) for i in range(len(peak_id))]
174
175 all_peaks.append(peaks_with_score_and_id)
176 peak_counter += len(peaks)
177
178 # Define limb connections for body25 model
179 if model_type == 'body25':
180 limb_sequence = [
181 [1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],
182 [10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],
183 [11,24],[11,22],[14,21],[14,19],[22,23],[19,20]
184 ]
185 map_index = [
186 [30,31],[14,15],[16,17],[18,19],[22,23],[24,25],[26,27],[0,1],[6,7],
187 [2,3],[4,5],[8,9],[10,11],[12,13],[32,33],[34,35],[36,37],[38,39],
188 [50,51],[46,47],[44,45],[40,41],[48,49],[42,43]
189 ]
190
191 # Find connections between body parts
192 connection_all = []
193 special_k = []
194 mid_num = 10
195
196 for k in range(len(map_index)):
197 score_mid = paf_average[:, :, map_index[k]]
198 candA = all_peaks[limb_sequence[k][0]]
199 candB = all_peaks[limb_sequence[k][1]]
200
201 nA = len(candA)
202 nB = len(candB)
203 indexA, indexB = limb_sequence[k]
204
205 if nA != 0 and nB != 0:
206 connection_candidate = []
207 for i in range(nA):
208 for j in range(nB):
209 vec = np.subtract(candB[j][:2], candA[i][:2])
210 norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1])
211 norm = max(0.001, norm)
212 vec = np.divide(vec, norm)
213
214 startend = list(zip(
215 np.linspace(candA[i][0], candB[j][0], num=mid_num),
216 np.linspace(candA[i][1], candB[j][1], num=mid_num)
217 ))
218
219 vec_x = np.array([
220 score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0]
221 for I in range(len(startend))
222 ])
223 vec_y = np.array([
224 score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1]
225 for I in range(len(startend))
226 ])
227
228 score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1])
229 score_with_dist_prior = (sum(score_midpts) / len(score_midpts) +
230 min(0.5 * input_image.shape[0] / norm - 1, 0))
231
232 criterion1 = len(np.nonzero(score_midpts > threshold_2)[0]) > 0.8 * len(score_midpts)
233 criterion2 = score_with_dist_prior > 0
234
235 if criterion1 and criterion2:
236 connection_candidate.append([
237 i, j, score_with_dist_prior,
238 score_with_dist_prior + candA[i][2] + candB[j][2]
239 ])
240
241 connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True)
242 connection = np.zeros((0, 5))
243
244 for c in range(len(connection_candidate)):
245 i, j, s = connection_candidate[c][0:3]
246 if i not in connection[:, 3] and j not in connection[:, 4]:
247 connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]])
248 if len(connection) >= min(nA, nB):
249 break
250
251 connection_all.append(connection)
252 else:
253 special_k.append(k)
254 connection_all.append([])
255
256 # Create human pose subsets
257 subset = -1 * np.ones((0, self.num_body_joints + 1))
258 candidate = np.array([item for sublist in all_peaks for item in sublist])
259
260 for k in range(len(map_index)):
261 if k not in special_k:
262 partAs = connection_all[k][:, 0]
263 partBs = connection_all[k][:, 1]
264 indexA, indexB = np.array(limb_sequence[k])
265
266 for i in range(len(connection_all[k])):
267 found = 0
268 subset_idx = [-1, -1]
269
270 for j in range(len(subset)):
271 if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]:
272 subset_idx[found] = j
273 found += 1
274
275 if found == 1:
276 j = subset_idx[0]
277 if subset[j][indexB] != partBs[i]:
278 subset[j][indexB] = partBs[i]
279 subset[j][-1] += 1
280 subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
281 elif found == 2:
282 j1, j2 = subset_idx
283 membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2]
284 if len(np.nonzero(membership == 2)[0]) == 0:
285 subset[j1][:-2] += (subset[j2][:-2] + 1)
286 subset[j1][-2:] += subset[j2][-2:]
287 subset[j1][-2] += connection_all[k][i][2]
288 subset = np.delete(subset, j2, 0)
289 else:
290 subset[j1][indexB] = partBs[i]
291 subset[j1][-1] += 1
292 subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2]
293 elif not found and k < self.num_body_joints - 2:
294 row = -1 * np.ones(self.num_body_joints + 1)
295 row[indexA] = partAs[i]
296 row[indexB] = partBs[i]
297 row[-1] = 2
298 row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2]
299 subset = np.vstack([subset, row])
300
301 # Filter out low-quality detections
302 deleteIdx = []
303 for i in range(len(subset)):
304 if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4:
305 deleteIdx.append(i)
306 subset = np.delete(subset, deleteIdx, axis=0)
307
308 return candidate, subset
309
310 def extract_hand_pose(self, input_image):
311 """
312 Extract hand pose keypoints from input image region
313
314 Args:
315 input_image: Cropped hand region image
316
317 Returns:
318 numpy.ndarray: Hand keypoint coordinates
319 """
320 scale_factors = [0.5, 1.0, 1.5, 2.0]
321 box_size = 368
322 stride = 8
323 padding_value = 128
324 threshold = 0.05
325
326 multiplier = [x * box_size / input_image.shape[0] for x in scale_factors]
327 heatmap_average = np.zeros((input_image.shape[0], input_image.shape[1], 22))
328
329 for m in range(len(multiplier)):
330 scale = multiplier[m]
331 test_image = cv2.resize(input_image, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
332 padded_image, pad = utils.pad_image_corner(test_image, stride, padding_value)
333
334 # Prepare image tensor
335 image_tensor = np.transpose(np.float32(padded_image[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
336 image_tensor = np.ascontiguousarray(image_tensor)
337
338 data = torch.from_numpy(image_tensor).float()
339 if torch.cuda.is_available():
340 data = data.cuda()
341
342 with torch.no_grad():
343 output = self.pytorch_hand_wrapper(data).cpu().numpy()
344
345 # Process heatmaps
346 heatmap = np.transpose(np.squeeze(output), (1, 2, 0))
347 heatmap = cv2.resize(heatmap, (0, 0), fx=stride, fy=stride, interpolation=cv2.INTER_CUBIC)
348 heatmap = heatmap[:padded_image.shape[0] - pad[2], :padded_image.shape[1] - pad[3], :]
349 heatmap = cv2.resize(heatmap, (input_image.shape[1], input_image.shape[0]), interpolation=cv2.INTER_CUBIC)
350
351 heatmap_average += heatmap / len(multiplier)
352
353 # Extract hand keypoints
354 all_peaks = []
355 for part in range(21):
356 original_map = heatmap_average[:, :, part]
357 smoothed_heatmap = gaussian_filter(original_map, sigma=3)
358 binary = np.ascontiguousarray(smoothed_heatmap > threshold, dtype=np.uint8)
359
360 if np.sum(binary) == 0:
361 all_peaks.append([0, 0])
362 continue
363
364 label_img, label_numbers = label(binary, return_num=True, connectivity=binary.ndim)
365 max_index = np.argmax([np.sum(original_map[label_img == i]) for i in range(1, label_numbers + 1)]) + 1
366 label_img[label_img != max_index] = 0
367 original_map[label_img == 0] = 0
368
369 y, x = utils.find_array_maximum(original_map)
370 all_peaks.append([x, y])
371
372 return np.array(all_peaks)
373
374
375class ISLTranslationModel(keras.Model):
376 """
377 Complete ISL Translation Model combining pose estimation and LSTM translation
378 Developed by TechMatrix Solvers for end-to-end sign language translation
379 """
380
381 def __init__(self, body_model, hand_model, translation_model):
382 super().__init__()
383 self.pytorch_body_wrapper = TorchModuleWrapper(body_model)
384 self.pytorch_body_wrapper.trainable = False
385 self.pytorch_hand_wrapper = TorchModuleWrapper(hand_model)
386 self.pytorch_hand_wrapper.trainable = False
387
388 self.num_body_joints = 26
389 self.num_body_pafs = 52
390 self.model_type = 'body25'
391 self.translation_network = translation_model
392
393 def call(self, frame_sequence):
394 """
395 Process a sequence of frames and return translation prediction
396
397 Args:
398 frame_sequence: Sequence of video frames
399
400 Returns:
401 Translation prediction probabilities
402 """
403 window_size = 20
404 feature_sequence = []
405 blank_frame = np.zeros((1, 156))
406
407 for idx, frame in enumerate(frame_sequence.cpu()):
408 # Extract pose features from current frame
409 candidate, subset = self.extract_body_pose(frame.cpu().numpy())
410 hand_regions = utils.detect_hand_regions(candidate, subset, frame.cpu().numpy())
411
412 all_hand_keypoints = []
413 for x, y, w, is_left in hand_regions:
414 peaks = self.extract_hand_pose(frame.cpu().numpy()[y:y+w, x:x+w, :])
415 peaks[:, 0] = np.where(peaks[:, 0] == 0, peaks[:, 0], peaks[:, 0] + x)
416 peaks[:, 1] = np.where(peaks[:, 1] == 0, peaks[:, 1], peaks[:, 1] + y)
417 all_hand_keypoints.append(peaks)
418
419 # Extract structured pose data
420 body_circles, body_sticks = utils.extract_body_pose_data(candidate, subset, self.model_type)
421 hand_edges, hand_peaks = utils.extract_hand_pose_data(all_hand_keypoints)
422
423 # Convert to feature vector
424 feature_vector = self.create_feature_vector(body_circles, hand_peaks)
425 feature_sequence.append(feature_vector)
426
427 # Pad sequence if needed
428 if len(feature_sequence) < window_size:
429 for _ in range(window_size - len(feature_sequence)):
430 feature_sequence.append(blank_frame)
431
432 # Run translation model
433 return self.translation_network(np.array(feature_sequence).reshape(1, 20, 156))
434
435 def create_feature_vector(self, body_circles, hand_peaks):
436 """
437 Create feature vector from pose data
438
439 Args:
440 body_circles: Body keypoint coordinates
441 hand_peaks: Hand keypoint data
442
443 Returns:
444 numpy.ndarray: 156-dimensional feature vector
445 """
446 features = []
447
448 # Body keypoint x-coordinates (15 points)
449 for idx in range(15):
450 if idx < len(body_circles):
451 features.append(body_circles[idx][0])
452 else:
453 features.append(0)
454
455 # Body keypoint y-coordinates (15 points)
456 for idx in range(15):
457 if idx < len(body_circles):
458 features.append(body_circles[idx][1])
459 else:
460 features.append(0)
461
462 # Hand features for both hands
463 for hand_idx in range(2):
464 # Hand x-coordinates (21 points)
465 for idx in range(21):
466 if idx < len(hand_peaks[hand_idx]):
467 features.append(float(hand_peaks[hand_idx][idx][0]))
468 else:
469 features.append(0)
470
471 # Hand y-coordinates (21 points)
472 for idx in range(21):
473 if idx < len(hand_peaks[hand_idx]):
474 features.append(float(hand_peaks[hand_idx][idx][1]))
475 else:
476 features.append(0)
477
478 # Hand peak text/confidence (21 points)
479 for idx in range(21):
480 if idx < len(hand_peaks[hand_idx]):
481 features.append(float(hand_peaks[hand_idx][idx][2]))
482 else:
483 features.append(0)
484
485 return np.array(features)
486
487 def extract_body_pose(self, input_image):
488 """Extract body pose - same implementation as ISLPoseEstimator"""
489 # This method would contain the same implementation as in ISLPoseEstimator
490 # For brevity, using a placeholder that calls the same logic
491 pose_estimator = ISLPoseEstimator(None, None)
492 pose_estimator.pytorch_body_wrapper = self.pytorch_body_wrapper
493 pose_estimator.num_body_joints = self.num_body_joints
494 pose_estimator.num_body_pafs = self.num_body_pafs
495 return pose_estimator.extract_body_pose(input_image)
496
497 def extract_hand_pose(self, input_image):
498 """Extract hand pose - same implementation as ISLPoseEstimator"""
499 # This method would contain the same implementation as in ISLPoseEstimator
500 # For brevity, using a placeholder that calls the same logic
501 pose_estimator = ISLPoseEstimator(None, None)
502 pose_estimator.pytorch_hand_wrapper = self.pytorch_hand_wrapper
503 return pose_estimator.extract_hand_pose(input_image)