Abs6187/ISL_Sign_Language_Translation
0
1"""
2ISL Sign Language Translation - TechMatrix Solvers Initiative
3Utility functions for pose processing and visualization
4Developed by: TechMatrix Solvers Team
5"""
6
7import numpy as np
8import math
9import cv2
10import matplotlib
11from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
12from matplotlib.figure import Figure
13import matplotlib.pyplot as plt
14import copy
15import seaborn as sns
16
17
18def pad_image_corner(img, stride, pad_value):
19 """
20 Pad image to ensure dimensions are divisible by stride
21
22 Args:
23 img: Input image array
24 stride: Stride value for padding calculation
25 pad_value: Value to use for padding
26 """
27 h, w = img.shape[:2]
28
29 pad = [0, 0, 0, 0] # [up, left, down, right]
30 pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down
31 pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right
32
33 img_padded = img
34
35 # Add padding
36 if pad[0] > 0: # up
37 pad_up = np.tile(img_padded[0:1, :, :] * 0 + pad_value, (pad[0], 1, 1))
38 img_padded = np.concatenate((pad_up, img_padded), axis=0)
39
40 if pad[1] > 0: # left
41 pad_left = np.tile(img_padded[:, 0:1, :] * 0 + pad_value, (1, pad[1], 1))
42 img_padded = np.concatenate((pad_left, img_padded), axis=1)
43
44 if pad[2] > 0: # down
45 pad_down = np.tile(img_padded[-2:-1, :, :] * 0 + pad_value, (pad[2], 1, 1))
46 img_padded = np.concatenate((img_padded, pad_down), axis=0)
47
48 if pad[3] > 0: # right
49 pad_right = np.tile(img_padded[:, -2:-1, :] * 0 + pad_value, (1, pad[3], 1))
50 img_padded = np.concatenate((img_padded, pad_right), axis=1)
51
52 return img_padded, pad
53
54
55def transfer_model_weights(model, model_weights):
56 """
57 Transfer weights from caffe model to pytorch model format
58
59 Args:
60 model: PyTorch model
61 model_weights: Dictionary of weights from caffe model
62 """
63 transferred_weights = {}
64 for weights_name in model.state_dict().keys():
65 if len(weights_name.split('.')) > 4: # body25 format
66 transferred_weights[weights_name] = model_weights['.'.join(
67 weights_name.split('.')[3:])]
68 else:
69 transferred_weights[weights_name] = model_weights['.'.join(
70 weights_name.split('.')[1:])]
71 return transferred_weights
72
73
74def draw_body_pose_visualization(canvas, candidate, subset, model_type='body25'):
75 """
76 Draw body pose keypoints and connections on image
77
78 Args:
79 canvas: Image to draw on
80 candidate: Detected keypoint candidates
81 subset: Valid keypoint connections
82 model_type: Type of pose model ('body25' or 'coco')
83 """
84 stick_width = 4
85
86 if model_type == 'body25':
87 limb_sequence = [
88 [1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],
89 [10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],
90 [11,24],[11,22],[14,21],[14,19],[22,23],[19,20]
91 ]
92 num_joints = 25
93 else:
94 limb_sequence = [
95 [1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9],
96 [9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16],
97 [0, 15], [15, 17], [2, 16], [5, 17]
98 ]
99 num_joints = 18
100
101 # Color scheme for different joints
102 colors = [
103 [255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0],
104 [85, 255, 0], [0, 255, 0], [0, 255, 85], [0, 255, 170], [0, 255, 255],
105 [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], [170, 0, 255],
106 [255, 0, 255], [255, 0, 170], [255, 0, 85], [255,255,0], [255,255,85],
107 [255,255,170], [255,255,255], [170,255,255], [85,255,255], [0,255,255]
108 ]
109
110 # Draw keypoints
111 for i in range(num_joints):
112 for n in range(len(subset)):
113 index = int(subset[n][i])
114 if index == -1:
115 continue
116 x, y = candidate[index][0:2]
117 cv2.circle(canvas, (int(x), int(y)), 4, colors[i], thickness=-1)
118
119 # Draw limbs
120 for i in range(num_joints - 1):
121 for n in range(len(subset)):
122 index = subset[n][np.array(limb_sequence[i])]
123 if -1 in index:
124 continue
125 current_canvas = canvas.copy()
126 Y = candidate[index.astype(int), 0]
127 X = candidate[index.astype(int), 1]
128 mean_x = np.mean(X)
129 mean_y = np.mean(Y)
130 length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
131 angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
132 polygon = cv2.ellipse2Poly((int(mean_y), int(mean_x)),
133 (int(length / 2), stick_width),
134 int(angle), 0, 360, 1)
135 cv2.fillConvexPoly(current_canvas, polygon, colors[i])
136 canvas = cv2.addWeighted(canvas, 0.4, current_canvas, 0.6, 0)
137
138 return canvas
139
140
141def extract_body_pose_data(candidate, subset, model_type='body25'):
142 """
143 Extract body pose data without drawing
144
145 Returns:
146 tuple: (keypoint_circles, limb_sticks) data for further processing
147 """
148 stick_width = 4
149
150 if model_type == 'body25':
151 limb_sequence = [
152 [1,0],[1,2],[2,3],[3,4],[1,5],[5,6],[6,7],[1,8],[8,9],[9,10],
153 [10,11],[8,12],[12,13],[13,14],[0,15],[0,16],[15,17],[16,18],
154 [11,24],[11,22],[14,21],[14,19],[22,23],[19,20]
155 ]
156 num_joints = 25
157 else:
158 limb_sequence = [
159 [1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9],
160 [9, 10], [1, 11], [11, 12], [12, 13], [1, 0], [0, 14], [14, 16],
161 [0, 15], [15, 17], [2, 16], [5, 17]
162 ]
163 num_joints = 18
164
165 # Extract keypoint coordinates
166 keypoint_circles = []
167 for i in range(num_joints):
168 for n in range(len(subset)):
169 index = int(subset[n][i])
170 if index == -1:
171 continue
172 x, y = candidate[index][0:2]
173 keypoint_circles.append((x, y))
174
175 # Extract limb stick data
176 limb_sticks = []
177 for i in range(num_joints - 1):
178 for n in range(len(subset)):
179 index = subset[n][np.array(limb_sequence[i])]
180 if -1 in index:
181 continue
182 Y = candidate[index.astype(int), 0]
183 X = candidate[index.astype(int), 1]
184 mean_x = np.mean(X)
185 mean_y = np.mean(Y)
186 length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5
187 angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1]))
188 limb_sticks.append((mean_y, mean_x, angle, length))
189
190 return keypoint_circles, limb_sticks
191
192
193def draw_hand_pose_visualization(canvas, all_hand_peaks, show_numbers=False):
194 """
195 Draw hand pose keypoints and connections
196
197 Args:
198 canvas: Image to draw on
199 all_hand_peaks: Detected hand keypoints for both hands
200 show_numbers: Whether to show keypoint numbers
201 """
202 edges = [
203 [0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10],
204 [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]
205 ]
206
207 fig = Figure(figsize=plt.figaspect(canvas))
208 fig.subplots_adjust(0, 0, 1, 1)
209 bg = FigureCanvas(fig)
210 ax = fig.subplots()
211 ax.axis('off')
212 ax.imshow(canvas)
213
214 width, height = ax.figure.get_size_inches() * ax.figure.get_dpi()
215
216 for peaks in all_hand_peaks:
217 for ie, e in enumerate(edges):
218 if np.sum(np.all(peaks[e], axis=1) == 0) == 0:
219 x1, y1 = peaks[e[0]]
220 x2, y2 = peaks[e[1]]
221 ax.plot([x1, x2], [y1, y2],
222 color=matplotlib.colors.hsv_to_rgb([ie/float(len(edges)), 1.0, 1.0]))
223
224 for i, keypoint in enumerate(peaks):
225 x, y = keypoint
226 ax.plot(x, y, 'r.')
227 if show_numbers:
228 ax.text(x, y, str(i))
229
230 bg.draw()
231 canvas = np.fromstring(bg.tostring_rgb(), dtype='uint8').reshape(int(height), int(width), 3)
232 return canvas
233
234
235def extract_hand_pose_data(all_hand_peaks, show_numbers=False):
236 """
237 Extract hand pose data without drawing
238
239 Returns:
240 tuple: (hand_edges, hand_peaks) data for further processing
241 """
242 edges = [
243 [0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10],
244 [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]
245 ]
246
247 export_edges = [[], []]
248 export_peaks = [[], []]
249
250 for idx, peaks in enumerate(all_hand_peaks):
251 for ie, e in enumerate(edges):
252 if np.sum(np.all(peaks[e], axis=1) == 0) == 0:
253 x1, y1 = peaks[e[0]]
254 x2, y2 = peaks[e[1]]
255 export_edges[idx].append((ie, (x1, y1), (x2, y2)))
256
257 for i, keypoint in enumerate(peaks):
258 x, y = keypoint
259 export_peaks[idx].append((x, y, str(i)))
260
261 return export_edges, export_peaks
262
263
264def detect_hand_regions(candidate, subset, original_image):
265 """
266 Detect hand regions based on body pose keypoints
267
268 Args:
269 candidate: Body pose candidates
270 subset: Valid body pose connections
271 original_image: Original input image
272
273 Returns:
274 List of detected hand regions [x, y, width, is_left_hand]
275 """
276 ratio_wrist_elbow = 0.33
277 detection_results = []
278
279 image_height, image_width = original_image.shape[0:2]
280
281 for person in subset.astype(int):
282 # Check if left hand keypoints exist (shoulder, elbow, wrist)
283 has_left_hand = np.sum(person[[5, 6, 7]] == -1) == 0
284 has_right_hand = np.sum(person[[2, 3, 4]] == -1) == 0
285
286 if not (has_left_hand or has_right_hand):
287 continue
288
289 hands = []
290
291 # Process left hand
292 if has_left_hand:
293 left_shoulder_idx, left_elbow_idx, left_wrist_idx = person[[5, 6, 7]]
294 x1, y1 = candidate[left_shoulder_idx][:2]
295 x2, y2 = candidate[left_elbow_idx][:2]
296 x3, y3 = candidate[left_wrist_idx][:2]
297 hands.append([x1, y1, x2, y2, x3, y3, True])
298
299 # Process right hand
300 if has_right_hand:
301 right_shoulder_idx, right_elbow_idx, right_wrist_idx = person[[2, 3, 4]]
302 x1, y1 = candidate[right_shoulder_idx][:2]
303 x2, y2 = candidate[right_elbow_idx][:2]
304 x3, y3 = candidate[right_wrist_idx][:2]
305 hands.append([x1, y1, x2, y2, x3, y3, False])
306
307 for x1, y1, x2, y2, x3, y3, is_left in hands:
308 # Calculate hand region based on wrist and elbow positions
309 x = x3 + ratio_wrist_elbow * (x3 - x2)
310 y = y3 + ratio_wrist_elbow * (y3 - y2)
311
312 distance_wrist_elbow = math.sqrt((x3 - x2) ** 2 + (y3 - y2) ** 2)
313 distance_elbow_shoulder = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)
314 width = 1.5 * max(distance_wrist_elbow, 0.9 * distance_elbow_shoulder)
315
316 # Adjust to top-left corner
317 x -= width / 2
318 y -= width / 2
319
320 # Ensure bounds are within image
321 x = max(0, x)
322 y = max(0, y)
323
324 width1 = width if x + width <= image_width else image_width - x
325 width2 = width if y + width <= image_height else image_height - y
326 width = min(width1, width2)
327
328 # Only include if region is large enough
329 if width >= 20:
330 detection_results.append([int(x), int(y), int(width), is_left])
331
332 return detection_results
333
334
335def render_stick_model(original_img, keypoint_circles, limb_sticks, hand_edges, hand_peaks):
336 """
337 Render complete stick model with body and hand poses
338
339 Args:
340 original_img: Original image
341 keypoint_circles: Body keypoint coordinates
342 limb_sticks: Body limb stick data
343 hand_edges: Hand connection data
344 hand_peaks: Hand keypoint data
345 """
346 canvas = copy.deepcopy(original_img)
347
348 colors = [
349 [255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0],
350 [85, 255, 0], [0, 255, 0], [0, 255, 85], [0, 255, 170], [0, 255, 255],
351 [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], [170, 0, 255],
352 [255, 0, 255], [255, 0, 170], [255, 0, 85], [255,255,0], [255,255,85],
353 [255,255,170], [255,255,255], [170,255,255], [85,255,255], [0,255,255]
354 ]
355 stick_width = 4
356
357 # Draw body limbs
358 for idx, (mean_x, mean_y, angle, length) in enumerate(limb_sticks):
359 current_canvas = canvas.copy()
360 polygon = cv2.ellipse2Poly(
361 (int(mean_x), int(mean_y)),
362 (int(length / 2), stick_width),
363 int(angle), 0, 360, 1
364 )
365 cv2.fillConvexPoly(current_canvas, polygon, colors[idx])
366 canvas = cv2.addWeighted(canvas, 0.4, current_canvas, 0.6, 0)
367
368 # Draw body keypoints
369 for idx, (x, y) in enumerate(keypoint_circles):
370 cv2.circle(canvas, (int(x), int(y)), 4, colors[idx], thickness=-1)
371
372 # Draw hand poses using matplotlib
373 fig = Figure(figsize=plt.figaspect(canvas))
374 fig.subplots_adjust(0, 0, 1, 1)
375 ax = fig.subplots()
376 ax.axis('off')
377 ax.imshow(canvas)
378
379 edges = [
380 [0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9],
381 [9, 10], [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16],
382 [0, 17], [17, 18], [18, 19], [19, 20]
383 ]
384
385 for hand_edge_set in hand_edges:
386 for (ie, (x1, y1), (x2, y2)) in hand_edge_set:
387 ax.plot([x1, x2], [y1, y2],
388 color=matplotlib.colors.hsv_to_rgb([ie/float(len(edges)), 1.0, 1.0]))
389
390 for hand_peak_set in hand_peaks:
391 for (x, y, text) in hand_peak_set:
392 ax.plot(x, y, 'r.')
393
394 # Convert figure to numpy array
395 bg = FigureCanvas(fig)
396 bg.draw()
397
398 width, height = fig.get_size_inches() * fig.get_dpi()
399 buf = bg.buffer_rgba()
400 canvas = np.frombuffer(buf, dtype=np.uint8).reshape(int(height), int(width), 4)
401 canvas = canvas[:, :, :3] # Keep only RGB channels
402
403 plt.close(fig) # Clean up
404 return cv2.resize(canvas, (math.ceil(width), math.ceil(height)))
405
406
407def create_bar_plot_visualization(image, predictions, title, orig_img):
408 """
409 Create bar plot visualization below the image
410
411 Args:
412 image: Input image
413 predictions: Dictionary of prediction probabilities
414 title: Plot title
415 orig_img: Original image for sizing
416 """
417 # Handle empty predictions case
418 if not predictions or len(predictions) == 0:
419 # Create a simple plot showing "No predictions available"
420 fig, ax = plt.subplots(figsize=(orig_img.shape[1]/100, orig_img.shape[0]/200), dpi=100)
421 ax.text(0.5, 0.5, 'No Predictions Available',
422 horizontalalignment='center', verticalalignment='center',
423 transform=ax.transAxes, fontsize=14)
424 ax.set_title(title)
425 ax.set_xlim(0, 1)
426 ax.set_ylim(0, 1)
427 ax.set_xticks([])
428 ax.set_yticks([])
429 else:
430 fig, ax = plt.subplots(figsize=(orig_img.shape[1]/100, orig_img.shape[0]/200), dpi=100)
431 plt.title(title)
432
433 # Create bar plot data
434 labels = list(predictions.keys())
435 probabilities = list(predictions.values())
436
437 # Create seaborn bar plot
438 sns.barplot(x=labels, y=probabilities, ax=ax)
439
440 fig.canvas.draw()
441
442 # Convert plot to numpy array
443 plot_image = np.array(fig.canvas.renderer.buffer_rgba())[:, :, :3] # Remove alpha
444 plt.close(fig) # Close to avoid memory leaks
445
446 # Combine image and plot vertically
447 combined_image = np.vstack((image, cv2.resize(plot_image, (image.shape[1], plot_image.shape[0]))))
448
449 return combined_image
450
451
452def add_bottom_padding(image, pad_value, pad_height):
453 """
454 Add padding to the bottom of an image
455
456 Args:
457 image: Input image
458 pad_value: Color value for padding (tuple or int)
459 pad_height: Height of padding to add
460 """
461 height, width, channels = image.shape
462 padding = np.zeros((pad_height, width, channels), dtype=image.dtype)
463 padding[:, :, :] = pad_value
464
465 return np.vstack((image, padding))
466
467
468def find_array_maximum(array):
469 """
470 Get maximum index of 2D array
471
472 Args:
473 array: 2D numpy array
474
475 Returns:
476 tuple: (row_index, col_index) of maximum value
477 """
478 array_index = array.argmax(1)
479 array_value = array.max(1)
480 i = array_value.argmax()
481 j = array_index[i]
482 return i, j