sky-variable/action-recognition
0
1"""2Human Action Recognition App3Powered by BiLSTM + Conv1D + MediaPipe Pose4Compatible with Gradio 6.x5"""6 7import gradio as gr8import numpy as np9import cv210import pickle11import json12import os13import tempfile14from collections import deque15import tensorflow as tf16import mediapipe as mp17 18# ── Load artefacts ────────────────────────────────────────────19BASE = os.path.dirname(__file__)20 21model = tf.keras.models.load_model(os.path.join(BASE, "action_model.keras"))22with open(os.path.join(BASE, "label_encoder.pkl"), "rb") as f:23 le = pickle.load(f)24with open(os.path.join(BASE, "scaler.pkl"), "rb") as f:25 scaler = pickle.load(f)26with open(os.path.join(BASE, "meta.json")) as f:27 meta = json.load(f)28 29CLASSES = meta["classes"]30N_LANDMARKS = meta["n_landmarks"]31N_FEAT_PER_LM = meta["n_feat_per_lm"]32 33mp_pose = mp.solutions.pose34mp_drawing = mp.solutions.drawing_utils35mp_styles = mp.solutions.drawing_styles36 37# ══════════════════════════════════════════════════════════════38# Core helpers39# ══════════════════════════════════════════════════════════════40 41def extract_keypoints(results):42 if results.pose_landmarks is None:43 return None44 kp = []45 for p in results.pose_landmarks.landmark:46 kp.extend([p.x, p.y, p.z, p.visibility])47 return np.array(kp, dtype=np.float32)48 49 50def predict_from_keypoints(kp, confidence_thresh=0.5, smoothing_window=None):51 scaled = scaler.transform(kp.reshape(1, -1))52 seq = scaled.reshape(1, N_LANDMARKS, N_FEAT_PER_LM)53 proba = model.predict(seq, verbose=0)[0]54 if smoothing_window is not None:55 smoothing_window.append(proba.copy())56 proba = np.mean(smoothing_window, axis=0)57 idx = int(np.argmax(proba))58 confidence = float(proba[idx])59 label = CLASSES[idx] if confidence >= confidence_thresh else "Uncertain"60 proba_dict = {c: float(p) for c, p in zip(CLASSES, proba)}61 return label, confidence, proba_dict62 63 64def draw_overlay(frame, results, label, confidence,65 show_skeleton=True, show_label=True):66 if show_skeleton and results.pose_landmarks:67 mp_drawing.draw_landmarks(68 frame, results.pose_landmarks, mp_pose.POSE_CONNECTIONS,69 landmark_drawing_spec=mp_styles.get_default_pose_landmarks_style()70 )71 if show_label:72 color = (0, 200, 0) if confidence >= 0.75 else (0, 165, 255)73 cv2.rectangle(frame, (0, 0), (420, 50), (0, 0, 0), -1)74 cv2.putText(frame, f"{label} {confidence*100:.1f}%",75 (10, 34), cv2.FONT_HERSHEY_SIMPLEX, 1.0, color, 2, cv2.LINE_AA)76 return frame77 78 79# ══════════════════════════════════════════════════════════════80# Video Upload Inference81# ══════════════════════════════════════════════════════════════82 83def process_video(video_path, confidence_thresh, smoothing_size,84 show_skeleton, show_label):85 if video_path is None:86 return None, "No video uploaded."87 88 cap = cv2.VideoCapture(video_path)89 fps = cap.get(cv2.CAP_PROP_FPS) or 2590 width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))91 height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))92 93 out_path = tempfile.mktemp(suffix=".mp4")94 writer = cv2.VideoWriter(out_path, cv2.VideoWriter_fourcc(*"mp4v"),95 fps, (width, height))96 97 buffer = deque(maxlen=max(1, int(smoothing_size)))98 stats = {}99 pose = mp_pose.Pose(min_detection_confidence=0.5,100 min_tracking_confidence=0.5)101 frame_count = 0102 103 while True:104 ret, frame = cap.read()105 if not ret:106 break107 frame_count += 1108 rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)109 results = pose.process(rgb)110 kp = extract_keypoints(results)111 112 if kp is not None:113 label, conf, _ = predict_from_keypoints(kp, confidence_thresh, buffer)114 else:115 label, conf = "No Pose Detected", 0.0116 117 stats[label] = stats.get(label, 0) + 1118 frame = draw_overlay(frame, results, label, conf, show_skeleton, show_label)119 writer.write(frame)120 121 cap.release(); writer.release(); pose.close()122 123 dominant = max(stats, key=stats.get) if stats else "N/A"124 lines = [f"**Frames processed**: {frame_count}",125 f"**Dominant action**: {dominant}", "---",126 "**Frame distribution**:"]127 for lbl, cnt in sorted(stats.items(), key=lambda x: -x[1]):128 pct = cnt / frame_count * 100 if frame_count else 0129 lines.append(f"- {lbl}: {cnt} frames ({pct:.1f}%)")130 return out_path, "\n".join(lines)131 132 133# ══════════════════════════════════════════════════════════════134# Live Stream — Gradio 6.x uses gr.Image with streaming=True135# and .stream() event. Frame arrives as numpy RGB array.136# ══════════════════════════════════════════════════════════════137 138_live_buffer = deque(maxlen=5)139_live_pose = None140 141def process_live_frame(frame, confidence_thresh, smoothing_size,142 show_skeleton, show_label):143 global _live_buffer, _live_pose144 if frame is None:145 return None146 147 if _live_pose is None:148 _live_pose = mp_pose.Pose(min_detection_confidence=0.5,149 min_tracking_confidence=0.5)150 if int(smoothing_size) != _live_buffer.maxlen:151 _live_buffer = deque(maxlen=max(1, int(smoothing_size)))152 153 results = _live_pose.process(frame) # expects RGB154 kp = extract_keypoints(results)155 156 bgr = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)157 if kp is not None:158 label, conf, _ = predict_from_keypoints(kp, confidence_thresh, _live_buffer)159 else:160 label, conf = "No Pose Detected", 0.0161 162 bgr = draw_overlay(bgr, results, label, conf, show_skeleton, show_label)163 return cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)164 165 166# ══════════════════════════════════════════════════════════════167# UI168# ══════════════════════════════════════════════════════════════169 170ABOUT_MD = """171# 🏃 Human Action Recognition172 173Real-time 8-class human action recognition using **MediaPipe Pose** + **BiLSTM + Conv1D + Multi-Head Attention**.174 175## Recognisable Actions176`Arms Crossed` · `Bending` · `Clapping` · `Hand Raised` · `Jumping` · `Sitting` · `Standing` · `Walking`177 178## How It Works1791. **MediaPipe Pose** extracts 33 body landmarks (x, y, z, visibility) = 132 features per frame1802. Features are **normalised** and reshaped to a **(33 × 4) sequence**1813. The sequence passes through:182 - **Conv1D** → local spatial patterns across adjacent landmarks183 - **BiLSTM × 2** → bidirectional temporal modelling184 - **Multi-Head Attention** → global body-part relationship weighting185 - **Dense softmax head** → 8-class prediction186 187## Model Performance188| Property | Value |189|----------|-------|190| Architecture | Conv1D → BiLSTM × 2 → Attention → Dense |191| Training samples | 7,163 pose frames |192| Test accuracy | **98.7%** |193| Inference speed | ~5 ms / frame (CPU) |194 195## Optimisations Used196L2 regularisation · Dropout · BatchNorm · Class-weight balancing · Gradient clipping · ReduceLROnPlateau · EarlyStopping197 198## Tech Stack199`TensorFlow 2.21` · `MediaPipe` · `OpenCV` · `scikit-learn` · `Gradio 6`200"""201 202with gr.Blocks(203 title="Human Action Recognition",204 theme=gr.themes.Soft(primary_hue="violet", secondary_hue="blue"),205) as demo:206 207 gr.Markdown("# 🏃 Human Action Recognition")208 gr.Markdown("*Real-time 8-class action detection — MediaPipe Pose + BiLSTM*")209 210 with gr.Tabs():211 212 # ── About ────────────────────────────────────────────213 with gr.Tab("ℹ️ About"):214 gr.Markdown(ABOUT_MD)215 216 # ── Upload Video ─────────────────────────────────────217 with gr.Tab("📤 Upload Video"):218 gr.Markdown("Upload a video and run action recognition on every frame.")219 with gr.Row():220 with gr.Column():221 vid_in = gr.Video(label="Upload Video")222 with gr.Accordion("⚙️ Settings", open=False):223 v_conf = gr.Slider(0.3, 1.0, value=0.60, step=0.05,224 label="Confidence threshold")225 v_smth = gr.Slider(1, 20, value=5, step=1,226 label="Smoothing window (frames)")227 v_skel = gr.Checkbox(value=True, label="Show skeleton")228 v_lbl = gr.Checkbox(value=True, label="Show label")229 run_btn = gr.Button("▶️ Run Inference", variant="primary")230 with gr.Column():231 vid_out = gr.Video(label="Annotated Output")232 summary = gr.Markdown()233 234 run_btn.click(235 fn=process_video,236 inputs=[vid_in, v_conf, v_smth, v_skel, v_lbl],237 outputs=[vid_out, summary]238 )239 240 # ── Live Stream ──────────────────────────────────────241 with gr.Tab("📷 Live Stream"):242 gr.Markdown(243 "Stand in front of your camera — actions are detected in real time.\n\n"244 "> **Tip:** Make sure your full body is visible for best results."245 )246 with gr.Row():247 with gr.Column():248 webcam_in = gr.Image(sources=["webcam"], streaming=True,249 label="Camera Feed", mirror_webcam=True)250 with gr.Column():251 webcam_out = gr.Image(label="Annotated Output")252 253 with gr.Row():254 l_conf = gr.Slider(0.3, 1.0, value=0.60, step=0.05,255 label="Confidence threshold")256 l_smth = gr.Slider(1, 20, value=5, step=1,257 label="Smoothing window")258 l_skel = gr.Checkbox(value=True, label="Show skeleton")259 l_lbl = gr.Checkbox(value=True, label="Show label")260 261 webcam_in.stream(262 fn=process_live_frame,263 inputs=[webcam_in, l_conf, l_smth, l_skel, l_lbl],264 outputs=webcam_out,265 time_limit=300,266 stream_every=0.04267 )268 269 # ── Settings & Help ──────────────────────────────────270 with gr.Tab("⚙️ Settings & Help"):271 gr.Markdown("""272## Settings Guide273 274### Confidence Threshold275- **0.50–0.65** — More permissive; shows predictions even when uncertain276- **0.70–0.85** — Balanced (recommended)277- **0.90+** — Only very confident predictions; shows "Uncertain" more often278 279### Smoothing Window280Averages probabilities across the last N frames for a stable label.281- **1** — No smoothing (fastest, most jitter)282- **5** — Good default for live use283- **10–15** — Very smooth but adds ~0.4 s lag at 25 fps284 285### Tips for Best Results286- Ensure **full body is in frame** — torso to feet287- Use good lighting; avoid strong backlighting288- Perform actions **clearly and fully** — e.g. raise hand fully above head289- For walking/running, move across the frame rather than toward the camera290 291## Troubleshooting292 293| Issue | Fix |294|-------|-----|295| "No Pose Detected" always | Full body not in frame; improve lighting |296| Predictions jitter a lot | Increase smoothing window to 8–12 |297| Wrong action predicted | Check body visibility; try lowering confidence threshold |298| Webcam not working | Allow browser camera permissions; use Chrome or Edge |299| Video inference is slow | Use a shorter clip or reduce resolution before uploading |300""")301 302 gr.Markdown("<center><small>Built with TensorFlow · MediaPipe · Gradio</small></center>")303 304if __name__ == "__main__":305 demo.launch()306