YasithaDulara/Letters
0
1import io2import os3import torch4import torch.nn as nn5from PIL import Image6from flask import Flask, request, jsonify7from flask_cors import CORS8from torchvision import transforms, models9from datetime import datetime10from email_service import send_parent_email11from gemini_service import generate_child_feedback, generate_comprehensive_summary12import base6413 14# =========================15# APP SETUP16# =========================17app = Flask(__name__)18CORS(app)19 20UPLOAD_FOLDER = "uploads"21CONFIDENCE_THRESHOLD = 0.8 # 80%22 23LETTERS = ["A", "B", "C", "D", "E", "F", "G", "H","J", "L", "N", "O", "P", "R", "S", "U", "V"]24FLIPPABLE_LETTERS = ["B","C","D","E","F","G","J","L","P","R","S"]25 26# Store quiz sessions in memory (in production, use a database)27quiz_sessions = {}28 29if not os.path.exists(UPLOAD_FOLDER):30 os.makedirs(UPLOAD_FOLDER)31 32# =========================33# IMAGE TRANSFORM34# =========================35transform = transforms.Compose([36 transforms.Grayscale(num_output_channels=1),37 transforms.ToTensor(),38 transforms.Normalize([0.5], [0.5])39])40 41# =========================42# MODEL43# =========================44letters_model = models.resnet18(weights="IMAGENET1K_V1")45letters_model.conv1 = nn.Conv2d(46 1, 64, kernel_size=7, stride=2, padding=3, bias=False47)48letters_model.fc = nn.Linear(letters_model.fc.in_features, len(LETTERS))49 50letters_model.load_state_dict(51 torch.load("letters_models/letters_model.pth", map_location=torch.device("cpu"))52)53letters_model.eval()54 55letters_flipped_model = models.resnet18(weights="IMAGENET1K_V1")56letters_flipped_model.conv1 = nn.Conv2d(57 1, 64, kernel_size=7, stride=2, padding=3, bias=False58)59letters_flipped_model.fc = nn.Linear(letters_flipped_model.fc.in_features, len(FLIPPABLE_LETTERS))60 61letters_flipped_model.load_state_dict(62 torch.load("letters_models/fliped_letters_model.pth", map_location=torch.device("cpu"))63)64letters_flipped_model.eval()65 66 67# =========================68# HELPER FUNCTIONS69# =========================70def get_session_key(parent_email):71 """Generate a session key based on parent email"""72 return parent_email.lower().strip()73 74# =========================75# KID FRIENDLY MESSAGES76# =========================77def get_kid_message(actual, predicted, confidence, is_flipped=False):78 79 if predicted == actual:80 if is_flipped:81 if confidence >= 0.8:82 return {83 "message": f"🔄 Great try! You wrote {actual}, but it's backwards. Try writing it the other way!",84 "emoji": "🪞",85 "confidence_level": "high",86 "is_flipped": True87 }88 elif confidence >= 0.5:89 return {90 "message": f"🔄 Nice effort! That's {actual} flipped around. Let's practice writing it the correct way!",91 "emoji": "✨",92 "confidence_level": "medium",93 "is_flipped": True94 }95 else:96 return {97 "message": f"🔄 I see {actual} backwards! Let's try writing it facing the right direction!",98 "emoji": "💛",99 "confidence_level": "low",100 "is_flipped": True101 }102 else:103 if confidence >= 0.8:104 return {105 "message": f"🎉 Awesome job! You wrote the letter {actual} perfectly!",106 "emoji": "🌟",107 "confidence_level": "high",108 "is_flipped": False109 }110 elif confidence >= 0.5:111 return {112 "message": f"😊 Nice work! That looks like {actual}. Let's try once more to make it even better!",113 "emoji": "✨",114 "confidence_level": "medium",115 "is_flipped": False116 }117 else:118 return {119 "message": f"👍 Good try! I can see {actual}. Let's practice it again together!",120 "emoji": "💛",121 "confidence_level": "low",122 "is_flipped": False123 }124 else:125 return {126 "message": "💪 Good effort! Let's try again and make the letter even clearer!",127 "emoji": "🌈",128 "confidence_level": "low",129 "is_flipped": False130 }131 132# =========================133# PREDICT ROUTE134# =========================135@app.route("/predict", methods=["POST"])136def predict_letter():137 if "image" not in request.files:138 return jsonify({"error": "No image uploaded"}), 400139 140 if "actual_letter" not in request.form:141 return jsonify({"error": "Actual letter missing"}), 400142 143 file = request.files["image"]144 actual_letter = request.form["actual_letter"].upper()145 parent_email = request.form.get("parent_email", "").strip()146 147 try:148 image_bytes = file.read()149 image = Image.open(io.BytesIO(image_bytes)).convert("L")150 151 timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")152 filename = f"{actual_letter}_{timestamp}.png"153 image.save(os.path.join(UPLOAD_FOLDER, filename))154 155 input_tensor = transform(image).unsqueeze(0)156 157 with torch.no_grad():158 outputs = letters_model(input_tensor)159 probs = torch.softmax(outputs, dim=1)160 161 top_prob, top_idx = torch.max(probs, 1)162 predicted_letter = LETTERS[top_idx.item()]163 confidence = top_prob.item()164 165 is_flipped = False166 flipped_confidence = 0.0167 168 if predicted_letter != actual_letter and actual_letter in FLIPPABLE_LETTERS:169 with torch.no_grad():170 flipped_outputs = letters_flipped_model(input_tensor)171 flipped_probs = torch.softmax(flipped_outputs, dim=1)172 173 flipped_top_prob, flipped_top_idx = torch.max(flipped_probs, 1)174 flipped_predicted_letter = FLIPPABLE_LETTERS[flipped_top_idx.item()]175 flipped_confidence = flipped_top_prob.item()176 177 if flipped_predicted_letter == actual_letter:178 predicted_letter = flipped_predicted_letter179 confidence = flipped_confidence180 is_flipped = True181 182 is_correct = (predicted_letter == actual_letter)183 184 kid_response = get_kid_message(actual_letter, predicted_letter, confidence, is_flipped)185 186 gemini_feedback = generate_child_feedback(187 letter=actual_letter,188 confidence=confidence,189 is_correct=is_correct,190 flip_used=is_flipped191 )192 193 if gemini_feedback:194 kid_response["message"] = gemini_feedback195 196 if parent_email:197 session_key = get_session_key(parent_email)198 if session_key not in quiz_sessions:199 quiz_sessions[session_key] = {200 "parent_email": parent_email,201 "start_time": datetime.now().isoformat(),202 "attempts": []203 }204 205 quiz_sessions[session_key]["attempts"].append({206 "actual_letter": actual_letter,207 "predicted_letter": predicted_letter,208 "confidence": confidence,209 "is_correct": is_correct,210 "is_flipped": is_flipped,211 "timestamp": datetime.now().isoformat(),212 "image_path": os.path.join(UPLOAD_FOLDER, filename)213 })214 215 os.remove(os.path.join(UPLOAD_FOLDER, filename))216 217 return jsonify({218 "predicted_letter": predicted_letter,219 "confidence": f"{confidence * 100:.2f}%",220 "is_correct": is_correct,221 "is_flipped": is_flipped,222 "kid_message": kid_response["message"],223 "emoji": kid_response["emoji"],224 "confidence_level": kid_response["confidence_level"]225 }), 200226 227 except Exception as e:228 return jsonify({"error": str(e)}), 500229 230# =========================231# SEND SUMMARY EMAIL232# =========================233@app.route("/send-summary", methods=["POST"])234def send_summary():235 try:236 data = request.json237 parent_email = data.get("parent_email", "").strip()238 239 if not parent_email:240 return jsonify({"error": "Parent email required"}), 400241 242 session_key = get_session_key(parent_email)243 244 if session_key not in quiz_sessions:245 return jsonify({"error": "No quiz session found for this email"}), 404246 247 session_data = quiz_sessions[session_key]248 attempts = session_data["attempts"]249 250 if not attempts:251 return jsonify({"error": "No attempts recorded"}), 400252 253 total_attempts = len(attempts)254 correct_attempts = sum(1 for a in attempts if a["is_correct"])255 flipped_attempts = sum(1 for a in attempts if a["is_flipped"])256 accuracy = (correct_attempts / total_attempts * 100) if total_attempts > 0 else 0257 258 letter_performance = {}259 for attempt in attempts:260 letter = attempt["actual_letter"]261 if letter not in letter_performance:262 letter_performance[letter] = {263 "attempts": 0,264 "correct": 0,265 "flipped": 0,266 "avg_confidence": 0,267 "confidences": []268 }269 270 letter_performance[letter]["attempts"] += 1271 if attempt["is_correct"]:272 letter_performance[letter]["correct"] += 1273 if attempt["is_flipped"]:274 letter_performance[letter]["flipped"] += 1275 letter_performance[letter]["confidences"].append(attempt["confidence"])276 277 for letter in letter_performance:278 confidences = letter_performance[letter]["confidences"]279 letter_performance[letter]["avg_confidence"] = sum(confidences) / len(confidences)280 letter_performance[letter]["accuracy"] = (281 letter_performance[letter]["correct"] / letter_performance[letter]["attempts"] * 100282 )283 284 strong_letters = [285 letter for letter, perf in letter_performance.items() 286 if perf["accuracy"] >= 80287 ]288 weak_letters = [289 letter for letter, perf in letter_performance.items() 290 if perf["accuracy"] < 60291 ]292 flipped_letters = [293 letter for letter, perf in letter_performance.items() 294 if perf["flipped"] > 0295 ]296 297 gemini_result = generate_comprehensive_summary(298 total_attempts=total_attempts,299 correct_attempts=correct_attempts,300 accuracy=accuracy,301 flipped_attempts=flipped_attempts,302 strong_letters=strong_letters,303 weak_letters=weak_letters,304 flipped_letters=flipped_letters,305 letter_performance=letter_performance306 )307 308 gemini_summary = gemini_result.get("summary", "")309 suggestions = gemini_result.get("suggestions", [])310 311 analysis_highlights = [312 f"Completed {total_attempts} letters with {accuracy:.1f}% accuracy",313 f"{correct_attempts} letters recognized correctly",314 ]315 316 if strong_letters:317 analysis_highlights.append(318 f"Strong performance on: {', '.join(strong_letters[:5])}"319 )320 321 if flipped_letters:322 analysis_highlights.append(323 f"Mirror writing detected in {len(flipped_letters)} letter(s): {', '.join(flipped_letters)}"324 )325 326 email_summary = {327 "predicted_letter": f"{correct_attempts}/{total_attempts} Correct",328 "confidence": f"{accuracy:.1f}%",329 "analysis_highlights": analysis_highlights,330 "kid_message": gemini_summary,331 "flip_used": flipped_attempts > 0,332 "strong_letters": strong_letters,333 "weak_letters": weak_letters,334 "flipped_letters": flipped_letters,335 "letter_performance": letter_performance336 }337 338 email_result = send_parent_email(339 parent_email=parent_email,340 summary=email_summary,341 suggestions=suggestions342 )343 344 if email_result["status"] == "sent":345 return jsonify({346 "status": "success",347 "message": "Summary email sent successfully",348 "summary": {349 "total_attempts": total_attempts,350 "correct_attempts": correct_attempts,351 "accuracy": accuracy,352 "strong_letters": strong_letters,353 "weak_letters": weak_letters,354 "flipped_letters": flipped_letters,355 "gemini_summary": gemini_summary,356 "letter_performance": letter_performance357 }358 }), 200359 else:360 return jsonify({361 "status": "error",362 "message": "Failed to send email",363 "error": email_result.get("error", "Unknown error")364 }), 500365 366 except Exception as e:367 import traceback368 print(f"Error in send_summary: {str(e)}")369 print(traceback.format_exc())370 return jsonify({"error": str(e), "trace": traceback.format_exc()}), 500371 372 373# =========================374# GET SESSION SUMMARY375# =========================376@app.route("/get-summary/<parent_email>", methods=["GET"])377def get_summary(parent_email):378 try:379 session_key = get_session_key(parent_email)380 381 if session_key not in quiz_sessions:382 return jsonify({"error": "No session found"}), 404383 384 session_data = quiz_sessions[session_key]385 attempts = session_data["attempts"]386 387 total_attempts = len(attempts)388 correct_attempts = sum(1 for a in attempts if a["is_correct"])389 flipped_attempts = sum(1 for a in attempts if a["is_flipped"])390 391 letter_performance = {}392 for attempt in attempts:393 letter = attempt["actual_letter"]394 if letter not in letter_performance:395 letter_performance[letter] = {396 "attempts": 0,397 "correct": 0,398 "flipped": 0,399 "confidences": []400 }401 402 letter_performance[letter]["attempts"] += 1403 if attempt["is_correct"]:404 letter_performance[letter]["correct"] += 1405 if attempt["is_flipped"]:406 letter_performance[letter]["flipped"] += 1407 letter_performance[letter]["confidences"].append(attempt["confidence"])408 409 for letter in letter_performance:410 confidences = letter_performance[letter]["confidences"]411 letter_performance[letter]["avg_confidence"] = sum(confidences) / len(confidences)412 letter_performance[letter]["accuracy"] = (413 letter_performance[letter]["correct"] / letter_performance[letter]["attempts"] * 100414 )415 416 return jsonify({417 "total_attempts": total_attempts,418 "correct_attempts": correct_attempts,419 "accuracy": (correct_attempts / total_attempts * 100) if total_attempts > 0 else 0,420 "flipped_attempts": flipped_attempts,421 "letter_performance": letter_performance,422 "attempts": attempts423 }), 200424 425 except Exception as e:426 return jsonify({"error": str(e)}), 500427 428 429# =========================430# HEALTH CHECK431# =========================432@app.route("/", methods=["GET"])433def health():434 return jsonify({"status": "ok", "message": "Little Learners Hub Backend is running"}), 200435 436 437# =========================438# RUN APP439# =========================440if __name__ == "__main__":441 port = int(os.environ.get("PORT", 7860))442 app.run(host="0.0.0.0", port=port, debug=False)443