evolvingtransformer/ats_demo

1# model_load.py2import os3import shutil4import json5import regex as re6import torch7from transformers import AutoTokenizer, AutoModelForCausalLM8 9local_model_dir = "./mistral_model"10model_name = "mistralai/Mistral-7B-Instruct-v0.3"11 12def load_model():13    global tokenizer, model14 15    # ---- 1. Check if the folder exists and looks healthy ----16    required_files = ["config.json", "generation_config.json", "tokenizer.json"]17    is_healthy = os.path.isdir(local_model_dir) and all(18        os.path.exists(os.path.join(local_model_dir, f)) for f in required_files19    )20 21    # ---- 2. If not healthy → delete and redownload ----22    if not is_healthy:23        print("Model folder is missing or corrupted → deleting and redownloading...")24        if os.path.exists(local_model_dir):25            shutil.rmtree(local_model_dir)  # full clean26        os.makedirs(local_model_dir, exist_ok=True)27 28        tokenizer = AutoTokenizer.from_pretrained(model_name)29        tokenizer.save_pretrained(local_model_dir)30 31        model = AutoModelForCausalLM.from_pretrained(32            model_name,33            torch_dtype=torch.float16,34            device_map="auto",35            trust_remote_code=True36        )37        model.save_pretrained(local_model_dir)38        print("Fresh model downloaded and saved locally.")39    else:40        print("Loading model from healthy local folder...")41        tokenizer = AutoTokenizer.from_pretrained(local_model_dir)42        model = AutoModelForCausalLM.from_pretrained(43            local_model_dir,44            torch_dtype=torch.float16,45            device_map="auto",46            trust_remote_code=True47        ).eval()48 49    return tokenizer, model50 51 52# Load once at import time53tokenizer, model = load_model()54 55# Keep all your other functions (summarize_jd, extract_skills, etc.) unchanged below56# === Rest of your functions (score_candidate_v4, skills_matcher, etc.) remain unchanged ===57# But here's the FIXED summarize_jd with robust parsing:58 59def summarize_jd(jd_text, max_tokens=512):60    prompt = f"""[INST] You are an expert recruiter. Rewrite the following job description in clean, structured markdown format.61 62Job Description:63{jd_text}64 65Output exactly in this format:66 67# Job Title68<clear one-line title>69 70# Job Summary71<3-4 sentence summary>72 73# Key Responsibilities74- Point one75- Point two76 77# Required Skills78- Skill one79- Skill two80 81# Experience82<Number> years83 84# Preferred Skills85- Nice to have86 87# Skills (JSON)88{{"skills": ["skill1", "skill2", "skill3"]}}89[/INST]"""90 91    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)92    outputs = model.generate(93        **inputs,94        max_new_tokens=max_tokens,95        temperature=0.1,96        do_sample=False,97        pad_token_id=tokenizer.eos_token_id98    )99    response = tokenizer.decode(outputs[0], skip_special_tokens=True)100 101    # Extract JSON block safely102    json_match = re.search(r'\{.*"skills".*\}', response, re.DOTALL)103    skills = []104    exp = "0"105    full_text = response.split("[/INST]")[-1] if "[/INST]" in response else response106 107    if json_match:108        try:109            skills = json.loads(json_match.group(0))["skills"]110        except:111            skills = []112 113    # Extract experience number114    exp_match = re.search(r'# Experience\s*\n\s*([0-9]+)', response)115    if exp_match:116        exp = exp_match.group(1)117 118    return {"Status": "OK", "output": [full_text.strip(), skills, exp]}119 120def score_candidate_v4(data:dict, required_years:int, critical_skills=None):121    if critical_skills is None:122        critical_skills = []123 124    # --- 1. SKILL SCORING (Max 60 pts) ---125    # We create a bucket. Exact fills it fast; Semantic fills it slower.126    # We stop adding points once we hit 60 to prevent "keyword stuffing" inflation.127    128    skill_pts = 0129    skill_pts += len(data.get("exact_matches", [])) * 10130    skill_pts += len(data.get("semantic_matches", [])) * 6  # Lowered from 8 to 6131    skill_pts += len(data.get("partial_matches", [])) * 2   # Lowered from 3 to 2132    133    # Hard cap on positive skill points134    skill_score = min(skill_pts, 60)135 136    # --- 2. EXPERIENCE SCORING (Max 40 pts) ---137    # Adjusted to sum to 40 so 60+40=100138    actual_years = data.get("candidate_relevant_years", 0)139    gap = actual_years - required_years140 141    if gap >= 3:142        exp_pts = 40       # Bonus for solid seniority143    elif gap >= 1:144        exp_pts = 35       # Meets + buffer145    elif gap >= 0:146        exp_pts = 30       # Meets exactly147    elif gap >= -2:148        exp_pts = 15       # Slightly under (Seniority Penalty)149    else:150        exp_pts = 0        # Significantly under151 152    # --- 3. PENALTIES (Subtractive) ---153    missing = data.get("missing_skills", [])154    155    # Critical misses hit hard156    crit_miss_count = sum(1 for skill in missing if skill in critical_skills)157    penalty_critical = crit_miss_count * 25 158    159    # Non-critical misses shouldn't hurt too much (cap the penalty)160    non_crit_count = len(missing) - crit_miss_count161    penalty_general = min(non_crit_count * 1, 10) # Max 10 pt penalty for fluff skills162 163    # --- FINAL CALC ---164    raw_score = (skill_score + exp_pts) - (penalty_critical + penalty_general)165    166    return max(0, min(100, raw_score)) 167 168def extract_skills(resume_text, max_tokens=512):169    # Function calling schema (manually enforced)170    schema_description = """171Your task is to extract ONLY the technical skills, experience and e-mail id from the user's resume.172 173Return JSON following exactly this schema:174 175Expected output:176 177{{178  "skills": ["Skill1", "Skill2", "Skill3"],179  "E-mail":"user's e-mail id",180  "experience": "Years of experience"181}}182 183Rules:184- ONLY return valid JSON.185- Remove the special characters.186- Do NOT add explanations or commentary.187- Expand abbreviations (CNN → Convolutional Neural Network).188- Include every technical skill found in the resume.189- Remove duplicates.190- If no skills found, return {{"skills": [], "E-mail":"user's e-mail id"}}191- If no email found return {{"skills": ["Skill1", "Skill2", "Skill3"], "E-mail":""}}192- If no skills and no email found return {{"skills": [], "E-mail":""}}193- If the user has no experience return the experience as 0.194"""195 196    prompt = schema_description + "\n\nResume text:\n" + resume_text197 198    # Prepare input199    device = next(model.parameters()).device200    inputs = tokenizer(prompt, return_tensors="pt")201    inputs = {k: v.to(device) for k, v in inputs.items()}202 203    # Generate204    outputs = model.generate(205        **inputs,206        max_new_tokens=max_tokens,207        temperature=0.1,        # deterministic208        do_sample=False209    )210 211    response = tokenizer.decode(outputs[0], skip_special_tokens=True)212    #return response213 214    # Extract JSON from output215    try:216        json_start = response.rfind("{")217        json_end = response.rfind("}") + 1218        json_str = response[json_start:json_end]219 220        result = json.loads(json_str)221        return {"Status":"OK", "output":result}222 223    except Exception as e:224        return {"Status":"ERROR", "output":[str(e)]}225def skills_matcher(js, cs, cex, jex, max_tokens=512):226 227    prompt = f"""You are an expert ATS (Applicant Tracking System) scoring engine.228Your task is to compare candidate skills with job description skills and classify each skill according to how well they match.229 230You MUST return ONLY valid JSON using the exact format shown below.231 232Expected output:233{{234  "exact_matches": [],235  "semantic_matches": [],236  "partial_matches": [],237  "missing_skills": [],238  "candidate_relevant_years": {cex}239}}240 241### Classification Rules242Classify each required job skill into one of the categories below:243 2441. EXACT MATCH:245   - Candidate skill matches the job skill word-for-word or very close variation.246 2472. SEMANTIC MATCH:248   - Meaning is the same even if wording differs.249 2503. PARTIAL MATCH:251   - Related but not fully covering the job requirement.252 2534. MISSING:254   - The job skill is not present in any form in the candidate skills.255 256### Input Data257Candidate Skills:258{cs}259 260Candidate Experience (years):261{cex}262 263Job Description Skills:264{js}265 266Required Experience (years):267{jex}268### Output data269 270"""271 272    device = next(model.parameters()).device273    inputs = tokenizer(prompt, return_tensors="pt")274    inputs = {k: v.to(device) for k, v in inputs.items()}275 276    outputs = model.generate(277        **inputs,278        max_new_tokens=max_tokens,279        temperature=0.1,280        do_sample=False281    )282 283    response = tokenizer.decode(outputs[0], skip_special_tokens=True)284    285 286    # Extract JSON only287    json_start = response.rfind("### Output data")288    json_str = response[json_start+len("### Output data"):]289 290    try:291        return json.loads(json_str)292    except:293        return {"error": "Failed to parse JSON", "raw": response}294