samurvivor-07/llm-evaluation-framework
0
1import gradio as gr2import pandas as pd3import plotly.graph_objects as go4from plotly.subplots import make_subplots5 6# ── Load Results ───────────────────────────────────────────────7df = pd.read_csv("eval_results.csv")8 9# ── Build Dashboard ────────────────────────────────────────────10def build_dashboard():11 fig = make_subplots(12 rows=2, cols=2,13 subplot_titles=[14 "Overall ROUGE Score Comparison",15 "Performance by Category",16 "Question-level Improvement (Δ ROUGE)",17 "Win Rate Distribution"18 ],19 specs=[20 [{"type": "bar"}, {"type": "bar"}],21 [{"type": "bar"}, {"type": "pie"}]22 ],23 vertical_spacing=0.18,24 horizontal_spacing=0.1225 )26 27 colors = {"base": "#636EFA", "ft": "#00CC96", "pos": "#00CC96", "neg": "#EF553B"}28 29 # Plot 1 — Overall30 for name, col, key in [("Base Mistral 7B", colors["base"], "base"), ("Fine-tuned", colors["ft"], "ft")]:31 fig.add_trace(go.Bar(32 name=name,33 x=["ROUGE-1", "ROUGE-2", "ROUGE-L", "Avg"],34 y=[df[f'{key}_rouge1'].mean(), df[f'{key}_rouge2'].mean(),35 df[f'{key}_rougeL'].mean(), df[f'{key}_avg_rouge'].mean()],36 marker_color=col,37 ), row=1, col=1)38 39 # Plot 2 — Per category40 cat_groups = df.groupby('category').agg(41 base_avg=('base_avg_rouge', 'mean'),42 ft_avg=('ft_avg_rouge', 'mean')43 ).reset_index()44 45 fig.add_trace(go.Bar(name="Base Mistral 7B", x=cat_groups['category'],46 y=cat_groups['base_avg'], marker_color=colors["base"], showlegend=False), row=1, col=2)47 fig.add_trace(go.Bar(name="Fine-tuned", x=cat_groups['category'],48 y=cat_groups['ft_avg'], marker_color=colors["ft"], showlegend=False), row=1, col=2)49 50 # Plot 3 — Per question51 imp_colors = [colors["pos"] if x > 0 else colors["neg"] for x in df['improvement']]52 fig.add_trace(go.Bar(53 x=list(range(1, 21)), y=df['improvement'].tolist(),54 marker_color=imp_colors, showlegend=False,55 hovertext=[q[:50] for q in df['question']], hoverinfo="text+y"56 ), row=2, col=1)57 58 # Plot 4 — Win rate59 wins = (df['improvement'] > 0).sum()60 losses = (df['improvement'] <= 0).sum()61 fig.add_trace(go.Pie(62 labels=["Fine-tuned Wins", "Base Wins"],63 values=[wins, losses],64 marker_colors=[colors["pos"], colors["neg"]],65 hole=0.4, textinfo="label+percent"66 ), row=2, col=2)67 68 fig.update_layout(69 height=650, barmode="group", template="plotly_dark",70 paper_bgcolor="#1e293b", plot_bgcolor="#1e293b",71 font={"color": "white", "size": 11},72 title={"text": "🔬 Medical LLM Evaluation — Base Mistral 7B vs Fine-tuned LoRA", "x": 0.5}73 )74 fig.update_xaxes(tickangle=-25, row=1, col=2)75 fig.update_xaxes(title_text="Question #", row=2, col=1)76 77 return fig78 79def get_details(question_idx):80 if question_idx is None:81 return "", "", ""82 row = df.iloc[int(question_idx)]83 q = f"**Question:** {row['question']}"84 base_ans = f"**Base Model Answer:**\n\n{row['base_answer']}"85 ft_ans = f"**Fine-tuned Answer:**\n\n{row['ft_answer']}\n\n---\n**ROUGE Improvement: {row['improvement']:+.4f}**"86 return q, base_ans, ft_ans87 88css = """89@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800;900&display=swap');90:root {91 --bg-primary:#ffffff;--bg-card:#ffffff;--bg-elevated:#f1f5f9;92 --border:rgba(0,0,0,0.08);--text-primary:#0f172a;--text-secondary:#475569;93 --text-tertiary:#94a3b8;--accent:#6366f1;--accent-dim:rgba(99,102,241,0.1);94 --positive:#10b981;--negative:#ef4444;95 --shadow:0 4px 6px -1px rgba(0,0,0,0.07);--shadow-lg:0 10px 15px -3px rgba(0,0,0,0.08);96 --radius:16px;97}98[data-theme="dark"] {99 --bg-primary:#0f172a;--bg-card:#1e293b;--bg-elevated:#334155;100 --border:rgba(255,255,255,0.08);--text-primary:#f1f5f9;101 --text-secondary:#94a3b8;--text-tertiary:#475569;102 --shadow:0 4px 6px -1px rgba(0,0,0,0.3);--shadow-lg:0 10px 15px -3px rgba(0,0,0,0.4);103}104* { font-family:"Inter",sans-serif !important; box-sizing:border-box; }105body,.gradio-container { background:var(--bg-primary) !important; color:var(--text-primary) !important; transition:all 0.3s ease; }106.gradio-container { max-width:1100px !important; margin:0 auto !important; padding:24px !important; }107.app-header { text-align:center; padding:52px 24px 36px; background:linear-gradient(135deg,var(--accent-dim),transparent); border-radius:var(--radius); border:1px solid var(--border); margin-bottom:28px; }108.app-header h1 { font-size:2.4em !important; font-weight:900 !important; background:linear-gradient(135deg,var(--accent),#8b5cf6,#10b981); -webkit-background-clip:text !important; -webkit-text-fill-color:transparent !important; margin-bottom:14px !important; }109.app-header p { color:var(--text-secondary) !important; font-size:1em !important; max-width:650px; margin:0 auto !important; line-height:1.8 !important; }110.badges { display:flex; justify-content:center; gap:10px; flex-wrap:wrap; margin-top:20px; }111.badge { padding:6px 16px; border-radius:20px; font-size:0.75em; font-weight:600; border:1px solid var(--border); background:var(--bg-elevated); color:var(--text-secondary); }112.badge.accent { background:var(--accent-dim); color:var(--accent); border-color:rgba(99,102,241,0.25); }113.badge.positive { background:rgba(16,185,129,0.1); color:#10b981; border-color:rgba(16,185,129,0.25); }114.stats-row { display:grid; grid-template-columns:repeat(4,1fr); gap:16px; margin-bottom:28px; }115.stat-card { background:var(--bg-card); border:1px solid var(--border); border-radius:var(--radius); padding:20px 16px; text-align:center; box-shadow:var(--shadow); transition:all 0.2s ease; }116.stat-card:hover { transform:translateY(-2px); box-shadow:var(--shadow-lg); }117.stat-value { font-size:1.9em; font-weight:900; color:var(--accent); line-height:1; }118.stat-label { font-size:0.7em; color:var(--text-tertiary); font-weight:600; text-transform:uppercase; letter-spacing:0.8px; margin-top:6px; }119.card { background:var(--bg-card) !important; border:1px solid var(--border) !important; border-radius:var(--radius) !important; padding:24px !important; box-shadow:var(--shadow) !important; }120.theme-toggle { position:fixed !important; top:20px !important; right:20px !important; background:var(--bg-card) !important; border:1px solid var(--border) !important; border-radius:50% !important; width:46px !important; height:46px !important; cursor:pointer !important; font-size:1.2em !important; box-shadow:var(--shadow-lg) !important; transition:all 0.2s !important; z-index:1000 !important; }121.theme-toggle:hover { transform:scale(1.12) rotate(15deg) !important; }122label { color:var(--text-secondary) !important; font-weight:700 !important; font-size:0.78em !important; text-transform:uppercase !important; letter-spacing:1px !important; }123.footer { text-align:center; padding:28px; color:var(--text-tertiary); font-size:0.78em; border-top:1px solid var(--border); margin-top:36px; line-height:2; }124"""125 126js = """127function toggleTheme(){128 const root=document.documentElement;129 const btn=document.getElementById("theme-btn");130 const dark=root.getAttribute("data-theme")==="dark";131 root.setAttribute("data-theme",dark?"light":"dark");132 btn.textContent=dark?"🌙":"☀️";133 localStorage.setItem("eval-theme",dark?"light":"dark");134}135document.addEventListener("DOMContentLoaded",()=>{136 const saved=localStorage.getItem("eval-theme")||"light";137 document.documentElement.setAttribute("data-theme",saved);138 const btn=document.getElementById("theme-btn");139 if(btn)btn.textContent=saved==="dark"?"☀️":"🌙";140});141"""142 143with gr.Blocks(css=css, title="🔬 LLM Evaluation Framework") as demo:144 145 gr.HTML(f"""146 <button class="theme-toggle" id="theme-btn" onclick="toggleTheme()">🌙</button>147 <script>{js}</script>148 <div class="app-header">149 <h1>🔬 LLM Evaluation Framework</h1>150 <p>Research-grade evaluation comparing Base Mistral 7B vs LoRA Fine-tuned Medical AI151 across 20 questions in 5 clinical domains. Interactive results dashboard.</p>152 <div class="badges">153 <span class="badge accent">Mistral 7B</span>154 <span class="badge accent">LoRA Fine-tuned</span>155 <span class="badge positive">+17.8% ROUGE</span>156 <span class="badge positive">65% Win Rate</span>157 <span class="badge">5 Medical Domains</span>158 <span class="badge">20 Questions</span>159 </div>160 </div>161 162 <div class="stats-row">163 <div class="stat-card">164 <div class="stat-value">+17.8%</div>165 <div class="stat-label">ROUGE Improvement</div>166 </div>167 <div class="stat-card">168 <div class="stat-value">65%</div>169 <div class="stat-label">Win Rate</div>170 </div>171 <div class="stat-card">172 <div class="stat-value">20</div>173 <div class="stat-label">Questions Evaluated</div>174 </div>175 <div class="stat-card">176 <div class="stat-value">5</div>177 <div class="stat-label">Medical Domains</div>178 </div>179 </div>180 """)181 182 with gr.Column(elem_classes=["card"]):183 gr.Markdown("### 📊 Evaluation Dashboard")184 dashboard_plot = gr.Plot(value=build_dashboard())185 186 with gr.Column(elem_classes=["card"]):187 gr.Markdown("### 🔍 Question-level Analysis")188 question_slider = gr.Slider(189 minimum=0, maximum=19, step=1, value=0,190 label="Select Question (0-19)"191 )192 with gr.Row():193 question_out = gr.Markdown()194 with gr.Row():195 base_out = gr.Markdown(label="Base Model")196 ft_out = gr.Markdown(label="Fine-tuned Model")197 198 gr.HTML("""199 <div style="margin-top:28px;background:var(--bg-card);border:1px solid var(--border);border-radius:16px;padding:28px;box-shadow:var(--shadow);">200 <h3 style="color:var(--text-primary);margin-bottom:20px;font-size:1em;font-weight:700;">📋 Evaluation Categories</h3>201 <div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:12px;">202 <div style="padding:16px;background:var(--bg-elevated);border-radius:12px;text-align:center;">203 <div style="font-size:1.8em;margin-bottom:8px;">💊</div>204 <div style="font-weight:700;color:var(--text-primary);font-size:0.85em;">Pharmacology</div>205 <div style="color:var(--text-tertiary);font-size:0.75em;margin-top:4px;">4 questions</div>206 </div>207 <div style="padding:16px;background:var(--bg-elevated);border-radius:12px;text-align:center;">208 <div style="font-size:1.8em;margin-bottom:8px;">🩺</div>209 <div style="font-weight:700;color:var(--text-primary);font-size:0.85em;">Symptoms & Diagnosis</div>210 <div style="color:var(--text-tertiary);font-size:0.75em;margin-top:4px;">4 questions</div>211 </div>212 <div style="padding:16px;background:var(--bg-elevated);border-radius:12px;text-align:center;">213 <div style="font-size:1.8em;margin-bottom:8px;">🧬</div>214 <div style="font-weight:700;color:var(--text-primary);font-size:0.85em;">Pathophysiology</div>215 <div style="color:var(--text-tertiary);font-size:0.75em;margin-top:4px;">4 questions</div>216 </div>217 <div style="padding:16px;background:var(--bg-elevated);border-radius:12px;text-align:center;">218 <div style="font-size:1.8em;margin-bottom:8px;">💉</div>219 <div style="font-weight:700;color:var(--text-primary);font-size:0.85em;">Treatment</div>220 <div style="color:var(--text-tertiary);font-size:0.75em;margin-top:4px;">4 questions</div>221 </div>222 <div style="padding:16px;background:var(--bg-elevated);border-radius:12px;text-align:center;">223 <div style="font-size:1.8em;margin-bottom:8px;">🫀</div>224 <div style="font-weight:700;color:var(--text-primary);font-size:0.85em;">Anatomy & Physiology</div>225 <div style="color:var(--text-tertiary);font-size:0.75em;margin-top:4px;">4 questions</div>226 </div>227 </div>228 </div>229 230 <div class="footer">231 Base Mistral 7B vs LoRA Fine-tuned • ROUGE-1, ROUGE-2, ROUGE-L Metrics •232 20 Medical QA Questions • 5 Clinical Domains •233 <a href="https://github.com/Boatengs/llm-evaluation-framework" target="_blank" style="color:var(--accent);">GitHub</a> •234 <a href="https://huggingface.co/samboateng190/medical-mistral-lora" target="_blank" style="color:var(--accent);">Fine-tuned Model</a>235 </div>236 """)237 238 question_slider.change(239 fn=get_details,240 inputs=question_slider,241 outputs=[question_out, base_out, ft_out]242 )243 244demo.launch()245