DevMastersZA/Marco_Professional_Profile
0
1{2 "cells": [3 {4 "cell_type": "markdown",5 "metadata": {},6 "source": [7 "## Welcome to the Second Lab - Week 1, Day 3\n",8 "\n",9 "Today we will work with lots of models! This is a way to get comfortable with APIs."10 ]11 },12 {13 "cell_type": "code",14 "execution_count": null,15 "metadata": {},16 "outputs": [],17 "source": [18 "# Start with imports - ask ChatGPT to explain any package that you don't know\n",19 "import os\n",20 "import json\n",21 "from dotenv import load_dotenv\n",22 "from openai import OpenAI\n",23 "from anthropic import Anthropic\n",24 "from IPython.display import Markdown, display"25 ]26 },27 {28 "cell_type": "code",29 "execution_count": null,30 "metadata": {},31 "outputs": [],32 "source": [33 "# Always remember to do this!\n",34 "load_dotenv(override=True)"35 ]36 },37 {38 "cell_type": "code",39 "execution_count": null,40 "metadata": {},41 "outputs": [],42 "source": [43 "# Print the key prefixes to help with any debugging\n",44 "\n",45 "openai_api_key = os.getenv('OPENAI_API_KEY')\n",46 "anthropic_api_key = os.getenv('ANTHROPIC_API_KEY')\n",47 "google_api_key = os.getenv('GOOGLE_API_KEY')\n",48 "deepseek_api_key = os.getenv('DEEPSEEK_API_KEY')\n",49 "groq_api_key = os.getenv('GROQ_API_KEY')\n",50 "\n",51 "if openai_api_key:\n",52 " print(f\"OpenAI API Key exists and begins {openai_api_key[:8]}\")\n",53 "else:\n",54 " print(\"OpenAI API Key not set\")\n",55 " \n",56 "if anthropic_api_key:\n",57 " print(f\"Anthropic API Key exists and begins {anthropic_api_key[:7]}\")\n",58 "else:\n",59 " print(\"Anthropic API Key not set (and this is optional)\")\n",60 "\n",61 "if google_api_key:\n",62 " print(f\"Google API Key exists and begins {google_api_key[:2]}\")\n",63 "else:\n",64 " print(\"Google API Key not set (and this is optional)\")\n",65 "\n",66 "if deepseek_api_key:\n",67 " print(f\"DeepSeek API Key exists and begins {deepseek_api_key[:3]}\")\n",68 "else:\n",69 " print(\"DeepSeek API Key not set (and this is optional)\")\n",70 "\n",71 "if groq_api_key:\n",72 " print(f\"Groq API Key exists and begins {groq_api_key[:4]}\")\n",73 "else:\n",74 " print(\"Groq API Key not set (and this is optional)\")"75 ]76 },77 {78 "cell_type": "code",79 "execution_count": null,80 "metadata": {},81 "outputs": [],82 "source": [83 "request = \"Please come up with a challenging, nuanced question that I can ask a number of LLMs to evaluate their intelligence. \"\n",84 "request += \"Answer only with the question, no explanation.\"\n",85 "messages = [{\"role\": \"user\", \"content\": request}]"86 ]87 },88 {89 "cell_type": "code",90 "execution_count": null,91 "metadata": {},92 "outputs": [],93 "source": [94 "messages"95 ]96 },97 {98 "cell_type": "code",99 "execution_count": null,100 "metadata": {},101 "outputs": [],102 "source": [103 "openai = OpenAI()\n",104 "response = openai.chat.completions.create(\n",105 " model=\"gpt-5-mini\",\n",106 " messages=messages,\n",107 ")\n",108 "question = response.choices[0].message.content\n",109 "print(question)"110 ]111 },112 {113 "cell_type": "code",114 "execution_count": null,115 "metadata": {},116 "outputs": [],117 "source": [118 "competitors = []\n",119 "answers = []\n",120 "messages = [{\"role\": \"user\", \"content\": question}]"121 ]122 },123 {124 "cell_type": "code",125 "execution_count": null,126 "metadata": {},127 "outputs": [],128 "source": [129 "# The API we know well\n",130 "# I've updated this with the latest model, but it can take some time because it likes to think!\n",131 "# Replace the model with gpt-4.1-mini if you'd prefer not to wait 1-2 mins\n",132 "\n",133 "model_name = \"gpt-5-nano\"\n",134 "\n",135 "response = openai.chat.completions.create(model=model_name, messages=messages)\n",136 "answer = response.choices[0].message.content\n",137 "\n",138 "display(Markdown(answer))\n",139 "competitors.append(model_name)\n",140 "answers.append(answer)"141 ]142 },143 {144 "cell_type": "code",145 "execution_count": null,146 "metadata": {},147 "outputs": [],148 "source": [149 "gemini = OpenAI(api_key=google_api_key, base_url=\"https://generativelanguage.googleapis.com/v1beta/openai/\")\n",150 "model_name = \"gemini-2.5-flash\"\n",151 "\n",152 "response = gemini.chat.completions.create(model=model_name, messages=messages)\n",153 "answer = response.choices[0].message.content\n",154 "\n",155 "display(Markdown(answer))\n",156 "competitors.append(model_name)\n",157 "answers.append(answer)"158 ]159 },160 {161 "cell_type": "code",162 "execution_count": null,163 "metadata": {},164 "outputs": [],165 "source": [166 "model_name = \"gemini-3-flash-preview\"\n",167 "\n",168 "response = gemini.chat.completions.create(model=model_name, messages=messages)\n",169 "answer = response.choices[0].message.content\n",170 "\n",171 "display(Markdown(answer))\n",172 "competitors.append(model_name)\n",173 "answers.append(answer)"174 ]175 },176 {177 "cell_type": "code",178 "execution_count": null,179 "metadata": {},180 "outputs": [],181 "source": [182 "# So where are we?\n",183 "print(competitors)\n",184 "print(answers)"185 ]186 },187 {188 "cell_type": "code",189 "execution_count": null,190 "metadata": {},191 "outputs": [],192 "source": [193 "# It's nice to know how to use \"zip\"\n",194 "for competitor, answer in zip(competitors, answers):\n",195 " print(f\"Competitor: {competitor}\\n\\n{answer}\")\n"196 ]197 },198 {199 "cell_type": "code",200 "execution_count": null,201 "metadata": {},202 "outputs": [],203 "source": [204 "# Let's bring this together - note the use of \"enumerate\"\n",205 "together = \"\"\n",206 "for index, answer in enumerate(answers):\n",207 " together += f\"# Response from competitor {index+1}\\n\\n\"\n",208 " together += answer + \"\\n\\n\""209 ]210 },211 {212 "cell_type": "code",213 "execution_count": null,214 "metadata": {},215 "outputs": [],216 "source": [217 "print(together)"218 ]219 },220 {221 "cell_type": "code",222 "execution_count": null,223 "metadata": {},224 "outputs": [],225 "source": [226 "judge = f\"\"\"You are judging a competition between {len(competitors)} competitors.\n",227 "Each model has been given this question:\n",228 "\n",229 "{question}\n",230 "\n",231 "Your job is to evaluate each response for clarity and strength of argument, and rank them in order of best to worst.\n",232 "Respond with JSON, and only JSON, with the following format:\n",233 "{{\"results\": [\"best competitor number\", \"second best competitor number\", \"third best competitor number\", ...]}}\n",234 "\n",235 "Here are the responses from each competitor:\n",236 "\n",237 "{together}\n",238 "\n",239 "Now respond with the JSON with the ranked order of the competitors, nothing else. Do not include markdown formatting or code blocks.\"\"\"\n"240 ]241 },242 {243 "cell_type": "code",244 "execution_count": null,245 "metadata": {},246 "outputs": [],247 "source": [248 "print(judge)"249 ]250 },251 {252 "cell_type": "code",253 "execution_count": null,254 "metadata": {},255 "outputs": [],256 "source": [257 "judge_messages = [{\"role\": \"user\", \"content\": judge}]"258 ]259 },260 {261 "cell_type": "code",262 "execution_count": null,263 "metadata": {},264 "outputs": [],265 "source": [266 "# Judgement time!\n",267 "openai = OpenAI()\n",268 "response = openai.chat.completions.create(\n",269 " model=\"gpt-5-mini\",\n",270 " messages=judge_messages,\n",271 ")\n",272 "results = response.choices[0].message.content\n",273 "print(results)\n"274 ]275 },276 {277 "cell_type": "code",278 "execution_count": null,279 "metadata": {},280 "outputs": [],281 "source": [282 "# OK let's turn this into results!\n",283 "results_dict = json.loads(results)\n",284 "ranks = results_dict[\"results\"]\n",285 "for index, result in enumerate(ranks):\n",286 " competitor = competitors[int(result)-1]\n",287 " print(f\"Rank {index+1}: {competitor}\")"288 ]289 },290 {291 "cell_type": "markdown",292 "metadata": {},293 "source": [294 "### Pattern(s) already used in this notebook\n",295 "\n",296 "| Pattern | Where it appears |\n",297 "|---|---|\n",298 "| **Multi-Agent Collaboration** | Multiple LLMs (GPT, Gemini Flash, Gemini Pro) independently answer the same question |\n",299 "| **LLM-as-a-Judge / Orchestration** | A separate GPT instance acts as an orchestrator: it generates the question, collects all responses, then evaluates and ranks them |\n",300 "\n",301 "Together these form the **\"parallel generation + judge\"** agentic workflow.\n",302 "\n",303 "---\n",304 "\n",305 "### New pattern being added below: **Reflection**\n",306 "\n",307 "The Reflection pattern adds a *feedback loop*:\n",308 "1. **Critique** — the judge analyses *why* the worst answer lost\n",309 "2. **Reflect & Revise** — the losing model sees the critique and rewrites its answer\n",310 "3. **Re-judge** — the revised answer is compared against the original winner\n",311 "\n",312 "This loop can be iterated until quality converges."313 ]314 },315 {316 "cell_type": "code",317 "execution_count": null,318 "metadata": {},319 "outputs": [],320 "source": [321 "# ============================================================\n",322 "# REFLECTION PATTERN\n",323 "# ============================================================\n",324 "# Pattern summary:\n",325 "# 1. CRITIQUE — Ask the judge WHY the last-place competitor lost\n",326 "# and what specifically was weak in its response.\n",327 "# 2. REFLECT — Feed that critique back to the losing competitor\n",328 "# so it can revise its answer.\n",329 "# 3. RE-JUDGE — Compare the revised answer against the original\n",330 "# winner to see whether quality improved.\n",331 "#\n",332 "# This closes the \"generate → evaluate → improve\" loop, which is\n",333 "# the defining characteristic of the Reflection agentic pattern.\n",334 "# ============================================================\n",335 "\n",336 "import json\n",337 "from openai import OpenAI\n",338 "\n",339 "openai_client = OpenAI()\n",340 "\n",341 "# ----------------------------------------------------------\n",342 "# Step 0: Identify the loser from the previous judge ranking\n",343 "# ----------------------------------------------------------\n",344 "# 'ranks' — list of competitor numbers ordered best→worst (from previous cells)\n",345 "# 'competitors' — list of model names in the same positional order\n",346 "# 'answers' — list of model answers in the same positional order\n",347 "# 'question' — the original question every competitor answered\n",348 "\n",349 "# The last element in `ranks` is the worst-ranked competitor number (1-based string)\n",350 "loser_rank_number = ranks[-1] # e.g. \"3\"\n",351 "loser_index = int(loser_rank_number) - 1 # convert to 0-based index\n",352 "loser_model = competitors[loser_index]\n",353 "loser_answer = answers[loser_index]\n",354 "\n",355 "winner_rank_number = ranks[0]\n",356 "winner_index = int(winner_rank_number) - 1\n",357 "winner_model = competitors[winner_index]\n",358 "winner_answer = answers[winner_index]\n",359 "\n",360 "print(f\"Winner : {winner_model}\")\n",361 "print(f\"Loser : {loser_model}\")\n",362 "\n",363 "# ----------------------------------------------------------\n",364 "# Step 1: CRITIQUE — ask the judge to explain the loser's flaws\n",365 "# ----------------------------------------------------------\n",366 "critique_prompt = f\"\"\"You previously judged responses to this question:\n",367 "\n",368 "{question}\n",369 "\n",370 "The weakest response was from Competitor {loser_rank_number}:\n",371 "\n",372 "{loser_answer}\n",373 "\n",374 "Please provide specific, constructive critique. Explain exactly what was unclear,\n",375 "missing, or logically weak. Be direct so the model can act on your feedback.\"\"\"\n",376 "\n",377 "critique_messages = [{\"role\": \"user\", \"content\": critique_prompt}]\n",378 "\n",379 "critique_response = openai_client.chat.completions.create(\n",380 " model=\"gpt-4.1-mini\", # Judge model — can use any capable LLM\n",381 " messages=critique_messages,\n",382 ")\n",383 "critique = critique_response.choices[0].message.content\n",384 "print(\"\\n--- CRITIQUE ---\\n\", critique)\n",385 "\n",386 "# ----------------------------------------------------------\n",387 "# Step 2: REFLECT — send the critique back to the losing model\n",388 "# so it can revise its answer (Reflection loop)\n",389 "# ----------------------------------------------------------\n",390 "# We re-use whichever client matches the losing competitor.\n",391 "# For simplicity we route through the gemini client if it is a\n",392 "# Gemini model, otherwise fall back to the OpenAI-compatible client.\n",393 "\n",394 "if \"gemini\" in loser_model.lower():\n",395 " reflect_client = OpenAI(\n",396 " api_key=google_api_key,\n",397 " base_url=\"https://generativelanguage.googleapis.com/v1beta/openai/\",\n",398 " )\n",399 "else:\n",400 " reflect_client = openai_client # Works for any OpenAI model\n",401 "\n",402 "reflect_prompt = f\"\"\"You previously answered this question:\n",403 "\n",404 "{question}\n",405 "\n",406 "Your original answer was:\n",407 "\n",408 "{loser_answer}\n",409 "\n",410 "A judge reviewed your answer and gave the following critique:\n",411 "\n",412 "{critique}\n",413 "\n",414 "Please reflect on this critique and write an improved answer.\n",415 "Focus on addressing every point raised.\"\"\"\n",416 "\n",417 "reflect_messages = [{\"role\": \"user\", \"content\": reflect_prompt}]\n",418 "\n",419 "reflect_response = reflect_client.chat.completions.create(\n",420 " model=loser_model, # Same model attempts to self-improve\n",421 " messages=reflect_messages,\n",422 ")\n",423 "revised_answer = reflect_response.choices[0].message.content\n",424 "print(\"\\n--- REVISED ANSWER ---\\n\", revised_answer)\n",425 "\n",426 "# ----------------------------------------------------------\n",427 "# Step 3: RE-JUDGE — compare the revised answer against the\n",428 "# original winner to see whether the Reflection loop helped\n",429 "# ----------------------------------------------------------\n",430 "rejudge_prompt = f\"\"\"You are evaluating two responses to this question:\n",431 "\n",432 "{question}\n",433 "\n",434 "Response A (original winner, from {winner_model}):\n",435 "{winner_answer}\n",436 "\n",437 "Response B (revised answer, from {loser_model} after reflection):\n",438 "{revised_answer}\n",439 "\n",440 "Evaluate which response is better: clearer, more accurate, and more insightful.\n",441 "Respond with JSON only, in this format:\n",442 "{{\"winner\": \"A or B\", \"reason\": \"one sentence explanation\"}}\"\"\"\n",443 "\n",444 "rejudge_messages = [{\"role\": \"user\", \"content\": rejudge_prompt}]\n",445 "\n",446 "rejudge_response = openai_client.chat.completions.create(\n",447 " model=\"gpt-4.1-mini\",\n",448 " messages=rejudge_messages,\n",449 ")\n",450 "rejudge_result = json.loads(rejudge_response.choices[0].message.content)\n",451 "\n",452 "print(\"\\n--- RE-JUDGE RESULT ---\")\n",453 "print(f\"Winner after Reflection loop: Response {rejudge_result['winner']}\")\n",454 "print(f\"Reason: {rejudge_result['reason']}\")\n",455 "\n",456 "if rejudge_result[\"winner\"] == \"B\":\n",457 " print(f\"\\n✓ Reflection worked — {loser_model} improved enough to beat {winner_model}!\")\n",458 "else:\n",459 " print(f\"\\n✗ Reflection did not flip the result — {winner_model} still leads.\")"460 ]461 }462 ],463 "metadata": {464 "kernelspec": {465 "display_name": "agents",466 "language": "python",467 "name": "python3"468 },469 "language_info": {470 "codemirror_mode": {471 "name": "ipython",472 "version": 3473 },474 "file_extension": ".py",475 "mimetype": "text/x-python",476 "name": "python",477 "nbconvert_exporter": "python",478 "pygments_lexer": "ipython3",479 "version": "3.12.12"480 }481 },482 "nbformat": 4,483 "nbformat_minor": 2484}485