Sathya77/Telecom_Plan_RAG_based
011
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 1,6 "id": "3bf1d8de",7 "metadata": {},8 "outputs": [],9 "source": [10 "#import numpy as np\n",11 "#print(np.__version__)"12 ]13 },14 {15 "cell_type": "code",16 "execution_count": 1,17 "id": "b47ee338",18 "metadata": {},19 "outputs": [20 {21 "name": "stderr",22 "output_type": "stream",23 "text": [24 "E:\\Anaconda\\Lib\\site-packages\\pandas\\core\\arrays\\masked.py:61: UserWarning: Pandas requires version '1.3.6' or newer of 'bottleneck' (version '1.3.5' currently installed).\n",25 " from pandas.core import (\n"26 ]27 },28 {29 "data": {30 "text/plain": [31 "{'Provider': 'Virgin Plus',\n",32 " 'Plan Name': 'BYOP 40GB + 10GB Bonus (Add-a-line)',\n",33 " 'Price': 40,\n",34 " 'Data': '50GB',\n",35 " 'BYOD': True,\n",36 " 'Contract': 'No',\n",37 " 'Hotspot': True,\n",38 " 'Notes': 'Add-a-line'}"39 ]40 },41 "execution_count": 1,42 "metadata": {},43 "output_type": "execute_result"44 }45 ],46 "source": [47 "from datasets import load_dataset\n",48 "\n",49 "data_files = \"E:/Hugging_Face/telecom_plans.csv\"\n",50 "plan_data = load_dataset(\"csv\", data_files = data_files, split = \"train\")\n",51 "plan_data[13]"52 ]53 },54 {55 "cell_type": "code",56 "execution_count": 2,57 "id": "6cf11fc3",58 "metadata": {},59 "outputs": [],60 "source": [61 "def description(example):\n",62 " if example['Notes'] == 'New':\n",63 " customer_note = 'new customers migrating from other service providers'\n",64 " elif example['Notes'] == 'Add-a-line':\n",65 " customer_note = f\"adding a line on existing {example['Provider']} account\"\n",66 " else:\n",67 " customer_note = 'hardware upgrade customers only'\n",68 "\n",69 " return {\n",70 " \"Description\": (\n",71 " f\"{example['Provider']} offers a plan \"\n",72 " f\"{example['Plan Name']} for the price of ${example['Price']} a month, which includes {example['Data']} \"\n",73 " f\"data with {'BYOD' if example['BYOD'] else 'device financing over 24 months'} and \"\n",74 " f\"{'allows you to data-share with Hotspot' if example['Hotspot'] else 'does not include data-share with Hotspot'}. \"\n",75 " f\"This plan is for {customer_note}.\"\n",76 " )\n",77 " }\n",78 "\n",79 "plan_data_mod = plan_data.map(description)\n",80 " "81 ]82 },83 {84 "cell_type": "code",85 "execution_count": 3,86 "id": "167e31ce",87 "metadata": {},88 "outputs": [89 {90 "data": {91 "text/plain": [92 "{'Provider': ['Bell', 'Bell', 'Bell', 'Bell', 'Bell'],\n",93 " 'Plan Name': ['Elite Lite (New)',\n",94 " 'Elite Lite (Add-a-line)',\n",95 " 'Elite Lite (HUG)',\n",96 " 'ExtraElite (New)',\n",97 " 'ExtraElite (Add-a-line)'],\n",98 " 'Price': [60, 60, 60, 45, 45],\n",99 " 'Data': ['60GB', '60GB', '60GB', '50GB', '50GB'],\n",100 " 'BYOD': [False, False, False, False, False],\n",101 " 'Contract': ['SmartPay', 'SmartPay', 'SmartPay', 'SmartPay', 'SmartPay'],\n",102 " 'Hotspot': [True, True, True, True, True],\n",103 " 'Notes': ['New', 'Add-a-line', 'HUG', 'New', 'Add-a-line'],\n",104 " 'Description': ['Bell offers a plan Elite Lite (New) for the price of $60 a month, which includes 60GB data with device financing over 24 months and allows you to data-share with Hotspot. This plan is for new customers migrating from other service providers.',\n",105 " 'Bell offers a plan Elite Lite (Add-a-line) for the price of $60 a month, which includes 60GB data with device financing over 24 months and allows you to data-share with Hotspot. This plan is for adding a line on existing Bell account.',\n",106 " 'Bell offers a plan Elite Lite (HUG) for the price of $60 a month, which includes 60GB data with device financing over 24 months and allows you to data-share with Hotspot. This plan is for hardware upgrade customers only.',\n",107 " 'Bell offers a plan ExtraElite (New) for the price of $45 a month, which includes 50GB data with device financing over 24 months and allows you to data-share with Hotspot. This plan is for new customers migrating from other service providers.',\n",108 " 'Bell offers a plan ExtraElite (Add-a-line) for the price of $45 a month, which includes 50GB data with device financing over 24 months and allows you to data-share with Hotspot. This plan is for adding a line on existing Bell account.']}"109 ]110 },111 "execution_count": 3,112 "metadata": {},113 "output_type": "execute_result"114 }115 ],116 "source": [117 "plan_data_mod[:5]"118 ]119 },120 {121 "cell_type": "code",122 "execution_count": 4,123 "id": "9664ba16",124 "metadata": {},125 "outputs": [],126 "source": [127 "#!pip uninstall urllib3 -y\n",128 "#!pip install urllib3==1.26.18"129 ]130 },131 {132 "cell_type": "code",133 "execution_count": 5,134 "id": "82bcbeea",135 "metadata": {},136 "outputs": [],137 "source": [138 "#!pip install --upgrade accelerate transformers"139 ]140 },141 {142 "cell_type": "code",143 "execution_count": 6,144 "id": "a7a41116",145 "metadata": {},146 "outputs": [147 {148 "name": "stdout",149 "output_type": "stream",150 "text": [151 "WARNING:tensorflow:From E:\\Anaconda\\Lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",152 "\n"153 ]154 }155 ],156 "source": [157 "from sentence_transformers import SentenceTransformer\n",158 "\n",159 "embedder = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",160 "\n",161 "descriptions = plan_data_mod[\"Description\"]\n",162 "embeddings = embedder.encode(descriptions, convert_to_numpy = True)"163 ]164 },165 {166 "cell_type": "code",167 "execution_count": 7,168 "id": "e5742a93",169 "metadata": {},170 "outputs": [171 {172 "data": {173 "text/plain": [174 "array([[-0.04481903, -0.06913085, 0.02706613, ..., -0.08726593,\n",175 " -0.05549479, 0.00731448],\n",176 " [-0.05391502, -0.07286713, -0.00130842, ..., -0.07893781,\n",177 " -0.05440836, -0.02790258],\n",178 " [-0.0552901 , -0.04580358, 0.04743244, ..., -0.10154203,\n",179 " -0.05256264, 0.02625924],\n",180 " ...,\n",181 " [ 0.00983924, 0.01232204, 0.03798797, ..., -0.080321 ,\n",182 " -0.1193777 , 0.00596426],\n",183 " [ 0.01246224, 0.01364668, 0.04105236, ..., -0.07725951,\n",184 " -0.13012475, 0.01362491],\n",185 " [ 0.01581116, 0.0131523 , 0.04183232, ..., -0.07409089,\n",186 " -0.12856905, 0.01441113]], dtype=float32)"187 ]188 },189 "execution_count": 7,190 "metadata": {},191 "output_type": "execute_result"192 }193 ],194 "source": [195 "embeddings"196 ]197 },198 {199 "cell_type": "code",200 "execution_count": 8,201 "id": "3aaf4323",202 "metadata": {},203 "outputs": [],204 "source": [205 "import faiss, numpy as np\n",206 "\n",207 "X = np.asarray(embeddings, dtype=\"float32\")\n",208 "faiss.normalize_L2(X)\n",209 "\n",210 "index = faiss.IndexFlatIP(X.shape[1])\n",211 "index.add(X) "212 ]213 },214 {215 "cell_type": "code",216 "execution_count": 9,217 "id": "b39fda8e",218 "metadata": {},219 "outputs": [220 {221 "data": {222 "text/plain": [223 "<faiss.swigfaiss_avx2.IndexFlatIP; proxy of <Swig Object of type 'faiss::IndexFlatIP *' at 0x0000011B20686B20> >"224 ]225 },226 "execution_count": 9,227 "metadata": {},228 "output_type": "execute_result"229 }230 ],231 "source": [232 "index"233 ]234 },235 {236 "cell_type": "code",237 "execution_count": 10,238 "id": "f933cfd5",239 "metadata": {},240 "outputs": [],241 "source": [242 "def retrieve(query, k=5):\n",243 " q = embedder.encode([query]).astype(\"float32\")\n",244 " faiss.normalize_L2(q)\n",245 " scores, idx = index.search(q, k) # scores in [0,1] ~ cosine\n",246 " return [(int(i), float(s)) for i, s in zip(idx[0], scores[0])]\n"247 ]248 },249 {250 "cell_type": "code",251 "execution_count": 11,252 "id": "34a49b4c",253 "metadata": {},254 "outputs": [255 {256 "name": "stdout",257 "output_type": "stream",258 "text": [259 "[(65, 0.7283002138137817), (59, 0.7259033918380737), (67, 0.7257879376411438), (9, 0.725434422492981), (68, 0.725420355796814)]\n"260 ]261 }262 ],263 "source": [264 "ex_1 = \"what are the best BYOD plans in Virgin plus below 60$\"\n",265 "\n",266 "pair_1 = retrieve(ex_1)\n",267 "print(pair_1)"268 ]269 },270 {271 "cell_type": "code",272 "execution_count": 12,273 "id": "1028f1cd",274 "metadata": {},275 "outputs": [],276 "source": [277 "#from huggingface_hub import notebook_login\n",278 "\n",279 "#notebook_login()"280 ]281 },282 {283 "cell_type": "code",284 "execution_count": 13,285 "id": "7aad76c2",286 "metadata": {},287 "outputs": [],288 "source": [289 "#pip install hf_xet"290 ]291 },292 {293 "cell_type": "code",294 "execution_count": 14,295 "id": "d9164db6",296 "metadata": {},297 "outputs": [298 {299 "data": {300 "application/vnd.jupyter.widget-view+json": {301 "model_id": "23f5ceb7f749450c930c69dd9e60a212",302 "version_major": 2,303 "version_minor": 0304 },305 "text/plain": [306 "tokenizer_config.json: 0.00B [00:00, ?B/s]"307 ]308 },309 "metadata": {},310 "output_type": "display_data"311 },312 {313 "name": "stderr",314 "output_type": "stream",315 "text": [316 "E:\\Anaconda\\Lib\\site-packages\\huggingface_hub\\file_download.py:143: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\kompe\\.cache\\huggingface\\hub\\models--MBZUAI--LaMini-Flan-T5-783M. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",317 "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",318 " warnings.warn(message)\n"319 ]320 },321 {322 "data": {323 "application/vnd.jupyter.widget-view+json": {324 "model_id": "b01a872f4fb241f6a7a18e75972695ca",325 "version_major": 2,326 "version_minor": 0327 },328 "text/plain": [329 "spiece.model: 0%| | 0.00/792k [00:00<?, ?B/s]"330 ]331 },332 "metadata": {},333 "output_type": "display_data"334 },335 {336 "data": {337 "application/vnd.jupyter.widget-view+json": {338 "model_id": "e738878aef244ae9bd592815125f30f9",339 "version_major": 2,340 "version_minor": 0341 },342 "text/plain": [343 "tokenizer.json: 0.00B [00:00, ?B/s]"344 ]345 },346 "metadata": {},347 "output_type": "display_data"348 },349 {350 "data": {351 "application/vnd.jupyter.widget-view+json": {352 "model_id": "1b0a3272dd3440779b10b6f02f07c4a3",353 "version_major": 2,354 "version_minor": 0355 },356 "text/plain": [357 "special_tokens_map.json: 0.00B [00:00, ?B/s]"358 ]359 },360 "metadata": {},361 "output_type": "display_data"362 },363 {364 "data": {365 "application/vnd.jupyter.widget-view+json": {366 "model_id": "b2651bd90ce64e4a8eb5066524b6e6f0",367 "version_major": 2,368 "version_minor": 0369 },370 "text/plain": [371 "config.json: 0%| | 0.00/860 [00:00<?, ?B/s]"372 ]373 },374 "metadata": {},375 "output_type": "display_data"376 },377 {378 "data": {379 "application/vnd.jupyter.widget-view+json": {380 "model_id": "4873faa21df54292a014f30aa0d00af9",381 "version_major": 2,382 "version_minor": 0383 },384 "text/plain": [385 "pytorch_model.bin: 0%| | 0.00/3.13G [00:00<?, ?B/s]"386 ]387 },388 "metadata": {},389 "output_type": "display_data"390 },391 {392 "data": {393 "application/vnd.jupyter.widget-view+json": {394 "model_id": "ee14382ae6a3490c9caa41f00f3b4c25",395 "version_major": 2,396 "version_minor": 0397 },398 "text/plain": [399 "generation_config.json: 0%| | 0.00/147 [00:00<?, ?B/s]"400 ]401 },402 "metadata": {},403 "output_type": "display_data"404 },405 {406 "data": {407 "application/vnd.jupyter.widget-view+json": {408 "model_id": "072832b6b66348569c4bd99293c731ca",409 "version_major": 2,410 "version_minor": 0411 },412 "text/plain": [413 "model.safetensors: 0%| | 0.00/3.13G [00:00<?, ?B/s]"414 ]415 },416 "metadata": {},417 "output_type": "display_data"418 },419 {420 "name": "stderr",421 "output_type": "stream",422 "text": [423 "Device set to use cuda:0\n"424 ]425 },426 {427 "data": {428 "text/plain": [429 "<function transformers.pipelines.pipeline(task: Optional[str] = None, model: Union[str, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel'), NoneType] = None, config: Union[str, transformers.configuration_utils.PretrainedConfig, NoneType] = None, tokenizer: Union[str, transformers.tokenization_utils.PreTrainedTokenizer, ForwardRef('PreTrainedTokenizerFast'), NoneType] = None, feature_extractor: Union[str, ForwardRef('SequenceFeatureExtractor'), NoneType] = None, image_processor: Union[str, transformers.image_processing_utils.BaseImageProcessor, NoneType] = None, processor: Union[str, transformers.processing_utils.ProcessorMixin, NoneType] = None, framework: Optional[str] = None, revision: Optional[str] = None, use_fast: bool = True, token: Union[str, bool, NoneType] = None, device: Union[int, str, ForwardRef('torch.device'), NoneType] = None, device_map: Union[str, dict[str, Union[int, str]], NoneType] = None, torch_dtype: Union[str, ForwardRef('torch.dtype'), NoneType] = 'auto', trust_remote_code: Optional[bool] = None, model_kwargs: Optional[dict[str, Any]] = None, pipeline_class: Optional[Any] = None, **kwargs: Any) -> transformers.pipelines.base.Pipeline>"430 ]431 },432 "execution_count": 14,433 "metadata": {},434 "output_type": "execute_result"435 }436 ],437 "source": [438 "from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM\n",439 "\n",440 "tokenizer = AutoTokenizer.from_pretrained(\"MBZUAI/LaMini-Flan-T5-783M\")\n",441 "model = AutoModelForSeq2SeqLM.from_pretrained(\"MBZUAI/LaMini-Flan-T5-783M\")\n",442 "qa_pipeline = pipeline(\"text2text-generation\",model = model, tokenizer = tokenizer)\n",443 "pipeline"444 ]445 },446 {447 "cell_type": "code",448 "execution_count": 15,449 "id": "7c939d25",450 "metadata": {},451 "outputs": [],452 "source": [453 "def answer_question(query):\n",454 " top_k = retrieve(query, k=5)\n",455 " context = \"\\n\".join([plan_data_mod[i][\"Description\"] for i, _ in top_k])\n",456 " prompt = (\n",457 " f\"You are a telecom assistant helping a user choose the best phone plan. \"\n",458 " f\"Only include plans that match the question clearly. Be concise and friendly.\\n\\n\"\n",459 " f\"Plans:\\n{context}\\n\\n\"\n",460 " f\"Question: {query}\\n\"\n",461 " f\"Answer in full sentences:\"\n",462 ")\n",463 "\n",464 " response = qa_pipeline(prompt, max_new_tokens=100, do_sample=False)\n",465 " raw_answer = response[0]['generated_text']\n",466 "\n",467 " lines = raw_answer.strip().split('\\n')\n",468 " unique_lines = list(dict.fromkeys(lines))\n",469 " cleaned_answer = \"\\n\".join(unique_lines)\n",470 "\n",471 " return cleaned_answer\n"472 ]473 },474 {475 "cell_type": "code",476 "execution_count": 19,477 "id": "8e0b5b11",478 "metadata": {},479 "outputs": [],480 "source": [481 "import evaluate\n",482 "rouge = evaluate.load(\"rouge\")\n",483 "bleu = evaluate.load(\"bleu\")\n",484 "bertscore = evaluate.load(\"bertscore\")\n",485 "\n",486 "def eval_metrics(prediction: str, reference: str):\n",487 " bleu_res = bleu.compute(predictions=[prediction], references=[reference])\n",488 " rouge_res = rouge.compute(predictions=[prediction], references=[reference])\n",489 " bert_res = bertscore.compute(predictions=[prediction], references=[reference], lang=\"en\")\n",490 " return {\n",491 " \"BLEU\": bleu_res[\"bleu\"],\n",492 " \"ROUGE-1\": rouge_res[\"rouge1\"],\n",493 " \"ROUGE-2\": rouge_res[\"rouge2\"],\n",494 " \"ROUGE-L\": rouge_res[\"rougeL\"],\n",495 " \"BERTScore-F1\": float(sum(bert_res[\"f1\"]) / len(bert_res[\"f1\"]))\n",496 " }"497 ]498 },499 {500 "cell_type": "code",501 "execution_count": 20,502 "id": "43a670d4",503 "metadata": {},504 "outputs": [505 {506 "name": "stdout",507 "output_type": "stream",508 "text": [509 "Ask your plan-related question: bell student plan in alberta?\n",510 "\n",511 "Answer:\n",512 " Bell offers three student plans in Alberta: Elite Lite for $52 a month, Elite Lite for $35 a month, ExtraElite+ for $55 a month, and ExtraElite for $45 a month. These plans are for hardware upgrade customers only and include unlimited data with device financing over 24 months and data-sharing with Hotspot.\n"513 ]514 },515 {516 "data": {517 "application/vnd.jupyter.widget-view+json": {518 "model_id": "84315c6640e848598284ac5f4ea5f0fb",519 "version_major": 2,520 "version_minor": 0521 },522 "text/plain": [523 "tokenizer_config.json: 0%| | 0.00/25.0 [00:00<?, ?B/s]"524 ]525 },526 "metadata": {},527 "output_type": "display_data"528 },529 {530 "name": "stderr",531 "output_type": "stream",532 "text": [533 "E:\\Anaconda\\Lib\\site-packages\\huggingface_hub\\file_download.py:143: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\kompe\\.cache\\huggingface\\hub\\models--roberta-large. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",534 "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",535 " warnings.warn(message)\n"536 ]537 },538 {539 "data": {540 "application/vnd.jupyter.widget-view+json": {541 "model_id": "82feabb3257c4bf8912b314e440d38fb",542 "version_major": 2,543 "version_minor": 0544 },545 "text/plain": [546 "config.json: 0%| | 0.00/482 [00:00<?, ?B/s]"547 ]548 },549 "metadata": {},550 "output_type": "display_data"551 },552 {553 "data": {554 "application/vnd.jupyter.widget-view+json": {555 "model_id": "ac91b156d70c4a19b7c5d9c919d8c3be",556 "version_major": 2,557 "version_minor": 0558 },559 "text/plain": [560 "vocab.json: 0%| | 0.00/899k [00:00<?, ?B/s]"561 ]562 },563 "metadata": {},564 "output_type": "display_data"565 },566 {567 "data": {568 "application/vnd.jupyter.widget-view+json": {569 "model_id": "cee120b285cd45fd9b1f9f5b0f85a23c",570 "version_major": 2,571 "version_minor": 0572 },573 "text/plain": [574 "merges.txt: 0%| | 0.00/456k [00:00<?, ?B/s]"575 ]576 },577 "metadata": {},578 "output_type": "display_data"579 },580 {581 "data": {582 "application/vnd.jupyter.widget-view+json": {583 "model_id": "b331347672c443aaa7a22c8f63448a72",584 "version_major": 2,585 "version_minor": 0586 },587 "text/plain": [588 "tokenizer.json: 0%| | 0.00/1.36M [00:00<?, ?B/s]"589 ]590 },591 "metadata": {},592 "output_type": "display_data"593 },594 {595 "data": {596 "application/vnd.jupyter.widget-view+json": {597 "model_id": "cb3e1988b26143068137b582ad4ec073",598 "version_major": 2,599 "version_minor": 0600 },601 "text/plain": [602 "model.safetensors: 0%| | 0.00/1.42G [00:00<?, ?B/s]"603 ]604 },605 "metadata": {},606 "output_type": "display_data"607 },608 {609 "name": "stderr",610 "output_type": "stream",611 "text": [612 "Some weights of RobertaModel were not initialized from the model checkpoint at roberta-large and are newly initialized: ['pooler.dense.bias', 'pooler.dense.weight']\n",613 "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"614 ]615 },616 {617 "name": "stdout",618 "output_type": "stream",619 "text": [620 "Evaluation:\n",621 "BLEU: 0.2613 | ROUGE-1: 0.5745 | ROUGE-2: 0.3478 | ROUGE-L: 0.4043 | BERTScore-F1: 0.9322\n",622 "\n",623 "--------------------------------------------------\n",624 "\n",625 "Ask your plan-related question: cheapest plan in virgin plus byod?\n",626 "\n",627 "Answer:\n",628 " The cheapest plan in Virgin Plus byod is BYOP 40GB (Student, Alberta) for the price of $39 a month, which includes Unlimited data with BYOD and allows you to data-share with Hotspot. This plan is for hardware upgrade customers only.\n",629 "Evaluation:\n",630 "BLEU: 0.8449 | ROUGE-1: 0.9114 | ROUGE-2: 0.8571 | ROUGE-L: 0.8861 | BERTScore-F1: 0.9670\n",631 "\n",632 "--------------------------------------------------\n",633 "\n",634 "Ask your plan-related question: lucky plan under 30$?\n",635 "\n",636 "Answer:\n",637 " Yes, Lucky offers a plan under $30 for the Student, Alberta plan which includes Unlimited data with BYOD and does not include data-share with Hotspot.\n",638 "Evaluation:\n",639 "BLEU: 0.3511 | ROUGE-1: 0.6667 | ROUGE-2: 0.5574 | ROUGE-L: 0.6349 | BERTScore-F1: 0.9292\n",640 "\n",641 "--------------------------------------------------\n",642 "\n",643 "Ask your plan-related question: what is the amount of data provided for $30 plan with lucky?\n",644 "\n",645 "Answer:\n",646 " The $30 plan with Lucky provides unlimited data with BYOD and does not include data-share with Hotspot.\n",647 "Evaluation:\n",648 "BLEU: 0.1539 | ROUGE-1: 0.5455 | ROUGE-2: 0.4151 | ROUGE-L: 0.4727 | BERTScore-F1: 0.9221\n",649 "\n",650 "--------------------------------------------------\n",651 "\n",652 "Ask your plan-related question: exit\n",653 "\n",654 "\n",655 "Goodbye!\n"656 ]657 }658 ],659 "source": [660 "while True:\n",661 " user_query = input(\"Ask your plan-related question: \")\n",662 " if user_query.strip().lower() in [\"exit\", \"quit\"]:\n",663 " print(\"\\n\\nGoodbye!\")\n",664 " break\n",665 "\n",666 " answer = answer_question(user_query)\n",667 " print(\"\\nAnswer:\\n\", answer)\n",668 " \n",669 " prediction = answer\n",670 " top1 = retrieve(user_query, k=1)\n",671 " reference = plan_data_mod[top1[0][0]][\"Description\"] if top1 else \"\"\n",672 "\n",673 " scores = eval_metrics(prediction, reference)\n",674 " print(\"Evaluation:\")\n",675 " print(f\"BLEU: {scores['BLEU']:.4f} | ROUGE-1: {scores['ROUGE-1']:.4f} | \"\n",676 " f\"ROUGE-2: {scores['ROUGE-2']:.4f} | ROUGE-L: {scores['ROUGE-L']:.4f} | \"\n",677 " f\"BERTScore-F1: {scores['BERTScore-F1']:.4f}\")\n",678 " print(\"\\n\" + \"-\"*50 + \"\\n\")\n"679 ]680 },681 {682 "cell_type": "code",683 "execution_count": 21,684 "id": "fed628c7",685 "metadata": {},686 "outputs": [],687 "source": [688 "tokenizer.save_pretrained(\"Plan_Retrieval_RAG\")\n",689 "model.save_pretrained(\"RAG_based_Model\")"690 ]691 },692 {693 "cell_type": "code",694 "execution_count": 22,695 "id": "9f24caf8",696 "metadata": {},697 "outputs": [],698 "source": [699 "faiss.write_index(index, \"RAG_model.index\")\n",700 "np.save(\"plan_embeddings.npy\", embeddings)"701 ]702 },703 {704 "cell_type": "code",705 "execution_count": null,706 "id": "6bcb6953",707 "metadata": {},708 "outputs": [],709 "source": []710 }711 ],712 "metadata": {713 "kernelspec": {714 "display_name": "Python 3 (ipykernel)",715 "language": "python",716 "name": "python3"717 },718 "language_info": {719 "codemirror_mode": {720 "name": "ipython",721 "version": 3722 },723 "file_extension": ".py",724 "mimetype": "text/x-python",725 "name": "python",726 "nbconvert_exporter": "python",727 "pygments_lexer": "ipython3",728 "version": "3.11.4"729 }730 },731 "nbformat": 4,732 "nbformat_minor": 5733}734