BarinkDev/LargeLanguageModels
0
1{2 "cells": [3 {4 "cell_type": "markdown",5 "id": "70d6cd6c-56a0-45bf-aedb-0c9f69a4c81d",6 "metadata": {},7 "source": [8 "# RAG model for php8 docs using BERT"9 ]10 },11 {12 "cell_type": "code",13 "execution_count": 3,14 "id": "7d4b872d-68de-43d2-8443-9ffab0fcde4e",15 "metadata": {},16 "outputs": [17 {18 "name": "stdout",19 "output_type": "stream",20 "text": [21 "Requirement already satisfied: opencv-python in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (4.10.0.84)\n",22 "Requirement already satisfied: numpy>=1.21.2 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from opencv-python) (1.26.4)\n",23 "Requirement already satisfied: tensorflow-text==2.17.* in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (2.17.0)\n",24 "Requirement already satisfied: tensorflow<2.18,>=2.17.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow-text==2.17.*) (2.17.0)\n",25 "Requirement already satisfied: absl-py>=1.0.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (1.4.0)\n",26 "Requirement already satisfied: astunparse>=1.6.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (1.6.3)\n",27 "Requirement already satisfied: flatbuffers>=24.3.25 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (24.3.25)\n",28 "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (0.6.0)\n",29 "Requirement already satisfied: google-pasta>=0.1.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (0.2.0)\n",30 "Requirement already satisfied: h5py>=3.10.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (3.11.0)\n",31 "Requirement already satisfied: libclang>=13.0.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (18.1.1)\n",32 "Requirement already satisfied: ml-dtypes<0.5.0,>=0.3.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (0.4.1)\n",33 "Requirement already satisfied: opt-einsum>=2.3.2 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (3.3.0)\n",34 "Requirement already satisfied: packaging in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (24.1)\n",35 "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (4.25.5)\n",36 "Requirement already satisfied: requests<3,>=2.21.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (2.32.3)\n",37 "Requirement already satisfied: setuptools in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (66.1.1)\n",38 "Requirement already satisfied: six>=1.12.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (1.16.0)\n",39 "Requirement already satisfied: termcolor>=1.1.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (2.4.0)\n",40 "Requirement already satisfied: typing-extensions>=3.6.6 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (4.12.2)\n",41 "Requirement already satisfied: wrapt>=1.11.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (1.16.0)\n",42 "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (1.66.1)\n",43 "Requirement already satisfied: tensorboard<2.18,>=2.17 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (2.17.1)\n",44 "Requirement already satisfied: keras>=3.2.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (3.5.0)\n",45 "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (0.37.1)\n",46 "Requirement already satisfied: numpy<2.0.0,>=1.23.5 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (1.26.4)\n",47 "Requirement already satisfied: wheel<1.0,>=0.23.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from astunparse>=1.6.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (0.44.0)\n",48 "Requirement already satisfied: rich in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from keras>=3.2.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (13.8.1)\n",49 "Requirement already satisfied: namex in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from keras>=3.2.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (0.0.8)\n",50 "Requirement already satisfied: optree in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from keras>=3.2.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (0.12.1)\n",51 "Requirement already satisfied: charset-normalizer<4,>=2 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from requests<3,>=2.21.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (3.3.2)\n",52 "Requirement already satisfied: idna<4,>=2.5 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from requests<3,>=2.21.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (3.10)\n",53 "Requirement already satisfied: urllib3<3,>=1.21.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from requests<3,>=2.21.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (2.2.3)\n",54 "Requirement already satisfied: certifi>=2017.4.17 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from requests<3,>=2.21.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (2024.8.30)\n",55 "Requirement already satisfied: markdown>=2.6.8 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorboard<2.18,>=2.17->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (3.7)\n",56 "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorboard<2.18,>=2.17->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (0.7.2)\n",57 "Requirement already satisfied: werkzeug>=1.0.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorboard<2.18,>=2.17->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (3.0.4)\n",58 "Requirement already satisfied: MarkupSafe>=2.1.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from werkzeug>=1.0.1->tensorboard<2.18,>=2.17->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (2.1.5)\n",59 "Requirement already satisfied: markdown-it-py>=2.2.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from rich->keras>=3.2.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (3.0.0)\n",60 "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from rich->keras>=3.2.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (2.18.0)\n",61 "Requirement already satisfied: mdurl~=0.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.2.0->tensorflow<2.18,>=2.17.0->tensorflow-text==2.17.*) (0.1.2)\n",62 "Requirement already satisfied: tf-models-official in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (2.17.0)\n",63 "Requirement already satisfied: Cython in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (3.0.11)\n",64 "Requirement already satisfied: Pillow in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (10.4.0)\n",65 "Requirement already satisfied: gin-config in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (0.5.0)\n",66 "Requirement already satisfied: google-api-python-client>=1.6.7 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (2.146.0)\n",67 "Requirement already satisfied: immutabledict in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (4.2.0)\n",68 "Requirement already satisfied: kaggle>=1.3.9 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (1.6.17)\n",69 "Requirement already satisfied: matplotlib in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (3.9.2)\n",70 "Requirement already satisfied: numpy>=1.20 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (1.26.4)\n",71 "Requirement already satisfied: oauth2client in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (4.1.3)\n",72 "Requirement already satisfied: opencv-python-headless in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (4.10.0.84)\n",73 "Requirement already satisfied: pandas>=0.22.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (2.2.3)\n",74 "Requirement already satisfied: psutil>=5.4.3 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (6.0.0)\n",75 "Requirement already satisfied: py-cpuinfo>=3.3.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (9.0.0)\n",76 "Requirement already satisfied: pycocotools in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (2.0.8)\n",77 "Requirement already satisfied: pyyaml>=6.0.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (6.0.2)\n",78 "Requirement already satisfied: sacrebleu in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (2.4.3)\n",79 "Requirement already satisfied: scipy>=0.19.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (1.14.1)\n",80 "Requirement already satisfied: sentencepiece in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (0.2.0)\n",81 "Requirement already satisfied: seqeval in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (1.2.2)\n",82 "Requirement already satisfied: six in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (1.16.0)\n",83 "Requirement already satisfied: tensorflow-datasets in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (4.9.6)\n",84 "Requirement already satisfied: tensorflow-hub>=0.6.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (0.16.1)\n",85 "Requirement already satisfied: tensorflow-model-optimization>=0.4.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (0.8.0)\n",86 "Requirement already satisfied: tensorflow-text~=2.17.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (2.17.0)\n",87 "Requirement already satisfied: tensorflow~=2.17.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (2.17.0)\n",88 "Requirement already satisfied: tf-keras>=2.16.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (2.17.0)\n",89 "Requirement already satisfied: tf-slim>=1.1.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tf-models-official) (1.1.0)\n",90 "Requirement already satisfied: httplib2<1.dev0,>=0.19.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from google-api-python-client>=1.6.7->tf-models-official) (0.22.0)\n",91 "Requirement already satisfied: google-auth!=2.24.0,!=2.25.0,<3.0.0.dev0,>=1.32.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from google-api-python-client>=1.6.7->tf-models-official) (2.35.0)\n",92 "Requirement already satisfied: google-auth-httplib2<1.0.0,>=0.2.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from google-api-python-client>=1.6.7->tf-models-official) (0.2.0)\n",93 "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from google-api-python-client>=1.6.7->tf-models-official) (2.20.0)\n",94 "Requirement already satisfied: uritemplate<5,>=3.0.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from google-api-python-client>=1.6.7->tf-models-official) (4.1.1)\n",95 "Requirement already satisfied: bleach in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from kaggle>=1.3.9->tf-models-official) (6.1.0)\n",96 "Requirement already satisfied: certifi>=2023.7.22 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from kaggle>=1.3.9->tf-models-official) (2024.8.30)\n",97 "Requirement already satisfied: python-dateutil in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from kaggle>=1.3.9->tf-models-official) (2.9.0.post0)\n",98 "Requirement already satisfied: python-slugify in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from kaggle>=1.3.9->tf-models-official) (8.0.4)\n",99 "Requirement already satisfied: requests in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from kaggle>=1.3.9->tf-models-official) (2.32.3)\n",100 "Requirement already satisfied: tqdm in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from kaggle>=1.3.9->tf-models-official) (4.66.5)\n",101 "Requirement already satisfied: urllib3 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from kaggle>=1.3.9->tf-models-official) (2.2.3)\n",102 "Requirement already satisfied: pytz>=2020.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from pandas>=0.22.0->tf-models-official) (2024.2)\n",103 "Requirement already satisfied: tzdata>=2022.7 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from pandas>=0.22.0->tf-models-official) (2024.1)\n",104 "Requirement already satisfied: absl-py>=1.0.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (1.4.0)\n",105 "Requirement already satisfied: astunparse>=1.6.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (1.6.3)\n",106 "Requirement already satisfied: flatbuffers>=24.3.25 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (24.3.25)\n",107 "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (0.6.0)\n",108 "Requirement already satisfied: google-pasta>=0.1.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (0.2.0)\n",109 "Requirement already satisfied: h5py>=3.10.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (3.11.0)\n",110 "Requirement already satisfied: libclang>=13.0.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (18.1.1)\n",111 "Requirement already satisfied: ml-dtypes<0.5.0,>=0.3.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (0.4.1)\n",112 "Requirement already satisfied: opt-einsum>=2.3.2 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (3.3.0)\n",113 "Requirement already satisfied: packaging in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (24.1)\n",114 "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (4.25.5)\n",115 "Requirement already satisfied: setuptools in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (66.1.1)\n",116 "Requirement already satisfied: termcolor>=1.1.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (2.4.0)\n",117 "Requirement already satisfied: typing-extensions>=3.6.6 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (4.12.2)\n",118 "Requirement already satisfied: wrapt>=1.11.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (1.16.0)\n",119 "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (1.66.1)\n",120 "Requirement already satisfied: tensorboard<2.18,>=2.17 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (2.17.1)\n",121 "Requirement already satisfied: keras>=3.2.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (3.5.0)\n",122 "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow~=2.17.0->tf-models-official) (0.37.1)\n",123 "Requirement already satisfied: dm-tree~=0.1.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow-model-optimization>=0.4.1->tf-models-official) (0.1.8)\n",124 "Requirement already satisfied: contourpy>=1.0.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from matplotlib->tf-models-official) (1.3.0)\n",125 "Requirement already satisfied: cycler>=0.10 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from matplotlib->tf-models-official) (0.12.1)\n",126 "Requirement already satisfied: fonttools>=4.22.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from matplotlib->tf-models-official) (4.53.1)\n",127 "Requirement already satisfied: kiwisolver>=1.3.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from matplotlib->tf-models-official) (1.4.7)\n",128 "Requirement already satisfied: pyparsing>=2.3.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from matplotlib->tf-models-official) (3.1.4)\n",129 "Requirement already satisfied: pyasn1>=0.1.7 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from oauth2client->tf-models-official) (0.6.1)\n",130 "Requirement already satisfied: pyasn1-modules>=0.0.5 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from oauth2client->tf-models-official) (0.4.1)\n",131 "Requirement already satisfied: rsa>=3.1.4 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from oauth2client->tf-models-official) (4.9)\n",132 "Requirement already satisfied: portalocker in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from sacrebleu->tf-models-official) (2.10.1)\n",133 "Requirement already satisfied: regex in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from sacrebleu->tf-models-official) (2024.9.11)\n",134 "Requirement already satisfied: tabulate>=0.8.9 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from sacrebleu->tf-models-official) (0.9.0)\n",135 "Requirement already satisfied: colorama in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from sacrebleu->tf-models-official) (0.4.6)\n",136 "Requirement already satisfied: lxml in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from sacrebleu->tf-models-official) (5.3.0)\n",137 "Requirement already satisfied: scikit-learn>=0.21.3 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from seqeval->tf-models-official) (1.5.2)\n",138 "Requirement already satisfied: click in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow-datasets->tf-models-official) (8.1.7)\n",139 "Requirement already satisfied: promise in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow-datasets->tf-models-official) (2.3)\n",140 "Requirement already satisfied: pyarrow in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow-datasets->tf-models-official) (17.0.0)\n",141 "Requirement already satisfied: simple-parsing in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow-datasets->tf-models-official) (0.1.6)\n",142 "Requirement already satisfied: tensorflow-metadata in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow-datasets->tf-models-official) (1.15.0)\n",143 "Requirement already satisfied: toml in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow-datasets->tf-models-official) (0.10.2)\n",144 "Requirement already satisfied: array-record>=0.5.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow-datasets->tf-models-official) (0.5.1)\n",145 "Requirement already satisfied: etils[enp,epath,epy,etree]>=1.9.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorflow-datasets->tf-models-official) (1.9.4)\n",146 "Requirement already satisfied: wheel<1.0,>=0.23.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from astunparse>=1.6.0->tensorflow~=2.17.0->tf-models-official) (0.44.0)\n",147 "Requirement already satisfied: fsspec in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from etils[enp,epath,epy,etree]>=1.9.1->tensorflow-datasets->tf-models-official) (2024.9.0)\n",148 "Requirement already satisfied: importlib_resources in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from etils[enp,epath,epy,etree]>=1.9.1->tensorflow-datasets->tf-models-official) (6.4.5)\n",149 "Requirement already satisfied: zipp in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from etils[enp,epath,epy,etree]>=1.9.1->tensorflow-datasets->tf-models-official) (3.20.2)\n",150 "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5->google-api-python-client>=1.6.7->tf-models-official) (1.65.0)\n",151 "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5->google-api-python-client>=1.6.7->tf-models-official) (1.24.0)\n",152 "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from google-auth!=2.24.0,!=2.25.0,<3.0.0.dev0,>=1.32.0->google-api-python-client>=1.6.7->tf-models-official) (5.5.0)\n",153 "Requirement already satisfied: rich in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from keras>=3.2.0->tensorflow~=2.17.0->tf-models-official) (13.8.1)\n",154 "Requirement already satisfied: namex in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from keras>=3.2.0->tensorflow~=2.17.0->tf-models-official) (0.0.8)\n",155 "Requirement already satisfied: optree in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from keras>=3.2.0->tensorflow~=2.17.0->tf-models-official) (0.12.1)\n",156 "Requirement already satisfied: charset-normalizer<4,>=2 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from requests->kaggle>=1.3.9->tf-models-official) (3.3.2)\n",157 "Requirement already satisfied: idna<4,>=2.5 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from requests->kaggle>=1.3.9->tf-models-official) (3.10)\n",158 "Requirement already satisfied: joblib>=1.2.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from scikit-learn>=0.21.3->seqeval->tf-models-official) (1.4.2)\n",159 "Requirement already satisfied: threadpoolctl>=3.1.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from scikit-learn>=0.21.3->seqeval->tf-models-official) (3.5.0)\n",160 "Requirement already satisfied: markdown>=2.6.8 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorboard<2.18,>=2.17->tensorflow~=2.17.0->tf-models-official) (3.7)\n",161 "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorboard<2.18,>=2.17->tensorflow~=2.17.0->tf-models-official) (0.7.2)\n",162 "Requirement already satisfied: werkzeug>=1.0.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from tensorboard<2.18,>=2.17->tensorflow~=2.17.0->tf-models-official) (3.0.4)\n",163 "Requirement already satisfied: webencodings in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from bleach->kaggle>=1.3.9->tf-models-official) (0.5.1)\n",164 "Requirement already satisfied: text-unidecode>=1.3 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from python-slugify->kaggle>=1.3.9->tf-models-official) (1.3)\n",165 "Requirement already satisfied: docstring-parser<1.0,>=0.15 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from simple-parsing->tensorflow-datasets->tf-models-official) (0.16)\n",166 "Requirement already satisfied: MarkupSafe>=2.1.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from werkzeug>=1.0.1->tensorboard<2.18,>=2.17->tensorflow~=2.17.0->tf-models-official) (2.1.5)\n",167 "Requirement already satisfied: markdown-it-py>=2.2.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from rich->keras>=3.2.0->tensorflow~=2.17.0->tf-models-official) (3.0.0)\n",168 "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from rich->keras>=3.2.0->tensorflow~=2.17.0->tf-models-official) (2.18.0)\n",169 "Requirement already satisfied: mdurl~=0.1 in /mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.2.0->tensorflow~=2.17.0->tf-models-official) (0.1.2)\n"170 ]171 }172 ],173 "source": [174 "# Install necessary python packages \n",175 "!pip install opencv-python\n",176 "!pip install -U \"tensorflow-text==2.17.*\"\n",177 "!pip install tf-models-official"178 ]179 },180 {181 "cell_type": "code",182 "execution_count": 4,183 "id": "039e5bdf-1486-44f2-a31c-f11f4e3eee7a",184 "metadata": {},185 "outputs": [186 {187 "name": "stderr",188 "output_type": "stream",189 "text": [190 "2024-09-22 15:39:22.582985: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",191 "2024-09-22 15:39:23.933322: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",192 "2024-09-22 15:39:24.526078: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",193 "2024-09-22 15:39:24.645627: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",194 "2024-09-22 15:39:25.481278: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",195 "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",196 "2024-09-22 15:39:32.888848: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"197 ]198 }199 ],200 "source": [201 "# Import libraries\n",202 "import os \n",203 "\n",204 "import numpy as np \n",205 "import matplotlib.pyplot as plt\n",206 "\n",207 "import tensorflow as tf\n",208 "import tensorflow_models as tfm\n",209 "import tensorflow_hub as hub\n",210 "import tensorflow_datasets as tfds\n",211 "tfds.disable_progress_bar()"212 ]213 },214 {215 "cell_type": "code",216 "execution_count": 5,217 "id": "54484154-3570-4901-99d7-6f78a85d5425",218 "metadata": {},219 "outputs": [220 {221 "name": "stderr",222 "output_type": "stream",223 "text": [224 "2024-09-22 15:40:06.071405: W external/local_tsl/tsl/platform/cloud/google_auth_provider.cc:184] All attempts to get a Google authentication bearer token failed, returning an empty token. Retrieving token from files failed with \"NOT_FOUND: Could not locate the credentials file.\". Retrieving token from GCE failed with \"FAILED_PRECONDITION: Error executing an HTTP request: libcurl code 6 meaning 'Couldn't resolve host name', error details: Could not resolve host: metadata.google.internal\".\n"225 ]226 },227 {228 "data": {229 "text/plain": [230 "['bert_config.json',\n",231 " 'bert_model.ckpt.data-00000-of-00001',\n",232 " 'bert_model.ckpt.index',\n",233 " 'vocab.txt']"234 ]235 },236 "execution_count": 5,237 "metadata": {},238 "output_type": "execute_result"239 }240 ],241 "source": [242 "gs_folder_bert = \"gs://cloud-tpu-checkpoints/bert/v3/uncased_L-12_H-768_A-12\"\n",243 "tf.io.gfile.listdir(gs_folder_bert)"244 ]245 },246 {247 "cell_type": "code",248 "execution_count": 6,249 "id": "f4b8c8ed-d2ab-4f46-9b3c-2d9343ee66aa",250 "metadata": {},251 "outputs": [252 {253 "name": "stdout",254 "output_type": "stream",255 "text": [256 "\u001b[1mDownloading and preparing dataset 1.43 MiB (download: 1.43 MiB, generated: 1.74 MiB, total: 3.17 MiB) to /home/nigel/tensorflow_datasets/glue/mrpc/2.0.0...\u001b[0m\n",257 "\u001b[1mDataset glue downloaded and prepared to /home/nigel/tensorflow_datasets/glue/mrpc/2.0.0. Subsequent calls will reuse this data.\u001b[0m\n"258 ]259 },260 {261 "name": "stderr",262 "output_type": "stream",263 "text": [264 "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",265 "I0000 00:00:1727012426.079668 2587 cuda_executor.cc:1001] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",266 "Your kernel may have been built without NUMA support.\n",267 "I0000 00:00:1727012426.362373 2587 cuda_executor.cc:1001] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",268 "Your kernel may have been built without NUMA support.\n",269 "I0000 00:00:1727012426.362444 2587 cuda_executor.cc:1001] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",270 "Your kernel may have been built without NUMA support.\n",271 "I0000 00:00:1727012426.366065 2587 cuda_executor.cc:1001] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",272 "Your kernel may have been built without NUMA support.\n",273 "I0000 00:00:1727012426.366196 2587 cuda_executor.cc:1001] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",274 "Your kernel may have been built without NUMA support.\n",275 "I0000 00:00:1727012426.366381 2587 cuda_executor.cc:1001] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",276 "Your kernel may have been built without NUMA support.\n",277 "I0000 00:00:1727012426.506578 2587 cuda_executor.cc:1001] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",278 "Your kernel may have been built without NUMA support.\n",279 "I0000 00:00:1727012426.506665 2587 cuda_executor.cc:1001] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",280 "Your kernel may have been built without NUMA support.\n",281 "2024-09-22 15:40:26.506673: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2112] Could not identify NUMA node of platform GPU id 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",282 "I0000 00:00:1727012426.506710 2587 cuda_executor.cc:1001] could not open file to read NUMA node: /sys/bus/pci/devices/0000:01:00.0/numa_node\n",283 "Your kernel may have been built without NUMA support.\n",284 "2024-09-22 15:40:26.509102: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 6096 MB memory: -> device: 0, name: NVIDIA GeForce RTX 2070, pci bus id: 0000:01:00.0, compute capability: 7.5\n"285 ]286 }287 ],288 "source": [289 "batch_size=32\n",290 "glue, info = tfds.load('glue/mrpc',\n",291 " with_info=True,\n",292 " batch_size=32)"293 ]294 },295 {296 "cell_type": "code",297 "execution_count": 7,298 "id": "191ea971-d9c8-4edf-a92f-59da426a88e3",299 "metadata": {},300 "outputs": [301 {302 "data": {303 "text/plain": [304 "{Split('train'): <_PrefetchDataset element_spec={'idx': TensorSpec(shape=(None,), dtype=tf.int32, name=None), 'label': TensorSpec(shape=(None,), dtype=tf.int64, name=None), 'sentence1': TensorSpec(shape=(None,), dtype=tf.string, name=None), 'sentence2': TensorSpec(shape=(None,), dtype=tf.string, name=None)}>,\n",305 " Split('validation'): <_PrefetchDataset element_spec={'idx': TensorSpec(shape=(None,), dtype=tf.int32, name=None), 'label': TensorSpec(shape=(None,), dtype=tf.int64, name=None), 'sentence1': TensorSpec(shape=(None,), dtype=tf.string, name=None), 'sentence2': TensorSpec(shape=(None,), dtype=tf.string, name=None)}>,\n",306 " Split('test'): <_PrefetchDataset element_spec={'idx': TensorSpec(shape=(None,), dtype=tf.int32, name=None), 'label': TensorSpec(shape=(None,), dtype=tf.int64, name=None), 'sentence1': TensorSpec(shape=(None,), dtype=tf.string, name=None), 'sentence2': TensorSpec(shape=(None,), dtype=tf.string, name=None)}>}"307 ]308 },309 "execution_count": 7,310 "metadata": {},311 "output_type": "execute_result"312 }313 ],314 "source": [315 "glue"316 ]317 },318 {319 "cell_type": "code",320 "execution_count": 8,321 "id": "c32d1e49-590a-443c-8995-444ea6263e73",322 "metadata": {},323 "outputs": [324 {325 "data": {326 "text/plain": [327 "FeaturesDict({\n",328 " 'idx': int32,\n",329 " 'label': ClassLabel(shape=(), dtype=int64, num_classes=2),\n",330 " 'sentence1': Text(shape=(), dtype=string),\n",331 " 'sentence2': Text(shape=(), dtype=string),\n",332 "})"333 ]334 },335 "execution_count": 8,336 "metadata": {},337 "output_type": "execute_result"338 }339 ],340 "source": [341 "info.features"342 ]343 },344 {345 "cell_type": "code",346 "execution_count": 9,347 "id": "71e0a84e-78b6-47dd-a3bd-2764d88e55f4",348 "metadata": {},349 "outputs": [350 {351 "data": {352 "text/plain": [353 "['not_equivalent', 'equivalent']"354 ]355 },356 "execution_count": 9,357 "metadata": {},358 "output_type": "execute_result"359 }360 ],361 "source": [362 "info.features['label'].names"363 ]364 },365 {366 "cell_type": "code",367 "execution_count": 10,368 "id": "7d557357-52cc-4f8d-bdbf-a55d64482ec3",369 "metadata": {},370 "outputs": [371 {372 "name": "stdout",373 "output_type": "stream",374 "text": [375 "idx : 1680\n",376 "label : 0\n",377 "sentence1: b'The identical rovers will act as robotic geologists , searching for evidence of past water .'\n",378 "sentence2: b'The rovers act as robotic geologists , moving on six wheels .'\n"379 ]380 },381 {382 "name": "stderr",383 "output_type": "stream",384 "text": [385 "2024-09-22 15:41:09.336833: W tensorflow/core/kernels/data/cache_dataset_ops.cc:913] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead.\n"386 ]387 }388 ],389 "source": [390 "example_batch = next(iter(glue['train']))\n",391 "\n",392 "for key, value in example_batch.items():\n",393 " print(f\"{key:9s}: {value[0].numpy()}\")"394 ]395 },396 {397 "cell_type": "code",398 "execution_count": 11,399 "id": "e0530302-8d22-4166-bf6d-4ef7d216f8a3",400 "metadata": {},401 "outputs": [],402 "source": [403 "tokenizer = tfm.nlp.layers.FastWordpieceBertTokenizer(\n",404 " vocab_file=os.path.join(gs_folder_bert, \"vocab.txt\"),\n",405 " lower_case=True)"406 ]407 },408 {409 "cell_type": "code",410 "execution_count": 12,411 "id": "40114915-48b0-44a6-b292-077fe6e10f3a",412 "metadata": {},413 "outputs": [414 {415 "data": {416 "text/plain": [417 "<tf.RaggedTensor [[[7592], [23435, 12314], [999]]]>"418 ]419 },420 "execution_count": 12,421 "metadata": {},422 "output_type": "execute_result"423 }424 ],425 "source": [426 "tokens = tokenizer(tf.constant([\"Hello TensorFlow!\"]))\n",427 "tokens"428 ]429 },430 {431 "cell_type": "code",432 "execution_count": 13,433 "id": "0983ffbe-6b67-4b62-b46f-1077af9b5ee6",434 "metadata": {},435 "outputs": [436 {437 "data": {438 "text/plain": [439 "{'vocab_size': 30522,\n",440 " 'start_of_sequence_id': 101,\n",441 " 'end_of_segment_id': 102,\n",442 " 'padding_id': 0,\n",443 " 'mask_id': 103}"444 ]445 },446 "execution_count": 13,447 "metadata": {},448 "output_type": "execute_result"449 }450 ],451 "source": [452 "special = tokenizer.get_special_tokens_dict()\n",453 "special"454 ]455 },456 {457 "cell_type": "code",458 "execution_count": 14,459 "id": "c3cf0cdb-75e1-456e-bbd5-7814937449d2",460 "metadata": {},461 "outputs": [],462 "source": [463 "max_seq_length = 128\n",464 "\n",465 "packer = tfm.nlp.layers.BertPackInputs(\n",466 " seq_length=max_seq_length,\n",467 " special_tokens_dict = tokenizer.get_special_tokens_dict())"468 ]469 },470 {471 "cell_type": "code",472 "execution_count": 15,473 "id": "2d071cf4-25fe-43b7-a43b-856400ab9b8e",474 "metadata": {},475 "outputs": [476 {477 "data": {478 "text/plain": [479 "<tf.RaggedTensor [[[7592], [23435, 12314]]]>"480 ]481 },482 "execution_count": 15,483 "metadata": {},484 "output_type": "execute_result"485 }486 ],487 "source": [488 "sentences1 = [\"hello tensorflow\"]\n",489 "tok1 = tokenizer(sentences1)\n",490 "tok1"491 ]492 },493 {494 "cell_type": "code",495 "execution_count": 16,496 "id": "a546b668-15cb-4ef7-a935-a1ca27673a8f",497 "metadata": {},498 "outputs": [499 {500 "data": {501 "text/plain": [502 "<tf.RaggedTensor [[[9119], [23435, 12314]]]>"503 ]504 },505 "execution_count": 16,506 "metadata": {},507 "output_type": "execute_result"508 }509 ],510 "source": [511 "sentences2 = [\"goodbye tensorflow\"]\n",512 "tok2 = tokenizer(sentences2)\n",513 "tok2"514 ]515 },516 {517 "cell_type": "code",518 "execution_count": 17,519 "id": "d668b2b1-771a-470c-bfc5-91dfdffebbbd",520 "metadata": {},521 "outputs": [522 {523 "name": "stdout",524 "output_type": "stream",525 "text": [526 "input_word_ids : [[ 101 7592 23435 12314 102 9119 23435 12314 102 0 0 0]]\n",527 "input_mask : [[1 1 1 1 1 1 1 1 1 0 0 0]]\n",528 "input_type_ids : [[0 0 0 0 0 1 1 1 1 0 0 0]]\n"529 ]530 }531 ],532 "source": [533 "packed = packer([tok1, tok2])\n",534 "\n",535 "for key, tensor in packed.items():\n",536 " print(f\"{key:15s}: {tensor[:, :12]}\")"537 ]538 },539 {540 "cell_type": "code",541 "execution_count": 18,542 "id": "8359b2db-0ea1-4e7c-8740-f66204798dec",543 "metadata": {},544 "outputs": [],545 "source": [546 "class BertInputProcessor(tf.keras.layers.Layer):\n",547 " def __init__(self, tokenizer, packer):\n",548 " super().__init__()\n",549 " self.tokenizer = tokenizer\n",550 " self.packer = packer\n",551 "\n",552 " def call(self, inputs):\n",553 " tok1 = self.tokenizer(inputs['sentence1'])\n",554 " tok2 = self.tokenizer(inputs['sentence2'])\n",555 "\n",556 " packed = self.packer([tok1, tok2])\n",557 "\n",558 " if 'label' in inputs:\n",559 " return packed, inputs['label']\n",560 " else:\n",561 " return packed"562 ]563 },564 {565 "cell_type": "code",566 "execution_count": 19,567 "id": "d4ccbb51-fbff-443b-9791-aa214a490afe",568 "metadata": {},569 "outputs": [],570 "source": [571 "bert_inputs_processor = BertInputProcessor(tokenizer, packer)"572 ]573 },574 {575 "cell_type": "code",576 "execution_count": 20,577 "id": "a6d7f15e-563a-448c-9af5-ea8c589e699d",578 "metadata": {},579 "outputs": [],580 "source": [581 "glue_train = glue['train'].map(bert_inputs_processor).prefetch(1)"582 ]583 },584 {585 "cell_type": "code",586 "execution_count": 21,587 "id": "8e70fe61-d47d-40d8-80d1-fe553d65d09f",588 "metadata": {},589 "outputs": [590 {591 "name": "stderr",592 "output_type": "stream",593 "text": [594 "2024-09-22 15:43:09.161960: E tensorflow/core/util/util.cc:131] oneDNN supports DT_INT64 only on platforms with AVX-512. Falling back to the default Eigen-based implementation if present.\n",595 "2024-09-22 15:43:09.241604: W tensorflow/core/kernels/data/cache_dataset_ops.cc:913] The calling iterator did not fully read the dataset being cached. In order to avoid unexpected truncation of the dataset, the partially cached contents of the dataset will be discarded. This can happen if you have an input pipeline similar to `dataset.cache().take(k).repeat()`. You should use `dataset.take(k).cache().repeat()` instead.\n"596 ]597 }598 ],599 "source": [600 "example_inputs, example_labels = next(iter(glue_train))"601 ]602 },603 {604 "cell_type": "code",605 "execution_count": 22,606 "id": "dab648c3-f458-4250-9758-70edbf545b08",607 "metadata": {},608 "outputs": [609 {610 "data": {611 "text/plain": [612 "{'input_word_ids': <tf.Tensor: shape=(32, 128), dtype=int32, numpy=\n",613 " array([[ 101, 1996, 7235, ..., 0, 0, 0],\n",614 " [ 101, 2625, 2084, ..., 0, 0, 0],\n",615 " [ 101, 6804, 1011, ..., 0, 0, 0],\n",616 " ...,\n",617 " [ 101, 2021, 2049, ..., 0, 0, 0],\n",618 " [ 101, 2274, 2062, ..., 0, 0, 0],\n",619 " [ 101, 2043, 1037, ..., 0, 0, 0]], dtype=int32)>,\n",620 " 'input_mask': <tf.Tensor: shape=(32, 128), dtype=int32, numpy=\n",621 " array([[1, 1, 1, ..., 0, 0, 0],\n",622 " [1, 1, 1, ..., 0, 0, 0],\n",623 " [1, 1, 1, ..., 0, 0, 0],\n",624 " ...,\n",625 " [1, 1, 1, ..., 0, 0, 0],\n",626 " [1, 1, 1, ..., 0, 0, 0],\n",627 " [1, 1, 1, ..., 0, 0, 0]], dtype=int32)>,\n",628 " 'input_type_ids': <tf.Tensor: shape=(32, 128), dtype=int32, numpy=\n",629 " array([[0, 0, 0, ..., 0, 0, 0],\n",630 " [0, 0, 0, ..., 0, 0, 0],\n",631 " [0, 0, 0, ..., 0, 0, 0],\n",632 " ...,\n",633 " [0, 0, 0, ..., 0, 0, 0],\n",634 " [0, 0, 0, ..., 0, 0, 0],\n",635 " [0, 0, 0, ..., 0, 0, 0]], dtype=int32)>}"636 ]637 },638 "execution_count": 22,639 "metadata": {},640 "output_type": "execute_result"641 }642 ],643 "source": [644 "example_inputs"645 ]646 },647 {648 "cell_type": "code",649 "execution_count": 23,650 "id": "3edfc705-5cc3-45a0-8899-f31243664b8b",651 "metadata": {},652 "outputs": [653 {654 "data": {655 "text/plain": [656 "<tf.Tensor: shape=(32,), dtype=int64, numpy=\n",657 "array([0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1,\n",658 " 1, 1, 1, 1, 1, 0, 0, 1, 0, 1])>"659 ]660 },661 "execution_count": 23,662 "metadata": {},663 "output_type": "execute_result"664 }665 ],666 "source": [667 "example_labels"668 ]669 },670 {671 "cell_type": "code",672 "execution_count": 24,673 "id": "86170575-92af-4dab-a268-3f7542e8c8c2",674 "metadata": {},675 "outputs": [676 {677 "name": "stdout",678 "output_type": "stream",679 "text": [680 "input_word_ids shape: (32, 128)\n",681 "input_mask shape: (32, 128)\n",682 "input_type_ids shape: (32, 128)\n",683 "labels shape: (32,)\n"684 ]685 }686 ],687 "source": [688 "for key, value in example_inputs.items():\n",689 " print(f'{key:15s} shape: {value.shape}')\n",690 "\n",691 "print(f'{\"labels\":15s} shape: {example_labels.shape}')"692 ]693 },694 {695 "cell_type": "code",696 "execution_count": 25,697 "id": "362ac21a-ee7c-4c1a-b54d-a0bf5189c58e",698 "metadata": {},699 "outputs": [700 {701 "data": {702 "text/plain": [703 "<matplotlib.collections.QuadMesh at 0x7fda641e8290>"704 ]705 },706 "execution_count": 25,707 "metadata": {},708 "output_type": "execute_result"709 }710 ],711 "source": [712 "plt.pcolormesh(example_inputs['input_word_ids'])"713 ]714 },715 {716 "cell_type": "code",717 "execution_count": 26,718 "id": "9c5d4844-efd8-4235-8487-e984c6a00030",719 "metadata": {},720 "outputs": [],721 "source": [722 "glue_validation = glue['validation'].map(bert_inputs_processor).prefetch(1)\n",723 "glue_test = glue['test'].map(bert_inputs_processor).prefetch(1)"724 ]725 },726 {727 "cell_type": "code",728 "execution_count": 27,729 "id": "ca14bd16-c9ea-4f02-bacc-1333b5068e94",730 "metadata": {},731 "outputs": [732 {733 "data": {734 "text/plain": [735 "{'attention_probs_dropout_prob': 0.1,\n",736 " 'hidden_act': 'gelu',\n",737 " 'hidden_dropout_prob': 0.1,\n",738 " 'hidden_size': 768,\n",739 " 'initializer_range': 0.02,\n",740 " 'intermediate_size': 3072,\n",741 " 'max_position_embeddings': 512,\n",742 " 'num_attention_heads': 12,\n",743 " 'num_hidden_layers': 12,\n",744 " 'type_vocab_size': 2,\n",745 " 'vocab_size': 30522}"746 ]747 },748 "execution_count": 27,749 "metadata": {},750 "output_type": "execute_result"751 }752 ],753 "source": [754 "import json\n",755 "\n",756 "bert_config_file = os.path.join(gs_folder_bert, \"bert_config.json\")\n",757 "config_dict = json.loads(tf.io.gfile.GFile(bert_config_file).read())\n",758 "config_dict"759 ]760 },761 {762 "cell_type": "code",763 "execution_count": 28,764 "id": "159574d8-051d-43f6-9cde-b05cc5b3e9aa",765 "metadata": {},766 "outputs": [],767 "source": [768 "encoder_config = tfm.nlp.encoders.EncoderConfig({\n",769 " 'type':'bert',\n",770 " 'bert': config_dict\n",771 "})"772 ]773 },774 {775 "cell_type": "code",776 "execution_count": 29,777 "id": "171fbff9-62e2-4b2c-b6b7-614e2c216a13",778 "metadata": {},779 "outputs": [780 {781 "data": {782 "text/plain": [783 "<official.nlp.modeling.networks.bert_encoder.BertEncoder at 0x7fda3c393b10>"784 ]785 },786 "execution_count": 29,787 "metadata": {},788 "output_type": "execute_result"789 }790 ],791 "source": [792 "bert_encoder = tfm.nlp.encoders.build_encoder(encoder_config)\n",793 "bert_encoder"794 ]795 },796 {797 "cell_type": "code",798 "execution_count": 30,799 "id": "63474540-48eb-4b83-82f3-1099fe417382",800 "metadata": {},801 "outputs": [],802 "source": [803 "bert_classifier = tfm.nlp.models.BertClassifier(network=bert_encoder, num_classes=2)"804 ]805 },806 {807 "cell_type": "code",808 "execution_count": 31,809 "id": "6fddb52b-5da7-457a-a598-d439b6c520d5",810 "metadata": {},811 "outputs": [812 {813 "data": {814 "text/plain": [815 "array([[ 0.4356808 , 1.4117762 ],\n",816 " [-0.03644767, 1.7365586 ],\n",817 " [ 0.5098716 , 0.9173252 ],\n",818 " [-0.02355742, 0.6873396 ],\n",819 " [-0.03669066, 0.5846623 ],\n",820 " [ 0.6288214 , 0.5022205 ],\n",821 " [ 0.7397315 , 0.99099755],\n",822 " [-0.9988446 , 0.84491223],\n",823 " [ 0.50856286, 0.9842516 ],\n",824 " [-0.03501675, 0.7552087 ]], dtype=float32)"825 ]826 },827 "execution_count": 31,828 "metadata": {},829 "output_type": "execute_result"830 }831 ],832 "source": [833 "bert_classifier(\n",834 " example_inputs, training=True).numpy()[:10]"835 ]836 },837 {838 "cell_type": "code",839 "execution_count": 32,840 "id": "45f1a9ea-015d-438a-bb0e-4e0d53ec99f1",841 "metadata": {},842 "outputs": [843 {844 "name": "stdout",845 "output_type": "stream",846 "text": [847 "You must install pydot (`pip install pydot`) for `plot_model` to work.\n"848 ]849 }850 ],851 "source": [852 "tf.keras.utils.plot_model(bert_encoder, show_shapes=True, dpi=48)"853 ]854 },855 {856 "cell_type": "code",857 "execution_count": 33,858 "id": "d31dfd62-bfc1-4062-bb75-562b6f1ee665",859 "metadata": {},860 "outputs": [861 {862 "data": {863 "text/plain": [864 "<tensorflow.python.checkpoint.checkpoint.CheckpointLoadStatus at 0x7fda3463b8d0>"865 ]866 },867 "execution_count": 33,868 "metadata": {},869 "output_type": "execute_result"870 }871 ],872 "source": [873 "checkpoint = tf.train.Checkpoint(encoder=bert_encoder)\n",874 "checkpoint.read(\n",875 " os.path.join(gs_folder_bert, 'bert_model.ckpt')).assert_consumed()"876 ]877 },878 {879 "cell_type": "code",880 "execution_count": 34,881 "id": "94e2b7be-e8cc-426d-bbb0-dbf3a94b41c3",882 "metadata": {},883 "outputs": [],884 "source": [885 "# Set up epochs and steps\n",886 "epochs = 5\n",887 "batch_size = 32\n",888 "eval_batch_size = 32\n",889 "\n",890 "train_data_size = info.splits['train'].num_examples\n",891 "steps_per_epoch = int(train_data_size / batch_size)\n",892 "num_train_steps = steps_per_epoch * epochs\n",893 "warmup_steps = int(0.1 * num_train_steps)\n",894 "initial_learning_rate=2e-5"895 ]896 },897 {898 "cell_type": "code",899 "execution_count": 39,900 "id": "157b6c11-169d-4d20-8aa0-11e831b4b280",901 "metadata": {},902 "outputs": [],903 "source": [904 "linear_decay = tf.keras.optimizers.schedules.PolynomialDecay(\n",905 " initial_learning_rate=initial_learning_rate,\n",906 " end_learning_rate=0,\n",907 " decay_steps=num_train_steps)"908 ]909 },910 {911 "cell_type": "code",912 "execution_count": 40,913 "id": "a2f806cc-ac03-4a84-9db4-35b742aae2f7",914 "metadata": {},915 "outputs": [916 {917 "ename": "ValueError",918 "evalue": "Attempt to convert a value (<keras.src.optimizers.schedules.learning_rate_schedule.PolynomialDecay object at 0x7fda34388850>) with an unsupported type (<class 'keras.src.optimizers.schedules.learning_rate_schedule.PolynomialDecay'>) to a Tensor.",919 "output_type": "error",920 "traceback": [921 "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",922 "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",923 "Cell \u001b[0;32mIn[40], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m warmup_schedule \u001b[38;5;241m=\u001b[39m \u001b[43mtfm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptimization\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlr_schedule\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLinearWarmup\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[43mwarmup_learning_rate\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mafter_warmup_lr_sched\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mlinear_decay\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43mwarmup_steps\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mwarmup_steps\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m)\u001b[49m\n",924 "File \u001b[0;32m/mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages/official/modeling/optimization/lr_schedule.py:131\u001b[0m, in \u001b[0;36mLinearWarmup.__init__\u001b[0;34m(self, after_warmup_lr_sched, warmup_steps, warmup_learning_rate, name)\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_final_warmup_lr \u001b[38;5;241m=\u001b[39m after_warmup_lr_sched(warmup_steps)\n\u001b[1;32m 130\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 131\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_final_warmup_lr \u001b[38;5;241m=\u001b[39m \u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcast\u001b[49m\u001b[43m(\u001b[49m\u001b[43mafter_warmup_lr_sched\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat32\u001b[49m\u001b[43m)\u001b[49m\n",925 "File \u001b[0;32m/mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages/tensorflow/python/util/traceback_utils.py:153\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 152\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[0;32m--> 153\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 154\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 155\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",926 "File \u001b[0;32m/mnt/f/wkspc-linux/tf-gpu/.venv/lib/python3.11/site-packages/tensorflow/python/framework/constant_op.py:108\u001b[0m, in \u001b[0;36mconvert_to_eager_tensor\u001b[0;34m(value, ctx, dtype)\u001b[0m\n\u001b[1;32m 106\u001b[0m dtype \u001b[38;5;241m=\u001b[39m dtypes\u001b[38;5;241m.\u001b[39mas_dtype(dtype)\u001b[38;5;241m.\u001b[39mas_datatype_enum\n\u001b[1;32m 107\u001b[0m ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[0;32m--> 108\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mops\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mEagerTensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdevice_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[43m)\u001b[49m\n",927 "\u001b[0;31mValueError\u001b[0m: Attempt to convert a value (<keras.src.optimizers.schedules.learning_rate_schedule.PolynomialDecay object at 0x7fda34388850>) with an unsupported type (<class 'keras.src.optimizers.schedules.learning_rate_schedule.PolynomialDecay'>) to a Tensor."928 ]929 }930 ],931 "source": [932 "warmup_schedule = tfm.optimization.lr_schedule.LinearWarmup(\n",933 " warmup_learning_rate = 0,\n",934 " after_warmup_lr_sched = linear_decay,\n",935 " warmup_steps = warmup_steps\n",936 ")"937 ]938 },939 {940 "cell_type": "code",941 "execution_count": 37,942 "id": "82a43876-ee7f-40e7-b598-2ab6f54fc077",943 "metadata": {},944 "outputs": [945 {946 "ename": "NameError",947 "evalue": "name 'warmup_schedule' is not defined",948 "output_type": "error",949 "traceback": [950 "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",951 "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",952 "Cell \u001b[0;32mIn[37], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m x \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mlinspace(\u001b[38;5;241m0\u001b[39m, num_train_steps, \u001b[38;5;241m1001\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m y \u001b[38;5;241m=\u001b[39m \u001b[43m[\u001b[49m\u001b[43mwarmup_schedule\u001b[49m\u001b[43m(\u001b[49m\u001b[43mxi\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mxi\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mx\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 3\u001b[0m plt\u001b[38;5;241m.\u001b[39mplot(x,y)\n\u001b[1;32m 4\u001b[0m plt\u001b[38;5;241m.\u001b[39mxlabel(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTrain step\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",953 "Cell \u001b[0;32mIn[37], line 2\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1\u001b[0m x \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mlinspace(\u001b[38;5;241m0\u001b[39m, num_train_steps, \u001b[38;5;241m1001\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m y \u001b[38;5;241m=\u001b[39m [\u001b[43mwarmup_schedule\u001b[49m(xi) \u001b[38;5;28;01mfor\u001b[39;00m xi \u001b[38;5;129;01min\u001b[39;00m x]\n\u001b[1;32m 3\u001b[0m plt\u001b[38;5;241m.\u001b[39mplot(x,y)\n\u001b[1;32m 4\u001b[0m plt\u001b[38;5;241m.\u001b[39mxlabel(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTrain step\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",954 "\u001b[0;31mNameError\u001b[0m: name 'warmup_schedule' is not defined"955 ]956 }957 ],958 "source": [959 "x = tf.linspace(0, num_train_steps, 1001)\n",960 "y = [warmup_schedule(xi) for xi in x]\n",961 "plt.plot(x,y)\n",962 "plt.xlabel('Train step')\n",963 "plt.ylabel('Learning rate')"964 ]965 },966 {967 "cell_type": "code",968 "execution_count": null,969 "id": "9340e1f3-2229-4f79-9ed5-70541c10eff7",970 "metadata": {},971 "outputs": [],972 "source": [973 "optimizer = tf.keras.optimizers.experimental.Adam(\n",974 " learning_rate = warmup_schedule)"975 ]976 },977 {978 "cell_type": "code",979 "execution_count": null,980 "id": "b560dd86-1f03-4adc-97d4-62c6dedfd69d",981 "metadata": {},982 "outputs": [],983 "source": [984 "metrics = [tf.keras.metrics.SparseCategoricalAccuracy('accuracy', dtype=tf.float32)]\n",985 "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",986 "\n",987 "bert_classifier.compile(\n",988 " optimizer=optimizer,\n",989 " loss=loss,\n",990 " metrics=metrics)"991 ]992 },993 {994 "cell_type": "code",995 "execution_count": null,996 "id": "402548eb-0d27-40d7-b69f-94cacb162ab0",997 "metadata": {},998 "outputs": [],999 "source": []1000 }1001 ],1002 "metadata": {1003 "kernelspec": {1004 "display_name": "Python 3 (ipykernel)",1005 "language": "python",1006 "name": "python3"1007 },1008 "language_info": {1009 "codemirror_mode": {1010 "name": "ipython",1011 "version": 31012 },1013 "file_extension": ".py",1014 "mimetype": "text/x-python",1015 "name": "python",1016 "nbconvert_exporter": "python",1017 "pygments_lexer": "ipython3",1018 "version": "3.11.2"1019 }1020 },1021 "nbformat": 4,1022 "nbformat_minor": 51023}1024