camilin29/github_pull_request_classifier
0
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 1,6 "metadata": {},7 "outputs": [8 {9 "name": "stderr",10 "output_type": "stream",11 "text": [12 "/home/camilo/anaconda3/envs/diplom/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",13 " from .autonotebook import tqdm as notebook_tqdm\n"14 ]15 }16 ],17 "source": [18 "import torch\n",19 "from transformers import AutoConfig, AutoModel\n",20 "from transformers import RobertaModel, RobertaTokenizer\n",21 "import numpy as np"22 ]23 },24 {25 "cell_type": "code",26 "execution_count": 2,27 "metadata": {},28 "outputs": [29 {30 "data": {31 "text/plain": [32 "'cuda'"33 ]34 },35 "execution_count": 2,36 "metadata": {},37 "output_type": "execute_result"38 }39 ],40 "source": [41 "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",42 "device"43 ]44 },45 {46 "cell_type": "code",47 "execution_count": 3,48 "metadata": {},49 "outputs": [50 {51 "name": "stderr",52 "output_type": "stream",53 "text": [54 "Some weights of RobertaModel were not initialized from the model checkpoint at roberta-base and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",55 "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"56 ]57 },58 {59 "data": {60 "text/plain": [61 "BERTClass(\n",62 " (bert_model): RobertaModel(\n",63 " (embeddings): RobertaEmbeddings(\n",64 " (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",65 " (position_embeddings): Embedding(514, 768, padding_idx=1)\n",66 " (token_type_embeddings): Embedding(1, 768)\n",67 " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",68 " (dropout): Dropout(p=0.1, inplace=False)\n",69 " )\n",70 " (encoder): RobertaEncoder(\n",71 " (layer): ModuleList(\n",72 " (0-11): 12 x RobertaLayer(\n",73 " (attention): RobertaAttention(\n",74 " (self): RobertaSelfAttention(\n",75 " (query): Linear(in_features=768, out_features=768, bias=True)\n",76 " (key): Linear(in_features=768, out_features=768, bias=True)\n",77 " (value): Linear(in_features=768, out_features=768, bias=True)\n",78 " (dropout): Dropout(p=0.1, inplace=False)\n",79 " )\n",80 " (output): RobertaSelfOutput(\n",81 " (dense): Linear(in_features=768, out_features=768, bias=True)\n",82 " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",83 " (dropout): Dropout(p=0.1, inplace=False)\n",84 " )\n",85 " )\n",86 " (intermediate): RobertaIntermediate(\n",87 " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",88 " (intermediate_act_fn): GELUActivation()\n",89 " )\n",90 " (output): RobertaOutput(\n",91 " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",92 " (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",93 " (dropout): Dropout(p=0.1, inplace=False)\n",94 " )\n",95 " )\n",96 " )\n",97 " )\n",98 " (pooler): RobertaPooler(\n",99 " (dense): Linear(in_features=768, out_features=768, bias=True)\n",100 " (activation): Tanh()\n",101 " )\n",102 " )\n",103 " (dropout): Dropout(p=0.3, inplace=False)\n",104 " (linear): Linear(in_features=768, out_features=4, bias=True)\n",105 ")"106 ]107 },108 "execution_count": 3,109 "metadata": {},110 "output_type": "execute_result"111 }112 ],113 "source": [114 "\n",115 "\n",116 "class BERTClass(torch.nn.Module):\n",117 " def __init__(self):\n",118 " super(BERTClass, self).__init__()\n",119 " self.config = AutoConfig.from_pretrained('roberta-base')\n",120 " self.bert_model = AutoModel.from_pretrained('roberta-base', return_dict=True)\n",121 " self.dropout = torch.nn.Dropout(0.3)\n",122 " self.linear = torch.nn.Linear(768,4)\n",123 " \n",124 " def forward(self, ids, mask, token_type_ids):\n",125 " output = self.bert_model(\n",126 " ids, \n",127 " attention_mask=mask, \n",128 " token_type_ids=token_type_ids\n",129 " )\n",130 "\n",131 " output_dropout = self.dropout(output.pooler_output)\n",132 " output = self.linear(output_dropout)\n",133 " return output\n",134 "\n",135 "# Load the model\n",136 "model = BERTClass()\n",137 "model.load_state_dict(torch.load('roberta_model.pth'))\n",138 "model.to(device)"139 ]140 },141 {142 "cell_type": "code",143 "execution_count": 4,144 "metadata": {},145 "outputs": [],146 "source": [147 "loaded_tokenizer = RobertaTokenizer.from_pretrained('roberta_tokenizer', local_files_only=True)"148 ]149 },150 {151 "cell_type": "code",152 "execution_count": 5,153 "metadata": {},154 "outputs": [],155 "source": [156 "input_text = \"ENH: Support export PyArray_API and PyUFunc_API from shared libraries\"\n",157 "\n",158 "# Encode the input text\n",159 "input_ids = loaded_tokenizer.encode(input_text, return_tensors=\"pt\")\n",160 "\n",161 "# Add attention mask\n",162 "attention_mask = input_ids.ne(loaded_tokenizer.pad_token_id)\n",163 "\n",164 "# Set token type ids to zeros (for a single sentence)\n",165 "token_type_ids = torch.zeros_like(input_ids)\n",166 "\n",167 "# Pass the input through the model\n",168 "output = model(input_ids.to(device), attention_mask.to(device), token_type_ids.to(device))"169 ]170 },171 {172 "cell_type": "code",173 "execution_count": 6,174 "metadata": {},175 "outputs": [],176 "source": [177 "#target_cols = ['deprecated', 'features', 'fix', 'maintenance']"178 ]179 },180 {181 "cell_type": "code",182 "execution_count": 7,183 "metadata": {},184 "outputs": [185 {186 "data": {187 "text/plain": [188 "tensor([-6.6440, -5.9061, -6.3155, 5.6980], device='cuda:0',\n",189 " grad_fn=<SelectBackward0>)"190 ]191 },192 "execution_count": 7,193 "metadata": {},194 "output_type": "execute_result"195 }196 ],197 "source": [198 "output[0]"199 ]200 },201 {202 "cell_type": "code",203 "execution_count": 10,204 "metadata": {},205 "outputs": [206 {207 "name": "stdout",208 "output_type": "stream",209 "text": [210 "[-6.643972 -5.9061 -6.3154554 5.69799 ]\n"211 ]212 }213 ],214 "source": [215 "output_tensor = output[0]\n",216 "output_tensor_cpu = output_tensor.detach().cpu() # Copy the tensor to the CPU and detach it from the computation graph\n",217 "output_array = output_tensor_cpu.numpy() # Convert the CPU tensor to a NumPy array\n",218 "print(output_array)"219 ]220 },221 {222 "cell_type": "code",223 "execution_count": 11,224 "metadata": {},225 "outputs": [226 {227 "data": {228 "text/plain": [229 "3"230 ]231 },232 "execution_count": 11,233 "metadata": {},234 "output_type": "execute_result"235 }236 ],237 "source": [238 "np.argmax(output_array)"239 ]240 },241 {242 "cell_type": "code",243 "execution_count": null,244 "metadata": {},245 "outputs": [],246 "source": []247 }248 ],249 "metadata": {250 "kernelspec": {251 "display_name": "diplom",252 "language": "python",253 "name": "python3"254 },255 "language_info": {256 "codemirror_mode": {257 "name": "ipython",258 "version": 3259 },260 "file_extension": ".py",261 "mimetype": "text/x-python",262 "name": "python",263 "nbconvert_exporter": "python",264 "pygments_lexer": "ipython3",265 "version": "3.10.13"266 }267 },268 "nbformat": 4,269 "nbformat_minor": 2270}271