Berbex/FinalProject
1
1""" CODE TO TRY IN COLAB2!pip install -q transformers datasets torch gradio console_logging numpy3 4import gradio as gr5import torch6from datasets import load_dataset7from console_logging.console import Console8import numpy as np9from transformers import AutoModelForSequenceClassification, AutoTokenizer10from transformers import TrainingArguments, Trainer11from sklearn.metrics import f1_score, roc_auc_score, accuracy_score12from transformers import EvalPrediction13import torch14console = Console()15 16dataset = load_dataset("zeroshot/twitter-financial-news-sentiment", )17 18 19model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased")20tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")21 22#labels = [label for label in dataset['train'].features.keys() if label not in ['text']]23 24labels = ["Bearish", "Bullish", "Neutral"]25 26def preprocess_data(examples):27 # take a batch of texts28 text = examples["text"]29 # encode them30 encoding = tokenizer(text, padding="max_length", truncation=True, max_length=128)31 # add labels32 #labels_batch = {k: examples[k] for k in examples.keys() if k in labels}33 labels_batch = {'Bearish': [], 'Bullish': [], 'Neutral': []}34 for i in range (len(examples['label'])):35 labels_batch["Bearish"].append(False)36 labels_batch["Bullish"].append(False)37 labels_batch["Neutral"].append(False)38 39 if examples['label'][i] == 0:40 labels_batch["Bearish"][i] = True41 42 elif examples['label'][i] == 1:43 labels_batch["Bullish"][i] = True44 45 else:46 labels_batch["Neutral"][i] = True47 48 # create numpy array of shape (batch_size, num_labels)49 labels_matrix = np.zeros((len(text), len(labels)))50 # fill numpy array51 for idx, label in enumerate(labels):52 labels_matrix[:, idx] = labels_batch[label]53 54 encoding["labels"] = labels_matrix.tolist()55 56 return encoding57 58encoded_dataset = dataset.map(preprocess_data, batched=True, remove_columns=dataset['train'].column_names)59 60encoded_dataset.set_format("torch")61 62id2label = {idx:label for idx, label in enumerate(labels)}63label2id = {label:idx for idx, label in enumerate(labels)}64 65model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased",66 problem_type="multi_label_classification", 67 num_labels=len(labels),68 id2label=id2label,69 label2id=label2id)70 71batch_size = 872metric_name = "f1"73 74args = TrainingArguments(75 f"bert-finetuned-sem_eval-english",76 evaluation_strategy = "epoch",77 save_strategy = "epoch",78 learning_rate=2e-5,79 per_device_train_batch_size=batch_size,80 per_device_eval_batch_size=batch_size,81 num_train_epochs=5,82 weight_decay=0.01,83 load_best_model_at_end=True,84 metric_for_best_model=metric_name,85 #push_to_hub=True,86)87 88# source: https://jesusleal.io/2021/04/21/Longformer-multilabel-classification/89def multi_label_metrics(predictions, labels, threshold=0.5):90 # first, apply sigmoid on predictions which are of shape (batch_size, num_labels)91 sigmoid = torch.nn.Sigmoid()92 probs = sigmoid(torch.Tensor(predictions))93 # next, use threshold to turn them into integer predictions94 y_pred = np.zeros(probs.shape)95 y_pred[np.where(probs >= threshold)] = 196 # finally, compute metrics97 y_true = labels98 f1_micro_average = f1_score(y_true=y_true, y_pred=y_pred, average='micro')99 roc_auc = roc_auc_score(y_true, y_pred, average = 'micro')100 accuracy = accuracy_score(y_true, y_pred)101 # return as dictionary102 metrics = {'f1': f1_micro_average,103 'roc_auc': roc_auc,104 'accuracy': accuracy}105 return metrics106 107def compute_metrics(p: EvalPrediction):108 preds = p.predictions[0] if isinstance(p.predictions, 109 tuple) else p.predictions110 result = multi_label_metrics(111 predictions=preds, 112 labels=p.label_ids)113 return result114 115 116trainer = Trainer(117 model,118 args,119 train_dataset=encoded_dataset["train"],120 eval_dataset=encoded_dataset["validation"],121 tokenizer=tokenizer,122 compute_metrics=compute_metrics123)124 125trainer.train()126 127trainer.evaluate()128"""129 130# Version to gradio and HuggingFace, doesn't works like the colab version, this version use the exported model, possible without the fine tuning131 132import torch133from datasets import load_dataset134from console_logging.console import Console135import numpy as np136from transformers import AutoModelForSequenceClassification, AutoTokenizer137from transformers import TrainingArguments, Trainer138from sklearn.metrics import f1_score, roc_auc_score, accuracy_score139from transformers import EvalPrediction140import torch141import gradio as gr142 143console = Console()144 145dataset = load_dataset("zeroshot/twitter-financial-news-sentiment", )146 147 148model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased")149tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")150 151#labels = [label for label in dataset['train'].features.keys() if label not in ['text']]152 153labels = ["Bearish", "Bullish", "Neutral"]154 155def preprocess_data(examples):156 # take a batch of texts157 text = examples["text"]158 # encode them159 encoding = tokenizer(text, padding="max_length", truncation=True, max_length=128)160 # add labels161 #labels_batch = {k: examples[k] for k in examples.keys() if k in labels}162 labels_batch = {'Bearish': [], 'Bullish': [], 'Neutral': []}163 for i in range (len(examples['label'])):164 labels_batch["Bearish"].append(False)165 labels_batch["Bullish"].append(False)166 labels_batch["Neutral"].append(False)167 168 if examples['label'][i] == 0:169 labels_batch["Bearish"][i] = True170 171 elif examples['label'][i] == 1:172 labels_batch["Bullish"][i] = True173 174 else:175 labels_batch["Neutral"][i] = True176 177 # create numpy array of shape (batch_size, num_labels)178 labels_matrix = np.zeros((len(text), len(labels)))179 # fill numpy array180 for idx, label in enumerate(labels):181 labels_matrix[:, idx] = labels_batch[label]182 183 encoding["labels"] = labels_matrix.tolist()184 185 return encoding186 187encoded_dataset = dataset.map(preprocess_data, batched=True, remove_columns=dataset['train'].column_names)188 189encoded_dataset.set_format("torch")190 191id2label = {idx:label for idx, label in enumerate(labels)}192label2id = {label:idx for idx, label in enumerate(labels)}193 194model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased",195 problem_type="multi_label_classification", 196 num_labels=len(labels),197 id2label=id2label,198 label2id=label2id)199 200batch_size = 8201metric_name = "f1"202 203args = TrainingArguments(204 f"bert-finetuned-sem_eval-english",205 evaluation_strategy = "epoch",206 save_strategy = "epoch",207 learning_rate=2e-5,208 per_device_train_batch_size=batch_size,209 per_device_eval_batch_size=batch_size,210 num_train_epochs=5,211 weight_decay=0.01,212 load_best_model_at_end=True,213 metric_for_best_model=metric_name,214 #push_to_hub=True,215)216 217# source: https://jesusleal.io/2021/04/21/Longformer-multilabel-classification/218def multi_label_metrics(predictions, labels, threshold=0.5):219 # first, apply sigmoid on predictions which are of shape (batch_size, num_labels)220 sigmoid = torch.nn.Sigmoid()221 probs = sigmoid(torch.Tensor(predictions))222 # next, use threshold to turn them into integer predictions223 y_pred = np.zeros(probs.shape)224 y_pred[np.where(probs >= threshold)] = 1225 # finally, compute metrics226 y_true = labels227 f1_micro_average = f1_score(y_true=y_true, y_pred=y_pred, average='micro')228 roc_auc = roc_auc_score(y_true, y_pred, average = 'micro')229 accuracy = accuracy_score(y_true, y_pred)230 # return as dictionary231 metrics = {'f1': f1_micro_average,232 'roc_auc': roc_auc,233 'accuracy': accuracy}234 return metrics235 236def compute_metrics(p: EvalPrediction):237 preds = p.predictions[0] if isinstance(p.predictions, 238 tuple) else p.predictions239 result = multi_label_metrics(240 predictions=preds, 241 labels=p.label_ids)242 return result243 244 245text_ = "Bitcoin to the moon"246model = torch.load("./model.pt", map_location=torch.device('cpu'))247 248trainer = Trainer(249 model,250 args,251 train_dataset=encoded_dataset["train"],252 eval_dataset=encoded_dataset["validation"],253 tokenizer=tokenizer,254 compute_metrics=compute_metrics255)256 257tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")258 259def predict(text):260 261 encoding = tokenizer(text, return_tensors="pt")262 encoding = {k: v.to(trainer.model.device) for k,v in encoding.items()}263 264 outputs = trainer.model(**encoding)265 266 logits = outputs.logits267 logits.shape268 269 270 # apply sigmoid + threshold271 sigmoid = torch.nn.Sigmoid()272 probs = sigmoid(logits.squeeze().cpu())273 predictions = np.zeros(probs.shape)274 predictions[np.where(probs >= 0.5)] = 1275 # turn predicted id's into actual label names276 return([id2label[idx] for idx, label in enumerate(predictions) if label == 1.0])277 278demo = gr.Blocks()279 280 281 282with demo:283 gr.Markdown(284 """285 # Sentiment text!!!286 """)287 inp = [gr.Textbox(label='Text or tweet text', placeholder="Insert text")]288 out = gr.Textbox(label='Output')289 text_button = gr.Button("Get the text sentiment")290 text_button.click(predict, inputs=inp, outputs=out)291 292 293demo.launch()294 295###############296 297 298 299 300trainer.train()301 302trainer.evaluate()303