loldota2iii/valueDetection
0
1import torch2import torch.nn as nn3import torch.nn.functional as F4import torch.optim as optim5import numpy as np6import matplotlib.pyplot as plt7import torchtext8import pandas as pd9import spacy10import argparse11import os12import csv13import re14from copy import copy, deepcopy15from transformers import AutoTokenizer, AutoModel, AutoConfig16from transformers import TrainingArguments, Trainer17from transformers import set_seed, AdamW18from torchtext import data19from torch.utils.data import Dataset, DataLoader20from tqdm.notebook import tqdm21from transformers import get_linear_schedule_with_warmup22import locale23import random24import gradio as gr25import time26def getpreferredencoding(do_setlocale = True):27 return "UTF-8"28locale.getpreferredencoding = getpreferredencoding29#print(locale.getpreferredencoding())30 31bert_tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")32 33final_value_list = ['ACHIEVEMENT', 'BENEVOLENCE', 'CONFORMITY', 'HEDONISM', 'POWER', 'SELF-DIRECTION', 'TRADITION', 'SECURITY', 'STIMULATION', 'UNIVERSALISM']34value_index = {35 'SECURITY': 0,36 'BENEVOLENCE': 1,37 'ACHIEVEMENT': 2,38 'SELF-DIRECTION': 3,39 'POWER': 4,40 'UNIVERSALISM': 5,41 'STIMULATION': 6,42 'CONFORMITY': 7,43 'TRADITION': 8,44 'HEDONISM': 945}46 47thresholds = {'SECURITY':0.5, 'BENEVOLENCE':0.66, 'ACHIEVEMENT':0.94, 'SELF-DIRECTION':0.52, 'POWER':0.72, 'UNIVERSALISM':0.64, 'STIMULATION':0.54, 'CONFORMITY':0.55, 'TRADITION':0.81, 'HEDONISM':0.52}48 49softmax = nn.Softmax(dim=1)50########################################## Global Variable ##############################################51# Get GPT2 model tokenizer52gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")53# GPT2 uses the last token for prediction so we need to pad to the left.54gpt2_tokenizer.padding_side = "left"55# Define PAD Token = EOS Token = 5025656gpt2_tokenizer.pad_token = gpt2_tokenizer.eos_token57#########################################################################################################58 59gpt2_model = AutoModel.from_pretrained('gpt2')60# resize model embedding to match new tokenizer61gpt2_model.resize_token_embeddings(len(gpt2_tokenizer))62# fix model padding token id63gpt2_model.config.pad_token_id = gpt2_model.config.eos_token_id64 65for param in gpt2_model.parameters():66 param.requires_grad = False67 68def is_english_statement(input):69 english_letters_or_symbols = re.compile(r'^[A-Za-z0-9 .,!?:;\-_=+@#$%^&*()"\']*$')70 return bool(english_letters_or_symbols.match(input))71 72def obtain_output(input, models):73 if is_english_statement(input) == False:74 # print("Invalid input. The input is not English.")75 return76 if len(input) == 0:77 # print("Invalid input.")78 return79 sigmoid = nn.Sigmoid()80 output_list = {}81 encoded_input = gpt2_tokenizer(input, return_tensors='pt').input_ids82 if len(encoded_input[0]) < 4:83 num = 4 - len(encoded_input[0])84 token_to_add = torch.full((1, num), gpt2_model.config.eos_token_id)85 encoded_input = torch.cat((token_to_add, encoded_input), dim=1)86 87 for i in range(len(final_value_list)):88 value = final_value_list[i]89 model = models[i]90 output = model(encoded_input).squeeze(1)91 prob = sigmoid(output)[0].item()92 # print(f"{value}: {prob: .2f}")93 if(prob > thresholds[value]): 94 output_list[value] = prob95 96 # if len(output_list) == 0:97 # print("No value detected from the statement.")98 99 output_list = sorted(output_list, key=output_list.get, reverse=True)100 return output_list[:3]101 102def load_model_state(model_path):103 state = torch.load(model_path)104 transformer_model = getNewGPT2Model()105 classifier_model = CNN(768, 384, [2, 4])106 model = GPT2_CNN(transformer_model, classifier_model)107 model.load_state_dict(state)108 return model109 110def getNewGPT2Model():111 # Get GPT2 model tokenizer112 gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")113 # GPT2 uses the last token for prediction so we need to pad to the left.114 gpt2_tokenizer.padding_side = "left"115 # Define PAD Token = EOS Token = 50256116 gpt2_tokenizer.pad_token = gpt2_tokenizer.eos_token117 118 gpt2_model = AutoModel.from_pretrained('gpt2')119 # resize model embedding to match new tokenizer120 gpt2_model.resize_token_embeddings(len(gpt2_tokenizer))121 # fix model padding token id122 gpt2_model.config.pad_token_id = gpt2_model.config.eos_token_id123 124 for param in gpt2_model.parameters():125 param.requires_grad = False126 127 return gpt2_model128 129class GPT2_CNN(nn.Module):130 def __init__(self, GPT2_model, CNN_model):131 super(GPT2_CNN, self).__init__()132 133 self.transformer = GPT2_model134 self.classifier = CNN_model135 136 def forward(self, x, lengths=None):137 138 output = self.transformer(x).last_hidden_state # x=[sentence length, batch size] -> [sentence length, batch size, embedding_dim]139 output = output.permute(1, 0, 2)140 output = self.classifier(output)141 return output142 return output143 144class CNN(nn.Module):145 def __init__(self, embedding_dim, n_filters, filter_sizes):146 super(CNN, self).__init__()147 148 self.conv1 = nn.Conv2d(1, n_filters, kernel_size=(embedding_dim, filter_sizes[0]))149 self.conv2 = nn.Conv2d(1, n_filters, kernel_size=(embedding_dim, filter_sizes[1]))150 self.fc = nn.Linear(embedding_dim, 1)151 152 def forward(self, x, lengths=None):153 154 #x = self.embedding(x) # x=[sentence length, batch size] -> [sentence length, batch size, embedding_dim]155 x = x.unsqueeze(0) # x=[1, sentence length, batch size, embedding_dim]156 x = x.permute(2, 0, 3, 1) # x=[batch_size, 1, embedding_dim, sentence length]157 C1 = F.relu(self.conv1(x)) # C1=[batch_size, n_filters, 1, num_words]158 C2 = F.relu(self.conv2(x)) # C2=[batch_size, n_filters, 1, num_words]159 C1 = C1.squeeze(2) 160 C2 = C2.squeeze(2) # C1, C2=[batch_size, n_filters, num_words]161 pool1 = nn.MaxPool1d(C1.size()[2])162 pool2 = nn.MaxPool1d(C2.size()[2])163 C1 = pool1(C1).squeeze(2)164 C2 = pool2(C2).squeeze(2) # C1, C2=[batch_size, n_filters]165 C3 = torch.cat((C1, C2), 1) # C3 = [batch_size, n_filters+n_filters]166 C3 = self.fc(C3) # C3 = [batch_size]167 return C3168 169device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')170#print(device)171best_model_for_achievement = load_model_state("ACHIEVEMENT_lr0.001_epoch22_backtranslation")172best_model_for_benevolence = load_model_state("BENEVOLENCE_lr0.001_epoch21_backtranslation")173best_model_for_conformity = load_model_state("CONFORMITY_lr0.001_epoch19_backtranslation")174best_model_for_hedonism = load_model_state("HEDONISM_lr0.001_epoch23_backtranslation")175best_model_for_power = load_model_state("POWER_lr0.001_epoch21_backtranslation")176best_model_for_self_d = load_model_state("SELF-DIRECTION_lr0.001_epoch16_backtranslation")177best_model_for_tradition = load_model_state("TRADITION_lr0.001_epoch14_backtranslation")178best_model_for_security = load_model_state("SECURITY_lr0.001_epoch14_backtranslation")179best_model_for_stimulation = load_model_state("STIMULATION_lr0.001_epoch12_backtranslation")180best_model_for_universalism = load_model_state("UNIVERSALISM_lr0.001_epoch18_backtranslation")181final_models = [best_model_for_achievement, best_model_for_benevolence, best_model_for_conformity, best_model_for_hedonism, best_model_for_power, best_model_for_self_d, 182best_model_for_tradition, best_model_for_security, best_model_for_stimulation, best_model_for_universalism]183 184 185definitions = {186 'ACHIEVEMENT' : "personal success through demonstrating competence according to social standards",187 'BENEVOLENCE' : "preservation and enhancement of the people with whom one is in frequent personal contact",188 'CONFORMITY' : "restraint of actions, inclinations, and impulses likely to upset or harm others and violate social expectations or norms",189 'HEDONISM' : "pleasure and sensuous gratification for oneself",190 'POWER' : "social status and prestige, control or dominance over people and resources",191 'SECURITY' : "safety, harmony, and stability of society, of relationships, and of self",192 'SELF-DIRECTION' : "independent thought and action choosing, creating, exploring",193 'STIMULATION' : "excitement, novelty, and challenge in life",194 'TRADITION' : "respect, commitment, and acceptance of the customs and ideas that traditional culture or religion provides",195 'UNIVERSALISM' : "understanding, appreciation, tolerance and protection for the welfare of all people and for nature",196}197 198begin = '''199# Value Detector Slim\n200Thank you for trying out Value Detector Slim.\n201This is a simplified interface to detect human value(s) in an English sentence.202You can enter your statement in the input box and click "Submit". \n203Based on your response, our product will output the human value (s) detected in your statement and their definition. \n204If you would like to learn more about human values and our product, you can check them in 'About Project' page.\n205'''206 207def detect_value(input):208 response = ''209 if len(input) == 0:210 raise gr.Error("Please enter a valid statement with at least one word.")211 elif not is_english_statement(input):212 raise gr.Error("Please enter the statement in English.")213 else:214 output = obtain_output(input, final_models)215 if len(output) == 0:216 response = 'None'217 else:218 for value in output:219 response += f"{value.title()}, which represents {definitions[value]}.\n\n"220 return response221 222 223with gr.Blocks() as demo:224 gr.Markdown(begin)225 with gr.Row():226 with gr.Column():227 input = gr.Textbox(label="Your Statement")228 with gr.Row():229 clear_button = gr.Button(value="Clear")230 submit_button = gr.Button(value="Submit")231 with gr.Column():232 output = gr.Textbox(label="Detected Value(s)")233 234 235 submit_button.click(detect_value, inputs=input, outputs=output)236 clear_button.click(lambda x: gr.update(value=''), inputs=input, outputs=input)237 238 239 240demo.launch()