CatZM/value-detection
0
1import torch2import torch.nn as nn3import torch.nn.functional as F4import torch.optim as optim5import numpy as np6import matplotlib.pyplot as plt7import torchtext8import pandas as pd9import spacy10import argparse11import os12import csv13import re14from copy import copy, deepcopy15from transformers import AutoTokenizer, AutoModel, AutoConfig16from transformers import TrainingArguments, Trainer17from transformers import set_seed, AdamW18from torchtext import data19from torch.utils.data import Dataset, DataLoader20from tqdm.notebook import tqdm21from transformers import get_linear_schedule_with_warmup22import locale23import random24import gradio as gr25import time26 27 28def getpreferredencoding(do_setlocale=True):29 return "UTF-8"30 31 32locale.getpreferredencoding = getpreferredencoding33# print(locale.getpreferredencoding())34 35bert_tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")36 37final_value_list = ['ACHIEVEMENT', 'BENEVOLENCE', 'CONFORMITY', 'HEDONISM',38 'POWER', 'SELF-DIRECTION', 'TRADITION', 'SECURITY', 'STIMULATION', 'UNIVERSALISM']39value_index = {40 'SECURITY': 0,41 'BENEVOLENCE': 1,42 'ACHIEVEMENT': 2,43 'SELF-DIRECTION': 3,44 'POWER': 4,45 'UNIVERSALISM': 5,46 'STIMULATION': 6,47 'CONFORMITY': 7,48 'TRADITION': 8,49 'HEDONISM': 950}51 52thresholds = {'SECURITY': 0.5, 'BENEVOLENCE': 0.66, 'ACHIEVEMENT': 0.94, 'SELF-DIRECTION': 0.52, 'POWER': 0.72, 'UNIVERSALISM': 0.64, 'STIMULATION': 0.54, 'CONFORMITY': 0.55, 'TRADITION': 0.81, 'HEDONISM': 0.52}53 54softmax = nn.Softmax(dim=1)55########################################## Global Variable ##############################################56# Get GPT2 model tokenizer57gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")58# GPT2 uses the last token for prediction so we need to pad to the left.59gpt2_tokenizer.padding_side = "left"60# Define PAD Token = EOS Token = 5025661gpt2_tokenizer.pad_token = gpt2_tokenizer.eos_token62#########################################################################################################63 64gpt2_model = AutoModel.from_pretrained('gpt2')65# resize model embedding to match new tokenizer66gpt2_model.resize_token_embeddings(len(gpt2_tokenizer))67# fix model padding token id68gpt2_model.config.pad_token_id = gpt2_model.config.eos_token_id69 70for param in gpt2_model.parameters():71 param.requires_grad = False72 73 74def is_english_statement(input):75 english_letters_or_symbols = re.compile(r'^[A-Za-z0-9 .,!?:;\-_=+@#$%^&*()"\']*$')76 return bool(english_letters_or_symbols.match(input))77 78 79def obtain_output(input, models):80 if is_english_statement(input) == False:81 # print("Invalid input. The input is not English.")82 return83 if len(input) == 0:84 # print("Invalid input.")85 return86 sigmoid = nn.Sigmoid()87 output_list = {}88 encoded_input = gpt2_tokenizer(input, return_tensors='pt').input_ids89 if len(encoded_input[0]) < 4:90 num = 4 - len(encoded_input[0])91 token_to_add = torch.full((1, num), gpt2_model.config.eos_token_id)92 encoded_input = torch.cat((token_to_add, encoded_input), dim=1)93 94 for i in range(len(final_value_list)):95 value = final_value_list[i]96 model = models[i]97 output = model(encoded_input).squeeze(1)98 prob = sigmoid(output)[0].item()99 # print(f"{value}: {prob: .2f}")100 if (prob > thresholds[value]):101 output_list[value] = prob102 103 # if len(output_list) == 0:104 # print("No value detected from the statement.")105 106 output_list = sorted(output_list, key=output_list.get, reverse=True)107 return output_list[:3]108 109 110def load_model_state(model_path):111 state = torch.load(model_path)112 transformer_model = getNewGPT2Model()113 classifier_model = CNN(768, 384, [2, 4])114 model = GPT2_CNN(transformer_model, classifier_model)115 model.load_state_dict(state)116 return model117 118 119def getNewGPT2Model():120 # Get GPT2 model tokenizer121 gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")122 # GPT2 uses the last token for prediction so we need to pad to the left.123 gpt2_tokenizer.padding_side = "left"124 # Define PAD Token = EOS Token = 50256125 gpt2_tokenizer.pad_token = gpt2_tokenizer.eos_token126 127 gpt2_model = AutoModel.from_pretrained('gpt2')128 # resize model embedding to match new tokenizer129 gpt2_model.resize_token_embeddings(len(gpt2_tokenizer))130 # fix model padding token id131 gpt2_model.config.pad_token_id = gpt2_model.config.eos_token_id132 133 for param in gpt2_model.parameters():134 param.requires_grad = False135 136 return gpt2_model137 138 139class GPT2_CNN(nn.Module):140 def __init__(self, GPT2_model, CNN_model):141 super(GPT2_CNN, self).__init__()142 143 self.transformer = GPT2_model144 self.classifier = CNN_model145 146 def forward(self, x, lengths=None):147 148 # x=[sentence length, batch size] -> [sentence length, batch size, embedding_dim]149 output = self.transformer(x).last_hidden_state150 output = output.permute(1, 0, 2)151 output = self.classifier(output)152 return output153 154 155class CNN(nn.Module):156 def __init__(self, embedding_dim, n_filters, filter_sizes):157 super(CNN, self).__init__()158 159 self.conv1 = nn.Conv2d(1, n_filters, kernel_size=(160 embedding_dim, filter_sizes[0]))161 self.conv2 = nn.Conv2d(1, n_filters, kernel_size=(162 embedding_dim, filter_sizes[1]))163 self.fc = nn.Linear(embedding_dim, 1)164 165 def forward(self, x, lengths=None):166 167 # x = self.embedding(x) # x=[sentence length, batch size] -> [sentence length, batch size, embedding_dim]168 x = x.unsqueeze(0) # x=[1, sentence length, batch size, embedding_dim]169 # x=[batch_size, 1, embedding_dim, sentence length]170 x = x.permute(2, 0, 3, 1)171 C1 = F.relu(self.conv1(x)) # C1=[batch_size, n_filters, 1, num_words]172 C2 = F.relu(self.conv2(x)) # C2=[batch_size, n_filters, 1, num_words]173 C1 = C1.squeeze(2)174 C2 = C2.squeeze(2) # C1, C2=[batch_size, n_filters, num_words]175 pool1 = nn.MaxPool1d(C1.size()[2])176 pool2 = nn.MaxPool1d(C2.size()[2])177 C1 = pool1(C1).squeeze(2)178 C2 = pool2(C2).squeeze(2) # C1, C2=[batch_size, n_filters]179 C3 = torch.cat((C1, C2), 1) # C3 = [batch_size, n_filters+n_filters]180 C3 = self.fc(C3) # C3 = [batch_size]181 return C3182 183 184device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')185# print(device)186best_model_for_achievement = load_model_state(187 "./train_result/ACHIEVEMENT_backtranslation")188best_model_for_benevolence = load_model_state(189 "./train_result/BENEVOLENCE_backtranslation")190best_model_for_conformity = load_model_state(191 "./train_result/CONFORMITY_backtranslation")192best_model_for_hedonism = load_model_state(193 "./train_result/HEDONISM_backtranslation")194best_model_for_power = load_model_state(195 "./train_result/POWER_backtranslation")196best_model_for_self_d = load_model_state(197 "./train_result/SELF-DIRECTION_backtranslation")198best_model_for_tradition = load_model_state(199 "./train_result/TRADITION_backtranslation")200best_model_for_security = load_model_state(201 "./train_result/SECURITY_backtranslation")202best_model_for_stimulation = load_model_state(203 "./train_result/STIMULATION_backtranslation")204best_model_for_universalism = load_model_state(205 "./train_result/UNIVERSALISM_backtranslation")206final_models = [best_model_for_achievement, best_model_for_benevolence, best_model_for_conformity, best_model_for_hedonism, best_model_for_power, best_model_for_self_d, best_model_for_tradition, best_model_for_security, best_model_for_stimulation, best_model_for_universalism]207 208 209definitions = {210 'ACHIEVEMENT': "personal success through demonstrating competence according to social standards",211 'BENEVOLENCE': "preservation and enhancement of the people with whom one is in frequent personal contact",212 'CONFORMITY': "restraint of actions, inclinations, and impulses likely to upset or harm others and violate social expectations or norms",213 'HEDONISM': "pleasure and sensuous gratification for oneself",214 'POWER': "social status and prestige, control or dominance over people and resources",215 'SECURITY': "safety, harmony, and stability of society, of relationships, and of self",216 'SELF-DIRECTION': "independent thought and action choosing, creating, exploring",217 'STIMULATION': "excitement, novelty, and challenge in life",218 'TRADITION': "respect, commitment, and acceptance of the customs and ideas that traditional culture or religion provides",219 'UNIVERSALISM': "understanding, appreciation, tolerance and protection for the welfare of all people and for nature",220}221 222begin = '''223# Value Detector Slim\n224Thank you for trying out Value Detector Slim.\n225This is a simplified interface to detect human value(s) in an English sentence.226You can enter your statement in the input box and click "Submit". \n227Based on your response, our product will output the human value (s) detected in your statement and their definition. \n228If you would like to learn more about human values and our product, you can check them in 'About Project' page.\n229'''230 231 232def detect_value(input):233 response = ''234 if len(input) == 0:235 raise gr.Error(236 "Please enter a valid statement with at least one word.")237 elif not is_english_statement(input):238 raise gr.Error("Please enter the statement in English.")239 else:240 output = obtain_output(input, final_models)241 if len(output) == 0:242 response = 'None'243 else:244 for value in output:245 response += f"{value.title()}, which represents {definitions[value]}.\n\n"246 return response247 248 249with gr.Blocks() as demo:250 gr.Markdown(begin)251 with gr.Row():252 with gr.Column():253 input = gr.Textbox(label="Your Statement")254 with gr.Row():255 clear_button = gr.Button(value="Clear")256 submit_button = gr.Button(value="Submit")257 with gr.Column():258 output = gr.Textbox(label="Detected Value(s)")259 260 submit_button.click(detect_value, inputs=input, outputs=output)261 clear_button.click(lambda x: gr.update(value=''),262 inputs=input, outputs=input)263 264 265demo.launch()266 267# input = "Start"268# obtain_output(input, final_models)269 270# begin = "Hi, thank you for trying Value Detector (beta). \n" + "This is a chatbot-alike interface to detect human value(s) in an English sentence." + "To simulate a conversation, the chatbot will ask you a question. " + \271# "Based on your response, our product will output the human value (s) detected in your statement. \n" + \272# "If you would like to learn more about human values and our product, you can check them in 'About Project' page. \n" + \273# "Please click 'New Question' to prompt a new question. "274 275 276# achievement = "Personal success through demonstrating competence according to social standards."277# benevolence = "Preservation and enhancement of the people with whom one is in frequent personal contact."278# conformity = "Restraint of actions, inclinations, and impulses likely to upset or harm others and violate social expectations or norms."279# hedonism = "Pleasure and sensuous gratification for oneself."280# power = "Social status and prestige, control or dominance over people and resources."281# security = "Safety, harmony, and stability of society, of relationships, and of self."282# self_direction = "Independent thought and action choosing, creating, exploring."283# stimulation = "Excitement, novelty, and challenge in life."284# tradition = "Respect, commitment, and acceptance of the customs and ideas that traditional culture or religion provides."285# universalism = "Understanding, appreciation, tolerance and protection for the welfare of all people and for nature."286 287# # output = "The statement holds the following values:" + list of value + definition288 289 290# questions = [291# "What are your aspirations in life?",292# "Tell about a short-term goal you have.",293# "What do you like most about yourself?",294# "What kind of a person do you want to be?",295# "What are your strengths?",296# "What are you grateful for?",297# "What is your biggest fear?",298# "How do you think others see you?",299# "Tell about a time you were happy.",300# "Who do you admire and why?"301# ]302 303 304# def respond(chat_history, message, sstate):305# output_message = obtain_output(message, final_models)306# output_message = output_message.title()307# if message.lower() == 'quit' or message.lower() == 'exit':308# sstate = 5309# else:310# if sstate == -1:311# if message.lower() == 'start':312# sstate = 0313# elif sstate == 0:314# if output_message == 'Null':315# sstate = 1316# elif output_message == 'None':317# sstate = 2318# else:319# sstate = 3320# elif sstate == 1:321# if output_message == 'Null':322# sstate = 1323# elif output_message == 'None':324# sstate = 2325# else:326# sstate = 3327# elif sstate == 2:328# if message.lower() == 'yes':329# sstate = 0330# elif message.lower() == 'no':331# sstate = 5332# else:333# sstate = 4334# elif sstate == 3:335# if message.lower() == 'yes':336# sstate = 0337# elif message.lower() == 'no':338# sstate = 5339# else:340# sstate = 4341# elif sstate == 4:342# if message.lower() == 'yes':343# sstate = 0344# elif message.lower() == 'no':345# sstate = 5346# else:347# sstate = 4348# elif sstate == 5:349# if message.lower() == 'reset':350# sstate = -1351 352# if sstate == -1:353# response = 'Please Enter start'354# elif sstate == 0:355# response = random.choice(questions)356# elif sstate == 1:357# response = 'Please enter in English'358# elif sstate == 2:359# response = 'No value detected from the statement. Would you like to a new question?'360# elif sstate == 3:361# response = 'Values: ' + output_message + '. Would you like to a new question?'362# elif sstate == 4:363# response = 'Please enter yes/no'364# elif sstate == 5:365# response = 'END'366 367# return chat_history + [[message, response]], sstate368 369 370# with gr.Blocks() as demo:371# sstate = gr.State(value=-1)372# gr.Markdown(373# """374# # Value Detector (beta)375# This is a chatbot-alike interface to detect human value(s) in an English human-spoken sentence.376 377# To view the full list of human values please refer to "About Project".378# """)379# chatbot = gr.Chatbot()380# msg = gr.Textbox()381# clear = gr.Button("Clear Conversation")382# msg.submit(respond, [chatbot, msg, sstate], [chatbot, sstate])383# clear.click(lambda: None, None, chatbot, queue=False)384 385# demo.launch(share=True)386