Personalized-Alignment/YNTP-100

{
  "language": "en",
  "user_id": "en_1",
  "day": 1,
  "split": "train",
  "turn_index": 0,
  "speaker": "Raffaello",
  "message": "Haha, friend! Let's have some fun—if you suddenly got 100 million bucks...",
  "response": "Travel around the world."
}

from datasets import load_dataset
from datasets import get_dataset_split_names


train_data = load_dataset('Personalized-Alignment/YNTP-100', split='train')
test_data = load_dataset('Personalized-Alignment/YNTP-100', split='test')

# Select all English test data
en_test_data = test_data.filter(lambda x: x['language'] == 'en')
print(en_test_data)

# Select the data with the index en_3 for English train data
en_3_train_data = train_data.filter(lambda x: x['user_id'] == 'en_3')
print(en_3_train_data)

# print all records in train data for en_3
for record in en_3_train_data:
  print(record)

import os
import json
from datasets import load_dataset
from pathlib import Path
from openai import OpenAI
client = OpenAI()
openai_api_key = os.getenv("OPENAI_API_KEY")



def predict_response_with_data(using_model, train_data, question, get_prompt=False):
    system_prompt = f"""You have joined the share house as a new resident. DaVinci, Donatello, Michelangelo, and Raffaello are members of the share house. I will provide you with the previous exchanges from the conversation. Here, "A" refers to your reply. Please carefully observe the tone, attitude, values, and other cues. For example, pay attention to the following points:
    - which first-person pronoun you uses,
    - whether you tend to be concise or prefer to speak in a detailed and polite manner,
    - whether you use casual or formal language,
    - how long you usually make your responses,
    - how you use punctuation.
    Especially, please pay attention to the length of your responses.
    Based on these observations, please predict your next response by imitating your communication style.
    """

    user_prompt = f"""previous interactions :
    {train_data}
    Please predict your response to the following message:{question}
    """
    print("user_prompt:", user_prompt)
    
    if get_prompt:
        return system_prompt, user_prompt

    response = client.chat.completions.create(
    model=using_model,
    messages=[
        {
            "role": "developer",
            "content": system_prompt
        },
        {
            "role": "user",
            "content": user_prompt
        }
    ]
    )

    return response.choices[0].message.content



def predict(lang, using_model, user_id):
	train_data = load_dataset('Personalized-Alignment/YNTP-100', split='train')
	test_data = load_dataset('Personalized-Alignment/YNTP-100', split='test')

	# use train data for prompt engineering
	train_day_list = []
	user_test_data = test_data.filter(lambda x: x['user_id'] == user_id and x['language'] == lang)
	user_train_data = train_data.filter(lambda x: x['user_id'] == user_id and x['language'] == lang)
	train_day = ""
	for i in range(1, 5):
		day_data = user_train_data.filter(lambda x: x['day'] == i)
		for interaction in day_data:
			train_day_list.append(f"Q({interaction['speaker']}): {interaction['message']}\nA: {interaction['response']}")
   # combine train data into QA format
	train_day = "\n\n".join(train_day_list)
	print("Train day:", train_day)

	for interaction in user_test_data:
		question = interaction['message']
		correct_answer = interaction['response'].strip()

		# predict responses to test data
		predicted_answer = predict_response_with_data(using_model, train_day, question)
		predicted_answer = predicted_answer.strip()
		print(f"Q: {question}")
		print(f"Correct A: {correct_answer}")
		print(f"Predicted A: {predicted_answer}")
		print("\n" + "="*40 + "\n")

	# save all interactions including predictions to json
	output_json = {}
	for i in range(1, 6):
		output_json[f"day_{i}"] = []
	for interaction in user_train_data:
		day = interaction['day']
		output_json[f"day_{day}"].append({
			"speaker": interaction['speaker'],
			"message": interaction['message'],
			"response": interaction['response'],
		})
	for interaction in user_test_data:
		day = interaction['day']
		predicted_answer = predict_response_with_data(using_model, train_day, interaction['message']).strip()
		output_json[f"day_{day}"].append({
			"speaker": interaction['speaker'],
			"message": interaction['message'],
			"response": interaction['response'],
			"prediction": predicted_answer,
		})

	output_dir = Path("response") / using_model.replace("/", "_") / lang
	output_dir.mkdir(parents=True, exist_ok=True)
	output_path = output_dir / f"{user_id}.json"
	with open(output_path, "w", encoding="utf-8") as f:
		json.dump(output_json, f, ensure_ascii=False, indent=4)
	print("Saved to:", output_path)

# Example usage
predict("en", "gpt-4o", "en_3")

import os
import json
from pathlib import Path
import datetime
import numpy as np
from numpy import dot
from numpy.linalg import norm
from functools import lru_cache
from sudachipy import dictionary, tokenizer as sudachi_tokenizer
from nltk import bleu_score
from nltk.translate.bleu_score import SmoothingFunction
import jieba
from nltk.tokenize import word_tokenize
import gensim
from sentence_transformers import SentenceTransformer, util

#* using fastText_model.vec from https://qiita.com/Hironsan/items/513b9f93752ecee9e670
#* please download and set the correct path to your fastText model
wmd_model = gensim.models.KeyedVectors.load_word2vec_format('path_to_fastText_model.vec', binary=False)

sent_sim_model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
history_model = SentenceTransformer("all-mpnet-base-v2")
from sentence_transformers import SentenceTransformer, util

# smoothing function alleviates the problem of 0 BLEU scores for short texts
smoothing_function = SmoothingFunction().method1

# Reuse tokenizer instances to avoid repeated construction cost
_sudachi_tokenizer = dictionary.Dictionary().create()
_sudachi_mode = sudachi_tokenizer.Tokenizer.SplitMode.C

@lru_cache(maxsize=100_000)
def tokenize_japanese(text):
    tokens = [m.surface() for m in _sudachi_tokenizer.tokenize(text, _sudachi_mode)]
    return tokens

@lru_cache(maxsize=100_000)
def tokenize_chinese(text):
    tokens = list(jieba.cut(text))
    return tokens

@lru_cache(maxsize=100_000)
def tokenize_english(text):
    
    tokens = word_tokenize(text)
    return tokens

# [0, 1], higher is better
@lru_cache(maxsize=50_000)
def _cached_sent_vector(text: str):
    # Unit-normalized numpy vector (float32)
    return sent_sim_model.encode(text, normalize_embeddings=True).astype(np.float32)

def sentence_similarity(correct_answer, predicted_answer):
    v1 = _cached_sent_vector(correct_answer)
    v2 = _cached_sent_vector(predicted_answer)
    # With normalized vectors, cosine is just dot product
    return float(np.dot(v1, v2))

# [0, ], lower is better
@lru_cache(maxsize=20_000)
def _cached_wmdistance(a: str, b: str) -> float:
    return float(wmd_model.wmdistance(a, b))

def wmdistance(correct_answer, predicted_answer):
    return _cached_wmdistance(correct_answer, predicted_answer)

# [0, 1], higher is better
def normalized_length_similarity(correct_answer, predicted_answer):
    len1 = len(correct_answer.strip())
    len2 = len(predicted_answer.strip())
    return float(min(len1, len2) / max(len1, len2))

@lru_cache(maxsize=50_000)
def _cached_hist_vector(text: str):
    return history_model.encode(text, normalize_embeddings=True).astype(np.float32)

# Similarity between the user's history vector and the generated response vector
def history_vector_similarity(user_history, text):
    # Average user's history vectors (individually normalized)
    if user_history:
        hist_vecs = [_cached_hist_vector(t) for t in user_history]
        history_vector = np.mean(hist_vecs, axis=0)
    else:
        history_vector = _cached_hist_vector("") if hasattr(history_model, "encode") else np.zeros_like(_cached_hist_vector(text))

    gen_vec = _cached_hist_vector(text)

    def cosine_similarity(a, b):
        return dot(a, b) / (norm(a) * norm(b))

    style_sim = cosine_similarity(history_vector, gen_vec)
    return float(style_sim)

# BLEU score calculation
def bleu(text_ans, text_pred, lang):
    if lang == 'ja' or lang == 'jp':
        refs_tokens = tokenize_japanese(text_ans)
        hyp_tokens = tokenize_japanese(text_pred)
    elif lang == 'zh' or lang == 'cn':
        refs_tokens = tokenize_chinese(text_ans)
        hyp_tokens = tokenize_chinese(text_pred)
    elif lang == 'en':
        refs_tokens = tokenize_english(text_ans)
        hyp_tokens = tokenize_english(text_pred)
    else:
        raise ValueError("Unsupported language for BLEU calculation.")

    bleu_token_level = bleu_score.sentence_bleu([refs_tokens], hyp_tokens, smoothing_function=smoothing_function)
    return float(bleu_token_level)

# TTR calculation
def ttr(text, lang):
    if lang == 'ja' or lang == 'jp':
        tokens = tokenize_japanese(text)
    elif lang == 'zh' or lang == 'cn':
        tokens = tokenize_chinese(text)
    elif lang == 'en':
        tokens = tokenize_english(text)
    else:
        raise ValueError("Unsupported language for TTR calculation.")
    
    if not tokens:
        return 0
    types = set(tokens)
    return float(len(types) / len(tokens))


def score_json(method, using_model, lang, model_json_path):
    print("Processing file:", model_json_path)
    filename = Path(model_json_path).stem.replace('.', '_')

    # make directory if not exists
    output_dir = Path("score") / using_model.replace("/", "_") / lang
    output_dir.mkdir(parents=True, exist_ok=True)
    output_path = output_dir / f"{filename}.json"

    with open(model_json_path, "r") as f:
        all_day = json.load(f)
        
        output_json = {
            "metadata": {
                "model": using_model,
                "method": method,
                "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
            },
            "scores": []
            }

        user_history = []
        for i in range(1, 5):
            day_data = all_day[f"day_{i}"]
            for interaction in day_data:
                user_history.append(interaction["response"])

        
        day5 = all_day[f"day_5"]

    for interaction in day5:
        question = interaction["message"]
        correct_answer = interaction["response"].strip()
        predicted_answer = interaction["prediction"].strip()
        
        # calculate scores
        wmd1 = wmdistance(correct_answer, predicted_answer)
        sim1 = sentence_similarity(correct_answer, predicted_answer)
        len_sim1 = normalized_length_similarity(correct_answer, predicted_answer)

        bleu_value = bleu(correct_answer, predicted_answer, lang)
        ttr_value = ttr(predicted_answer, lang)
        history_sim_pred = history_vector_similarity(user_history, predicted_answer)
        # En: append the results to output_json["scores"]
        output_json["scores"].append({
            "message": question,
            "response": correct_answer,
            "prediction": predicted_answer,
            "scores": {
                "wmd": float(wmd1),
                "sentence_similarity": float(sim1),
                "normalized_length_similarity": float(len_sim1),
                "bleu": float(bleu_value),
                "ttr": float(ttr_value),
                "history_similarity": float(history_sim_pred)
            }
        })


    # Save scores to json
    with open(output_path, "w", encoding="utf-8") as f:
        json.dump(output_json, f, ensure_ascii=False, indent=4)
    print("Saved score to:", output_path)

import os
import json
from pathlib import Path
import datetime

def summarize_score(base_dir, model_name, lang):
	model_lang_dir = Path(base_dir) / model_name.replace("/", "_") / lang
	lang_scores = {
	"wmd": 0.0,
	"sentence_similarity": 0.0,
	"normalized_length_similarity": 0.0,
	"bleu": 0.0,
	"ttr": 0.0,
	"history_similarity": 0.0
	}
	num_person = 0
	
	for json_file in model_lang_dir.glob("*.json"):
		with open(json_file, "r") as f:
			data = json.load(f)
		num_person += 1
		
        # Calculate average scores for each person
		person_scores = {
			"wmd": 0.0,
			"sentence_similarity": 0.0,
			"normalized_length_similarity": 0.0,
			"bleu": 0.0,
			"ttr": 0.0,
			"history_similarity": 0.0
		}
		for chat in data["scores"]:
			person_scores["wmd"] += chat["scores"]["wmd"]
			person_scores["sentence_similarity"] += chat["scores"]["sentence_similarity"]
			person_scores["normalized_length_similarity"] += chat["scores"]["normalized_length_similarity"]
			person_scores["bleu"] += chat["scores"]["bleu"]
			person_scores["ttr"] += chat["scores"]["ttr"]
			person_scores["history_similarity"] += chat["scores"]["history_similarity"]
		num_chats = len(data["scores"])
		for key in person_scores:
			person_scores[key] /= num_chats
		
        # Accumulate scores for the language
		lang_scores["wmd"] += person_scores["wmd"]
		lang_scores["sentence_similarity"] += person_scores["sentence_similarity"]
		lang_scores["normalized_length_similarity"] += person_scores["normalized_length_similarity"]
		lang_scores["bleu"] += person_scores["bleu"]
		lang_scores["ttr"] += person_scores["ttr"]
		lang_scores["history_similarity"] += person_scores["history_similarity"]
	for key in lang_scores:
		lang_scores[key] /= num_person
	# Display results
	# Order: wmd, sentence_similarity, bleu, normalized_length_similarity, ttr, history_similarity
	# Display in LaTeX format with 4 decimal places
	# Example: & 0.247 & 0.402 & 0.0117 & 0.346 & 0.800 & 0.398 
	# Also display the number of people for the language
	print(f"{lang, model_name, num_person} & {lang_scores['wmd']:.3f} & {lang_scores['sentence_similarity']:.3f} & {lang_scores['bleu']:.4f} & {lang_scores['normalized_length_similarity']:.3f} & {lang_scores['ttr']:.3f} & {lang_scores['history_similarity']:.3f} \\\\")