CalebKoster/Translation_Note_Alignment
0
1import json2import csv3import re4from langdetect import detect5import pycountry6from sklearn.feature_extraction.text import TfidfVectorizer7from guidance import models, gen, select, instruction, system, user, assistant # use llama-cpp-python==0.2.268import openai9from romanize import uroman10from ScriptureReference import ScriptureReference as SR11import stanza12import difflib13import requests14from TrainingData import greek_to_lang15 16 17class TranslationNoteFinder:18 verses = SR.verse_ones19 20 greek_bible_path = 'bibles/grc-grctcgnt.txt'21 22 # Bibles in various languages can be downloaded from https://github.com/BibleNLP/ebible/tree/main/corpus23 # lang_code follows ISO 639-1 standard24 def __init__(self, bible_text_path, api_key, model_path=None, lang_code=None):25 26 # Load Bibles27 self.verses = TranslationNoteFinder.verses28 self.greek_bible_text = self.load_bible(self.greek_bible_path)29 self.target_bible_text = self.load_bible(bible_text_path)30 first_line_nt = self.target_bible_text.splitlines()[23213]31 32 # Auto-detect language of target Bible text (occassionally incorrect, so lang_code can be passed in)33 if lang_code:34 self.language = lang_code35 self.lang_name = pycountry.languages.get(alpha_2=self.language).name36 print(f'Language of target Bible text: {self.lang_name}')37 else:38 self.language = detect(first_line_nt)39 self.lang_name = pycountry.languages.get(alpha_2=self.language).name40 print(f'Detected language of target Bible text: {self.lang_name}')41 42 # Local model currently not in use43 if model_path:44 self.model_path = model_path45 46 # Download target language data for use in tokenizer47 stanza.download(self.language)48 self.nlp = stanza.Pipeline(lang=self.language, processors='tokenize')49 50 # Assign instance variables51 self.target_bible_text = self.load_bible(bible_text_path)52 self.api_key = api_key53 54 # Get tf-idf vectorizer, matrix for target Bible text55 self.tfidf_vectorizer, self.tfidf_matrix = self.create_tfidf_vectorizer_matrix()56 57 58 def parse_tsv_to_json(self, file_content, book_abbrev):59 result = [] # Initialize an empty list to store the dictionaries.60 61 # Turn tsv content into reader62 tsv_reader = csv.reader(file_content.splitlines(), delimiter='\t')63 64 for row in tsv_reader:65 # Check if the row contains a Greek term (non-empty) in the expected position.66 if row and len(row) > 3 and row[4].strip():67 # Construct a dictionary for the current row.68 entry = {69 "source_term": row[4].strip(),70 "translation_note": row[6].strip(),71 "verse": book_abbrev + row[0].strip()72 }73 # Append the dictionary to the result list.74 result.append(entry)75 76 return result77 78 79 def load_translation_notes(self, book_abbrev):80 # If filepath ends with json81 translation_notes_path = f'https://git.door43.org/unfoldingWord/en_tn/raw/branch/master/tn_{book_abbrev}.tsv'82 response = requests.get(translation_notes_path)83 if response.status_code == 200:84 translation_notes_raw = response.text85 else:86 translation_notes_raw = ''87 88 translation_notes = self.parse_tsv_to_json(translation_notes_raw, book_abbrev)89 90 return translation_notes91 92 93 def load_bible(self, bible_path):94 # Check if the path starts with "http://" or "https://"95 if bible_path.startswith('http'):96 # Use requests to fetch the Bible text from the URL97 response = requests.get(bible_path)98 # Check if the request was successful99 if response.status_code == 200:100 bible_text = response.text101 else:102 bible_text = '' # Or handle errors as needed103 else:104 # Load the Bible text from a local file105 with open(bible_path, 'r', encoding='utf-8') as file:106 bible_text = file.read()107 return bible_text108 109 110 # Transforms loaded Bible text from file into a list of documents/books (prep for tf-idf)111 # i.e., documents = [Genesis content, Exodus content, ...]112 def segment_corpus(self, bible_text):113 documents = []114 current_document = []115 verse_lines = bible_text.splitlines()116 for i, line in enumerate(verse_lines, start=1):117 if i in self.verses:118 if current_document:119 joined_doc_string = " ".join(current_document)120 documents.append(joined_doc_string)121 current_document = []122 current_document.append(line.strip())123 # Add the last document124 if current_document:125 joined_doc_string = " ".join(current_document)126 documents.append(joined_doc_string)127 return documents128 129 130 # A method created for the tokenizer arg of the TfidfVectorizer class constructor131 # See create_tfidf_vectorizer_matrix method132 def stanza_tokenizer(self, text):133 # Use the Stanza pipeline to process the text134 doc = self.nlp(text)135 # Extract tokens from the Stanza Document object136 tokens = [word.text for sent in doc.sentences for word in sent.words]137 return tokens138 139 140 # Create a tf-idf vectorizer and matrix for the target Bible text141 def create_tfidf_vectorizer_matrix(self):142 tfidf_vectorizer = TfidfVectorizer(tokenizer=self.stanza_tokenizer, ngram_range=(1, 10)) 143 segmented_corpus = self.segment_corpus(self.target_bible_text)144 tfidf_matrix = tfidf_vectorizer.fit_transform(segmented_corpus)145 return tfidf_vectorizer, tfidf_matrix146 147 148 # Use the tf-idf matrix to get the tf-idf scores for the features (n-grams) of a specific book149 def get_tfidf_book_features(self, book_code):150 book_index = list(SR.book_codes.keys()).index(book_code)151 feature_names = self.tfidf_vectorizer.get_feature_names_out()152 dense = self.tfidf_matrix[book_index].todense()153 document_tfidf_scores = dense.tolist()[0]154 feature_scores = dict(zip(feature_names, document_tfidf_scores))155 156 # Filter out zero scores157 filtered_feature_scores = {feature: score for feature, score in feature_scores.items() if score > 0}158 # Sort by score in descending order (just because...)159 sorted_feature_scores = dict(sorted(filtered_feature_scores.items(), key=lambda item: item[1], reverse=True))160 return sorted_feature_scores161 162 163 # For each translation note in verse, use difflib to select the verse ngram which best matches the AI-translated Greek term164 def best_ngram_for_note(self, note, verse_ngrams, language):165 # local_llm = models.LlamaCpp(self.model_path, n_gpu_layers=1) # n_ctx=4096 to increase prompt size from 512 tokens166 167 openai_llm = models.OpenAI("gpt-4", api_key=self.api_key) # To use OPENAI_API_KEY environment variable, omit api_key argument168 openai_lm = openai_llm169 170 print(f'All ngrams in verse guidance is selecting from: {[key for key in verse_ngrams.keys()]}')171 # print(f'All ngrams in verse guidance is selecting from: {[uroman(key) for key in verse_ngrams.keys()]}')172 source_term = note['source_term'].strip()173 # source_term = uroman(note['source_term']).strip()174 175 with system():176 openai_lm += f'You are an expert at translating from Greek into {language}.'177 openai_lm += 'When asked to translate, provide only the translation of the term. Nothing else. Do not provide any additional information or context.'178 openai_lm += 'Be extrememly succinct in your translations.'179 openai_lm += 'You must choose only from the list of translation options you are given. Choose the single best option.'180 # with instruction():181 with user():182 openai_lm += f'What is a good translation of {source_term} from Greek into {language} and is found here: {verse_ngrams.keys()}?'183 with assistant(): 184 openai_lm += gen('openai_translation', stop='.')185 print(f'OpenAI translation: {openai_lm["openai_translation"]}')186 187 try:188 ngram = difflib.get_close_matches(openai_lm["openai_translation"].strip(), verse_ngrams.keys(), n=1, cutoff=0.3)[0]189 except IndexError:190 ngram = "No close match found"191 192 193 print(f'Best ngram found for note: {ngram}')194 return ngram195 196 197 def verse_notes(self, verse_ref):198 # Get the Greek form of the verse199 v_ref = SR(verse_ref)200 gk_verse_text = self.greek_bible_text.splitlines()[v_ref.line_number - 1]201 202 # Get all relevant translation notes for the verse (based on Greek terms found in Greek verse)203 # with open('translation_notes.json', 'r', encoding='utf-8') as file:204 # translation_notes = json.load(file)205 translation_notes_in_verse = []206 print(f'Let\'s see if there are any translation notes for this verse: \n\t {gk_verse_text}')207 translation_notes = self.load_translation_notes(v_ref.structured_ref['bookCode'])208 for note in translation_notes:209 note_v_ref = SR(note['verse'])210 if note_v_ref.line_number != v_ref.line_number:211 continue212 print('Note verse:', note_v_ref.structured_ref)213 print(f'Checking for existence of: {note["source_term"]}')214 if note['source_term'].lower() in gk_verse_text.lower():215 translation_notes_in_verse.append(note)216 print(f'Greek terms for all translation notes in verse: {[note["source_term"] for note in translation_notes_in_verse]}')217 218 # Get the target language form of the verse219 target_verse_text = self.target_bible_text.splitlines()[v_ref.line_number - 1]220 221 # Find n-grams from the book of the verse which exist in the verse222 bookCode = v_ref.structured_ref['bookCode']223 book_ngrams = self.get_tfidf_book_features(bookCode)224 print(f'First 30 n-grams of the book: {list(book_ngrams.keys())[:30]}')225 verse_ngrams = {feature: score for feature, score in book_ngrams.items() if feature.lower() in target_verse_text.lower()}226 print(f'First five n-grams of the verse along with their scores: {list(verse_ngrams.items())[:5]}')227 228 ngrams = []229 for note in translation_notes_in_verse:230 ngram = self.best_ngram_for_note(note, verse_ngrams, self.lang_name)231 start_pos = target_verse_text.lower().find(ngram.lower())232 end_pos = start_pos + len(ngram)233 source_term = note['source_term']234 trans_note = note['translation_note']235 ngrams.append(236 {237 'ngram': ngram,238 'start_pos': start_pos,239 'end_pos': end_pos,240 'source_term': source_term,241 'trans_note': trans_note242 })243 244 print(f'Verse notes to be returned: {ngrams}')245 return {246 'target_verse_text': target_verse_text,247 'verse_ref': v_ref.structured_ref,248 'line_number': v_ref.line_number,249 'ngrams': ngrams250 }251 252 253