CalebKoster/Translation_Note_Alignment

1import json2import csv3import re4from langdetect import detect5import pycountry6from LanguageTool import Lang7from sklearn.feature_extraction.text import TfidfVectorizer8from guidance import models, gen, select, instruction, system, user, assistant # use llama-cpp-python==0.2.269import openai10from romanize import uroman11from ScriptureReference import ScriptureReference as SR12import stanza13import difflib14import requests15# from TrainingData import greek_to_lang16 17 18class TranslationNoteFinder:19    verses = SR.verse_ones20 21    # greek_bible_path = 'bibles/grc-grctcgnt.txt'22    # hebrew_bible_path = 'bibles/heb-hebrewtanakh.txt'23    # english_bible_path = 'bibles/eng-web.txt'24    25    # Bibles in various languages can be downloaded from https://github.com/BibleNLP/ebible/tree/main/corpus26    # lang_code follows ISO 639-1 standard27    def __init__(self, bible_text_path, api_key, lang_code=None):28        29        # Load Bibles30        self.verses = TranslationNoteFinder.verses31        # self.greek_bible_text = self.load_bible('bibles/grc-grctcgnt.txt')32        # self.hebrew_bible_text = self.load_bible('bibles/heb-heb.txt')33        # self.english_bible_text = self.load_bible('bibles/eng-engwebp.txt')34        self.target_bible_text = self.load_bible(bible_text_path)35 36        # Auto-detect language of target Bible text (occassionally incorrect, so lang_code can be passed in)37        if lang_code:38            self.language = lang_code39            self.lang_name = pycountry.languages.get(alpha_2=self.language).name40            print(f'Language of target Bible text: {self.lang_name}')41        else:42            first_line_nt = self.target_bible_text.splitlines()[23213]43            self.language = detect(first_line_nt)44            self.lang_name = pycountry.languages.get(alpha_2=self.language).name45            print(f'Detected language of target Bible text: {self.lang_name}')46 47        # Assign instance variables48        self.target_bible_text = self.load_bible(bible_text_path)49        self.api_key = api_key50 51 52    def parse_tsv_to_json(self, file_content, book_abbrev):53        result = []  # Initialize an empty list to store the dictionaries.54 55        # Turn tsv content into reader56        tsv_reader = csv.reader(file_content.splitlines(), delimiter='\t')57        58        for row in tsv_reader:59            # Check if the row contains a source term (non-empty) in the expected position.60            if row and len(row) > 3 and row[4].strip():61                # Construct a dictionary for the current row.62                entry = {63                    "source_term": row[4].strip(),64                    "translation_note": row[6].strip(),65                    "verse": book_abbrev + row[0].strip()66                }67                # Append the dictionary to the result list.68                result.append(entry)69        70        return result71 72            73    def load_translation_notes(self, book_abbrev):74        # If filepath ends with json75        translation_notes_path = f'https://git.door43.org/unfoldingWord/en_tn/raw/branch/master/tn_{book_abbrev}.tsv'76        response = requests.get(translation_notes_path)77        if response.status_code == 200:78            translation_notes_raw = response.text79        else:80            translation_notes_raw = ''81 82        translation_notes = self.parse_tsv_to_json(translation_notes_raw, book_abbrev)83        84        return translation_notes85    86 87    def load_bible(self, bible_path):88        # Check if the path starts with "http://" or "https://"89        if bible_path.startswith('http'):90            # Use requests to fetch the Bible text from the URL91            response = requests.get(bible_path)92            # Check if the request was successful93            if response.status_code == 200:94                bible_text = response.text95            else:96                bible_text = ''  # Or handle errors as needed97        else:98            # Load the Bible text from a local file99            with open(bible_path, 'r', encoding='utf-8') as file:100                bible_text = file.read()101        return bible_text102 103 104    # Transforms loaded Bible text from file into a list of documents/books (prep for tf-idf)105    # i.e., documents = [Genesis content, Exodus content, ...]106    def segment_corpus(self, bible_text):107        documents = []108        current_document = []109        verse_lines = bible_text.splitlines()110        for i, line in enumerate(verse_lines, start=1):111            if i in self.verses:112                if current_document:113                    joined_doc_string = " ".join(current_document)114                    documents.append(joined_doc_string)115                    current_document = []116            current_document.append(line.strip())117        # Add the last document118        if current_document:119            joined_doc_string = " ".join(current_document)120            documents.append(joined_doc_string)121        return documents122 123 124    # For each translation note in verse, use difflib to select the verse ngram which best matches the AI-translated source term125    def best_ngram_for_note(self, note, target_verse_text, language):126        # local_llm = models.LlamaCpp(self.model_path, n_gpu_layers=1) # n_ctx=4096 to increase prompt size from 512 tokens127 128        openai_llm = models.OpenAI("gpt-4", api_key=self.api_key) # To use OPENAI_API_KEY environment variable, omit api_key argument129        openai_lm = openai_llm130        131        source_term = note['source_term'].strip()132        source_lang = Lang(source_term, options=['en', 'he', 'el']).lang_name   # Can only choose between English, Hebrew, and Greek133        print(f'Source term: {source_term}, \nSource language: {source_lang}')134        # source_term = uroman(note['source_term']).strip()135        136        with system():137            openai_lm += f'You are an expert at translating between {source_lang} and {language}.'138            openai_lm += f'When asked to translate, provide only the {language} translation of the {source_lang} term found in the {language} verse.'139            openai_lm += 'Nothing else. Do not provide any additional information or context. Be extrememly succinct in your translations.'140            openai_lm += f'You must choose only an N-gram which already exists in the {language} verse.'141        142        with user():143            openai_lm += f'What is a good translation of {source_term} from {source_lang} into {language} and is also found within this verse: {target_verse_text}?'144            # openai_lm += f'What part of the verse \"{target_verse_text}\" is a good translation of {source_term} from {source_lang} into {language}?'145        146        with assistant():    147            openai_lm += gen('openai_translation', stop='.')148        print(f'OpenAI translation: {openai_lm["openai_translation"]}')149        150        # If openai_lm["openai_translation"] can be found in the verse, return it151        llm_output = openai_lm["openai_translation"].strip()152        print(f'LLM output: {llm_output}')153        if llm_output in target_verse_text:154            print(f'LLM output found in verse: {llm_output}')155            return llm_output156        else:157            print(f'LLM output not found in verse: {llm_output}')158            return ''159 160 161    def verse_notes(self, verse_ref):162        # Get the source form of the verse163        v_ref = SR(verse_ref)164        # source_verse_text = self.source_bible_text.splitlines()[v_ref.line_number - 1]165        166        translation_notes_in_verse = []167        # print(f'Let\'s see if there are any translation notes for this verse: \n\t {source_verse_text}')168        translation_notes = self.load_translation_notes(v_ref.structured_ref['bookCode'])169        # for note in translation_notes:170        #     note_v_ref = SR(note['verse'])171        #     if note_v_ref.line_number != v_ref.line_number:172        #         continue173        #     print('Note verse:', note_v_ref.structured_ref)174        #     print(f'Checking for existence of: {note["source_term"]}')175        #     if note['source_term'].lower() in source_verse_text.lower():176        #         translation_notes_in_verse.append(note)177        for note in translation_notes:178            note_v_ref = SR(note['verse'])179            if note_v_ref.line_number == v_ref.line_number: # Not checking for existence assumes there is a verse reference180                translation_notes_in_verse.append(note)181        print(f'Source terms for all translation notes in verse: {[note["source_term"] for note in translation_notes_in_verse]}')182        183        # Get the target language form of the verse184        target_verse_text = self.target_bible_text.splitlines()[v_ref.line_number - 1]185 186        ngrams = []187        for note in translation_notes_in_verse:188            source_term = note['source_term']189            trans_note = note['translation_note']190            ngram = self.best_ngram_for_note(note, target_verse_text, self.lang_name)191            start_pos = target_verse_text.lower().find(ngram.lower())192            end_pos = start_pos + len(ngram)193            ngrams.append(194            {195                'ngram': ngram,196                'start_pos': start_pos,197                'end_pos': end_pos,198                'source_term': source_term,199                'trans_note': trans_note200            })201 202        203        print('Verse notes to be returned:')204        print(json.dumps(ngrams, indent=4))205        return {206            'target_verse_text': target_verse_text,207            'verse_ref': v_ref.structured_ref,208            'line_number': v_ref.line_number,209            'ngrams': ngrams210        }211            212 213