CalebKoster/Translation_Note_Alignment

1from guidance import models, select2 3import nltk4from nltk import FreqDist5from nltk.util import ngrams6from nltk.tokenize import word_tokenize7from nltk.corpus import stopwords8import string9 10nltk.download('punkt')11 12from romanize import uroman13 14llm = models.LlamaCpp('models/neural-chat-7b-v3-3.Q2_K.gguf', n_gpu_layers=1)15 16hin_str = 'हमारे परमेश्‍वर और प्रभु यीशु मसीह के पिता का धन्यवाद हो कि उसने हमें मसीह में स्वर्गीय स्थानों में सब प्रकार की आत्मिक आशीष* दी है।'17hin_str = uroman(hin_str)18greek_term = "ἐν Χριστῷ"19greek_term = uroman(greek_term)20 21 22tokens= word_tokenize(hin_str.lower())  # Tokenize and normalize case23tokens = [token for token in tokens if token not in string.punctuation]24all_ngrams = []25for n in range(2, 4):26    all_ngrams.extend(ngrams(tokens, n))27 28all_ngrams = [x[0] + x[1] for x in all_ngrams]29all_ngrams = []30print(all_ngrams)31 32lm = llm33lm += f'The best translation of {greek_term} from Greek into Hindi is '34lm += select(all_ngrams, name='ngram')35 36print(lm['ngram'])