CalebKoster/Translation_Note_Alignment

1from sklearn.feature_extraction.text import TfidfVectorizer2import pandas as pd3import numpy as np4from itertools import islice5from romanize import uroman6 7 8verses = [9    1,10    1534,11    2747,12    3606,13    4895,14    5854,15    6512,16    7130,17    7215,18    8026,19    8721,20    9538,21    10257,22    11200,23    12022,24    12302,25    12707,26    12874,27    13944,28    16471,29    17608,30    17725,31    19016,32    20380,33    20534,34    21807,35    22164,36    22361,37    22434,38    22580,39    22601,40    22649,41    22754,42    22857,43    22910,44    22948,45    23159,46    23214,47    24285,48    24963,49    26114,50    26993,51    27999,52    28432,53    28869,54    29125,55    29274,56    29429,57    29533,58    29628,59    29717,60    29764,61    29877,62    29960,63    30006,64    30031,65    30334,66    30442,67    30547,68    30608,69    30713,70    30726,71    30741,72    30766,73    3117174]75 76# Adjust verses to be zero-indexed for Python77verses = [x-1 for x in verses]78 79# Function to extract the verse of interest from the corpus80def extract_interested_verse(file_path, line_number, romanize=False):81    with open(file_path, 'r', encoding='utf-8') as file:82        for i, line in enumerate(file):83            if i == line_number:84                if romanize:85                    return uroman(line.strip())86                else:87                    return line.strip()88    return None89 90 91# Function to segment the corpus into documents based on the verses list92def segment_corpus(file_path, romanize=False):93    documents = []94    current_document = []95    with open(file_path, 'r', encoding='utf-8') as file:96        for i, line in enumerate(file, start=1):97            if i in verses:98                if current_document:99                    joined_doc_string = " ".join(current_document)100                    if romanize:101                        joined_doc_string = uroman(joined_doc_string)102                    documents.append(joined_doc_string)103                    current_document = []104            current_document.append(line.strip())105        # Don't forget to add the last document106        if current_document:107            joined_doc_string = " ".join(current_document)108            if romanize:109                joined_doc_string = uroman(joined_doc_string)110            documents.append(joined_doc_string)111    return documents112 113# Function to perform TF-IDF on the corpus and extract scores for a specific verse114def analyze_verse_in_corpus(file_path, interested_line, romanize=False):115    documents = segment_corpus(file_path, romanize=romanize)116    tfidf_vectorizer = TfidfVectorizer(ngram_range=(2, 4))117    tfidf_matrix = tfidf_vectorizer.fit_transform(documents)118    feature_names = tfidf_vectorizer.get_feature_names_out()119 120    # Identify the document index for the interested line121    document_index = next(i for i, v in enumerate(verses) if v > interested_line) - 1122 123    # Extract TF-IDF scores for the document containing the interested line124    scores = np.array(tfidf_matrix[document_index].todense()).flatten()125    scores_dict = dict(zip(feature_names, scores))126 127    # Extract the interested verse text128    interested_verse = extract_interested_verse(file_path, interested_line - 1, romanize=romanize)  129    130    # Map n-grams in verse to their TF-IDF scores131    if interested_verse:132        tfidf_vectorizer_verse = TfidfVectorizer(ngram_range=(2, 4))133        tfidf_vectorizer_verse.fit([interested_verse])134        verse_ngrams = tfidf_vectorizer_verse.get_feature_names_out()135        verse_scores = {ngram: scores_dict.get(ngram, 0) for ngram in verse_ngrams}136        # Get ngrams and respective scores in the verse in descending score order137        sorted_verse_scores = dict(sorted(verse_scores.items(), key=lambda item: item[1], reverse=True))138        return sorted_verse_scores139    else:140        return "Verse not found."141 142 143# file_path = 'bibles/eng-engkjvcpb.txt'144# interested_line = 29276  # Example line number145# verse_scores = analyze_verse_in_corpus(file_path, kjv_verses, interested_line)146 147# Print or return the results148# print(verse_scores)149 150# Print ngrams and respective scores in the verse in descending score order151# for ngram, score in islice(sorted_verse_scores.items(), 30):152#     print(f"{ngram}: {score:.4f}")