CoolFace
Apppublic

CalebKoster/Translation_Note_Alignment

sourceHugging Faceupdated 3y agoView on Hugging Face
0likes
TranslationNoteFinder.cpython-312.pyc64 linesDownload Raw Back to __pycache__
12}B�e�-���ddlZddlZddlZddlmZddlZddlmZddlm	Z	m3Z4mZmZm
Z
mZmZddlZddlmZddlmZddlZddlZddlZddlmZGd�d	�Zy)5�N)�detect)�TfidfVectorizer)�models�gen�select�instruction�system�user�	assistant)�uroman)�ScriptureReference)�
greek_to_langc�f�eZdZejZdZd
d�Zd�Zd�Z	d�Z6d�Zd�Zd	�Z
d7�Zd�Zd�Zy)�TranslationNoteFinderzbibles/grc-grctcgnt.txtNc�\�tj|_|j|j�|_|j|�|_|j8j
�d}|rY||_tjj|j��j|_td|j���nat|�|_tjj|j��j|_td|j���|r||_t!j"|j�t!j$|jd��|_|j|�|_||_|j+�\|_|_y)Ni�Z)�alpha_2zLanguage of target Bible text: z(Detected language of target Bible text: �tokenize)�lang�9processors)r�verses�10load_bible�greek_bible_path�greek_bible_text�target_bible_text�11splitlines�language�	pycountry�	languages�get�name�	lang_name�printr�12model_path�stanza�download�Pipeline�nlp�api_key�create_tfidf_vectorizer_matrix�tfidf_vectorizer�tfidf_matrix)�self�bible_text_pathr(r#�	lang_code�
first_line_nts      �Jc:\Users\caleb\Bible Translation Project\guidance\TranslationNoteFinder.py�__init__zTranslationNoteFinder.__init__sG��,�2�2��� $����0E�0E� F���!%����!A����.�.�9�9�;�E�B�
��%�D�M�&�0�0�4�4�T�]�]�4�K�P�P�D�N��3�D�N�N�3C�D�E�"�=�1�D�M�&�0�0�4�4�T�]�]�4�K�P�P�D�N��<�T�^�^�<L�M�N��(�D�O�	����
�
�&��?�?��
�
�*�M���"&����!A������48�3V�3V�3X�0���t�0�c�H�g}tj|j�d��}|D]u}|s�t|�dkDs�|dj	�s�)|dj	�|dj	�||dj	�zd�}|j|��w|S)N�	)�	delimiter���r)�13greek_term�translation_note�verse)�csv�readerr�len�strip�append)r,�file_content�book_abbrev�result�14tsv_reader�row�entrys       r0�parse_tsv_to_jsonz'TranslationNoteFinder.parse_tsv_to_json:s������Z�Z�� 7� 7� 9�T�J�15��16	%�C��s�3�x�!�|��A�����#&�a�&�,�,�.�(+�A�����(�3�q�6�<�<�>�9����
�
�e�$�17	%��
r2c��d|�d�}tj|�}|jdk(r
|j}nd}|j	||�}|S)Nz@https://git.door43.org/unfoldingWord/en_tn/raw/branch/master/tn_z.tsv���)�requestsr�status_code�textrG)r,rB�translation_notes_path�response�translation_notes_raw�translation_notess      r0�load_translation_notesz,TranslationNoteFinder.load_translation_notesOs[��#c�do�cp�pt�!u���<�<� 6�7�����3�&�$,�M�M�!�$&�!� �2�2�3H�+�V�� � r2c���|jd�r6tj|�}|jdk(r|j}|Sd}|St|dd��5}|j
�}ddd�|S#1swYSxYw)N�httprIrJ�rzutf-8)�encoding)�18startswithrKrrLrM�open�read)r,�19bible_pathrO�20bible_text�files     r0rz TranslationNoteFinder.load_bible]s���� � ��(��|�|�J�/�H��#�#�s�*�%�]�]�21��� �2223���j�#��8�
)�D�!�Y�Y�[�24�
)���
)���s�A1�1A;c�F�g}g}|j�}t|d��D]X\}}||jvr&|r$dj|�}|j	|�g}|j	|j���Z|r"dj|�}|j	|�|S)N�)�start� )r�	enumerater�joinr@r?)r,r[�	documents�current_document�verse_lines�i�line�joined_doc_strings        r0�segment_corpusz$TranslationNoteFinder.segment_corpusps����	��� �+�+�-�� ��A�6�	2�G�A�t��D�K�K��#�(+���1A�(B�%��$�$�%6�7�')�$��#�#�D�J�J�L�1�
	2�� #���)9� :�����.�/��r2c��|j|�}|jD��cgc]}|jD]}|j���!}}}|Scc}}w)N)r'�	sentences�wordsrM)r,rM�doc�sent�word�tokenss      r0�stanza_tokenizerz&TranslationNoteFinder.stanza_tokenizer�sE���h�h�t�n��(+�
�
�M��$�*�*�M�$�$�)�)�M�)�M��M��
��Ns�$A25c��t|jd��}|j|j�}|j	|�}||fS)N)r^�26)�	tokenizer�ngram_range)rrqrir�
fit_transform)r,r*�segmented_corpusr+s    r0r)z4TranslationNoteFinder.create_tfidf_vectorizer_matrix�sI��*�T�5J�5J�X_�`���.�.�t�/E�/E�F��'�5�5�6F�G����-�-r2c���ttjj��j	|�}|j27j
�}|j|j�}|j�d}tt||��}|j�D��cic]\}}|dkDs�||��}	}}tt|	j�d�d���}28|29Scc}}w)Nrc��|dS)Nr^�)�items r0�<lambda>z?TranslationNoteFinder.get_tfidf_book_features.<locals>.<lambda>�s
��^b�cd�^e�r2T)�key�reverse)�list�SR�30book_codes�keys�indexr*�get_feature_names_outr+�todense�tolist�dict�zip�items�sorted)r,�	book_code�31book_index�
feature_names�dense�document_tfidf_scores�feature_scores�feature�score�filtered_feature_scores�sorted_feature_scoress           r0�get_tfidf_book_featuresz-TranslationNoteFinder.get_tfidf_book_features�s����"�-�-�,�,�.�/�5�5�i�@�32��-�-�C�C�E�
��!�!�*�-�5�5�7�� %����q� 1���c�-�1F�G�H��IW�H\�H\�H^�"l�n�g�u�bg�jk�bk�7�E�>�"l��"l� $�V�,C�,I�,I�,K�Qe�os�%t� u��$�$��#ms�)
C'�7C'c	���tjd|j��}|}td|j	�D�cgc]}|��c}���|dj�}t
�5|d|�d�z
}|dz
}|dz
}|d	z
}ddd�t�5|d33|�d|�d|j	��d
�z
}ddd�t�5|tdd��z
}ddd�td|d���	tj|dj�|j	�dd��d}td|���|Scc}w#1swY��xYw#1swY��xYw#1swY��xYw#t$rd}Y�FwxYw)Nzgpt-4)r(z0All ngrams in verse guidance is selecting from: r9z1You are an expert at translating from Greek into �.z�When asked to translate, provide only the translation of the term. Nothing else. Do not provide any additional information or context.z,Be extrememly succinct in your translations.zgYou must choose only from the list of translation options you are given. Choose the single best option.zWhat is a good translation of z from Greek into z and is found here: �?�openai_translation)�stopzOpenAI translation: r^g333333�?)�n�cutoffrzNo close match foundzBest ngram found for note: )
r�OpenAIr(r"r�r?r	r34rr�difflib�get_close_matches�35IndexError)	r,�note�verse_ngramsr�36openai_llm�	openai_lmr}r9�ngrams	         r0�best_ngram_for_notez)TranslationNoteFinder.best_ngram_for_note�s����]�]�7�D�L�L�A�37��	�
�@�Q]�Qb�Qb�Qd�Ae�#�#�Ae�@f�g�h��,�'�-�-�/�38��X�	C��L�X�J�VW�X�X�I��b�
b�I��G�G�I��C�
C�I�		C��V�	L��9�*��EV�W_�V`�`t�vB�vG�vG�vI�uJ�JK�L�
L�I�	L�
�[�	=���1��<�<�I�	=�
�$�Y�/C�%D�$E�F�G�	+��-�-�i�8L�.M�.S�.S�.U�Wc�Wh�Wh�Wj�no�x{�|�}~��E�39	�+�E�7�3�4����1Bf�40	C�	C��	L�	L��	=�	=���	+�*�E�	+�s;�	D<41�+E�E
�E�1;E%�E42�
E�E"�%E3�2E3c	�D�t|�}|jj�|jdz43}g}t	d|���|j|jd�}|D]�}t|d�}|j|jk7r�+t	d|j�t	d|d���|dj�|j�vs�v|j|���t	d|D�cgc]}|d��	c}���|jj�|jdz44}|jd}	|j|	�}45t	d	t|46j��dd47���|48j�D��cic])\}}|j�|j�vs�'||��+}
}}t	dt|
j��dd���g}|D]{}|j||
|j�}|j�j!|j��}|t#|�z}|d}|d
}|j|||||d���}t	d|���||j|j|d�Scc}wcc}}w)Nr^z@Let's see if there are any translation notes for this verse: 49	 �bookCoder;zNote verse:zChecking for existence of: r9z0Greek terms for all translation notes in verse: zFirst 30 n-grams of the book: �z9First five n-grams of the verse along with their scores: �r:)r��	start_pos�end_posr9�50trans_notezVerse notes to be returned: )�target_verse_text�	verse_ref�line_number�ngrams)r�rrr�r"rR�structured_ref�lowerr@rr�rr�r�r�r!�findr>)r,r��v_ref�
gk_verse_text�translation_notes_in_verserQr��51note_v_refr�r��book_ngramsr�r�r�r�r�r�r�r9r�s                    r0�verse_notesz!TranslationNoteFinder.verse_notes�s����9�
���-�-�8�8�:�5�;L�;L�q�;P�Q�
�52&(�"�
�S�Ta�Sb�c�d� �7�7��8L�8L�Z�8X�Y��%�	8�D��D��M�*�J��%�%��):�):�:���-��!:�!:�;��/��\�0B�/C�D�E��L�!�'�'�)�]�-@�-@�-B�B�*�1�1�$�7�	8�	�@�a{�A|�Y]�$�|�BT�A|�@}�~��!�2�2�=�=�?��@Q�@Q�TU�@U�V���'�'�53�3���2�2�8�<��
�.�t�K�4D�4D�4F�/G���/L�.M�N�O�=H�=N�=N�=P�B�>�7�E�T[�Ta�Ta�Tc�gx�g~�g~�hA�UA����B��B�
�I�$�|�Oa�Oa�Oc�Jd�eg�fg�Jh�Ii�j�k���.�
	�D��,�,�T�<����P�E�)�/�/�1�6�6�u�{�{�}�E�I��#�e�*�,�G��l�+�J��0�1�J��M�M��&�"�(�(�
�
�

	�	�,�V�H�5�6�!2��-�-� �,�,��	54�	55��;B}��Bs�9J56�(J�<J)NN)�__name__�57__module__�__qualname__r��58verse_onesrrr1rGrRrrirqr)r�r�r�rzr2r0rrsF��
�]�]�F�0��Y�D�*!��&�(�.�%��B559r2r)�jsonr<�re�60langdetectrr�sklearn.feature_extraction.textr�guidancerrrrr	r61r�openai�romanizerr
r�r$r�rK�TrainingDatarrrzr2r0�<module>r�sA���62�	���;�N�N�N�
��7�
���&�i63�i64r2