CoolFace
Apppublic

CalebKoster/Translation_Note_Alignment

sourceHugging Faceupdated 3y agoView on Hugging Face
0likes
TranslationNoteFinderLLMOnly.cpython-312.pyc48 linesDownload Raw Back to __pycache__
12�N�eG&���ddlZddlZddlZddlmZddlZddlmZddlm	Z	ddl3mZmZm
Z
mZmZmZmZddlZddlmZddlmZddlZddlZddlZGd�d	�Zy)4�N)�detect)�Lang)�TfidfVectorizer)�models�gen�select�instruction�system�user�	assistant)�uroman)�ScriptureReferencec�P�eZdZejZd	d�Zd�Zd�Zd�Z	d�Z5d�Zd�Zy)6�TranslationNoteFinderNc�F�tj|_|j|�|_|rY||_t7jj|j��j|_	td|j���n~|jj�d}t|�|_t8jj|j��j|_	td|j���|j|�|_||_
y)N)�alpha_2zLanguage of target Bible text: i�Zz(Detected language of target Bible text: )r�verses�9load_bible�target_bible_text�language�	pycountry�	languages�get�name�	lang_name�print�10splitlinesr�api_key)�self�bible_text_pathr�	lang_code�
first_line_nts     �Qc:\Users\caleb\Bible Translation Project\guidance\TranslationNoteFinderLLMOnly.py�__init__zTranslationNoteFinder.__init__s���,�2�2���"&����!A����%�D�M�&�0�0�4�4�T�]�]�4�K�P�P�D�N��3�D�N�N�3C�D�E� �2�2�=�=�?��F�M�"�=�1�D�M�&�0�0�4�4�T�]�]�4�K�P�P�D�N��<�T�^�^�<L�M�N�"&����!A������c�H�g}tj|j�d��}|D]u}|s�t|�dkDs�|dj	�s�)|dj	�|dj	�||dj	�zd�}|j|��w|S)N�	)�	delimiter���r)�source_term�translation_note�verse)�csv�readerr�len�strip�append)r�file_content�book_abbrev�result�11tsv_reader�row�entrys       r#�parse_tsv_to_jsonz'TranslationNoteFinder.parse_tsv_to_json4s������Z�Z�� 7� 7� 9�T�J�12��13	%�C��s�3�x�!�|��A�����$'�q�6�<�<�>�(+�A�����(�3�q�6�<�<�>�9����
�
�e�$�14	%��
r%c��d|�d�}tj|�}|jdk(r
|j}nd}|j	||�}|S)Nz@https://git.door43.org/unfoldingWord/en_tn/raw/branch/master/tn_z.tsv���)�requestsr�status_code�textr:)rr5�translation_notes_path�response�translation_notes_raw�translation_notess      r#�load_translation_notesz,TranslationNoteFinder.load_translation_notesIs[��#c�do�cp�pt�!u���<�<� 6�7�����3�&�$,�M�M�!�$&�!� �2�2�3H�+�V�� � r%c���|jd�r6tj|�}|jdk(r|j}|Sd}|St|dd��5}|j
�}ddd�|S#1swYSxYw)N�httpr<r=�rzutf-8)�encoding)�15startswithr>rr?r@�open�read)r�16bible_pathrB�17bible_text�files     r#rz TranslationNoteFinder.load_bibleWs���� � ��(��|�|�J�/�H��#�#�s�*�%�]�]�18��� �1920���j�#��8�
)�D�!�Y�Y�[�21�
)���
)���s�A1�1A;c�F�g}g}|j�}t|d��D]X\}}||jvr&|r$dj|�}|j	|�g}|j	|j���Z|r"dj|�}|j	|�|S)N�)�start� )r�	enumerater�joinr3r2)rrN�	documents�current_document�verse_lines�i�line�joined_doc_strings        r#�segment_corpusz$TranslationNoteFinder.segment_corpusjs����	��� �+�+�-�� ��A�6�	2�G�A�t��D�K�K��#�(+���1A�(B�%��$�$�%6�7�')�$��#�#�D�J�J�L�1�
	2�� #���)9� :�����.�/��r%c��tjd|j��}|}|dj�}t	|gd���j22}t
d|�d|���t�5|d|�d	|�d23�z
}|d|�d|�d
|�d�z
}|dz
}|d|�d�z
}ddd�t�5|d|�d|�d|�d|�d�	z
}ddd�t�5|tdd24��z
}ddd�t
d|d���|dj�}t
d|���||vrt
d|���|St
d|���y#1swY��xYw#1swY��xYw#1swY�vxYw)Nzgpt-4)rr,)�en�he�el)�optionsz
Source term: z, 25Source language: z)You are an expert at translating between z and �.z*When asked to translate, provide only the z translation of the z term found in the z verse.zpNothing else. Do not provide any additional information or context. Be extrememly succinct in your translations.z;You must choose only an N-gram which already exists in the zWhat is a good translation of z from z into z& and is also found within this verse: �?�openai_translation)�stopzOpenAI translation: zLLM output: zLLM output found in verse: zLLM output not found in verse: r=)r�OpenAIrr2rrrr26rrr)	r�note�target_verse_textr�27openai_llm�	openai_lmr,�source_lang�28llm_outputs	         r#�best_ngram_for_notez)TranslationNoteFinder.best_ngram_for_note}s����]�]�7�D�L�L�A�29��	��=�)�/�/�1���;�0B�C�M�M��
�
�k�]�*?��}�M�N��X�	i��D�[�M�QV�W_�V`�`a�b�b�I��E�h�Z�Oc�do�cp�qD�EM�DN�NU�V�
V�I��L�
L�I��V�W_�V`�`g�h�h�I�		i��V�	e��9�+��f�[�M�Y_�`h�_i�jP�Qb�Pc�cd�e�
e�I�	e��[�	=���1��<�<�I�	=�
�$�Y�/C�%D�$E�F�G��3�4�:�:�<�30�
��Z�L�)�*��*�*��/�31�|�<�=����3�J�<�@�A��-	i�	i��	e�	e��	=�	=�s$�+*D:�'E�E�:E�E�Ec	��t|�}g}|j|jd�}|D];}t|d�}|j|jk(s�+|j	|��=td|D�cgc]}|d��	c}���|jj�|jdz32}g}|D]{}|d}	|d}33|j|||j�}|j�j|j��}|t|�z}
|j	|||
|	|34d���}td�ttj|d	�35��||j|j|d�Scc}w)N�bookCoder.z1Source terms for all translation notes in verse: r,rQr-)�ngram�	start_pos�end_posr,�36trans_notezVerse notes to be returned:r*)�indent)rh�	verse_ref�line_number�ngrams)�SRrE�structured_refrvr3rrrrmr�lower�findr1�json�dumps)rru�v_ref�translation_notes_in_verserDrg�37note_v_refrhrwr,rsrprqrrs              r#�verse_notesz!TranslationNoteFinder.verse_notes�s����9�
��&(�"� �7�7��8L�8L�Z�8X�Y��&�	8�D��D��M�*�J��%�%��):�):�:�*�1�1�$�7�	8�	�A�c}�B~�[_�4�
�CV�B~�A�A�	B�!�2�2�=�=�?��@Q�@Q�TU�@U�V����.�
	�D��}�-�K��0�1�J��,�,�T�3D�d�n�n�U�E�)�/�/�1�6�6�u�{�{�}�E�I��#�e�*�,�G��M�M��&�"�*�(�
�
�

	� 	�+�,�
�d�j�j���*�+�!2��-�-� �,�,��	38�	39��1Cs�6E=40)N)
�__name__�41__module__�__qualname__rx�42verse_onesrr$r:rErr\rmr��r%r#rrs/��
�]�]�F��2�*!��&�&!�H143r%r)r|r/�re�44langdetectrr�LanguageToolr�sklearn.feature_extraction.textr�guidancerrrr	r45rr�openai�romanizer
rrx�stanza�difflibr>rr�r%r#�<module>r�sA���46�	����;�N�N�N�
��7�
���@47�@48r%