CoolFace
Modelpublic

aigenrec/luminabackend

sourceHugging Faceupdated 10mo agoView on Hugging Face
0likes
document_service.cpython-312.pyc49 linesDownload Raw Back to __pycache__
1�

2jv5i����ddlZddlZddlmZddlmZddlmZddlm	Z	ddl3mZddlm
Z
ddlmZdd	lmZGd4�d�Ze�Zy)�N)�Optional)�supabase_client)�settings)�5FileParser)�TextChunker)�embedding_service)�qdrant_service)�loggerc�X�eZdZd�Zdedededefd�Z	ddeded	eefd6�Zdedefd�Zy)
�DocumentServicec��t|_t�|_t	t7jt8j��|_y)N)�9chunk_size�overlap)	r�clientr�file_parserrr�10CHUNK_SIZE�
CHUNK_OVERLAP�text_chunker)�selfs �.D:\lumina\backend\services\document_service.py�__init__zDocumentService.__init__
s2��%���%�<���'��*�*��*�*�11����document_id�12project_id�	file_path�filenamec13����������K�	�j�d��d{���tj�}tjd�����j�dd��d{���|jd�jj|��d{�����s�j�dd��d{���ytjd�����j�dd��d{���|jd��fd	���d{���}|s�j�dd14��d{���ytjdt|��d��d
|���tj���d{���tjdt|��d���j�ddt|��d���d{���d}g}tdt|�|�D]}	||	|	|z}15|j|	|16f��t|��tjd�������fd�}t|�D��
�cgc]\}\}
}|||
|���}}
}}tj|��d{����j�d��d{���tjd��d��	ddlm}|j%|���d{���y7���7��N7��"7��7���7���7���7��`7��cc}}
}w7�{7�c7�+#t&$r%}tj(d��d|���Yd}~yd}~wwxYw#t&$rS}tj(d��dt+|�����j�dt+|���d{���7Yd}~yd}~wwxYw�w)z@Process uploaded document: extract text, chunk, embed, and store�17processingNzExtracting text from zExtracting text...�failedzFailed to extract textzChunking text from zChunking text...c�:���jj��S�N)r�18chunk_text)r�texts��r�<lambda>z2DocumentService.process_document.<locals>.<lambda>2s����)�)�4�4�T�:rzNo chunks generatedzGenerating embeddings for z chunks�project_z(Generating embeddings and upserting for zGenerating embeddings (z chunks)...�r�19c��L�K��
4�d{���d}t|�D]�}	tjd|dz�d����tj|��d{���}tt|��D�cgc]}��||zd���
}}t
j�20|||���d{���ddd��d{���yddd��d{���y7��7�scc}w7�17�"#t$r�}dt|�vs
dt|�vrY||dz21krQd	|zd22|dzzz}	tjd|dz�d
|	�d��tj|	��d{���7Yd}~��5tjd|dz�d|dz�d|���||dz23k(r|�Yd}~��id}~wwxYw7��#1�d{���7swYyxYw�w)N�zProcessing batch ��/)r�
document_name�chunk_id)�collection_name�chunks�24embeddings�metadata�429zToo Many Requests�g�������?�zRate limit for batch z, retrying in zs...zError in batch z25 (Attempt z): )�ranger26�infor�generate_embeddings�lenr	�
upsert_chunks�	Exception�str�warning�asyncio�sleep�error)�	batch_idx�start_index�27batch_data�retries�attempt�batch_embeddings�k�batch_metadata�e�	wait_timer.rr�	semaphore�
total_batchess          �����r�
process_batchz7DocumentService.process_document.<locals>.process_batchRs������$�9��G�#(��>��"(�"�K�K�*;�I��M�?�!�M�?�([�\�6G�5Z�5Z�[e�5f�/f�,�*/�s�:��)?�
.�*@�A�	4?�5=�0;�a��!"�28*@�
+�.�#1�">�">�0?�'1�+;�)7�	#���#�5%�9�9�#1�%�9�9��0g��.���)%��8 )�	(�$��A���2E��Q��2O�#*�W�q�[�#8�12�g��#��UV��BW�0X�I�$*�N�N�5J�9�WX�=�/�Yg�hq�gr�rv�3w�$x�*1�-�-�	�*B�$B�$B�$,�"�L�L�?�9�q�=�/��T[�\]�T]�S^�^a�bc�ad�)e�f�&�'�A�+�5�&'�� 6��	(��9%��9�9�9�s��F$�C�F$�F�6C�C29�C�1C�C�C� C�$F�%F$�0C�1F$�6F�7F$�F
�F$�30C�C�F$�	F31�A'F�E32�F�F�.F�?F�F33�34F�
F$�F!�F�F!�F$�	completedz	Document z processed successfully)�mcq_servicezFailed to generate topics for z: zError processing document )�_update_document_statusr=�get_running_loopr35r6�run_in_executorr�extract_textr8r	�create_collectionr5�append�	Semaphore�	enumerate�gather�services.mcq_servicerN�generate_document_topicsr:r?r;)rrrrr�loopr/�36batch_size�batches�irBrL�idx�	start_idx�b�tasksrN�	topic_errrHr.rJr#rKs``  `              @@@@r�process_documentz DocumentService.process_documents`�����n	N��.�.�{�L�I�I�I��+�+�-�D�
�K�K�/��z�:�;��.�.�{�L�J^�_�_�_��-�-�d�D�4D�4D�4Q�4Q�S\�]�]�D���2�2�;��Jb�c�c�c��
�K�K�-�h�Z�8�9��.�.�{�L�J\�]�]�]��/�/��:���F�37��2�2�;��J_�`�`�`��
�K�K�4�S��[�M��I�J�!)���5�O� �2�2�?�C�C�C�
�K�K�B�3�v�;�-�w�W�X��.�.�{�L�Lc�dg�hn�do�cp�p{�J|�}�}�}��J��G��1�c�&�k�:�6��#�A�a�*�n�5�38�����:��/�7� ��L�M� �)�)�"�-�I�&
(�&
(�RR[�[b�Qc�d�Qc�:M�#�~�	�1�]�3�	�1�5�Qc�E�d��.�.�%�(�(�(��.�.�{�K�H�H�H��K�K�)�H�:�-D�E�F�
W�<�!�:�:�:�{�S�S�S�M
J��
`��]��d��
^����a��
D��
~��re�(��
I��T���
W����=�h�Z�r�)��U�V�V��
W���	N��L�L�5�h�Z�r�#�a�&��J�K��.�.�{�H�c�!�f�M�M�M��	N�sb�M*�L�J2�AL�'J5�(.L�J8�L�3J;�4L�8M*�9.L�'J>�(L�K�L�"K�#L�'M*�(?L�'K�(A	L�1K39�2A6L�(K
�?L�K�L�2K�3L�K�,K�-K�1M*�2L�5L�8L�;L�>L�L�L�L�40L�
L�L�K�	L�#L�>L�M*�L�L�	M'�AM"�M�M"�M*�"M'�'M*N�status�messagec��8K�	d|i}|dk(rd|d<n|r||d<|jjd�j|�jd|�j	�y#t41$r+}t
jdt|����Yd}~yd}~wwxYw�w)z-Update document processing status in database�
upload_statusrMN�
error_message�	documents�idz Error updating document status: )	r�table�update�eq�executer:r42r?r;)rrrdre�update_datarHs      rrOz'DocumentService._update_document_status�s�����	F�*�F�3�K���$�04��_�-��/6��O�,��K�K���k�*�1�1�+�>�A�A��k�
��g�i���	F��L�L�;�C��F�8�D�E�E��	F�s/�B�AA#�"B�#	B�,!B�
B�B�Bc��K�	d|��}tj||��d{���|jjd�j	�jd|�j
�tjd|�d|���y7�g#t$r'}tjdt|�����d}~wwxYw�w)z(Delete document from DB and Vector Storer%NrirjzDeleted document z from project zError deleting document: )r	�delete_vectorsrrk�deletermrnr43r6r:r?r;)rrrr.rHs     r�delete_documentzDocumentService.delete_document�s�����	� (���5�O� �/�/���M�M�M�
�K�K���k�*�1�1�3�6�6�t�[�I�Q�Q�S��K�K�+�K�=��z�l�S�T�
N���	��L�L�4�S��V�H�=�>���	�s9�B?�B�B44�A&B�	B?�45B�	B<�"B7�7B<�<B?r!)	�__name__�46__module__�__qualname__rr;rcrrOrs�rrrrs~��47�vN��vN��vN��	vN�48�vN�x"&�	F��F��F��#��	F�*���#�rr)�osr=�typingr�	db.clientr�config.settingsr�utils.file_parserr�utils.text_chunkerr�services.embedding_servicer�services.qdrant_servicer	�utils.loggerr49r�document_servicerwrr�<module>r�s;��	���%�$�(�*�8�2��g�g�R#�$�r