CoolFace
Apppublic

VTdevelops/rag-vs-full-context-comparator

sourceHugging Faceupdated 10mo agoView on Hugging Face
0likes
app.cpython-312.pyc357 linesDownload Raw Back to __pycache__
1�

2B�6i����	�dZddlZddlZddlZddlZddlZddlZddlZddlm	Z	ddl3mZmZddl
mZmZmZmZmZmZmZddlZ	ddlmZ	ddlZddlmZe�eGd	�d4��ZeGd�d��Zej@d
�Z!dZ"dZ#de$de%de%fd�Z&e'de&dd��Z(de$dee$fd�Z)de$dee$fd�Z*de	de	de+fd�Z,dee+dee+de+fd�Z-de$dee$effd�Z.d}dede%d e%deefd!�Z/d}d"eede%d e%deefd#�Z0d~d"eed$ee%dee$e1ffd%�Z2dd&e$d'eed(e%deefd)�Z3d*e$de$de$fd+�Z4d,eedeeeee$ffd-�Z5d.e$d/eee+d0d1de+fd2�Z6d.e$d3e$d0d1dee+fd4�Z7d5e$d6eee+d7eed8eede+f5d9�Z8d:e+d;e%de+fd<�Z9	d�d7eed8eed6eee+d=e%dee$e+ff6d>�Z:d?ee+d@ee+dAe1de$fdB�Z;d5e$d"eed8eedCeedDe$dEe$dFe$d0d1d6eee+de<fdG�Z=d:ee+de$fdH�Z>dIe<dJe1de$fdK�Z?d7eede$fdL�Z@dMe$dNee$de$fdO�ZAGdP�d1�ZBGdQ�dR�ZCeC�ZDeBeDj��S�ZFdTed5e$d.e$d3e$dUe$dee$effdV�ZGdWZHdXZId5e$dFe$fdY�ZJej�dZd[�\�5ZLej�d]�ej�d^eI�d_��ej�d`�ej��5ej�dadbdc�d�ZQej�dedfdg�d�ZRddd�ej�dhdigdj��k�ZTej�dldm�n�ZVej�do�ej�dpg�q�5ej�drg�q�5ej�ds�ej�d[�t�ZXddd�ej�drg�q�5ej�du�ej�d[�t�ZYddd�ddd�ej�dv�w�ZZej�dx�w�Z[ej�dyd[�z�5ej�eH�ddd�eVj�eJeQeTeRgeXeYeZe[g�{�ddd�e^d|k(reLj��yy#e$rd�ZY��/wxYw#e$rdZdZY��4wxYw#1swY���xYw#1swY��xYw#1swY��xYw#1swY��xYw#1swY��xYw#1swY�}xYw)�z*RAG vs full-context comparator Gradio app.�N)�Counter)�	dataclass�field)�Any�Dict�Iterable�List�Optional�Sequence�Tuple)�load_dotenvc��y�N�)�_args�_kwargss  �app.pyr
r
s���)�Settingsc�"�eZdZUeed<eed<y)�Document�name�textN)�__name__�7__module__�__qualname__�str�__annotations__rrrrr"s��8
�I�9
�Irrc�z�eZdZUeed<eed<ee��Zeed<dZe	ed<dZ10eee	ed<dZ
eeed	<y)11�Chunkr�source)�default_factory�tokens��scoreN�	embedding�uid)rrrrrrrr#r%�floatr&r12r	r'rrrr r (sF��13
�I��K��G�4�F�G�4��E�5��'+�I�x��U��$�+��C��#��rr z
[A-Za-z0-9']+zFull-context answerz14RAG answerr�default�returnc�v�	ttj|t|���S#t$r|cYSwxYwr)�int�os�getenvr�15ValueError)rr)s  r�_env_intr08s5����2�9�9�T�3�w�<�0�1�1�������s�'*�8�8�FULL_CONTEXT_CHAR_LIMITi�.rc�P�|sgStj|j��Sr)�TOKEN_RE�findall�lower�rs r�tokenizer7Bs!����	����D�J�J�L�)�)rc��|sgStjd|j��}|D�cgc]#}|j�s�|j���%c}Scc}w)Nz
(?<=[.!?])\s+)�re�split�strip)r�	sentences�sentences   r�split_into_sentencesr>HsI����	����)�4�:�:�<�8�I�-6�K�Y��(�.�.�:J�H�N�N��Y�K�K��Ks�A�A�vec_a�vec_bc������r�syt�j��t�j��z}|syt��fd�|D��}tjtd��j�D���}tjtd��j�D���}|dk(s|dk(ry|||zz}t
dtd|��S)Nr$c3�4�K�|]}�|�|z���y�wrr)�.0�tokenr?r@s  ��r�	<genexpr>z$cosine_similarity.<locals>.<genexpr>Us�����
>�v�e�e�E�l�U�5�\�)�v�s�c3�&K�|]	}||z���y�wrr�rC�values  rrEz$cosine_similarity.<locals>.<genexpr>V�����E�n�U�5�5�=�n���c3�&K�|]	}||z���y�wrrrGs  rrEz$cosine_similarity.<locals>.<genexpr>WrIrJ��?)�set�keys�sum�math�sqrt�values�max�min)r?r@�shared�dot�norm_a�norm_brHs``     r�cosine_similarityrYOs�������
�����
��U�Z�Z�\�!2�
2�F���16
�
>�v�
>�17>�C�
�Y�Y�s�E�e�l�l�n�E�E�
F�F�
�Y�Y�s�E�e�l�l�n�E�E�
F�F�
��}��#�
���6�F�?�#�E��s�C��U�O�$�$rc�Z�|r|syt|�t|�k7rytd�t||�D��}tjtd�|D���}tjtd�|D���}|dk(s|dk(ry|||zz}tdt
d|��S)Nr$c3�,K�|]\}}||z���y�wrr)rC�a�bs   rrEz*cosine_similarity_dense.<locals>.<genexpr>cs����
2� 1���1�a�!�e� 1�s�c3�&K�|]	}||z���y�wrr)rCr\s  rrEz*cosine_similarity_dense.<locals>.<genexpr>d�����0�%�Q�1�q�5�%�rJc3�&K�|]	}||z���y�wrr)rCr]s  rrEz*cosine_similarity_dense.<locals>.<genexpr>er_rJrL)�lenrO�ziprPrQrSrT)r?r@rVrWrXrHs      r�cosine_similarity_denserc^s������18�5�z�S��Z���19
�
2��E�5� 1�
2�202�C�
�Y�Y�s�0�%�0�0�
1�F�
�Y�Y�s�0�%�0�0�
1�F�
��}��#�
���6�F�?�#�E��s�C��U�O�$�$rc�x�|xsdj�}|std��	tj|�S#tj$rYnwxYwtjd|t21jt22jz��}|D]2}	tj|�cS#tj$rY�0wxYw|jd�}|jd�}|dk7rK|dk7rF||kDrA|||dz}	tj|�S#tj$r
Ytd	��wxYwtd	��)23N�zEmpty response bodyz```(?:json)?\s*(\{.*?\})\s*```)�flags�{�}������z"Could not parse JSON from response)r;r/�json�loads�JSONDecodeErrorr9r4�24IGNORECASE�DOTALL�find�rfind)r�	candidate�fenced�block�start�end�snippets       r�parse_json_blockrxls1�����"�"�$�I���.�/�/�
��z�z�)�$�$�����
��
���Z�Z�9�9�B�M�M�\^�\e�\e�Le�
f�F���	��:�:�e�$�$���#�#�	��	���
�N�N�3��E�25�/�/�#�26�C���{�s�b�y�S�5�[��E�C�!�G�,��	��:�:�g�&�&���#�#�	��27�9�28:�:�	���9�29:�:s3�8�A�
A�B%�%B;�:B;�9D�D.�-D.�document�30chunk_size�
chunk_overlapc�*�t|j�}|sS|jxsdj�}|sgSt|�}|sgSt	||j31t
|���gSg}g}g}d}	|D�].}32t|33�}|s�t|�}|	|z|kDr�|r�dj|�j�}|rIt
�}
|D]}|
j|��|jt	||j34|
���|dkDrlg}g}d}tt|�t|��D]>\}}|jd|�|jd|�|t|�z
}||k\s�>n|}|}|}	ng}g}d}	|j|35�|j|�|	|z
}	��1|rjdj|�j�}|rIt
�}
|D]}|
j|��|jt	||j36|
���|S)Nre)rr!r#r� )r>rr;r7r rrra�join�update�appendrb�reversed�insert)ryrzr{r<�strippedr#�chunks�current_sentences�current_tokens�current_lenr=�token_count�37chunk_text�
chunk_counter�38token_list�retained_sentences�retained_tokens�retained_len�
prev_sentence�prev_tokenss                    r�chunk_documentr��s��$�X�]�]�3�I���M�M�'�R�.�.�0����I��(�#����I��8�H�M�M�'�&�/�R�S�S��F�#%��&(�N��K����(�#�����&�k����$�z�1�6G����"3�4�:�:�<�J�� '�	�
�"0�J�!�(�(��4�#1��
�
�e��H�M�M�R_�`�a��q� �02�"�35�� ��25�h�?P�6Q�S[�\j�Sk�2l�.�M�;�&�-�-�a��?�#�*�*�1�k�:� �C��$4�4�L�#�}�4��3m�%7�!�!0��*��$&�!�!#����� � ��*����f�%��{�"��?�B��X�X�/�0�6�6�8�39��#�I�M�,�40��$�$�Z�0�-��M�M�%�Z��
�
�m�\�]��Mr�	documentsc��g}d}|D]A}t|||��D].}|j�d|��|_|dz
}|j|��0�C|S)Nr)rzr{�-rj)r�rr'r�)r�rzr{r��counterry�chunks       r�build_corpus_chunksr��s]���F��G���#�H��S`�a�E�#�=�=�/��7�)�4�E�I��q�L�G��M�M�%� �b��41�Mr�	max_charsc�x�|sy|r|dkDr|nd}g}d}d}|D]}d|j�d|jj��d�}t|�}|�/||z|kDr'||z42}	|	dkDr|j	|d|	�||	z
}d}n|j	|�||z
}��dj|�j�}43|44|fS)	N)reFrFzSource: �45�46 47Tre)rrr;rar�r~)r�r��limit�combined_parts�
running_total�	truncatedry�section�section_length�	remaining�
combined_texts           r�assemble_full_contextr��s�����"�y�1�}�I�$�E� "�N��M��I����X�]�]�O�2�h�m�m�.A�.A�.C�-D�D�I���W�������!?�%�!G��
�-�I��1�}��%�%�g�j�y�&9�:���*�
��I�����g�&���'�
���G�G�N�+�1�1�3�M��)�#�#r�queryr��top_kc�`�tt|��}|sgSg}|D]w}|js�t||j�}|j	t|j|j|j||j|j����y|jd�d��|d|S)N�rr!r#r%r&r'c��|jSr�r%��items r�<lambda>z)retrieve_chunks_lexical.<locals>.<lambda>�s����rT��key�reverse)rr7r#rYr�r rr!r&r'�sort)r�r�r��query_tokens�scoredr�r%s       r�retrieve_chunks_lexicalr��s����8�E�?�+�L���	��F����|�|��!�,����=���
�
���Z�Z��|�|��|�|���/�/��I�I�

�		48�	��K�K�+�T�K�:��&�5�>�r�pathc��tjj|�dj�}|dvr)t	|ddd��5}|j�cddd�S|dk(rTddl}t	|ddd��5}|j|�}ddd�tt�r|S|j|d	�49�S|dk(rX	ddlm}||�}g}	|jD]'}50|51j!�xsd}|	j#|��)dj%|	�Std|xsd�d���#1swY��xYw#1swY��xYw#t$r}td
�|�d}~wwxYw)Nrj>�.md�.csv�.txt�	.markdown�rzutf-8�ignore)�encoding�errors�.jsonr�)�indent�.pdf)�	PdfReaderzMPDF support requires the 'pypdf' package to be installed in this environment.rer�zUnsupported file type: '�unknownz?'. Upload text-based formats such as .txt, .md, .json, or .pdf.)r-r��splitextr5�open�readrk�load�52isinstancer�dumps�pypdfr��	Exception�RuntimeError�pages�extract_textr�r~)r�r�ext�handlerk�datar��exc�readerr��pagers            r�extract_text_from_uploadr�sU��53�'�'�54�55�4�56 ��57#�58)�59)�60+�C�61�2�2�
�$��g�h�
?�6��;�;�=�@�
?�62�g�~��
�$��g�h�
?�6��4�9�9�V�$�D�@��d�C� ��K��t�z�z�$�q�)�)�63�f�}�	y�'��4������L�L�D��$�$�&�,�"�D��L�L���!��y�y����64�65"�3�#3�)�"4�4s�t���+@�
?��66@�
?���	y��n�o�ux�x��	y�s0�D'�6D3�=D?�'D0�3D<�?	E�E�E�filesc���g}g}|s||fS|D]�}t|dd�}|s|jd��$t|dd�xstjj	|�}	t||�}|j
�}|s|j|�d����|jt||�����||fS#t$r }|j|�d|���Yd}~��d}~wwxYw)Nrz*Encountered an upload without a file path.�	orig_namez,: file was empty after stripping whitespace.)rrz: )	�getattrr�r-r��basenamer�r;rr�)	r�r�r��uploadedr��display_namer�cleanedr�s	         r�read_uploaded_filesr�s��� "�I��F���&� � ����x���.����M�M�F�G���x��d�;�U�r�w�w�?O�?O�PT�?U��	4�+�D�,�?�D��j�j�l�G���
�
���.Z�[�\�����X�<�g�F�G���f�����	4��M�M�\�N�"�S�E�2�3�3��	4�s� 2B5�B5�5	C�>C�C�answer�corpus_embeddings�embedder�EmbeddingServicec�l�t|�}|r|sy|j|�}|sy|D�cgc]}|s�|��	}}|syg}|D];}|s�|D�	cgc]}	t||	���}67}	|68s�"|jt	|69���=|syt|�t
|�z}t	dtd|��Scc}wcc}	w)Nr$rL)r>�embed_textsrcr�rSrOrarT)r�r�r��answer_sentences�answer_embeddings�vector�filtered_corpus�sentence_scores�sentence_vector�70corpus_vec�similarities�averages            r�compute_support_scorer�4s���71,�F�3���#4�� �,�,�-=�>����,=�H�,=�&��v�,=�O�H���#%�O�,����_n�o�_n�Q[�/���L�_n��o���"�"�3�|�#4�5�-����/�"�S��%9�9�G��s�C��W�%�&�&��I��ps�B,�B,�B1�	referencec�|�|r|sy|j||g�}t|�dk7ry|\}}|r|syt||�S)Nr�)r�rarc)r�r�r��72embeddings�
answer_vector�reference_vectors      r�compute_semantic_similarityr�OsP������%�%�v�y�&9�:�J�73�:��!���&0�#�M�#�� 0��"�=�2B�C�Cr�question�question_embedding�	retrieved�74all_chunksc	�H�|r�td�|D��r�|D�cgc]$}|jrt||j���&}}|D�cgc]$}|jrt||j���&}}|r|syt|�t	|�z}t|�t	|�z}tdt
d|||zdzz��Stt|��}	|	r|sy|D�cgc]%}|js�t|	|j���'}}|D�cgc]%}|js�t|	|j���'}}|r|syt|�t	|�z}t|�t	|�z}tdt
d|||zdzz��Scc}wcc}wcc}wcc}w)Nc3�4K�|]}|j���y�wr)r&�rCr�s  rrEz2compute_retrieval_effectiveness.<locals>.<genexpr>as����!I�y�e�%�/�/�y���r$rLg:�0�yE>)�anyr&rcrOrarSrTrr7r#rY)75r�r�r�r�r��retrieved_scores�baseline_scores�top_avg�baseline_avg�query_vectors76          r�compute_retrieval_effectivenessr[s����c�!I�y�!I�I�#�77�"�����
$�$6����H�"�	�78�$�79�#�����
$�$6����H�#�	�80�81 ����&�'�#�.>�*?�?���?�+�c�/�.B�B���3��C��G�l�,B�T�,I�!J�K�L�L��8�H�-�.�L��y��S\�m�S\�%�`e�`l�`l�)�,����E�S\��m�R\�m�R\��`e�`l�`l�(��u�|�|�D�R\�O�m��?���"�#�c�*:�&;�;�G���'�#�o�*>�>�L��s�C��W��,�(>��(E�F�G�H�H��582��8384��n��ms#�)F�)F�"F�4F�F�$Fr%�rankc�R�|dkDr!d|zdz85tj|dz�zSdS)Nrr�rjr$)rP�log2)r%rs  r�discounted_gainrs-��59�A�X�A��J��N�d�i�i��q��1�1�F�3�Fr�max_relevantc86��|sddddd�Sg}|D]T}|jr|js�t||j�}|dkr�8|j|j|f��V|sddddd�S|j	d�d��|d|D��chc]\}}|��	}	}}|D��cic]\}}||��87}88}}tt
|	�d�}|D�cgc]}|js�|j��}}|sddddd�S|D�cgc]89}||	vrdnd��}
}|D�cgc]}|90j|d���}}t|
�t
|
�z}t|
�|z}d}t|
�D]\}}|s�	d|dzz}nd}t|�D]\}}|t||dz�z
}�|dt
|�D��cgc]\}}|��	}}}d}t|�D]\}}|t||dz�z
}�|dkDr||znd}tdtd|��tdtd|��tdtd|��tdtd|��d�Scc}}wcc}}wcc}wcc}wcc}wcc}}w)	Nr$)�	precision�recall�mrr�ndcgc��|dS)Nrjrr�s rr�z&compute_rank_metrics.<locals>.<lambda>�s���a�rTr�rjrrL)r&r'rcr�r�rSra�getrO�	enumeraterrT)r�r�r�r	r�r�r%r'�_�top_relevant�relevant_scores�total_relevant�retrieved_uids�binary_relevance�graded_relevancerrr
�idx�is_rel�dcg�rel�ideal_scores�idcgrs                         r�compute_rank_metricsr�s���� �C��S�I�I�&(�F������e�i�i��'�(:�E�O�O�L���C�<���
�
�u�y�y�%�(�)�
�� �C��S�I�I�91�K�K�(�$�K�7�&,�^�|�&<�=�&<�F�C��C�&<�L�=�4:�;�F�j�c�5�s�E�z�F�O�;���\�*�A�.�N�-6�D�Y�E�%�)�)�e�i�i�Y�N�D�� �C��S�I�I�CQ�R�>�C�S�L�0��a�7�>��R�AO�P��#��+�+�C��5���P��$�%��,<�(=�=�I�
�!�
"�^�
3�F�92
�C� �!1�2���V����q��/�C��3�93�C��.�/���S���s�C�!�G�,�,��0�+1�1H�3�7G�3H�*I�J�*I�h�a��E�*I�L�J��D��l�+���S����S�1�W�-�-��,����3��:�s�D���c�#�y�1�2��c�3�s�F�+�,��3��C��
�&��C��S�$��(�	���=>��;��E��S��P��Ks*�94H=�
I�I	�I	�6I�I�$I�score_full_context�	score_rag�higher_is_betterc��|�|�y|�tS|�tSd}t||z95�|kry|r||kDrtStS||krtStS)N�Tieg�����ư>)�	RAG_LABEL�FULL_CONTEXT_LABEL�abs)r r!r"�	tolerances    r�winner_labelr)�sn���!�i�&7���!�����!�!��I�96��	�)�*�i�7���%�(:�:�y�R�@R�R�!�$6�6�9�N�<N�Nr�retrieved_chunks�full_context_answer�97rag_answer�reference_answerc	�"�g}	|D]&}98|	jt|99j���(|	r|j|	�ng}t	|||�}t	|||�}
tdd|z100�}tdd|
z101�}|j
�}|r
t|||�nd}|r
t|||�nd}t||||�}t|||�}t||
d��t||d��|rt||d��nd|d�}|||d�|
||d�||t|�|d	�S)102Nr$rLT)r"FzReference answer not provided)�
more_grounded�fewer_hallucinations�closest_to_reference�retrieval_effectiveness)�
support_score�hallucination_score�semantic_similarity)�full_context�ragr2�retrieval_rank�retrieved_k�103comparison)�extendr>rr�r�rSr;r�rrr)ra)r�r�r�r*r+r,r-r�r��corpus_sentencesryr��support_full�support_rag�hallucination_full�hallucination_rag�reference_clean�
semantic_full�semantic_rag�retrieval_score�rank_metricsr:s                      r�evaluate_answersrF�s\��#%������� 4�X�]�]� C�D��BR��,�,�-=�>�XZ��(�)<�>O�QY�Z�L�'�104�4E�x�P�K��S�#��"4�5���C��{�!2�3��&�,�,�.�O�cr�/�0C�_�V^�_�x|�M�Yh�.�z�?�H�U�nr�L�5�h�@R�Td�fp�q�O�'�(8�*�FX�Y�L�%�l�K�RV�W� ,�-?�AR�ej� k��
���t�L�0�#2�	�J�*�#5�#0�105�)�#4�#/�106�107$3�&��+�,� ��rc��|�y|dzd�d�S)N�—�dz.1f�%rr�s r�format_scorerKs���}���c�k�#�
�a� � r�metrics�reference_availablec�B108�dt|dd�t|dd�dfdt|dd�t|dd�dfg}|r3|jd	t|dd109�t|dd110�df�|jdd
t|d�df�gd�}|D]%\}}}}|jd|�d|�d|�d|�d�	��'|jd�|d}|jd�|jd�|jd|d�d��|jd|d�d��|r|jd|d�d��n|jd�t|d�}	|jd |	�d��|jd!�|jd"i�}111|112�r|jd#�xst|113�}|jd$�|jd�|jd%|�d&t|114jd'���d��|jd(t|115jd)���d��|jd*t|116jd+���d��|jd,t|117jd-���d��|jd!�|jd.i�}|�r'|jd/�|jd0�|jd1�|jd2�d3t|jdi�jd4��t|jdi�jd4��d5fd6t|jdi�jd7��t|jdi�jd7��d8fd9t|jdi�jd:��t|jdi�jd:��d;fg}
|
D]%\}}}}|jd|�d|�d|�d|�d�	��'|jd�|jdi�jd<�}|jdi�jd<�}|s|ra|jd=�|jd�|r|jd>|�d?��|r|jd@|�d?��|jd!�|jdA�|jdB�dCj	|�S)DNz
Support scorer6r3r7uhHigher is better — measures how well each answer sentence embedding is backed by the uploaded sources.zHallucination scorer4uZLower is better — 1 minus the support score, indicating embedding-level gaps in support.z Semantic similarity to referencer5uNHigher is better — embedding similarity to the provided ground-truth answer.zRetrieval effectivenessrHr2ujHigher is better — compares retrieved chunk embeddings to the average embedding match for this question.)z<div class='metric-summary'>�<table class='metric-table'>z_<thead><tr><th>Metric</th><th>Full context</th><th>RAG</th><th>Interpretation</th></tr></thead>�<tbody>z<tr><td>z	</td><td>z118</td></tr>z</tbody></table>r:z3<p class='comparison-note'>Automatic comparison</p>z<ul>z3<li class='comparison-note'>More grounded: <strong>r/z</strong></li>z:<li class='comparison-note'>Fewer hallucinations: <strong>r0z@<li class='comparison-note'>Closer to reference answer: <strong>r1zZ<li>Closer to reference answer: Provide a reference answer to enable this comparison.</li>zC<li class='comparison-note'>Retrieval effectiveness score: <strong>z</ul>r8r9z8<p class='comparison-note'>Retrieval ranking metrics</p>z<li>Precision@z119: <strong>rz<li>Recall estimate: <strong>rz<li>MRR: <strong>r
z<li>nDCG: <strong>r�judgez1<p class='comparison-note'>LLM judge verdicts</p>rOzb<thead><tr><th>Criterion</th><th>Full context</th><th>RAG</th><th>Interpretation</th></tr></thead>rP�Accuracy�accuracyz9Higher means the answer aligns with ground truth/context.�Completeness�completenessz.Higher means the answer covers all key points.�	Relevance�	relevancez8Higher means the answer stays on topic for the question.�feedbackz4<p class='metric-disclaimer'>LLM judge feedback:</p>z<li>Full context: z</li>z	<li>RAG: z�<p class='metric-disclaimer'>Scores are heuristic: embedding-based metrics use OpenAI or hashed fallback vectors, and the LLM judge relies on prompting best effort. Use them for rapid feedback rather than formal evaluation.</p>z</div>re)rKr�rrar~)rLrM�rows�120html_parts�metric_name�no_value�	rag_value�noter:�retrieval_percentrEr9rQ�121judge_rows�feedback_no�feedback_rags                r�format_metrics_htmlrcs���
����0��A�B������8�9�v�		122�
"����0�1F�G�H�����(=�>�?�h�		123�
�D�����2��W�^�4�5J�K�L��W�U�^�,A�B�C�`�	
�	124�	�K�K�%����!:�;�<�x�		125���J�37�.��X�y�$�����{�m�9�X�J�i�	�{�)�TX�SY�Yc�d�	126�37����(�)���&�J����K�L����f�����127=�j��>Y�=Z�Zh�i�����128D�Z�Pf�Eg�Dh�hv�w������N�z�Zp�Oq�Nr�sA�
B�	129�	���h�	130�%�W�-F�%G�H�����131M�N_�M`�`n�o�����g���;�;�/��4�L���k�k�-�0�E�C��4E�����T�U����&�!�����[�M��L��AQ�AQ�R]�A^�4_�3`�`n�o�	132�	���+�L��9I�9I�(�9S�,T�+U�Uc�d�	133�	�����\�-=�-=�e�-D� E�F�n�U�	134�	��� ��l�.>�.>�v�.F�!G� H��W�	135�	���'�"��K�K���$�E�����M�N����8�9����~�����)�$���U�Y�Y�~�r�:�>�>�z�J�K��U�Y�Y�u�b�1�5�5�j�A�B�K�	
���U�Y�Y�~�r�:�>�>�~�N�O��U�Y�Y�u�b�1�5�5�n�E�F�@�	
���U�Y�Y�~�r�:�>�>�{�K�L��U�Y�Y�u�b�1�5�5�k�B�C�J�	
�136�137�(7A�2�K��9�d�����;�-�y��138�)�I�;�i�X\�W]�]g�h�
�7A�	���,�-��i�i���3�7�7�139�C���y�y���+�/�/�140�;���,����T�U����f�%���!�!�$6�{�m�5�"I�J���!�!�I�l�^�5�"A�B����g�&����	I�����h��
�7�7�:��rc���|syg}t|d��D]R\}}|jd�}|jd|�d|j�d|�d|jj�����Td	j
|�S)141NzX_No supporting context retrieved yet. Upload richer documents or rephrase the question._rj)ruz.2fz**Chunk u — z
 (similarity z)**142 143z144 145---146 147)rr%r�r!rr;r~)r��partsrr��
score_displays     r�format_context_markdownrg�s����i��E��	��3�148��U� �;�;�s�+�
�
����s�e�5�����m�M�?�'�RW�R\�R\�Rb�Rb�Rd�Qe�f�	149�4�150���e�$�$r�markdownr�c�X�|s|Sdjd�|D��}d|�d�}||xsdzS)Nr�c3�&K�|]	}d|�����y�w)z- Nr)rC�messages  rrEz#prepend_warnings.<locals>.<genexpr>�s����A�&�w�b��	�N�&�rJz'**Warnings while reading your files:**151r�re)r~)rhr��bullet_list�
warning_blocks    r�prepend_warningsrn�s;������)�)�A�&�A�A�K�>�{�m�4�P�M��H�N��+�+rc�|�eZdZdZddeeddfd�Zdeede	e	e152fd�Zdedee	e153fd	�Zdede	e154fd155�Z
y)r�zCUtility for generating embeddings with OpenAI or a hashed fallback.N�existing_clientr*c	�f�tjdd�|_tdt	tjdd���|_||_|j�,tjd�}|r	ddlm}||�	�|_|jdu|_156y#t$r157d|_Y�&wxYw)158N�OPENAI_EMBED_MODELztext-embedding-3-large�@�FALLBACK_EMBED_DIM�512�OPENAI_API_KEYr��OpenAI��api_key)r-r.�modelrSr,�dim�client�openairxr��
fallback_mode)�selfrprzrxs    r�__init__zEmbeddingService.__init__�s����Y�Y�3�5M�N��159��r�3�r�y�y�)=�u�E�F�G���%����;�;���i�i� 0�1�G��'�-�"(��"9�D�K�"�[�[�D�0����!�'�"&�D�K�'�s�5B�B0�/B0�textsc���|D�cgc]}|��|xsdj���}}|sgS|D�cgc]}|r|nd��160}}|j�^	|jjj|j|��}|j161D�cgc]}|j��}}d|_|S|D�cgc]}|j|���c}Scc}wcc}wcc}w#t$rd|_d|_Y�GwxYwcc}w)Nrer})r{�inputFT)162r;r}r��creater{r�r&rr��_fallback_embedding)r�r�rr��normalized_inputs�response�recordr�s        rr�zEmbeddingService.embed_texts�s��49�N�E�D�T�=M�D�J�B�%�%�'�E��N���I�?F�G�w�t�T�T�s�2�w��G��;�;�"�	
*��;�;�1�1�8�8�t�z�z�Qb�8�c��=E�]�]�K�]�6�f�.�.�]�163�K�%*��"�!�!�164<M�M�;L�4��(�(��.�;L�M�M��O��H��L���
*�"���%)��"�
*��Ns9�C�C�
C165�	?C�C�166C�*C1�C�C.�-C.rc�8�|j|g�}|r|dSdS)Nr)r�)r�rr�s   r�	embed_onezEmbeddingService.embed_one�s%���%�%�t�f�-�167� *�z�!�}�4��4rc��dg|jz}t|�}|s|S|D]'}t|�|jz}||xxdz
cc<�)tjtd�|D���}|r|D�cgc]}||z��	}}|Scc}w)Nr$rLc3�&K�|]	}||z���y�wrrrGs  rrEz7EmbeddingService._fallback_embedding.<locals>.<genexpr>�s����?��u�U�U�]��rJ)r|r7�hashrPrQrO)r�rr�r#rDr�normrHs        rr�z$EmbeddingService._fallback_embedding�s��������!���$�����M��E��u�+����(�C��3�K�3��K���y�y��?��?�?�@���06�7��u�e�d�l��F�7��
��8s�7Br)rrr�__doc__r168rr�rrr	r(r�r�r�rrrr�r��sr��M�
1���
�
1��
1�N��#��N�4��U��3D�N�$5�c�5�h�t�E�{�&;�5�����U��rc�P�eZdZdZd	d�Zd169dedeedefd�Zd170dedeedefd�Zy)�171LLMServicezLSimple wrapper that prefers OpenAI if configured, otherwise uses heuristics.r*Nc��d|_tjdd�|_t	tjdd��|_tjd�}|r	ddlm}||��|_yy#t$r172d|_YywxYw)	N�OPENAI_MODEL�gpt-4o-mini�LLM_TEMPERATUREz0.2rvrrwry)	r}r-r.r{r(�temperaturer~rxr�)r�rzrxs   rr�zLLMService.__init__�sw������Y�Y�~�}�=��173� ����+<�e�!D�E����)�)�,�-���
#�)�$�W�5���	��174�
#�"���
#�s�A5�5B�Br��contextc��|xsdj�}|sy|jr�d}g}|r"|jd|j�z�|jd|z�|jd�	|jjj	|j175d|d�d	d176j
|�d�g|j��}|jj�S|j||�S#t$r}d|�d
�cYd}~Sd}~wwxYw)NrezNo question provided.z�You are an assistant that compares retrieval augmented generation to plain LLM answers. Always ground responses in the supplied context when available. If context is missing, answer from general knowledge and call out uncertainty explicitly.z	Context:177z178Question:179zAnswer:�system��role�content�userr��r{r�r�z[LLM call failed: �])r;r}r��	responsesr�r{r~r��output_textr��_fallback_answer)r�r�r��
system_prompt�user_contentr�r�s       rr�zLLMService.answer�s���N��)�)�+���*��;�;�I�
�180')�L���#�#�L�7�=�=�?�$B�C����
�� 8�9����	�*�
3��;�;�0�0�7�7��*�*�!)�m�D�!'�F�K�K��4M�N��!%� 0� 0�
8��� �+�+�1�1�3�3��$�$�X�w�7�7���
3�+�C�5��2�2��
3�s�2A,C1�1	D181�:D�?D182�D183c���|r�t|�}|s	ytt|���g}|D]B}t|�}|s�t�fd�|D��}|j	|t|�|f��D|sy|j
�|ddD�cgc]}|d��	}	}dj|	�j�}184|185sd}186d|187��Sd	|�d188�Scc}w)NzeContext grounded answer (heuristic): The retrieved context did not contain enough readable sentences.c3�,�K�|]}|�vs�d���
y�w)rjNr)rCrD�keywordss  �rrEz.LLMService._fallback_answer.<locals>.<genexpr>s�����I�V�E�u��7H�a�V�s�	�z`Context grounded answer (heuristic): No overlapping signals were found in the retrieved context.�����r�r}zqThe uploaded material does not include details that match the question closely, so this answer may be incomplete.zAContext grounded answer (heuristic, no external LLM configured): zZSpeculative answer without retrieval (heuristic): I do not have supporting documents for 'z!', so this guess may hallucinate.)	r>rMr7rOr�rar�r~r;)r�r�r�r<r�r=r#�overlap�entry�
top_sentencesrwr�s           @rr�zLLMService._fallback_answers�����,�W�5�I��{���8�H�-�.�H�13�F�%��!�(�+�����I�V�I�I���
�
�w��V���h�?�@�&��y��K�K�M�39�"�#�;�?�;�%�U�1�X�;�M�?��h�h�}�-�3�3�5�G��H��T��)��
�189
7�7?�j�@a�
c�	190��@s�191C)r*Nr)	rrrr�r�rr192r�r�rrrr�r��sG��V�#�8�s�8�X�c�]�8�c�8�:193��194�x��}�195�PS�196rr�)rpr}r�c����K���dddddd�Stjd|�d|xsd�d|�d|�d	�	�j��	tj�}|jd��fd197���d{���}|jj�}t|�}dt|jdd
��t|jdd
��t|jdd
��|jdd�d�S7��#t$rJ}	dt�vrt�dkDr|dddzndt�vrnd}198ddddd|	�d|199��d�cYd}	~	Sd}	~	wwxYw�w)N�unavailablez.LLM judge unavailable (no API key configured).)�statusrSrUrWrXae200        You are evaluating answers in a Retrieval-Augmented Generation (RAG) demo.201        Rate the given answer from 0 to 1 on accuracy, completeness, and relevance, using the reference answer and retrieved context.202        Return a compact JSON dictionary with keys accuracy, completeness, relevance, and a short feedback note.203 204        Context:205        z/206 207        Reference answer (optional):208        z[not provided]z!209 210        User question:211        z&212 213        Answer to evaluate:214        z	215        c�x���jjtjdd�ddd�d�d�gd��S)	N�OPENAI_JUDGE_MODELr�r�zReturn valid JSON only.r�r�g�������?r�)r�r�r-r.)r}�prompts��rr�z!async_llm_judge.<locals>.<lambda>SsN���F�$�$�+�+��i�i� 4�m�D�!)�#<��216!'�#)��	� �,�
r�okrSr$rUrWrXre�raw��u…�errorz
Judge error: z. Raw: )
�textwrap�dedentr;�asyncio�get_event_loop�run_in_executorr�rxr(rr��localsra)r}r�r�r�r��loopr�r�r�r��raw_previewr�s`          @r�async_llm_judger�+s�������~�#�� ��H�217�	218��_�_�
	�219�	�	�220�	&�&�'�(	�221�222�	�223��	�	��$�e�g�%�($224��%�%�'���-�-��

�225�226��"�"�"�(�(�*����$����d�h�h�z�3�7�8�!�$�(�(�>�3�"?�@��t�x�x��S�9�:�����R�0�227�	228�'229��4�230�-2�f�h�->�3�s�8�c�>�s�4�C�y�5�(�^c�gm�go�^o�WZ�uw���� ��'��u�G�K�=�A�231�	232��233�sH�?E�-D�1C>�2BD�=E�>D�	E�	?E�E�	E�E�Eu�234**Hallucination Score** — 1 minus the support score. Lower is better; 0 means every answer sentence has strong embedding support in the uploaded sources.235 236**Support Score** — Average cosine similarity between each answer sentence embedding and its best matching sentence embedding from the uploaded documents. Higher is better and indicates factual grounding.237 238**Semantic Similarity** — Cosine similarity between the answer embedding and the provided reference answer embedding. Higher is better. This metric is only available when you supply a ground-truth answer.239 240**Retrieval Effectiveness** — Compares how well the retrieved chunk embeddings match the question embedding relative to the average chunk in the corpus. Higher is better; 0.5 means retrieval is performing on par with a random chunk.241 242**Precision / Recall / MRR / nDCG** — Ranking metrics derived from embedding similarity that estimate how well the retriever surfaces the most relevant chunks. Higher is better.243 244When the OpenAI embedding API is unavailable, the app falls back to hashed lexical vectors so scores remain approximate but still indicative. If the LLM judge cannot be reached, its scores are replaced with a warning.245 246The full-context baseline feeds the concatenated uploads directly to the LLM (trimmed to the `FULL_CONTEXT_CHAR_LIMIT` environment variable, default 12,000 characters) so you can see what retrieval adds over simply pasting the entire document set into the prompt.247a#248.metric-table {249    width: 100%;250    border-collapse: collapse;251}252.metric-table th, .metric-table td {253    border: 1px solid #ddd;254    padding: 0.4rem 0.6rem;255    text-align: left;256}257.metric-table th {258    background-color: #f8f8f8;259}260.metric-summary ul {261    margin-top: 0.4rem;262}263.metric-summary li {264    margin-bottom: 0.2rem;265}266.metric-disclaimer {267    font-size: 0.85rem;268    color: #555;269}270.answer-section {271    display: flex;272    gap: 1rem;273    flex-wrap: wrap;274}275.answer-card {276    background-color: #f9fafc;277    border: 1px solid #d8dee9;278    border-radius: 8px;279    padding: 0.8rem;280    flex: 1 1 280px;281    box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.6);282}283.answer-card h3 {284    margin-top: 0;285}286.answer-card p {287    white-space: pre-wrap;288}289.comparison-note {290    font-weight: 600;291}292c��0K�|xsdj�}|sd}||ddfSt|�\}}t|�}|svd}tjr|jd�|j|�tj|d��}tj|d��}	d}293td	|�}||	||294fSt|t�295�\}}
|
rtr|jdt�d��t|�}|D�cgc]}|j��}}|rtj|�ng}t|�D]!\}}|t|�kr||nd}||_�#tj#|�}g}d
}d
}t$���t&���|�r�|���d}	t)t$d�rt%j*�}nt%j,t'd
���}|j/dt1j2���ddi��}t5t|��D�cgc]}d|����	}}|j7||||D�cgc]}d|j8i��c}��t;dt|��}|j=|g|��}t|�D��cic]\}}|||��
}}}|j?dgg�} t| r| dng�D]�\}}!|j?|!�}"|"��d}#|r"|"j rtA||"j �}#|jtC|"j|"j8|"jD|#|"j |"jF�����tI|�}nEt$�t&�|jd �n'|�|jd!�n|s|jd"�|s#|r|s|jd#�tM||d�$�}tjr|jd�d%jOd&�|D��}%g}&|%r|&jd'|%z�|r|&jd(|z�|&rd%jO|&�nd)}'tj||r|nd��}tj||%r|%nd��}	|xsdj�}(tQ||||||	|(t|�*�	})tSjTtWtjX|||(|'��tSjTtWtjX||	|(|'��g}*tSjZ|*��d{���\}+},|+|,d+�}-|-j]�D]h\}.}/|/j?d,�}0|0d-k(r&d.d/jO|�vs�0|jd0��B|0d1k(s�H|j|/j?d2d3���j|-|)d4<t_|)tI|(��5�}1ta|�}t||�}||	||1fScc}wcc}wcc}wcc}}w#tJ$r }$|jd|$�d��Yd}$~$��qd}$~$wwxYw7���w)6Nrez0Please provide a question to run the comparison.z"_Awaiting question and documents._zC<p>Enter a question and upload documents to see the comparison.</p>z`No documents were loaded. Upload at least one text-friendly file (.txt, .md, .json, .pdf, .csv).z`OpenAI embedding API not configured; using hashed fallback vectors until an API key is provided.)r�z5<p>Upload documents to compute grounding metrics.</p>z_No context available._)r�z!Full-context prompt truncated to z( characters to stay within model limits.FT�EphemeralClient)�anonymized_telemetryzrag-session-z296hnsw:space�cosine)r�metadatazchunk-r!)�idsr�r��	metadatas�)�query_embeddings�	n_resultsr�rr$r�zChroma retrieval failed (z"); falling back to lexical search.z:Chroma not installed; using lexical retrieval as fallback.zJCould not compute question embedding; using lexical retrieval as fallback.z=No chunks were embedded; using lexical retrieval as fallback.zJVector retrieval returned no matches; using lexical retrieval as fallback.)r�r�c3�4K�|]}|j���y�wrr6r�s  rrEzcompare.<locals>.<genexpr>*s����A�y�e�u�z�z�y�r�zRetrieved context:297zFull document context:298z+No context was available for this question.)	r�r�r�r*r+r,r-r�r�)r6r7r�r�zLLM judge unavailabler}zDLLM judge unavailable (no API key configured); judge scores omitted.r�rXzLLM judge encountered an error.rQ)rM)1r;r��list�EMBEDDERrr��LLMr�rnr�r1r�rr�rrar&r��chromadbr�hasattrr��Client�create_collection�uuid�uuid4�range�addr!rTr�rrcr r#r'�boolr�r�r~rFr��create_taskr�r}�gather�itemsrcrg)2r�r�r-rkr��file_errors�warnings�no_docs_messager+r,�metrics_placeholder�context_markdown�full_context_text�truncated_full_contextr�r��chunk_texts�chunk_embeddings�indexr&r�r��vector_retrieval_used�vector_attemptedr}�299collectionrr�r��results�id_value�	id_lookup�
retrieved_ids�chunk_id�300base_chunkr%r��context_text�context_sections�context_for_judgerArL�judge_tasks�judge_no�	judge_rag�
judge_results�label�resultr��metrics_htmls2                                                  r�comparer��sS������B�%�%�'�H��D����0�Q�	301�	302�1��7��I�{��K� �H��n�	��!�!��O�O�r�
�	����(�!�j�j��4�j�@���Z�Z��$�Z�7�303�U��+�,E�x�P��"�J�0@�BU�U�U�0E�i�[r�0s�-��-��"9����/�0G�/H�Hp�q�	304�%�Y�/�J�+5�6�:�%�5�:�:�:�K�6�<G�x�+�+�K�8�R��!�*�-���u�/4�s�;K�7L�/L�$�U�+�RV�	�#���.�"�+�+�H�5���I�!����	��� ���*���+	a��x�!2�3�!�1�1�3��!����-2����305 �1�1�#�D�J�J�L�>�2�&��1�2��J�.3�3�z�?�-C�D�-C�c�V�C�5�>�-C�C�D��N�N��+�%�AK�L���H�e�l�l�3��L�	
�
��A�s�:��/�I� �&�&�9K�8L�Xa�&�b�G�Zc�dg�Zh�*i�Zh���h�8�Z��_�+D�Zh�I�*i�#�K�K���t�4�M�!*�}�=��+;�RT�!U�
��X�&�]�]�8�4�306��%����%�*�*>�*>�3�4F�307�H\�H\�]�E�� � ��'�_�_�)�0�0�)�0�0�#�",�"6�"6�&�N�N�
�	�"V�"%)��O�!���x�/��O�O�X�Y�
�
'��O�O�h�i�!��O�O�[�\� ��I��O�O�h�i�+�H�j��J�	�������n�	308��;�;�A�y�A�A�L�"$������ 6�� E�F����� :�=N� N�O��	���$�%�
:���*�*��%6�!�D�%������H�l�l�PT��U�J�'�-�2�4�4�6�O�����"�/��(��-�309�G�	�����310�311��#��!�
�	312�	�����313�314����!�
�	315��K�(!(���� <�<��H�i�%-�i�@�M�&�,�,�.�
��v����H�%���]�"�&�c�h�h�x�.@�@����Z���w�
��O�O�F�J�J�z�3T�U�V�/�%�G�G��&�w�D��DY�Z�L�.�y�9��'�(8�(�C���316�,<�l�J�J��e7��>E��317M��+j��(�	a��O�O�7��u�<^�_�`�`��	a��L=�s��C4X�6W�	A;X�BW'�	W�W'�)W�>?W'�=W!�
CW'�F>X�X�AX�X�7A X�W'�'	X�0X�X�X�XzRAG vs Full-Context ComparatorF)�title�analytics_enabledz # RAG vs Full-Context Comparatorz<style>z</style>a
Upload documents, ask a question, and compare a full-document LLM answer against a retrieval-augmented answer. Vector retrieval runs on OpenAI text-embedding-3-large + Chroma, with automatic evaluations highlighting grounding, hallucination risk, and retrieval quality.�Questionz3e.g. What are the main obligations in the contract?r�)r��placeholder�lineszReference answer (optional)z1Provide a ground-truth answer to compare against.�zUpload documents�multiple)r�r�r�r�r�r�)r��318file_count�319file_typeszRun comparison�primary)�variantz## Answer comparisonzanswer-section)�elem_classeszanswer-cardz(### Full-context answer (entire uploads))�320show_labelz'### RAG answer (with retrieved context)zRetrieved context)r�zEvaluation metricszWhat do these metrics mean?)r�)�fn�inputs�outputs�__main__)r��(r)r�)�)`r�r�rkr-rPr9r�r��collectionsr�dataclassesrr�typingrrrr	r321rr�gradio�gr�dotenvr
�ImportErrorr��chromadb.configrr�rr �compiler3r&r%rr,r0rSr1r7r>r(rYrcrxr�r�r�r�r�r�r�r�r�rrrr)�dictrFrKrcrgrnr�r�r�r}r�r��METRIC_DETAILS_MD�322CUSTOM_CSSr��Blocks�demo�Markdown�HTML�Row�Textbox�question_input�reference_input�File�document_input�Button�323run_button�Group�full_context_output�324rag_output�context_output�metrics_output�	Accordion�clickr�launchrrr�<module>r+s|��0���	��	����(�G�G�G���"�325��(��
������326������2�:�:�&�'��*���	��3�������a��*C�U�!K�L��*�3�*�4��9�*�L�s�L�t�C�y�L�%�W�%�W�%��%�%�8�E�?�%�8�E�?�%�u�%�;�3�;�4��S��>�;�89�X�9�3�9�S�9�Z^�_d�Ze�9�x�8�H�#5��3��]`��jn�ot�ju��$�X�h�%7�$�H�S�M�$�]b�cf�hl�cl�]m�$�4�3��������TX�Y^�T_��.�3��c��c��:�x��1��e�D��N�D�QT�I�<U�6V��.'��'�����0�'�!�'��	'�6	D��	D��	D�GY�	D�^f�gl�^m�	D�!I��!I� ��%��1�!I����!I����	!I�327�!I�HG�5�G��G��G��	9����9����9�!��%��1�9��	9�328329�#�u�*��9�xO�X�e�_�O��%��O�dh�O�mp�O�1��1���!�1����1��u�o�	1�330�1��
1��1�!�1�!��%��1�1�331�1�h!����!�C�!�C��C�D�C�S�C�L	%�x���	%�3�	%�,�s�,�H�S�M�,�c�,�3�3�lL332�L333�^�l���C�J�J�7��H334��H335��H336�
�H337��	H338�339�H340�341�#�s�(�^�
H342�V��"-�343�`uK�C�uK�#�uK�p�R�Y�Y�5��O�SW��B�K�K�2�3��B�G�G�g�j�\��*�+��B�K�K�	i��344345�����#�����M��346��347%�"�*�*�/�K��348��
349��R�W�W� ��H��N�����+�Y�?�J��B�K�K�&�'�	����.�/�	0�
�R�X�X�M�?�
3��B�K�K�B�C�"-�"�+�+��"?��4��R�X�X�M�?�
3��B�K�K�A�B�$�����6�J�4�	3501�!�R�[�[�':�;�N��R�W�W�#7�8�N�	����3�%�	@�����%�&�351A��������@�$�j�.�.�Q���WP�d�z���K�K�M���]$��������H��H���z"352���,4�
3��4�
3��	3531�	0��354A�	@��QP�O�s��Q.�355Q=�AS�+R
�9AS�R3�'&R�
R3�)&R'�R3�AS�R?�,#S�.Q:�9Q:�=	R356�	R357�
R	�S�R$�R3�'R0�,R3�3R<	�8S�?S	�S�S
VTdevelops/rag-vs-full-context-comparator · CoolFace