Teradata/multilingual-e5-base

1import sys2import teradataml as tdml3from tabulate import tabulate4 5import json6 7 8with open('conversion_config.json') as json_file:9    conversion_config = json.load(json_file)10 11 12    model_id = conversion_config["model_id"]13    number_of_generated_embeddings = conversion_config["number_of_generated_embeddings"]14    precision_to_filename_map = conversion_config["precision_to_filename_map"]15    16    host = sys.argv[1]17    username = sys.argv[2]18    password = sys.argv[3]19    20    print("Setting up connection to teradata...")21    tdml.create_context(host = host, username = username, password = password)22    print("Done\n\n")23    24    25    print("Deploying tokenizer...")26    try:27        tdml.db_drop_table('tokenizer_table')28    except:29        print("Can't drop tokenizers table - it's not existing")30    tdml.save_byom('tokenizer',31                  'tokenizer.json',32                  'tokenizer_table')33    print("Done\n\n")34    35    print("Testing models...")36    try:37        tdml.db_drop_table('model_table')38    except:39        print("Can't drop models table - it's not existing")40    41    for precision, file_name in precision_to_filename_map.items():42        print(f"Deploying {precision} model...")43        tdml.save_byom(precision,44                      file_name,45                      'model_table')46        print(f"Model {precision} is deployed\n")47    48        print(f"Calculating embeddings with {precision} model...")49        try:50            tdml.db_drop_table('emails_embeddings_store')51        except:52            print("Can't drop embeddings table - it's not existing")53        54        tdml.execute_sql(f"""55            create volatile table emails_embeddings_store as (56                select 57                    *58            from mldb.ONNXEmbeddings(59                    on emails.emails as InputTable60                    on (select * from model_table where model_id = '{precision}') as ModelTable DIMENSION61                    on (select model as tokenizer from tokenizer_table where model_id = 'tokenizer') as TokenizerTable DIMENSION62               63                    using64                        Accumulate('id', 'txt') 65                        ModelOutputTensor('sentence_embedding')66                        EnableMemoryCheck('false')67                        OutputFormat('FLOAT32({number_of_generated_embeddings})')68                        OverwriteCachedModel('true')69                ) a 70        ) with data on commit preserve rows71        72        """)73        print("Embeddings calculated")74        print(f"Testing semantic search with cosine similiarity on the output of the model with precision '{precision}'...")75        tdf_embeddings_store = tdml.DataFrame('emails_embeddings_store')76        tdf_embeddings_store_tgt = tdf_embeddings_store[tdf_embeddings_store.id == 3]77        78        tdf_embeddings_store_ref = tdf_embeddings_store[tdf_embeddings_store.id != 3]79        80        cos_sim_pd = tdml.DataFrame.from_query(f"""81            SELECT 82                dt.target_id, 83                dt.reference_id,84                e_tgt.txt as target_txt,85                e_ref.txt as reference_txt,86                (1.0 - dt.distance) as similiarity 87            FROM88                TD_VECTORDISTANCE (89                    ON ({tdf_embeddings_store_tgt.show_query()}) AS TargetTable90                    ON ({tdf_embeddings_store_ref.show_query()}) AS ReferenceTable DIMENSION91                    USING92                        TargetIDColumn('id')93                        TargetFeatureColumns('[emb_0:emb_{number_of_generated_embeddings - 1}]')94                        RefIDColumn('id')95                        RefFeatureColumns('[emb_0:emb_{number_of_generated_embeddings - 1}]')96                        DistanceMeasure('cosine')97                        topk(3)98                ) AS dt99            JOIN emails.emails e_tgt on e_tgt.id = dt.target_id100            JOIN emails.emails e_ref on e_ref.id = dt.reference_id;101            """).to_pandas()102        print(tabulate(cos_sim_pd, headers='keys', tablefmt='fancy_grid'))103        print("Done\n\n")104    105    106    tdml.remove_context()