sadit/TextEmbeddings

1import h5py as h52import json3import pandas as pd4import torch5from sentence_transformers import SentenceTransformer6 7#torch.set_num_threads(32)8 9D = pd.read_parquet("ccnews/pair")10# title, article11modelname = 'sentence-transformers/all-MiniLM-L6-v2'12model = SentenceTransformer(modelname)13 14print(D.columns)15 16#print("embeddings title")17#embeddings = model.encode(D.title)18#19#with h5.File("ccnews.h5", "w") as f:20#    f["title"] = embeddings21#    f.attrs["model"] = modelname22 23print("embeddings article")24embeddings = model.encode(D.article)25 26with h5.File("ccnews.h5", "a") as f:27    f["article"] = embeddings28