eagle0504/quantized-retrieval

1 2from datasets import load_dataset3import numpy as np4from faiss import IndexBinaryFlat, write_index_binary5from sentence_transformers.util import quantize_embeddings6 7dataset = load_dataset("mixedbread-ai/wikipedia-2023-11-embed-en-pre-1", split="train")8embeddings = np.array(dataset["emb"], dtype=np.float32)9 10ubinary_embeddings = quantize_embeddings(embeddings, "ubinary")11index = IndexBinaryFlat(1024)12index.add(ubinary_embeddings)13write_index_binary(index, "wikipedia_ubinary_faiss_1m.index")14