moghit/word2vec-embeddings

1from joblib import load
2import spacy 
3from gensim.models import KeyedVectors
4import numpy as np
5import string
6
7
8model = load('logistic_model.joblib')
9wv = KeyedVectors.load("word2vec-google-news-100-pca-fp16.model")
10nlp_small = spacy.load("en_core_web_sm")
11stop_words_small = nlp_small.Defaults.stop_words
12
13
14def tokenizer(sentence):
15    punctuations = string.punctuation
16                   
17    doc = nlp_small(sentence)  # -> tokenization
18
19    mytokens = []
20    for word in doc:
21        lemma = word.lemma_.lower().strip() # removing the whitespace & lemmatization and lowercuase
22        mytokens.append(lemma)
23
24    filtered_tokens = []
25    for word in mytokens:
26        if word not in stop_words_small and word not in punctuations:
27            filtered_tokens.append(word)
28
29    return filtered_tokens
30
31def avg_vector(sent):
32    vector_size = wv.vectors.shape[1]
33    average_vector = np.zeros(vector_size)
34    valid_word_count = 0
35
36    for word in sent:
37        if word in wv:
38            average_vector += wv[word]
39            valid_word_count += 1
40
41    if valid_word_count > 0:
42        average_vector /= valid_word_count
43    return average_vector
44
45def predict_(sentence):
46    words = tokenizer(sentence)
47    average_vector = avg_vector(words)
48    return model.predict([average_vector])
49
50
51print ("Prediction module loaded successfully.")
52
53#uv pip install numpy==1.26.4
54#uv pip install spacy gensim