patellae/Sentimental_Analysis_From_Scratch_Using_Stacked_LSTMs

1import torch2from tokenizer import tokenizer3from torchtext.vocab import Vectors4 5max_len = 646 7glove_file = 'glove.6B.100d.txt'8 9# Load the pre-trained GloVe embeddings from the local file10glove = Vectors(glove_file)11 12def embed(sentence):13  14  tok = tokenizer(sentence.lower()) # tokenization15 16 17  if len(tok) >= max_len:  # truncation18    tok = tok[1:max_len + 1]19  20  pad = max_len - len(tok)21 22  output = []23    24  for i in range(len(tok)):25    if not(tok[i].text in glove.stoi):26        pad = pad+127 28    else:29      word_embedding = glove.vectors[glove.stoi[tok[i].text]]30      output.append(word_embedding)31  32  for i in range(pad): # padding33      output.append(torch.zeros(100))34  35  return torch.stack(output)36 37