project-sign-language/Sign_language

1import spacy2import pickle3from nltk.corpus import wordnet4 5 6def load_spacy_values(model = "en_core_web_md", filepath_docs_spacy = 'dict_spacy_object.pkl'):7    '''8    Loads a spaCy language model and a dictionary of spaCy Doc objects from a pickle file.9 10    Parameters11    ----------12    model : str13        The name or local path of the spaCy model to be loaded for processing text. 14        For example, "en_core_web_sm" or a custom model path.15 16    filepath_docs_spacy : str17        The path to the pickle file containing a dictionary where the keys are tokens 18        (strings) and the values are the corresponding serialized spaCy Doc objects.19 20    Returns21    -------22    nlp : spacy.language.Language23        The loaded spaCy language model.24 25    dict_docs_spacy : dict26        A dictionary where the keys are tokens (strings) and the values are spaCy Doc 27        objects reconstructed from the serialized bytes stored in the pickle file.28    '''29    30    # ---- Load the spaCy NLP model31    #32    nlp = spacy.load(model)33    34    # ---- Load pickle file and reconstruct the dictionary with tokens as keys and spaCy Doc objects as values35    #36    with open(filepath_docs_spacy, 'rb') as file:37        dict_docs_spacy_bytes = pickle.load(file)38    39    dict_docs_spacy = {key: spacy.tokens.Doc(nlp.vocab).from_bytes(doc_bytes) for key, doc_bytes in dict_docs_spacy_bytes.items()}40    41    return nlp, dict_docs_spacy42 43 44def find_antonyms(word):45    '''46    Generate a set of all the antonyms of a given word47 48    Parameters49    ----------50    word : str51        The word that we want to find the antonyms52 53    Returns54    -------55    antonyms : set of str56        A set of all the antonym detected using nltk and WordNet57    '''58    59    antonyms = set()60 61    # ---- Load all the set of synonyms of the word recorded from wordnet62    #63    syn_set = wordnet.synsets(word)64 65    # ---- Loop over each set of synonyms66    #67    for syn in syn_set:68        # ---- Loop over each synonym69        #70        for lemma in syn.lemmas():71            # ---- Add antonyms of the synonyms to the antonyms set72            #73            if lemma.antonyms():74                antonyms.add(lemma.antonyms()[0].name())75 76    return antonyms77 78 79def find_synonyms(word, model, dict_embedding, list_2000_tokens):80    '''81    Finds the most similar token to a given word.82 83    Parameters84    ----------85    word : str86        The word that we want to find the most similar word87 88    model : spacy.language.Language89        spaCy language model to use for the detection of the synonym90    91    dict_embedding: dict92        A dictionary where the keys are tokens (str) and the values are spaCy Doc objects93 94    list_2000_tokens : list of str95        A list of 2000 tokens against which the gloss will be checked.96    97    Returns98    -------99    most_similar_token : str100        The most similar token to the given word 101    '''102 103    # ---- Skip synonym detection if the word is already in the list_2000_token104    #105    if word in list_2000_tokens:106        return word107    else:108        # ---- Remove antonyms of the given word of the list_2000_tokens (a word and an antonym might be similar in embedding representation)109        #110        antonyms = find_antonyms(word)111        list_2000_tokens_less_antonyms = [token for token in list_2000_tokens if token not in antonyms]112 113        # ---- Generate a list of tuple (token, similarities values between the embedding of the given word and the embedding of each token of the list_2000_tokens)114        #115        word_embedding = model(word)116        similarities=[]117    118        for token in list_2000_tokens_less_antonyms:119            similarities.append((token, dict_embedding.get(token).similarity(word_embedding)))120 121        # ---- Extract the most similar token of the list122        #123        most_similar_token = sorted(similarities, key=lambda item: -item[1])[0][0]124 125        return most_similar_token