ValadisCERTH/NaturalLanguageModule_complete
0
1 2import spacy3import re4import nltk5from nltk.corpus import wordnet6import numpy as np7 8from sklearn.metrics.pairwise import cosine_similarity9 10spacy.cli.download("en_core_web_sm")11 12# use spacy small because in that way we are closer to a BOW model which is the one we care in our case since we just compare words13nlp_comparatives = spacy.load('en_core_web_sm', disable=["parser", "ner"])14 15 16def find_comptives_symbols(sentence):17 """18 Capture unique cases of symbols like <, >, =, <=, >= and ==19 If more than one symbol exists, return []20 """21 22 # symbols regex pattern23 pattern = r"(?<![<=>])<=|>=|==|(?<![<=>])<|>|(?<!<)=|=(?![<=>])"24 25 matches = re.findall(pattern, sentence)26 27 # if len(matches) > 1:28 # return []29 30 found_symbols = []31 for matching in matches:32 # found_symbols.append({'comparative': ['symbol', matching]})33 found_symbols.append({'comparative': matching})34 35 return found_symbols36 37 38def find_comptives_straight_patterns(sentence):39 """40 Function to identivy mentions of comparatives. The form is "comparative adverbs/adjectives followed by than", "words like more/less followed by than", "equal to"41 """42 43 doc = nlp_comparatives(sentence)44 comparatives = []45 46 for token in doc:47 48 # find mentions of "equal" followed by "to"49 if token.text.lower() == "equal":50 51 next_token = token.nbor()52 53 if next_token.text.lower() == "to":54 55 prev_token = token.nbor(-1)56 57 if prev_token.pos_ == "NOUN":58 59 # comparatives.append({'comparative': ["equal to", "="]})60 comparatives.append({'comparative': "="})61 62 # find mentions of "more"/"less" followed by "than"63 elif token.text.lower() in ["more", "less"]:64 65 next_token = token.nbor()66 67 if next_token.text.lower() == "than":68 69 prev_token = token.nbor(-1)70 71 if token.text.lower() == 'more':72 73 # comparatives.append({'comparative': [token.text + " " + next_token.text, '>']})74 comparatives.append({'comparative': '>'})75 76 elif token.text.lower() == 'less':77 78 # comparatives.append({'comparative': [token.text + " " + next_token.text, '<']})79 comparatives.append({'comparative': '<'})80 81 # find mentions of comparative adjectives or comparative adverbs followed by "than"82 elif token.tag_ == "JJR" or token.tag_ == "RBR":83 84 next_token = token.nbor()85 86 if next_token.text.lower() == "than" and next_token.nbor().pos_ != "NOUN":87 88 # check if the token is a synonym of "bigger"89 90 # retrieve a set of synonyms for the concepts of 'big' and 'bigger'91 big_synonyms = set(wordnet.synsets('big') + wordnet.synsets('large') + wordnet.synsets('great') + wordnet.synsets('huge') + wordnet.synsets('enormous') + wordnet.synsets('heavy') + wordnet.synsets(92 'strong') + wordnet.synsets('enormous') + wordnet.synsets('massive') + wordnet.synsets(93 'immense') + wordnet.synsets('substantial'))94 bigger_synonyms = set(wordnet.synsets('bigger') + wordnet.synsets('larger') + wordnet.synsets(95 'greater') + wordnet.synsets('higher') + wordnet.synsets('taller') + wordnet.synsets(96 'heavier') + wordnet.synsets('stronger'))97 98 bigger_related_words = big_synonyms.union(bigger_synonyms)99 100 bigger_rel_words = [word.name().split('.')[0] for word in bigger_related_words]101 102 flag_bigger = 0103 104 if token.text.lower() in bigger_rel_words:105 106 flag_bigger = 1107 # comparatives.append({'comparative': [token.text + " " + next_token.text, '>']})108 comparatives.append({'comparative': '>'})109 110 # if no synonym of bigger was found, check for smaller synsets111 if flag_bigger==0:112 113 # retrieve a set of synonyms for the concepts of 'small' and 'smaller'114 small_synonyms = set(wordnet.synsets('small') + wordnet.synsets('little') + wordnet.synsets(115 'tiny') + wordnet.synsets('petite') + wordnet.synsets('miniature') + wordnet.synsets(116 'slight') + wordnet.synsets('meager') + wordnet.synsets('inconsequential') + wordnet.synsets(117 'minor'))118 smaller_synonyms = set(wordnet.synsets('smaller') + wordnet.synsets('lesser') + wordnet.synsets(119 'lower') + wordnet.synsets('shorter') + wordnet.synsets('lighter') + wordnet.synsets('weaker'))120 121 smaller_related_words = small_synonyms.union(smaller_synonyms)122 smaller_rel_words = [word.name().split('.')[0] for word in smaller_related_words]123 124 if token.text.lower() in smaller_rel_words:125 126 flag_bigger = 0127 # comparatives.append({'comparative': [token.text + " " + next_token.text, '<']})128 comparatives.append({'comparative': '<'})129 130 return comparatives131 132 133# helper functions for 'identify_pattern_bigger_smaller'134 135# helper functions for 'identify_pattern_bigger_smaller'136 137def identify_comparison(sentence):138 """139 Capture patterns of 'word-er' followed by 'than' (e.g. 'better than', 'lesser than', etc)140 """141 142 pattern = r'\b(\w+er than)\b'143 matches = re.findall(pattern, sentence)144 145 if matches:146 return matches147 else:148 return 0149 150 151def find_more_than_reference(sentence):152 """153 Capture patterns of 'more' followed by 'word' followed by 'than' (e.g. more advanced than)154 """155 156 pattern = r"(more) (\w+) than"157 matches = re.findall(pattern, sentence)158 159 if matches:160 return [' '.join(match) for match in matches]161 else:162 return 0163 164 165def find_less_than_reference(sentence):166 """167 Capture patterns of 'less' followed by 'word' followed by 'than' (e.g. less advanced than)168 """169 170 pattern = r"(less) (\w+) than"171 matches = re.findall(pattern, sentence)172 173 if matches:174 return [' '.join(match) for match in matches]175 else:176 return 0177 178 179def is_related_to(word, target_word):180 """181 Returns True if the input 'word' is semantically related to the 'target_word', otherwise False.182 """183 184 target_synsets = set(wordnet.synsets(target_word))185 word_synsets = set(wordnet.synsets(word))186 187 if word_synsets.intersection(target_synsets):188 return True189 else:190 return False191 192 193def is_related_to_bigger(word):194 """195 Returns True if the input 'word' is semantically related to the concept 'bigger', otherwise False.196 """197 198 if word.lower() == "more" or word.lower().startswith("more "):199 return True200 201 # retrieve a set of synonyms for the concepts of 'big' and 'bigger'202 big_synonyms = set(wordnet.synsets('big') + wordnet.synsets('large') + wordnet.synsets('great') + wordnet.synsets(203 'huge') + wordnet.synsets('enormous') + wordnet.synsets('heavy') + wordnet.synsets('strong') + wordnet.synsets(204 'enormous') + wordnet.synsets('massive') + wordnet.synsets('immense') + wordnet.synsets('substantial'))205 bigger_synonyms = set(206 wordnet.synsets('bigger') + wordnet.synsets('larger') + wordnet.synsets('greater') + wordnet.synsets(207 'higher') + wordnet.synsets('taller') + wordnet.synsets('heavier') + wordnet.synsets('stronger'))208 209 related_words = big_synonyms.union(bigger_synonyms)210 211 # Check if the input word is semantically related to any of those 'big'/'bigger' synonyms212 for related_word in related_words:213 if is_related_to(word, related_word.name().split('.')[0]):214 return True215 return False216 217 218def is_related_to_smaller(word):219 """220 Returns True if the input word is semantically related to the concept of 'smaller', otherwise False.221 """222 if word.lower() == "less" or word.lower().startswith("less "):223 return True224 225 # retrieve a set of synonyms for the concepts of 'small' and 'smaller'226 small_synonyms = set(227 wordnet.synsets('small') + wordnet.synsets('little') + wordnet.synsets('tiny') + wordnet.synsets(228 'petite') + wordnet.synsets('miniature') + wordnet.synsets('slight') + wordnet.synsets(229 'meager') + wordnet.synsets('inconsequential') + wordnet.synsets('minor'))230 smaller_synonyms = set(231 wordnet.synsets('smaller') + wordnet.synsets('lesser') + wordnet.synsets('lower') + wordnet.synsets(232 'shorter') + wordnet.synsets('lighter') + wordnet.synsets('weaker'))233 234 related_words = small_synonyms.union(smaller_synonyms)235 236 # Check if the input word is semantically related to any of those 'small'/'smaller' synonyms237 for related_word in related_words:238 if is_related_to(word, related_word.name().split('.')[0]):239 return True240 return False241 242 243def identify_bigger_smaller_advanced(sentence):244 """245 This is a complementary function to capture cases of 'words ending with -er' followed by 'than' and cases of 'more'/'less' followed 'word' followed by 'than'246 """247 248 # pattern 'words ending with -er' followed by 'than' (pattern1)249 word_er_than = identify_comparison(sentence)250 251 # pattern 'more' followed 'word' followed by 'than' (pattern2)252 more_word_than = find_more_than_reference(sentence)253 254 # pattern 'less' followed 'word' followed by 'than' (pattern3)255 less_word_than = find_less_than_reference(sentence)256 257 bigger_list = []258 smaller_list = []259 260 # in case any pattern is captured261 if word_er_than or more_word_than or less_word_than:262 263 # in case of pattern1264 if word_er_than:265 for word in word_er_than:266 267 # perform relevant substitutions268 target_word = word.replace("than", "").strip()269 270 # examine if it is a bigger-related or smaller-related word271 bigger_word = is_related_to_bigger(target_word)272 smaller_word = is_related_to_smaller(target_word)273 274 # case of bigger word275 if bigger_word and not smaller_word:276 # bigger_list.append({"comparative": [word, ">"]})277 bigger_list.append({"comparative": ">"})278 279 # case of smaller word280 elif smaller_word and not bigger_word:281 # smaller_list.append({"comparative": [word, "<"]})282 smaller_list.append({"comparative": "<"})283 284 # in case of pattern2285 if more_word_than:286 for word in more_word_than:287 288 # perform relevant substitutions289 target_word = word.replace("than", "").replace("more", "").strip()290 291 # in this case it must be a bigger-related word292 bigger_word = is_related_to_bigger(target_word)293 294 # case of bigger word295 if bigger_word:296 # bigger_list.append({"comparative": [word, ">"]})297 bigger_list.append({"comparative": ">"})298 299 # in case of pattern3300 if less_word_than:301 for word in less_word_than:302 303 # perform relevant substitutions304 target_word = word.replace("than", "").replace("less", "").strip()305 306 # in this case it must be a lesser-related word307 lesser_word = is_related_to_smaller(target_word)308 309 # case of bigger word310 if lesser_word:311 # smaller_list.append({"comparative": [word, "<"]})312 smaller_list.append({"comparative": "<"})313 314 # return the combined list315 return bigger_list + smaller_list316 317 318def find_equal_to_comptives_ngrams(sentence):319 """320 This function takes a sentence as input and returns a reference phrase based on semantic similarity using n-grams.321 The possible reference phrases are provided as a list.322 """323 324 # This is a reference list for the concept of 'equal to'. It has many references to perform on them the semantic similarity examination325 possible_references = ["equal to", "same as", "similar to", "identical to", "equivalent to", "tantamount to",326 "corresponding to", "comparable to", "akin to", "commensurate with", "in line with",327 "on a par with", "indistinguishable from", "corresponding with", "congruent with"]328 329 # that thershold is enough empirically330 max_similarity = 0.85331 332 possible_reference_list = []333 334 # parse with the spacy model (embeddings each of the references)335 embedding_references = []336 for reference in possible_references:337 reference_doc = nlp_comparatives(reference)338 embedding_references.append(reference_doc)339 340 # Check 2-grams, 3-grams, and 4-grams341 for n in range(2, 5):342 343 # get n-grams344 sentence_ngrams = list(nltk.ngrams(sentence.split(), n))345 346 for sent_ngram in sentence_ngrams:347 sentence_ngram_str = ' '.join(sent_ngram)348 sentence_ngram_doc = nlp_comparatives(sentence_ngram_str)349 350 for emb_ref in embedding_references:351 similarity = sentence_ngram_doc.similarity(emb_ref)352 353 if similarity >= max_similarity:354 # possible_reference_list.append({'comparative': [sentence_ngram_str, "="]})355 possible_reference_list.append({'comparative': "="})356 break357 358 # if we have found a possible refernce that is similar enough with an n-gram of the input sentence, return the comparative '=', otherwise return 0359 if possible_reference_list:360 return possible_reference_list361 else:362 return []363 364 365def single_verb_comptives(sentence):366 """367 This function takes a sentence and identifies any mention of bigger than, smaller than, equal to, expressed368 as single-word verb. It uses wordnet synsets to examine for synonyms and antonyms369 """370 371 # base references372 bigger_references_sg = ["surpass", "exceed", "outstrip", "outdo", "outrank", "transcend"]373 lesser_references_sg = ["subside", "depreciate", "curtail"]374 equal_references_sg = ["match", "equal", "agree", "comply"]375 376 doc = nlp_comparatives(sentence)377 378 bigger_list = []379 smaller_list = []380 equal_list = []381 382 # search for all verbs and examine their lemma with all the synonyms of each of the previous references. Assign a label accordingly383 for token in doc:384 385 # first examine for 1-1 pair matching and 1-1 lemma pair matching386 if token.text in bigger_references_sg or token.lemma_ in bigger_references_sg:387 # bigger_list.append({'comparative': [token.text, ">"]})388 bigger_list.append({'comparative': ">"})389 break390 391 elif token.text in lesser_references_sg or token.lemma_ in lesser_references_sg:392 # smaller_list.append({'comparative': [token.text, "<"]})393 smaller_list.append({'comparative': "<"})394 break395 396 elif token.text in equal_references_sg or token.lemma_ in equal_references_sg:397 # equal_list.append({'comparative': [token.text, "="]})398 equal_list.append({'comparative': "="})399 break400 401 else:402 403 # if not, then try with synonyms only for verbs404 if token.pos_ == "VERB":405 406 for lemma in token.lemma_.split('|'):407 synsets = wordnet.synsets(lemma, pos='v')408 409 for syn in synsets:410 if any(lemma in bigger_references_sg for lemma in syn.lemma_names()):411 # bigger_list.append({'comparative': [token.text, ">"]})412 bigger_list.append({'comparative': ">"})413 break414 415 elif any(lemma in lesser_references_sg for lemma in syn.lemma_names()):416 # smaller_list.append({'comparative': [token.text, "<"]})417 smaller_list.append({'comparative': "<"})418 break419 420 elif any(lemma in equal_references_sg for lemma in syn.lemma_names()):421 # equal_list.append({'comparative': [token.text, "="]})422 equal_list.append({'comparative': "="})423 break424 425 final_list = bigger_list + smaller_list + equal_list426 427 if final_list:428 return final_list429 else:430 return []431 432 433# helper functions for 'identify_multi_word_verbs'434 435# Define multi-word verb lists436bigger_list = ["is a cut above", "is ahead of", "is superior to", "is greater than", "is a class apart"]437smaller_list = ["fall behind", "is inferior to", "is smaller than", "lag behind", "trail behind", "fall short", "fall beneath"]438equal_list = ["is in line with", "is equal to", "is on a par with", "is the same as", "is comparable to", "is in sync with", "is in harmony with", "is in step with", "is in tune with", "is in accord with", "is consistent with", "is consonant with", "is equivalent to"]439 440# Calculate embeddings of multi-word verbs441bigger_embeddings = [np.mean([token.vector for token in nlp_comparatives(verb)], axis=0) for verb in bigger_list]442smaller_embeddings = [np.mean([token.vector for token in nlp_comparatives(verb)], axis=0) for verb in smaller_list]443equal_embeddings = [np.mean([token.vector for token in nlp_comparatives(verb)], axis=0) for verb in equal_list]444 445 446# Define function to check if n-gram is in multi-word verb list447def check_list(ngram, verb_list):448 """449 This is a function to check if n-gram is in multi-word verb list450 """451 452 if ngram in verb_list:453 return True454 else:455 return False456 457 458def cosine_sim(a, b):459 """460 This is a function to calculate cosine similarity461 """462 463 return cosine_similarity(a.reshape(1,-1), b.reshape(1,-1))[0][0]464 465 466# we examine the n-grams reversely and any time we find a match, we "delete" that match, so that lesser ngrams will not be matched \467# (e.g. is on a par with, would also match afterwords on a par with, par with, etc)468 469def multiword_verb_comptives(sentence):470 """471 This function takes a sentence and identifies any mention of bigger than, smaller than, equal to, expressed472 as multi-word verbs. Based on three refernces lists it performs initially a simple string comparison with each473 of their elements and the ngrams of the input sentence. If there is no match there, it performs the same procedure474 with cosine similarity to identify any similar ngrams.475 """476 477 # Split sentence into tokens478 tokens = sentence.split()479 480 # Initialize variables to store label and max similarity481 label = None482 max_sim = 0483 484 # these lists are used to capture any possible reference485 bigger_l = []486 smaller_l = []487 equal_l = []488 489 # Define set to keep track of matched ngrams490 matched_ngrams = set()491 492 # Iterate through n-grams of sentence, starting with the largest n-grams493 for n in range(5, 0, -1):494 for i in range(len(tokens)-n+1):495 ngram = ' '.join(tokens[i:i+n])496 497 # Skip ngrams that have already been matched498 if ngram in matched_ngrams:499 continue500 501 # Check if n-gram is in bigger_list502 if check_list(ngram, bigger_list):503 matched_ngrams.update(set(ngram.split()))504 # bigger_l.append({"comparative": [ngram, '>']})505 bigger_l.append({"comparative": '>'})506 507 # Check if n-gram is in smaller_list508 elif check_list(ngram, smaller_list):509 matched_ngrams.update(set(ngram.split()))510 # smaller_l.append({"comparative":[ngram, '<']})511 smaller_l.append({"comparative":'<'})512 513 # Check if n-gram is in equal_list514 elif check_list(ngram, equal_list):515 matched_ngrams.update(set(ngram.split()))516 # equal_l.append({"comparative":[ngram, '=']})517 equal_l.append({"comparative": '='})518 519 # Check if n-gram is similar to any verb in bigger_list using pre-calculated embeddings520 else:521 ngram_emb = np.mean([token.vector for token in nlp_comparatives(ngram)], axis=0)522 similarities_bigger = [cosine_sim(ngram_emb, verb_emb) for verb_emb in bigger_embeddings]523 max_sim_bigger = max(similarities_bigger)524 525 # Check if n-gram is similar to any verb in smaller_list using pre-calculated embeddings526 similarities_smaller = [cosine_sim(ngram_emb, verb_emb) for verb_emb in smaller_embeddings]527 max_sim_smaller = max(similarities_smaller)528 529 # Check if n-gram is similar to any verb in equal_list using pre-calculated embeddings530 similarities_equal = [cosine_sim(ngram_emb, verb_emb) for verb_emb in equal_embeddings]531 max_sim_equal = max(similarities_equal)532 533 # Determine the maximum similarity value among the three lists534 if max_sim_bigger > max_sim_smaller and max_sim_bigger > max_sim_equal and max_sim_bigger > max_sim:535 max_sim = max_sim_bigger536 if max_sim > 0.9:537 matched_ngrams.update(set(ngram.split()))538 # bigger_l.append({"comparative":[ngram, '>']})539 bigger_l.append({"comparative":'>'})540 else:541 matched_ngrams.update(set(ngram.split()))542 543 544 elif max_sim_smaller > max_sim_bigger and max_sim_smaller > max_sim_equal and max_sim_smaller > max_sim:545 max_sim = max_sim_smaller546 if max_sim > 0.9:547 matched_ngrams.update(set(ngram.split()))548 # smaller_l.append({"comparative":[ngram, '<']})549 smaller_l.append({"comparative":'<'})550 else:551 matched_ngrams.update(set(ngram.split()))552 553 554 elif max_sim_equal > max_sim_bigger and max_sim_equal > max_sim_smaller and max_sim_equal > max_sim:555 max_sim = max_sim_smaller556 if max_sim > 0.9:557 matched_ngrams.update(set(ngram.split()))558 # equal_l.append({"comparative":[ngram, '=']})559 equal_l.append({"comparative":'='})560 else:561 matched_ngrams.update(set(ngram.split()))562 563 564 return bigger_l + smaller_l + equal_l565 566 567def identify_double_symbol_comparisons(sentence):568 """569 Identifies comparison phrases in a given sentence.570 Returns a list of matched phrases and their corresponding operators.571 """572 573 comparison_phrases = [574 ["less than or equal to", "less or equal to", "smaller than or equal to",575 "smaller or equal to", "lower than or equal to", "lower or equal to",576 "inferior to or equal to", "inferior or equal to", "lesser or equal to"],577 ["greater than or equal to", "greater or equal to", "more than or equal to",578 "more or equal to", "higher than or equal to", "higher or equal to",579 "above than or equal to", "above or equal to", "larger than or equal to",580 "larger or equal to", "superior to or equal to", "superior or equal to",581 "bigger or equal to", "over or equal to", "surpassing or equal to"]582 ]583 584 operators = {585 "less than or equal to": "<=",586 "less or equal to": "<=",587 "smaller than or equal to": "<=",588 "smaller or equal to": "<=",589 "lower than or equal to": "<=",590 "lower or equal to": "<=",591 "inferior to or equal to": "<=",592 "inferior or equal to": "<=",593 "greater than or equal to": ">=",594 "greater or equal to": ">=",595 "more than or equal to": ">=",596 "more or equal to": ">=",597 "higher than or equal to": ">=",598 "higher or equal to": ">=",599 "above than or equal to": ">=",600 "above or equal to": ">=",601 "larger than or equal to": ">=",602 "larger or equal to": ">=",603 "superior to or equal to": ">=",604 "superior or equal to": ">=",605 "bigger or equal to": ">=",606 "over or equal to": ">=",607 "lesser or equal to": "<=",608 "surpassing or equal to": ">="609 }610 611 found_phrases = []612 found_operators = []613 for variations in comparison_phrases:614 pattern = r"\b(" + "|".join([re.escape(v) for v in variations]) + r")\b"615 matches = re.findall(pattern, sentence, re.IGNORECASE)616 if matches:617 for match in matches:618 found_phrases.append(match)619 found_operators.append(operators[match])620 621 comparative_list = [{'comparative': []}]622 for phrase, operator in zip(found_phrases, found_operators):623 # comparative_list[0]['comparative'].append(phrase)624 comparative_list[0]['comparative'].append((phrase, operator))625 626 final_comptives_list = [{'comparative': comparative_list[0]['comparative'][i:i + 2]} for i in range(0, len(comparative_list[0]['comparative']), 2)]627 628 final_clean_list = []629 for item in final_comptives_list:630 for value in item['comparative']:631 final_clean_list.append({'comparative': value})632 633 return final_clean_list634 635 636def check_substrings(lst):637 """638 This function checks all the elements of a list and if any substring exist in any other element it returns a list of tuples639 where the first element is the substring and the second the string that contains the substring640 """641 substring_tuples = []642 for i, comp1 in enumerate(lst):643 for j, comp2 in enumerate(lst):644 if i == j:645 continue646 if comp1['comparative'][0] in comp2['comparative'][0]:647 substring_tuples.append((comp1, comp2))648 return substring_tuples649 650 651def identify_comparatives(sentence):652 """653 This function combines the results of all the aforementioned techniques (simple and advance) to identify bigger than, smaller than, equal to patterns654 """655 656 # first identify the double symbols (<= >= ==)657 identify_double_symbols_initial = identify_double_symbol_comparisons(sentence)658 659 # this is because (for example) bigger than is a subset of bigger or equal than (and it returns conflicts)660 if identify_double_symbols_initial:661 for elem in identify_double_symbols_initial:662 sentence = sentence.replace(elem['comparative'][0], " ")663 664 identify_double_symbols = []665 666 for item in identify_double_symbols_initial:667 for k, v in item.items():668 if isinstance(v, tuple):669 item[k] = v[1]670 identify_double_symbols.append(item)671 672 # Identify straightforward patterns673 straight_comptives = find_comptives_straight_patterns(sentence)674 675 # Identify advanced bigger/smaller comparativesunknown_error676 bigger_smaller_comparatives = identify_bigger_smaller_advanced(sentence)677 678 # Identify advanced equal-to comparatives679 equal_to_comparatives = find_equal_to_comptives_ngrams(sentence)680 681 single_verb = single_verb_comptives(sentence)682 683 multi_verb = multiword_verb_comptives(sentence)684 685 # return all the patterns that were captured686 comparatives = straight_comptives + bigger_smaller_comparatives + equal_to_comparatives + single_verb + multi_verb + identify_double_symbols687 688 # since those different techniques might capture similar patterns, we keep only unique references. More precisely689 # we discard any unique reference while also any reference thay may exist as a substring on any other reference690 691 # sort the list by length of the comparatives, in descending order692 comparatives.sort(key=lambda item: len(item['comparative'][0]), reverse=False)693 694 unique_comparatives = {}695 for i, item in enumerate(comparatives):696 comparative = item['comparative'][0]697 # check if the comparative is already in the dictionary or a substring/similar string of an existing comparative698 is_unique = True699 for existing_comp in unique_comparatives:700 if (comparative in existing_comp) or (existing_comp in comparative):701 is_unique = False702 break703 if is_unique:704 unique_comparatives[comparative] = item705 elif i == len(comparatives) - 1:706 # if it's the last item and it's not unique, replace the first unique item in the list with this item707 for j, existing_item in enumerate(unique_comparatives.values()):708 if (existing_item['comparative'][0] in comparative) or (comparative in existing_item['comparative'][0]):709 unique_comparatives.pop(list(unique_comparatives.keys())[j])710 unique_comparatives[comparative] = item711 break712 713 unique_output = list(unique_comparatives.values())714 715 clean_unique_output = []716 717 # this snippet is to handle the extra cases of smaller than or equal to etc718 # in case a reference of eg "smaller than" is found by the previous modules, while also a reference of "smaller than or equal to"719 # then the snippet checks whether the "smaller than" reference exists only as a substring of "smaller than or equal to" or if it720 # also exists as a seperate, standalone reference on the initial sentence (in which case it is kept, otherwise it is dismissed)721 if len(unique_output) > 1:722 list_of_tuples = check_substrings(unique_output)723 724 for elem in list_of_tuples:725 dupl_sent = sentence726 dupl_sent = dupl_sent.replace(elem[1]['comparative'][0], " ")727 728 clean_unique_output.append(elem[1])729 730 if elem[0]['comparative'][0] in dupl_sent:731 clean_unique_output.append(elem[0])732 733 if clean_unique_output:734 return clean_unique_output735 736 else:737 return unique_output738 739 740def comparatives_binding(sentence):741 #742 try:743 comparative_symbols = find_comptives_symbols(sentence)744 comparative_mentions = identify_comparatives(sentence)745 746 # starting with the symbols, if one was captured747 if len(comparative_symbols) == 1:748 749 # if the rest of the functions are empty (meaning that there are no other references)750 if len(comparative_mentions) == 0:751 return comparative_symbols[0]752 753 else:754 return (0, "COMPARATIVES", "more_comparatives_mentions")755 756 # in case that there is no symbol757 elif len(comparative_symbols) == 0:758 759 # we need only one mention of comparatives760 if len(comparative_mentions) == 1:761 return comparative_mentions[0]762 763 # case of no comparative mentions764 elif len(comparative_mentions) == 0:765 return (0, "COMPARATIVES", "no_comparatives")766 767 # case of no more than one comparative mentions768 else:769 return (0, "COMPARATIVES", "more_comparatives_mentions")770 771 # case of multiple symbol references772 else:773 return (0, "COMPARATIVES", "more_symbol_comparatives")774 775 except:776 return (0, "COMPARATIVES", "unknown_error")