ValadisCERTH/NaturalLanguageModule_complete
0
1import spacy2 3from geopy.geocoders import Nominatim4import geonamescache5import pycountry6 7from geotext import GeoText8 9import re10 11spacy.cli.download("en_core_web_lg")12 13# Load the spacy model with GloVe embeddings14nlp = spacy.load("en_core_web_lg")15 16# Load valid city names from geonamescache17gc = geonamescache.GeonamesCache()18 19# There is a bug with geonamescache where some countries exist as cities (e.g. albania)20# So initially we delete any country reference from the cities21 22# Get a list of all country names23original_countries = set(country['name'] for country in gc.get_countries().values())24 25# Get a list of all the original city names26original_cities = set(city['name'] for city in gc.get_cities().values())27 28# Get a list of all country names that appear as city names29country_names = set(30 country['name'] for country in gc.get_countries().values() if country['name'] not in original_cities)31 32# We also add these two cases because they have been asked by SERCO33country_names.add("Guinea Bissau")34country_names.add("Guinea bissau")35country_names.add("guinea Bissau")36country_names.add("guinea bissau")37country_names.add("Timor Leste")38country_names.add("Timor leste")39country_names.add("timor Leste")40country_names.add("timor leste")41country_names.add("UAE")42country_names.add("uae")43country_names.add("Uae")44country_names.add("Uk")45country_names.add("uK")46country_names.add("uk")47country_names.add("USa")48country_names.add("Usa")49country_names.add("usa")50country_names.add("uSa")51country_names.add("usA")52country_names.add("uSA")53country_names.add("Palestine")54 55# Get a list of all city names, excluding country names56city_names = set(city['name'] for city in gc.get_cities().values() if city['name'] not in original_countries)57 58city_names.add("Puebla de sanabria")59 60 61def flatten(lst):62 """63 Define a helper function to flatten the list recursively64 """65 66 for item in lst:67 if isinstance(item, list):68 yield from flatten(item)69 else:70 yield item71 72 73def is_country(reference):74 """75 Check if a given reference is a valid country name76 """77 try:78 # Check if the reference is a valid city name from the first geoparse library79 if reference in country_names:80 return True81 82 else:83 # if not then use the pycountry library to verify if an input is a country84 country = pycountry.countries.search_fuzzy(reference)[0]85 86 temp_country_names = []87 88 if country:89 if hasattr(country, 'name') or hasattr(country, 'official_name') or hasattr(country, 'common_name'):90 91 if hasattr(country, 'official_name'):92 temp_country_names.append(country.official_name.lower())93 if hasattr(country, 'name'):94 temp_country_names.append(country.name.lower())95 if hasattr(country, 'common_name'):96 temp_country_names.append(country.common_name.lower())97 if any(reference.lower()==elem for elem in temp_country_names):98 return True99 100 return False101 102 except LookupError:103 return False104 105 106def is_city(reference):107 """108 Check if a given reference is a valid city name109 """110 111 reference = reference.replace("x$x", "").strip()112 113 # Check if the reference is a valid city name114 if reference in city_names:115 return True116 117 # Load the Nomatim (open street maps) api118 geolocator = Nominatim(user_agent="certh_serco_validate_city_app")119 location = geolocator.geocode(reference, language="en", timeout=10)120 121 # If a reference is identified as a 'city', 'town', or 'village', then it is indeed a city122 if location.raw['type'] in ['city', 'town', 'village']:123 return True124 125 # If a reference is identified as 'administrative' (e.g. administrative area),126 # then we further examine if the retrieved info is a single token (meaning a country) or a series of tokens (meaning a city)127 # that condition takes place to separate some cases where small cities were identified as administrative areas128 elif location.raw['type'] == 'administrative':129 130 if len(location.raw['display_name'].split(",")) > 1:131 return True132 133 return False134 135 136def validate_locations(locations):137 """138 Validate that the identified references are indeed a Country and a City139 """140 141 validated_loc = []142 143 for location in locations:144 145 # validate whether it is a country146 if is_country(location):147 validated_loc.append((location, 'country'))148 149 # validate whether it is a city150 elif is_city(location):151 validated_loc.append((location, 'city'))152 153 else:154 # Check if the location is a multi-word name155 words = location.split()156 if len(words) > 1:157 158 # Try to find the country or city name among the words159 for i in range(len(words)):160 name = ' '.join(words[i:])161 162 if is_country(name):163 validated_loc.append((name, 'country'))164 break165 166 elif is_city(name):167 validated_loc.append((name, 'city'))168 break169 170 return validated_loc171 172 173def identify_loc_ner(sentence):174 """175 Identify all the geopolitical and location entities with the spacy tool176 """177 178 doc = nlp(sentence)179 180 ner_locations = []181 182 # GPE and LOC are the labels for location entities in spaCy183 for ent in doc.ents:184 if ent.label_ in ['GPE', 'LOC']:185 186 if len(ent.text.split()) > 1:187 ner_locations.append(ent.text)188 else:189 for token in ent:190 if token.ent_type_ == 'GPE':191 ner_locations.append(ent.text)192 break193 194 return ner_locations195 196 197def identify_loc_geoparselibs(sentence):198 """199 Identify cities and countries with 3 different geoparsing libraries200 """201 202 geoparse_locations = []203 204 # Geoparsing library 1205 206 # Load geonames cache to check if a city name is valid207 gc = geonamescache.GeonamesCache()208 209 # Get a list of many countries/cities210 countries = gc.get_countries()211 cities = gc.get_cities()212 213 city_names = [city['name'] for city in cities.values()]214 country_names = [country['name'] for country in countries.values()]215 216 # if any word sequence in our sentence is one of those countries/cities identify it217 words = sentence.split()218 for i in range(len(words)):219 for j in range(i + 1, len(words) + 1):220 word_seq = ' '.join(words[i:j])221 if word_seq in city_names or word_seq in country_names:222 geoparse_locations.append(word_seq)223 224 # Geoparsing library 2225 226 # similarly with the pycountry library227 for country in pycountry.countries:228 if country.name in sentence:229 geoparse_locations.append(country.name)230 231 # Geoparsing library 3232 233 # similarly with the geotext library234 places = GeoText(sentence)235 cities = list(places.cities)236 countries = list(places.countries)237 238 if cities:239 geoparse_locations += cities240 if countries:241 geoparse_locations += countries242 243 return (geoparse_locations, countries, cities)244 245 246def identify_loc_regex(sentence):247 """248 Identify cities and countries with regular expression matching249 """250 251 regex_locations = []252 253 # Country and cities references can be preceded by 'in', 'from' or 'of'254 pattern = r"\b(in|from|of)\b\s([\w\s]+)"255 additional_refs = re.findall(pattern, sentence)256 257 for match in additional_refs:258 regex_locations.append(match[1])259 260 return regex_locations261 262 263 264def multiple_country_city_identifications_solve(country_city_dict):265 """266 This is a function to solve the appearance of multiple identification of countries and cities.267 It checks all the elements of the input dictionary and if any smaller length element exists as a substring inside268 a bigger length element of it, it deletes the smaller size one. In that sense, a dictionary of the sort269 {'city': ['Port moresby', 'Port'], 'country': ['Guinea', 'Papua new guinea']} will be converted into270 {'city': ['Port moresby'], 'country': ['Papua new guinea']}.271 272 The reason for that function, is because such type of incosistencies were identified during country/city identification,273 propably relevant to the geoparsing libraries in use274 """275 276 try:277 278 country_flag = False279 city_flag = False280 281 # to avoid examining any element in any case, we validate that both a country and a city exist282 # on the input dictionary and that they are of length more than one (which is the target case for us)283 if 'country' in country_city_dict:284 if len(country_city_dict['country']) > 1:285 country_flag = True286 287 if 'city' in country_city_dict:288 if len(country_city_dict['city']) > 1:289 city_flag = True290 291 # at first cope with country multiple iterative references292 if country_flag:293 294 # Sort the countries by length, longest first295 country_city_dict['country'].sort(key=lambda x: len(x), reverse=True)296 297 # Create a new list of countries that don't contain any substrings298 cleaned_countries = []299 for i in range(len(country_city_dict['country'])):300 is_substring = False301 for j in range(len(cleaned_countries)):302 if country_city_dict['country'][i].lower().find(cleaned_countries[j].lower()) != -1:303 # If the i-th country is a substring of an already-cleaned country, skip it304 is_substring = True305 break306 if not is_substring:307 cleaned_countries.append(country_city_dict['country'][i])308 309 # Replace the original list of countries with the cleaned one310 country_city_dict['country'] = cleaned_countries311 312 # Create a new list of countries that are not substrings of other countries313 final_countries = []314 for i in range(len(country_city_dict['country'])):315 is_superstring = False316 for j in range(len(country_city_dict['country'])):317 if i == j:318 continue319 if country_city_dict['country'][j].lower().find(country_city_dict['country'][i].lower()) != -1:320 # If the i-th country is a substring of a different country, skip it321 is_superstring = True322 break323 if not is_superstring:324 final_countries.append(country_city_dict['country'][i])325 326 # Replace the original list of countries with the final one327 country_city_dict['country'] = final_countries328 329 # then cope with city multiple iterative references330 if city_flag:331 332 # Sort the cities by length, longest first333 country_city_dict['city'].sort(key=lambda x: len(x), reverse=True)334 335 # Create a new list of cities that don't contain any substrings336 cleaned_cities = []337 for i in range(len(country_city_dict['city'])):338 is_substring = False339 for j in range(len(cleaned_cities)):340 if country_city_dict['city'][i].lower().find(cleaned_cities[j].lower()) != -1:341 # If the i-th city is a substring of an already-cleaned city, skip it342 is_substring = True343 break344 if not is_substring:345 cleaned_cities.append(country_city_dict['city'][i])346 347 # Replace the original list of cities with the cleaned one348 country_city_dict['city'] = cleaned_cities349 350 # Create a new list of cities that are not substrings of other cities351 final_cities = []352 for i in range(len(country_city_dict['city'])):353 is_superstring = False354 for j in range(len(country_city_dict['city'])):355 if i == j:356 continue357 if country_city_dict['city'][j].lower().find(country_city_dict['city'][i].lower()) != -1:358 # If the i-th city is a substring of a different city, skip it359 is_superstring = True360 break361 if not is_superstring:362 final_cities.append(country_city_dict['city'][i])363 364 # Replace the original list of cities with the final one365 country_city_dict['city'] = final_cities366 367 # return the final dictionary368 if country_city_dict:369 return country_city_dict370 371 except:372 return (0, "LOCATION", "unknown_error")373 374 375def helper_resolve_cities(sentence, locations):376 """377 Verify that the city captured does not belong to the capture country. If so delete it, unless there is also a second reference on the original sentence378 (which might be the case of a city with a similar name/substring of a country)379 """380 381 if 'country' in locations and 'city' in locations:382 383 # Check if any city names are also present in the corresponding country name384 for country in locations['country']:385 for city in locations['city']:386 387 if city.lower() in country.lower():388 # If the city name is found in the country name, check how many times it appears in the sentence389 city_count = len(re.findall(city, sentence, re.IGNORECASE))390 if city_count == 1:391 # If the city appears only once, remove it from the locations dictionary392 locations['city'] = [c for c in locations['city'] if c != city]393 394 return locations395 396 397def helper_delete_city_reference(locations):398 """399 If the 'city' reference was captured by mistake by the system, delete it, unless it belongs to the cities that should contain it (e.g. Mexico city)400 """401 402 city_cities = ["Adamstown City", "Alexander City", "Angeles City", "Antipolo City", "Arizona City", "Arkansas City",403 "Ashley City", "Atlantic City", "Bacolod City", "Bacoor City", "Bago City", "Baguio City",404 "Baker City", "Baltimore City", "Batangas City", "Bay City", "Belgrade City", "Belize City",405 "Benin City", "Big Bear City", "Bossier City", "Boulder City", "Brazil City", "Bridge City",406 "Brigham City", "Brighton City", "Bristol City", "Buckeye City", "Bullhead City", "Butuan City",407 "Cabanatuan City", "Calamba City", "Calbayog City", "California City", "Caloocan City",408 "Calumet City", "Candon City", "Canon City", "Carcar City", "Carson City", "Castries City",409 "Cathedral City", "Cavite City", "Cebu City", "Cedar City", "Central Falls City", "Century City",410 "Cestos City", "City Bell", "City Terrace", "City of Balikpapan", "City of Calamba",411 "City of Gold Coast", "City of Industry", "City of Isabela", "City of Orange", "City of Paranaque",412 "City of Parramatta", "City of Shoalhaven", "Collier City", "Columbia City", "Commerce City",413 "Cooper City", "Cotabato City", "Crescent City", "Crescent City North", "Culver City",414 "Dagupan City", "Dale City", "Dali City", "Daly City", "Danao City", "Dasmariñas City", "Davao City",415 "De Forest City", "Del City", "Dhaka City", "Dipolog City", "Dodge City", "Dumaguete City",416 "El Centro City", "Elizabeth City", "Elk City", "Ellicott City", "Emeryville City", "Fernley City",417 "Florida City", "Forest City", "Forrest City", "Foster City", "Freeport City", "Garden City",418 "Gdynia City", "General Santos City", "General Trias City", "Gloucester City", "Granite City",419 "Green City", "Grove City", "Guatemala City", "Haines City", "Haltom City", "Harbor City",420 "Havre City", "Highland City", "Ho Chi Minh City", "Holiday City", "Horizon City", "Hyderabad City",421 "Iligan City", "Iloilo City", "Imus City", "Iowa City", "Iriga City", "Isabela City", "Jacinto City",422 "James City County", "Jefferson City", "Jersey City", "Jhang City", "Jincheng City", "Johnson City",423 "Junction City", "Kaiyuan City", "Kansas City", "King City", "Kingman City", "Kingston City",424 "Koror City", "Kowloon City", "Kuwait City", "Lake City", "Lake Havasu City", "Laoag City",425 "Lapu-Lapu City", "Las Pinas City", "Las Piñas City", "League City", "Legazpi City", "Leisure City",426 "Lenoir City", "Ligao City", "Lincoln City", "Linyi City", "Lipa City", "Loma Linda City",427 "Lucena City", "Madrid City", "Makati City", "Malabon City", "Mandaluyong City", "Mandaue City",428 "Manukau City", "Marawi City", "Marikina City", "Maryland City", "Mason City", "McKee City",429 "Mexico City", "Mexico City Beach", "Michigan City", "Midwest City", "Mineral City", "Missouri City",430 "Morehead City", "Morgan City", "Muntinlupa City", "Naga City", "Nagasaki City", "National City",431 "Navotas City", "Nay Pyi Taw City", "Nevada City", "New City", "New York City", "Norwich City",432 "Ocean City", "Oil City", "Oklahoma City", "Olongapo City", "Orange City", "Oregon City",433 "Ozamiz City", "Pagadian City", "Palayan City", "Palm City", "Panabo City", "Panama City",434 "Panama City", "Panama City Beach", "Parañaque City", "Park City", "Pasay City", "Peachtree City",435 "Pearl City", "Pell City", "Phenix City", "Plant City", "Ponca City", "Port Augusta City",436 "Port Pirie City", "Quad Cities", "Quartzsite City", "Quebec City", "Quezon City", "Quezon City",437 "Rainbow City", "Rapid City", "Red City", "Redwood City", "Richmond City", "Rio Grande City",438 "Roxas City", "Royse City", "Salt Lake City", "Salt Lake City", "Samal City", "San Carlos City",439 "San Carlos City", "San Fernando City", "San Fernando City", "San Fernando City", "San Jose City",440 "San Jose City", "San Juan City", "San Juan City", "San Pedro City", "Santa Rosa City",441 "Science City of Munoz", "Shelby City", "Sialkot City", "Silver City", "Sioux City",442 "South Lake Tahoe City", "South Sioux City", "Studio City", "Suisun City", "Summit Park City",443 "Sun City", "Sun City Center", "Sun City West", "Sun City West", "Suva City", "Tabaco City",444 "Tacloban City", "Tagbilaran City", "Taguig City", "Tagum City", "Talisay City", "Tanauan City",445 "Tarlac City", "Tauranga City", "Tayabas City", "Temple City", "Texas City", "Thomas City",446 "Tipp City", "Toledo City", "Traverse City", "Trece Martires City", "Tuba City", "Union City",447 "Universal City", "University City", "Upper Hutt City", "Valencia City", "Valenzuela City",448 "Vatican City", "Vatican City", "Ventnor City", "Webb City", "Wellington City", "Welwyn Garden City",449 "West Valley City", "White City", "Yazoo City", "Yuba City", "Zamboanga City"]450 451 if 'city' in locations:452 for city in locations['city']:453 if 'city' in city:454 if not city in city_cities:455 city = city.replace("city", "")456 457 elif 'City' in city:458 if not city in city_cities:459 city = city.replace("City", "")460 461 locations['city'] = city462 463 # Convert city values to a list464 if isinstance(locations['city'], str):465 locations['city'] = [locations['city']]466 467 return locations468 469 470def helper_delete_country_reference(locations):471 """472 If the 'country' reference was captured by mistake by the system and exists in a city name, delete it473 """474 475 country_city_same = ["djibouti", "guatemala", "mexico", "panama", "san marino", "singapore", "vatican"]476 477 if 'country' in locations:478 for i, country in enumerate(locations['country']):479 480 if country.lower() not in country_city_same:481 split_country = country.lower().split()482 483 if 'city' in locations:484 for j, city in enumerate(locations['city']):485 split_city = city.lower().split()486 487 for substring in split_country:488 if substring in split_city:489 split_city.remove(substring)490 new_city = ' '.join(split_city)491 locations['city'][j] = new_city.strip()492 493 return locations494 495 496def identify_locations(sentence):497 """498 Identify all the possible Country and City references in the given sentence, using different approaches in a hybrid manner499 """500 501 locations = []502 extra_serco_countries = False503 504 try:505 # # # this is because there were cases were a city followed by comma was not understood by the system506 507 sentence = sentence.replace(",", " x$x ")508 509 # Serco wanted to also handle these two cases without the symbol "-". The only way to do that is by hardcoding it510 if "Timor Leste" in sentence:511 extra_serco_countries = True512 locations.append("Timor Leste")513 514 if "Guinea Bissau" in sentence:515 extra_serco_countries = True516 locations.append("Guinea Bissau")517 518 # ner519 locations.append(identify_loc_ner(sentence))520 521 # geoparse libs522 geoparse_list, countries, cities = identify_loc_geoparselibs(sentence)523 locations.append(geoparse_list)524 525 # flatten the geoparse list526 locations_flat_1 = list(flatten(locations))527 528 # regex529 locations_flat_1.append(identify_loc_regex(sentence))530 531 # flatten the regex list532 locations_flat_2 = list(flatten(locations))533 534 # remove duplicates while also taking under consideration capitalization (e.g. a reference of italy should be valid, while also a reference of Italy and italy)535 # Lowercase the words and get their unique references using set()536 loc_unique = set([loc.lower() for loc in locations_flat_2])537 538 # Create a new list of locations with initial capitalization, removing duplicates539 loc_capitalization = list(540 set([loc.capitalize() if loc.lower() in loc_unique else loc.lower() for loc in locations_flat_2]))541 542 # That calculation checks whether there are substrings contained in another string. E.g. for the case of [timor leste, timor], it should remove "timor"543 if extra_serco_countries:544 loc_capitalization_cp = loc_capitalization.copy()545 for i, loc1 in enumerate(loc_capitalization):546 for j, loc2 in enumerate(loc_capitalization):547 if i != j and loc1 in loc2:548 loc_capitalization_cp.remove(loc1)549 break550 551 loc_capitalization = loc_capitalization_cp552 553 # validate that indeed each one of the countries/cities are indeed countries/cities554 validated_locations = validate_locations(loc_capitalization)555 556 # create a proper dictionary with country/city tags and the relevant entries as a result557 loc_dict = {}558 for location, loc_type in validated_locations:559 if loc_type not in loc_dict:560 loc_dict[loc_type] = []561 loc_dict[loc_type].append(location)562 563 # bring sentence on previous form564 sentence = sentence.replace(" x$x ", ",")565 566 # cope with cases of iterative country or city reference due to geoparse lib issues567 locations_dict = multiple_country_city_identifications_solve(loc_dict)568 569 if locations_dict == None:570 return (0, "LOCATION", "no_country")571 # return {'city':[], 'country':[]}572 573 else:574 # conditions for multiple references575 # it is mandatory that a country will exist576 if 'country' in locations_dict:577 578 # if a city exists579 if 'city' in locations_dict:580 581 resolved_dict = helper_resolve_cities(sentence, locations_dict)582 583 # we accept one country and one city584 if len(resolved_dict['country']) == 1 and len(resolved_dict['city']) == 1:585 586 # capitalize because there may be cases that it will return 'italy'587 resolved_dict['country'][0] = resolved_dict['country'][0].capitalize()588 589 # there were some cases that the 'x$x' was not removed590 for key, values in resolved_dict.items():591 for i, value in enumerate(values):592 if 'x$x' in value:593 values[i] = value.replace('x$x', '')594 595 delete_city = helper_delete_city_reference(resolved_dict)596 597 return helper_delete_country_reference(delete_city)598 599 600 # we can accept an absence of city but a country is always mandatory601 elif len(resolved_dict['country']) == 1 and len(resolved_dict['city']) == 0:602 603 resolved_dict['country'][0] = resolved_dict['country'][0].capitalize()604 resolved_dict['city'] = ['0']605 606 # there were some cases that the 'x$x' was not removed607 for key, values in resolved_dict.items():608 for i, value in enumerate(values):609 if 'x$x' in value:610 values[i] = value.replace('x$x', '')611 612 delete_city = helper_delete_city_reference(resolved_dict)613 614 return helper_delete_country_reference(delete_city)615 616 # error if more than one country or city617 else:618 return (0, "LOCATION", "more_city_or_country")619 620 621 # if a city does not exist622 else:623 # we only accept for one country624 if len(locations_dict['country']) == 1:625 626 locations_dict['country'][0] = locations_dict['country'][0].capitalize()627 628 # there were some cases that the 'x$x' was not removed629 for key, values in locations_dict.items():630 for i, value in enumerate(values):631 if 'x$x' in value:632 values[i] = value.replace('x$x', '')633 634 resolved_cities = helper_resolve_cities(sentence, locations_dict)635 delete_city = helper_delete_city_reference(resolved_cities)636 637 help_city = helper_delete_country_reference(delete_city)638 639 if not 'city' in help_city:640 help_city['city'] = [0]641 642 return help_city643 644 # error if more than one country645 else:646 return (0, "LOCATION", "more_country")647 648 # error if no country is referred649 else:650 return (0, "LOCATION", "no_country")651 652 except:653 # handle the exception if any errors occur while identifying a country/city654 return (0, "LOCATION", "unknown_error")