huggingface/text-data-filtering
37
1import re2 3import numpy as np4 5import fasttext6 7import sentencepiece8import kenlm9 10import pathlib11 12from languages_id import langs_id13from parameters_filtering import parameters_filtering14from normalization import normalization15from stopwords import stopwords16from flagged_words import flagged_words17 18 19class LoadParameters:20 @staticmethod21 def load_parameters(lang_dataset_id):22 if lang_dataset_id in parameters_filtering:23 param = parameters_filtering[lang_dataset_id]24 else:25 param = parameters_filtering["default"]26 return param27 28 @staticmethod29 def load_stopwords(lang_dataset_id):30 stopwords_lang_id = langs_id.loc[31 langs_id["dataset_id"] == lang_dataset_id, "stopwords_id"32 ].iloc[0]33 if stopwords_lang_id:34 stopwords_lang = set(stopwords[stopwords_lang_id])35 else:36 stopwords_lang = None37 return stopwords_lang38 39 @staticmethod40 def load_flagged_words(lang_dataset_id):41 flagged_words_lang_id = langs_id.loc[42 langs_id["dataset_id"] == lang_dataset_id, "flagged_words_id"43 ].iloc[0]44 if flagged_words_lang_id:45 flagged_words_lang = set(flagged_words[flagged_words_lang_id])46 else:47 flagged_words_lang = None48 return flagged_words_lang49 50 @staticmethod51 def load_model_lang_id(lang_dataset_id, path_fasttext_model):52 fasttext_lang_id = langs_id.loc[53 langs_id["dataset_id"] == lang_dataset_id, "fasttext_id"54 ].iloc[0]55 if fasttext_lang_id:56 model_lang_id = fasttext.load_model(path_fasttext_model)57 else:58 model_lang_id = None59 return model_lang_id60 61 @staticmethod62 def load_sentencepiece_model(lang_dataset_id, path_sentencepiece_model):63 sentencepiece_lang_id = langs_id.loc[64 langs_id["dataset_id"] == lang_dataset_id, "sentencepiece_id"65 ].iloc[0]66 if sentencepiece_lang_id:67 sentencepiece_model = sentencepiece.SentencePieceProcessor()68 sentencepiece_model.load(path_sentencepiece_model)69 else:70 sentencepiece_model = None71 return sentencepiece_model72 73 @staticmethod74 def load_kenlm_model(lang_dataset_id, path_kenlm_model):75 kenlm_lang_id = langs_id.loc[76 langs_id["dataset_id"] == lang_dataset_id, "kenlm_id"77 ].iloc[0]78 if kenlm_lang_id:79 kenlm_model = kenlm.Model(path_kenlm_model)80 else:81 kenlm_model = None82 return kenlm_model83 84 85class ModifyingDocuments:86 @staticmethod87 def remove_empty_el_from_list(list_):88 return [el for el in list_ if el]89 90 @staticmethod91 def remove_non_printing_characters(document, non_printing_characters_re):92 return non_printing_characters_re.sub("", document)93 94 @staticmethod95 def uniform_whitespace(96 document,97 whitespace=[98 " ",99 " ",100 " ",101 " ",102 " ",103 " ",104 " ",105 " ",106 " ",107 " ",108 "",109 "",110 ],111 ):112 """There are different whitespace characters."""113 whitespace = set(whitespace)114 document = "".join(115 [char if char not in whitespace else " " for char in document]116 )117 return document118 119 @staticmethod120 def replace_digits_with_zeros(document, digits_re):121 return digits_re.sub("0", document)122 123 @staticmethod124 def replace_unicode_punctuation(document, unicode_punctuation):125 return "".join(unicode_punctuation.get(c, c) for c in document)126 127 @staticmethod128 def normalization(129 document,130 remove_non_printing_characters,131 strip,132 lower_case,133 uniform_whitespace,134 replace_digits_with_zeros,135 replace_unicode_punctuation,136 non_printing_characters_re=normalization["non_printing_characters_re"],137 digits_re=normalization["digits_re"],138 unicode_punctuation=normalization["unicode_punctuation"],139 ):140 if remove_non_printing_characters:141 document = ModifyingDocuments.remove_non_printing_characters(142 document, non_printing_characters_re143 )144 if strip:145 document = document.strip()146 if not document:147 return document148 if lower_case:149 document = document.lower()150 if uniform_whitespace:151 document = ModifyingDocuments.uniform_whitespace(document)152 if replace_digits_with_zeros:153 document = ModifyingDocuments.replace_digits_with_zeros(document, digits_re)154 if replace_unicode_punctuation:155 document = ModifyingDocuments.replace_unicode_punctuation(156 document, unicode_punctuation157 )158 return document159 160 @staticmethod161 def tokenization(document, sentencepiece_model, join_on_whitespace):162 document_tokenized = sentencepiece_model.encode_as_pieces(document)163 if join_on_whitespace:164 document_tokenized = " ".join(document_tokenized)165 return document_tokenized166 167 @staticmethod168 def split_on_whitespace(169 document,170 new_line=False,171 tab=False,172 ):173 """This method also removes concatenated spaces."""174 sep = [" "] + new_line * ["\n"] + tab * ["\t"]175 sep = "|".join(sep)176 split_document = re.split(sep, document)177 split_document = ModifyingDocuments.remove_empty_el_from_list(split_document)178 return split_document179 180 @staticmethod181 def strip(document, strip_characters):182 """Way faster than document.strip(strip_characters)183 since strip_characters is now a set instead of a str,184 and it contains a lot of elements (all the emojis)."""185 if not document:186 return document187 beg_ind = 0188 end_ind = len(document)189 for i in range(len(document)):190 if document[i] in strip_characters:191 beg_ind += 1192 else:193 break194 for i in range(1, len(document) + 1):195 if document[-i] in strip_characters:196 end_ind -= 1197 else:198 break199 document_stripped = document[beg_ind:end_ind]200 return document_stripped201 202 @staticmethod203 def get_words_from_document(204 document, sentencepiece_model_tok, lower_case, strip_characters205 ):206 """Get words from a document. Non reversible since the document207 is split on multiple characters, words are stripped of208 special characters and characters are converted to lower case.209 Useful to compute ratios, like the stopwords ratio."""210 if sentencepiece_model_tok:211 document_normalized = ModifyingDocuments.normalization(212 document=document,213 remove_non_printing_characters=True,214 strip=True,215 lower_case=True,216 uniform_whitespace=True,217 replace_digits_with_zeros=True,218 replace_unicode_punctuation=True,219 )220 words = ModifyingDocuments.tokenization(221 document_normalized, sentencepiece_model_tok, join_on_whitespace=False222 )223 else:224 words = ModifyingDocuments.split_on_whitespace(225 document, new_line=True, tab=True226 )227 if lower_case:228 words = [word.lower() for word in words]229 if strip_characters:230 words = [ModifyingDocuments.strip(word, strip_characters) for word in words]231 words = ModifyingDocuments.remove_empty_el_from_list(words)232 return words233 234 @staticmethod235 def words_augmentation(words, group_size, join_char):236 """Augment words, especially for Chinese (without a space between words)237 and Vietnamese (with a space between syllables)."""238 augmentation = [239 join_char.join(words[i : i + group_size])240 for i in range(len(words) - group_size + 1)241 ]242 return augmentation243 244 @staticmethod245 def split_on_newline_tab_whitespace(document):246 """First split on "\n", then on "\t", then on " "."""247 sentences = document.split("\n")248 sentences = [sentence.split("\t") for sentence in sentences]249 sentences = [250 [251 ModifyingDocuments.split_on_whitespace(subsentence)252 for subsentence in sentence253 ]254 for sentence in sentences255 ]256 return sentences257 258 @staticmethod259 def merge_on_whitespace_tab_newline(sentences):260 """Invert the method split_on_newline_tab_whitespace.261 Removes concatenated separators."""262 sentences = [263 [" ".join(subsentence) for subsentence in sentence if subsentence]264 for sentence in sentences265 ]266 sentences = ["\t".join(sentence) for sentence in sentences if sentence]267 if not sentences:268 return ""269 document = "\n".join(sentences)270 return document271 272 @staticmethod273 def should_keep_word_with_incorrect_substrings(274 word, strip_characters, incorrect_word_substrings275 ):276 word = ModifyingDocuments.strip(word, strip_characters)277 should_keep = all(278 [(i_substr not in word) for i_substr in incorrect_word_substrings]279 )280 return should_keep281 282 @staticmethod283 def remove_words_with_incorrect_substrings(284 document,285 strip_characters,286 incorrect_word_substrings,287 ):288 sentences = ModifyingDocuments.split_on_newline_tab_whitespace(document)289 sentences = [290 [291 [292 word293 for word in subsentence294 if ModifyingDocuments.should_keep_word_with_incorrect_substrings(295 word, strip_characters, incorrect_word_substrings296 )297 ]298 for subsentence in sentence299 ]300 for sentence in sentences301 ]302 document = ModifyingDocuments.merge_on_whitespace_tab_newline(sentences)303 return document304 305 @staticmethod306 def should_keep_long_word(word, strip_characters, length_word_max_cutoff):307 """If the word is too long but it contains only one308 special character, it might be a concatenation of one word,309 a punctuation, and another word, with no space between them.310 In this case, we give the word a pass."""311 if len(word) <= length_word_max_cutoff:312 return True313 word = ModifyingDocuments.strip(word, strip_characters)314 if not word: # The word consisted only of strip characters315 return False316 if len(word) <= length_word_max_cutoff:317 return True318 return False319 320 def remove_long_words(321 document,322 strip_characters,323 length_word_max_cutoff,324 ):325 sentences = ModifyingDocuments.split_on_newline_tab_whitespace(document)326 sentences = [327 [328 [329 word330 for word in subsentence331 if ModifyingDocuments.should_keep_long_word(332 word,333 strip_characters,334 length_word_max_cutoff,335 )336 ]337 for subsentence in sentence338 ]339 for sentence in sentences340 ]341 document = ModifyingDocuments.merge_on_whitespace_tab_newline(sentences)342 return document343 344 @staticmethod345 def modifying_documents(346 document,347 cond_uniform_whitespace,348 cond_replace_unicode_punctuation,349 cond_remove_words_with_incorrect_substrings,350 strip_characters,351 incorrect_word_substrings,352 cond_remove_long_words,353 length_word_max_cutoff,354 ):355 document = ModifyingDocuments.normalization(356 document=document,357 remove_non_printing_characters=False,358 strip=True,359 lower_case=False,360 uniform_whitespace=cond_uniform_whitespace,361 replace_digits_with_zeros=False,362 replace_unicode_punctuation=cond_replace_unicode_punctuation,363 )364 if cond_remove_words_with_incorrect_substrings:365 document = ModifyingDocuments.remove_words_with_incorrect_substrings(366 document,367 strip_characters,368 incorrect_word_substrings,369 )370 if cond_remove_long_words:371 document = ModifyingDocuments.remove_long_words(372 document,373 strip_characters,374 length_word_max_cutoff,375 )376 return document377 378 379class FunctionDatasetModifyingDocuments:380 def __init__(self, lang_dataset_id):381 self.lang_dataset_id = lang_dataset_id382 self.param = LoadParameters.load_parameters(lang_dataset_id)383 384 def __call__(self, example):385 example["text"] = ModifyingDocuments.modifying_documents(386 document=example["text"],387 cond_uniform_whitespace=self.param["cond_uniform_whitespace"],388 cond_replace_unicode_punctuation=self.param[389 "cond_replace_unicode_punctuation"390 ],391 cond_remove_words_with_incorrect_substrings=self.param[392 "cond_remove_words_with_incorrect_substrings"393 ],394 strip_characters=self.param["strip_characters"],395 incorrect_word_substrings=self.param["incorrect_word_substrings"],396 cond_remove_long_words=self.param["cond_remove_long_words"],397 length_word_max_cutoff=self.param["length_word_max_cutoff"],398 )399 return example400 401 def __reduce__(self):402 return (self.__class__, (self.lang_dataset_id,))403 404 405class Filtering:406 @staticmethod407 def check_number_words(408 document,409 sentencepiece_model_tok,410 strip_characters,411 number_words_min_cutoff,412 number_words_max_cutoff,413 ):414 words = ModifyingDocuments.get_words_from_document(415 document,416 sentencepiece_model_tok,417 lower_case=False,418 strip_characters=strip_characters,419 )420 cond = (len(words) >= number_words_min_cutoff) and (421 len(words) <= number_words_max_cutoff422 )423 return cond424 425 @staticmethod426 def compute_character_repetition_ratio(document, character_repetition_length):427 def get_freq_character_ngrams(document, n):428 character_ngrams = [429 document[i : i + n] for i in range(len(document) - n + 1)430 ]431 freq_character_ngrams = {}432 for character_ngram in character_ngrams:433 freq_character_ngrams[character_ngram] = (434 freq_character_ngrams.get(character_ngram, 0) + 1435 )436 return freq_character_ngrams437 438 freq_character_ngrams = get_freq_character_ngrams(439 document, character_repetition_length440 )441 if len(freq_character_ngrams) == 0:442 return 0443 freq_character_ngrams = list(freq_character_ngrams.values())444 freq_character_ngrams = sorted(freq_character_ngrams, reverse=True)445 val_less_than_one = len([el for el in freq_character_ngrams if el > 1])446 num_rep_character_ngrams = min(447 int(np.sqrt(len(freq_character_ngrams))),448 len(freq_character_ngrams) - val_less_than_one,449 )450 character_repetition_ratio = sum(451 freq_character_ngrams[:num_rep_character_ngrams]452 ) / sum(freq_character_ngrams)453 return character_repetition_ratio454 455 @staticmethod456 def check_character_repetition_removal(457 document,458 character_repetition_length,459 character_repetition_max_cutoff,460 ):461 character_repetition_ratio = Filtering.compute_character_repetition_ratio(462 document, character_repetition_length463 )464 cond = character_repetition_ratio <= character_repetition_max_cutoff465 return cond466 467 @staticmethod468 def compute_word_repetition_ratio(469 document, sentencepiece_model_tok, strip_characters, word_repetition_length470 ):471 def get_freq_word_ngrams(472 document, sentencepiece_model_tok, strip_characters, n473 ):474 words = ModifyingDocuments.get_words_from_document(475 document,476 sentencepiece_model_tok,477 lower_case=True,478 strip_characters=strip_characters,479 )480 word_ngrams = [481 " ".join(words[i : i + n]) for i in range(len(words) - n + 1)482 ]483 freq_word_ngrams = {}484 for word_ngram in word_ngrams:485 freq_word_ngrams[word_ngram] = freq_word_ngrams.get(word_ngram, 0) + 1486 return freq_word_ngrams487 488 freq_word_ngrams = get_freq_word_ngrams(489 document, sentencepiece_model_tok, strip_characters, word_repetition_length490 )491 if len(freq_word_ngrams) == 0:492 return 0493 freq_word_ngrams = list(freq_word_ngrams.values())494 word_repetition_ratio = sum(495 freq for freq in freq_word_ngrams if freq > 1496 ) / sum(freq_word_ngrams)497 return word_repetition_ratio498 499 @staticmethod500 def check_word_repetition_removal(501 document,502 sentencepiece_model_tok,503 strip_characters,504 word_repetition_length,505 word_repetition_max_cutoff,506 ):507 word_repetition_ratio = Filtering.compute_word_repetition_ratio(508 document, sentencepiece_model_tok, strip_characters, word_repetition_length509 )510 cond = word_repetition_ratio <= word_repetition_max_cutoff511 return cond512 513 @staticmethod514 def compute_special_characters_ratio(document, special_characters):515 if len(document) == 0:516 return 0517 special_characters_ratio = len(518 [char for char in document if char in special_characters]519 ) / len(document)520 return special_characters_ratio521 522 @staticmethod523 def check_special_characters(524 document,525 special_characters,526 special_characters_max_cutoff,527 ):528 special_characters_ratio = Filtering.compute_special_characters_ratio(529 document, special_characters530 )531 cond = special_characters_ratio <= special_characters_max_cutoff532 return cond533 534 @staticmethod535 def compute_stopwords_ratio(536 document,537 sentencepiece_model_tok,538 strip_characters,539 cond_words_augmentation,540 words_augmentation_group_sizes,541 words_augmentation_join_char,542 stopwords,543 ):544 words = ModifyingDocuments.get_words_from_document(545 document,546 sentencepiece_model_tok,547 lower_case=True,548 strip_characters=strip_characters,549 )550 if not words:551 return 0552 augmentation = []553 if cond_words_augmentation:554 augmentation = [555 ModifyingDocuments.words_augmentation(556 words, group_size, words_augmentation_join_char557 )558 for group_size in words_augmentation_group_sizes559 ]560 augmentation = [word for augm in augmentation for word in augm]561 stopwords_ratio = len(562 [word for word in words + augmentation if word in stopwords]563 ) / len(words)564 if stopwords_ratio > 1.0:565 stopwords_ratio = 1.0566 return stopwords_ratio567 568 @staticmethod569 def check_stopwords(570 document,571 sentencepiece_model_tok,572 strip_characters,573 cond_words_augmentation,574 words_augmentation_group_sizes,575 words_augmentation_join_char,576 stopwords,577 stopwords_min_cutoff,578 ):579 cond = True580 if stopwords:581 stopwords_ratio = Filtering.compute_stopwords_ratio(582 document,583 sentencepiece_model_tok,584 strip_characters,585 cond_words_augmentation,586 words_augmentation_group_sizes,587 words_augmentation_join_char,588 stopwords,589 )590 cond = stopwords_ratio >= stopwords_min_cutoff591 return cond592 593 @staticmethod594 def compute_flagged_words_ratio(595 document,596 sentencepiece_model_tok,597 strip_characters,598 cond_words_augmentation,599 words_augmentation_group_sizes,600 words_augmentation_join_char,601 flagged_words,602 ):603 words = ModifyingDocuments.get_words_from_document(604 document,605 sentencepiece_model_tok,606 lower_case=True,607 strip_characters=strip_characters,608 )609 if not words:610 return 0611 augmentation = []612 if cond_words_augmentation:613 augmentation = [614 ModifyingDocuments.words_augmentation(615 words, group_size, words_augmentation_join_char616 )617 for group_size in words_augmentation_group_sizes618 ]619 augmentation = [word for augm in augmentation for word in augm]620 flagged_words_ratio = len(621 [word for word in words + augmentation if word in flagged_words]622 ) / len(words)623 if flagged_words_ratio > 1.0:624 flagged_words_ratio = 1.0625 return flagged_words_ratio626 627 @staticmethod628 def check_flagged_words(629 document,630 sentencepiece_model_tok,631 strip_characters,632 cond_words_augmentation,633 words_augmentation_group_sizes,634 words_augmentation_join_char,635 flagged_words,636 flagged_words_max_cutoff,637 ):638 cond = True639 if flagged_words:640 flagged_words_ratio = Filtering.compute_flagged_words_ratio(641 document,642 sentencepiece_model_tok,643 strip_characters,644 cond_words_augmentation,645 words_augmentation_group_sizes,646 words_augmentation_join_char,647 flagged_words,648 )649 cond = flagged_words_ratio <= flagged_words_max_cutoff650 return cond651 652 @staticmethod653 def compute_lang_id_pred_score(document, model_lang_id):654 document = document.lower().replace("\n", " ")655 pred = model_lang_id.predict(document)656 lang_pred_fasttext_id = pred[0][0].replace("__label__", "")657 score_pred = pred[1][0]658 lang_pred_dataset_id = langs_id.loc[659 langs_id["fasttext_id"] == lang_pred_fasttext_id, "dataset_id"660 ]661 if len(lang_pred_dataset_id) > 0:662 lang_pred_dataset_id = lang_pred_dataset_id.iloc[0]663 else:664 lang_pred_dataset_id = "unknown"665 return lang_pred_dataset_id, score_pred666 667 @staticmethod668 def check_lang_id(669 document,670 lang_dataset_id,671 model_lang_id,672 lang_id_min_cutoff,673 ):674 cond = True675 if model_lang_id:676 lang_pred_dataset_id, score_pred = Filtering.compute_lang_id_pred_score(677 document, model_lang_id678 )679 cond = (lang_pred_dataset_id == lang_dataset_id) and (680 score_pred >= lang_id_min_cutoff681 )682 return cond683 684 @staticmethod685 def compute_perplexity_score(document, sentencepiece_model, kenlm_model):686 document = ModifyingDocuments.normalization(687 document=document,688 remove_non_printing_characters=True,689 strip=True,690 lower_case=False,691 uniform_whitespace=True,692 replace_digits_with_zeros=True,693 replace_unicode_punctuation=True,694 )695 document = ModifyingDocuments.tokenization(696 document, sentencepiece_model, join_on_whitespace=True697 )698 doc_log_score, doc_length = 0, 0699 for line in document.split("\n"):700 log_score = kenlm_model.score(line)701 length = len(line.split()) + 1702 doc_log_score += log_score703 doc_length += length704 pp_score = 10.0 ** (-doc_log_score / doc_length)705 pp_score = round(pp_score, 1)706 return pp_score707 708 @staticmethod709 def check_perplexity(710 document,711 sentencepiece_model,712 kenlm_model,713 perplexity_max_cutoff,714 ):715 cond = True716 if kenlm_model:717 score = Filtering.compute_perplexity_score(718 document, sentencepiece_model, kenlm_model719 )720 cond = score <= perplexity_max_cutoff721 return cond722 723 @staticmethod724 def filtering(725 document,726 cond_check_number_words,727 sentencepiece_model_tok,728 strip_characters,729 number_words_min_cutoff,730 number_words_max_cutoff,731 cond_check_character_repetition_removal,732 character_repetition_length,733 character_repetition_max_cutoff,734 cond_check_word_repetition_removal,735 word_repetition_length,736 word_repetition_max_cutoff,737 cond_check_special_characters,738 special_characters,739 special_characters_max_cutoff,740 cond_words_augmentation,741 words_augmentation_group_sizes,742 words_augmentation_join_char,743 cond_check_stopwords,744 stopwords,745 stopwords_min_cutoff,746 cond_check_flagged_words,747 flagged_words,748 flagged_words_max_cutoff,749 cond_check_lang_id,750 lang_dataset_id,751 model_lang_id,752 lang_id_min_cutoff,753 cond_check_perplexity,754 sentencepiece_model,755 kenlm_model,756 perplexity_max_cutoff,757 ):758 if cond_check_number_words:759 if not Filtering.check_number_words(760 document,761 sentencepiece_model_tok,762 strip_characters,763 number_words_min_cutoff,764 number_words_max_cutoff,765 ):766 return False767 if cond_check_character_repetition_removal:768 if not Filtering.check_character_repetition_removal(769 document,770 character_repetition_length,771 character_repetition_max_cutoff,772 ):773 return False774 if cond_check_word_repetition_removal:775 if not Filtering.check_word_repetition_removal(776 document,777 sentencepiece_model_tok,778 strip_characters,779 word_repetition_length,780 word_repetition_max_cutoff,781 ):782 return False783 if cond_check_special_characters:784 if not Filtering.check_special_characters(785 document,786 special_characters,787 special_characters_max_cutoff,788 ):789 return False790 if cond_check_stopwords:791 if not Filtering.check_stopwords(792 document,793 sentencepiece_model_tok,794 strip_characters,795 cond_words_augmentation,796 words_augmentation_group_sizes,797 words_augmentation_join_char,798 stopwords,799 stopwords_min_cutoff,800 ):801 return False802 if cond_check_flagged_words:803 if not Filtering.check_flagged_words(804 document,805 sentencepiece_model_tok,806 strip_characters,807 cond_words_augmentation,808 words_augmentation_group_sizes,809 words_augmentation_join_char,810 flagged_words,811 flagged_words_max_cutoff,812 ):813 return False814 if cond_check_lang_id:815 if not Filtering.check_lang_id(816 document,817 lang_dataset_id,818 model_lang_id,819 lang_id_min_cutoff,820 ):821 return False822 if cond_check_perplexity:823 if not Filtering.check_perplexity(824 document,825 sentencepiece_model,826 kenlm_model,827 perplexity_max_cutoff,828 ):829 return False830 return True831 832 833class FunctionDatasetFiltering:834 def __init__(835 self,836 lang_dataset_id,837 path_fasttext_model,838 path_sentencepiece_model,839 path_kenlm_model,840 ):841 self.lang_dataset_id = lang_dataset_id842 self.path_fasttext_model = path_fasttext_model843 self.path_sentencepiece_model = path_sentencepiece_model844 self.path_kenlm_model = path_kenlm_model845 846 self.param = LoadParameters.load_parameters(lang_dataset_id)847 self.stopwords = LoadParameters.load_stopwords(lang_dataset_id)848 self.flagged_words = LoadParameters.load_flagged_words(lang_dataset_id)849 self.model_lang_id = LoadParameters.load_model_lang_id(850 lang_dataset_id, path_fasttext_model851 )852 self.sentencepiece_model = LoadParameters.load_sentencepiece_model(853 lang_dataset_id, path_sentencepiece_model854 )855 self.sentencepiece_model_tok = (856 self.sentencepiece_model if self.param["tokenization"] else None857 )858 self.kenlm_model = LoadParameters.load_kenlm_model(859 lang_dataset_id, path_kenlm_model860 )861 862 def __call__(self, example):863 keep_example = Filtering.filtering(864 document=example["text"],865 cond_check_number_words=self.param["cond_check_number_words"],866 sentencepiece_model_tok=self.sentencepiece_model_tok,867 strip_characters=self.param["strip_characters"],868 number_words_min_cutoff=self.param["number_words_min_cutoff"],869 number_words_max_cutoff=self.param["number_words_max_cutoff"],870 cond_check_character_repetition_removal=self.param[871 "cond_check_character_repetition_removal"872 ],873 character_repetition_length=self.param["character_repetition_length"],874 character_repetition_max_cutoff=self.param[875 "character_repetition_max_cutoff"876 ],877 cond_check_word_repetition_removal=self.param[878 "cond_check_word_repetition_removal"879 ],880 word_repetition_length=self.param["word_repetition_length"],881 word_repetition_max_cutoff=self.param["word_repetition_max_cutoff"],882 cond_check_special_characters=self.param["cond_check_special_characters"],883 special_characters=self.param["special_characters"],884 special_characters_max_cutoff=self.param["special_characters_max_cutoff"],885 cond_words_augmentation=self.param["cond_words_augmentation"],886 words_augmentation_group_sizes=self.param["words_augmentation_group_sizes"],887 words_augmentation_join_char=self.param["words_augmentation_join_char"],888 cond_check_stopwords=self.param["cond_check_stopwords"],889 stopwords=self.stopwords,890 stopwords_min_cutoff=self.param["stopwords_min_cutoff"],891 cond_check_flagged_words=self.param["cond_check_flagged_words"],892 flagged_words=self.flagged_words,893 flagged_words_max_cutoff=self.param["flagged_words_max_cutoff"],894 cond_check_lang_id=self.param["cond_check_lang_id"],895 lang_dataset_id=self.lang_dataset_id,896 model_lang_id=self.model_lang_id,897 lang_id_min_cutoff=self.param["lang_id_min_cutoff"],898 cond_check_perplexity=self.param["cond_check_perplexity"],899 sentencepiece_model=self.sentencepiece_model,900 kenlm_model=self.kenlm_model,901 perplexity_max_cutoff=self.param["perplexity_max_cutoff"],902 )903 return keep_example904 905 def __reduce__(self):906 return (907 self.__class__,908 (909 self.lang_dataset_id,910 self.path_fasttext_model,911 self.path_sentencepiece_model,912 self.path_kenlm_model,913 ),914 )915 916 917class DatasetFiltering:918 def __init__(919 self,920 dataset,921 lang_dataset_id,922 path_fasttext_model,923 path_sentencepiece_model,924 path_kenlm_model,925 num_proc,926 path_dir_save_dataset,927 ):928 self.ds = dataset929 self.lang_dataset_id = lang_dataset_id930 self.path_fasttext_model = path_fasttext_model931 self.path_sentencepiece_model = path_sentencepiece_model932 self.path_kenlm_model = path_kenlm_model933 self.num_proc = num_proc934 self.path_dir_save_dataset = path_dir_save_dataset935 936 def modifying_documents(self):937 func_dataset_modifying_documents = FunctionDatasetModifyingDocuments(938 self.lang_dataset_id939 )940 self.ds = self.ds.map(func_dataset_modifying_documents, num_proc=self.num_proc)941 942 def filtering(self):943 func_dataset_filtering = FunctionDatasetFiltering(944 self.lang_dataset_id,945 self.path_fasttext_model,946 self.path_sentencepiece_model,947 self.path_kenlm_model,948 )949 self.ds = self.ds.filter(func_dataset_filtering, num_proc=self.num_proc)950 951 def save_dataset(self):952 pathlib.Path(self.path_dir_save_dataset).mkdir(parents=True, exist_ok=True)953 path_dir_save_dataset = pathlib.PurePath(954 self.path_dir_save_dataset, self.lang_dataset_id955 )956 pathlib.Path(path_dir_save_dataset).mkdir(parents=True, exist_ok=True)957 self.ds.save_to_disk(path_dir_save_dataset)958