huggingface/text-data-filtering
37
1# Run with: streamlit run visualization.py2 3import streamlit as st4 5import os6 7from io import StringIO8import base649import json10import pandas as pd11 12pd.options.mode.chained_assignment = None13 14import numpy as np15 16import matplotlib.pyplot as plt17 18from filtering import LoadParameters, ModifyingDocuments, Filtering19from languages_id import langs_id20 21 22class Visualization_for_lang:23 def __init__(24 self,25 path_data,26 lang,27 num_docs,28 num_docs_for_words,29 max_len_text_display,30 lang_dataset_id,31 path_fasttext_model,32 path_sentencepiece_model,33 path_kenlm_model,34 ):35 self.path_data = path_data36 self.lang = lang37 self.num_docs = num_docs38 self.num_docs_for_words = num_docs_for_words39 self.max_len_text_display = max_len_text_display40 41 self.lang_dataset_id = lang_dataset_id42 self.param = LoadParameters.load_parameters(lang_dataset_id)43 self.stopwords = LoadParameters.load_stopwords(lang_dataset_id)44 self.flagged_words = LoadParameters.load_flagged_words(lang_dataset_id)45 self.model_lang_id = LoadParameters.load_model_lang_id(46 lang_dataset_id, path_fasttext_model47 )48 self.sentencepiece_model = LoadParameters.load_sentencepiece_model(49 lang_dataset_id, path_sentencepiece_model50 )51 self.sentencepiece_model_tok = (52 self.sentencepiece_model if self.param["tokenization"] else None53 )54 self.kenlm_model = LoadParameters.load_kenlm_model(55 lang_dataset_id, path_kenlm_model56 )57 58 def set_title(self):59 st.title(f"Filtering visualization for {self.lang}")60 61 def open_data(self):62 with open(self.path_data) as json_file:63 data = json.load(json_file)64 65 self.num_docs = min(self.num_docs, len(data))66 self.num_docs_for_words = min(self.num_docs_for_words, len(data))67 68 if "words" in data[0]:69 words = [doc["words"] for doc in data[: self.num_docs_for_words]]70 words = [word for doc in words for word in doc]71 self.words = pd.DataFrame(words)72 else:73 self.words = None74 75 docs = data[: self.num_docs]76 for doc in docs:77 if not (self.words is None):78 del doc["words"]79 if len(doc["text"]) > self.max_len_text_display:80 doc["text"] = (81 doc["text"][: self.max_len_text_display]82 + " [...] [THIS LONG TEXT HAS BEEN TRUNCATED FOR DISPLAY REASONS]"83 )84 self.docs_checkpoint = pd.DataFrame(docs)85 self.docs = self.docs_checkpoint86 87 @staticmethod88 def print_discarded_by_cond(cond):89 st.caption(90 f"{(len(cond) - np.sum(1*cond)) / len(cond) * 100:.2f}% of the total is discarded with this filter."91 )92 93 @staticmethod94 def plot_hist(dataframe, key, num_bins=50):95 checkbox = st.checkbox(96 "Diplay distribution", value=True, key=f"display_distribution_{key[0]}"97 )98 if checkbox:99 fig, ax = plt.subplots()100 val = dataframe[key[0]].values101 if np.median(val) != 0:102 val = val[103 abs(val - np.median(val))104 < 9 * np.median(np.absolute(val - np.median(val)))105 ]106 ax.hist(val, bins=num_bins, density=True)107 ax.set_title(" ".join(key[0].split("_")))108 ax.axvline(x=key[1], color="r", linestyle="dashed")109 st.pyplot(fig)110 111 @staticmethod112 def display_dataset(dataframe, cond, description, type_of_examples):113 displayed_examples = dataframe.loc[cond]114 st.subheader(115 f"{description}: {len(displayed_examples)} {type_of_examples} ({len(displayed_examples) / len(dataframe.index) * 100:.2f}%)"116 )117 st.markdown(118 "Click on a column to sort by it, place the cursor on the text to display it."119 )120 st.dataframe(displayed_examples)121 122 def filtering_of_docs(self):123 def set_sliders():124 columns = list(self.docs)125 keys = []126 conds = {}127 128 def get_cond(key, cutoff, max_cutoff):129 if max_cutoff:130 return self.docs[key] <= cutoff131 return self.docs[key] >= cutoff132 133 if "number_words" in columns:134 with st.sidebar.expander("Number of words"):135 cutoff_def = "If the number of words of a document is lower than this number, the document is removed."136 max_nb_words = int(np.max(self.docs["number_words"])) + 1137 cutoff_min_number_words = st.slider(138 cutoff_def, 0, min(max_nb_words, 500), 0139 )140 new_key = ("number_words", cutoff_min_number_words, False)141 keys.append(new_key)142 Visualization_for_lang.plot_hist(self.docs, new_key)143 cond_1 = get_cond(new_key[0], new_key[1], new_key[2])144 Visualization_for_lang.print_discarded_by_cond(cond_1)145 146 cutoff_def = "If the number of words of a document is higher than this number, the document is removed."147 cutoff_max_number_words = st.slider(148 cutoff_def, 0, max_nb_words, max_nb_words149 )150 new_key = ("number_words", cutoff_max_number_words, True)151 keys.append(new_key)152 cond_2 = get_cond(new_key[0], new_key[1], new_key[2])153 Visualization_for_lang.print_discarded_by_cond(cond_2)154 155 conds["number_words"] = [cond_1, cond_2]156 157 if "character_repetition_ratio" in columns:158 with st.sidebar.expander("Character repetition ratio"):159 val_repetitions_lengths = list(160 self.docs["character_repetition_ratio"].iloc[0].keys()161 )162 default_index = (163 val_repetitions_lengths.index("10")164 if "10" in val_repetitions_lengths165 else 0166 )167 label_selectbox = "Length of repetitions in characters (that will influence the character repetition ratio)."168 repetitions_length = st.selectbox(169 label=label_selectbox,170 options=val_repetitions_lengths,171 index=default_index,172 )173 st.caption(174 "Choosing a higher or lower number does not mean that the filtering "175 "is stronger or weaker. Be careful, choosing a low number (below 5 for languages like English) "176 "tends to associate a high character repetition ratio to very long documents (like book chapters), but with "177 "few or no repetitions, simply because their length gives them more diversity, and we do "178 "not want to discard such documents. It is generally better to increase this number, so that false "179 "positives are very short documents (which we want to delete anyway) rather than long ones. However, "180 "a low number can be useful for Chinese, where a character can designate a whole word."181 )182 self.docs["character_repetition_ratio"] = self.docs_checkpoint[183 "character_repetition_ratio"184 ]185 for i in range(len(self.docs["character_repetition_ratio"])):186 self.docs["character_repetition_ratio"].iloc[i] = self.docs[187 "character_repetition_ratio"188 ].iloc[i][repetitions_length]189 190 cutoff_def = "If the character repetition ratio of a document is higher than this number, the document is removed."191 cutoff_character_repetition_ratio = st.slider(192 cutoff_def, 0.0, 1.0, 1.0, step=0.01193 )194 new_key = (195 "character_repetition_ratio",196 cutoff_character_repetition_ratio,197 True,198 repetitions_length,199 )200 keys.append(new_key)201 Visualization_for_lang.plot_hist(self.docs, new_key)202 cond = get_cond(new_key[0], new_key[1], new_key[2])203 Visualization_for_lang.print_discarded_by_cond(cond)204 conds["character_repetition_ratio"] = [cond]205 206 if "word_repetition_ratio" in columns:207 with st.sidebar.expander("Word repetition ratio"):208 val_repetitions_lengths = list(209 self.docs["word_repetition_ratio"].iloc[0].keys()210 )211 default_index = (212 val_repetitions_lengths.index("5")213 if "5" in val_repetitions_lengths214 else 0215 )216 label_selectbox = "Length of repetitions in words (that will influence the word repetition ratio)."217 repetitions_length = st.selectbox(218 label=label_selectbox,219 options=val_repetitions_lengths,220 index=default_index,221 )222 st.caption(223 "Choosing a higher or lower number does not mean that the filtering "224 "is stronger or weaker. Be careful, choosing a low number (like 3) could "225 "tend to associate a high word repetition ratio to very long documents (like book chapters), but with "226 "few or no repetitions, simply because their length gives them more diversity, and we do "227 "not want to discard such documents. It is generally better to increase a bit this number, so that false "228 "positives are very short documents (which we want to delete anyway) rather than long ones."229 )230 self.docs["word_repetition_ratio"] = self.docs_checkpoint[231 "word_repetition_ratio"232 ]233 for i in range(len(self.docs["word_repetition_ratio"])):234 self.docs["word_repetition_ratio"].iloc[i] = self.docs[235 "word_repetition_ratio"236 ].iloc[i][repetitions_length]237 238 cutoff_def = "If the word repetition ratio of a document is higher than this number, the document is removed."239 cutoff_word_repetition_ratio = st.slider(240 cutoff_def, 0.0, 1.0, 1.0, step=0.01241 )242 new_key = (243 "word_repetition_ratio",244 cutoff_word_repetition_ratio,245 True,246 repetitions_length,247 )248 keys.append(new_key)249 Visualization_for_lang.plot_hist(self.docs, new_key)250 cond = get_cond(new_key[0], new_key[1], new_key[2])251 Visualization_for_lang.print_discarded_by_cond(cond)252 conds["word_repetition_ratio"] = [cond]253 254 if "special_characters_ratio" in columns:255 with st.sidebar.expander("Special characters ratio"):256 cutoff_def = "If the special characters ratio of a document is higher than this number, the document is removed."257 cutoff_special_characters_ratio = st.slider(258 cutoff_def, 0.0, 1.0, 1.0, step=0.01259 )260 new_key = (261 "special_characters_ratio",262 cutoff_special_characters_ratio,263 True,264 )265 keys.append(new_key)266 Visualization_for_lang.plot_hist(self.docs, new_key)267 cond = get_cond(new_key[0], new_key[1], new_key[2])268 Visualization_for_lang.print_discarded_by_cond(cond)269 conds["special_characters_ratio"] = [cond]270 271 if "stopwords_ratio" in columns:272 with st.sidebar.expander("Stop words ratio"):273 stopwords_file = st.file_uploader(274 "Upload your own list of stop words (one per line). If there is none, the default one is used."275 )276 if stopwords_file:277 new_stopwords = StringIO(278 stopwords_file.getvalue().decode("utf-8")279 ).read()280 new_stopwords = set(new_stopwords.split("\n"))281 self.docs["stopwords_ratio"] = self.docs_checkpoint[282 "stopwords_ratio"283 ]284 for i in range(len(self.docs["stopwords_ratio"])):285 self.docs["stopwords_ratio"].iloc[286 i287 ] = Filtering.compute_stopwords_ratio(288 self.docs["text"].iloc[i],289 self.sentencepiece_model_tok,290 self.param["strip_characters"],291 self.param["cond_words_augmentation"],292 self.param["words_augmentation_group_sizes"],293 self.param["words_augmentation_join_char"],294 new_stopwords,295 )296 cutoff_def = "If the stop words ratio of a document is lower than this number, the document is removed."297 cutoff_stopwords_ratio = st.slider(298 cutoff_def, 0.0, 1.0, 0.0, step=0.01299 )300 new_key = ("stopwords_ratio", cutoff_stopwords_ratio, False)301 keys.append(new_key)302 Visualization_for_lang.plot_hist(self.docs, new_key)303 cond = get_cond(new_key[0], new_key[1], new_key[2])304 Visualization_for_lang.print_discarded_by_cond(cond)305 conds["stopwords_ratio"] = [cond]306 307 if "flagged_words_ratio" in columns:308 with st.sidebar.expander("Flagged words ratio"):309 flagged_words_file = st.file_uploader(310 "Upload your own list of flagged words (one per line). If there is none, the default one is used."311 )312 if flagged_words_file:313 new_flagged_words = StringIO(314 flagged_words_file.getvalue().decode("utf-8")315 ).read()316 new_flagged_words = set(new_flagged_words.split("\n"))317 self.docs["flagged_words_ratio"] = self.docs_checkpoint[318 "flagged_words_ratio"319 ]320 for i in range(len(self.docs["flagged_words_ratio"])):321 self.docs["flagged_words_ratio"].iloc[322 i323 ] = Filtering.compute_flagged_words_ratio(324 self.docs["text"].iloc[i],325 self.sentencepiece_model_tok,326 self.param["strip_characters"],327 self.param["cond_words_augmentation"],328 self.param["words_augmentation_group_sizes"],329 self.param["words_augmentation_join_char"],330 new_flagged_words,331 )332 cutoff_def = "If the flagged words ratio of a document is higher than this number, the document is removed."333 max_fwr = np.max(self.docs["flagged_words_ratio"])334 max_fwr = np.ceil(max_fwr * 1000) / 1000335 max_fwr = float(max_fwr)336 cutoff_flagged_words_ratio = st.slider(337 cutoff_def,338 0.000,339 max_fwr,340 max_fwr,341 step=0.001,342 format="%f",343 )344 new_key = ("flagged_words_ratio", cutoff_flagged_words_ratio, True)345 keys.append(new_key)346 Visualization_for_lang.plot_hist(self.docs, new_key)347 cond = get_cond(new_key[0], new_key[1], new_key[2])348 Visualization_for_lang.print_discarded_by_cond(cond)349 conds["flagged_words_ratio"] = [cond]350 351 if "lang_id_score" in columns:352 with st.sidebar.expander("Language ID confidence score"):353 cutoff_def = "If the confidence score for the language identification prediction of a document is lower than this number, the document is removed."354 cutoff_lang_id_score = st.slider(355 cutoff_def, 0.0, 1.0, 0.0, step=0.01356 )357 new_key = ("lang_id_score", cutoff_lang_id_score, False)358 keys.append(new_key)359 Visualization_for_lang.plot_hist(self.docs, new_key)360 cond = get_cond(new_key[0], new_key[1], new_key[2])361 Visualization_for_lang.print_discarded_by_cond(cond)362 conds["lang_id_score"] = [cond]363 364 if "perplexity_score" in columns:365 with st.sidebar.expander("Perplexity score"):366 cutoff_def = "If the perplexity score of a document is higher than this number, the document is removed."367 max_pp = int(np.max(self.docs["perplexity_score"])) + 1368 cutoff_perplexity_score = st.slider(cutoff_def, 0, max_pp, max_pp)369 new_key = ("perplexity_score", cutoff_perplexity_score, True)370 keys.append(new_key)371 Visualization_for_lang.plot_hist(self.docs, new_key)372 cond = get_cond(new_key[0], new_key[1], new_key[2])373 Visualization_for_lang.print_discarded_by_cond(cond)374 conds["perplexity_score"] = [cond]375 376 return keys, conds377 378 with st.expander(379 f"Filtering on documents, for {self.num_docs} {self.lang} documents"380 ):381 st.header(382 f"Filtering on documents, for {self.num_docs} {self.lang} documents"383 )384 385 if "labels" in list(self.docs):386 chosen_label = st.selectbox(387 label="Consider only documents that include the following label",388 options=[389 "All",390 "NA: Narrative",391 "IN: Informational Description",392 "OP: Opinion",393 "ID: Interactive Discussion",394 "HI: How-to/Instruction",395 "IP: Informational Persuasion",396 "LY: Lyrical",397 "SP: Spoken",398 ],399 )400 chosen_label = chosen_label.split(":")[0]401 if chosen_label != "All":402 cond_label = list(403 self.docs["labels"].apply(404 lambda x: True if chosen_label in x else False405 )406 )407 self.docs = self.docs[cond_label]408 409 if self.docs.empty:410 st.markdown(411 "No document to display, please try to select a different label."412 )413 self.keys = []414 self.parameters = []415 416 else:417 st.sidebar.subheader("Parameters of the filtering on documents")418 self.keys, conds = set_sliders()419 self.parameters = self.keys * 1420 421 all_conds = [422 subcond for cond in list(conds.values()) for subcond in cond423 ]424 all_conds = np.all(all_conds, axis=0)425 426 Visualization_for_lang.display_dataset(427 self.docs, np.invert(all_conds), "Discarded documents", "docs"428 )429 430 # st.subheader("Display discarded documents by filter")431 display_discarded_documents_by_filter = st.checkbox(432 "Display discarded documents by filter"433 )434 435 if display_discarded_documents_by_filter:436 columns = list(self.docs)437 438 if "number_words" in columns:439 cond_filter = np.invert(np.all(conds["number_words"], axis=0))440 Visualization_for_lang.display_dataset(441 self.docs,442 cond_filter,443 "Discarded documents for the filter on the number of words",444 "docs",445 )446 447 if "character_repetition_ratio" in columns:448 cond_filter = np.invert(449 np.all(conds["character_repetition_ratio"], axis=0)450 )451 Visualization_for_lang.display_dataset(452 self.docs,453 cond_filter,454 "Discarded documents for the filter on the character repetition ratio",455 "docs",456 )457 458 if "word_repetition_ratio" in columns:459 cond_filter = np.invert(460 np.all(conds["word_repetition_ratio"], axis=0)461 )462 Visualization_for_lang.display_dataset(463 self.docs,464 cond_filter,465 "Discarded documents for the filter on the word repetition ratio",466 "docs",467 )468 469 if "special_characters_ratio" in columns:470 cond_filter = np.invert(471 np.all(conds["special_characters_ratio"], axis=0)472 )473 Visualization_for_lang.display_dataset(474 self.docs,475 cond_filter,476 "Discarded documents for the filter on the special characters ratio",477 "docs",478 )479 480 if "stopwords_ratio" in columns:481 cond_filter = np.invert(482 np.all(conds["stopwords_ratio"], axis=0)483 )484 Visualization_for_lang.display_dataset(485 self.docs,486 cond_filter,487 "Discarded documents for the filter on the stop words ratio",488 "docs",489 )490 491 if "flagged_words_ratio" in columns:492 cond_filter = np.invert(493 np.all(conds["flagged_words_ratio"], axis=0)494 )495 Visualization_for_lang.display_dataset(496 self.docs,497 cond_filter,498 "Discarded documents for the filter on the flagged words ratio",499 "docs",500 )501 502 if "lang_id_score" in columns:503 cond_filter = np.invert(np.all(conds["lang_id_score"], axis=0))504 Visualization_for_lang.display_dataset(505 self.docs,506 cond_filter,507 "Discarded documents for the filter on the language identification confidence score",508 "docs",509 )510 511 if "perplexity_score" in columns:512 cond_filter = np.invert(513 np.all(conds["perplexity_score"], axis=0)514 )515 Visualization_for_lang.display_dataset(516 self.docs,517 cond_filter,518 "Discarded documents for the filter on the perplexity score",519 "docs",520 )521 522 Visualization_for_lang.display_dataset(523 self.docs, all_conds, "Retained documents", "docs"524 )525 526 st.header("Download data")527 528 with open(self.path_data) as json_file:529 btn = st.download_button(530 label="Download data as json",531 data=json_file,532 file_name="data.json",533 )534 535 def filtering_of_words(self):536 if not (self.words is None):537 columns = list(self.words)538 539 st.sidebar.subheader("Parameter of the filtering on words")540 541 conds_words = {}542 543 if "len_word" in columns:544 with st.sidebar.expander("Length of words"):545 cutoff_def = "If the length of a word is higher than this number, the word is removed."546 max_len_word = min(int(np.max(self.words["len_word"])) + 1, 200)547 cutoff_word = st.slider(cutoff_def, 0, max_len_word, max_len_word)548 new_key = ("len_word", cutoff_word, True)549 self.parameters.append(new_key)550 Visualization_for_lang.plot_hist(self.words, new_key)551 cond_len_words = self.words["len_word"] <= cutoff_word552 Visualization_for_lang.print_discarded_by_cond(cond_len_words)553 conds_words["len_word"] = cond_len_words554 555 if "incorrect_substrings" in columns:556 with st.sidebar.expander("Words with incorrect substrings"):557 incorrect_substrings = st.checkbox(558 "Remove words with incorrect substrings."559 )560 self.parameters.append(561 ("incorrect_substrings", incorrect_substrings)562 )563 564 checkbox = st.checkbox(565 "Diplay distribution",566 value=True,567 key="display_distribution_incorrect_substrings",568 )569 if checkbox:570 incor_sub = np.array(self.words["incorrect_substrings"]) * 1571 with_incor_sub = np.sum(incor_sub)572 without_incor_sub = len(incor_sub) - with_incor_sub573 st.markdown(574 f"Number of words with incorrect substrings: {with_incor_sub}"575 )576 st.markdown(577 f"Number of words without incorrect substrings: {without_incor_sub}"578 )579 580 if incorrect_substrings:581 cond_incorrect_substrings = np.invert(582 self.words["incorrect_substrings"]583 )584 else:585 cond_incorrect_substrings = np.array(586 [587 True588 for i in range(len(self.words["incorrect_substrings"]))589 ]590 )591 Visualization_for_lang.print_discarded_by_cond(592 cond_incorrect_substrings593 )594 conds_words["incorrect_substrings"] = cond_incorrect_substrings595 596 all_conds_words = np.all(list(conds_words.values()), axis=0)597 598 with st.expander(599 f"Filtering on words, for {self.num_docs_for_words} {self.lang} documents"600 ):601 st.header(602 f"Filtering on words, for {self.num_docs_for_words} {self.lang} documents"603 )604 605 st.markdown(606 f"Since the number of words is way larger than the number of documents, "607 f"we consider in this section words for only {self.num_docs_for_words} documents."608 )609 610 Visualization_for_lang.display_dataset(611 self.words, np.invert(all_conds_words), "Discarded words", "words"612 )613 614 # st.subheader("Display discarded words by filter")615 display_discarded_words_by_filter = st.checkbox(616 "Display discarded words by filter"617 )618 619 if display_discarded_words_by_filter:620 621 if "len_word" in columns:622 cond_filter = np.invert(conds_words["len_word"])623 Visualization_for_lang.display_dataset(624 self.words,625 cond_filter,626 "Discarded words for the filter on length",627 "words",628 )629 630 if "incorrect_substrings" in columns:631 cond_filter = np.invert(conds_words["incorrect_substrings"])632 Visualization_for_lang.display_dataset(633 self.words,634 cond_filter,635 "Discarded words for the filter on incorrect substrings",636 "words",637 )638 639 Visualization_for_lang.display_dataset(640 self.words, all_conds_words, "Retained words", "words"641 )642 643 def download_parameters(self):644 st.sidebar.subheader("Download parameters")645 btn = st.sidebar.download_button(646 label="Download current parameters as json",647 data=json.dumps(self.parameters),648 file_name=f"parameters_{self.lang_dataset_id}.json",649 )650 651 """652 def plot_zipf_law(self):653 if not (self.words is None):654 st.header("Zipf's Law")655 656 display_zipf_law = st.checkbox("Display Zipf's Law")657 658 if display_zipf_law:659 660 freq_words = {}661 for _, row in self.words.iterrows():662 freq_words[row["word"]] = freq_words.get(row["word"], 0) + 1663 freq_words = np.array(list(freq_words.values()))664 freq_words = -np.sort(-freq_words)665 666 fig, ax = plt.subplots()667 ax.loglog(freq_words)668 ax.set_title("Zipf's Law")669 ax.set_xlabel("$i$-th most frequent word")670 ax.set_ylabel("frequency in the documents")671 st.pyplot(fig)672 """673 674 def analyse_personal_doc(self):675 with st.expander("Analyse your own document"):676 st.header("Analyse your own document")677 678 personal_doc = st.text_area(679 label="Paste here the document you want to analyse",680 value="",681 max_chars=10000,682 )683 684 is_discarded = False685 686 def is_doc_discarded(key, score):687 if key[2]: # max cutoff688 return score > key[1]689 else:690 return score < key[1]691 692 if personal_doc:693 694 st.markdown("Statistics of the document:")695 696 for key in self.keys:697 if key[0] == "number_words":698 words = ModifyingDocuments.get_words_from_document(699 personal_doc,700 self.sentencepiece_model_tok,701 lower_case=False,702 strip_characters=self.param["strip_characters"],703 )704 if key[2]:705 st.markdown(f"Number of words: {len(words)}")706 if is_doc_discarded(key, len(words)):707 is_discarded = True708 709 elif key[0] == "character_repetition_ratio":710 character_repetition_ratio = (711 Filtering.compute_character_repetition_ratio(712 personal_doc, int(key[3])713 )714 )715 character_repetition_ratio = round(716 character_repetition_ratio, 3717 )718 st.markdown(719 f"Character repetition ratio: {character_repetition_ratio}"720 )721 if is_doc_discarded(key, character_repetition_ratio):722 is_discarded = True723 724 elif key[0] == "word_repetition_ratio":725 word_repetition_ratio = Filtering.compute_word_repetition_ratio(726 personal_doc,727 self.sentencepiece_model_tok,728 self.param["strip_characters"],729 int(key[3]),730 )731 word_repetition_ratio = round(word_repetition_ratio, 3)732 st.markdown(f"Word repetition ratio: {word_repetition_ratio}")733 if is_doc_discarded(key, word_repetition_ratio):734 is_discarded = True735 736 elif key[0] == "special_characters_ratio":737 special_characters_ratio = (738 Filtering.compute_special_characters_ratio(739 personal_doc, self.param["special_characters"]740 )741 )742 special_characters_ratio = round(special_characters_ratio, 3)743 st.markdown(744 f"Special characters ratio: {special_characters_ratio}"745 )746 if is_doc_discarded(key, special_characters_ratio):747 is_discarded = True748 749 elif key[0] == "stopwords_ratio":750 stopwords_ratio = Filtering.compute_stopwords_ratio(751 personal_doc,752 self.sentencepiece_model_tok,753 self.param["strip_characters"],754 self.param["cond_words_augmentation"],755 self.param["words_augmentation_group_sizes"],756 self.param["words_augmentation_join_char"],757 self.stopwords,758 )759 stopwords_ratio = round(stopwords_ratio, 3)760 st.markdown(f"Stop words ratio: {stopwords_ratio}")761 if is_doc_discarded(key, stopwords_ratio):762 is_discarded = True763 764 elif key[0] == "flagged_words_ratio":765 flagged_words_ratio = Filtering.compute_flagged_words_ratio(766 personal_doc,767 self.sentencepiece_model_tok,768 self.param["strip_characters"],769 self.param["cond_words_augmentation"],770 self.param["words_augmentation_group_sizes"],771 self.param["words_augmentation_join_char"],772 self.flagged_words,773 )774 flagged_words_ratio = round(flagged_words_ratio, 3)775 st.markdown(f"Flagged words ratio: {flagged_words_ratio}")776 if is_doc_discarded(key, flagged_words_ratio):777 is_discarded = True778 779 elif key[0] == "lang_id_score":780 (781 lang_pred_dataset_id,782 lang_id_score,783 ) = Filtering.compute_lang_id_pred_score(784 personal_doc, self.model_lang_id785 )786 lang_id_score = round(lang_id_score, 3)787 st.markdown(788 f"Language identification confidence score: {lang_id_score}"789 )790 if is_doc_discarded(key, flagged_words_ratio) or (791 self.lang_dataset_id != lang_pred_dataset_id792 ):793 is_discarded = True794 795 elif key[0] == "perplexity_score":796 perplexity_score = Filtering.compute_perplexity_score(797 personal_doc,798 self.sentencepiece_model,799 self.kenlm_model,800 )801 perplexity_score = round(perplexity_score, 3)802 st.markdown(f"Perplexity score: {perplexity_score}")803 if is_doc_discarded(key, perplexity_score):804 is_discarded = True805 806 is_discarded = "" if is_discarded else "not "807 st.markdown(808 f"With the current filtering parameters, this document **is {is_discarded}discarded**."809 )810 811 def visualization_for_lang(self):812 self.set_title()813 self.open_data()814 self.filtering_of_docs()815 self.filtering_of_words()816 self.download_parameters()817 self.analyse_personal_doc()818 819 820class Visualization:821 def __init__(self, path_instructions, param_visu_langs):822 self.path_instructions = path_instructions823 self.param_visu_langs = param_visu_langs824 825 def preamble(self):826 def get_binary_file_downloader_html(bin_file, file_label="File"):827 with open(bin_file, "rb") as f:828 data = f.read()829 bin_str = base64.b64encode(data).decode()830 href = f'<a href="data:application/octet-stream;base64,{bin_str}" download="{os.path.basename(bin_file)}">{file_label}</a>'831 return href832 833 st.markdown(834 "Before diving into this demo, you might want to take a look at how the filtering pipeline looks like in more detail in this "835 + get_binary_file_downloader_html(836 self.path_instructions,837 "pdf",838 )839 + ".",840 unsafe_allow_html=True,841 )842 843 def warning_preamble(self):844 st.markdown(845 "This demo can be a little slow, and only allows you to process up to 5000 documents "846 "for a decent speed. If you want to display up to three times more documents and have "847 "a faster visualization, we invite you to run this "848 "[code](https://github.com/bigscience-workshop/data-preparation/tree/main/preprocessing/filtering/visualization) "849 "on your computer."850 )851 852 def choose_lang(self):853 options = [854 self.param_visu_langs[lang_dataset_id]["lang"]855 for lang_dataset_id in self.param_visu_langs856 ]857 index = options.index("English") if ("English" in options) else 0858 lang_chosen = st.selectbox(859 label="Select the language for visualization",860 options=options,861 index=index,862 )863 if lang_chosen != "None":864 lang_chosen_dataset_id = langs_id.loc[865 langs_id["lang"] == lang_chosen, "dataset_id"866 ].iloc[0]867 visualization_for_lang = Visualization_for_lang(868 path_data=self.param_visu_langs[lang_chosen_dataset_id]["path_data"],869 lang=self.param_visu_langs[lang_chosen_dataset_id]["lang"],870 num_docs=self.param_visu_langs[lang_chosen_dataset_id]["num_docs"],871 num_docs_for_words=self.param_visu_langs[lang_chosen_dataset_id][872 "num_docs_for_words"873 ],874 max_len_text_display=self.param_visu_langs[lang_chosen_dataset_id][875 "max_len_text_display"876 ],877 lang_dataset_id=self.param_visu_langs[lang_chosen_dataset_id][878 "lang_dataset_id"879 ],880 path_fasttext_model=self.param_visu_langs[lang_chosen_dataset_id][881 "path_fasttext_model"882 ],883 path_sentencepiece_model=self.param_visu_langs[lang_chosen_dataset_id][884 "path_sentencepiece_model"885 ],886 path_kenlm_model=self.param_visu_langs[lang_chosen_dataset_id][887 "path_kenlm_model"888 ],889 )890 visualization_for_lang.visualization_for_lang()891 892 def visualization(self):893 self.preamble()894 self.warning_preamble()895 self.choose_lang()896 897 898path_instructions = "./explanation_filtering_pipeline.pdf"899 900param_visu_langs = {901 lang_dataset_id: {902 "path_data": f"./{lang_dataset_id}_examples_with_stats.json",903 "lang": langs_id.loc[langs_id["dataset_id"] == lang_dataset_id, "lang"].iloc[0],904 "num_docs": 5000,905 "num_docs_for_words": 500,906 "max_len_text_display": 10000,907 "lang_dataset_id": lang_dataset_id,908 "path_fasttext_model": "./lid.176.bin",909 "path_sentencepiece_model": f"./{lang_dataset_id}.sp.model",910 "path_kenlm_model": f"./{lang_dataset_id}.arpa.bin",911 }912 for lang_dataset_id in ["eu", "ca", "zh", "en", "fr", "id", "pt", "es"]913}914 915visualization = Visualization(path_instructions, param_visu_langs)916visualization.visualization()917