bigscience/SourcingCatalog
8
1import json2 3import streamlit as st4from datasets import load_dataset5from streamlit_folium import folium_static6 7from catalogue import make_choro_map, region_tree8 9##################10## streamlit11##################12st.set_page_config(13 page_title="BigScience Language Resource Catalogue Input Form",14 page_icon="https://avatars.githubusercontent.com/u/82455566",15 layout="wide",16 initial_sidebar_state="auto",17)18 19query_params = st.experimental_get_query_params()20 21 22def main():23 if "save_state" not in st.session_state:24 st.session_state.save_state = {}25 26 viz_page()27 28 29##################30## SECTION: Explore the current catalogue31##################32 33app_categories = {34 "entry_types": {35 "primary": "Primary source",36 "processed": "Processed language dataset",37 "organization": "Language organization or advocate",38 },39 "language_lists": json.load(40 open("resources/language_lists.json", encoding="utf-8")41 ),42 "programming_languages": [43 x44 for x in json.load(45 open("resources/programming_languages.json", encoding="utf-8")46 )["itemListElement"]47 ],48 "languages_bcp47": [49 x50 for x in json.load(open("resources/bcp47.json", encoding="utf-8"))["subtags"]51 if x["type"] == "language"52 ],53 "custodian_types": [54 "A private individual",55 "A commercial entity",56 "A library, museum, or archival institute",57 "A university or research institution",58 "A nonprofit/NGO (other)",59 "A government organization",60 ],61 "pii_categories": json.load(62 open("resources/pii_categories.json", encoding="utf-8")63 ),64 "licenses": json.load(open("resources/licenses.json", encoding="utf-8")),65 "primary_taxonomy": json.load(66 open("resources/primary_source_taxonomy.json", encoding="utf-8")67 ),68 "file_formats": json.load(open("resources/file_formats.json", encoding="utf-8")),69}70 71 72def filter_entry(entry, filter_dct):73 res = True74 for k, v in entry.items():75 if k in filter_dct:76 if isinstance(v, dict):77 res = res and filter_entry(v, filter_dct[k])78 elif isinstance(v, list):79 res = res and (80 len(filter_dct[k]) == 0 or any([e in filter_dct[k] for e in v])81 )82 else:83 res = res and (len(filter_dct[k]) == 0 or v in filter_dct[k])84 return res85 86 87def filter_catalogue_visualization(catalogue, options):88 st.markdown("### Select entries to visualize")89 st.markdown(90 "##### Select entries by category, language, type of custodian or media"91 )92 st.markdown(93 "You can select specific parts of the catalogue to visualize in this window."94 + " Leave a field empty to select all values, or select specific options to only select entries that have one of the chosen values."95 )96 filter_by_options = [97 "resource type",98 "language names",99 "custodian type",100 "available for download",101 "license type",102 "source type",103 "media type",104 ]105 filter_by = st.multiselect(106 key="viz_filter_by",107 label="You can filter the catalogue to only visualize entries that have certain properties, such as:",108 options=filter_by_options,109 )110 filter_dict = {}111 if "resource type" in filter_by:112 filter_dict["type"] = st.multiselect(113 key="viz_filter_type",114 label="I want to only see entries that are of the following category:",115 options=options["entry_types"],116 format_func=lambda x: options["entry_types"][x],117 )118 if "language names" in filter_by:119 filter_dict["languages"] = {}120 filter_dict["languages"]["language_names"] = st.multiselect(121 key="viz_filter_languages_language_names",122 label="I want to only see entries that have one of the following languages:",123 options=list(options["language_lists"]["language_groups"].keys())124 + options["language_lists"]["niger_congo_languages"]125 + options["language_lists"]["indic_languages"],126 )127 if "custodian type" in filter_by:128 filter_dict["custodian"] = {}129 filter_dict["custodian"]["type"] = st.multiselect(130 key="viz_filter_custodian_type",131 label="I want to only see entries that corresponds to organizations or to data that id owned/managed by organizations of the following types:",132 options=options["custodian_types"],133 )134 if "available for download" in filter_by:135 filter_dict["availability"] = filter_dict.get("availability", {})136 filter_dict["availability"]["procurement"] = {}137 download_options = [138 "No - but the current owners/custodians have contact information for data queries",139 "No - we would need to spontaneously reach out to the current owners/custodians",140 "Yes - it has a direct download link or links",141 "Yes - after signing a user agreement",142 ]143 filter_dict["availability"]["procurement"]["for_download"] = st.multiselect(144 key="viz_availability_procurement_for_download",145 label="Select based on whether the data can be obtained online:",146 options=download_options,147 )148 if "license type" in filter_by:149 filter_dict["availability"] = filter_dict.get("availability", {})150 filter_dict["availability"]["licensing"] = {}151 filter_dict["availability"]["licensing"]["license_properties"] = st.multiselect(152 key="viz_availability_licensing_license_properties",153 label="Select primary entries that have the following license types",154 options=[155 "public domain",156 "multiple licenses",157 "copyright - all rights reserved",158 "open license",159 "research use",160 "non-commercial use",161 "do not distribute",162 ],163 )164 primary_license_options = [165 "Unclear / I don't know",166 "Yes - the source material has an open license that allows re-use",167 "Yes - the dataset has the same license as the source material",168 "Yes - the dataset curators have obtained consent from the source material owners",169 "No - the license of the source material actually prohibits re-use in this manner",170 ]171 filter_dict["processed_from_primary"] = filter_dict.get(172 "processed_from_primary", {}173 )174 filter_dict["processed_from_primary"]["primary_license"] = st.multiselect(175 key="viz_processed_from_primary_primary_license",176 label="For datasets, selected based on: Is the license or commercial status of the source material compatible with the license of the dataset?",177 options=primary_license_options,178 )179 if "source type" in filter_by:180 filter_dict["source_category"] = {}181 filter_dict["source_category"]["category_type"] = st.multiselect(182 key="viz_source_category_category_type",183 label="Select primary sources that correspond to:",184 options=["collection", "website"],185 )186 filter_dict["source_category"]["category_web"] = st.multiselect(187 key="viz_source_category_category_web",188 label="Select web-based primary sources that contain:",189 options=options["primary_taxonomy"]["website"],190 )191 filter_dict["source_category"]["category_media"] = st.multiselect(192 key="viz_source_category_category_media",193 label="Select primary sources that are collections of:",194 options=options["primary_taxonomy"]["collection"],195 )196 filter_dict["processed_from_primary"] = filter_dict.get(197 "processed_from_primary", {}198 )199 filter_dict["processed_from_primary"]["primary_types"] = st.multiselect(200 key="viz_processed_from_primary_primary_types",201 label="Select processed datasets whose primary sources contain:",202 options=[f"web | {w}" for w in options["primary_taxonomy"]["website"]]203 + options["primary_taxonomy"]["collection"],204 )205 if "media type" in filter_by:206 filter_dict["media"] = {}207 filter_dict["media"]["category"] = st.multiselect(208 key="viz_media_category",209 label="Select language data resources that contain:",210 options=["text", "audiovisual", "image"],211 help="Media data provided with transcription should go into **text**, then select the *transcribed* option. PDFs that have pre-extracted text information should go into **text**, PDFs that need OCR should go into **images**, select the latter if you're unsure",212 )213 filtered_catalogue = [214 entry215 for entry in catalogue216 if filter_entry(entry, filter_dict) and not (entry["uid"] == "")217 ]218 st.markdown(219 f"##### Your query matched **{len(filtered_catalogue)}** entries in the current catalogue."220 )221 return filtered_catalogue222 223 224def viz_page():225 st.title("🌸 - BigScience Catalog of Language Resources")226 st.markdown("---\n")227 catalogue = load_dataset("bigscience/collaborative_catalog")["train"]228 with st.sidebar:229 filtered_catalogue = filter_catalogue_visualization(catalogue, app_categories)230 entry_location_type = st.radio(231 label="I want to visualize",232 options=[233 "Where the organizations or data custodians are located",234 "Where the language data creators are located",235 ],236 key="viz_show_location_type",237 )238 show_by_org = (239 entry_location_type240 == "Where the organizations or data custodians are located"241 )242 with st.expander("Map of entries", expanded=True):243 filtered_counts = {}244 for entry in filtered_catalogue:245 locations = (246 [entry["custodian"]["location"]]247 if show_by_org248 else entry["languages"]["language_locations"]249 )250 # be as specific as possible251 locations = [252 loc253 for loc in locations254 if not any([l in region_tree.get(loc, []) for l in locations])255 ]256 for loc in locations:257 filtered_counts[loc] = filtered_counts.get(loc, 0) + 1258 world_map = make_choro_map(filtered_counts)259 folium_static(world_map, width=900, height=600)260 with st.expander("View selected resources", expanded=False):261 st.write("You can further select locations to select entries from here:")262 filter_region_choices = sorted(263 set(264 [265 loc266 for entry in filtered_catalogue267 for loc in (268 [entry["custodian"]["location"]]269 if show_by_org270 else entry["languages"]["language_locations"]271 )272 ]273 )274 )275 filter_locs = st.multiselect(276 "View entries from the following locations:",277 options=filter_region_choices,278 key="viz_select_location",279 )280 filter_loc_dict = (281 {"custodian": {"location": filter_locs}}282 if show_by_org283 else {"languages": {"language_locations": filter_locs}}284 )285 filtered_catalogue_by_loc = [286 entry287 for entry in filtered_catalogue288 if filter_entry(entry, filter_loc_dict)289 ]290 view_entry = st.selectbox(291 label="Select an entry to see more detail:",292 options=filtered_catalogue_by_loc,293 format_func=lambda entry: f"{entry['uid']} | {entry['description']['name']} -- {entry['description']['description']}",294 key="viz_select_entry",295 )296 st.markdown(297 f"##### *Type:* {view_entry['type']} *UID:* {view_entry['uid']} - *Name:* {view_entry['description']['name']}\n\n{view_entry['description']['description']}"298 )299 st.write(view_entry)300 301 302if __name__ == "__main__":303 main()304