bigcode/in-the-commitpack
1
1import gradio as gr2from huggingface_hub import hf_hub_download3import json4import urllib5import os6 7filepath = hf_hub_download(repo_id="bigcode/commits-username-to-repo", filename="username_to_repo.json",8 repo_type="dataset", token=os.environ.get("api_token"))9with open(filepath, 'r') as f:10 username_to_repo = json.load(f)11 12usernames = set(username_to_repo.keys())13 14text = """\1516**_CommitPack is is a 4TB dataset of commits scraped from GitHub repositories that are permissively licensed._**17 18# Am I in The CommitPack?19 20As part of the BigCode project, we released and maintain [CommitPack](https://huggingface.co/datasets/bigcode/commitpack), a 4 TB dataset of permissively licensed Git commits covering 350 programming languages. One of our goals in this project is to give people agency over their source code by letting them decide whether or not it should be used to develop and evaluate machine learning models, as we acknowledge that not all developers may wish to have their data used for that purpose.21""" + """\22 23This tool lets you check if a repository under a given username is part of the CommitPack dataset. Would you like to have your data removed from future versions of CommitPack? The CommitPack uses the same opt-out as The Stack, so you can opt-out by following the instructions [here](https://www.bigcode-project.org/docs/about/the-stack/#how-can-i-request-that-my-data-be-removed-from-the-stack).24"""25 26opt_out_text_template = """\27### Opt-out28 29If you want your data to be removed from the CommitPack and model training \30open an issue with <a href="https://github.com/bigcode-project/opt-out-v2/issues/new?title={title}&body={body}" target="_blank">this link</a> \31(if the link doesn't work try right a right click and open it in a new tab) or visit [https://github.com/bigcode-project/opt-out-v2/issues/new?&template=opt-out-request.md](https://github.com/bigcode-project/opt-out-v2/issues/new?&template=opt-out-request.md) .\32"""33 34opt_out_issue_title = """Opt-out request for {username}"""35opt_out_issue_body = """\36I request that the following data is removed from the CommitPack:37 38 - Commits39 - GitHub issue40{repo_list}41 42_Note_: If you don't want all resources to be included just remove the elements from the list above. If you would like to exclude all repositories and resources just add a single element "all" to the list.43"""44 45 46def issue_url(username, repos):47 title = urllib.parse.quote(opt_out_issue_title.format(username=username))48 body = urllib.parse.quote(opt_out_issue_body.format(repo_list=" - " + "\n - ".join(repos)))49 50 opt_out_text = opt_out_text_template.format(title=title, body=body)51 52 return opt_out_text53 54 55def check_username(username):56 output_md = ""57 if username in usernames and len(username_to_repo[username]) > 0:58 repos = username_to_repo[username]59 repo_word = "repository" if len(repos) == 1 else "repositories"60 output_md += f"**Yes**, there is code from **{len(repos)} {repo_word}** in the CommitPack:\n\n"61 for repo in repos:62 output_md += f"_{repo}_\n\n"63 64 return output_md.strip(), issue_url(username, repos)65 else:66 output_md += "**No**, your code is not in the CommitPack."67 return output_md.strip(), ""68 69 70with gr.Blocks() as demo:71 with gr.Row():72 _, colum_2, _ = gr.Column(scale=1), gr.Column(scale=6), gr.Column(scale=1)73 with colum_2:74 gr.Markdown(text)75 username = gr.Text("", label="Your GitHub username:")76 check_button = gr.Button("Check!")77 repos = gr.Markdown()78 opt_out = gr.Markdown()79 80 check_button.click(check_username, [username], [repos, opt_out])81 82demo.launch()83