peter2000/audit_uganda_config_space

1import json2from tempfile import mktemp3 4import argilla as rg5from huggingface_hub import HfApi6 7from defaults import REMOTE_CODE_PATHS, SEED_DATA_PATH8 9 10hf_api = HfApi()11 12with open("DATASET_README_BASE.md") as f:13    DATASET_README_BASE = f.read()14 15 16def create_readme(domain_seed_data, project_name, domain):17    # create a readme for the project that shows the domain and project name18    readme = DATASET_README_BASE19    readme += f"# {project_name}\n\n## Domain: {domain}"20    perspectives = domain_seed_data.get("perspectives")21    topics = domain_seed_data.get("topics")22    examples = domain_seed_data.get("examples")23    if perspectives:24        readme += "\n\n## Perspectives\n\n"25        for p in perspectives:26            readme += f"- {p}\n"27    if topics:28        readme += "\n\n## Topics\n\n"29        for t in topics:30            readme += f"- {t}\n"31    if examples:32        readme += "\n\n## Examples\n\n"33        for example in examples:34            readme += f"### {example['question']}\n\n{example['answer']}\n\n"35    temp_file = mktemp()36 37    with open(temp_file, "w") as f:38        f.write(readme)39    return temp_file40 41 42def setup_dataset_on_hub(repo_id, hub_token):43    # create an empty dataset repo on the hub44    hf_api.create_repo(45        repo_id=repo_id,46        token=hub_token,47        repo_type="dataset",48        exist_ok=True,49    )50 51 52def push_dataset_to_hub(53    domain_seed_data_path,54    project_name,55    domain,56    pipeline_path,57    hub_username,58    hub_token: str,59):60    repo_id = f"{hub_username}/{project_name}"61 62    setup_dataset_on_hub(repo_id=repo_id, hub_token=hub_token)63 64    #  upload the seed data and readme to the hub65    hf_api.upload_file(66        path_or_fileobj=domain_seed_data_path,67        path_in_repo="seed_data.json",68        token=hub_token,69        repo_id=repo_id,70        repo_type="dataset",71    )72 73    # upload the readme to the hub74    domain_seed_data = json.load(open(domain_seed_data_path))75    hf_api.upload_file(76        path_or_fileobj=create_readme(77            domain_seed_data=domain_seed_data, project_name=project_name, domain=domain78        ),79        path_in_repo="README.md",80        token=hub_token,81        repo_id=repo_id,82        repo_type="dataset",83    )84 85 86def push_pipeline_to_hub(87    pipeline_path,88    hub_username,89    hub_token: str,90    project_name,91):92    repo_id = f"{hub_username}/{project_name}"93 94    # upload the pipeline to the hub95    hf_api.upload_file(96        path_or_fileobj=pipeline_path,97        path_in_repo="pipeline.py",98        token=hub_token,99        repo_id=repo_id,100        repo_type="dataset",101    )102 103    for code_path in REMOTE_CODE_PATHS:104        hf_api.upload_file(105            path_or_fileobj=code_path,106            path_in_repo=code_path,107            token=hub_token,108            repo_id=repo_id,109            repo_type="dataset",110        )111 112    print(f"Dataset uploaded to {repo_id}")113 114 115def pull_seed_data_from_repo(repo_id, hub_token):116    # pull the dataset repo from the hub117    hf_api.hf_hub_download(118        repo_id=repo_id, token=hub_token, repo_type="dataset", filename=SEED_DATA_PATH119    )120    return json.load(open(SEED_DATA_PATH))121 122 123def push_argilla_dataset_to_hub(124    name: str,125    repo_id: str,126    url: str,127    api_key: str,128    hub_token: str,129    workspace: str = "admin",130):131    rg.init(api_url=url, api_key=api_key)132    feedback_dataset = rg.FeedbackDataset.from_argilla(name=name, workspace=workspace)133    local_dataset = feedback_dataset.pull()134    local_dataset.push_to_huggingface(repo_id=repo_id, token=hub_token)135 136 137def push_pipeline_params(138    pipeline_params,139    hub_username,140    hub_token: str,141    project_name,142):143    repo_id = f"{hub_username}/{project_name}"144    temp_path = mktemp()145    with open(temp_path, "w") as f:146        json.dump(pipeline_params, f)147    # upload the pipeline to the hub148    hf_api.upload_file(149        path_or_fileobj=temp_path,150        path_in_repo="pipeline_params.json",151        token=hub_token,152        repo_id=repo_id,153        repo_type="dataset",154    )155 156    print(f"Pipeline params uploaded to {repo_id}")157