rodrigomasini/data_only_hallucination_leaderboard

1#!/usr/bin/env python2 3import os4import fnmatch5 6import json7from huggingface_hub import HfApi8 9 10def find_json_files(directory):11    matches = []12    for root, dirnames, filenames in os.walk(directory):13        for filename in fnmatch.filter(filenames, '*.json'):14            matches.append(os.path.join(root, filename))15    return matches16 17 18directory_path = '/Users/pasquale/workspace/eval/requests'19json_files = find_json_files(directory_path)20 21api = HfApi()22model_lst = api.list_models()23 24model_lst = [m for m in model_lst]25 26id_to_model = {m.id: m for m in model_lst}27 28for path in json_files:29    with open(path, 'r') as fr:30        data = json.load(fr)31 32        model_id = data['model']33        if model_id in id_to_model:34            model = id_to_model[model_id]35 36            to_overwrite = False37 38            is_finetuned = any(tag.startswith('base_model:') for tag in id_to_model[data['model']].tags)39 40            if is_finetuned:41                data["model_type"] = "fine-tuned"42                to_overwrite = True43 44            is_instruction_tuned = ('nstruct' in model_id) or ('chat' in model_id)45            if is_instruction_tuned:46                data["model_type"] = "instruction-tuned"47                to_overwrite = True48 49            if to_overwrite is True:50                with open(path, 'w') as fw:51                    json.dump(data, fw)52 53        else:54            print(f'Model {model_id} not found')55