CoreyMorris/MMLU-by-task-Leaderboard

1import unittest2from details_data_processor import DetailsDataProcessor3import pandas as pd4import requests5import os6 7class TestDetailsDataProcessor(unittest.TestCase):8 9    def setUp(self):10        self.processor = DetailsDataProcessor()11 12    # check that the result is a pandas dataframe13    def test_process_data(self):14        pass15        # data = self.processor.data16        # self.assertIsInstance(data, pd.DataFrame)17 18    def test_download_file(self):19        DetailsDataProcessor.download_file('https://www.google.com', 'test_file_please_remove')20        self.assertTrue(os.path.exists('test.html'))21        os.remove('test.html')22 23    # queries_harness is in the url24    def test_download_file_queries(self):25        file_path_with_error = 'results/shaohang/Sparse0.5_OPT-1.3/results_2023-07-19T19:10:31.005235.json'26        url = self.processor.build_url(file_path_with_error)27        DetailsDataProcessor.download_file(url, 'test_file_please_remove')28 29    # details harness is in the url30    def test_download_file_details(self):31        file_path = 'results/v2ray/LLaMA-2-Wizard-70B-QLoRA/results_2023-08-18T07:09:43.451689.json'32        url = self.processor.build_url(file_path)33        DetailsDataProcessor.download_file(url, 'test_file_please_remove')34 35 36    def test_build_url(self):37        test_cases = [38            ('results/64bits/LexPodLM-13B/results_2023-07-25T13:41:51.227672.json',39            'https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/64bits/LexPodLM-13B/details_harness%7ChendrycksTest-moral_scenarios%7C5_2023-07-25T13%3A41%3A51.227672.json'),40            ('results/AlpinDale/pygmalion-instruct/results_2023-08-17T11:20:15.687659.json',41            'https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/AlpinDale/pygmalion-instruct/details_harness%7ChendrycksTest-moral_scenarios%7C5_2023-08-17T11%3A20%3A15.687659.json')42        ]43    44        for file_path, expected in test_cases:45            assert self.processor.build_url(file_path) == expected, f"Test failed for file_path: {file_path}"46 47    def test_pipeline(self):48        df = self.processor.pipeline()49        print(100 * "****")50        print(df)51        self.assertIsInstance(df, pd.DataFrame)52 53    def test_find_files(self):54        directory = 'results'55        pattern = 'results*.json'56        files = self.processor._find_files(directory, pattern)57        # breakpoint()58        # print(files)59        self.assertIsInstance(files, list)60 61    def test_build_url_harness_types(self):62        test_cases = [63            ('results/shaohang/Sparse0.5_OPT-1.3/results_2023-07-19T19:10:31.005235.json', 'details',64             'https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/shaohang/Sparse0.5_OPT-1.3/details_harness%7ChendrycksTest-moral_scenarios%7C5_2023-07-19T19%3A10%3A31.005235.json'),65            ('results/shaohang/Sparse0.5_OPT-1.3/results_2023-07-19T19:10:31.005235.json', 'queries',66             'https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/shaohang/Sparse0.5_OPT-1.3/queries_harness%7ChendrycksTest-moral_scenarios%7C5_2023-07-19T19%3A10%3A31.005235.json')67        ]68 69        for file_path, harness_type, expected in test_cases:70            self.assertEqual(self.processor.build_url(file_path, harness_type), expected,71                             f"Test failed for file_path: {file_path}, harness_type: {harness_type}")72 73    def test_download_file_filename_format(self):74        url = "https://huggingface.co/datasets/open-llm-leaderboard/details/resolve/main/64bits/LexPodLM-13B/details_harness%7ChendrycksTest-moral_scenarios%7C5_2023-07-25T13%3A41%3A51.227672.json"75        directory = 'details_data'76        error_count, success_count = self.processor.download_file(url, directory)77 78        # Check that the download was successful79        self.assertEqual(success_count, 1)80        self.assertEqual(error_count, 0)81 82        # Expected file name83        expected_file_name = "64bits_LexPodLM-13B_moral_scenarios.json"84 85        # Check that the file was created with the expected name86        self.assertTrue(expected_file_name in os.listdir(directory),87                        f"File with expected name {expected_file_name} not found in directory {directory}")88 89if __name__ == '__main__':90    unittest.main()