asahi417/multi-domain-document-classification

1from statistics import mean2import pandas as pd3from datasets import load_dataset4 5 6def count_word(text):7    return len(text.split())8 9 10if __name__ == '__main__':11    data = ["chemprot", "citation_intent", "hyperpartisan_news", "rct_sample", "sciie", "amcd", 'yelp_review',12            'tweet_eval_irony', 'tweet_eval_hate', 'tweet_eval_emotion']13    stats = {}14    for d in data:15        _data = load_dataset('asahi417/multi_domain_document_classification', d)16        stats[d] = {17            'word/validation': mean([count_word(k['text']) for k in _data['validation']]),18            'word/test': mean([count_word(k['text']) for k in _data['test']]),19            'word/train': mean([count_word(k['text']) for k in _data['train']]),20            'instance/validation': len(_data['validation']),21            'instance/test': len(_data['test']),22            'instance/train': len(_data['train'])23        }24    df = pd.DataFrame(stats).astype(int)25    df.to_csv('stats.csv')26    print(df.to_markdown())27