research-backup/conceptnet_high_confidence

1import pandas as pd2from datasets import load_dataset3 4data = load_dataset('relbert/conceptnet_high_confidence')5stats = []6for k in data.keys():7    for i in data[k]:8        stats.append({'relation_type': i['relation_type'], 'split': k, 'positives': len(i['positives']), 'negatives': len(i['negatives'])})9df = pd.DataFrame(stats)10df_train = df[df['split'] == 'train']11df_valid = df[df['split'] == 'validation']12stats = []13for r in df['relation_type'].unique():14    _df_t = df_train[df_train['relation_type'] == r]15    _df_v = df_valid[df_valid['relation_type'] == r]16    stats.append({17        'relation_type': r,18        'positive (train)': 0 if len(_df_t) == 0 else _df_t['positives'].values[0],19        'negative (train)': 0 if len(_df_t) == 0 else _df_t['negatives'].values[0],20        'positive (validation)': 0 if len(_df_v) == 0 else _df_v['positives'].values[0],21        'negative (validation)': 0 if len(_df_v) == 0 else _df_v['negatives'].values[0],22    })23 24df = pd.DataFrame(stats).sort_values(by=['relation_type'])25df.index = df.pop('relation_type')26sum_pairs = df.sum(0)27df = df.T28df['SUM'] = sum_pairs29df = df.T30 31df.to_csv('stats.csv')32with open('stats.md', 'w') as f:33    f.write(df.to_markdown())34 35 36