APPFL/Illinois_load_datasets

1import os, sys2import torch3import numpy as np4 5# import the dataset generation functions6from custom_dataset import get_data_and_generate_train_val_test_sets as multivariate_dataset7from custom_dataset_univariate import get_data_and_generate_train_val_test_sets as univariate_dataset8 9# names of features10feat_names = ['energy consumption (kwh)', '15-min interval of day [0..96]', 'day of week [0..6]', 'temperature (celsius)', 'windspeed (m/s)', 'floor area (ft2)', 'wall area (m2)', 'window area (m2)']11 12# load raw numpy data13heterogenous_data, homogenous_data = np.load('./IllinoisHeterogenous.npz')['data'], np.load('./IllinoisHomogenous.npz')['data']14 15# generate train-val-test datasets16# CASE 1: multivariate, with the time indices also normalized. heterogenous dataset17train_1, val_1, test_1, mean_1, std_1 = multivariate_dataset(18    data_array=heterogenous_data, # choose the appropriate file - homogenous or heterogenous19    split_ratios=[0.8,0.1,0.1], # ratios that add up to 1 - the split is made along all buildings' time axis20    dataset_kwargs={21        'num_bldg': heterogenous_data.shape[0],22        'lookback': 512,23        'lookahead': 48,24        'normalize': True,25        'dtype': torch.float32,26        'transformer': False # time indices are not normalized - use in non-Transformer scenarios where index embedding is not needed27    }28)29# CASE 2: multivariate, with the time indices not normalized. heterogenous dataset30train_2, val_2, test_2, mean_2, std_2 = multivariate_dataset(31    data_array=heterogenous_data, # choose the appropriate file - homogenous or heterogenous32    split_ratios=[0.8,0.1,0.1], # ratios that add up to 1 - the split is made along all buildings' time axis33    dataset_kwargs={34        'num_bldg': heterogenous_data.shape[0],35        'lookback': 512,36        'lookahead': 48,37        'normalize': True,38        'dtype': torch.float32,39        'transformer': True # time indices are normalized - use in Transformer scenarios where index is embedded40    }41)42# CASE 3: univariate. heterogenous dataset43train_3, val_3, test_3, mean_3, std_3 = univariate_dataset(44    data_array=heterogenous_data, # choose the appropriate file - homogenous or heterogenous45    split_ratios=[0.8,0.1,0.1], # ratios that add up to 1 - the split is made along all buildings' time axis46    dataset_kwargs={47        'num_bldg': heterogenous_data.shape[0],48        'lookback': 512,49        'lookahead': 48,50        'normalize': True,51        'dtype': torch.float32,52    }53)54# CASE 4: multivariate, with the time indices also normalized. homogenous dataset55train_4, val_4, test_4, mean_4, std_4 = multivariate_dataset(56    data_array=homogenous_data, # choose the appropriate file - homogenous or heterogenous57    split_ratios=[0.8,0.1,0.1], # ratios that add up to 1 - the split is made along all buildings' time axis58    dataset_kwargs={59        'num_bldg': homogenous_data.shape[0],60        'lookback': 512,61        'lookahead': 48,62        'normalize': True,63        'dtype': torch.float32,64        'transformer': False # time indices are not normalized - use in non-Transformer scenarios where index embedding is not needed65    }66)67# CASE 5: multivariate, with the time indices not normalized. homogenous dataset68train_5, val_5, test_5, mean_5, std_5 = multivariate_dataset(69    data_array=homogenous_data, # choose the appropriate file - homogenous or heterogenous70    split_ratios=[0.8,0.1,0.1], # ratios that add up to 1 - the split is made along all buildings' time axis71    dataset_kwargs={72        'num_bldg': homogenous_data.shape[0],73        'lookback': 512,74        'lookahead': 48,75        'normalize': True,76        'dtype': torch.float32,77        'transformer': True # time indices are normalized - use in Transformer scenarios where index is embedded78    }79)80# CASE 6: multivariate. heterogenous dataset81train_6, val_6, test_6, mean_6, std_6 = univariate_dataset(82    data_array=homogenous_data, # choose the appropriate file - homogenous or heterogenous83    split_ratios=[0.8,0.1,0.1], # ratios that add up to 1 - the split is made along all buildings' time axis84    dataset_kwargs={85        'num_bldg': homogenous_data.shape[0],86        'lookback': 512,87        'lookahead': 48,88        'normalize': True,89        'dtype': torch.float32,90    }91)92 93 94if __name__ == "__main__":95 96    # Create dataloaders97    dl_1 = torch.utils.data.DataLoader(train_1, batch_size=32, shuffle=False)98    dl_2 = torch.utils.data.DataLoader(train_2, batch_size=32, shuffle=False)99    dl_3 = torch.utils.data.DataLoader(train_3, batch_size=32, shuffle=False)100    dl_4 = torch.utils.data.DataLoader(train_4, batch_size=32, shuffle=False)101    dl_5 = torch.utils.data.DataLoader(train_5, batch_size=32, shuffle=False)102    dl_6 = torch.utils.data.DataLoader(train_6, batch_size=32, shuffle=False)103 104    # print out of the shapes of elements in the first dataloader105    for inp, label, future_time in dl_1:106        print("Case 1: Each dataloader item contains input, label, future_time. Here time indices are normalized. Dataset is IL-HET.")107        print(f"Input shape is (including batch size of 32): {inp.shape}.")108        print(f"Label shape is (including batch size of 32): {label.shape}.")109        print(f"Future time shape is (including batch size of 32): {future_time.shape}.\n")110        for m,s,n,i in zip(mean_1.flatten().tolist(),std_1.flatten().tolist(), feat_names, range(1,len(feat_names)+1)):111            print(f"Feature number: {i}, name: {n}, mean: {m}, std: {s}."+("(unnormalized)" if m==0 and s==1 else ""))112        print('----------------\n')113        break114 115    # print out of the shapes of elements in the second dataloader116    for inp, label, future_time in dl_2:117        print("Case 2: Each dataloader item contains input, label, future_time. Here time indices are not normalized to allow embedding. Dataset is IL-HET.")118        print(f"Input shape is (including batch size of 32): {inp.shape}.")119        print(f"Label shape is (including batch size of 32): {label.shape}.")120        print(f"Future time shape is (including batch size of 32): {future_time.shape}.\n")121        for m,s,n,i in zip(mean_2.flatten().tolist(),std_2.flatten().tolist(), feat_names, range(1,len(feat_names)+1)):122            print(f"Feature number: {i}, name: {n}, mean: {m}, std: {s}."+("(unnormalized)" if m==0 and s==1 else ""))123        print('----------------\n')124        break125 126    # print out of the shapes of elements in the third dataloader127    for inp, label in dl_3:128        print("Case 3: Each dataloader item contains input, label. Dataset is IL-HET.")129        print(f"Input shape is (including batch size of 32): {inp.shape}.")130        print(f"Label shape is (including batch size of 32): {label.shape}.\n")131        print(f"Feature number: 1, name: {feat_names[0]}, mean: {mean_3.item()}, std: {std_3.item()}.")132        print('----------------\n')133        break134 135    # print out of the shapes of elements in the first dataloader136    for inp, label, future_time in dl_4:137        print("Case 4: Each dataloader item contains input, label, future_time. Here time indices are normalized. Dataset is IL-HOM.")138        print(f"Input shape is (including batch size of 32): {inp.shape}.")139        print(f"Label shape is (including batch size of 32): {label.shape}.")140        print(f"Future time shape is (including batch size of 32): {future_time.shape}.\n")141        for m,s,n,i in zip(mean_4.flatten().tolist(),std_4.flatten().tolist(), feat_names, range(1,len(feat_names)+1)):142            print(f"Feature number: {i}, name: {n}, mean: {m}, std: {s}."+("(unnormalized)" if m==0 and s==1 else ""))143        print('----------------\n')144        break145 146    # print out of the shapes of elements in the second dataloader147    for inp, label, future_time in dl_5:148        print("Case 5: Each dataloader item contains input, label, future_time. Here time indices are not normalized to allow embedding. Dataset is IL-HOM.")149        print(f"Input shape is (including batch size of 32): {inp.shape}.")150        print(f"Label shape is (including batch size of 32): {label.shape}.")151        print(f"Future time shape is (including batch size of 32): {future_time.shape}.\n")152        for m,s,n,i in zip(mean_5.flatten().tolist(),std_5.flatten().tolist(), feat_names, range(1,len(feat_names)+1)):153            print(f"Feature number: {i}, name: {n}, mean: {m}, std: {s}."+("(unnormalized)" if m==0 and s==1 else ""))154        print('----------------\n')155        break156 157    # print out of the shapes of elements in the third dataloader158    for inp, label in dl_6:159        print("Case 6: Each dataloader item contains input, label. Dataset is IL-HOM.")160        print(f"Input shape is (including batch size of 32): {inp.shape}.")161        print(f"Label shape is (including batch size of 32): {label.shape}.")162        print(f"Feature number: 1, name: {feat_names[0]}, mean: {mean_6.item()}, std: {std_6.item()}.")163        print('----------------\n')164        break165 166