chrisjay/mnist-adversarial
10
1 2import json3import hashlib4import random5import string6import warnings7import matplotlib.pyplot as plt8 9TITLE = "# MNIST Adversarial: Try to fool this MNIST model"10description = """This project is about dynamic adversarial data collection (DADC). 11The basic idea is to collect “adversarial data” - the kind of data that is difficult for a model to predict correctly. 12This kind of data is presumably the most valuable for a model, so this can be helpful in low-resource settings where data is hard to collect and label.13"""14WHAT_TO_DO="""15### What to do:161. Draw any number from 0-9. The model will automatically try to predict it after drawing.172. If the model misclassifies it, Flag that example.183. This will add your (adversarial) example to a dataset on which the model will be trained later.194. The model will finetune on the adversarial samples after every __{num_samples}__ samples have been generated. 20"""21 22MODEL_IS_WRONG = """23--- 24### Did the model get it wrong or has a low confidence? Choose the correct prediction below and flag it. When you flag it, the instance is saved [here](https://huggingface.co/datasets/chrisjay/mnist-adversarial-dataset) and the model learns from it periodically.25"""26DEFAULT_TEST_METRIC = "<html> Current test metric - Avg. loss: 1000, Accuracy: 30/1000 (30%) </html>"27 28DASHBOARD_EXPLANATION="To test the effect of adversarial training on out-of-distribution data, we track the performance progress of the model on the [MNIST Corrupted test dataset](https://zenodo.org/record/3239543). We are using {TEST_PER_SAMPLE} samples per digit."29DASHBOARD_EXPLANATION_TEST="Test accuracy on out-of-distribution data for all numbers combined."30 31STATS_EXPLANATION = "Here is the distribution of the __{num_adv_samples}__ adversarial samples we've got. The dataset can be found [here](https://huggingface.co/datasets/chrisjay/mnist-adversarial-dataset)."32 33def get_unique_name():34 return ''.join([random.choice(string.ascii_letters35 + string.digits) for n in range(32)])36 37 38def read_json(file):39 with open(file,'r',encoding="utf8") as f:40 return json.load(f)41 42def read_json_lines(file):43 try:44 with open(file,'r',encoding="utf8") as f:45 lines = f.readlines()46 data=[]47 for l in lines:48 data.append(json.loads(l))49 return data50 except Exception as err:51 warnings.warn(f"{err}")52 return None 53 54 55def json_dump(thing):56 return json.dumps(thing,57 ensure_ascii=False,58 sort_keys=True,59 indent=None,60 separators=(',', ':'))61 62def get_hash(thing): # stable-hashing63 return str(hashlib.md5(json_dump(thing).encode('utf-8')).hexdigest())64 65 66def dump_json(thing,file):67 with open(file,'w+',encoding="utf8") as f:68 json.dump(thing,f)69 70 71def plot_bar(value,name,x_name,y_name,title,set_yticks=False,set_xticks=False):72 fig, ax = plt.subplots(tight_layout=True)73 74 ax.set(xlabel=x_name, ylabel=y_name,title=title)75 76 if set_yticks:77 ax.set_yticks(range(min(name), max(name)+1, 1))78 if set_xticks:79 ax.set_xticks(range(min(name), max(name)+1, 1))80 81 82 ax.barh(name, value)83 84 return ax.figure 85