chendl/compositional_test

1## The relevant files are currently on a shared Google2## drive at https://drive.google.com/drive/folders/1kC0I2UGl2ltrluI9NqDjaQJGw5iliw_J3## Monitor for changes and eventually migrate to use the `datasets` library4curl -L 'https://drive.google.com/uc?export=download&id=1Jjhbal535VVz2ap4v4r_rN1UEHTdLK5P' \5| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > train.txt.tmp6curl -L 'https://drive.google.com/uc?export=download&id=1ZfRcQThdtAR5PPRjIDtrVP7BtXSCUBbm' \7| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > dev.txt.tmp8curl -L 'https://drive.google.com/uc?export=download&id=1u9mb7kNJHWQCWyweMDRMuTFoOHOfeBTH' \9| grep -v "^#" | cut -f 2,3 | tr '\t' ' ' > test.txt.tmp10 11export MAX_LENGTH=12812export BERT_MODEL=bert-base-multilingual-cased13python3 scripts/preprocess.py train.txt.tmp $BERT_MODEL $MAX_LENGTH > train.txt14python3 scripts/preprocess.py dev.txt.tmp $BERT_MODEL $MAX_LENGTH > dev.txt15python3 scripts/preprocess.py test.txt.tmp $BERT_MODEL $MAX_LENGTH > test.txt16cat train.txt dev.txt test.txt | cut -d " " -f 2 | grep -v "^$"| sort | uniq > labels.txt17export OUTPUT_DIR=germeval-model18export BATCH_SIZE=3219export NUM_EPOCHS=320export SAVE_STEPS=75021export SEED=122 23python3 run_ner.py \24--task_type NER \25--data_dir . \26--labels ./labels.txt \27--model_name_or_path $BERT_MODEL \28--output_dir $OUTPUT_DIR \29--max_seq_length  $MAX_LENGTH \30--num_train_epochs $NUM_EPOCHS \31--per_gpu_train_batch_size $BATCH_SIZE \32--save_steps $SAVE_STEPS \33--seed $SEED \34--do_train \35--do_eval \36--do_predict37