LumenAI/trainer_test2

1{2  "additional_special_tokens": [3    "<s_iitcdip>",4    "<s_synthdog>"5  ],6  "bos_token": {7    "content": "<s>",8    "lstrip": false,9    "normalized": false,10    "rstrip": false,11    "single_word": false12  },13  "cls_token": {14    "content": "<s>",15    "lstrip": false,16    "normalized": false,17    "rstrip": false,18    "single_word": false19  },20  "eos_token": {21    "content": "</s>",22    "lstrip": false,23    "normalized": false,24    "rstrip": false,25    "single_word": false26  },27  "mask_token": {28    "content": "<mask>",29    "lstrip": true,30    "normalized": true,31    "rstrip": false,32    "single_word": false33  },34  "pad_token": {35    "content": "<pad>",36    "lstrip": false,37    "normalized": false,38    "rstrip": false,39    "single_word": false40  },41  "sep_token": {42    "content": "</s>",43    "lstrip": false,44    "normalized": false,45    "rstrip": false,46    "single_word": false47  },48  "unk_token": {49    "content": "<unk>",50    "lstrip": false,51    "normalized": false,52    "rstrip": false,53    "single_word": false54  }55}56