ko-yoshida/sample1
02.7k
1{2 "version": "1.0",3 "truncation": null,4 "padding": null,5 "added_tokens": [6 {7 "id": 0,8 "content": "<pad>",9 "single_word": false,10 "lstrip": false,11 "rstrip": false,12 "normalized": false,13 "special": true14 },15 {16 "id": 1,17 "content": "<cls>",18 "single_word": false,19 "lstrip": false,20 "rstrip": false,21 "normalized": false,22 "special": true23 },24 {25 "id": 2,26 "content": "<sep>",27 "single_word": false,28 "lstrip": false,29 "rstrip": false,30 "normalized": false,31 "special": true32 },33 {34 "id": 3,35 "content": "<s>",36 "single_word": false,37 "lstrip": false,38 "rstrip": false,39 "normalized": false,40 "special": true41 },42 {43 "id": 4,44 "content": "</s>",45 "single_word": false,46 "lstrip": false,47 "rstrip": false,48 "normalized": false,49 "special": true50 },51 {52 "id": 5,53 "content": "<unk>",54 "single_word": false,55 "lstrip": false,56 "rstrip": false,57 "normalized": false,58 "special": true59 }60 ],61 "normalizer": {62 "type": "NFKC"63 },64 "pre_tokenizer": {65 "type": "Metaspace",66 "replacement": "▁",67 "prepend_scheme": "always",68 "split": true69 },70 "post_processor": {71 "type": "TemplateProcessing",72 "single": [73 {74 "SpecialToken": {75 "id": "<s>",76 "type_id": 077 }78 },79 {80 "Sequence": {81 "id": "A",82 "type_id": 083 }84 },85 {86 "SpecialToken": {87 "id": "</s>",88 "type_id": 089 }90 }91 ],92 "pair": [93 {94 "SpecialToken": {95 "id": "<s>",96 "type_id": 097 }98 },99 {100 "Sequence": {101 "id": "A",102 "type_id": 0103 }104 },105 {106 "SpecialToken": {107 "id": "</s>",108 "type_id": 0109 }110 },111 {112 "SpecialToken": {113 "id": "<s>",114 "type_id": 1115 }116 },117 {118 "Sequence": {119 "id": "B",120 "type_id": 1121 }122 },123 {124 "SpecialToken": {125 "id": "</s>",126 "type_id": 1127 }128 }129 ],130 "special_tokens": {131 "</s>": {132 "id": "</s>",133 "ids": [134 4135 ],136 "tokens": [137 "</s>"138 ]139 },140 "<s>": {141 "id": "<s>",142 "ids": [143 3144 ],145 "tokens": [146 "<s>"147 ]148 }149 }150 },151 "decoder": null,152 "model": {153 "type": "Unigram",154 "unk_id": 5,155 "vocab": [156 [157 "<pad>",158 0.0159 ],160 [161 "<cls>",162 0.0163 ],164 [165 "<sep>",166 0.0167 ],168 [169 "<s>",170 0.0171 ],172 [173 "</s>",174 0.0175 ],176 [177 "<unk>",178 0.0179 ],180 [181 "▁",182 -2.9447451172333707183 ],184 [185 "▁the",186 -3.255716666652152187 ],188 [189 "▁and",190 -3.3952242732350513191 ],192 [193 "a",194 -3.487301029869881195 ],196 [197 "▁to",198 -3.595536420969363199 ],200 [201 "▁was",202 -4.08733750767742203 ],204 [205 ".",206 -4.370001070946293207 ],208 [209 ",",210 -4.595086522544255211 ],212 [213 "▁it",214 -4.73979799920402215 ],216 [217 "\n",218 -4.743481399036691219 ],220 [221 "▁The",222 -4.749280203433306223 ],224 [225 "▁She",226 -4.837218290653265227 ],228 [229 "▁He",230 -4.865643299153314231 ],232 [233 "s",234 -4.876993864600209235 ],236 [237 "▁They",238 -4.920866210489274239 ],240 [241 "▁her",242 -4.921809059345181243 ],244 [245 "▁in",246 -4.922214243238258247 ],248 [249 "▁you",250 -4.950753686101075251 ],252 [253 "▁with",254 -4.994948170903607255 ],256 [257 "▁of",258 -5.035126123859852259 ],260 [261 "▁he",262 -5.1092391356927263 ],264 [265 "I",266 -5.116824396502212267 ],268 [269 "▁for",270 -5.193760498673701271 ],272 [273 "▁that",274 -5.199856159764728275 ],276 [277 "▁she",278 -5.276579793627182279 ],280 [281 "▁his",282 -5.289850856084543283 ],284 [285 "▁is",286 -5.345648427662905287 ],288 [289 "▁so",290 -5.36626455400415291 ],292 [293 "▁on",294 -5.379244563220475295 ],296 [297 "▁they",298 -5.392491319518285299 ],300 [301 "t",302 -5.428056602887686303 ],304 [305 "▁had",306 -5.478295460993962307 ],308 [309 "!",310 -5.554146124156585311 ],312 [313 "▁can",314 -5.609472542745726315 ],316 [317 "▁but",318 -5.617384679208659319 ],320 [321 "▁Lily",322 -5.620128292191268323 ],324 [325 "▁not",326 -5.645349642931274327 ],328 [329 "▁little",330 -5.664697437843602331 ],332 [333 "▁we",334 -5.692465048353892335 ],336 [337 "▁be",338 -5.706739713433828339 ],340 [341 "▁very",342 -5.728724477651241343 ],344 [345 "▁It",346 -5.746357624615218347 ],348 [349 "▁like",350 -5.7530622453259035351 ],352 [353 "▁have",354 -5.806694142971276355 ],356 [357 "▁there",358 -5.830677680260774359 ],360 [361 "▁big",362 -5.84999814158517363 ],364 [365 "▁are",366 -5.854296116462564367 ],368 [369 "▁said,",370 -5.8798455515944354371 ],372 [373 "▁But",374 -5.89002317691337375 ],376 [377 "▁saw",378 -5.907839325248927379 ],380 [381 "▁play",382 -5.921163059584545383 ],384 [385 "▁day,",386 -5.932758393818494387 ],388 [389 "▁were",390 -5.94561316098981391 ],392 [393 "▁One",394 -5.984141588355236395 ],396 [397 "!\"",398 -6.002704493300557399 ],400 [401 "▁your",402 -6.00844921972341403 ],404 [405 "▁their",406 -6.056894657722175407 ],408 [409 "▁all",410 -6.058750369392342411 ],412 [413 "▁Once",414 -6.0618853800130665415 ],416 [417 "▁mom",418 -6.0736712860548465419 ],420 [421 "?",422 -6.094935397736323423 ],424 [425 "▁go",426 -6.131936311094345427 ],428 [429 "▁up",430 -6.147313203361431431 ],432 [433 "▁about",434 -6.151926777964455435 ],436 [437 ".\"",438 -6.157619110439349439 ],440 [441 "▁wanted",442 -6.159320878935013443 ],444 [445 "▁upon",446 -6.16654678201472447 ],448 [449 "▁make",450 -6.215053230550705451 ],452 [453 "▁said",454 -6.2234996724372955455 ],456 [457 "e",458 -6.223529980213371459 ],460 [461 "▁want",462 -6.228328546525347463 ],464 [465 "▁happy",466 -6.236381219294042467 ],468 [469 "▁girl",470 -6.2640514382016471 ],472 [473 "▁time,",474 -6.312744724434008475 ],476 [477 "▁went",478 -6.330373537278994479 ],480 [481 "▁them",482 -6.351106563171857483 ],484 [485 "▁what",486 -6.363855742753893487 ],488 [489 "▁You",490 -6.38061185433142491 ],492 [493 "?\"",494 -6.392568140043814495 ],496 [497 "▁it.",498 -6.399592691155329499 ],500 [501 "▁when",502 -6.409469586570484503 ],504 [505 "▁could",506 -6.41486380019912507 ],508 [509 "▁do",510 -6.416594453229987511 ],512 [513 "▁get",514 -6.422357899571525515 ],516 [517 "▁my",518 -6.42472709516249519 ],520 [521 "▁from",522 -6.42805901632812523 ],524 [525 "▁some",526 -6.431174456332165527 ],528 [529 "▁see",530 -6.441164887553489531 ],532 [533 "d",534 -6.488287631231399535 ],536 [537 "▁help",538 -6.498633064810697539 ],540 [541 "▁Tom",542 -6.49915302857219543 ],544 [545 "▁him",546 -6.502112559368033547 ],548 [549 "▁out",550 -6.509261904721312551 ],552 [553 "▁Ben",554 -6.525541848540788555 ],556 [557 "▁named",558 -6.5357617532893535559 ],560 [561 "▁loved",562 -6.541063119494396563 ],564 [565 "▁time",566 -6.558475615085307567 ],568 [569 "▁back",570 -6.560272118453648571 ],572 [573 "▁day",574 -6.5628940421497095575 ],576 [577 "▁did",578 -6.56769130176677579 ],580 [581 "▁just",582 -6.571248555442244583 ],584 [585 "▁felt",586 -6.5772280270497845587 ],588 [589 "▁would",590 -6.579336361544888591 ],592 [593 "▁We",594 -6.586485761226735595 ],596 [597 "▁looked",598 -6.606217153236187599 ],600 [601 "▁it's",602 -6.615165531089005603 ],604 [605 "▁put",606 -6.619312543697394607 ],608 [609 "▁something",610 -6.622372575868811611 ],612 [613 "▁made",614 -6.625630090836097615 ],616 [617 "'s",618 -6.649017049146382619 ],620 [621 "▁new",622 -6.649993162611212623 ],624 [625 "▁who",626 -6.684017870153578627 ],628 [629 "▁this",630 -6.688982464012339631 ],632 [633 "▁And",634 -6.6927801393779625635 ],636 [637 "▁an",638 -6.697088611839266639 ],640 [641 "▁around",642 -6.710178409258505643 ],644 [645 "▁It's",646 -6.737394973284889647 ],648 [649 "▁started",650 -6.746626816787595651 ],652 [653 "▁found",654 -6.770753458984997655 ],656 [657 "▁if",658 -6.772886981885691659 ],660 [661 "▁how",662 -6.773824934266088663 ],664 [665 "▁got",666 -6.786535384766584667 ],668 [669 "▁boy",670 -6.789128030736062671 ],672 [673 "▁me",674 -6.790990098707081675 ],676 [677 "▁ran",678 -6.791479711647007679 ],680 [681 "▁find",682 -6.8049810131291615683 ],684 [685 "▁one",686 -6.819075522870991687 ],688 [689 "▁will",690 -6.831354432173651691 ],692 [693 "▁then",694 -6.835483805013306695 ],696 [697 "▁asked",698 -6.843225449081364699 ],700 [701 "▁smiled",702 -6.8457130987635235703 ],704 [705 "▁don't",706 -6.867638676677847707 ],708 [709 "▁Timmy",710 -6.869135912042999711 ],712 [713 "▁\"I",714 -6.8702045906689495715 ],716 [717 "▁more",718 -6.873447873550102719 ],720 [721 "▁good",722 -6.891203584228272723 ],724 [725 "▁too",726 -6.9078472510541555727 ],728 [729 "▁said.",730 -6.929573853665417731 ],732 [733 "▁or",734 -6.941695322456404735 ],736 [737 "n",738 -6.944718245466213739 ],740 [741 ",\"",742 -6.956664747409215743 ],744 [745 "▁take",746 -6.9589082401027795747 ],748 [749 "i",750 -6.965989983659428751 ],752 [753 "▁decided",754 -6.974442854143387755 ],756 [757 "▁playing",758 -6.981413072105845759 ],760 [761 "m",762 -6.9819506765486015763 ],764 [765 "ing",766 -6.982068810835557767 ],768 [769 "▁need",770 -6.982699811808171771 ],772 [773 "y",774 -6.985936472364049775 ],776 [777 "▁Can",778 -6.989339644735331779 ],780 [781 "▁always",782 -6.995194982937486783 ],784 [785 "▁friends",786 -7.00887969401936787 ],788 [789 "▁fun",790 -7.015532926795544791 ],792 [793 "▁didn't",794 -7.0264014711423215795 ],796 [797 "▁because",798 -7.032530367434898799 ],800 [801 "▁liked",802 -7.032854687757544803 ],804 [805 "▁old",806 -7.036472857700456807 ],808 [809 "o",810 -7.042263685591065811 ],812 [813 "▁Let's",814 -7.050135883571478815 ],816 [817 "▁came",818 -7.050250673276727819 ],820 [821 "▁into",822 -7.05569673041281823 ],824 [825 "▁by",826 -7.086946634480542827 ],828 [829 "▁know",830 -7.11192593441436831 ],832 [833 "▁I'm",834 -7.113830558696593835 ],836 [837 "▁our",838 -7.11453329544176839 ],840 [841 "▁other",842 -7.122006983521395843 ],844 [845 "▁let's",846 -7.12466695117762847 ],848 [849 "▁bird",850 -7.126535465745279851 ],852 [853 "▁Tim",854 -7.132409614847553855 ],856 [857 "▁took",858 -7.14359717974089859 ],860 [861 "▁happy.",862 -7.143941133942686863 ],864 [865 "▁down",866 -7.147747988085445867 ],868 [869 "▁over",870 -7.156067744584153871 ],872 [873 "▁When",874 -7.164121871042296875 ],876 [877 "▁things",878 -7.178042866337174879 ],880 [881 "\"",882 -7.182898356660175883 ],884 [885 "▁\"Yes,",886 -7.183285275144744887 ],888 [889 "▁look",890 -7.191528231188245891 ],892 [893 "▁man",894 -7.196821568351515895 ],896 [897 "▁Her",898 -7.21238405313003899 ],900 [901 "▁thought",902 -7.219442725529323903 ],904 [905 "l",906 -7.256432329600525907 ],908 [909 "h",910 -7.266857519644121911 ],912 [913 "▁Lily.",914 -7.276330478529662915 ],916 [917 "▁even",918 -7.282229601943902919 ],920 [921 "▁That's",922 -7.284867471572822923 ],924 [925 "▁I'll",926 -7.289255249177645927 ],928 [929 "▁Mom",930 -7.290540766323231931 ],932 [933 "▁friends.",934 -7.293579267561976935 ],936 [937 "r",938 -7.296425838512606939 ],940 [941 "▁each",942 -7.30369593655514943 ],944 [945 "▁knew",946 -7.318304228078171947 ],948 [949 "▁again.",950 -7.318955835624315951 ],952 [953 "▁no",954 -7.32389773829528955 ],956 [957 "▁special",958 -7.332527310784444959 ],960 [961 "▁mommy",962 -7.352028622729412963 ],964 [965 "▁them.",966 -7.368850024769735967 ],968 [969 "▁Anna",970 -7.372196716802096971 ],972 [973 "▁Sam",974 -7.375341076867555975 ],976 [977 "▁gave",978 -7.381382465684453979 ],980 [981 "▁many",982 -7.384931441296896983 ],984 [985 "▁What",986 -7.387415783912445987 ],988 [989 "▁never",990 -7.391475637994949991 ],992 [993 "▁together.",994 -7.394326677383619995 ],996 [997 "▁heard",998 -7.39672724709424999 ],1000 [1001 "▁told",1002 -7.4107861511960421003 ],1004 [1005 "▁tried",1006 -7.4124254438142551007 ],1008 [1009 "▁should",1010 -7.417160159715151011 ],1012 [1013 "▁has",1014 -7.4190205447764971015 ],1016 [1017 "▁much",1018 -7.4266080943128171019 ],1020 [1021 "▁going",1022 -7.4286116301743981023 ],1024 [1025 "▁Max",1026 -7.4302479186704021027 ],1028 [1029 "▁dog",1030 -7.4313102761562121031 ],1032 [1033 "▁toy",1034 -7.435508828972471035 ],1036 [1037 "▁played",1038 -7.4364497229946721039 ],1040 [1041 "▁sad",1042 -7.4391973084945491043 ],1044 [1045 "▁after",1046 -7.44652790868104651047 ],1048 [1049 "▁him.",1050 -7.4514719005384581051 ],1052 [1053 "▁eat",1054 -7.4527321843230911055 ],1056 [1057 "-",1058 -7.4635864663876481059 ],1060 [1061 "▁together",1062 -7.4651711441458171063 ],1064 [1065 "▁His",1066 -7.4705037220883441067 ],1068 [1069 "▁still",1070 -7.471213272483081071 ],1072 [1073 "▁love",1074 -7.4712532022238241075 ],1076 [1077 "er",1078 -7.48266060452483651079 ],1080 [1081 "▁on,",1082 -7.483523932293191083 ],1084 [1085 "▁excited",1086 -7.4908092860751181087 ],1088 [1089 "▁best",1090 -7.4922276252759531091 ],1092 [1093 "▁couldn't",1094 -7.499945520788191095 ],1096 [1097 "▁too.",1098 -7.5003709241237541099 ],1100 [1101 "▁come",1102 -7.5012287569942431103 ],1104 [1105 "▁Now,",1106 -7.5027711273349151107 ],1108 [1109 "▁car",1110 -7.506602481098461111 ],1112 [1113 "▁until",1114 -7.5106157420143481115 ],1116 [1117 "▁both",1118 -7.5218370329889891119 ],1120 [1121 "▁away",1122 -7.5241213574698591123 ],1124 [1125 "▁feel",1126 -7.5287748050571931127 ],1128 [1129 "▁outside",1130 -7.5293895632437221131 ],1132 [1133 "▁story",1134 -7.5387292429484011135 ],1136 [1137 "▁keep",1138 -7.5408685375330841139 ],1140 [1141 "▁say",1142 -7.5423015027744251143 ],1144 [1145 "▁says.",1146 -7.5450377127336451147 ],1148 [1149 "'",1150 -7.5458409069548371151 ],1152 [1153 "▁away.",1154 -7.5459225409899721155 ],1156 [1157 "▁small",1158 -7.5507916812556581159 ],1160 [1161 "▁try",1162 -7.550886319901511163 ],1164 [1165 "▁might",1166 -7.5616813107587521167 ],1168 [1169 "▁use",1170 -7.5755002074675681171 ],1172 [1173 "▁people",1174 -7.5775922733735361175 ],1176 [1177 "▁proud",1178 -7.5786007017332861179 ],1180 [1181 "▁hugged",1182 -7.5790343066819781183 ],1184 [1185 "▁From",1186 -7.5793234777983291187 ],1188 [1189 "▁Jack",1190 -7.5811481301580291191 ],1192 [1193 "▁sure",1194 -7.5824222353832941195 ],1196 [1197 "▁toys",1198 -7.5834430529717171199 ],1200 [