{ | |
"version": "1.0", | |
"truncation": null, | |
"padding": null, | |
"added_tokens": [ | |
{ | |
"id": 0, | |
"content": "[BOS]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 1, | |
"content": "[UNK]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
}, | |
{ | |
"id": 2, | |
"content": "[EOS]", | |
"single_word": false, | |
"lstrip": false, | |
"rstrip": false, | |
"normalized": false, | |
"special": true | |
} | |
], | |
"normalizer": null, | |
"pre_tokenizer": null, | |
"post_processor": { | |
"type": "TemplateProcessing", | |
"single": [ | |
{ | |
"Sequence": { | |
"id": "A", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"SpecialToken": { | |
"id": "[EOS]", | |
"type_id": 0 | |
} | |
} | |
], | |
"pair": [ | |
{ | |
"Sequence": { | |
"id": "A", | |
"type_id": 0 | |
} | |
}, | |
{ | |
"Sequence": { | |
"id": "B", | |
"type_id": 1 | |
} | |
} | |
], | |
"special_tokens": { | |
"[EOS]": { | |
"id": "[EOS]", | |
"ids": [ | |
2 | |
], | |
"tokens": [ | |
"[EOS]" | |
] | |
} | |
} | |
}, | |
"decoder": { | |
"type": "ByteLevel", | |
"add_prefix_space": true, | |
"trim_offsets": true, | |
"use_regex": true | |
}, | |
"model": { | |
"type": "BPE", | |
"dropout": null, | |
"unk_token": "[UNK]", | |
"continuing_subword_prefix": null, | |
"end_of_word_suffix": null, | |
"fuse_unk": false, | |
"vocab": { | |
"[BOS]": 0, | |
"[UNK]": 1, | |
"[EOS]": 2, | |
" ": 3, | |
"#": 4, | |
"*": 5, | |
"-": 6, | |
".": 7, | |
">": 8, | |
"A": 9, | |
"B": 10, | |
"C": 11, | |
"E": 12, | |
"L": 13, | |
"M": 14, | |
"N": 15, | |
"O": 16, | |
"P": 17, | |
"Q": 18, | |
"R": 19, | |
"S": 20, | |
"T": 21, | |
"U": 22, | |
"V": 23, | |
"W": 24, | |
"Z": 25, | |
"`": 26, | |
"a": 27, | |
"b": 28, | |
"c": 29, | |
"d": 30, | |
"e": 31, | |
"f": 32, | |
"g": 33, | |
"h": 34, | |
"i": 35, | |
"j": 36, | |
"k": 37, | |
"l": 38, | |
"m": 39, | |
"n": 40, | |
"o": 41, | |
"p": 42, | |
"q": 43, | |
"r": 44, | |
"s": 45, | |
"t": 46, | |
"u": 47, | |
"v": 48, | |
"w": 49, | |
"x": 50, | |
"y": 51, | |
"z": 52, | |
"|": 53, | |
"us": 54, | |
"ti": 55, | |
"ri": 56, | |
"um": 57, | |
"co": 58, | |
"ra": 59, | |
"li": 60 | |
}, | |
"merges": [ | |
"u s", | |
"t i", | |
"r i", | |
"u m", | |
"c o", | |
"r a", | |
"l i" | |
] | |
} | |
} |