|
{ |
|
"version": "1.0", |
|
"truncation": null, |
|
"padding": null, |
|
"added_tokens": [ |
|
{ |
|
"id": 0, |
|
"content": "[BOS]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 1, |
|
"content": "[UNK]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
}, |
|
{ |
|
"id": 2, |
|
"content": "[EOS]", |
|
"single_word": false, |
|
"lstrip": false, |
|
"rstrip": false, |
|
"normalized": false, |
|
"special": true |
|
} |
|
], |
|
"normalizer": null, |
|
"pre_tokenizer": null, |
|
"post_processor": { |
|
"type": "TemplateProcessing", |
|
"single": [ |
|
{ |
|
"Sequence": { |
|
"id": "A", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"SpecialToken": { |
|
"id": "[EOS]", |
|
"type_id": 0 |
|
} |
|
} |
|
], |
|
"pair": [ |
|
{ |
|
"Sequence": { |
|
"id": "A", |
|
"type_id": 0 |
|
} |
|
}, |
|
{ |
|
"Sequence": { |
|
"id": "B", |
|
"type_id": 1 |
|
} |
|
} |
|
], |
|
"special_tokens": { |
|
"[EOS]": { |
|
"id": "[EOS]", |
|
"ids": [ |
|
2 |
|
], |
|
"tokens": [ |
|
"[EOS]" |
|
] |
|
} |
|
} |
|
}, |
|
"decoder": { |
|
"type": "ByteLevel", |
|
"add_prefix_space": true, |
|
"trim_offsets": true, |
|
"use_regex": true |
|
}, |
|
"model": { |
|
"type": "BPE", |
|
"dropout": null, |
|
"unk_token": "[UNK]", |
|
"continuing_subword_prefix": null, |
|
"end_of_word_suffix": null, |
|
"fuse_unk": false, |
|
"vocab": { |
|
"[BOS]": 0, |
|
"[UNK]": 1, |
|
"[EOS]": 2, |
|
" ": 3, |
|
"#": 4, |
|
"*": 5, |
|
"-": 6, |
|
".": 7, |
|
">": 8, |
|
"A": 9, |
|
"B": 10, |
|
"C": 11, |
|
"E": 12, |
|
"L": 13, |
|
"M": 14, |
|
"N": 15, |
|
"O": 16, |
|
"P": 17, |
|
"Q": 18, |
|
"R": 19, |
|
"S": 20, |
|
"T": 21, |
|
"U": 22, |
|
"V": 23, |
|
"W": 24, |
|
"Z": 25, |
|
"`": 26, |
|
"a": 27, |
|
"b": 28, |
|
"c": 29, |
|
"d": 30, |
|
"e": 31, |
|
"f": 32, |
|
"g": 33, |
|
"h": 34, |
|
"i": 35, |
|
"j": 36, |
|
"k": 37, |
|
"l": 38, |
|
"m": 39, |
|
"n": 40, |
|
"o": 41, |
|
"p": 42, |
|
"q": 43, |
|
"r": 44, |
|
"s": 45, |
|
"t": 46, |
|
"u": 47, |
|
"v": 48, |
|
"w": 49, |
|
"x": 50, |
|
"y": 51, |
|
"z": 52, |
|
"|": 53, |
|
"us": 54, |
|
"ti": 55, |
|
"ri": 56, |
|
"um": 57, |
|
"co": 58, |
|
"ra": 59, |
|
"li": 60 |
|
}, |
|
"merges": [ |
|
"u s", |
|
"t i", |
|
"r i", |
|
"u m", |
|
"c o", |
|
"r a", |
|
"l i" |
|
] |
|
} |
|
} |