user XLMRobertaTokenizerFast

special_tokens_map.json

@@ -1,12 +1,18 @@
 {
   "additional_special_tokens": [
-    "[EOD]"
+    "<EOD>"
   ],
   "bos_token": "<s>",
-  "cls_token": "[CLS]",
+  "cls_token": "<CLS>",
   "eos_token": "</s>",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
+  "mask_token": {
+    "content": "<MASK>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<PAD>",
+  "sep_token": "<SEP>",
   "unk_token": "<unk>"
 }

tokenizer.json

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "[PAD]",
+      "content": "<PAD>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -23,7 +23,7 @@
     },
     {
       "id": 2,
-      "content": "[CLS]",
+      "content": "<CLS>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,7 +32,7 @@
     },
     {
       "id": 3,
-      "content": "[SEP]",
+      "content": "<SEP>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,16 +41,16 @@
     },
     {
       "id": 4,
-      "content": "[MASK]",
+      "content": "<MASK>",
       "single_word": false,
-      "lstrip": false,
+      "lstrip": true,
       "rstrip": false,
       "normalized": false,
       "special": true
     },
     {
       "id": 5,
-      "content": "[EOD]",
+      "content": "<EOD>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -77,11 +77,8 @@
     }
   ],
   "normalizer": {
-    "type": "BertNormalizer",
-    "clean_text": false,
-    "handle_chinese_chars": true,
-    "strip_accents": false,
-    "lowercase": false
+    "type": "Sequence",
+    "normalizers": []
   },
   "pre_tokenizer": null,
   "post_processor": {
@@ -89,7 +86,7 @@
     "single": [
       {
         "SpecialToken": {
-          "id": "[CLS]",
+          "id": "<CLS>",
           "type_id": 0
         }
       },
@@ -101,7 +98,7 @@
       },
       {
         "SpecialToken": {
-          "id": "[SEP]",
+          "id": "<SEP>",
           "type_id": 0
         }
       }
@@ -109,7 +106,7 @@
     "pair": [
       {
         "SpecialToken": {
-          "id": "[CLS]",
+          "id": "<CLS>",
           "type_id": 0
         }
       },
@@ -121,7 +118,7 @@
       },
       {
         "SpecialToken": {
-          "id": "[SEP]",
+          "id": "<SEP>",
           "type_id": 0
         }
       },
@@ -133,28 +130,28 @@
       },
       {
         "SpecialToken": {
-          "id": "[SEP]",
+          "id": "<SEP>",
           "type_id": 1
         }
       }
     ],
     "special_tokens": {
-      "[CLS]": {
-        "id": "[CLS]",
+      "<CLS>": {
+        "id": "<CLS>",
         "ids": [
           2
         ],
         "tokens": [
-          "[CLS]"
+          "<CLS>"
         ]
       },
-      "[SEP]": {
-        "id": "[SEP]",
+      "<SEP>": {
+        "id": "<SEP>",
         "ids": [
           3
         ],
         "tokens": [
-          "[SEP]"
+          "<SEP>"
         ]
       }
     }
@@ -175,7 +172,7 @@
     "unk_id": 0,
     "vocab": [
       [
-        "[PAD]",
+        "<PAD>",
         -1000.0
       ],
       [
@@ -183,19 +180,19 @@
         -1000.0
       ],
       [
-        "[CLS]",
+        "<CLS>",
         -1000.0
       ],
       [
-        "[SEP]",
+        "<SEP>",
         -1000.0
       ],
       [
-        "[MASK]",
+        "<MASK>",
         -1000.0
       ],
       [
-        "[EOD]",
+        "<EOD>",
         -1000.0
       ],
       [

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "[PAD]",
+      "content": "<PAD>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -17,7 +17,7 @@
       "special": true
     },
     "2": {
-      "content": "[CLS]",
+      "content": "<CLS>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -25,7 +25,7 @@
       "special": true
     },
     "3": {
-      "content": "[SEP]",
+      "content": "<SEP>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -33,15 +33,15 @@
       "special": true
     },
     "4": {
-      "content": "[MASK]",
-      "lstrip": false,
+      "content": "<MASK>",
+      "lstrip": true,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
     "5": {
-      "content": "[EOD]",
+      "content": "<EOD>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -66,21 +66,17 @@
     }
   },
   "additional_special_tokens": [
-    "[EOD]"
+    "<EOD>"
   ],
   "bos_token": "<s>",
-  "clean_text": false,
   "clean_up_tokenization_spaces": false,
-  "cls_token": "[CLS]",
-  "do_lower_case": false,
-  "eod_token": "[EOD]",
+  "cls_token": "<CLS>",
+  "eod_token": "<EOD>",
   "eos_token": "</s>",
-  "mask_token": "[MASK]",
+  "mask_token": "<MASK>",
   "model_max_length": 2048,
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "strip_accents": false,
-  "tokenize_chinese_chars": true,
-  "tokenizer_class": "BertTokenizerFast",
+  "pad_token": "<PAD>",
+  "sep_token": "<SEP>",
+  "tokenizer_class": "XLMRobertaTokenizer",
   "unk_token": "<unk>"
 }