train tokenizer

scripts/train_tokenizer.py

@@ -1,6 +1,6 @@
 import gc
 import sys
-import string
+# import string
 
 from datasets import load_dataset
 from transformers import PreTrainedTokenizerFast
@@ -183,15 +183,15 @@ def batch_iterator():
     # gc.collect()
 
 
-bpe = BPE(unk_token='<unk>', fuse_unk=False, byte_fallback=False)
+bpe = BPE(unk_token='<unk>', fuse_unk=True, byte_fallback=True)
 tokenizer = Tokenizer(bpe)
 
 special_tokens = [
     '<unk>',
     '<s>',
     '</s>',
-    '<|im_end|>',
     '<|im_start|>',
+    '<|im_end|>',
     '<tools>',
     '</tools>',
     '<tool_call>',
@@ -202,6 +202,19 @@ special_tokens = [
     'user',
     'assistant',
     'tool',
+    
+    '"arguments"',
+    '<arguments>',
+    '<argument>',
+    '<argument-name>',
+    '<parameter>',
+    '<parameter-name>',
+    '<value>',
+    '<argument-value>',
+    '<parameter-value>',
+    '"name"',
+    '<function>',
+    '<function-name>',
 ]
 
 for i in range(2, 25):
@@ -210,21 +223,24 @@ for i in range(2, 25):
 for i in range(64 - len(special_tokens)):
     special_tokens.append(f'<|reserved_{i}|>')
 
-# ascii
-ascii_chars = list(string.ascii_letters + string.ascii_lowercase + string.ascii_uppercase + string.digits + string.punctuation)
+## ascii
+# ascii_chars = list(string.ascii_letters + string.ascii_lowercase + string.ascii_uppercase + string.digits + string.punctuation)
 
 # emoji
 dataset = load_dataset('badrex/llm-emoji-dataset', split='train')
 emoji_chars = [row['character'] for row in dataset if len(row['character']) == 1]
 del dataset
 
+# programming languages
+dataset = load_dataset('Tanvir1337/programming-languages', split='train')
+programming_languages = [n for row in dataset for n in row['text']]
+del dataset
+
 # programming languages keywords
 dataset = load_dataset('bigcode/programming-languages-keywords', split='train')
 code_keywords = [n for row in dataset for n in row['keywords']]
 del dataset
 
-tokenizer.normalizer = normalizers.NFC()
-
 tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False, trim_offsets=True, use_regex=True)
 
 tokenizer.post_processor = TemplateProcessing(
@@ -237,11 +253,9 @@ tokenizer.decoder = decoders.ByteLevel(add_prefix_space=False, trim_offsets=True
 
 trainer = BpeTrainer(
     vocab_size=32000,
-    # min_frequency=2,
+    min_frequency=2,
     special_tokens=special_tokens,
-    initial_alphabet=ascii_chars + emoji_chars + code_keywords,
-    # continuing_subword_prefix=None,
-    # end_of_word_suffix=None,
+    initial_alphabet=emoji_chars + programming_languages + code_keywords,
 )
 
 tokenizer.train_from_iterator(batch_iterator(), trainer)
@@ -261,12 +275,10 @@ fast_tokenizer = PreTrainedTokenizerFast(
     tokenizer_object=tokenizer,
     chat_template=CHATML_CHAT_TEMPLATE,
     bos_token='<s>',
-    eos_token='<|im_end|>',
+    eos_token='</s>',
     unk_token='<unk>',
     pad_token='</s>',
     clean_up_tokenization_spaces=False,
-    # spaces_between_special_tokens=False,
-    # use_default_system_prompt=False,
 )
 
-fast_tokenizer.save_pretrained('../')
+fast_tokenizer.save_pretrained('../')

special_tokens_map.json

@@ -1,6 +1,6 @@
 {
   "bos_token": "<s>",
-  "eos_token": "<|im_end|>",
+  "eos_token": "</s>",
   "pad_token": "</s>",
   "unk_token": "<unk>"
 }

tokenizer_config.json

@@ -25,7 +25,7 @@
       "special": true
     },
     "3": {
-      "content": "<|im_end|>",
+      "content": "<|im_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -33,7 +33,7 @@
       "special": true
     },
     "4": {
-      "content": "<|im_start|>",
+      "content": "<|im_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -121,7 +121,7 @@
       "special": true
     },
     "15": {
-      "content": "  ",
+      "content": "\"arguments\"",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -129,7 +129,7 @@
       "special": true
     },
     "16": {
-      "content": "   ",
+      "content": "<arguments>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -137,7 +137,7 @@
       "special": true
     },
     "17": {
-      "content": "    ",
+      "content": "<argument>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -145,7 +145,7 @@
       "special": true
     },
     "18": {
-      "content": "     ",
+      "content": "<argument-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -153,7 +153,7 @@
       "special": true
     },
     "19": {
-      "content": "      ",
+      "content": "<parameter>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -161,7 +161,7 @@
       "special": true
     },
     "20": {
-      "content": "       ",
+      "content": "<parameter-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -169,7 +169,7 @@
       "special": true
     },
     "21": {
-      "content": "        ",
+      "content": "<value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -177,7 +177,7 @@
       "special": true
     },
     "22": {
-      "content": "         ",
+      "content": "<argument-value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -185,7 +185,7 @@
       "special": true
     },
     "23": {
-      "content": "          ",
+      "content": "<parameter-value>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -193,7 +193,7 @@
       "special": true
     },
     "24": {
-      "content": "           ",
+      "content": "\"name\"",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -201,7 +201,7 @@
       "special": true
     },
     "25": {
-      "content": "            ",
+      "content": "<function>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -209,7 +209,7 @@
       "special": true
     },
     "26": {
-      "content": "             ",
+      "content": "<function-name>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -217,7 +217,7 @@
       "special": true
     },
     "27": {
-      "content": "              ",
+      "content": "  ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -225,7 +225,7 @@
       "special": true
     },
     "28": {
-      "content": "               ",
+      "content": "   ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -233,7 +233,7 @@
       "special": true
     },
     "29": {
-      "content": "                ",
+      "content": "    ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -241,7 +241,7 @@
       "special": true
     },
     "30": {
-      "content": "                 ",
+      "content": "     ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -249,7 +249,7 @@
       "special": true
     },
     "31": {
-      "content": "                  ",
+      "content": "      ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -257,7 +257,7 @@
       "special": true
     },
     "32": {
-      "content": "                   ",
+      "content": "       ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -265,7 +265,7 @@
       "special": true
     },
     "33": {
-      "content": "                    ",
+      "content": "        ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -273,7 +273,7 @@
       "special": true
     },
     "34": {
-      "content": "                     ",
+      "content": "         ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -281,7 +281,7 @@
       "special": true
     },
     "35": {
-      "content": "                      ",
+      "content": "          ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -289,7 +289,7 @@
       "special": true
     },
     "36": {
-      "content": "                       ",
+      "content": "           ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -297,7 +297,7 @@
       "special": true
     },
     "37": {
-      "content": "                        ",
+      "content": "            ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -305,7 +305,7 @@
       "special": true
     },
     "38": {
-      "content": "<|reserved_0|>",
+      "content": "             ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -313,7 +313,7 @@
       "special": true
     },
     "39": {
-      "content": "<|reserved_1|>",
+      "content": "              ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -321,7 +321,7 @@
       "special": true
     },
     "40": {
-      "content": "<|reserved_2|>",
+      "content": "               ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -329,7 +329,7 @@
       "special": true
     },
     "41": {
-      "content": "<|reserved_3|>",
+      "content": "                ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -337,7 +337,7 @@
       "special": true
     },
     "42": {
-      "content": "<|reserved_4|>",
+      "content": "                 ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -345,7 +345,7 @@
       "special": true
     },
     "43": {
-      "content": "<|reserved_5|>",
+      "content": "                  ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -353,7 +353,7 @@
       "special": true
     },
     "44": {
-      "content": "<|reserved_6|>",
+      "content": "                   ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -361,7 +361,7 @@
       "special": true
     },
     "45": {
-      "content": "<|reserved_7|>",
+      "content": "                    ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -369,7 +369,7 @@
       "special": true
     },
     "46": {
-      "content": "<|reserved_8|>",
+      "content": "                     ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -377,7 +377,7 @@
       "special": true
     },
     "47": {
-      "content": "<|reserved_9|>",
+      "content": "                      ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -385,7 +385,7 @@
       "special": true
     },
     "48": {
-      "content": "<|reserved_10|>",
+      "content": "                       ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -393,7 +393,7 @@
       "special": true
     },
     "49": {
-      "content": "<|reserved_11|>",
+      "content": "                        ",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -401,7 +401,7 @@
       "special": true
     },
     "50": {
-      "content": "<|reserved_12|>",
+      "content": "<|reserved_0|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -409,7 +409,7 @@
       "special": true
     },
     "51": {
-      "content": "<|reserved_13|>",
+      "content": "<|reserved_1|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -417,7 +417,7 @@
       "special": true
     },
     "52": {
-      "content": "<|reserved_14|>",
+      "content": "<|reserved_2|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -425,7 +425,7 @@
       "special": true
     },
     "53": {
-      "content": "<|reserved_15|>",
+      "content": "<|reserved_3|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -433,7 +433,7 @@
       "special": true
     },
     "54": {
-      "content": "<|reserved_16|>",
+      "content": "<|reserved_4|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -441,7 +441,7 @@
       "special": true
     },
     "55": {
-      "content": "<|reserved_17|>",
+      "content": "<|reserved_5|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -449,7 +449,7 @@
       "special": true
     },
     "56": {
-      "content": "<|reserved_18|>",
+      "content": "<|reserved_6|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -457,7 +457,7 @@
       "special": true
     },
     "57": {
-      "content": "<|reserved_19|>",
+      "content": "<|reserved_7|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -465,7 +465,7 @@
       "special": true
     },
     "58": {
-      "content": "<|reserved_20|>",
+      "content": "<|reserved_8|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -473,7 +473,7 @@
       "special": true
     },
     "59": {
-      "content": "<|reserved_21|>",
+      "content": "<|reserved_9|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -481,7 +481,7 @@
       "special": true
     },
     "60": {
-      "content": "<|reserved_22|>",
+      "content": "<|reserved_10|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -489,7 +489,7 @@
       "special": true
     },
     "61": {
-      "content": "<|reserved_23|>",
+      "content": "<|reserved_11|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -497,7 +497,7 @@
       "special": true
     },
     "62": {
-      "content": "<|reserved_24|>",
+      "content": "<|reserved_12|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -505,7 +505,7 @@
       "special": true
     },
     "63": {
-      "content": "<|reserved_25|>",
+      "content": "<|reserved_13|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -516,7 +516,7 @@
   "bos_token": "<s>",
   "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
+  "eos_token": "</s>",
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "</s>",
   "tokenizer_class": "PreTrainedTokenizerFast",