Add tokenizer

special_tokens_map.json

@@ -18,7 +18,7 @@
     "single_word": false
   },
   "pad_token": {
-    "content": "<eos>",
+    "content": "<pad>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer_config.json

@@ -2000,11 +2000,11 @@
     "<end_of_turn>"
   ],
   "bos_token": "<bos>",
-  "chat_template": "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}",
+  "chat_template": "\n{%- macro render_json(d, indent=4) -%}\n{%- if d is string %}\n{{ '\"' + d + '\"' }}\n{%- elif d is mapping %}\n{%- for key, value in d.items() %}\n{%- if value is string %}\n{{ \" \" * indent + '\"' + key + '\": \"' + value + '\"' }}\n{%- elif value is mapping %}\n{{ \" \" * indent + '\"' + key + '\": {' }}\n{{ render_json(value, indent + 4) }}\n{{ \" \" * indent + \"}\" }}\n{%- elif value is sequence %}\n{{ \" \" * indent + '\"' + key + '\": [\n' }}\n{%- for item in value %}\n{{- \" \" * (indent + 4) + render_json(item, indent + 4) }}\n{%- if not loop.last %},\n{% endif %}\n{%- endfor %}\n{{ '\n' + \" \" * indent + \"]\" }}\n{%- else %}\n{{ \" \" * indent + '\"' + key + '\": ' + value|string }}\n{%- endif %}\n{%- if not loop.last %},\n{% endif %}\n{%- endfor %}\n{%- elif d is sequence %}\n{%- for item in d %}\n{{ \" \" * indent + render_json(item, indent + 4) }}\n{%- if not loop.last %},\n{% endif %}\n{%- endfor %}\n{%- else %}\n{{ \" \" * indent + d|string }}\n{%- endif %}\n{%- endmacro %}\n\n\n{#- This block checks system prompt and list of tools #}\n{%- if not add_generation_prompt is defined %}\n    {%- set add_generation_prompt = false %}\n{%- endif %}\n{%- if messages[0][\"role\"] == \"system\" %}\n    {%- set system_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n\n{#- This block checks for alternating user/model messages, skipping tool calling messages #}\n{%- set ns = namespace() %}\n{%- set ns.index = 0 %}\n{%- for message in loop_messages %}\n    {%- if (message[\"role\"] == \"user\" or message[\"role\"] == \"function-response\") != (ns.index % 2 == 0) %}\n        {{- raise_exception(\"After the optional system message, conversation roles must alternate user/model/user/model/... NB: 'function-call' is same as the 'model' and 'function-response' is same as the 'user'!\") }}\n    {%- endif %}\n    {%- set ns.index = ns.index + 1 %}\n{%- endfor %}\n\n\n{#- This block format full dialogue messages #}\n{{- bos_token }}\n{{- \"<start_of_turn>user\" + '\n' }}\n{%- if system_message != \"\" %}\n    {{- system_message }}\n{%- endif %}\n{%- if tools is not none %}\n    {%- for tool in tools %}\n        {%- set tool = tool.function %}\n        {{- '{\n' }}\n        {{- render_json(tool, 4) }}\n        {%- if not loop.last %}\n            {{- \"\n},\n\" }}\n        {%- else %}\n            {{- \"\n}\n\" }}\n        {% endif %}\n    {%- endfor %}\n    {%- elif system_message != \"\" %}\n        {{- '\n\n' }}\n{%- endif %}\n{{ loop_messages[0]['content'] + '<end_of_turn>\n' }}\n{%- for message in loop_messages[1:] %}\n    {%- if message[\"role\"] == \"user\" %}\n        {{- \"<start_of_turn>user\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"model\" %}\n        {{- \"<start_of_turn>model\" + '\n' + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"function-call\" %}\n        {{- \"<start_of_turn>model\" + '\n' + \"Function call: \" + message['content'] + '<end_of_turn>\n' }}\n    {%- elif message[\"role\"] == \"function-response\" %}\n        {{- \"<start_of_turn>user\" + '\n' + \"Function response: \" + message['content'] + '<end_of_turn>\n' }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {%- if loop_messages[-1]['role'] == \"user\" or loop_messages[-1]['role'] == \"function-response\" %}\n{{ '<start_of_turn>model\n' }}\n    {%- else %}\n        {{- raise_exception(\"For add_generation_prompt=True, the last message role must be 'user' or 'function-response'.\") }}\n    {%- endif %}\n{%- endif %}\n\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<eos>",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<eos>",
+  "pad_token": "<pad>",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "GemmaTokenizer",