Update tokenizer_config.json
Browse filesAdded Tekken v7 style chat template
- tokenizer_config.json +1 -0
tokenizer_config.json
CHANGED
|
@@ -9008,6 +9008,7 @@
|
|
| 9008 |
],
|
| 9009 |
"bos_token": "<s>",
|
| 9010 |
"clean_up_tokenization_spaces": false,
|
|
|
|
| 9011 |
"eos_token": "</s>",
|
| 9012 |
"extra_special_tokens": {},
|
| 9013 |
"legacy": true,
|
|
|
|
| 9008 |
],
|
| 9009 |
"bos_token": "<s>",
|
| 9010 |
"clean_up_tokenization_spaces": false,
|
| 9011 |
+
"chat_template": "{{- bos_token }}{%- if messages[0]['role'] == 'system' %}{{- '[SYSTEM_PROMPT]' + messages[0]['content'] + '[/SYSTEM_PROMPT]' }}{%- set loop_messages = messages[1:] %}{%- else %}{%- set loop_messages = messages %}{%- endif %}{%- for message in loop_messages %}{%- if message['role'] == 'user' %}{{- '[INST]' + message['content'] + '[/INST]' }}{%- elif message['role'] == 'system' %}{{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}{%- elif message['role'] == 'assistant' %}{{- message['content'] + eos_token }}{%- else %}{{- raise_exception('Only user, system and assistant roles are supported!') }}{%- endif %}{%- endfor %}",
|
| 9012 |
"eos_token": "</s>",
|
| 9013 |
"extra_special_tokens": {},
|
| 9014 |
"legacy": true,
|