Undi95 commited on
Commit
38afc4e
·
verified ·
1 Parent(s): c006087

Upload folder using huggingface_hub

Browse files
.ipynb_checkpoints/tokenizer_config-checkpoint.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -9007,7 +9007,8 @@
9007
  "<SPECIAL_999>"
9008
  ],
9009
  "bos_token": "<s>",
9010
- "chat_template": "{%- set today = strftime_now(\"%Y-%m-%d\") %}\n{%- set default_system_message = \"You are MistralThinker, a Large Language Model (LLM) created by Undi.\\nYour knowledge base was last updated on 2023-10-01. The current date is \" + today + \".\\n\\nWhen you're not sure about some information, you say that you don't have the information and don't make up anything.\" %}\n\n{{- bos_token }}\n\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content'] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set system_message = default_system_message %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\n\n{%- for message in loop_messages %}\n {%- if message['role'] == 'user' %}\n {{- '[INST]' + message['content'] + '[/INST]' }}\n {%- elif message['role'] == 'system' %}\n {{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}\n {%- elif message['role'] == 'assistant' %}\n {{- message['content'] + eos_token }}\n {%- else %}\n {{- raise_exception('Only user, system and assistant roles are supported!') }}\n {%- endif %}\n{%- endfor %}",
 
9011
  "clean_up_tokenization_spaces": false,
9012
  "eos_token": "</s>",
9013
  "extra_special_tokens": {},
 
9007
  "<SPECIAL_999>"
9008
  ],
9009
  "bos_token": "<s>",
9010
+ "chat_template": "{%- set today = strftime_now(\"%Y-%m-%d\") %}{%- set default_system_message = \"You are MistralThinker, a Large Language Model (LLM) created by Undi.\\nYour knowledge base was last updated on 2023-10-01. The current date is \" + today + \".\\n\\nWhen you're not sure about some information, you say that you don't have the information and don't make up anything.\" %}{{- bos_token}}{%- if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% set loop_messages = messages[1:] %}{% else %}{% set system_message = default_system_message %}{% set loop_messages = messages %}{% endif %}{{'[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]'}}{%- for message in loop_messages %}{%- if message['role'] == 'user' %}{{'[INST]' + message['content'] + '[/INST]'}}{%- elif message['role'] == 'system' %}{{'[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]'}}{%- elif message['role'] == 'assistant' %}{% if add_generation_prompt %}{{'<think>\\n'}}{% endif %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{ content + eos_token }}{%- else %}{{ raise_exception('Only user, system and assistant roles are supported!')}}{%- endif %}{% endfor %}"
9011
+
9012
  "clean_up_tokenization_spaces": false,
9013
  "eos_token": "</s>",
9014
  "extra_special_tokens": {},