Hugging Face's logo

codellama
/
CodeLlama-34b-hf

Text Generation
Transformers
PyTorch
Safetensors
code
llama
llama-2
text-generation-inference
Model card Files
xet
Community
25

Upload tokenizer

#23
by ArthurZ HF Staff - opened
base: refs/heads/main
←
from: refs/pr/23
Discussion Files changed
+38
-26
Files changed (2) hide show
  1. special_tokens_map.json +3 -3
  2. tokenizer_config.json +35 -23
special_tokens_map.json CHANGED
@@ -2,21 +2,21 @@
2
  "bos_token": {
3
  "content": "<s>",
4
  "lstrip": false,
5
- "normalized": true,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
  "content": "</s>",
11
  "lstrip": false,
12
- "normalized": true,
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
  "unk_token": {
17
  "content": "<unk>",
18
  "lstrip": false,
19
- "normalized": true,
20
  "rstrip": false,
21
  "single_word": false
22
  }
 
2
  "bos_token": {
3
  "content": "<s>",
4
  "lstrip": false,
5
+ "normalized": false,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
  "content": "</s>",
11
  "lstrip": false,
12
+ "normalized": false,
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
  "unk_token": {
17
  "content": "<unk>",
18
  "lstrip": false,
19
+ "normalized": false,
20
  "rstrip": false,
21
  "single_word": false
22
  }
tokenizer_config.json CHANGED
@@ -1,34 +1,46 @@
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
4
- "bos_token": {
5
- "__type": "AddedToken",
6
- "content": "<s>",
7
- "lstrip": false,
8
- "normalized": true,
9
- "rstrip": false,
10
- "single_word": false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  },
 
 
12
  "clean_up_tokenization_spaces": false,
13
- "eos_token": {
14
- "__type": "AddedToken",
15
- "content": "</s>",
16
- "lstrip": false,
17
- "normalized": true,
18
- "rstrip": false,
19
- "single_word": false
20
- },
21
  "legacy": null,
 
22
  "model_max_length": 1000000000000000019884624838656,
23
  "pad_token": null,
 
24
  "sp_model_kwargs": {},
 
25
  "tokenizer_class": "CodeLlamaTokenizer",
26
- "unk_token": {
27
- "__type": "AddedToken",
28
- "content": "<unk>",
29
- "lstrip": false,
30
- "normalized": true,
31
- "rstrip": false,
32
- "single_word": false
33
- }
34
  }
 
1
  {
2
  "add_bos_token": true,
3
  "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ }
29
  },
30
+ "additional_special_tokens": [],
31
+ "bos_token": "<s>",
32
  "clean_up_tokenization_spaces": false,
33
+ "eos_token": "</s>",
34
+ "eot_token": null,
35
+ "fill_token": null,
 
 
 
 
 
36
  "legacy": null,
37
+ "middle_token": null,
38
  "model_max_length": 1000000000000000019884624838656,
39
  "pad_token": null,
40
+ "prefix_token": null,
41
  "sp_model_kwargs": {},
42
+ "suffix_token": null,
43
  "tokenizer_class": "CodeLlamaTokenizer",
44
+ "unk_token": "<unk>",
45
+ "use_default_system_prompt": false
 
 
 
 
 
 
46
  }