weathon commited on
Commit
c66933f
·
verified ·
1 Parent(s): 15e636f

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +18 -18
tokenizer_config.json CHANGED
@@ -1,30 +1,30 @@
1
  {
2
  "char_ords": [
 
 
 
 
 
 
 
 
 
3
  67,
 
 
 
4
  61,
5
- 70,
 
 
 
6
  35,
7
  51,
8
- 41,
9
  108,
10
- 46,
11
- 78,
12
- 53,
13
- 72,
14
- 52,
15
- 93,
16
- 79,
17
- 40,
18
- 45,
19
  80,
20
  91,
21
- 50,
22
- 114,
23
- 49,
24
- 43,
25
- 54,
26
- 66,
27
- 83
28
  ],
29
  "model_max_length": 1024
30
  }
 
1
  {
2
  "char_ords": [
3
+ 45,
4
+ 93,
5
+ 41,
6
+ 52,
7
+ 70,
8
+ 49,
9
+ 40,
10
+ 66,
11
+ 72,
12
  67,
13
+ 54,
14
+ 46,
15
+ 43,
16
  61,
17
+ 114,
18
+ 79,
19
+ 53,
20
+ 78,
21
  35,
22
  51,
 
23
  108,
 
 
 
 
 
 
 
 
 
24
  80,
25
  91,
26
+ 83,
27
+ 50
 
 
 
 
 
28
  ],
29
  "model_max_length": 1024
30
  }