Rodimus-Plus-Coder-1.6B-Chat / special_tokens_map.json

Upload 5 files

cd7b1c4 verified over 1 year ago

314 Bytes

{
    "additional_special_tokens": [
        "<|fim_begin|>",
        "<|fim_hole|>",
        "<|fim_end|>",
        "<|fim_pad|>",
        "<|repo_name|>",
        "<|file_sep|>"
    ],
    "bos_token": "<|startoftext|>",
    "cls_token": "[CLS]",
    "eos_token": "<|endoftext|>",
    "gmask_token": "[gMASK]"
}

	{
	"additional_special_tokens": [
	"<\|fim_begin\|>",
	"<\|fim_hole\|>",
	"<\|fim_end\|>",
	"<\|fim_pad\|>",
	"<\|repo_name\|>",
	"<\|file_sep\|>"
	],
	"bos_token": "<\|startoftext\|>",
	"cls_token": "[CLS]",
	"eos_token": "<\|endoftext\|>",
	"gmask_token": "[gMASK]"
	}