use spm tokenizer

pretrain_llama_3b.sh

@@ -20,7 +20,7 @@ python3 -m EasyLM.models.llama.llama_train \
     --update_llama_config='' \
     --load_dataset_state='' \
     --load_checkpoint='' \
-    --tokenizer.pretrained_model_name_or_path='./' \
+    --tokenizer.vocab_file='tokenizer.model' \
     --optimizer.type='lion' \
     --optimizer.lion_optimizer.weight_decay=1.0 \
     --optimizer.lion_optimizer.lr_schedule_type='warmup_constant' \
@@ -31,6 +31,8 @@ python3 -m EasyLM.models.llama.llama_train \
     --optimizer.lion_optimizer.bf16_momentum=True \
     --train_dataset.type='huggingface' \
     --train_dataset.text_processor.fields='text' \
+    --train_dataset.text_processor.add_eos_token=True \
+    --train_dataset.text_processor.add_bos_token=True \
     --train_dataset.huggingface_dataset.path='/researchdisk/lm_training_dataset_first_stage' \
     --train_dataset.huggingface_dataset.split='train' \
     --train_dataset.huggingface_dataset.seq_length=2048 \