Upload 13 files

README.md

@@ -1,3 +1,60 @@
----
-license: apache-2.0
----
+---
+license: other
+library_name: peft
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+base_model: Qwen/CodeQwen1.5-7B
+model-index:
+- name: train_scopai1
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# train_scopai1
+
+This model is a fine-tuned version of [Qwen/CodeQwen1.5-7B](https://huggingface.co/Qwen/CodeQwen1.5-7B) on the scopai dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 8
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 5
+- num_epochs: 2.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.10.1.dev0
+- Transformers 4.41.0.dev0
+- Pytorch 2.1.2+cu121
+- Datasets 2.19.0
+- Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/CodeQwen1.5-7B",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 8,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 4,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e48cbadaf8b2bbc623e76624a745d30b09e817abeae778bbefe535f09722c692
+size 6570424

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.0,
+    "total_flos": 1341465195577344.0,
+    "train_loss": 0.8099669218063354,
+    "train_runtime": 334.6224,
+    "train_samples_per_second": 0.233,
+    "train_steps_per_second": 0.03
+}

running_log.txt

@@ -0,0 +1,170 @@
+05/15/2024 23:19:53 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /home/f200309/.cache/huggingface/hub/models--Qwen--CodeQwen1.5-7B/snapshots/5ce5a1554e50a9e3bb236de7c0b8a2a1746186e4/tokenizer.json
+
+05/15/2024 23:19:53 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None
+
+05/15/2024 23:19:53 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at None
+
+05/15/2024 23:19:53 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /home/f200309/.cache/huggingface/hub/models--Qwen--CodeQwen1.5-7B/snapshots/5ce5a1554e50a9e3bb236de7c0b8a2a1746186e4/tokenizer_config.json
+
+05/15/2024 23:19:55 - INFO - llmtuner.data.loader - Loading dataset bigsmoke05/optimized-solidity-dataset...
+
+05/15/2024 23:20:04 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /home/f200309/.cache/huggingface/hub/models--Qwen--CodeQwen1.5-7B/snapshots/5ce5a1554e50a9e3bb236de7c0b8a2a1746186e4/config.json
+
+05/15/2024 23:20:04 - INFO - transformers.configuration_utils - Model config Qwen2Config {
+  "_name_or_path": "Qwen/CodeQwen1.5-7B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 13440,
+  "max_position_embeddings": 65536,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000,
+  "rotary_emb_base": 1000000,
+  "seq_length": 65536,
+  "sliding_window": 65536,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.0.dev0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 92416
+}
+
+
+05/15/2024 23:20:04 - INFO - llmtuner.model.utils.quantization - Quantizing model to 4 bit.
+
+05/15/2024 23:20:04 - INFO - transformers.modeling_utils - loading weights file model.safetensors from cache at /home/f200309/.cache/huggingface/hub/models--Qwen--CodeQwen1.5-7B/snapshots/5ce5a1554e50a9e3bb236de7c0b8a2a1746186e4/model.safetensors.index.json
+
+05/15/2024 23:20:04 - INFO - transformers.modeling_utils - Instantiating Qwen2ForCausalLM model under default dtype torch.float16.
+
+05/15/2024 23:20:04 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig {
+  "bos_token_id": 2,
+  "eos_token_id": 2,
+  "use_cache": false
+}
+
+
+05/15/2024 23:20:18 - INFO - transformers.modeling_utils - All model checkpoint weights were used when initializing Qwen2ForCausalLM.
+
+
+05/15/2024 23:20:18 - INFO - transformers.modeling_utils - All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/CodeQwen1.5-7B.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.
+
+05/15/2024 23:20:18 - INFO - transformers.generation.configuration_utils - loading configuration file generation_config.json from cache at /home/f200309/.cache/huggingface/hub/models--Qwen--CodeQwen1.5-7B/snapshots/5ce5a1554e50a9e3bb236de7c0b8a2a1746186e4/generation_config.json
+
+05/15/2024 23:20:18 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig {
+  "bos_token_id": 2,
+  "eos_token_id": [
+    4,
+    2
+  ],
+  "pad_token_id": 92298,
+  "top_p": 0.95
+}
+
+
+05/15/2024 23:20:19 - INFO - llmtuner.model.utils.checkpointing - Gradient checkpointing enabled.
+
+05/15/2024 23:20:19 - INFO - llmtuner.model.utils.attention - Using torch SDPA for faster training and inference.
+
+05/15/2024 23:20:19 - INFO - llmtuner.model.adapter - Fine-tuning method: LoRA
+
+05/15/2024 23:20:19 - INFO - llmtuner.model.loader - trainable params: 1638400 || all params: 7251922944 || trainable%: 0.0226
+
+05/15/2024 23:20:19 - INFO - transformers.trainer - Using auto half precision backend
+
+05/15/2024 23:20:20 - INFO - transformers.trainer - ***** Running training *****
+
+05/15/2024 23:20:20 - INFO - transformers.trainer -   Num examples = 39
+
+05/15/2024 23:20:20 - INFO - transformers.trainer -   Num Epochs = 2
+
+05/15/2024 23:20:20 - INFO - transformers.trainer -   Instantaneous batch size per device = 2
+
+05/15/2024 23:20:20 - INFO - transformers.trainer -   Total train batch size (w. parallel, distributed & accumulation) = 8
+
+05/15/2024 23:20:20 - INFO - transformers.trainer -   Gradient Accumulation steps = 4
+
+05/15/2024 23:20:20 - INFO - transformers.trainer -   Total optimization steps = 10
+
+05/15/2024 23:20:20 - INFO - transformers.trainer -   Number of trainable parameters = 1,638,400
+
+05/15/2024 23:20:59 - INFO - llmtuner.extras.callbacks - {'loss': 0.7110, 'learning_rate': 4.0000e-05, 'epoch': 0.20}
+
+05/15/2024 23:21:31 - INFO - llmtuner.extras.callbacks - {'loss': 0.7668, 'learning_rate': 8.0000e-05, 'epoch': 0.40}
+
+05/15/2024 23:22:00 - INFO - llmtuner.extras.callbacks - {'loss': 0.9731, 'learning_rate': 1.2000e-04, 'epoch': 0.60}
+
+05/15/2024 23:22:33 - INFO - llmtuner.extras.callbacks - {'loss': 0.8388, 'learning_rate': 1.6000e-04, 'epoch': 0.80}
+
+05/15/2024 23:23:02 - INFO - llmtuner.extras.callbacks - {'loss': 0.8452, 'learning_rate': 2.0000e-04, 'epoch': 1.00}
+
+05/15/2024 23:23:46 - INFO - llmtuner.extras.callbacks - {'loss': 0.6138, 'learning_rate': 1.6000e-04, 'epoch': 1.20}
+
+05/15/2024 23:24:28 - INFO - llmtuner.extras.callbacks - {'loss': 0.5858, 'learning_rate': 1.2000e-04, 'epoch': 1.40}
+
+05/15/2024 23:25:07 - INFO - llmtuner.extras.callbacks - {'loss': 0.7195, 'learning_rate': 8.0000e-05, 'epoch': 1.60}
+
+05/15/2024 23:25:26 - INFO - llmtuner.extras.callbacks - {'loss': 1.0479, 'learning_rate': 4.0000e-05, 'epoch': 1.80}
+
+05/15/2024 23:25:55 - INFO - llmtuner.extras.callbacks - {'loss': 0.9978, 'learning_rate': 0.0000e+00, 'epoch': 2.00}
+
+05/15/2024 23:25:55 - INFO - transformers.trainer - 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+05/15/2024 23:25:55 - INFO - transformers.trainer - Saving model checkpoint to saves/Qwen1.5-Code-7B/lora/train_scopai1
+
+05/15/2024 23:25:57 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /home/f200309/.cache/huggingface/hub/models--Qwen--CodeQwen1.5-7B/snapshots/5ce5a1554e50a9e3bb236de7c0b8a2a1746186e4/config.json
+
+05/15/2024 23:25:57 - INFO - transformers.configuration_utils - Model config Qwen2Config {
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 13440,
+  "max_position_embeddings": 65536,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000,
+  "rotary_emb_base": 1000000,
+  "seq_length": 65536,
+  "sliding_window": 65536,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.0.dev0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 92416
+}
+
+
+05/15/2024 23:25:57 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in saves/Qwen1.5-Code-7B/lora/train_scopai1/tokenizer_config.json
+
+05/15/2024 23:25:57 - INFO - transformers.tokenization_utils_base - Special tokens file saved in saves/Qwen1.5-Code-7B/lora/train_scopai1/special_tokens_map.json
+
+05/15/2024 23:25:57 - INFO - transformers.modelcard - Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<fim_pad>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<fim_pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.0,
+    "total_flos": 1341465195577344.0,
+    "train_loss": 0.8099669218063354,
+    "train_runtime": 334.6224,
+    "train_samples_per_second": 0.233,
+    "train_steps_per_second": 0.03
+}

trainer_config.yaml

@@ -0,0 +1,29 @@
+cutoff_len: 512
+dataset: scopai
+dataset_dir: data
+do_train: true
+finetuning_type: lora
+flash_attn: auto
+fp16: true
+gradient_accumulation_steps: 4
+learning_rate: 0.0002
+logging_steps: 1
+lora_alpha: 8
+lora_dropout: 0.05
+lora_rank: 4
+lora_target: q_proj,v_proj
+lr_scheduler_type: linear
+max_grad_norm: 0.5
+max_samples: 100
+model_name_or_path: Qwen/CodeQwen1.5-7B
+num_train_epochs: 2.0
+optim: adamw_torch
+output_dir: saves/Qwen1.5-Code-7B/lora/train_scopai1
+packing: false
+per_device_train_batch_size: 2
+quantization_bit: 4
+report_to: none
+save_steps: 100
+stage: sft
+template: default
+warmup_steps: 5

trainer_log.jsonl

@@ -0,0 +1,11 @@
+{"current_steps": 1, "total_steps": 10, "loss": 0.711, "learning_rate": 4e-05, "epoch": 0.2, "percentage": 10.0, "elapsed_time": "0:00:39", "remaining_time": "0:05:54"}
+{"current_steps": 2, "total_steps": 10, "loss": 0.7668, "learning_rate": 8e-05, "epoch": 0.4, "percentage": 20.0, "elapsed_time": "0:01:10", "remaining_time": "0:04:42"}
+{"current_steps": 3, "total_steps": 10, "loss": 0.9731, "learning_rate": 0.00012, "epoch": 0.6, "percentage": 30.0, "elapsed_time": "0:01:39", "remaining_time": "0:03:52"}
+{"current_steps": 4, "total_steps": 10, "loss": 0.8388, "learning_rate": 0.00016, "epoch": 0.8, "percentage": 40.0, "elapsed_time": "0:02:12", "remaining_time": "0:03:19"}
+{"current_steps": 5, "total_steps": 10, "loss": 0.8452, "learning_rate": 0.0002, "epoch": 1.0, "percentage": 50.0, "elapsed_time": "0:02:42", "remaining_time": "0:02:42"}
+{"current_steps": 6, "total_steps": 10, "loss": 0.6138, "learning_rate": 0.00016, "epoch": 1.2, "percentage": 60.0, "elapsed_time": "0:03:26", "remaining_time": "0:02:17"}
+{"current_steps": 7, "total_steps": 10, "loss": 0.5858, "learning_rate": 0.00012, "epoch": 1.4, "percentage": 70.0, "elapsed_time": "0:04:08", "remaining_time": "0:01:46"}
+{"current_steps": 8, "total_steps": 10, "loss": 0.7195, "learning_rate": 8e-05, "epoch": 1.6, "percentage": 80.0, "elapsed_time": "0:04:46", "remaining_time": "0:01:11"}
+{"current_steps": 9, "total_steps": 10, "loss": 1.0479, "learning_rate": 4e-05, "epoch": 1.8, "percentage": 90.0, "elapsed_time": "0:05:05", "remaining_time": "0:00:33"}
+{"current_steps": 10, "total_steps": 10, "loss": 0.9978, "learning_rate": 0.0, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "0:05:34", "remaining_time": "0:00:00"}
+{"current_steps": 10, "total_steps": 10, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "0:05:34", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,100 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "eval_steps": 500,
+  "global_step": 10,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.0990709736943245,
+      "learning_rate": 4e-05,
+      "loss": 0.711,
+      "step": 1
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.120853953063488,
+      "learning_rate": 8e-05,
+      "loss": 0.7668,
+      "step": 2
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.13565421104431152,
+      "learning_rate": 0.00012,
+      "loss": 0.9731,
+      "step": 3
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.14744164049625397,
+      "learning_rate": 0.00016,
+      "loss": 0.8388,
+      "step": 4
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.16943834722042084,
+      "learning_rate": 0.0002,
+      "loss": 0.8452,
+      "step": 5
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.09526122361421585,
+      "learning_rate": 0.00016,
+      "loss": 0.6138,
+      "step": 6
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 0.11567643284797668,
+      "learning_rate": 0.00012,
+      "loss": 0.5858,
+      "step": 7
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.20003917813301086,
+      "learning_rate": 8e-05,
+      "loss": 0.7195,
+      "step": 8
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 0.4086407721042633,
+      "learning_rate": 4e-05,
+      "loss": 1.0479,
+      "step": 9
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.2869514226913452,
+      "learning_rate": 0.0,
+      "loss": 0.9978,
+      "step": 10
+    },
+    {
+      "epoch": 2.0,
+      "step": 10,
+      "total_flos": 1341465195577344.0,
+      "train_loss": 0.8099669218063354,
+      "train_runtime": 334.6224,
+      "train_samples_per_second": 0.233,
+      "train_steps_per_second": 0.03
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 10,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "total_flos": 1341465195577344.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e58748cd2f7b71f16f9a8755f1a3ba8bc7b8165c0f7a390639ad9a463cde2c7
+size 5176