ai-maker-space/llama2-instruct-tune-500s

README.md

@@ -1,7 +1,12 @@
 ---
-base_model: NousResearch/Llama-2-7b-hf
+library_name: peft
 tags:
+- trl
+- sft
 - generated_from_trainer
+datasets:
+- generator
+base_model: NousResearch/Llama-2-7b-hf
 model-index:
 - name: llama2_instruct_generation
   results: []
@@ -12,9 +17,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # llama2_instruct_generation
 
-This model is a fine-tuned version of [NousResearch/Llama-2-7b-hf](https://huggingface.co/NousResearch/Llama-2-7b-hf) on an unknown dataset.
+This model is a fine-tuned version of [NousResearch/Llama-2-7b-hf](https://huggingface.co/NousResearch/Llama-2-7b-hf) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.6733
+- Loss: 1.6759
 
 ## Model description
 
@@ -46,36 +51,37 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.9478        | 0.0   | 20   | 1.8129          |
-| 1.7992        | 0.0   | 40   | 1.7797          |
-| 1.8765        | 0.0   | 60   | 1.7642          |
-| 1.8488        | 0.01  | 80   | 1.7527          |
-| 1.8512        | 0.01  | 100  | 1.7384          |
-| 1.8856        | 0.01  | 120  | 1.7136          |
-| 1.8429        | 0.01  | 140  | 1.7035          |
-| 1.8258        | 0.01  | 160  | 1.6970          |
-| 1.8125        | 0.01  | 180  | 1.6923          |
-| 1.7902        | 0.01  | 200  | 1.6898          |
-| 1.8622        | 0.02  | 220  | 1.6895          |
-| 1.8823        | 0.02  | 240  | 1.6867          |
-| 1.7728        | 0.02  | 260  | 1.6837          |
-| 1.6331        | 0.02  | 280  | 1.6820          |
-| 1.8399        | 0.02  | 300  | 1.6821          |
-| 1.735         | 0.02  | 320  | 1.6810          |
-| 1.8192        | 0.02  | 340  | 1.6804          |
-| 1.7609        | 0.03  | 360  | 1.6771          |
-| 1.6517        | 0.03  | 380  | 1.6768          |
-| 1.6272        | 0.03  | 400  | 1.6738          |
-| 1.7947        | 0.03  | 420  | 1.6751          |
-| 1.7794        | 0.03  | 440  | 1.6740          |
-| 1.6371        | 0.03  | 460  | 1.6726          |
-| 1.7126        | 0.03  | 480  | 1.6730          |
-| 1.7321        | 0.04  | 500  | 1.6733          |
+| 1.8994        | 0.0   | 20   | 1.8109          |
+| 1.8521        | 0.01  | 40   | 1.7830          |
+| 1.8745        | 0.01  | 60   | 1.7694          |
+| 1.8092        | 0.01  | 80   | 1.7576          |
+| 1.8042        | 0.01  | 100  | 1.7436          |
+| 1.9305        | 0.02  | 120  | 1.7090          |
+| 1.7965        | 0.02  | 140  | 1.7034          |
+| 1.8457        | 0.02  | 160  | 1.6977          |
+| 1.823         | 0.02  | 180  | 1.6943          |
+| 1.7997        | 0.03  | 200  | 1.6922          |
+| 1.7614        | 0.03  | 220  | 1.6895          |
+| 1.7701        | 0.03  | 240  | 1.6886          |
+| 1.8093        | 0.04  | 260  | 1.6877          |
+| 1.8101        | 0.04  | 280  | 1.6847          |
+| 1.8109        | 0.04  | 300  | 1.6834          |
+| 1.7523        | 0.04  | 320  | 1.6807          |
+| 1.7575        | 0.05  | 340  | 1.6802          |
+| 1.8497        | 0.05  | 360  | 1.6783          |
+| 1.8347        | 0.05  | 380  | 1.6781          |
+| 1.8019        | 0.05  | 400  | 1.6766          |
+| 1.7267        | 0.06  | 420  | 1.6770          |
+| 1.7849        | 0.06  | 440  | 1.6767          |
+| 1.7727        | 0.06  | 460  | 1.6748          |
+| 1.7796        | 0.07  | 480  | 1.6744          |
+| 1.7963        | 0.07  | 500  | 1.6759          |
 
 
 ### Framework versions
 
-- Transformers 4.35.1
-- Pytorch 2.1.0+cu118
-- Datasets 2.14.6
-- Tokenizers 0.14.1
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.1.0+cu121
+- Datasets 2.16.1
+- Tokenizers 0.15.0

adapter_config.json

@@ -8,16 +8,19 @@
   "init_lora_weights": true,
   "layers_pattern": null,
   "layers_to_transform": null,
+  "loftq_config": {},
   "lora_alpha": 16,
   "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
   "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "q_proj"
+    "q_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e44ce263e6fd885f50d82ca515b9325375b43ee36ededb75acf161ce88bc2e41
-size 48
+oid sha256:bb9279cb010e4c54dff6a2480dfa4fc6d285a9df96a097c6631032b2b3628018
+size 134235048

runs/Jan18_22-19-50_09c86763fe3a/events.out.tfevents.1705616457.09c86763fe3a.251.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1038dc4f67fab820ca160b34cd00e5d70dab3cd5a5fc731990dc41e155056517
+size 19749

tokenizer_config.json

@@ -1,4 +1,6 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95aa17d15e436395bb17415ed4cb606077c6ea609cce8bb1f8da04d026217b9d
-size 4600
+oid sha256:a4f88f7da09dbe9f6769612e6c5e2adb43e29836217817946e1acd3840967886
+size 4728