Model save

README.md

@@ -14,7 +14,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 4.1674
+- Loss: 4.1349
 
 ## Model description
 
@@ -46,9 +46,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 4.7272        | 0.32  | 1000 | 4.6783          |
-| 4.2957        | 0.64  | 2000 | 4.2638          |
-| 4.1819        | 0.96  | 3000 | 4.1674          |
+| 4.7828        | 0.32  | 1000 | 4.7268          |
+| 4.3604        | 0.64  | 2000 | 4.3024          |
+| 4.1839        | 0.96  | 3000 | 4.1349          |
 
 
 ### Framework versions

config.json

@@ -37,10 +37,6 @@
   "linear_num_key_heads": 8,
   "linear_num_value_heads": 8,
   "linear_value_head_dim": 64,
-  "matrix_mlp_enabled": true,
-  "matrix_mlp_k_dim": 64,
-  "matrix_mlp_num_aspects": 4,
-  "matrix_mlp_v_dim": 128,
   "max_position_embeddings": 512,
   "model_type": "neollm",
   "num_attention_heads": 8,
@@ -51,6 +47,7 @@
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 10000.0,
+  "specific_dropout_rate": 0.05,
   "transformers_version": "4.57.0.dev0",
   "vocab_size": 151665
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5dd4b8cf706a0576d07f64a230bf9594f3f3d1c3ede26252ad1ff0c801bb3ec8
-size 256819968
+oid sha256:564577add96d6dd96662764bc6dc179ea5b2f7996134722f141383fc350206ef
+size 250512472

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6d80b02a2cee58a145347d0678d11d7d32ad02578fc52fd503884bc7ddb3332
+oid sha256:e7324fec5b1c9f1bf64d10ba1735257d89652697f9f72faad5159fd7d8d963b3
 size 5585