| { |
| "step": 21400, |
| "val_bpb": 0.6682459571973981, |
| "model_config": { |
| "sequence_len": 2048, |
| "vocab_size": 65536, |
| "n_layer": 20, |
| "n_head": 10, |
| "n_kv_head": 10, |
| "n_embd": 1280 |
| }, |
| "user_config": { |
| "run": "d20-jp-1760620493", |
| "depth": 20, |
| "max_seq_len": 2048, |
| "target_param_data_ratio": 20, |
| "num_iterations": -1, |
| "device_batch_size": 32, |
| "total_batch_size": 524288, |
| "embedding_lr": 0.2, |
| "unembedding_lr": 0.004, |
| "matrix_lr": 0.02, |
| "weight_decay": 0.0, |
| "eval_every": 250, |
| "eval_tokens": 10485760, |
| "DATASET_REPO_ID": "kajuma/ABEJA-CC-JA-edu", |
| "CONFIG_NAME": "10%", |
| "SPLIT": "train", |
| "TOTAL_SHARDS": 378, |
| "DOWNLOAD_CACHE_DIR": "download_cache_jp" |
| } |
| } |
