Add training records, tokenizer, configurations and evaluations

OpenSML-150M-training-records.tar.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45d442f2a9713b73f14d4301d96a0c4b1d963985c706b3c7a7ec9631029503a3
+size 91288296

configs/corpus.json

@@ -0,0 +1,15 @@
+{
+  "name": "sml-v2-educational-prose-v1",
+  "seed": 7337,
+  "shuffle_buffer": 10000,
+  "min_chars": 200,
+  "max_bytes": 200000,
+  "max_rows_per_source": 5000000,
+  "split_buckets": 10000,
+  "sources": [
+    {"label": "web", "repo": "HuggingFaceTB/smollm-corpus", "config": "fineweb-edu-dedup", "revision": "3ba9d605774198c5868892d7a8deda78031a781f", "split": "train", "text_field": "text", "weight": 55, "license": "ODC-BY; preserve underlying source notices"},
+    {"label": "dclm", "repo": "HuggingFaceTB/dclm-edu", "revision": "dbad8ad71224482740cd9c9d353591adbf62fe04", "split": "train", "text_field": "text", "weight": 25, "min_score": {"field": "edu_int_score", "value": 3}, "license": "CC-BY-4.0"},
+    {"label": "wiki", "repo": "HuggingFaceFW/finewiki", "config": "en", "revision": "8bd13e72e6a002407649b3e898535f42ceb1aeb9", "split": "train", "text_field": "text", "weight": 10, "license": "CC-BY-SA-4.0 / GFDL"},
+    {"label": "cosmopedia", "repo": "HuggingFaceTB/smollm-corpus", "config": "cosmopedia-v2", "revision": "3ba9d605774198c5868892d7a8deda78031a781f", "split": "train", "text_field": "text", "weight": 10, "format_contains": ["textbook", "tutorial", "blog", "educational"], "license": "ODC-BY; synthetic content; preserve provenance"}
+  ]
+}

configs/pilot.json

@@ -0,0 +1,33 @@
+{
+  "name": "sml-v2-150m-pilot-v1",
+  "data_mode": "hf_stream",
+  "stream": {"prefetch_batches": 4, "read_attempts": 5, "dedup_window": 10000},
+  "model": {"vocab_size": 32000, "max_seq_len": 2048, "d_model": 768, "n_layers": 20, "n_heads": 12, "n_kv_heads": 4, "mlp_ratio": 4.0, "mlp_multiple_of": 256, "bias": false, "qk_norm": true, "attention_impl": "fast", "loss_dtype": "float32", "ce_impl": "metal", "ffn_impl": "packed-metal"},
+  "seed": 7337,
+  "batches": [4, 3, 3, 3],
+  "grad_accum": 4,
+  "target_tokens": 500000000,
+  "peak_lr": 0.0003,
+  "warmup_fraction": 0.01,
+  "decay_start_fraction": 0.8,
+  "min_lr_ratio": 0.1,
+  "betas": [0.9, 0.95],
+  "eps": 1e-8,
+  "weight_decay": 0.1,
+  "grad_clip": 1.0,
+  "eval_every_tokens": 25000000,
+  "sample_every_tokens": 50000000,
+  "save_every_tokens": 10000000,
+  "eval_batches_per_source": 16,
+  "eval_batch_size": 2,
+  "keep_checkpoints": 2,
+  "reserve_gib": 8,
+  "log_every": 10,
+  "prompts": [
+    "The library was closed for repairs, so Maya decided to",
+    "Plants need sunlight because",
+    "A useful way to remember a new word is to",
+    "The train arrived late because of heavy snow. When the passengers stepped outside,",
+    "Clean drinking water is important because"
+  ]
+}

configs/training_recipe.json

@@ -0,0 +1,74 @@
+{
+  "batches": [
+    4,
+    3,
+    3,
+    3
+  ],
+  "betas": [
+    0.9,
+    0.95
+  ],
+  "continuation": {
+    "format": "sml-v2-continuation-v1",
+    "parent_contract": {
+      "code": "3af3f1a2405b7c447ee79cd1e9504ea9558d0f6636d7f9379f15cd216a7e4d58",
+      "data": "d9bfbfd69b065eafe0254fc36f96354461704f9bde0cbd6c0fd9f60383083614",
+      "recipe": "3c548ac699a0c6dc54c51536465cafd74487c2300fefc3da1fefae402204dc78",
+      "tokenizer": "1194292c6d906ac19e6f01c6ad2b42825143fad426a22aa8856d903776701ecb"
+    },
+    "parent_replica_hash": "5a3deeeede337b604e755127198f1e174b663e3e534f9cc332550c2806747730",
+    "parent_step": 4696,
+    "rewarm_tokens": 50000000,
+    "start_lr": 2.9999999999999997e-05,
+    "start_tokens": 500105216
+  },
+  "data_mode": "hf_stream",
+  "decay_start_fraction": 0.8,
+  "eps": 1e-08,
+  "eval_batch_size": 2,
+  "eval_batches_per_source": 16,
+  "eval_every_tokens": 25000000,
+  "grad_accum": 4,
+  "grad_clip": 1.0,
+  "keep_checkpoints": 2,
+  "log_every": 10,
+  "min_lr_ratio": 0.1,
+  "model": {
+    "attention_impl": "fast",
+    "bias": false,
+    "ce_impl": "metal",
+    "d_model": 768,
+    "ffn_impl": "packed-metal",
+    "loss_dtype": "float32",
+    "max_seq_len": 2048,
+    "mlp_multiple_of": 256,
+    "mlp_ratio": 4.0,
+    "n_heads": 12,
+    "n_kv_heads": 4,
+    "n_layers": 20,
+    "qk_norm": true,
+    "vocab_size": 32000
+  },
+  "name": "sml-v2-150m-pilot-v1-continued-to-15000000000",
+  "peak_lr": 0.0003,
+  "prompts": [
+    "The library was closed for repairs, so Maya decided to",
+    "Plants need sunlight because",
+    "A useful way to remember a new word is to",
+    "The train arrived late because of heavy snow. When the passengers stepped outside,",
+    "Clean drinking water is important because"
+  ],
+  "reserve_gib": 8,
+  "sample_every_tokens": 50000000,
+  "save_every_tokens": 10000000,
+  "seed": 7337,
+  "stream": {
+    "dedup_window": 10000,
+    "prefetch_batches": 4,
+    "read_attempts": 5
+  },
+  "target_tokens": 15000000000,
+  "warmup_fraction": 0.01,
+  "weight_decay": 0.1
+}

evaluations/post_training_20260927.json

@@ -0,0 +1,196 @@
+{
+  "date": "2026-09-27",
+  "training": false,
+  "models": {
+    "balanced-skills-1920": {
+      "bundle": "sft_skill_balance_v1/step_0001920_628a0a65099b",
+      "sha256": "4cac7f733e6ef63230e26e2aa66418bf0d9661ef55f46fa1e01f2ac602b17407",
+      "results": {
+        "multiple-choice": {
+          "completed": 15428,
+          "expected": 15428,
+          "status": "complete",
+          "tasks": {
+            "arc_challenge": {
+              "acc": 0.25341296928327645,
+              "acc_norm": 0.3003412969283277,
+              "completed": 1172,
+              "expected": 1172
+            },
+            "arc_easy": {
+              "acc": 0.5904882154882155,
+              "acc_norm": 0.5707070707070707,
+              "completed": 2376,
+              "expected": 2376
+            },
+            "hellaswag": {
+              "acc": 0.30999800836486757,
+              "acc_norm": 0.34534953196574386,
+              "completed": 10042,
+              "expected": 10042
+            },
+            "piqa": {
+              "acc": 0.6474428726877041,
+              "acc_norm": 0.6430903155603918,
+              "completed": 1838,
+              "expected": 1838
+            }
+          },
+          "training": false
+        },
+        "ifeval": {
+          "completed": 541,
+          "expected": 541,
+          "loose": {
+            "instruction_accuracy": 0.2637889688249401,
+            "instruction_correct": 220,
+            "instruction_total": 834,
+            "prompt_accuracy": 0.14417744916820702,
+            "prompt_correct": 78,
+            "prompt_total": 541
+          },
+          "status": "complete",
+          "stop_counts": {
+            "context_limit": 0,
+            "eos": 539,
+            "length": 2
+          },
+          "strict": {
+            "instruction_accuracy": 0.2637889688249401,
+            "instruction_correct": 220,
+            "instruction_total": 834,
+            "prompt_accuracy": 0.14417744916820702,
+            "prompt_correct": 78,
+            "prompt_total": 541
+          },
+          "training": false
+        }
+      }
+    },
+    "constraint-completion-2048": {
+      "bundle": "sft_constraint_completion_1920_v1/step_0002048_1267d9f4cf7a",
+      "sha256": "e4e8a0de28e2141f6a580b49cc950aa64492c9f94d0d3aa98e6a26a2f439764b",
+      "results": {
+        "multiple-choice": {
+          "completed": 15428,
+          "expected": 15428,
+          "status": "complete",
+          "tasks": {
+            "arc_challenge": {
+              "acc": 0.25426621160409557,
+              "acc_norm": 0.31399317406143346,
+              "completed": 1172,
+              "expected": 1172
+            },
+            "arc_easy": {
+              "acc": 0.5871212121212122,
+              "acc_norm": 0.5677609427609428,
+              "completed": 2376,
+              "expected": 2376
+            },
+            "hellaswag": {
+              "acc": 0.31029675363473413,
+              "acc_norm": 0.3472415853415654,
+              "completed": 10042,
+              "expected": 10042
+            },
+            "piqa": {
+              "acc": 0.64689880304679,
+              "acc_norm": 0.6381936887921654,
+              "completed": 1838,
+              "expected": 1838
+            }
+          },
+          "training": false
+        },
+        "ifeval": {
+          "completed": 541,
+          "expected": 541,
+          "loose": {
+            "instruction_accuracy": 0.29136690647482016,
+            "instruction_correct": 243,
+            "instruction_total": 834,
+            "prompt_accuracy": 0.16820702402957485,
+            "prompt_correct": 91,
+            "prompt_total": 541
+          },
+          "status": "complete",
+          "stop_counts": {
+            "context_limit": 0,
+            "eos": 532,
+            "length": 9
+          },
+          "strict": {
+            "instruction_accuracy": 0.28776978417266186,
+            "instruction_correct": 240,
+            "instruction_total": 834,
+            "prompt_accuracy": 0.16635859519408502,
+            "prompt_correct": 90,
+            "prompt_total": 541
+          },
+          "training": false
+        }
+      }
+    }
+  },
+  "protocol": {
+    "harness_reference_revision": "b954108c9baaaa934b4ad842033b31a97ee30816",
+    "datasets": {
+      "arc_easy": {
+        "repo": "allenai/ai2_arc",
+        "revision": "210d026faf9955653af8916fad021475a3f00453",
+        "file": "ARC-Easy/test-00000-of-00001.parquet",
+        "split": "test",
+        "rows": 2376,
+        "sha256": "4160597d618ae851c7eb04e281574f3f654776216ac6b6641588d64527b47177"
+      },
+      "arc_challenge": {
+        "repo": "allenai/ai2_arc",
+        "revision": "210d026faf9955653af8916fad021475a3f00453",
+        "file": "ARC-Challenge/test-00000-of-00001.parquet",
+        "split": "test",
+        "rows": 1172,
+        "sha256": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9"
+      },
+      "piqa": {
+        "repo": "baber/piqa",
+        "revision": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1",
+        "file": "piqa_validation.parquet",
+        "split": "validation",
+        "rows": 1838,
+        "sha256": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3"
+      },
+      "hellaswag": {
+        "repo": "Rowan/hellaswag",
+        "revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76",
+        "file": "data/validation-00000-of-00001.parquet",
+        "split": "validation",
+        "rows": 10042,
+        "sha256": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd"
+      },
+      "ifeval": {
+        "repo": "google/IFEval",
+        "revision": "966cd89545d6b6acfd7638bc708b98261ca58e84",
+        "file": "ifeval_input_data.jsonl",
+        "split": "train",
+        "rows": 541,
+        "sha256": "6a85310ca8ce15eff755aa08a3a4ff931c7e273e7515ebb3c492ea85fd8288f2"
+      }
+    },
+    "multiple_choice_normalization": "candidate log-likelihood divided by candidate character length",
+    "few_shot_examples": 0,
+    "ifeval": {
+      "prompt_template": "User: {prompt}\nAssistant:",
+      "max_context": 2048,
+      "max_new_tokens": 1280,
+      "greedy": true,
+      "repetition_penalty": 1
+    }
+  },
+  "limitations": [
+    "Custom MLX evaluator, not an official leaderboard submission.",
+    "Public suites repeatedly inspected during development, not an untouched final test.",
+    "No full contamination audit or statistical-significance claim.",
+    "Instruction compliance does not establish correct content or conversational quality."
+  ]
+}

tokenizer/corpus.json

@@ -0,0 +1,60 @@
+{
+  "max_bytes": 200000,
+  "max_rows_per_source": 5000000,
+  "min_chars": 200,
+  "name": "sml-v2-educational-prose-v1",
+  "seed": 7337,
+  "shuffle_buffer": 10000,
+  "sources": [
+    {
+      "config": "fineweb-edu-dedup",
+      "label": "web",
+      "license": "ODC-BY; preserve underlying source notices",
+      "repo": "HuggingFaceTB/smollm-corpus",
+      "revision": "3ba9d605774198c5868892d7a8deda78031a781f",
+      "split": "train",
+      "text_field": "text",
+      "weight": 55
+    },
+    {
+      "label": "dclm",
+      "license": "CC-BY-4.0",
+      "min_score": {
+        "field": "edu_int_score",
+        "value": 3
+      },
+      "repo": "HuggingFaceTB/dclm-edu",
+      "revision": "dbad8ad71224482740cd9c9d353591adbf62fe04",
+      "split": "train",
+      "text_field": "text",
+      "weight": 25
+    },
+    {
+      "config": "en",
+      "label": "wiki",
+      "license": "CC-BY-SA-4.0 / GFDL",
+      "repo": "HuggingFaceFW/finewiki",
+      "revision": "8bd13e72e6a002407649b3e898535f42ceb1aeb9",
+      "split": "train",
+      "text_field": "text",
+      "weight": 10
+    },
+    {
+      "config": "cosmopedia-v2",
+      "format_contains": [
+        "textbook",
+        "tutorial",
+        "blog",
+        "educational"
+      ],
+      "label": "cosmopedia",
+      "license": "ODC-BY; synthetic content; preserve provenance",
+      "repo": "HuggingFaceTB/smollm-corpus",
+      "revision": "3ba9d605774198c5868892d7a8deda78031a781f",
+      "split": "train",
+      "text_field": "text",
+      "weight": 10
+    }
+  ],
+  "split_buckets": 10000
+}

tokenizer/manifest.json

@@ -0,0 +1,31 @@
+{
+  "boundary_policy": "raw-text-then-one-document-EOS-v1",
+  "corpus_fingerprint": "a89f69d6f63cfa4385da233693a445404234a18194170fe1d2b4a74bf501f5d6",
+  "format": "sml-v2-tokenizer-v1",
+  "recipe": {
+    "corpus": "a89f69d6f63cfa4385da233693a445404234a18194170fe1d2b4a74bf501f5d6",
+    "min_frequency": 2,
+    "sample_manifest": "30ead6f502f11e99b479e3711a81ce7198787408d0700153615fc3255b244f46",
+    "specials": [
+      "<|pad|>",
+      "<|doc_end|>",
+      "<|turn_start|>",
+      "<|turn_end|>"
+    ],
+    "tokenizers": "0.22.2",
+    "version": 1,
+    "vocab_size": 32000
+  },
+  "sha256": {
+    "corpus.json": "e4c47b7b6aba08f948de7c2c7cb94c58dbd09dde2ded6ac185c60856ee686d27",
+    "tokenizer.json": "e095f0413a11fa91fbb58f37d00e4c6bcda59beb89b28f6f680db1436a7e8891",
+    "tokenizer_config.json": "619f500553367eb76f5be00a98625130ec857b9c367561334984096c029ba2a9"
+  },
+  "special_ids": {
+    "eos": 1,
+    "pad": 0,
+    "turn_end": 3,
+    "turn_start": 2
+  },
+  "vocab_size": 32000
+}

tokenizer/report.json

@@ -0,0 +1,38 @@
+{
+  "fixtures_passed": 9,
+  "sources": {
+    "cosmopedia": {
+      "bytes": 402259,
+      "bytes_per_token": 5.061007523716062,
+      "documents": 103,
+      "roundtrip_failures": 0,
+      "seconds": 0.09366637503262609,
+      "tokens": 79482
+    },
+    "dclm": {
+      "bytes": 1001509,
+      "bytes_per_token": 4.453209483494593,
+      "documents": 165,
+      "roundtrip_failures": 0,
+      "seconds": 0.26016941701527685,
+      "tokens": 224896
+    },
+    "web": {
+      "bytes": 2205062,
+      "bytes_per_token": 4.625467253451697,
+      "documents": 559,
+      "roundtrip_failures": 0,
+      "seconds": 0.5487007499905303,
+      "tokens": 476722
+    },
+    "wiki": {
+      "bytes": 401775,
+      "bytes_per_token": 4.059276397546905,
+      "documents": 143,
+      "roundtrip_failures": 0,
+      "seconds": 0.10723229206632823,
+      "tokens": 98977
+    }
+  },
+  "vocab_size": 32000
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,8 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|doc_end|>",
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}