diff --git a/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt index 8cbc5718f30ce0db33e9f76ad6083fbc637471af..481873a2f93eaf64d8b7181184390ce151e58d79 100644 --- a/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:98ab3c22434e1234ccd40a047cf201bb3681dd236dcc60ffe433b56961f6c0a6 +oid sha256:160c7027cfe3e90a3df9635f19c2b65336404e865ef161bd4aa87197b5cae91a size 442311792 diff --git a/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt index 97cb55000c5e6ddaf7c7c71e2a2af9c3978a60a8..bf479ec9712d85f64f4d7b019f31a64742cc541e 100644 --- a/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:65bc89a4f6a73d82390bf541399ce026af373282d078a8668db40592f04b42f2 +oid sha256:8da2d94cc143e4b780bf43ad2fc705ba0f2c2921292133468207cbaafbc43ea9 size 442311866 diff --git a/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt index bbed95bc7888ef342f95dbe5f88faa3f4d77777c..160a23062c5939d08cfb1ae08aaf0a04dfa8a162 100644 --- a/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d16bac4bd257dacc5883228665c94e59d599dca381ca965f53a08271f2b4e87b +oid sha256:056a862fd6ce2376fbb5b886925da19240d70ae5b54d3dec0f97ea10a9f7c724 size 442311994 diff --git a/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt index cb58ab8549259c473a3518098d668691120d2112..74fe8eaaaaa58f3155e7ddf7bfc97e9d88341294 100644 --- a/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a7dd517dad08e5e417bb8a0c85deed1c7a1daaf5ce892be3ab57c47c20a1490c +oid sha256:658304537778560e8210961cbebe4849062c1a7c6bee71dedd84a6e486b1ac9b size 442312058 diff --git a/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt index 04c80e96b6a62579ed61281e3cd59cf53775788e..692865def257a6c6c1903812cf722f8cc5046099 100644 --- a/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:41c0315c66f8a52fd01dfd6398d67ca9af22683c75155ed42ab6bcf800c1e1a8 +oid sha256:c707f8bb04356f7da9fd7311e5b64f71a15aa5ce4a5f434386f9e1c43e4f4270 size 442311866 diff --git a/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt index 8370166718f964eb685f31c442fa3ba4b80aa25f..184ebcad4c938c176951a03c24f1ca448ff329eb 100644 --- a/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bc3655166039abbcfe48704778ff4991cc7537dda795934ec7127f80b0a258fa +oid sha256:6c345bd6af30532601cb2e4b1968e2ad46aacc680666da1234dc127921a3077c size 442311994 diff --git a/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt index 947373217de90cf5d4e98db44f2b97baaeec0487..546c8cc71ce14b98d657d54d2cf1827096d5a8f5 100644 --- a/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9934b6395e2df8514e89f23f462d41776885694db136a8cab180dc2e99240e10 +oid sha256:228048b16f82fc2cb117c07ea382a9f0edeba25f26d21b7e8f7099694c465e0a size 442311866 diff --git a/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt index 5566668b34930f63e8290376396862e0dc38928e..57d9bb1714b5f4221350c13a94ac9af029426d35 100644 --- a/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:825207233b406a251a8c0fe1faa88d708b8bf6aad59437e7bbb011a656674c3e +oid sha256:04369b16e63dc8ed82c0111264bea3c335b478990e7380c4059e26518873ec8c size 442311930 diff --git a/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt index 60dd73f650e2256d2db809aceaf8817a926d83ab..09b66137693b1d4a236fe64231966b30053c7d93 100644 --- a/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e5d81fad20db325373d5b0eef2ccc90d69633f9b6394c9d826c585f8e2d7eb69 -size 442311994 +oid sha256:fcbae93f0c8ae6406ed785fe9f0d9e2f167b9923d10c6b74ee23f8be0510df32 +size 442312058 diff --git a/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt index f4b7bdcc904f96acec4425aba66f72849a652705..5fbfe7af04857fc9b75721589d96bc1dd3e91b39 100644 --- a/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:96464bf0d7981d02d261a7412fac77cbb0fd3ea2714f220b9c43208178696365 +oid sha256:4ea403e6eed9bfa7591f89b9f89dc75f6d408678a059df1679baf7bbca9d3a53 size 442311866 diff --git a/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt index 38e53b1697a87c3e9cc851184294e32e46056876..e86f56ddf986b8ed8ffdd86104565dd7359e77cb 100644 --- a/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:241e8aa4f96cd10f7dca7c16c4627c5318c111f7c4d0bf773dd5a776b1436705 +oid sha256:53f7d884781495834c443daf53c8ca5d188f52412606d0b0d2ef44f3ace8b8b4 size 442311994 diff --git a/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt index 22a9f43f4e43017fe3ba38ab7cf2029c2a57030f..1d1ec379ddb34575f079c631f364422d483b042c 100644 --- a/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:561c5d1ca5cec31ca8579e6217e7d5645ab8b7d65f401abc4ca52c830c09e629 +oid sha256:f8d9d61c458069e3e28419810e2acbe985660c5b1962229829058442586dbab6 size 442311792 diff --git a/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt index d6dee173e371ec5e228b74a65af57838c063ec36..d3fc0e0a53e32ceeebc0b54783ef4b4b0b7c78e0 100644 --- a/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b6933f0c6147ff2275894d9ff48f79a1fa231e0d6b53365db518eaa1a89158b9 +oid sha256:16c61ac2685f4bbb6091f93fd0688b3d8da58e0442c769c38a3efaf7587f39e7 size 442311930 diff --git a/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt index 560a6be10a38285bdf4318901468a14e8c9b8a27..6815b1f340c75906662b3f8d650e642685071e23 100644 --- a/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b7c6b5895ca9630d0ef360e2c84e7b8c8ec61ecb078a2f33948c3a1e5df82143 +oid sha256:0a24ccee8d7328591b62710480afc1a309be35f0f6d92809c3db754d43b8045b size 442311930 diff --git a/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt index aea232bcad5ff542803f86f9b1b34295d11bbc8e..2c14cfd8443b27fb5a9d1abb80453b8b1425f7eb 100644 --- a/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b7e51327a75ea3c4bd891c8ee7db18ab816817db6a68e9f15f0a521908ecce5a +oid sha256:fcaa7c4b27173c1ed517527d349d1ff3eae2bf52303d775871c9d5a94a77f39d size 442312058 diff --git a/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt index 0379247c527f6491d4afa2c4602a87ef93b6bd4d..0fdf919eb61653582e4d79eb30f9711b9a46ad52 100644 --- a/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3c63137b8cf322421d1d2c0fc9f02c4d595cd6f3964b8d8b7756272c933485a1 +oid sha256:cf9d44754ae28cdb31d10e0413968d62fa1651a063f8c6156d1aca1fddbe52e9 size 442311866 diff --git a/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt index 72b1d3f564218b5dfd94f51694d255b4db783326..e24ae6a08cf92f4e4642f7c66d4ce6c10a941f3d 100644 --- a/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:15df04b68f011f0a75cfb017ee7b39fd2f4273b5670369b69537bc91b5c47bc1 +oid sha256:23c7ae5af96362bc7b954859660c6b06ce8ba2ae5f725882d4ca5b7116a1e871 size 442311994 diff --git a/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt index 8e33183ef36d0ef92706a8d0e3627913ce930a0c..aeef2b76de9cada489279fc4b168e5af3469ea93 100644 --- a/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9913a1036cc807ea1f2245b0605d4f00da830d9bdc0337eaf1a8da9ee8d6b1ef +oid sha256:8e89d4b3c865fb3da793da63cdfb8ac85173b2912e75d80bffb76029836c1ce8 size 442311930 diff --git a/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt index d2ca89d5dcc904707be4d6e7a624e5337e7d4fb4..5772420188ba8ff8492d6e5f3788092eb14a2d78 100644 --- a/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b67e8ee2a5b5ad0746e58692aae31c3bd57024d62cb97141902a20ce238961a2 +oid sha256:d9a97bc7cd6cbc074755edf30b597d5f1ea593c820653f8b8bdf2a570a384ab5 size 442311930 diff --git a/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt index fcdb51ecc87f9a5de0a2c0d0bdcc396c5025e4d9..17a5a0feb409139bae30576191e942ac711e1889 100644 --- a/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0c23ab2b6ed450f0f4f70f985cfcf495a677aef497484bf991d27be9b0e08953 -size 442311994 +oid sha256:6636ec649d883734ef9f6edf8547b8922f973dab03d44d06f60d99c690b2aa00 +size 442312058 diff --git a/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt index bd6bd9b3b9054d45ad4f314e10f871fc3c1e41bc..ed06803e1de4569eb2a4b1058e07bda8959f4977 100644 --- a/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:99d9715c914d4227be794b5024eae6de40d2dab681c5e4e54bd1f691c4281265 +oid sha256:a33215ef93f70d42fd40c63803d37459b3ecbaf72b8f9743d04c1f5b51bd799b size 442311866 diff --git a/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt index 0a1c6fe122f4a0342d1d9e9b88c29b70412d6409..941bda425ea19ae98e74d293ed162dff3ea8bb40 100644 --- a/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cd6b42564ea1f65d9c7dae50d52b7775be6e33dcaaa4aaa081f10346679dddea +oid sha256:7847083ad6d4c700fb1cf97009bf7ff6e6dc8c9ed2b594c49d205e6f4a291f1b size 442311866 diff --git a/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt index 0377ec15492949ca48822e01c05c7393c9541c8c..f3c33fe487b3012aa353bcd87b1c468bd21020b6 100644 --- a/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cfd89562ed800c4631134c88499ad92ae793d43817a4644932057d08c420a870 +oid sha256:91629e9a0878ef93f112eb5d03ac39caddc6bf942e7a6e162694ecacf96eddd8 size 442311920 diff --git a/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt index 695f229b77394ec4f132038c4f247250704b6c08..3651aa972244421b249152e26be3860baf071ba9 100644 --- a/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c5c3fb9630fd6d072fde5704da0695d425503cee6f278a15d9abc18d5006b57c +oid sha256:771a118c862c97bb5ebc6ddc05f1e1cb661a33835ab5a50ea9c1b1d294d239f6 size 442311802 diff --git a/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt index 42c763157354df9133a8c0b6061e04374a5d4f38..0bf9a9464c621d83925499aff6eaf412f4d507f7 100644 --- a/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:43f41a54228d1edfa5c1deafb0862a4390dd818d71d56247f1d98bae83ff03ae +oid sha256:def8e6c5616be1ddbc76f80175de3d80d147e868050e3c534a7dfe2032d23227 size 442311802 diff --git a/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt index f8b07972a409266dc9d2ac3fe1aa4a852a0807ac..70c323706c99b519f03a786e382eb40c624e1230 100644 --- a/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:73af311a70d5a232587cefb54a63335560d3a124bf8f5f017f132fc4d4a30046 -size 442311856 +oid sha256:a44bf2950529b147cbdef6e5ce612eb9cd0aa8f67c0a2c2198c68d0929ec703f +size 442311920 diff --git a/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt index 2a0bf4ab16eb0d7de7eec182d6c369f5f40f20f4..de47a98647338287fe192cb999117dfb8c78ef3d 100644 --- a/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3ab4170ba7151e223c8e1233b5804e95f472d0ce1712652da6fbddba83d034d6 +oid sha256:f9ef9f49889b4e72637bc225d47282a7aacc064fed591dfc914cfe4980ec1364 size 442311984 diff --git a/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt index d45bafbefd9fbe91037d86e464a03db5b8bfb227..ba27485be352e0eaa4d741656e04593fa9e58dc6 100644 --- a/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:82dbad4def0f5e4701460e47611bdcfde0cd2307fc6d73cff4b641a6bb29bc36 +oid sha256:4e6f14ac16437fdcf25b83ba481e3ee5b542a78104e226ceb62ad26ebc009115 size 442311856 diff --git a/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt index 3263cc06e3929827f0b2e94d8351981194f57af6..d8ad089f987ae5583dd6a2fabe999d0060ad301b 100644 --- a/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:13860fbb39a0f8a625581b024085a16bcf684c09b8ef79d31a0c669874b16c9f +oid sha256:1f05d32215dc989b3d643c30cfaf2d3d054ab919f311345c39ad559a5f45db09 size 442311984 diff --git a/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt index 624ad39e1401d4b62bedb65d1a0cc0b30c4bacc0..155c28ddd8a82f7153a22337a4fcbcae6d76c075 100644 --- a/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0b5461cbcbcc9c1c94e60d596a331f1582eabeeedd2731d543a3d26b3124314b -size 442311920 +oid sha256:dcc2ad428df111a23b18db2c0a1ab991b71664cae6c2ff44128668f840afd257 +size 442311984 diff --git a/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt index 70a64e63d1f0e8759174a9b7a49eef1758fa10b7..a46a06deea3c04f32ac4bd81997b488b32a3bf21 100644 --- a/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1051ad6641b3020577cb3de6837959b0e4280521c827f053ca64b54d7d82efef +oid sha256:370d22ddf31eab8131cee7d81cec8d0e1d24e625af79cedb22565dbeb53a8bde size 442311920 diff --git a/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt index 4097fb0622aebe02dfffb8560382fed8027a8d71..9af2f7805a673f96982469f3ca7fba4b37b322e8 100644 --- a/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt +++ b/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f41503d14bd845b3320ef3544bf35d5974f52bd2d035a553cef57a9aa2770485 -size 442311920 +oid sha256:d79bc62104a691e71a42ee3da44fd84ca31695c0b1fab74c65dd81940c7bff4b +size 442311984 diff --git a/configs/local_setup_25_50.yml b/configs/local_setup_25_50.yml new file mode 100644 index 0000000000000000000000000000000000000000..f7e36634ee3b43b6c3c72b878c6fc0dcd445614b --- /dev/null +++ b/configs/local_setup_25_50.yml @@ -0,0 +1,36 @@ +{ + # Paths are relative to /lustre/fs0/scratch + + # Data etc. + "data_path": "/shared/data/neox-dclm_baseline-100B-perturbed-25-50/standard_text_document", + + # or for weighted datasets: + # "train-data-paths": ["data/enwik8/enwik8_text_document", "data/enwik8/enwik8_text_document"], + # "test-data-paths": ["data/enwik8/enwik8_text_document", "data/enwik8/enwik8_text_document"], + # "valid-data-paths": ["data/enwik8/enwik8_text_document", "data/enwik8/enwik8_text_document"], + # "train-data-weights": [1., 2.], + # "test-data-weights": [2., 1.], + # "valid-data-weights": [0.5, 0.4], + + # If weight_by_num_documents is True, Builds dataset weights from a multinomial distribution over groups of data according to the number of documents in each group. + # WARNING: setting this to True will override any user provided weights + # "weight_by_num_documents": false, + # "weighted_sampler_alpha": 0.3, + + # Vocab + "padded_vocab_size": 50304, + "vocab_file": "/shared/ameyagod/HubbleSuite/vocab-data/olmo-0724-hf/tokenizer.json", + "tokenizer_type": "HFTokenizer", + + "save": "/shared/pt_models/Hubble_1.1B/DCLM_100B/Perturbed-GBS_1024-SL_2048-DYNA_25_50", + "load": "/shared/pt_models/Hubble_1.1B/DCLM_100B/Perturbed-GBS_1024-SL_2048-DYNA_25_50", + "checkpoint_validation_with_forward_pass": False, + + # "tensorboard_dir": "tensorboard", + "log_dir": "logs", + "use_wandb": True, + "wandb_host": "https://api.wandb.ai", + "wandb_team": "usc_and_mpi", + "wandb_project": "Hubble", + "wandb_run_name": "Hubble_1.1B-DCLM_100B-Perturbed-GBS_1024-SL_2048-DYNA_25_50", +} diff --git a/layer_00-model_00-model_states.pt b/layer_00-model_00-model_states.pt index 425e961a5a32fbdc80d8b86441e440444eaaf4da..2c331c451e49f70f20bbce612de54d8c278ec926 100644 --- a/layer_00-model_00-model_states.pt +++ b/layer_00-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:77d751a0ceeec80047727ee801ba9a10bd82f12e9ccdc0f5dbf1e27aa8b854f8 +oid sha256:ad8dcd1efcb7605dfb7f5f1719fcd249a1b3fd2c666e7469b64d239b0fce6be6 size 206046607 diff --git a/layer_02-model_00-model_states.pt b/layer_02-model_00-model_states.pt index 45ee8405098474eafe1755b9caedd9ea9f554f2c..0cdc156a0a94acfd0d4e2b9f8d92817a8a1b5309 100644 --- a/layer_02-model_00-model_states.pt +++ b/layer_02-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6fa0bed62aad7c30d3957f7c11a07b397ad681131ed181f7336e9e04b4c3c356 +oid sha256:09b6ade2a7d2c8ec3c8c40202663e76ac40656a0237401ef1c1c6b4db8b4f44e size 121683348 diff --git a/layer_03-model_00-model_states.pt b/layer_03-model_00-model_states.pt index 0089153bd47cd0da37dabbcbf97a4ab2b0a95685..380003ace78807ea43ae49b5cde5f760cb7f9d32 100644 --- a/layer_03-model_00-model_states.pt +++ b/layer_03-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:30e98e2f064c36b5b980ed83080531f93163ae90ec93cc7fbcf55972807ad864 +oid sha256:28e1168c48e4502e9199503719f93d09de44e7272466bd791f084304b9f53901 size 121683348 diff --git a/layer_04-model_00-model_states.pt b/layer_04-model_00-model_states.pt index 1f35b8e2ce5f2ed3969a381bbd48faff5ef0aa49..68bf5d39e0057a5e3228df228b4acb9ed62eb09c 100644 --- a/layer_04-model_00-model_states.pt +++ b/layer_04-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d19d108ff7da9c56e2864917f0037fefcba5d04e58dd73ccfbb27dfa0c1503a7 +oid sha256:b8a80c306e298baa928e351ba067d3bf287491dd4e0a5379a9ab7335ffa959f7 size 121683348 diff --git a/layer_05-model_00-model_states.pt b/layer_05-model_00-model_states.pt index 50df171baaa81e642a6976ac29629a0d9f20ed07..0f682c171f5ddec039e91d75708d5246d327d07f 100644 --- a/layer_05-model_00-model_states.pt +++ b/layer_05-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1edf0c38e81393725eee8e51ab80cd6d8a71cd76582846faf9a46593ae8bcf80 +oid sha256:15d81d25f436250bbd776fea478eb4e423de8d7e8bd90b0d0cf57b9b5c735f02 size 121683348 diff --git a/layer_06-model_00-model_states.pt b/layer_06-model_00-model_states.pt index 41555437fe0a93d94b27c5163eb1d0029ace5c41..365460e351262d7f3480c22a8bc9d0be28356711 100644 --- a/layer_06-model_00-model_states.pt +++ b/layer_06-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6ec5aeff2aa09dad17caf0b4ca70d2f0f79cb4752589db3f635d9e2e61f29bed +oid sha256:0ccd9af41bd28dc541978fb650f15c18188837469219b84d0c829b25497cf806 size 121683348 diff --git a/layer_07-model_00-model_states.pt b/layer_07-model_00-model_states.pt index d806c7d40d6071fdd134771a179519e94123fdf5..e2c805eb8feb77ea5476203c4e514dca23116eb1 100644 --- a/layer_07-model_00-model_states.pt +++ b/layer_07-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4379ff9e65f7d3d1c487919e606c5931e203cdff146722209bb268f0a589f48a +oid sha256:035f3f87c22a1d43d1c7aed65ee79b74c57b542803f183db0f3703898b8f6aab size 121683348 diff --git a/layer_08-model_00-model_states.pt b/layer_08-model_00-model_states.pt index 790463ab6328dea5ebadedf6c0eb05a7003dd39f..1afa7ad504c43c28ec06c508af2811d40bc12429 100644 --- a/layer_08-model_00-model_states.pt +++ b/layer_08-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e6af4673d523cf86bc0ac6ed673fa79758eb9b9504e7282ec81c97593d9218b8 +oid sha256:ce5649cdbd39344ba04764fbf47aa32523c77ffc6559c03cdf3a0e15adca4de8 size 121683348 diff --git a/layer_09-model_00-model_states.pt b/layer_09-model_00-model_states.pt index 8692b38d18f5a01b17aaf6fc62011e26382fd7d8..4da16ab04d4ba5c7193747c05b4ede181239ff73 100644 --- a/layer_09-model_00-model_states.pt +++ b/layer_09-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c21b767bd4117ac4c7fb91afd852925ddcbe938e891a762831469c9a8bf11a53 +oid sha256:4f3212f8bf0f417839463167d1ddbde6359372052abc9a05219160447dbc78f9 size 121683348 diff --git a/layer_10-model_00-model_states.pt b/layer_10-model_00-model_states.pt index 602afc581ed21297ebe916f2ded053ce4685f725..d36864de8ff3dd97b6965d2fe2f07e32dabfc0c1 100644 --- a/layer_10-model_00-model_states.pt +++ b/layer_10-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3f5470d9a321113f9e95a961f71109b89ea7b0033dbc916453fb9898e2428b1d +oid sha256:10661ea62d8ce16bf9ce43cefbc08a616aee1ef2b50c97850dce1511af64a3eb size 121683348 diff --git a/layer_11-model_00-model_states.pt b/layer_11-model_00-model_states.pt index 89c2d2becd5375496664f40a3d081971d77ed871..a0e4c963ba7120345653452bf3a1553a57d9c4b7 100644 --- a/layer_11-model_00-model_states.pt +++ b/layer_11-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d482b2179483f4e89ebbfc49b35dc0b776578ce47dda91dcb494dce50ee78ac3 +oid sha256:7e77941f2f5f4f733ef181b222d3f144de5a5e00b79a1a4306b9f6cf573840dd size 121683348 diff --git a/layer_12-model_00-model_states.pt b/layer_12-model_00-model_states.pt index 551d18b9b411fa2b766479a8cb4d996dea42b28a..74a5b505d0fd8c49e33445d2091ccc7e47fc4947 100644 --- a/layer_12-model_00-model_states.pt +++ b/layer_12-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b6729e9e27ce307a630eb0a1ace765d6383ae7a7b4dce7e986145348d2505b14 +oid sha256:18d78572f0d1f71bb326e1866be346489c0b481c3bb432d24ddd5a14fe94a5f9 size 121683348 diff --git a/layer_13-model_00-model_states.pt b/layer_13-model_00-model_states.pt index 1076e9b481bff1fca5e37b29e5e8982dcc2a7e95..80c0e207b89b5a71444bfdf94bbb1b4429b7e29e 100644 --- a/layer_13-model_00-model_states.pt +++ b/layer_13-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a9d0b99dec369a938afc006a24c88d01501197b9d5cd7928f33b1055c9c671a4 +oid sha256:5db493fa02ac8db582c6098b065c24816842acc50242e72923d768ec7844a9c0 size 121683348 diff --git a/layer_14-model_00-model_states.pt b/layer_14-model_00-model_states.pt index 641017dd32031b9fa8cd317e2b62813a714ff5b2..b149997071f69f8c7c6aafa1eead821cdd07f12a 100644 --- a/layer_14-model_00-model_states.pt +++ b/layer_14-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b6964f39cace8f21e88a0715d1c7302b040d663929fba972806146330ff70afe +oid sha256:46a320a13b43ed5d5b5f007f680791a1bf6ccc8abadc73ea75afb50eb6d56430 size 121683348 diff --git a/layer_15-model_00-model_states.pt b/layer_15-model_00-model_states.pt index 3bb0adfb7d65a875954f719b02649335559de052..4efa46c6885c0272c9ed8e584d56ce339926d639 100644 --- a/layer_15-model_00-model_states.pt +++ b/layer_15-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5eb8beb39ec159a55fbbb1400cd556f7348ff925a5d1881639ba0af9570bb0fb +oid sha256:9ad57e1cbd4ad8d5af769a765501e3ec94f937a7e84d991c407f8c10320a0990 size 121683348 diff --git a/layer_16-model_00-model_states.pt b/layer_16-model_00-model_states.pt index 539cebbb7c8e02d982460f5580bf083eaa46aedf..b28597fbff99ff84a3668dd89bd0fe52df908230 100644 --- a/layer_16-model_00-model_states.pt +++ b/layer_16-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:05b14d8f735c8dbb5b81a72c3e93f7d0ae70c89adecccfb4d2b8be3d3c1a5646 +oid sha256:456b7266537dcad59ab0c5c62b7d442210de542c08580d0f1c6a73b7c3e9707a size 121683348 diff --git a/layer_17-model_00-model_states.pt b/layer_17-model_00-model_states.pt index 302e8d1243774ba078cd7393673907726c460eec..2cf9ee5500d07bdd6f2696a5f064c165a0fcef45 100644 --- a/layer_17-model_00-model_states.pt +++ b/layer_17-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f8410ddea715f3e4ae2cb26f51706f3a7a5afcac9282beea0db53e8ab0130552 +oid sha256:5d37da346301fa2ef60e73498f0fa4869857c4bb18d21c83f97a936be7e44ed7 size 121683348 diff --git a/layer_19-model_00-model_states.pt b/layer_19-model_00-model_states.pt index f5adaa419efe12f0b1a2ba48650acb54c832fda2..a32c52224a71dc66969efeccf765e82b034e3c18 100644 --- a/layer_19-model_00-model_states.pt +++ b/layer_19-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3fcdc85617c9b5f702662c5a674a38237157c40693f6cc5f2143051d1981fb99 +oid sha256:bf63163a1cdde516533f711b3b980199f9ff67b91fd9f9c74c7f3b3d1af93241 size 5519 diff --git a/layer_20-model_00-model_states.pt b/layer_20-model_00-model_states.pt index 32ed022dca1801cb98ebaee2ddefcdc421cd2ea3..05652bb6b5d062b75c99746fae033ac18d02ebde 100644 --- a/layer_20-model_00-model_states.pt +++ b/layer_20-model_00-model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:861ae73ef3f7ac9f740f11587216c041329eb0da9b3c298e51dcba6155267413 +oid sha256:8d76ff10f2ac2292b9f0dfa86b765e17f4704d4ed6990c9dbad1be74be300d92 size 206046607 diff --git a/mp_rank_00_model_states.pt b/mp_rank_00_model_states.pt index 2d541cde7e606abe5c6d7438289de546300e022e..cdfc89cce1d730851f344927395acdbd2fab7107 100644 --- a/mp_rank_00_model_states.pt +++ b/mp_rank_00_model_states.pt @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:85c6f2e597e533ba0cae70fb199a1acd5dc3571312d4751daf05ef0acc76a2c9 +oid sha256:ac941bf33bca0a3134d810c78e71910f3ba87db71aec1009601525310c374bcc size 23620