added the checkpoints

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+1b-model filter=lfs diff=lfs merge=lfs -text
+3b-model filter=lfs diff=lfs merge=lfs -text
+70m-model/ filter=lfs diff=lfs merge=lfs -text

1b-model/best-model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d7b790ce2c915b22fe7175c5d15bae67bb1e8b121147f0f9c6f38f37288304a
+size 5415539456

1b-model/collator_config.yml

@@ -0,0 +1,13 @@
+do_mlm: true
+num_bins: 51
+sampling: true
+pad_value: -2
+data_style: both
+do_binning: true
+do_padding: true
+mask_value: -1
+max_length: 1024
+right_binning: false
+mlm_probability: 0.5
+use_junk_tokens: false
+pad_token_id: 0

1b-model/model_config.yml

@@ -0,0 +1,40 @@
+mvc:
+  arch_style: inner product
+  query_activation: sigmoid
+  scaled_dot_product: true
+name: vevo_scgpt
+d_model: 2048
+n_heads: 16
+use_glu: false
+n_layers: 24
+attn_config:
+  attn_impl: triton
+  attn_type: grouped_query_attention
+  kv_nheads: 16
+  attn_pdrop: 0
+init_device: cpu
+norm_config:
+  eps: 1.0e-05
+  norm_type: layernorm
+norm_scheme: pre
+gene_encoder:
+  use_norm: true
+cell_emb_style: cls
+expansion_ratio: 4
+expression_decoder:
+  n_layers: 1
+  n_outputs: 1
+  activation: leaky_relu
+expression_encoder:
+  dropout: 0.1
+  use_norm: true
+  max_value: 512
+  activation: relu
+  input_emb_style: continuous
+standard_scale_outputs: false
+transformer_activation: relu
+use_generative_training: false
+use_cell_conditioned_generation: false
+vocab_size: 62720
+precision: amp_bf16
+wandb_id: vevotx/vevo-scgpt/26iormxc

3b-model/best-model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4c94fcc1f970688c44543888b9edc86fb26e0c9173e8c5591e07c99ccc04a64
+size 10868338326

3b-model/collator_config.yml

@@ -0,0 +1,18 @@
+do_padding: true
+pad_value: -2
+do_mlm: true
+do_binning: true
+mlm_probability: 0.5
+mask_value: -1
+max_length: 2048
+sampling: true
+data_style: both
+num_bins: 51
+right_binning: false
+use_junk_tokens: false
+use_chem_token: true
+drug_to_id_path:
+  remote: s3://vevo-ml-datasets/mosaicfm_v2/datasets/drug_to_id_pad.json
+  local: drug_to_id_pad.json
+keep_first_n_tokens: 2
+pad_token_id: 0

3b-model/model_config.yml

@@ -0,0 +1,48 @@
+name: mosaicfm
+d_model: 2560
+n_layers: 32
+init_device: cpu
+expansion_ratio: 4
+standard_scale_outputs: false
+transformer_activation: gelu
+n_heads: 20
+norm_scheme: pre
+use_generative_training: false
+use_cell_conditioned_generation: false
+use_glu: false
+cell_emb_style: cls
+attn_config:
+  attn_impl: flash
+  use_attn_mask: false
+  attn_type: grouped_query_attention
+  kv_nheads: 20
+  attn_pdrop: 0.0
+norm_config:
+  norm_type: layernorm
+  eps: 1.0e-05
+expression_encoder:
+  input_emb_style: continuous
+  dropout: 0.1
+  max_value: 512
+  activation: gelu
+  use_norm: true
+gene_encoder:
+  use_norm: true
+mvc:
+  arch_style: inner product
+  query_activation: sigmoid
+  scaled_dot_product: true
+expression_decoder:
+  n_outputs: 1
+  n_layers: 1
+  activation: gelu
+chemical_encoder:
+  drug_fps_path:
+    remote: s3://vevo-ml-datasets/mosaicfm_v2/datasets/drug_fps_pad.npy
+    local: drug_fps_pad.npy
+  activation: gelu
+  padding_idx: 0
+  freeze: false
+vocab_size: 62721
+precision: amp_bf16
+wandb_id: vevotx/vevo-MFM-v2/cdk744ih

70m-model/best-model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0452ea29a2d81dbdac83ccb05a30ff327dc947b471248f3f5bddb9566661c152
+size 284050604

70m-model/collator_config.yml

@@ -0,0 +1,13 @@
+do_mlm: true
+num_bins: 51
+sampling: true
+pad_value: -2
+data_style: both
+do_binning: true
+do_padding: true
+mask_value: -1
+max_length: 1024
+right_binning: false
+mlm_probability: 0.5
+use_junk_tokens: false
+pad_token_id: 0

70m-model/model_config.yml

@@ -0,0 +1,40 @@
+mvc:
+  arch_style: inner product
+  query_activation: sigmoid
+  scaled_dot_product: true
+name: vevo_scgpt
+d_model: 512
+n_heads: 8
+use_glu: false
+n_layers: 12
+attn_config:
+  attn_impl: triton
+  attn_type: grouped_query_attention
+  kv_nheads: 8
+  attn_pdrop: 0
+init_device: cpu
+norm_config:
+  eps: 1.0e-05
+  norm_type: layernorm
+norm_scheme: pre
+gene_encoder:
+  use_norm: true
+cell_emb_style: cls
+expansion_ratio: 4
+expression_decoder:
+  n_layers: 1
+  n_outputs: 1
+  activation: leaky_relu
+expression_encoder:
+  dropout: 0.1
+  use_norm: true
+  max_value: 512
+  activation: relu
+  input_emb_style: continuous
+standard_scale_outputs: false
+transformer_activation: relu
+use_generative_training: false
+use_cell_conditioned_generation: false
+vocab_size: 62720
+precision: amp_bf16
+wandb_id: vevotx/vevo-scgpt/ftb65le8