model_name: molmo
llm:
  d_model: 4096
  n_heads: 32
  n_kv_heads: null
  head_dim: null
  qkv_bias: false
  clip_qkv: null
  n_layers: 32
  mlp_ratio: 4
  mlp_hidden_size: 22016
  activation_type: swiglu
  block_type: sequential
  rope: true
  rope_full_precision: true
  rope_theta: 500000.0
  rope_type: default
  rope_factor: null
  rope_high_freq_factor: null
  rope_low_freq_factor: null
  rope_original_max_position_embeddings: null
  attention_type: sdpa
  float32_attention: true
  attention_dropout: 0.0
  attention_layer_norm: true
  attention_layer_norm_type: olmo
  residual_dropout: 0.1
  response_residual_dropout: 0.0
  layer_norm_type: rms
  layer_norm_with_affine: true
  layer_norm_eps: 1.0e-06
  attention_layer_norm_with_affine: true
  max_sequence_length: 4096
  max_position_embeddings: null
  include_bias: false
  bias_for_layer_norm: false
  norm_after: true
  moe_num_experts: 8
  moe_top_k: 2
  moe_mlp_impl: sparse
  moe_log_expert_assignment: false
  moe_shared_expert: false
  moe_lbl_in_fp32: false
  moe_interleave: false
  moe_loss_weight: 0.1
  moe_zloss_weight: null
  moe_dropless: true
  moe_capacity_factor: 1.25
  embedding_dropout: 0.0
  scale_logits: false
  vocab_size: 100278
  additional_vocab_size: 128
  weight_tying: false
  embedding_size: 100864
  use_position_ids: true
  tokenizer:
    identifier: allenai/OLMo-2-1124-7B
    tokenizer_dir: null
    depth_tokens: true
  init_path: /weka/oe-training-default/mm-olmo/pretrained_llms/olmo2-1124-7b.pt
  init_incremental: null
  new_embedding_init_range: 0.02
  initializer_range: 0.02
  normalize_input_embeds: false
  activation_checkpoint: whole_layer
  compile: blocks
  fix_pad_tokenizer: true
  resize_vocab: true
  init_std: 0.02
  init_fn: normal
  init_cutoff_factor: null
vision_backbone:
  vit:
    image_model_type: openai
    image_default_input_size:
    - 336
    - 336
    image_patch_size: 14
    image_pos_patch_size: 14
    image_emb_dim: 1024
    image_num_heads: 16
    image_num_key_value_heads: 16
    image_num_layers: 23
    image_head_dim: 64
    image_mlp_dim: 4096
    image_mlp_activations: quick_gelu
    image_dropout_rate: 0.0
    image_num_pos: 577
    image_norm_eps: 1.0e-05
    attention_dropout: 0.0
    residual_dropout: 0.0
    initializer_range: 0.02
    float32_attention: true
    attention_type: sdpa
    activation_checkpointing: true
    init_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt
    resize_mode: default
    pad_value: 0.0
    normalize: openai
  image_pooling_2d: attention_meanq
  pooling_attention_mask: false
  image_projector: mlp
  image_padding_embed: pad_and_partial_pad
  vit_layers:
  - -2
  - -9
  skip_unused_layers: true
  image_feature_dropout: 0.0
  connector_activation_checkpointing: true
  compile_vit: blocks
data_formatter:
  prompt_templates: uber_model
  message_format: role
  system_prompt: demo_or_style
  always_start_with_space: false
  default_inference_len: 65
  select_answer: best
  debug: false
  image_last: false
  format_message_list: null
  p_one_message: 0.0
mm_preprocessor:
  crop_mode: overlap-and-resize-c2
  max_crops: 8
  max_images: 2
  max_multi_image_crops: 8
  pooling_w: 2
  pooling_h: 2
  overlap_margins:
  - 4
  - 4
  use_col_tokens: true
  loss_token_weighting: root_subsegments
  legacy_image_mask: false
  max_answer_len: null
  img_aug: false
bi_directional_attn: null
lora_enable: false
lora_rank: 64
lora_alpha: 16
lora_dropout: 0.05
lora_bias: none
n_action_bins: 256
norm_stats:
  molmoact:
    action:
      mean:
      - 0.0005706787342205644
      - 0.0002448957529850304
      - -3.5987635783385485e-05
      - 0.00021597897284664214
      - -0.0004896928439848125
      - -0.000241481073317118
      - 0.5570635199546814
      std:
      - 0.005207270849496126
      - 0.007506529800593853
      - 0.006415561307221651
      - 0.013248044066131115
      - 0.010928540490567684
      - 0.014873150736093521
      - 0.49715080857276917
      min:
      - -0.07434078305959702
      - -0.07339745759963989
      - -0.06539416313171387
      - -0.1688285619020462
      - -0.10289879888296127
      - -0.2667275667190552
      - 0.0
      max:
      - 0.06042003631591797
      - 0.09417290985584259
      - 0.07019275426864624
      - 0.2616892158985138
      - 0.11751057207584381
      - 0.16968433558940887
      - 1.0
      q01:
      - -0.01538565568625927
      - -0.021047022193670273
      - -0.01688069850206375
      - -0.044314172118902206
      - -0.03890235349535942
      - -0.04788423702120781
      - 0.0
      q99:
      - 0.014661382883787155
      - 0.026515591889619827
      - 0.021398313343524933
      - 0.04216696694493294
      - 0.03401297703385353
      - 0.04957397282123566
      - 1.0
    num_entries: 1560068