{ "base": { "dir": "/workspace/models/model-2.5-6", "bpw": 2.5161406536856386 }, "alts": [ { "dir": "/workspace/models/model-3.5-6", "bpw": 3.515917257669038 } ], "groups": [ { "idx": 0, "layers": [ "model.layers.0.self_attn.q_proj", "model.layers.0.self_attn.k_proj", "model.layers.0.self_attn.v_proj", "model.layers.0.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0011784225702285711, "dbits": 109051904 } ] }, { "idx": 1, "layers": [ "model.layers.0.mlp.gate_proj", "model.layers.0.mlp.up_proj", "model.layers.0.mlp.down_proj" ], "candidates": [ { "dkld": -0.0027617249172180725, "dbits": 135266304 } ] }, { "idx": 2, "layers": [ "model.layers.1.self_attn.q_proj", "model.layers.1.self_attn.k_proj", "model.layers.1.self_attn.v_proj", "model.layers.1.self_attn.o_proj" ], "candidates": [ { "dkld": 0.002751186210662143, "dbits": 109051904 } ] }, { "idx": 3, "layers": [ "model.layers.1.mlp.shared_experts.gate_proj", "model.layers.1.mlp.shared_experts.up_proj", "model.layers.1.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.009815181046724325, "dbits": 17301504 } ] }, { "idx": 4, "layers": [ "model.layers.1.mlp.experts.0.gate_proj", "model.layers.1.mlp.experts.1.gate_proj", "model.layers.1.mlp.experts.2.gate_proj", "model.layers.1.mlp.experts.3.gate_proj", "model.layers.1.mlp.experts.4.gate_proj", "model.layers.1.mlp.experts.5.gate_proj", "model.layers.1.mlp.experts.6.gate_proj", "model.layers.1.mlp.experts.7.gate_proj", "model.layers.1.mlp.experts.8.gate_proj", "model.layers.1.mlp.experts.9.gate_proj", "model.layers.1.mlp.experts.10.gate_proj", "model.layers.1.mlp.experts.11.gate_proj", "model.layers.1.mlp.experts.12.gate_proj", "model.layers.1.mlp.experts.13.gate_proj", "model.layers.1.mlp.experts.14.gate_proj", "model.layers.1.mlp.experts.15.gate_proj", "model.layers.1.mlp.experts.16.gate_proj", "model.layers.1.mlp.experts.17.gate_proj", "model.layers.1.mlp.experts.18.gate_proj", "model.layers.1.mlp.experts.19.gate_proj", "model.layers.1.mlp.experts.20.gate_proj", "model.layers.1.mlp.experts.21.gate_proj", "model.layers.1.mlp.experts.22.gate_proj", "model.layers.1.mlp.experts.23.gate_proj", "model.layers.1.mlp.experts.24.gate_proj", "model.layers.1.mlp.experts.25.gate_proj", "model.layers.1.mlp.experts.26.gate_proj", "model.layers.1.mlp.experts.27.gate_proj", "model.layers.1.mlp.experts.28.gate_proj", "model.layers.1.mlp.experts.29.gate_proj", "model.layers.1.mlp.experts.30.gate_proj", "model.layers.1.mlp.experts.31.gate_proj", "model.layers.1.mlp.experts.32.gate_proj", "model.layers.1.mlp.experts.33.gate_proj", "model.layers.1.mlp.experts.34.gate_proj", "model.layers.1.mlp.experts.35.gate_proj", "model.layers.1.mlp.experts.36.gate_proj", "model.layers.1.mlp.experts.37.gate_proj", "model.layers.1.mlp.experts.38.gate_proj", "model.layers.1.mlp.experts.39.gate_proj", "model.layers.1.mlp.experts.40.gate_proj", "model.layers.1.mlp.experts.41.gate_proj", "model.layers.1.mlp.experts.42.gate_proj", "model.layers.1.mlp.experts.43.gate_proj", "model.layers.1.mlp.experts.44.gate_proj", "model.layers.1.mlp.experts.45.gate_proj", "model.layers.1.mlp.experts.46.gate_proj", "model.layers.1.mlp.experts.47.gate_proj", "model.layers.1.mlp.experts.48.gate_proj", "model.layers.1.mlp.experts.49.gate_proj", "model.layers.1.mlp.experts.50.gate_proj", "model.layers.1.mlp.experts.51.gate_proj", "model.layers.1.mlp.experts.52.gate_proj", "model.layers.1.mlp.experts.53.gate_proj", "model.layers.1.mlp.experts.54.gate_proj", "model.layers.1.mlp.experts.55.gate_proj", "model.layers.1.mlp.experts.56.gate_proj", "model.layers.1.mlp.experts.57.gate_proj", "model.layers.1.mlp.experts.58.gate_proj", "model.layers.1.mlp.experts.59.gate_proj", "model.layers.1.mlp.experts.60.gate_proj", "model.layers.1.mlp.experts.61.gate_proj", "model.layers.1.mlp.experts.62.gate_proj", "model.layers.1.mlp.experts.63.gate_proj", "model.layers.1.mlp.experts.64.gate_proj", "model.layers.1.mlp.experts.65.gate_proj", "model.layers.1.mlp.experts.66.gate_proj", "model.layers.1.mlp.experts.67.gate_proj", "model.layers.1.mlp.experts.68.gate_proj", "model.layers.1.mlp.experts.69.gate_proj", "model.layers.1.mlp.experts.70.gate_proj", "model.layers.1.mlp.experts.71.gate_proj", "model.layers.1.mlp.experts.72.gate_proj", "model.layers.1.mlp.experts.73.gate_proj", "model.layers.1.mlp.experts.74.gate_proj", "model.layers.1.mlp.experts.75.gate_proj", "model.layers.1.mlp.experts.76.gate_proj", "model.layers.1.mlp.experts.77.gate_proj", "model.layers.1.mlp.experts.78.gate_proj", "model.layers.1.mlp.experts.79.gate_proj", "model.layers.1.mlp.experts.80.gate_proj", "model.layers.1.mlp.experts.81.gate_proj", "model.layers.1.mlp.experts.82.gate_proj", "model.layers.1.mlp.experts.83.gate_proj", "model.layers.1.mlp.experts.84.gate_proj", "model.layers.1.mlp.experts.85.gate_proj", "model.layers.1.mlp.experts.86.gate_proj", "model.layers.1.mlp.experts.87.gate_proj", "model.layers.1.mlp.experts.88.gate_proj", "model.layers.1.mlp.experts.89.gate_proj", "model.layers.1.mlp.experts.90.gate_proj", "model.layers.1.mlp.experts.91.gate_proj", "model.layers.1.mlp.experts.92.gate_proj", "model.layers.1.mlp.experts.93.gate_proj", "model.layers.1.mlp.experts.94.gate_proj", "model.layers.1.mlp.experts.95.gate_proj", "model.layers.1.mlp.experts.96.gate_proj", "model.layers.1.mlp.experts.97.gate_proj", "model.layers.1.mlp.experts.98.gate_proj", "model.layers.1.mlp.experts.99.gate_proj", "model.layers.1.mlp.experts.100.gate_proj", "model.layers.1.mlp.experts.101.gate_proj", "model.layers.1.mlp.experts.102.gate_proj", "model.layers.1.mlp.experts.103.gate_proj", "model.layers.1.mlp.experts.104.gate_proj", "model.layers.1.mlp.experts.105.gate_proj", "model.layers.1.mlp.experts.106.gate_proj", "model.layers.1.mlp.experts.107.gate_proj", "model.layers.1.mlp.experts.108.gate_proj", "model.layers.1.mlp.experts.109.gate_proj", "model.layers.1.mlp.experts.110.gate_proj", "model.layers.1.mlp.experts.111.gate_proj", "model.layers.1.mlp.experts.112.gate_proj", "model.layers.1.mlp.experts.113.gate_proj", "model.layers.1.mlp.experts.114.gate_proj", "model.layers.1.mlp.experts.115.gate_proj", "model.layers.1.mlp.experts.116.gate_proj", "model.layers.1.mlp.experts.117.gate_proj", "model.layers.1.mlp.experts.118.gate_proj", "model.layers.1.mlp.experts.119.gate_proj", "model.layers.1.mlp.experts.120.gate_proj", "model.layers.1.mlp.experts.121.gate_proj", "model.layers.1.mlp.experts.122.gate_proj", "model.layers.1.mlp.experts.123.gate_proj", "model.layers.1.mlp.experts.124.gate_proj", "model.layers.1.mlp.experts.125.gate_proj", "model.layers.1.mlp.experts.126.gate_proj", "model.layers.1.mlp.experts.127.gate_proj", "model.layers.1.mlp.experts.0.up_proj", "model.layers.1.mlp.experts.1.up_proj", "model.layers.1.mlp.experts.2.up_proj", "model.layers.1.mlp.experts.3.up_proj", "model.layers.1.mlp.experts.4.up_proj", "model.layers.1.mlp.experts.5.up_proj", "model.layers.1.mlp.experts.6.up_proj", "model.layers.1.mlp.experts.7.up_proj", "model.layers.1.mlp.experts.8.up_proj", "model.layers.1.mlp.experts.9.up_proj", "model.layers.1.mlp.experts.10.up_proj", "model.layers.1.mlp.experts.11.up_proj", "model.layers.1.mlp.experts.12.up_proj", "model.layers.1.mlp.experts.13.up_proj", "model.layers.1.mlp.experts.14.up_proj", "model.layers.1.mlp.experts.15.up_proj", "model.layers.1.mlp.experts.16.up_proj", "model.layers.1.mlp.experts.17.up_proj", "model.layers.1.mlp.experts.18.up_proj", "model.layers.1.mlp.experts.19.up_proj", "model.layers.1.mlp.experts.20.up_proj", "model.layers.1.mlp.experts.21.up_proj", "model.layers.1.mlp.experts.22.up_proj", "model.layers.1.mlp.experts.23.up_proj", "model.layers.1.mlp.experts.24.up_proj", "model.layers.1.mlp.experts.25.up_proj", "model.layers.1.mlp.experts.26.up_proj", "model.layers.1.mlp.experts.27.up_proj", "model.layers.1.mlp.experts.28.up_proj", "model.layers.1.mlp.experts.29.up_proj", "model.layers.1.mlp.experts.30.up_proj", "model.layers.1.mlp.experts.31.up_proj", "model.layers.1.mlp.experts.32.up_proj", "model.layers.1.mlp.experts.33.up_proj", "model.layers.1.mlp.experts.34.up_proj", "model.layers.1.mlp.experts.35.up_proj", "model.layers.1.mlp.experts.36.up_proj", "model.layers.1.mlp.experts.37.up_proj", "model.layers.1.mlp.experts.38.up_proj", "model.layers.1.mlp.experts.39.up_proj", "model.layers.1.mlp.experts.40.up_proj", "model.layers.1.mlp.experts.41.up_proj", "model.layers.1.mlp.experts.42.up_proj", "model.layers.1.mlp.experts.43.up_proj", "model.layers.1.mlp.experts.44.up_proj", "model.layers.1.mlp.experts.45.up_proj", "model.layers.1.mlp.experts.46.up_proj", "model.layers.1.mlp.experts.47.up_proj", "model.layers.1.mlp.experts.48.up_proj", "model.layers.1.mlp.experts.49.up_proj", "model.layers.1.mlp.experts.50.up_proj", "model.layers.1.mlp.experts.51.up_proj", "model.layers.1.mlp.experts.52.up_proj", "model.layers.1.mlp.experts.53.up_proj", "model.layers.1.mlp.experts.54.up_proj", "model.layers.1.mlp.experts.55.up_proj", "model.layers.1.mlp.experts.56.up_proj", "model.layers.1.mlp.experts.57.up_proj", "model.layers.1.mlp.experts.58.up_proj", "model.layers.1.mlp.experts.59.up_proj", "model.layers.1.mlp.experts.60.up_proj", "model.layers.1.mlp.experts.61.up_proj", "model.layers.1.mlp.experts.62.up_proj", "model.layers.1.mlp.experts.63.up_proj", "model.layers.1.mlp.experts.64.up_proj", "model.layers.1.mlp.experts.65.up_proj", "model.layers.1.mlp.experts.66.up_proj", "model.layers.1.mlp.experts.67.up_proj", "model.layers.1.mlp.experts.68.up_proj", "model.layers.1.mlp.experts.69.up_proj", "model.layers.1.mlp.experts.70.up_proj", "model.layers.1.mlp.experts.71.up_proj", "model.layers.1.mlp.experts.72.up_proj", "model.layers.1.mlp.experts.73.up_proj", "model.layers.1.mlp.experts.74.up_proj", "model.layers.1.mlp.experts.75.up_proj", "model.layers.1.mlp.experts.76.up_proj", "model.layers.1.mlp.experts.77.up_proj", "model.layers.1.mlp.experts.78.up_proj", "model.layers.1.mlp.experts.79.up_proj", "model.layers.1.mlp.experts.80.up_proj", "model.layers.1.mlp.experts.81.up_proj", "model.layers.1.mlp.experts.82.up_proj", "model.layers.1.mlp.experts.83.up_proj", "model.layers.1.mlp.experts.84.up_proj", "model.layers.1.mlp.experts.85.up_proj", "model.layers.1.mlp.experts.86.up_proj", "model.layers.1.mlp.experts.87.up_proj", "model.layers.1.mlp.experts.88.up_proj", "model.layers.1.mlp.experts.89.up_proj", "model.layers.1.mlp.experts.90.up_proj", "model.layers.1.mlp.experts.91.up_proj", "model.layers.1.mlp.experts.92.up_proj", "model.layers.1.mlp.experts.93.up_proj", "model.layers.1.mlp.experts.94.up_proj", "model.layers.1.mlp.experts.95.up_proj", "model.layers.1.mlp.experts.96.up_proj", "model.layers.1.mlp.experts.97.up_proj", "model.layers.1.mlp.experts.98.up_proj", "model.layers.1.mlp.experts.99.up_proj", "model.layers.1.mlp.experts.100.up_proj", "model.layers.1.mlp.experts.101.up_proj", "model.layers.1.mlp.experts.102.up_proj", "model.layers.1.mlp.experts.103.up_proj", "model.layers.1.mlp.experts.104.up_proj", "model.layers.1.mlp.experts.105.up_proj", "model.layers.1.mlp.experts.106.up_proj", "model.layers.1.mlp.experts.107.up_proj", "model.layers.1.mlp.experts.108.up_proj", "model.layers.1.mlp.experts.109.up_proj", "model.layers.1.mlp.experts.110.up_proj", "model.layers.1.mlp.experts.111.up_proj", "model.layers.1.mlp.experts.112.up_proj", "model.layers.1.mlp.experts.113.up_proj", "model.layers.1.mlp.experts.114.up_proj", "model.layers.1.mlp.experts.115.up_proj", "model.layers.1.mlp.experts.116.up_proj", "model.layers.1.mlp.experts.117.up_proj", "model.layers.1.mlp.experts.118.up_proj", "model.layers.1.mlp.experts.119.up_proj", "model.layers.1.mlp.experts.120.up_proj", "model.layers.1.mlp.experts.121.up_proj", "model.layers.1.mlp.experts.122.up_proj", "model.layers.1.mlp.experts.123.up_proj", "model.layers.1.mlp.experts.124.up_proj", "model.layers.1.mlp.experts.125.up_proj", "model.layers.1.mlp.experts.126.up_proj", "model.layers.1.mlp.experts.127.up_proj", "model.layers.1.mlp.experts.0.down_proj", "model.layers.1.mlp.experts.1.down_proj", "model.layers.1.mlp.experts.2.down_proj", "model.layers.1.mlp.experts.3.down_proj", "model.layers.1.mlp.experts.4.down_proj", "model.layers.1.mlp.experts.5.down_proj", "model.layers.1.mlp.experts.6.down_proj", "model.layers.1.mlp.experts.7.down_proj", "model.layers.1.mlp.experts.8.down_proj", "model.layers.1.mlp.experts.9.down_proj", "model.layers.1.mlp.experts.10.down_proj", "model.layers.1.mlp.experts.11.down_proj", "model.layers.1.mlp.experts.12.down_proj", "model.layers.1.mlp.experts.13.down_proj", "model.layers.1.mlp.experts.14.down_proj", "model.layers.1.mlp.experts.15.down_proj", "model.layers.1.mlp.experts.16.down_proj", "model.layers.1.mlp.experts.17.down_proj", "model.layers.1.mlp.experts.18.down_proj", "model.layers.1.mlp.experts.19.down_proj", "model.layers.1.mlp.experts.20.down_proj", "model.layers.1.mlp.experts.21.down_proj", "model.layers.1.mlp.experts.22.down_proj", "model.layers.1.mlp.experts.23.down_proj", "model.layers.1.mlp.experts.24.down_proj", "model.layers.1.mlp.experts.25.down_proj", "model.layers.1.mlp.experts.26.down_proj", "model.layers.1.mlp.experts.27.down_proj", "model.layers.1.mlp.experts.28.down_proj", "model.layers.1.mlp.experts.29.down_proj", "model.layers.1.mlp.experts.30.down_proj", "model.layers.1.mlp.experts.31.down_proj", "model.layers.1.mlp.experts.32.down_proj", "model.layers.1.mlp.experts.33.down_proj", "model.layers.1.mlp.experts.34.down_proj", "model.layers.1.mlp.experts.35.down_proj", "model.layers.1.mlp.experts.36.down_proj", "model.layers.1.mlp.experts.37.down_proj", "model.layers.1.mlp.experts.38.down_proj", "model.layers.1.mlp.experts.39.down_proj", "model.layers.1.mlp.experts.40.down_proj", "model.layers.1.mlp.experts.41.down_proj", "model.layers.1.mlp.experts.42.down_proj", "model.layers.1.mlp.experts.43.down_proj", "model.layers.1.mlp.experts.44.down_proj", "model.layers.1.mlp.experts.45.down_proj", "model.layers.1.mlp.experts.46.down_proj", "model.layers.1.mlp.experts.47.down_proj", "model.layers.1.mlp.experts.48.down_proj", "model.layers.1.mlp.experts.49.down_proj", "model.layers.1.mlp.experts.50.down_proj", "model.layers.1.mlp.experts.51.down_proj", "model.layers.1.mlp.experts.52.down_proj", "model.layers.1.mlp.experts.53.down_proj", "model.layers.1.mlp.experts.54.down_proj", "model.layers.1.mlp.experts.55.down_proj", "model.layers.1.mlp.experts.56.down_proj", "model.layers.1.mlp.experts.57.down_proj", "model.layers.1.mlp.experts.58.down_proj", "model.layers.1.mlp.experts.59.down_proj", "model.layers.1.mlp.experts.60.down_proj", "model.layers.1.mlp.experts.61.down_proj", "model.layers.1.mlp.experts.62.down_proj", "model.layers.1.mlp.experts.63.down_proj", "model.layers.1.mlp.experts.64.down_proj", "model.layers.1.mlp.experts.65.down_proj", "model.layers.1.mlp.experts.66.down_proj", "model.layers.1.mlp.experts.67.down_proj", "model.layers.1.mlp.experts.68.down_proj", "model.layers.1.mlp.experts.69.down_proj", "model.layers.1.mlp.experts.70.down_proj", "model.layers.1.mlp.experts.71.down_proj", "model.layers.1.mlp.experts.72.down_proj", "model.layers.1.mlp.experts.73.down_proj", "model.layers.1.mlp.experts.74.down_proj", "model.layers.1.mlp.experts.75.down_proj", "model.layers.1.mlp.experts.76.down_proj", "model.layers.1.mlp.experts.77.down_proj", "model.layers.1.mlp.experts.78.down_proj", "model.layers.1.mlp.experts.79.down_proj", "model.layers.1.mlp.experts.80.down_proj", "model.layers.1.mlp.experts.81.down_proj", "model.layers.1.mlp.experts.82.down_proj", "model.layers.1.mlp.experts.83.down_proj", "model.layers.1.mlp.experts.84.down_proj", "model.layers.1.mlp.experts.85.down_proj", "model.layers.1.mlp.experts.86.down_proj", "model.layers.1.mlp.experts.87.down_proj", "model.layers.1.mlp.experts.88.down_proj", "model.layers.1.mlp.experts.89.down_proj", "model.layers.1.mlp.experts.90.down_proj", "model.layers.1.mlp.experts.91.down_proj", "model.layers.1.mlp.experts.92.down_proj", "model.layers.1.mlp.experts.93.down_proj", "model.layers.1.mlp.experts.94.down_proj", "model.layers.1.mlp.experts.95.down_proj", "model.layers.1.mlp.experts.96.down_proj", "model.layers.1.mlp.experts.97.down_proj", "model.layers.1.mlp.experts.98.down_proj", "model.layers.1.mlp.experts.99.down_proj", "model.layers.1.mlp.experts.100.down_proj", "model.layers.1.mlp.experts.101.down_proj", "model.layers.1.mlp.experts.102.down_proj", "model.layers.1.mlp.experts.103.down_proj", "model.layers.1.mlp.experts.104.down_proj", "model.layers.1.mlp.experts.105.down_proj", "model.layers.1.mlp.experts.106.down_proj", "model.layers.1.mlp.experts.107.down_proj", "model.layers.1.mlp.experts.108.down_proj", "model.layers.1.mlp.experts.109.down_proj", "model.layers.1.mlp.experts.110.down_proj", "model.layers.1.mlp.experts.111.down_proj", "model.layers.1.mlp.experts.112.down_proj", "model.layers.1.mlp.experts.113.down_proj", "model.layers.1.mlp.experts.114.down_proj", "model.layers.1.mlp.experts.115.down_proj", "model.layers.1.mlp.experts.116.down_proj", "model.layers.1.mlp.experts.117.down_proj", "model.layers.1.mlp.experts.118.down_proj", "model.layers.1.mlp.experts.119.down_proj", "model.layers.1.mlp.experts.120.down_proj", "model.layers.1.mlp.experts.121.down_proj", "model.layers.1.mlp.experts.122.down_proj", "model.layers.1.mlp.experts.123.down_proj", "model.layers.1.mlp.experts.124.down_proj", "model.layers.1.mlp.experts.125.down_proj", "model.layers.1.mlp.experts.126.down_proj", "model.layers.1.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.012276190612465127, "dbits": 2214592512 } ] }, { "idx": 5, "layers": [ "model.layers.2.self_attn.q_proj", "model.layers.2.self_attn.k_proj", "model.layers.2.self_attn.v_proj", "model.layers.2.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0016038351692259367, "dbits": 109051904 } ] }, { "idx": 6, "layers": [ "model.layers.2.mlp.shared_experts.gate_proj", "model.layers.2.mlp.shared_experts.up_proj", "model.layers.2.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.05919955638237298, "dbits": 17301504 } ] }, { "idx": 7, "layers": [ "model.layers.2.mlp.experts.0.gate_proj", "model.layers.2.mlp.experts.1.gate_proj", "model.layers.2.mlp.experts.2.gate_proj", "model.layers.2.mlp.experts.3.gate_proj", "model.layers.2.mlp.experts.4.gate_proj", "model.layers.2.mlp.experts.5.gate_proj", "model.layers.2.mlp.experts.6.gate_proj", "model.layers.2.mlp.experts.7.gate_proj", "model.layers.2.mlp.experts.8.gate_proj", "model.layers.2.mlp.experts.9.gate_proj", "model.layers.2.mlp.experts.10.gate_proj", "model.layers.2.mlp.experts.11.gate_proj", "model.layers.2.mlp.experts.12.gate_proj", "model.layers.2.mlp.experts.13.gate_proj", "model.layers.2.mlp.experts.14.gate_proj", "model.layers.2.mlp.experts.15.gate_proj", "model.layers.2.mlp.experts.16.gate_proj", "model.layers.2.mlp.experts.17.gate_proj", "model.layers.2.mlp.experts.18.gate_proj", "model.layers.2.mlp.experts.19.gate_proj", "model.layers.2.mlp.experts.20.gate_proj", "model.layers.2.mlp.experts.21.gate_proj", "model.layers.2.mlp.experts.22.gate_proj", "model.layers.2.mlp.experts.23.gate_proj", "model.layers.2.mlp.experts.24.gate_proj", "model.layers.2.mlp.experts.25.gate_proj", "model.layers.2.mlp.experts.26.gate_proj", "model.layers.2.mlp.experts.27.gate_proj", "model.layers.2.mlp.experts.28.gate_proj", "model.layers.2.mlp.experts.29.gate_proj", "model.layers.2.mlp.experts.30.gate_proj", "model.layers.2.mlp.experts.31.gate_proj", "model.layers.2.mlp.experts.32.gate_proj", "model.layers.2.mlp.experts.33.gate_proj", "model.layers.2.mlp.experts.34.gate_proj", "model.layers.2.mlp.experts.35.gate_proj", "model.layers.2.mlp.experts.36.gate_proj", "model.layers.2.mlp.experts.37.gate_proj", "model.layers.2.mlp.experts.38.gate_proj", "model.layers.2.mlp.experts.39.gate_proj", "model.layers.2.mlp.experts.40.gate_proj", "model.layers.2.mlp.experts.41.gate_proj", "model.layers.2.mlp.experts.42.gate_proj", "model.layers.2.mlp.experts.43.gate_proj", "model.layers.2.mlp.experts.44.gate_proj", "model.layers.2.mlp.experts.45.gate_proj", "model.layers.2.mlp.experts.46.gate_proj", "model.layers.2.mlp.experts.47.gate_proj", "model.layers.2.mlp.experts.48.gate_proj", "model.layers.2.mlp.experts.49.gate_proj", "model.layers.2.mlp.experts.50.gate_proj", "model.layers.2.mlp.experts.51.gate_proj", "model.layers.2.mlp.experts.52.gate_proj", "model.layers.2.mlp.experts.53.gate_proj", "model.layers.2.mlp.experts.54.gate_proj", "model.layers.2.mlp.experts.55.gate_proj", "model.layers.2.mlp.experts.56.gate_proj", "model.layers.2.mlp.experts.57.gate_proj", "model.layers.2.mlp.experts.58.gate_proj", "model.layers.2.mlp.experts.59.gate_proj", "model.layers.2.mlp.experts.60.gate_proj", "model.layers.2.mlp.experts.61.gate_proj", "model.layers.2.mlp.experts.62.gate_proj", "model.layers.2.mlp.experts.63.gate_proj", "model.layers.2.mlp.experts.64.gate_proj", "model.layers.2.mlp.experts.65.gate_proj", "model.layers.2.mlp.experts.66.gate_proj", "model.layers.2.mlp.experts.67.gate_proj", "model.layers.2.mlp.experts.68.gate_proj", "model.layers.2.mlp.experts.69.gate_proj", "model.layers.2.mlp.experts.70.gate_proj", "model.layers.2.mlp.experts.71.gate_proj", "model.layers.2.mlp.experts.72.gate_proj", "model.layers.2.mlp.experts.73.gate_proj", "model.layers.2.mlp.experts.74.gate_proj", "model.layers.2.mlp.experts.75.gate_proj", "model.layers.2.mlp.experts.76.gate_proj", "model.layers.2.mlp.experts.77.gate_proj", "model.layers.2.mlp.experts.78.gate_proj", "model.layers.2.mlp.experts.79.gate_proj", "model.layers.2.mlp.experts.80.gate_proj", "model.layers.2.mlp.experts.81.gate_proj", "model.layers.2.mlp.experts.82.gate_proj", "model.layers.2.mlp.experts.83.gate_proj", "model.layers.2.mlp.experts.84.gate_proj", "model.layers.2.mlp.experts.85.gate_proj", "model.layers.2.mlp.experts.86.gate_proj", "model.layers.2.mlp.experts.87.gate_proj", "model.layers.2.mlp.experts.88.gate_proj", "model.layers.2.mlp.experts.89.gate_proj", "model.layers.2.mlp.experts.90.gate_proj", "model.layers.2.mlp.experts.91.gate_proj", "model.layers.2.mlp.experts.92.gate_proj", "model.layers.2.mlp.experts.93.gate_proj", "model.layers.2.mlp.experts.94.gate_proj", "model.layers.2.mlp.experts.95.gate_proj", "model.layers.2.mlp.experts.96.gate_proj", "model.layers.2.mlp.experts.97.gate_proj", "model.layers.2.mlp.experts.98.gate_proj", "model.layers.2.mlp.experts.99.gate_proj", "model.layers.2.mlp.experts.100.gate_proj", "model.layers.2.mlp.experts.101.gate_proj", "model.layers.2.mlp.experts.102.gate_proj", "model.layers.2.mlp.experts.103.gate_proj", "model.layers.2.mlp.experts.104.gate_proj", "model.layers.2.mlp.experts.105.gate_proj", "model.layers.2.mlp.experts.106.gate_proj", "model.layers.2.mlp.experts.107.gate_proj", "model.layers.2.mlp.experts.108.gate_proj", "model.layers.2.mlp.experts.109.gate_proj", "model.layers.2.mlp.experts.110.gate_proj", "model.layers.2.mlp.experts.111.gate_proj", "model.layers.2.mlp.experts.112.gate_proj", "model.layers.2.mlp.experts.113.gate_proj", "model.layers.2.mlp.experts.114.gate_proj", "model.layers.2.mlp.experts.115.gate_proj", "model.layers.2.mlp.experts.116.gate_proj", "model.layers.2.mlp.experts.117.gate_proj", "model.layers.2.mlp.experts.118.gate_proj", "model.layers.2.mlp.experts.119.gate_proj", "model.layers.2.mlp.experts.120.gate_proj", "model.layers.2.mlp.experts.121.gate_proj", "model.layers.2.mlp.experts.122.gate_proj", "model.layers.2.mlp.experts.123.gate_proj", "model.layers.2.mlp.experts.124.gate_proj", "model.layers.2.mlp.experts.125.gate_proj", "model.layers.2.mlp.experts.126.gate_proj", "model.layers.2.mlp.experts.127.gate_proj", "model.layers.2.mlp.experts.0.up_proj", "model.layers.2.mlp.experts.1.up_proj", "model.layers.2.mlp.experts.2.up_proj", "model.layers.2.mlp.experts.3.up_proj", "model.layers.2.mlp.experts.4.up_proj", "model.layers.2.mlp.experts.5.up_proj", "model.layers.2.mlp.experts.6.up_proj", "model.layers.2.mlp.experts.7.up_proj", "model.layers.2.mlp.experts.8.up_proj", "model.layers.2.mlp.experts.9.up_proj", "model.layers.2.mlp.experts.10.up_proj", "model.layers.2.mlp.experts.11.up_proj", "model.layers.2.mlp.experts.12.up_proj", "model.layers.2.mlp.experts.13.up_proj", "model.layers.2.mlp.experts.14.up_proj", "model.layers.2.mlp.experts.15.up_proj", "model.layers.2.mlp.experts.16.up_proj", "model.layers.2.mlp.experts.17.up_proj", "model.layers.2.mlp.experts.18.up_proj", "model.layers.2.mlp.experts.19.up_proj", "model.layers.2.mlp.experts.20.up_proj", "model.layers.2.mlp.experts.21.up_proj", "model.layers.2.mlp.experts.22.up_proj", "model.layers.2.mlp.experts.23.up_proj", "model.layers.2.mlp.experts.24.up_proj", "model.layers.2.mlp.experts.25.up_proj", "model.layers.2.mlp.experts.26.up_proj", "model.layers.2.mlp.experts.27.up_proj", "model.layers.2.mlp.experts.28.up_proj", "model.layers.2.mlp.experts.29.up_proj", "model.layers.2.mlp.experts.30.up_proj", "model.layers.2.mlp.experts.31.up_proj", "model.layers.2.mlp.experts.32.up_proj", "model.layers.2.mlp.experts.33.up_proj", "model.layers.2.mlp.experts.34.up_proj", "model.layers.2.mlp.experts.35.up_proj", "model.layers.2.mlp.experts.36.up_proj", "model.layers.2.mlp.experts.37.up_proj", "model.layers.2.mlp.experts.38.up_proj", "model.layers.2.mlp.experts.39.up_proj", "model.layers.2.mlp.experts.40.up_proj", "model.layers.2.mlp.experts.41.up_proj", "model.layers.2.mlp.experts.42.up_proj", "model.layers.2.mlp.experts.43.up_proj", "model.layers.2.mlp.experts.44.up_proj", "model.layers.2.mlp.experts.45.up_proj", "model.layers.2.mlp.experts.46.up_proj", "model.layers.2.mlp.experts.47.up_proj", "model.layers.2.mlp.experts.48.up_proj", "model.layers.2.mlp.experts.49.up_proj", "model.layers.2.mlp.experts.50.up_proj", "model.layers.2.mlp.experts.51.up_proj", "model.layers.2.mlp.experts.52.up_proj", "model.layers.2.mlp.experts.53.up_proj", "model.layers.2.mlp.experts.54.up_proj", "model.layers.2.mlp.experts.55.up_proj", "model.layers.2.mlp.experts.56.up_proj", "model.layers.2.mlp.experts.57.up_proj", "model.layers.2.mlp.experts.58.up_proj", "model.layers.2.mlp.experts.59.up_proj", "model.layers.2.mlp.experts.60.up_proj", "model.layers.2.mlp.experts.61.up_proj", "model.layers.2.mlp.experts.62.up_proj", "model.layers.2.mlp.experts.63.up_proj", "model.layers.2.mlp.experts.64.up_proj", "model.layers.2.mlp.experts.65.up_proj", "model.layers.2.mlp.experts.66.up_proj", "model.layers.2.mlp.experts.67.up_proj", "model.layers.2.mlp.experts.68.up_proj", "model.layers.2.mlp.experts.69.up_proj", "model.layers.2.mlp.experts.70.up_proj", "model.layers.2.mlp.experts.71.up_proj", "model.layers.2.mlp.experts.72.up_proj", "model.layers.2.mlp.experts.73.up_proj", "model.layers.2.mlp.experts.74.up_proj", "model.layers.2.mlp.experts.75.up_proj", "model.layers.2.mlp.experts.76.up_proj", "model.layers.2.mlp.experts.77.up_proj", "model.layers.2.mlp.experts.78.up_proj", "model.layers.2.mlp.experts.79.up_proj", "model.layers.2.mlp.experts.80.up_proj", "model.layers.2.mlp.experts.81.up_proj", "model.layers.2.mlp.experts.82.up_proj", "model.layers.2.mlp.experts.83.up_proj", "model.layers.2.mlp.experts.84.up_proj", "model.layers.2.mlp.experts.85.up_proj", "model.layers.2.mlp.experts.86.up_proj", "model.layers.2.mlp.experts.87.up_proj", "model.layers.2.mlp.experts.88.up_proj", "model.layers.2.mlp.experts.89.up_proj", "model.layers.2.mlp.experts.90.up_proj", "model.layers.2.mlp.experts.91.up_proj", "model.layers.2.mlp.experts.92.up_proj", "model.layers.2.mlp.experts.93.up_proj", "model.layers.2.mlp.experts.94.up_proj", "model.layers.2.mlp.experts.95.up_proj", "model.layers.2.mlp.experts.96.up_proj", "model.layers.2.mlp.experts.97.up_proj", "model.layers.2.mlp.experts.98.up_proj", "model.layers.2.mlp.experts.99.up_proj", "model.layers.2.mlp.experts.100.up_proj", "model.layers.2.mlp.experts.101.up_proj", "model.layers.2.mlp.experts.102.up_proj", "model.layers.2.mlp.experts.103.up_proj", "model.layers.2.mlp.experts.104.up_proj", "model.layers.2.mlp.experts.105.up_proj", "model.layers.2.mlp.experts.106.up_proj", "model.layers.2.mlp.experts.107.up_proj", "model.layers.2.mlp.experts.108.up_proj", "model.layers.2.mlp.experts.109.up_proj", "model.layers.2.mlp.experts.110.up_proj", "model.layers.2.mlp.experts.111.up_proj", "model.layers.2.mlp.experts.112.up_proj", "model.layers.2.mlp.experts.113.up_proj", "model.layers.2.mlp.experts.114.up_proj", "model.layers.2.mlp.experts.115.up_proj", "model.layers.2.mlp.experts.116.up_proj", "model.layers.2.mlp.experts.117.up_proj", "model.layers.2.mlp.experts.118.up_proj", "model.layers.2.mlp.experts.119.up_proj", "model.layers.2.mlp.experts.120.up_proj", "model.layers.2.mlp.experts.121.up_proj", "model.layers.2.mlp.experts.122.up_proj", "model.layers.2.mlp.experts.123.up_proj", "model.layers.2.mlp.experts.124.up_proj", "model.layers.2.mlp.experts.125.up_proj", "model.layers.2.mlp.experts.126.up_proj", "model.layers.2.mlp.experts.127.up_proj", "model.layers.2.mlp.experts.0.down_proj", "model.layers.2.mlp.experts.1.down_proj", "model.layers.2.mlp.experts.2.down_proj", "model.layers.2.mlp.experts.3.down_proj", "model.layers.2.mlp.experts.4.down_proj", "model.layers.2.mlp.experts.5.down_proj", "model.layers.2.mlp.experts.6.down_proj", "model.layers.2.mlp.experts.7.down_proj", "model.layers.2.mlp.experts.8.down_proj", "model.layers.2.mlp.experts.9.down_proj", "model.layers.2.mlp.experts.10.down_proj", "model.layers.2.mlp.experts.11.down_proj", "model.layers.2.mlp.experts.12.down_proj", "model.layers.2.mlp.experts.13.down_proj", "model.layers.2.mlp.experts.14.down_proj", "model.layers.2.mlp.experts.15.down_proj", "model.layers.2.mlp.experts.16.down_proj", "model.layers.2.mlp.experts.17.down_proj", "model.layers.2.mlp.experts.18.down_proj", "model.layers.2.mlp.experts.19.down_proj", "model.layers.2.mlp.experts.20.down_proj", "model.layers.2.mlp.experts.21.down_proj", "model.layers.2.mlp.experts.22.down_proj", "model.layers.2.mlp.experts.23.down_proj", "model.layers.2.mlp.experts.24.down_proj", "model.layers.2.mlp.experts.25.down_proj", "model.layers.2.mlp.experts.26.down_proj", "model.layers.2.mlp.experts.27.down_proj", "model.layers.2.mlp.experts.28.down_proj", "model.layers.2.mlp.experts.29.down_proj", "model.layers.2.mlp.experts.30.down_proj", "model.layers.2.mlp.experts.31.down_proj", "model.layers.2.mlp.experts.32.down_proj", "model.layers.2.mlp.experts.33.down_proj", "model.layers.2.mlp.experts.34.down_proj", "model.layers.2.mlp.experts.35.down_proj", "model.layers.2.mlp.experts.36.down_proj", "model.layers.2.mlp.experts.37.down_proj", "model.layers.2.mlp.experts.38.down_proj", "model.layers.2.mlp.experts.39.down_proj", "model.layers.2.mlp.experts.40.down_proj", "model.layers.2.mlp.experts.41.down_proj", "model.layers.2.mlp.experts.42.down_proj", "model.layers.2.mlp.experts.43.down_proj", "model.layers.2.mlp.experts.44.down_proj", "model.layers.2.mlp.experts.45.down_proj", "model.layers.2.mlp.experts.46.down_proj", "model.layers.2.mlp.experts.47.down_proj", "model.layers.2.mlp.experts.48.down_proj", "model.layers.2.mlp.experts.49.down_proj", "model.layers.2.mlp.experts.50.down_proj", "model.layers.2.mlp.experts.51.down_proj", "model.layers.2.mlp.experts.52.down_proj", "model.layers.2.mlp.experts.53.down_proj", "model.layers.2.mlp.experts.54.down_proj", "model.layers.2.mlp.experts.55.down_proj", "model.layers.2.mlp.experts.56.down_proj", "model.layers.2.mlp.experts.57.down_proj", "model.layers.2.mlp.experts.58.down_proj", "model.layers.2.mlp.experts.59.down_proj", "model.layers.2.mlp.experts.60.down_proj", "model.layers.2.mlp.experts.61.down_proj", "model.layers.2.mlp.experts.62.down_proj", "model.layers.2.mlp.experts.63.down_proj", "model.layers.2.mlp.experts.64.down_proj", "model.layers.2.mlp.experts.65.down_proj", "model.layers.2.mlp.experts.66.down_proj", "model.layers.2.mlp.experts.67.down_proj", "model.layers.2.mlp.experts.68.down_proj", "model.layers.2.mlp.experts.69.down_proj", "model.layers.2.mlp.experts.70.down_proj", "model.layers.2.mlp.experts.71.down_proj", "model.layers.2.mlp.experts.72.down_proj", "model.layers.2.mlp.experts.73.down_proj", "model.layers.2.mlp.experts.74.down_proj", "model.layers.2.mlp.experts.75.down_proj", "model.layers.2.mlp.experts.76.down_proj", "model.layers.2.mlp.experts.77.down_proj", "model.layers.2.mlp.experts.78.down_proj", "model.layers.2.mlp.experts.79.down_proj", "model.layers.2.mlp.experts.80.down_proj", "model.layers.2.mlp.experts.81.down_proj", "model.layers.2.mlp.experts.82.down_proj", "model.layers.2.mlp.experts.83.down_proj", "model.layers.2.mlp.experts.84.down_proj", "model.layers.2.mlp.experts.85.down_proj", "model.layers.2.mlp.experts.86.down_proj", "model.layers.2.mlp.experts.87.down_proj", "model.layers.2.mlp.experts.88.down_proj", "model.layers.2.mlp.experts.89.down_proj", "model.layers.2.mlp.experts.90.down_proj", "model.layers.2.mlp.experts.91.down_proj", "model.layers.2.mlp.experts.92.down_proj", "model.layers.2.mlp.experts.93.down_proj", "model.layers.2.mlp.experts.94.down_proj", "model.layers.2.mlp.experts.95.down_proj", "model.layers.2.mlp.experts.96.down_proj", "model.layers.2.mlp.experts.97.down_proj", "model.layers.2.mlp.experts.98.down_proj", "model.layers.2.mlp.experts.99.down_proj", "model.layers.2.mlp.experts.100.down_proj", "model.layers.2.mlp.experts.101.down_proj", "model.layers.2.mlp.experts.102.down_proj", "model.layers.2.mlp.experts.103.down_proj", "model.layers.2.mlp.experts.104.down_proj", "model.layers.2.mlp.experts.105.down_proj", "model.layers.2.mlp.experts.106.down_proj", "model.layers.2.mlp.experts.107.down_proj", "model.layers.2.mlp.experts.108.down_proj", "model.layers.2.mlp.experts.109.down_proj", "model.layers.2.mlp.experts.110.down_proj", "model.layers.2.mlp.experts.111.down_proj", "model.layers.2.mlp.experts.112.down_proj", "model.layers.2.mlp.experts.113.down_proj", "model.layers.2.mlp.experts.114.down_proj", "model.layers.2.mlp.experts.115.down_proj", "model.layers.2.mlp.experts.116.down_proj", "model.layers.2.mlp.experts.117.down_proj", "model.layers.2.mlp.experts.118.down_proj", "model.layers.2.mlp.experts.119.down_proj", "model.layers.2.mlp.experts.120.down_proj", "model.layers.2.mlp.experts.121.down_proj", "model.layers.2.mlp.experts.122.down_proj", "model.layers.2.mlp.experts.123.down_proj", "model.layers.2.mlp.experts.124.down_proj", "model.layers.2.mlp.experts.125.down_proj", "model.layers.2.mlp.experts.126.down_proj", "model.layers.2.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.007077699992805719, "dbits": 2214592512 } ] }, { "idx": 8, "layers": [ "model.layers.3.self_attn.q_proj", "model.layers.3.self_attn.k_proj", "model.layers.3.self_attn.v_proj", "model.layers.3.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0036273919977247715, "dbits": 109051904 } ] }, { "idx": 9, "layers": [ "model.layers.3.mlp.shared_experts.gate_proj", "model.layers.3.mlp.shared_experts.up_proj", "model.layers.3.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.004769983794540161, "dbits": 17301504 } ] }, { "idx": 10, "layers": [ "model.layers.3.mlp.experts.0.gate_proj", "model.layers.3.mlp.experts.1.gate_proj", "model.layers.3.mlp.experts.2.gate_proj", "model.layers.3.mlp.experts.3.gate_proj", "model.layers.3.mlp.experts.4.gate_proj", "model.layers.3.mlp.experts.5.gate_proj", "model.layers.3.mlp.experts.6.gate_proj", "model.layers.3.mlp.experts.7.gate_proj", "model.layers.3.mlp.experts.8.gate_proj", "model.layers.3.mlp.experts.9.gate_proj", "model.layers.3.mlp.experts.10.gate_proj", "model.layers.3.mlp.experts.11.gate_proj", "model.layers.3.mlp.experts.12.gate_proj", "model.layers.3.mlp.experts.13.gate_proj", "model.layers.3.mlp.experts.14.gate_proj", "model.layers.3.mlp.experts.15.gate_proj", "model.layers.3.mlp.experts.16.gate_proj", "model.layers.3.mlp.experts.17.gate_proj", "model.layers.3.mlp.experts.18.gate_proj", "model.layers.3.mlp.experts.19.gate_proj", "model.layers.3.mlp.experts.20.gate_proj", "model.layers.3.mlp.experts.21.gate_proj", "model.layers.3.mlp.experts.22.gate_proj", "model.layers.3.mlp.experts.23.gate_proj", "model.layers.3.mlp.experts.24.gate_proj", "model.layers.3.mlp.experts.25.gate_proj", "model.layers.3.mlp.experts.26.gate_proj", "model.layers.3.mlp.experts.27.gate_proj", "model.layers.3.mlp.experts.28.gate_proj", "model.layers.3.mlp.experts.29.gate_proj", "model.layers.3.mlp.experts.30.gate_proj", "model.layers.3.mlp.experts.31.gate_proj", "model.layers.3.mlp.experts.32.gate_proj", "model.layers.3.mlp.experts.33.gate_proj", "model.layers.3.mlp.experts.34.gate_proj", "model.layers.3.mlp.experts.35.gate_proj", "model.layers.3.mlp.experts.36.gate_proj", "model.layers.3.mlp.experts.37.gate_proj", "model.layers.3.mlp.experts.38.gate_proj", "model.layers.3.mlp.experts.39.gate_proj", "model.layers.3.mlp.experts.40.gate_proj", "model.layers.3.mlp.experts.41.gate_proj", "model.layers.3.mlp.experts.42.gate_proj", "model.layers.3.mlp.experts.43.gate_proj", "model.layers.3.mlp.experts.44.gate_proj", "model.layers.3.mlp.experts.45.gate_proj", "model.layers.3.mlp.experts.46.gate_proj", "model.layers.3.mlp.experts.47.gate_proj", "model.layers.3.mlp.experts.48.gate_proj", "model.layers.3.mlp.experts.49.gate_proj", "model.layers.3.mlp.experts.50.gate_proj", "model.layers.3.mlp.experts.51.gate_proj", "model.layers.3.mlp.experts.52.gate_proj", "model.layers.3.mlp.experts.53.gate_proj", "model.layers.3.mlp.experts.54.gate_proj", "model.layers.3.mlp.experts.55.gate_proj", "model.layers.3.mlp.experts.56.gate_proj", "model.layers.3.mlp.experts.57.gate_proj", "model.layers.3.mlp.experts.58.gate_proj", "model.layers.3.mlp.experts.59.gate_proj", "model.layers.3.mlp.experts.60.gate_proj", "model.layers.3.mlp.experts.61.gate_proj", "model.layers.3.mlp.experts.62.gate_proj", "model.layers.3.mlp.experts.63.gate_proj", "model.layers.3.mlp.experts.64.gate_proj", "model.layers.3.mlp.experts.65.gate_proj", "model.layers.3.mlp.experts.66.gate_proj", "model.layers.3.mlp.experts.67.gate_proj", "model.layers.3.mlp.experts.68.gate_proj", "model.layers.3.mlp.experts.69.gate_proj", "model.layers.3.mlp.experts.70.gate_proj", "model.layers.3.mlp.experts.71.gate_proj", "model.layers.3.mlp.experts.72.gate_proj", "model.layers.3.mlp.experts.73.gate_proj", "model.layers.3.mlp.experts.74.gate_proj", "model.layers.3.mlp.experts.75.gate_proj", "model.layers.3.mlp.experts.76.gate_proj", "model.layers.3.mlp.experts.77.gate_proj", "model.layers.3.mlp.experts.78.gate_proj", "model.layers.3.mlp.experts.79.gate_proj", "model.layers.3.mlp.experts.80.gate_proj", "model.layers.3.mlp.experts.81.gate_proj", "model.layers.3.mlp.experts.82.gate_proj", "model.layers.3.mlp.experts.83.gate_proj", "model.layers.3.mlp.experts.84.gate_proj", "model.layers.3.mlp.experts.85.gate_proj", "model.layers.3.mlp.experts.86.gate_proj", "model.layers.3.mlp.experts.87.gate_proj", "model.layers.3.mlp.experts.88.gate_proj", "model.layers.3.mlp.experts.89.gate_proj", "model.layers.3.mlp.experts.90.gate_proj", "model.layers.3.mlp.experts.91.gate_proj", "model.layers.3.mlp.experts.92.gate_proj", "model.layers.3.mlp.experts.93.gate_proj", "model.layers.3.mlp.experts.94.gate_proj", "model.layers.3.mlp.experts.95.gate_proj", "model.layers.3.mlp.experts.96.gate_proj", "model.layers.3.mlp.experts.97.gate_proj", "model.layers.3.mlp.experts.98.gate_proj", "model.layers.3.mlp.experts.99.gate_proj", "model.layers.3.mlp.experts.100.gate_proj", "model.layers.3.mlp.experts.101.gate_proj", "model.layers.3.mlp.experts.102.gate_proj", "model.layers.3.mlp.experts.103.gate_proj", "model.layers.3.mlp.experts.104.gate_proj", "model.layers.3.mlp.experts.105.gate_proj", "model.layers.3.mlp.experts.106.gate_proj", "model.layers.3.mlp.experts.107.gate_proj", "model.layers.3.mlp.experts.108.gate_proj", "model.layers.3.mlp.experts.109.gate_proj", "model.layers.3.mlp.experts.110.gate_proj", "model.layers.3.mlp.experts.111.gate_proj", "model.layers.3.mlp.experts.112.gate_proj", "model.layers.3.mlp.experts.113.gate_proj", "model.layers.3.mlp.experts.114.gate_proj", "model.layers.3.mlp.experts.115.gate_proj", "model.layers.3.mlp.experts.116.gate_proj", "model.layers.3.mlp.experts.117.gate_proj", "model.layers.3.mlp.experts.118.gate_proj", "model.layers.3.mlp.experts.119.gate_proj", "model.layers.3.mlp.experts.120.gate_proj", "model.layers.3.mlp.experts.121.gate_proj", "model.layers.3.mlp.experts.122.gate_proj", "model.layers.3.mlp.experts.123.gate_proj", "model.layers.3.mlp.experts.124.gate_proj", "model.layers.3.mlp.experts.125.gate_proj", "model.layers.3.mlp.experts.126.gate_proj", "model.layers.3.mlp.experts.127.gate_proj", "model.layers.3.mlp.experts.0.up_proj", "model.layers.3.mlp.experts.1.up_proj", "model.layers.3.mlp.experts.2.up_proj", "model.layers.3.mlp.experts.3.up_proj", "model.layers.3.mlp.experts.4.up_proj", "model.layers.3.mlp.experts.5.up_proj", "model.layers.3.mlp.experts.6.up_proj", "model.layers.3.mlp.experts.7.up_proj", "model.layers.3.mlp.experts.8.up_proj", "model.layers.3.mlp.experts.9.up_proj", "model.layers.3.mlp.experts.10.up_proj", "model.layers.3.mlp.experts.11.up_proj", "model.layers.3.mlp.experts.12.up_proj", "model.layers.3.mlp.experts.13.up_proj", "model.layers.3.mlp.experts.14.up_proj", "model.layers.3.mlp.experts.15.up_proj", "model.layers.3.mlp.experts.16.up_proj", "model.layers.3.mlp.experts.17.up_proj", "model.layers.3.mlp.experts.18.up_proj", "model.layers.3.mlp.experts.19.up_proj", "model.layers.3.mlp.experts.20.up_proj", "model.layers.3.mlp.experts.21.up_proj", "model.layers.3.mlp.experts.22.up_proj", "model.layers.3.mlp.experts.23.up_proj", "model.layers.3.mlp.experts.24.up_proj", "model.layers.3.mlp.experts.25.up_proj", "model.layers.3.mlp.experts.26.up_proj", "model.layers.3.mlp.experts.27.up_proj", "model.layers.3.mlp.experts.28.up_proj", "model.layers.3.mlp.experts.29.up_proj", "model.layers.3.mlp.experts.30.up_proj", "model.layers.3.mlp.experts.31.up_proj", "model.layers.3.mlp.experts.32.up_proj", "model.layers.3.mlp.experts.33.up_proj", "model.layers.3.mlp.experts.34.up_proj", "model.layers.3.mlp.experts.35.up_proj", "model.layers.3.mlp.experts.36.up_proj", "model.layers.3.mlp.experts.37.up_proj", "model.layers.3.mlp.experts.38.up_proj", "model.layers.3.mlp.experts.39.up_proj", "model.layers.3.mlp.experts.40.up_proj", "model.layers.3.mlp.experts.41.up_proj", "model.layers.3.mlp.experts.42.up_proj", "model.layers.3.mlp.experts.43.up_proj", "model.layers.3.mlp.experts.44.up_proj", "model.layers.3.mlp.experts.45.up_proj", "model.layers.3.mlp.experts.46.up_proj", "model.layers.3.mlp.experts.47.up_proj", "model.layers.3.mlp.experts.48.up_proj", "model.layers.3.mlp.experts.49.up_proj", "model.layers.3.mlp.experts.50.up_proj", "model.layers.3.mlp.experts.51.up_proj", "model.layers.3.mlp.experts.52.up_proj", "model.layers.3.mlp.experts.53.up_proj", "model.layers.3.mlp.experts.54.up_proj", "model.layers.3.mlp.experts.55.up_proj", "model.layers.3.mlp.experts.56.up_proj", "model.layers.3.mlp.experts.57.up_proj", "model.layers.3.mlp.experts.58.up_proj", "model.layers.3.mlp.experts.59.up_proj", "model.layers.3.mlp.experts.60.up_proj", "model.layers.3.mlp.experts.61.up_proj", "model.layers.3.mlp.experts.62.up_proj", "model.layers.3.mlp.experts.63.up_proj", "model.layers.3.mlp.experts.64.up_proj", "model.layers.3.mlp.experts.65.up_proj", "model.layers.3.mlp.experts.66.up_proj", "model.layers.3.mlp.experts.67.up_proj", "model.layers.3.mlp.experts.68.up_proj", "model.layers.3.mlp.experts.69.up_proj", "model.layers.3.mlp.experts.70.up_proj", "model.layers.3.mlp.experts.71.up_proj", "model.layers.3.mlp.experts.72.up_proj", "model.layers.3.mlp.experts.73.up_proj", "model.layers.3.mlp.experts.74.up_proj", "model.layers.3.mlp.experts.75.up_proj", "model.layers.3.mlp.experts.76.up_proj", "model.layers.3.mlp.experts.77.up_proj", "model.layers.3.mlp.experts.78.up_proj", "model.layers.3.mlp.experts.79.up_proj", "model.layers.3.mlp.experts.80.up_proj", "model.layers.3.mlp.experts.81.up_proj", "model.layers.3.mlp.experts.82.up_proj", "model.layers.3.mlp.experts.83.up_proj", "model.layers.3.mlp.experts.84.up_proj", "model.layers.3.mlp.experts.85.up_proj", "model.layers.3.mlp.experts.86.up_proj", "model.layers.3.mlp.experts.87.up_proj", "model.layers.3.mlp.experts.88.up_proj", "model.layers.3.mlp.experts.89.up_proj", "model.layers.3.mlp.experts.90.up_proj", "model.layers.3.mlp.experts.91.up_proj", "model.layers.3.mlp.experts.92.up_proj", "model.layers.3.mlp.experts.93.up_proj", "model.layers.3.mlp.experts.94.up_proj", "model.layers.3.mlp.experts.95.up_proj", "model.layers.3.mlp.experts.96.up_proj", "model.layers.3.mlp.experts.97.up_proj", "model.layers.3.mlp.experts.98.up_proj", "model.layers.3.mlp.experts.99.up_proj", "model.layers.3.mlp.experts.100.up_proj", "model.layers.3.mlp.experts.101.up_proj", "model.layers.3.mlp.experts.102.up_proj", "model.layers.3.mlp.experts.103.up_proj", "model.layers.3.mlp.experts.104.up_proj", "model.layers.3.mlp.experts.105.up_proj", "model.layers.3.mlp.experts.106.up_proj", "model.layers.3.mlp.experts.107.up_proj", "model.layers.3.mlp.experts.108.up_proj", "model.layers.3.mlp.experts.109.up_proj", "model.layers.3.mlp.experts.110.up_proj", "model.layers.3.mlp.experts.111.up_proj", "model.layers.3.mlp.experts.112.up_proj", "model.layers.3.mlp.experts.113.up_proj", "model.layers.3.mlp.experts.114.up_proj", "model.layers.3.mlp.experts.115.up_proj", "model.layers.3.mlp.experts.116.up_proj", "model.layers.3.mlp.experts.117.up_proj", "model.layers.3.mlp.experts.118.up_proj", "model.layers.3.mlp.experts.119.up_proj", "model.layers.3.mlp.experts.120.up_proj", "model.layers.3.mlp.experts.121.up_proj", "model.layers.3.mlp.experts.122.up_proj", "model.layers.3.mlp.experts.123.up_proj", "model.layers.3.mlp.experts.124.up_proj", "model.layers.3.mlp.experts.125.up_proj", "model.layers.3.mlp.experts.126.up_proj", "model.layers.3.mlp.experts.127.up_proj", "model.layers.3.mlp.experts.0.down_proj", "model.layers.3.mlp.experts.1.down_proj", "model.layers.3.mlp.experts.2.down_proj", "model.layers.3.mlp.experts.3.down_proj", "model.layers.3.mlp.experts.4.down_proj", "model.layers.3.mlp.experts.5.down_proj", "model.layers.3.mlp.experts.6.down_proj", "model.layers.3.mlp.experts.7.down_proj", "model.layers.3.mlp.experts.8.down_proj", "model.layers.3.mlp.experts.9.down_proj", "model.layers.3.mlp.experts.10.down_proj", "model.layers.3.mlp.experts.11.down_proj", "model.layers.3.mlp.experts.12.down_proj", "model.layers.3.mlp.experts.13.down_proj", "model.layers.3.mlp.experts.14.down_proj", "model.layers.3.mlp.experts.15.down_proj", "model.layers.3.mlp.experts.16.down_proj", "model.layers.3.mlp.experts.17.down_proj", "model.layers.3.mlp.experts.18.down_proj", "model.layers.3.mlp.experts.19.down_proj", "model.layers.3.mlp.experts.20.down_proj", "model.layers.3.mlp.experts.21.down_proj", "model.layers.3.mlp.experts.22.down_proj", "model.layers.3.mlp.experts.23.down_proj", "model.layers.3.mlp.experts.24.down_proj", "model.layers.3.mlp.experts.25.down_proj", "model.layers.3.mlp.experts.26.down_proj", "model.layers.3.mlp.experts.27.down_proj", "model.layers.3.mlp.experts.28.down_proj", "model.layers.3.mlp.experts.29.down_proj", "model.layers.3.mlp.experts.30.down_proj", "model.layers.3.mlp.experts.31.down_proj", "model.layers.3.mlp.experts.32.down_proj", "model.layers.3.mlp.experts.33.down_proj", "model.layers.3.mlp.experts.34.down_proj", "model.layers.3.mlp.experts.35.down_proj", "model.layers.3.mlp.experts.36.down_proj", "model.layers.3.mlp.experts.37.down_proj", "model.layers.3.mlp.experts.38.down_proj", "model.layers.3.mlp.experts.39.down_proj", "model.layers.3.mlp.experts.40.down_proj", "model.layers.3.mlp.experts.41.down_proj", "model.layers.3.mlp.experts.42.down_proj", "model.layers.3.mlp.experts.43.down_proj", "model.layers.3.mlp.experts.44.down_proj", "model.layers.3.mlp.experts.45.down_proj", "model.layers.3.mlp.experts.46.down_proj", "model.layers.3.mlp.experts.47.down_proj", "model.layers.3.mlp.experts.48.down_proj", "model.layers.3.mlp.experts.49.down_proj", "model.layers.3.mlp.experts.50.down_proj", "model.layers.3.mlp.experts.51.down_proj", "model.layers.3.mlp.experts.52.down_proj", "model.layers.3.mlp.experts.53.down_proj", "model.layers.3.mlp.experts.54.down_proj", "model.layers.3.mlp.experts.55.down_proj", "model.layers.3.mlp.experts.56.down_proj", "model.layers.3.mlp.experts.57.down_proj", "model.layers.3.mlp.experts.58.down_proj", "model.layers.3.mlp.experts.59.down_proj", "model.layers.3.mlp.experts.60.down_proj", "model.layers.3.mlp.experts.61.down_proj", "model.layers.3.mlp.experts.62.down_proj", "model.layers.3.mlp.experts.63.down_proj", "model.layers.3.mlp.experts.64.down_proj", "model.layers.3.mlp.experts.65.down_proj", "model.layers.3.mlp.experts.66.down_proj", "model.layers.3.mlp.experts.67.down_proj", "model.layers.3.mlp.experts.68.down_proj", "model.layers.3.mlp.experts.69.down_proj", "model.layers.3.mlp.experts.70.down_proj", "model.layers.3.mlp.experts.71.down_proj", "model.layers.3.mlp.experts.72.down_proj", "model.layers.3.mlp.experts.73.down_proj", "model.layers.3.mlp.experts.74.down_proj", "model.layers.3.mlp.experts.75.down_proj", "model.layers.3.mlp.experts.76.down_proj", "model.layers.3.mlp.experts.77.down_proj", "model.layers.3.mlp.experts.78.down_proj", "model.layers.3.mlp.experts.79.down_proj", "model.layers.3.mlp.experts.80.down_proj", "model.layers.3.mlp.experts.81.down_proj", "model.layers.3.mlp.experts.82.down_proj", "model.layers.3.mlp.experts.83.down_proj", "model.layers.3.mlp.experts.84.down_proj", "model.layers.3.mlp.experts.85.down_proj", "model.layers.3.mlp.experts.86.down_proj", "model.layers.3.mlp.experts.87.down_proj", "model.layers.3.mlp.experts.88.down_proj", "model.layers.3.mlp.experts.89.down_proj", "model.layers.3.mlp.experts.90.down_proj", "model.layers.3.mlp.experts.91.down_proj", "model.layers.3.mlp.experts.92.down_proj", "model.layers.3.mlp.experts.93.down_proj", "model.layers.3.mlp.experts.94.down_proj", "model.layers.3.mlp.experts.95.down_proj", "model.layers.3.mlp.experts.96.down_proj", "model.layers.3.mlp.experts.97.down_proj", "model.layers.3.mlp.experts.98.down_proj", "model.layers.3.mlp.experts.99.down_proj", "model.layers.3.mlp.experts.100.down_proj", "model.layers.3.mlp.experts.101.down_proj", "model.layers.3.mlp.experts.102.down_proj", "model.layers.3.mlp.experts.103.down_proj", "model.layers.3.mlp.experts.104.down_proj", "model.layers.3.mlp.experts.105.down_proj", "model.layers.3.mlp.experts.106.down_proj", "model.layers.3.mlp.experts.107.down_proj", "model.layers.3.mlp.experts.108.down_proj", "model.layers.3.mlp.experts.109.down_proj", "model.layers.3.mlp.experts.110.down_proj", "model.layers.3.mlp.experts.111.down_proj", "model.layers.3.mlp.experts.112.down_proj", "model.layers.3.mlp.experts.113.down_proj", "model.layers.3.mlp.experts.114.down_proj", "model.layers.3.mlp.experts.115.down_proj", "model.layers.3.mlp.experts.116.down_proj", "model.layers.3.mlp.experts.117.down_proj", "model.layers.3.mlp.experts.118.down_proj", "model.layers.3.mlp.experts.119.down_proj", "model.layers.3.mlp.experts.120.down_proj", "model.layers.3.mlp.experts.121.down_proj", "model.layers.3.mlp.experts.122.down_proj", "model.layers.3.mlp.experts.123.down_proj", "model.layers.3.mlp.experts.124.down_proj", "model.layers.3.mlp.experts.125.down_proj", "model.layers.3.mlp.experts.126.down_proj", "model.layers.3.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.011354844272136677, "dbits": 2214592512 } ] }, { "idx": 11, "layers": [ "model.layers.4.self_attn.q_proj", "model.layers.4.self_attn.k_proj", "model.layers.4.self_attn.v_proj", "model.layers.4.self_attn.o_proj" ], "candidates": [ { "dkld": -0.003989484906196578, "dbits": 109051904 } ] }, { "idx": 12, "layers": [ "model.layers.4.mlp.shared_experts.gate_proj", "model.layers.4.mlp.shared_experts.up_proj", "model.layers.4.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.003063128050416697, "dbits": 17301504 } ] }, { "idx": 13, "layers": [ "model.layers.4.mlp.experts.0.gate_proj", "model.layers.4.mlp.experts.1.gate_proj", "model.layers.4.mlp.experts.2.gate_proj", "model.layers.4.mlp.experts.3.gate_proj", "model.layers.4.mlp.experts.4.gate_proj", "model.layers.4.mlp.experts.5.gate_proj", "model.layers.4.mlp.experts.6.gate_proj", "model.layers.4.mlp.experts.7.gate_proj", "model.layers.4.mlp.experts.8.gate_proj", "model.layers.4.mlp.experts.9.gate_proj", "model.layers.4.mlp.experts.10.gate_proj", "model.layers.4.mlp.experts.11.gate_proj", "model.layers.4.mlp.experts.12.gate_proj", "model.layers.4.mlp.experts.13.gate_proj", "model.layers.4.mlp.experts.14.gate_proj", "model.layers.4.mlp.experts.15.gate_proj", "model.layers.4.mlp.experts.16.gate_proj", "model.layers.4.mlp.experts.17.gate_proj", "model.layers.4.mlp.experts.18.gate_proj", "model.layers.4.mlp.experts.19.gate_proj", "model.layers.4.mlp.experts.20.gate_proj", "model.layers.4.mlp.experts.21.gate_proj", "model.layers.4.mlp.experts.22.gate_proj", "model.layers.4.mlp.experts.23.gate_proj", "model.layers.4.mlp.experts.24.gate_proj", "model.layers.4.mlp.experts.25.gate_proj", "model.layers.4.mlp.experts.26.gate_proj", "model.layers.4.mlp.experts.27.gate_proj", "model.layers.4.mlp.experts.28.gate_proj", "model.layers.4.mlp.experts.29.gate_proj", "model.layers.4.mlp.experts.30.gate_proj", "model.layers.4.mlp.experts.31.gate_proj", "model.layers.4.mlp.experts.32.gate_proj", "model.layers.4.mlp.experts.33.gate_proj", "model.layers.4.mlp.experts.34.gate_proj", "model.layers.4.mlp.experts.35.gate_proj", "model.layers.4.mlp.experts.36.gate_proj", "model.layers.4.mlp.experts.37.gate_proj", "model.layers.4.mlp.experts.38.gate_proj", "model.layers.4.mlp.experts.39.gate_proj", "model.layers.4.mlp.experts.40.gate_proj", "model.layers.4.mlp.experts.41.gate_proj", "model.layers.4.mlp.experts.42.gate_proj", "model.layers.4.mlp.experts.43.gate_proj", "model.layers.4.mlp.experts.44.gate_proj", "model.layers.4.mlp.experts.45.gate_proj", "model.layers.4.mlp.experts.46.gate_proj", "model.layers.4.mlp.experts.47.gate_proj", "model.layers.4.mlp.experts.48.gate_proj", "model.layers.4.mlp.experts.49.gate_proj", "model.layers.4.mlp.experts.50.gate_proj", "model.layers.4.mlp.experts.51.gate_proj", "model.layers.4.mlp.experts.52.gate_proj", "model.layers.4.mlp.experts.53.gate_proj", "model.layers.4.mlp.experts.54.gate_proj", "model.layers.4.mlp.experts.55.gate_proj", "model.layers.4.mlp.experts.56.gate_proj", "model.layers.4.mlp.experts.57.gate_proj", "model.layers.4.mlp.experts.58.gate_proj", "model.layers.4.mlp.experts.59.gate_proj", "model.layers.4.mlp.experts.60.gate_proj", "model.layers.4.mlp.experts.61.gate_proj", "model.layers.4.mlp.experts.62.gate_proj", "model.layers.4.mlp.experts.63.gate_proj", "model.layers.4.mlp.experts.64.gate_proj", "model.layers.4.mlp.experts.65.gate_proj", "model.layers.4.mlp.experts.66.gate_proj", "model.layers.4.mlp.experts.67.gate_proj", "model.layers.4.mlp.experts.68.gate_proj", "model.layers.4.mlp.experts.69.gate_proj", "model.layers.4.mlp.experts.70.gate_proj", "model.layers.4.mlp.experts.71.gate_proj", "model.layers.4.mlp.experts.72.gate_proj", "model.layers.4.mlp.experts.73.gate_proj", "model.layers.4.mlp.experts.74.gate_proj", "model.layers.4.mlp.experts.75.gate_proj", "model.layers.4.mlp.experts.76.gate_proj", "model.layers.4.mlp.experts.77.gate_proj", "model.layers.4.mlp.experts.78.gate_proj", "model.layers.4.mlp.experts.79.gate_proj", "model.layers.4.mlp.experts.80.gate_proj", "model.layers.4.mlp.experts.81.gate_proj", "model.layers.4.mlp.experts.82.gate_proj", "model.layers.4.mlp.experts.83.gate_proj", "model.layers.4.mlp.experts.84.gate_proj", "model.layers.4.mlp.experts.85.gate_proj", "model.layers.4.mlp.experts.86.gate_proj", "model.layers.4.mlp.experts.87.gate_proj", "model.layers.4.mlp.experts.88.gate_proj", "model.layers.4.mlp.experts.89.gate_proj", "model.layers.4.mlp.experts.90.gate_proj", "model.layers.4.mlp.experts.91.gate_proj", "model.layers.4.mlp.experts.92.gate_proj", "model.layers.4.mlp.experts.93.gate_proj", "model.layers.4.mlp.experts.94.gate_proj", "model.layers.4.mlp.experts.95.gate_proj", "model.layers.4.mlp.experts.96.gate_proj", "model.layers.4.mlp.experts.97.gate_proj", "model.layers.4.mlp.experts.98.gate_proj", "model.layers.4.mlp.experts.99.gate_proj", "model.layers.4.mlp.experts.100.gate_proj", "model.layers.4.mlp.experts.101.gate_proj", "model.layers.4.mlp.experts.102.gate_proj", "model.layers.4.mlp.experts.103.gate_proj", "model.layers.4.mlp.experts.104.gate_proj", "model.layers.4.mlp.experts.105.gate_proj", "model.layers.4.mlp.experts.106.gate_proj", "model.layers.4.mlp.experts.107.gate_proj", "model.layers.4.mlp.experts.108.gate_proj", "model.layers.4.mlp.experts.109.gate_proj", "model.layers.4.mlp.experts.110.gate_proj", "model.layers.4.mlp.experts.111.gate_proj", "model.layers.4.mlp.experts.112.gate_proj", "model.layers.4.mlp.experts.113.gate_proj", "model.layers.4.mlp.experts.114.gate_proj", "model.layers.4.mlp.experts.115.gate_proj", "model.layers.4.mlp.experts.116.gate_proj", "model.layers.4.mlp.experts.117.gate_proj", "model.layers.4.mlp.experts.118.gate_proj", "model.layers.4.mlp.experts.119.gate_proj", "model.layers.4.mlp.experts.120.gate_proj", "model.layers.4.mlp.experts.121.gate_proj", "model.layers.4.mlp.experts.122.gate_proj", "model.layers.4.mlp.experts.123.gate_proj", "model.layers.4.mlp.experts.124.gate_proj", "model.layers.4.mlp.experts.125.gate_proj", "model.layers.4.mlp.experts.126.gate_proj", "model.layers.4.mlp.experts.127.gate_proj", "model.layers.4.mlp.experts.0.up_proj", "model.layers.4.mlp.experts.1.up_proj", "model.layers.4.mlp.experts.2.up_proj", "model.layers.4.mlp.experts.3.up_proj", "model.layers.4.mlp.experts.4.up_proj", "model.layers.4.mlp.experts.5.up_proj", "model.layers.4.mlp.experts.6.up_proj", "model.layers.4.mlp.experts.7.up_proj", "model.layers.4.mlp.experts.8.up_proj", "model.layers.4.mlp.experts.9.up_proj", "model.layers.4.mlp.experts.10.up_proj", "model.layers.4.mlp.experts.11.up_proj", "model.layers.4.mlp.experts.12.up_proj", "model.layers.4.mlp.experts.13.up_proj", "model.layers.4.mlp.experts.14.up_proj", "model.layers.4.mlp.experts.15.up_proj", "model.layers.4.mlp.experts.16.up_proj", "model.layers.4.mlp.experts.17.up_proj", "model.layers.4.mlp.experts.18.up_proj", "model.layers.4.mlp.experts.19.up_proj", "model.layers.4.mlp.experts.20.up_proj", "model.layers.4.mlp.experts.21.up_proj", "model.layers.4.mlp.experts.22.up_proj", "model.layers.4.mlp.experts.23.up_proj", "model.layers.4.mlp.experts.24.up_proj", "model.layers.4.mlp.experts.25.up_proj", "model.layers.4.mlp.experts.26.up_proj", "model.layers.4.mlp.experts.27.up_proj", "model.layers.4.mlp.experts.28.up_proj", "model.layers.4.mlp.experts.29.up_proj", "model.layers.4.mlp.experts.30.up_proj", "model.layers.4.mlp.experts.31.up_proj", "model.layers.4.mlp.experts.32.up_proj", "model.layers.4.mlp.experts.33.up_proj", "model.layers.4.mlp.experts.34.up_proj", "model.layers.4.mlp.experts.35.up_proj", "model.layers.4.mlp.experts.36.up_proj", "model.layers.4.mlp.experts.37.up_proj", "model.layers.4.mlp.experts.38.up_proj", "model.layers.4.mlp.experts.39.up_proj", "model.layers.4.mlp.experts.40.up_proj", "model.layers.4.mlp.experts.41.up_proj", "model.layers.4.mlp.experts.42.up_proj", "model.layers.4.mlp.experts.43.up_proj", "model.layers.4.mlp.experts.44.up_proj", "model.layers.4.mlp.experts.45.up_proj", "model.layers.4.mlp.experts.46.up_proj", "model.layers.4.mlp.experts.47.up_proj", "model.layers.4.mlp.experts.48.up_proj", "model.layers.4.mlp.experts.49.up_proj", "model.layers.4.mlp.experts.50.up_proj", "model.layers.4.mlp.experts.51.up_proj", "model.layers.4.mlp.experts.52.up_proj", "model.layers.4.mlp.experts.53.up_proj", "model.layers.4.mlp.experts.54.up_proj", "model.layers.4.mlp.experts.55.up_proj", "model.layers.4.mlp.experts.56.up_proj", "model.layers.4.mlp.experts.57.up_proj", "model.layers.4.mlp.experts.58.up_proj", "model.layers.4.mlp.experts.59.up_proj", "model.layers.4.mlp.experts.60.up_proj", "model.layers.4.mlp.experts.61.up_proj", "model.layers.4.mlp.experts.62.up_proj", "model.layers.4.mlp.experts.63.up_proj", "model.layers.4.mlp.experts.64.up_proj", "model.layers.4.mlp.experts.65.up_proj", "model.layers.4.mlp.experts.66.up_proj", "model.layers.4.mlp.experts.67.up_proj", "model.layers.4.mlp.experts.68.up_proj", "model.layers.4.mlp.experts.69.up_proj", "model.layers.4.mlp.experts.70.up_proj", "model.layers.4.mlp.experts.71.up_proj", "model.layers.4.mlp.experts.72.up_proj", "model.layers.4.mlp.experts.73.up_proj", "model.layers.4.mlp.experts.74.up_proj", "model.layers.4.mlp.experts.75.up_proj", "model.layers.4.mlp.experts.76.up_proj", "model.layers.4.mlp.experts.77.up_proj", "model.layers.4.mlp.experts.78.up_proj", "model.layers.4.mlp.experts.79.up_proj", "model.layers.4.mlp.experts.80.up_proj", "model.layers.4.mlp.experts.81.up_proj", "model.layers.4.mlp.experts.82.up_proj", "model.layers.4.mlp.experts.83.up_proj", "model.layers.4.mlp.experts.84.up_proj", "model.layers.4.mlp.experts.85.up_proj", "model.layers.4.mlp.experts.86.up_proj", "model.layers.4.mlp.experts.87.up_proj", "model.layers.4.mlp.experts.88.up_proj", "model.layers.4.mlp.experts.89.up_proj", "model.layers.4.mlp.experts.90.up_proj", "model.layers.4.mlp.experts.91.up_proj", "model.layers.4.mlp.experts.92.up_proj", "model.layers.4.mlp.experts.93.up_proj", "model.layers.4.mlp.experts.94.up_proj", "model.layers.4.mlp.experts.95.up_proj", "model.layers.4.mlp.experts.96.up_proj", "model.layers.4.mlp.experts.97.up_proj", "model.layers.4.mlp.experts.98.up_proj", "model.layers.4.mlp.experts.99.up_proj", "model.layers.4.mlp.experts.100.up_proj", "model.layers.4.mlp.experts.101.up_proj", "model.layers.4.mlp.experts.102.up_proj", "model.layers.4.mlp.experts.103.up_proj", "model.layers.4.mlp.experts.104.up_proj", "model.layers.4.mlp.experts.105.up_proj", "model.layers.4.mlp.experts.106.up_proj", "model.layers.4.mlp.experts.107.up_proj", "model.layers.4.mlp.experts.108.up_proj", "model.layers.4.mlp.experts.109.up_proj", "model.layers.4.mlp.experts.110.up_proj", "model.layers.4.mlp.experts.111.up_proj", "model.layers.4.mlp.experts.112.up_proj", "model.layers.4.mlp.experts.113.up_proj", "model.layers.4.mlp.experts.114.up_proj", "model.layers.4.mlp.experts.115.up_proj", "model.layers.4.mlp.experts.116.up_proj", "model.layers.4.mlp.experts.117.up_proj", "model.layers.4.mlp.experts.118.up_proj", "model.layers.4.mlp.experts.119.up_proj", "model.layers.4.mlp.experts.120.up_proj", "model.layers.4.mlp.experts.121.up_proj", "model.layers.4.mlp.experts.122.up_proj", "model.layers.4.mlp.experts.123.up_proj", "model.layers.4.mlp.experts.124.up_proj", "model.layers.4.mlp.experts.125.up_proj", "model.layers.4.mlp.experts.126.up_proj", "model.layers.4.mlp.experts.127.up_proj", "model.layers.4.mlp.experts.0.down_proj", "model.layers.4.mlp.experts.1.down_proj", "model.layers.4.mlp.experts.2.down_proj", "model.layers.4.mlp.experts.3.down_proj", "model.layers.4.mlp.experts.4.down_proj", "model.layers.4.mlp.experts.5.down_proj", "model.layers.4.mlp.experts.6.down_proj", "model.layers.4.mlp.experts.7.down_proj", "model.layers.4.mlp.experts.8.down_proj", "model.layers.4.mlp.experts.9.down_proj", "model.layers.4.mlp.experts.10.down_proj", "model.layers.4.mlp.experts.11.down_proj", "model.layers.4.mlp.experts.12.down_proj", "model.layers.4.mlp.experts.13.down_proj", "model.layers.4.mlp.experts.14.down_proj", "model.layers.4.mlp.experts.15.down_proj", "model.layers.4.mlp.experts.16.down_proj", "model.layers.4.mlp.experts.17.down_proj", "model.layers.4.mlp.experts.18.down_proj", "model.layers.4.mlp.experts.19.down_proj", "model.layers.4.mlp.experts.20.down_proj", "model.layers.4.mlp.experts.21.down_proj", "model.layers.4.mlp.experts.22.down_proj", "model.layers.4.mlp.experts.23.down_proj", "model.layers.4.mlp.experts.24.down_proj", "model.layers.4.mlp.experts.25.down_proj", "model.layers.4.mlp.experts.26.down_proj", "model.layers.4.mlp.experts.27.down_proj", "model.layers.4.mlp.experts.28.down_proj", "model.layers.4.mlp.experts.29.down_proj", "model.layers.4.mlp.experts.30.down_proj", "model.layers.4.mlp.experts.31.down_proj", "model.layers.4.mlp.experts.32.down_proj", "model.layers.4.mlp.experts.33.down_proj", "model.layers.4.mlp.experts.34.down_proj", "model.layers.4.mlp.experts.35.down_proj", "model.layers.4.mlp.experts.36.down_proj", "model.layers.4.mlp.experts.37.down_proj", "model.layers.4.mlp.experts.38.down_proj", "model.layers.4.mlp.experts.39.down_proj", "model.layers.4.mlp.experts.40.down_proj", "model.layers.4.mlp.experts.41.down_proj", "model.layers.4.mlp.experts.42.down_proj", "model.layers.4.mlp.experts.43.down_proj", "model.layers.4.mlp.experts.44.down_proj", "model.layers.4.mlp.experts.45.down_proj", "model.layers.4.mlp.experts.46.down_proj", "model.layers.4.mlp.experts.47.down_proj", "model.layers.4.mlp.experts.48.down_proj", "model.layers.4.mlp.experts.49.down_proj", "model.layers.4.mlp.experts.50.down_proj", "model.layers.4.mlp.experts.51.down_proj", "model.layers.4.mlp.experts.52.down_proj", "model.layers.4.mlp.experts.53.down_proj", "model.layers.4.mlp.experts.54.down_proj", "model.layers.4.mlp.experts.55.down_proj", "model.layers.4.mlp.experts.56.down_proj", "model.layers.4.mlp.experts.57.down_proj", "model.layers.4.mlp.experts.58.down_proj", "model.layers.4.mlp.experts.59.down_proj", "model.layers.4.mlp.experts.60.down_proj", "model.layers.4.mlp.experts.61.down_proj", "model.layers.4.mlp.experts.62.down_proj", "model.layers.4.mlp.experts.63.down_proj", "model.layers.4.mlp.experts.64.down_proj", "model.layers.4.mlp.experts.65.down_proj", "model.layers.4.mlp.experts.66.down_proj", "model.layers.4.mlp.experts.67.down_proj", "model.layers.4.mlp.experts.68.down_proj", "model.layers.4.mlp.experts.69.down_proj", "model.layers.4.mlp.experts.70.down_proj", "model.layers.4.mlp.experts.71.down_proj", "model.layers.4.mlp.experts.72.down_proj", "model.layers.4.mlp.experts.73.down_proj", "model.layers.4.mlp.experts.74.down_proj", "model.layers.4.mlp.experts.75.down_proj", "model.layers.4.mlp.experts.76.down_proj", "model.layers.4.mlp.experts.77.down_proj", "model.layers.4.mlp.experts.78.down_proj", "model.layers.4.mlp.experts.79.down_proj", "model.layers.4.mlp.experts.80.down_proj", "model.layers.4.mlp.experts.81.down_proj", "model.layers.4.mlp.experts.82.down_proj", "model.layers.4.mlp.experts.83.down_proj", "model.layers.4.mlp.experts.84.down_proj", "model.layers.4.mlp.experts.85.down_proj", "model.layers.4.mlp.experts.86.down_proj", "model.layers.4.mlp.experts.87.down_proj", "model.layers.4.mlp.experts.88.down_proj", "model.layers.4.mlp.experts.89.down_proj", "model.layers.4.mlp.experts.90.down_proj", "model.layers.4.mlp.experts.91.down_proj", "model.layers.4.mlp.experts.92.down_proj", "model.layers.4.mlp.experts.93.down_proj", "model.layers.4.mlp.experts.94.down_proj", "model.layers.4.mlp.experts.95.down_proj", "model.layers.4.mlp.experts.96.down_proj", "model.layers.4.mlp.experts.97.down_proj", "model.layers.4.mlp.experts.98.down_proj", "model.layers.4.mlp.experts.99.down_proj", "model.layers.4.mlp.experts.100.down_proj", "model.layers.4.mlp.experts.101.down_proj", "model.layers.4.mlp.experts.102.down_proj", "model.layers.4.mlp.experts.103.down_proj", "model.layers.4.mlp.experts.104.down_proj", "model.layers.4.mlp.experts.105.down_proj", "model.layers.4.mlp.experts.106.down_proj", "model.layers.4.mlp.experts.107.down_proj", "model.layers.4.mlp.experts.108.down_proj", "model.layers.4.mlp.experts.109.down_proj", "model.layers.4.mlp.experts.110.down_proj", "model.layers.4.mlp.experts.111.down_proj", "model.layers.4.mlp.experts.112.down_proj", "model.layers.4.mlp.experts.113.down_proj", "model.layers.4.mlp.experts.114.down_proj", "model.layers.4.mlp.experts.115.down_proj", "model.layers.4.mlp.experts.116.down_proj", "model.layers.4.mlp.experts.117.down_proj", "model.layers.4.mlp.experts.118.down_proj", "model.layers.4.mlp.experts.119.down_proj", "model.layers.4.mlp.experts.120.down_proj", "model.layers.4.mlp.experts.121.down_proj", "model.layers.4.mlp.experts.122.down_proj", "model.layers.4.mlp.experts.123.down_proj", "model.layers.4.mlp.experts.124.down_proj", "model.layers.4.mlp.experts.125.down_proj", "model.layers.4.mlp.experts.126.down_proj", "model.layers.4.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.02036324068903922, "dbits": 2214592512 } ] }, { "idx": 14, "layers": [ "model.layers.5.self_attn.q_proj", "model.layers.5.self_attn.k_proj", "model.layers.5.self_attn.v_proj", "model.layers.5.self_attn.o_proj" ], "candidates": [ { "dkld": -9.421687573193949e-05, "dbits": 109051904 } ] }, { "idx": 15, "layers": [ "model.layers.5.mlp.shared_experts.gate_proj", "model.layers.5.mlp.shared_experts.up_proj", "model.layers.5.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.004700421448796982, "dbits": 17301504 } ] }, { "idx": 16, "layers": [ "model.layers.5.mlp.experts.0.gate_proj", "model.layers.5.mlp.experts.1.gate_proj", "model.layers.5.mlp.experts.2.gate_proj", "model.layers.5.mlp.experts.3.gate_proj", "model.layers.5.mlp.experts.4.gate_proj", "model.layers.5.mlp.experts.5.gate_proj", "model.layers.5.mlp.experts.6.gate_proj", "model.layers.5.mlp.experts.7.gate_proj", "model.layers.5.mlp.experts.8.gate_proj", "model.layers.5.mlp.experts.9.gate_proj", "model.layers.5.mlp.experts.10.gate_proj", "model.layers.5.mlp.experts.11.gate_proj", "model.layers.5.mlp.experts.12.gate_proj", "model.layers.5.mlp.experts.13.gate_proj", "model.layers.5.mlp.experts.14.gate_proj", "model.layers.5.mlp.experts.15.gate_proj", "model.layers.5.mlp.experts.16.gate_proj", "model.layers.5.mlp.experts.17.gate_proj", "model.layers.5.mlp.experts.18.gate_proj", "model.layers.5.mlp.experts.19.gate_proj", "model.layers.5.mlp.experts.20.gate_proj", "model.layers.5.mlp.experts.21.gate_proj", "model.layers.5.mlp.experts.22.gate_proj", "model.layers.5.mlp.experts.23.gate_proj", "model.layers.5.mlp.experts.24.gate_proj", "model.layers.5.mlp.experts.25.gate_proj", "model.layers.5.mlp.experts.26.gate_proj", "model.layers.5.mlp.experts.27.gate_proj", "model.layers.5.mlp.experts.28.gate_proj", "model.layers.5.mlp.experts.29.gate_proj", "model.layers.5.mlp.experts.30.gate_proj", "model.layers.5.mlp.experts.31.gate_proj", "model.layers.5.mlp.experts.32.gate_proj", "model.layers.5.mlp.experts.33.gate_proj", "model.layers.5.mlp.experts.34.gate_proj", "model.layers.5.mlp.experts.35.gate_proj", "model.layers.5.mlp.experts.36.gate_proj", "model.layers.5.mlp.experts.37.gate_proj", "model.layers.5.mlp.experts.38.gate_proj", "model.layers.5.mlp.experts.39.gate_proj", "model.layers.5.mlp.experts.40.gate_proj", "model.layers.5.mlp.experts.41.gate_proj", "model.layers.5.mlp.experts.42.gate_proj", "model.layers.5.mlp.experts.43.gate_proj", "model.layers.5.mlp.experts.44.gate_proj", "model.layers.5.mlp.experts.45.gate_proj", "model.layers.5.mlp.experts.46.gate_proj", "model.layers.5.mlp.experts.47.gate_proj", "model.layers.5.mlp.experts.48.gate_proj", "model.layers.5.mlp.experts.49.gate_proj", "model.layers.5.mlp.experts.50.gate_proj", "model.layers.5.mlp.experts.51.gate_proj", "model.layers.5.mlp.experts.52.gate_proj", "model.layers.5.mlp.experts.53.gate_proj", "model.layers.5.mlp.experts.54.gate_proj", "model.layers.5.mlp.experts.55.gate_proj", "model.layers.5.mlp.experts.56.gate_proj", "model.layers.5.mlp.experts.57.gate_proj", "model.layers.5.mlp.experts.58.gate_proj", "model.layers.5.mlp.experts.59.gate_proj", "model.layers.5.mlp.experts.60.gate_proj", "model.layers.5.mlp.experts.61.gate_proj", "model.layers.5.mlp.experts.62.gate_proj", "model.layers.5.mlp.experts.63.gate_proj", "model.layers.5.mlp.experts.64.gate_proj", "model.layers.5.mlp.experts.65.gate_proj", "model.layers.5.mlp.experts.66.gate_proj", "model.layers.5.mlp.experts.67.gate_proj", "model.layers.5.mlp.experts.68.gate_proj", "model.layers.5.mlp.experts.69.gate_proj", "model.layers.5.mlp.experts.70.gate_proj", "model.layers.5.mlp.experts.71.gate_proj", "model.layers.5.mlp.experts.72.gate_proj", "model.layers.5.mlp.experts.73.gate_proj", "model.layers.5.mlp.experts.74.gate_proj", "model.layers.5.mlp.experts.75.gate_proj", "model.layers.5.mlp.experts.76.gate_proj", "model.layers.5.mlp.experts.77.gate_proj", "model.layers.5.mlp.experts.78.gate_proj", "model.layers.5.mlp.experts.79.gate_proj", "model.layers.5.mlp.experts.80.gate_proj", "model.layers.5.mlp.experts.81.gate_proj", "model.layers.5.mlp.experts.82.gate_proj", "model.layers.5.mlp.experts.83.gate_proj", "model.layers.5.mlp.experts.84.gate_proj", "model.layers.5.mlp.experts.85.gate_proj", "model.layers.5.mlp.experts.86.gate_proj", "model.layers.5.mlp.experts.87.gate_proj", "model.layers.5.mlp.experts.88.gate_proj", "model.layers.5.mlp.experts.89.gate_proj", "model.layers.5.mlp.experts.90.gate_proj", "model.layers.5.mlp.experts.91.gate_proj", "model.layers.5.mlp.experts.92.gate_proj", "model.layers.5.mlp.experts.93.gate_proj", "model.layers.5.mlp.experts.94.gate_proj", "model.layers.5.mlp.experts.95.gate_proj", "model.layers.5.mlp.experts.96.gate_proj", "model.layers.5.mlp.experts.97.gate_proj", "model.layers.5.mlp.experts.98.gate_proj", "model.layers.5.mlp.experts.99.gate_proj", "model.layers.5.mlp.experts.100.gate_proj", "model.layers.5.mlp.experts.101.gate_proj", "model.layers.5.mlp.experts.102.gate_proj", "model.layers.5.mlp.experts.103.gate_proj", "model.layers.5.mlp.experts.104.gate_proj", "model.layers.5.mlp.experts.105.gate_proj", "model.layers.5.mlp.experts.106.gate_proj", "model.layers.5.mlp.experts.107.gate_proj", "model.layers.5.mlp.experts.108.gate_proj", "model.layers.5.mlp.experts.109.gate_proj", "model.layers.5.mlp.experts.110.gate_proj", "model.layers.5.mlp.experts.111.gate_proj", "model.layers.5.mlp.experts.112.gate_proj", "model.layers.5.mlp.experts.113.gate_proj", "model.layers.5.mlp.experts.114.gate_proj", "model.layers.5.mlp.experts.115.gate_proj", "model.layers.5.mlp.experts.116.gate_proj", "model.layers.5.mlp.experts.117.gate_proj", "model.layers.5.mlp.experts.118.gate_proj", "model.layers.5.mlp.experts.119.gate_proj", "model.layers.5.mlp.experts.120.gate_proj", "model.layers.5.mlp.experts.121.gate_proj", "model.layers.5.mlp.experts.122.gate_proj", "model.layers.5.mlp.experts.123.gate_proj", "model.layers.5.mlp.experts.124.gate_proj", "model.layers.5.mlp.experts.125.gate_proj", "model.layers.5.mlp.experts.126.gate_proj", "model.layers.5.mlp.experts.127.gate_proj", "model.layers.5.mlp.experts.0.up_proj", "model.layers.5.mlp.experts.1.up_proj", "model.layers.5.mlp.experts.2.up_proj", "model.layers.5.mlp.experts.3.up_proj", "model.layers.5.mlp.experts.4.up_proj", "model.layers.5.mlp.experts.5.up_proj", "model.layers.5.mlp.experts.6.up_proj", "model.layers.5.mlp.experts.7.up_proj", "model.layers.5.mlp.experts.8.up_proj", "model.layers.5.mlp.experts.9.up_proj", "model.layers.5.mlp.experts.10.up_proj", "model.layers.5.mlp.experts.11.up_proj", "model.layers.5.mlp.experts.12.up_proj", "model.layers.5.mlp.experts.13.up_proj", "model.layers.5.mlp.experts.14.up_proj", "model.layers.5.mlp.experts.15.up_proj", "model.layers.5.mlp.experts.16.up_proj", "model.layers.5.mlp.experts.17.up_proj", "model.layers.5.mlp.experts.18.up_proj", "model.layers.5.mlp.experts.19.up_proj", "model.layers.5.mlp.experts.20.up_proj", "model.layers.5.mlp.experts.21.up_proj", "model.layers.5.mlp.experts.22.up_proj", "model.layers.5.mlp.experts.23.up_proj", "model.layers.5.mlp.experts.24.up_proj", "model.layers.5.mlp.experts.25.up_proj", "model.layers.5.mlp.experts.26.up_proj", "model.layers.5.mlp.experts.27.up_proj", "model.layers.5.mlp.experts.28.up_proj", "model.layers.5.mlp.experts.29.up_proj", "model.layers.5.mlp.experts.30.up_proj", "model.layers.5.mlp.experts.31.up_proj", "model.layers.5.mlp.experts.32.up_proj", "model.layers.5.mlp.experts.33.up_proj", "model.layers.5.mlp.experts.34.up_proj", "model.layers.5.mlp.experts.35.up_proj", "model.layers.5.mlp.experts.36.up_proj", "model.layers.5.mlp.experts.37.up_proj", "model.layers.5.mlp.experts.38.up_proj", "model.layers.5.mlp.experts.39.up_proj", "model.layers.5.mlp.experts.40.up_proj", "model.layers.5.mlp.experts.41.up_proj", "model.layers.5.mlp.experts.42.up_proj", "model.layers.5.mlp.experts.43.up_proj", "model.layers.5.mlp.experts.44.up_proj", "model.layers.5.mlp.experts.45.up_proj", "model.layers.5.mlp.experts.46.up_proj", "model.layers.5.mlp.experts.47.up_proj", "model.layers.5.mlp.experts.48.up_proj", "model.layers.5.mlp.experts.49.up_proj", "model.layers.5.mlp.experts.50.up_proj", "model.layers.5.mlp.experts.51.up_proj", "model.layers.5.mlp.experts.52.up_proj", "model.layers.5.mlp.experts.53.up_proj", "model.layers.5.mlp.experts.54.up_proj", "model.layers.5.mlp.experts.55.up_proj", "model.layers.5.mlp.experts.56.up_proj", "model.layers.5.mlp.experts.57.up_proj", "model.layers.5.mlp.experts.58.up_proj", "model.layers.5.mlp.experts.59.up_proj", "model.layers.5.mlp.experts.60.up_proj", "model.layers.5.mlp.experts.61.up_proj", "model.layers.5.mlp.experts.62.up_proj", "model.layers.5.mlp.experts.63.up_proj", "model.layers.5.mlp.experts.64.up_proj", "model.layers.5.mlp.experts.65.up_proj", "model.layers.5.mlp.experts.66.up_proj", "model.layers.5.mlp.experts.67.up_proj", "model.layers.5.mlp.experts.68.up_proj", "model.layers.5.mlp.experts.69.up_proj", "model.layers.5.mlp.experts.70.up_proj", "model.layers.5.mlp.experts.71.up_proj", "model.layers.5.mlp.experts.72.up_proj", "model.layers.5.mlp.experts.73.up_proj", "model.layers.5.mlp.experts.74.up_proj", "model.layers.5.mlp.experts.75.up_proj", "model.layers.5.mlp.experts.76.up_proj", "model.layers.5.mlp.experts.77.up_proj", "model.layers.5.mlp.experts.78.up_proj", "model.layers.5.mlp.experts.79.up_proj", "model.layers.5.mlp.experts.80.up_proj", "model.layers.5.mlp.experts.81.up_proj", "model.layers.5.mlp.experts.82.up_proj", "model.layers.5.mlp.experts.83.up_proj", "model.layers.5.mlp.experts.84.up_proj", "model.layers.5.mlp.experts.85.up_proj", "model.layers.5.mlp.experts.86.up_proj", "model.layers.5.mlp.experts.87.up_proj", "model.layers.5.mlp.experts.88.up_proj", "model.layers.5.mlp.experts.89.up_proj", "model.layers.5.mlp.experts.90.up_proj", "model.layers.5.mlp.experts.91.up_proj", "model.layers.5.mlp.experts.92.up_proj", "model.layers.5.mlp.experts.93.up_proj", "model.layers.5.mlp.experts.94.up_proj", "model.layers.5.mlp.experts.95.up_proj", "model.layers.5.mlp.experts.96.up_proj", "model.layers.5.mlp.experts.97.up_proj", "model.layers.5.mlp.experts.98.up_proj", "model.layers.5.mlp.experts.99.up_proj", "model.layers.5.mlp.experts.100.up_proj", "model.layers.5.mlp.experts.101.up_proj", "model.layers.5.mlp.experts.102.up_proj", "model.layers.5.mlp.experts.103.up_proj", "model.layers.5.mlp.experts.104.up_proj", "model.layers.5.mlp.experts.105.up_proj", "model.layers.5.mlp.experts.106.up_proj", "model.layers.5.mlp.experts.107.up_proj", "model.layers.5.mlp.experts.108.up_proj", "model.layers.5.mlp.experts.109.up_proj", "model.layers.5.mlp.experts.110.up_proj", "model.layers.5.mlp.experts.111.up_proj", "model.layers.5.mlp.experts.112.up_proj", "model.layers.5.mlp.experts.113.up_proj", "model.layers.5.mlp.experts.114.up_proj", "model.layers.5.mlp.experts.115.up_proj", "model.layers.5.mlp.experts.116.up_proj", "model.layers.5.mlp.experts.117.up_proj", "model.layers.5.mlp.experts.118.up_proj", "model.layers.5.mlp.experts.119.up_proj", "model.layers.5.mlp.experts.120.up_proj", "model.layers.5.mlp.experts.121.up_proj", "model.layers.5.mlp.experts.122.up_proj", "model.layers.5.mlp.experts.123.up_proj", "model.layers.5.mlp.experts.124.up_proj", "model.layers.5.mlp.experts.125.up_proj", "model.layers.5.mlp.experts.126.up_proj", "model.layers.5.mlp.experts.127.up_proj", "model.layers.5.mlp.experts.0.down_proj", "model.layers.5.mlp.experts.1.down_proj", "model.layers.5.mlp.experts.2.down_proj", "model.layers.5.mlp.experts.3.down_proj", "model.layers.5.mlp.experts.4.down_proj", "model.layers.5.mlp.experts.5.down_proj", "model.layers.5.mlp.experts.6.down_proj", "model.layers.5.mlp.experts.7.down_proj", "model.layers.5.mlp.experts.8.down_proj", "model.layers.5.mlp.experts.9.down_proj", "model.layers.5.mlp.experts.10.down_proj", "model.layers.5.mlp.experts.11.down_proj", "model.layers.5.mlp.experts.12.down_proj", "model.layers.5.mlp.experts.13.down_proj", "model.layers.5.mlp.experts.14.down_proj", "model.layers.5.mlp.experts.15.down_proj", "model.layers.5.mlp.experts.16.down_proj", "model.layers.5.mlp.experts.17.down_proj", "model.layers.5.mlp.experts.18.down_proj", "model.layers.5.mlp.experts.19.down_proj", "model.layers.5.mlp.experts.20.down_proj", "model.layers.5.mlp.experts.21.down_proj", "model.layers.5.mlp.experts.22.down_proj", "model.layers.5.mlp.experts.23.down_proj", "model.layers.5.mlp.experts.24.down_proj", "model.layers.5.mlp.experts.25.down_proj", "model.layers.5.mlp.experts.26.down_proj", "model.layers.5.mlp.experts.27.down_proj", "model.layers.5.mlp.experts.28.down_proj", "model.layers.5.mlp.experts.29.down_proj", "model.layers.5.mlp.experts.30.down_proj", "model.layers.5.mlp.experts.31.down_proj", "model.layers.5.mlp.experts.32.down_proj", "model.layers.5.mlp.experts.33.down_proj", "model.layers.5.mlp.experts.34.down_proj", "model.layers.5.mlp.experts.35.down_proj", "model.layers.5.mlp.experts.36.down_proj", "model.layers.5.mlp.experts.37.down_proj", "model.layers.5.mlp.experts.38.down_proj", "model.layers.5.mlp.experts.39.down_proj", "model.layers.5.mlp.experts.40.down_proj", "model.layers.5.mlp.experts.41.down_proj", "model.layers.5.mlp.experts.42.down_proj", "model.layers.5.mlp.experts.43.down_proj", "model.layers.5.mlp.experts.44.down_proj", "model.layers.5.mlp.experts.45.down_proj", "model.layers.5.mlp.experts.46.down_proj", "model.layers.5.mlp.experts.47.down_proj", "model.layers.5.mlp.experts.48.down_proj", "model.layers.5.mlp.experts.49.down_proj", "model.layers.5.mlp.experts.50.down_proj", "model.layers.5.mlp.experts.51.down_proj", "model.layers.5.mlp.experts.52.down_proj", "model.layers.5.mlp.experts.53.down_proj", "model.layers.5.mlp.experts.54.down_proj", "model.layers.5.mlp.experts.55.down_proj", "model.layers.5.mlp.experts.56.down_proj", "model.layers.5.mlp.experts.57.down_proj", "model.layers.5.mlp.experts.58.down_proj", "model.layers.5.mlp.experts.59.down_proj", "model.layers.5.mlp.experts.60.down_proj", "model.layers.5.mlp.experts.61.down_proj", "model.layers.5.mlp.experts.62.down_proj", "model.layers.5.mlp.experts.63.down_proj", "model.layers.5.mlp.experts.64.down_proj", "model.layers.5.mlp.experts.65.down_proj", "model.layers.5.mlp.experts.66.down_proj", "model.layers.5.mlp.experts.67.down_proj", "model.layers.5.mlp.experts.68.down_proj", "model.layers.5.mlp.experts.69.down_proj", "model.layers.5.mlp.experts.70.down_proj", "model.layers.5.mlp.experts.71.down_proj", "model.layers.5.mlp.experts.72.down_proj", "model.layers.5.mlp.experts.73.down_proj", "model.layers.5.mlp.experts.74.down_proj", "model.layers.5.mlp.experts.75.down_proj", "model.layers.5.mlp.experts.76.down_proj", "model.layers.5.mlp.experts.77.down_proj", "model.layers.5.mlp.experts.78.down_proj", "model.layers.5.mlp.experts.79.down_proj", "model.layers.5.mlp.experts.80.down_proj", "model.layers.5.mlp.experts.81.down_proj", "model.layers.5.mlp.experts.82.down_proj", "model.layers.5.mlp.experts.83.down_proj", "model.layers.5.mlp.experts.84.down_proj", "model.layers.5.mlp.experts.85.down_proj", "model.layers.5.mlp.experts.86.down_proj", "model.layers.5.mlp.experts.87.down_proj", "model.layers.5.mlp.experts.88.down_proj", "model.layers.5.mlp.experts.89.down_proj", "model.layers.5.mlp.experts.90.down_proj", "model.layers.5.mlp.experts.91.down_proj", "model.layers.5.mlp.experts.92.down_proj", "model.layers.5.mlp.experts.93.down_proj", "model.layers.5.mlp.experts.94.down_proj", "model.layers.5.mlp.experts.95.down_proj", "model.layers.5.mlp.experts.96.down_proj", "model.layers.5.mlp.experts.97.down_proj", "model.layers.5.mlp.experts.98.down_proj", "model.layers.5.mlp.experts.99.down_proj", "model.layers.5.mlp.experts.100.down_proj", "model.layers.5.mlp.experts.101.down_proj", "model.layers.5.mlp.experts.102.down_proj", "model.layers.5.mlp.experts.103.down_proj", "model.layers.5.mlp.experts.104.down_proj", "model.layers.5.mlp.experts.105.down_proj", "model.layers.5.mlp.experts.106.down_proj", "model.layers.5.mlp.experts.107.down_proj", "model.layers.5.mlp.experts.108.down_proj", "model.layers.5.mlp.experts.109.down_proj", "model.layers.5.mlp.experts.110.down_proj", "model.layers.5.mlp.experts.111.down_proj", "model.layers.5.mlp.experts.112.down_proj", "model.layers.5.mlp.experts.113.down_proj", "model.layers.5.mlp.experts.114.down_proj", "model.layers.5.mlp.experts.115.down_proj", "model.layers.5.mlp.experts.116.down_proj", "model.layers.5.mlp.experts.117.down_proj", "model.layers.5.mlp.experts.118.down_proj", "model.layers.5.mlp.experts.119.down_proj", "model.layers.5.mlp.experts.120.down_proj", "model.layers.5.mlp.experts.121.down_proj", "model.layers.5.mlp.experts.122.down_proj", "model.layers.5.mlp.experts.123.down_proj", "model.layers.5.mlp.experts.124.down_proj", "model.layers.5.mlp.experts.125.down_proj", "model.layers.5.mlp.experts.126.down_proj", "model.layers.5.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.015746647492051125, "dbits": 2214592512 } ] }, { "idx": 17, "layers": [ "model.layers.6.self_attn.q_proj", "model.layers.6.self_attn.k_proj", "model.layers.6.self_attn.v_proj", "model.layers.6.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00037743616849184036, "dbits": 109051904 } ] }, { "idx": 18, "layers": [ "model.layers.6.mlp.shared_experts.gate_proj", "model.layers.6.mlp.shared_experts.up_proj", "model.layers.6.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0027984189800918102, "dbits": 17301504 } ] }, { "idx": 19, "layers": [ "model.layers.6.mlp.experts.0.gate_proj", "model.layers.6.mlp.experts.1.gate_proj", "model.layers.6.mlp.experts.2.gate_proj", "model.layers.6.mlp.experts.3.gate_proj", "model.layers.6.mlp.experts.4.gate_proj", "model.layers.6.mlp.experts.5.gate_proj", "model.layers.6.mlp.experts.6.gate_proj", "model.layers.6.mlp.experts.7.gate_proj", "model.layers.6.mlp.experts.8.gate_proj", "model.layers.6.mlp.experts.9.gate_proj", "model.layers.6.mlp.experts.10.gate_proj", "model.layers.6.mlp.experts.11.gate_proj", "model.layers.6.mlp.experts.12.gate_proj", "model.layers.6.mlp.experts.13.gate_proj", "model.layers.6.mlp.experts.14.gate_proj", "model.layers.6.mlp.experts.15.gate_proj", "model.layers.6.mlp.experts.16.gate_proj", "model.layers.6.mlp.experts.17.gate_proj", "model.layers.6.mlp.experts.18.gate_proj", "model.layers.6.mlp.experts.19.gate_proj", "model.layers.6.mlp.experts.20.gate_proj", "model.layers.6.mlp.experts.21.gate_proj", "model.layers.6.mlp.experts.22.gate_proj", "model.layers.6.mlp.experts.23.gate_proj", "model.layers.6.mlp.experts.24.gate_proj", "model.layers.6.mlp.experts.25.gate_proj", "model.layers.6.mlp.experts.26.gate_proj", "model.layers.6.mlp.experts.27.gate_proj", "model.layers.6.mlp.experts.28.gate_proj", "model.layers.6.mlp.experts.29.gate_proj", "model.layers.6.mlp.experts.30.gate_proj", "model.layers.6.mlp.experts.31.gate_proj", "model.layers.6.mlp.experts.32.gate_proj", "model.layers.6.mlp.experts.33.gate_proj", "model.layers.6.mlp.experts.34.gate_proj", "model.layers.6.mlp.experts.35.gate_proj", "model.layers.6.mlp.experts.36.gate_proj", "model.layers.6.mlp.experts.37.gate_proj", "model.layers.6.mlp.experts.38.gate_proj", "model.layers.6.mlp.experts.39.gate_proj", "model.layers.6.mlp.experts.40.gate_proj", "model.layers.6.mlp.experts.41.gate_proj", "model.layers.6.mlp.experts.42.gate_proj", "model.layers.6.mlp.experts.43.gate_proj", "model.layers.6.mlp.experts.44.gate_proj", "model.layers.6.mlp.experts.45.gate_proj", "model.layers.6.mlp.experts.46.gate_proj", "model.layers.6.mlp.experts.47.gate_proj", "model.layers.6.mlp.experts.48.gate_proj", "model.layers.6.mlp.experts.49.gate_proj", "model.layers.6.mlp.experts.50.gate_proj", "model.layers.6.mlp.experts.51.gate_proj", "model.layers.6.mlp.experts.52.gate_proj", "model.layers.6.mlp.experts.53.gate_proj", "model.layers.6.mlp.experts.54.gate_proj", "model.layers.6.mlp.experts.55.gate_proj", "model.layers.6.mlp.experts.56.gate_proj", "model.layers.6.mlp.experts.57.gate_proj", "model.layers.6.mlp.experts.58.gate_proj", "model.layers.6.mlp.experts.59.gate_proj", "model.layers.6.mlp.experts.60.gate_proj", "model.layers.6.mlp.experts.61.gate_proj", "model.layers.6.mlp.experts.62.gate_proj", "model.layers.6.mlp.experts.63.gate_proj", "model.layers.6.mlp.experts.64.gate_proj", "model.layers.6.mlp.experts.65.gate_proj", "model.layers.6.mlp.experts.66.gate_proj", "model.layers.6.mlp.experts.67.gate_proj", "model.layers.6.mlp.experts.68.gate_proj", "model.layers.6.mlp.experts.69.gate_proj", "model.layers.6.mlp.experts.70.gate_proj", "model.layers.6.mlp.experts.71.gate_proj", "model.layers.6.mlp.experts.72.gate_proj", "model.layers.6.mlp.experts.73.gate_proj", "model.layers.6.mlp.experts.74.gate_proj", "model.layers.6.mlp.experts.75.gate_proj", "model.layers.6.mlp.experts.76.gate_proj", "model.layers.6.mlp.experts.77.gate_proj", "model.layers.6.mlp.experts.78.gate_proj", "model.layers.6.mlp.experts.79.gate_proj", "model.layers.6.mlp.experts.80.gate_proj", "model.layers.6.mlp.experts.81.gate_proj", "model.layers.6.mlp.experts.82.gate_proj", "model.layers.6.mlp.experts.83.gate_proj", "model.layers.6.mlp.experts.84.gate_proj", "model.layers.6.mlp.experts.85.gate_proj", "model.layers.6.mlp.experts.86.gate_proj", "model.layers.6.mlp.experts.87.gate_proj", "model.layers.6.mlp.experts.88.gate_proj", "model.layers.6.mlp.experts.89.gate_proj", "model.layers.6.mlp.experts.90.gate_proj", "model.layers.6.mlp.experts.91.gate_proj", "model.layers.6.mlp.experts.92.gate_proj", "model.layers.6.mlp.experts.93.gate_proj", "model.layers.6.mlp.experts.94.gate_proj", "model.layers.6.mlp.experts.95.gate_proj", "model.layers.6.mlp.experts.96.gate_proj", "model.layers.6.mlp.experts.97.gate_proj", "model.layers.6.mlp.experts.98.gate_proj", "model.layers.6.mlp.experts.99.gate_proj", "model.layers.6.mlp.experts.100.gate_proj", "model.layers.6.mlp.experts.101.gate_proj", "model.layers.6.mlp.experts.102.gate_proj", "model.layers.6.mlp.experts.103.gate_proj", "model.layers.6.mlp.experts.104.gate_proj", "model.layers.6.mlp.experts.105.gate_proj", "model.layers.6.mlp.experts.106.gate_proj", "model.layers.6.mlp.experts.107.gate_proj", "model.layers.6.mlp.experts.108.gate_proj", "model.layers.6.mlp.experts.109.gate_proj", "model.layers.6.mlp.experts.110.gate_proj", "model.layers.6.mlp.experts.111.gate_proj", "model.layers.6.mlp.experts.112.gate_proj", "model.layers.6.mlp.experts.113.gate_proj", "model.layers.6.mlp.experts.114.gate_proj", "model.layers.6.mlp.experts.115.gate_proj", "model.layers.6.mlp.experts.116.gate_proj", "model.layers.6.mlp.experts.117.gate_proj", "model.layers.6.mlp.experts.118.gate_proj", "model.layers.6.mlp.experts.119.gate_proj", "model.layers.6.mlp.experts.120.gate_proj", "model.layers.6.mlp.experts.121.gate_proj", "model.layers.6.mlp.experts.122.gate_proj", "model.layers.6.mlp.experts.123.gate_proj", "model.layers.6.mlp.experts.124.gate_proj", "model.layers.6.mlp.experts.125.gate_proj", "model.layers.6.mlp.experts.126.gate_proj", "model.layers.6.mlp.experts.127.gate_proj", "model.layers.6.mlp.experts.0.up_proj", "model.layers.6.mlp.experts.1.up_proj", "model.layers.6.mlp.experts.2.up_proj", "model.layers.6.mlp.experts.3.up_proj", "model.layers.6.mlp.experts.4.up_proj", "model.layers.6.mlp.experts.5.up_proj", "model.layers.6.mlp.experts.6.up_proj", "model.layers.6.mlp.experts.7.up_proj", "model.layers.6.mlp.experts.8.up_proj", "model.layers.6.mlp.experts.9.up_proj", "model.layers.6.mlp.experts.10.up_proj", "model.layers.6.mlp.experts.11.up_proj", "model.layers.6.mlp.experts.12.up_proj", "model.layers.6.mlp.experts.13.up_proj", "model.layers.6.mlp.experts.14.up_proj", "model.layers.6.mlp.experts.15.up_proj", "model.layers.6.mlp.experts.16.up_proj", "model.layers.6.mlp.experts.17.up_proj", "model.layers.6.mlp.experts.18.up_proj", "model.layers.6.mlp.experts.19.up_proj", "model.layers.6.mlp.experts.20.up_proj", "model.layers.6.mlp.experts.21.up_proj", "model.layers.6.mlp.experts.22.up_proj", "model.layers.6.mlp.experts.23.up_proj", "model.layers.6.mlp.experts.24.up_proj", "model.layers.6.mlp.experts.25.up_proj", "model.layers.6.mlp.experts.26.up_proj", "model.layers.6.mlp.experts.27.up_proj", "model.layers.6.mlp.experts.28.up_proj", "model.layers.6.mlp.experts.29.up_proj", "model.layers.6.mlp.experts.30.up_proj", "model.layers.6.mlp.experts.31.up_proj", "model.layers.6.mlp.experts.32.up_proj", "model.layers.6.mlp.experts.33.up_proj", "model.layers.6.mlp.experts.34.up_proj", "model.layers.6.mlp.experts.35.up_proj", "model.layers.6.mlp.experts.36.up_proj", "model.layers.6.mlp.experts.37.up_proj", "model.layers.6.mlp.experts.38.up_proj", "model.layers.6.mlp.experts.39.up_proj", "model.layers.6.mlp.experts.40.up_proj", "model.layers.6.mlp.experts.41.up_proj", "model.layers.6.mlp.experts.42.up_proj", "model.layers.6.mlp.experts.43.up_proj", "model.layers.6.mlp.experts.44.up_proj", "model.layers.6.mlp.experts.45.up_proj", "model.layers.6.mlp.experts.46.up_proj", "model.layers.6.mlp.experts.47.up_proj", "model.layers.6.mlp.experts.48.up_proj", "model.layers.6.mlp.experts.49.up_proj", "model.layers.6.mlp.experts.50.up_proj", "model.layers.6.mlp.experts.51.up_proj", "model.layers.6.mlp.experts.52.up_proj", "model.layers.6.mlp.experts.53.up_proj", "model.layers.6.mlp.experts.54.up_proj", "model.layers.6.mlp.experts.55.up_proj", "model.layers.6.mlp.experts.56.up_proj", "model.layers.6.mlp.experts.57.up_proj", "model.layers.6.mlp.experts.58.up_proj", "model.layers.6.mlp.experts.59.up_proj", "model.layers.6.mlp.experts.60.up_proj", "model.layers.6.mlp.experts.61.up_proj", "model.layers.6.mlp.experts.62.up_proj", "model.layers.6.mlp.experts.63.up_proj", "model.layers.6.mlp.experts.64.up_proj", "model.layers.6.mlp.experts.65.up_proj", "model.layers.6.mlp.experts.66.up_proj", "model.layers.6.mlp.experts.67.up_proj", "model.layers.6.mlp.experts.68.up_proj", "model.layers.6.mlp.experts.69.up_proj", "model.layers.6.mlp.experts.70.up_proj", "model.layers.6.mlp.experts.71.up_proj", "model.layers.6.mlp.experts.72.up_proj", "model.layers.6.mlp.experts.73.up_proj", "model.layers.6.mlp.experts.74.up_proj", "model.layers.6.mlp.experts.75.up_proj", "model.layers.6.mlp.experts.76.up_proj", "model.layers.6.mlp.experts.77.up_proj", "model.layers.6.mlp.experts.78.up_proj", "model.layers.6.mlp.experts.79.up_proj", "model.layers.6.mlp.experts.80.up_proj", "model.layers.6.mlp.experts.81.up_proj", "model.layers.6.mlp.experts.82.up_proj", "model.layers.6.mlp.experts.83.up_proj", "model.layers.6.mlp.experts.84.up_proj", "model.layers.6.mlp.experts.85.up_proj", "model.layers.6.mlp.experts.86.up_proj", "model.layers.6.mlp.experts.87.up_proj", "model.layers.6.mlp.experts.88.up_proj", "model.layers.6.mlp.experts.89.up_proj", "model.layers.6.mlp.experts.90.up_proj", "model.layers.6.mlp.experts.91.up_proj", "model.layers.6.mlp.experts.92.up_proj", "model.layers.6.mlp.experts.93.up_proj", "model.layers.6.mlp.experts.94.up_proj", "model.layers.6.mlp.experts.95.up_proj", "model.layers.6.mlp.experts.96.up_proj", "model.layers.6.mlp.experts.97.up_proj", "model.layers.6.mlp.experts.98.up_proj", "model.layers.6.mlp.experts.99.up_proj", "model.layers.6.mlp.experts.100.up_proj", "model.layers.6.mlp.experts.101.up_proj", "model.layers.6.mlp.experts.102.up_proj", "model.layers.6.mlp.experts.103.up_proj", "model.layers.6.mlp.experts.104.up_proj", "model.layers.6.mlp.experts.105.up_proj", "model.layers.6.mlp.experts.106.up_proj", "model.layers.6.mlp.experts.107.up_proj", "model.layers.6.mlp.experts.108.up_proj", "model.layers.6.mlp.experts.109.up_proj", "model.layers.6.mlp.experts.110.up_proj", "model.layers.6.mlp.experts.111.up_proj", "model.layers.6.mlp.experts.112.up_proj", "model.layers.6.mlp.experts.113.up_proj", "model.layers.6.mlp.experts.114.up_proj", "model.layers.6.mlp.experts.115.up_proj", "model.layers.6.mlp.experts.116.up_proj", "model.layers.6.mlp.experts.117.up_proj", "model.layers.6.mlp.experts.118.up_proj", "model.layers.6.mlp.experts.119.up_proj", "model.layers.6.mlp.experts.120.up_proj", "model.layers.6.mlp.experts.121.up_proj", "model.layers.6.mlp.experts.122.up_proj", "model.layers.6.mlp.experts.123.up_proj", "model.layers.6.mlp.experts.124.up_proj", "model.layers.6.mlp.experts.125.up_proj", "model.layers.6.mlp.experts.126.up_proj", "model.layers.6.mlp.experts.127.up_proj", "model.layers.6.mlp.experts.0.down_proj", "model.layers.6.mlp.experts.1.down_proj", "model.layers.6.mlp.experts.2.down_proj", "model.layers.6.mlp.experts.3.down_proj", "model.layers.6.mlp.experts.4.down_proj", "model.layers.6.mlp.experts.5.down_proj", "model.layers.6.mlp.experts.6.down_proj", "model.layers.6.mlp.experts.7.down_proj", "model.layers.6.mlp.experts.8.down_proj", "model.layers.6.mlp.experts.9.down_proj", "model.layers.6.mlp.experts.10.down_proj", "model.layers.6.mlp.experts.11.down_proj", "model.layers.6.mlp.experts.12.down_proj", "model.layers.6.mlp.experts.13.down_proj", "model.layers.6.mlp.experts.14.down_proj", "model.layers.6.mlp.experts.15.down_proj", "model.layers.6.mlp.experts.16.down_proj", "model.layers.6.mlp.experts.17.down_proj", "model.layers.6.mlp.experts.18.down_proj", "model.layers.6.mlp.experts.19.down_proj", "model.layers.6.mlp.experts.20.down_proj", "model.layers.6.mlp.experts.21.down_proj", "model.layers.6.mlp.experts.22.down_proj", "model.layers.6.mlp.experts.23.down_proj", "model.layers.6.mlp.experts.24.down_proj", "model.layers.6.mlp.experts.25.down_proj", "model.layers.6.mlp.experts.26.down_proj", "model.layers.6.mlp.experts.27.down_proj", "model.layers.6.mlp.experts.28.down_proj", "model.layers.6.mlp.experts.29.down_proj", "model.layers.6.mlp.experts.30.down_proj", "model.layers.6.mlp.experts.31.down_proj", "model.layers.6.mlp.experts.32.down_proj", "model.layers.6.mlp.experts.33.down_proj", "model.layers.6.mlp.experts.34.down_proj", "model.layers.6.mlp.experts.35.down_proj", "model.layers.6.mlp.experts.36.down_proj", "model.layers.6.mlp.experts.37.down_proj", "model.layers.6.mlp.experts.38.down_proj", "model.layers.6.mlp.experts.39.down_proj", "model.layers.6.mlp.experts.40.down_proj", "model.layers.6.mlp.experts.41.down_proj", "model.layers.6.mlp.experts.42.down_proj", "model.layers.6.mlp.experts.43.down_proj", "model.layers.6.mlp.experts.44.down_proj", "model.layers.6.mlp.experts.45.down_proj", "model.layers.6.mlp.experts.46.down_proj", "model.layers.6.mlp.experts.47.down_proj", "model.layers.6.mlp.experts.48.down_proj", "model.layers.6.mlp.experts.49.down_proj", "model.layers.6.mlp.experts.50.down_proj", "model.layers.6.mlp.experts.51.down_proj", "model.layers.6.mlp.experts.52.down_proj", "model.layers.6.mlp.experts.53.down_proj", "model.layers.6.mlp.experts.54.down_proj", "model.layers.6.mlp.experts.55.down_proj", "model.layers.6.mlp.experts.56.down_proj", "model.layers.6.mlp.experts.57.down_proj", "model.layers.6.mlp.experts.58.down_proj", "model.layers.6.mlp.experts.59.down_proj", "model.layers.6.mlp.experts.60.down_proj", "model.layers.6.mlp.experts.61.down_proj", "model.layers.6.mlp.experts.62.down_proj", "model.layers.6.mlp.experts.63.down_proj", "model.layers.6.mlp.experts.64.down_proj", "model.layers.6.mlp.experts.65.down_proj", "model.layers.6.mlp.experts.66.down_proj", "model.layers.6.mlp.experts.67.down_proj", "model.layers.6.mlp.experts.68.down_proj", "model.layers.6.mlp.experts.69.down_proj", "model.layers.6.mlp.experts.70.down_proj", "model.layers.6.mlp.experts.71.down_proj", "model.layers.6.mlp.experts.72.down_proj", "model.layers.6.mlp.experts.73.down_proj", "model.layers.6.mlp.experts.74.down_proj", "model.layers.6.mlp.experts.75.down_proj", "model.layers.6.mlp.experts.76.down_proj", "model.layers.6.mlp.experts.77.down_proj", "model.layers.6.mlp.experts.78.down_proj", "model.layers.6.mlp.experts.79.down_proj", "model.layers.6.mlp.experts.80.down_proj", "model.layers.6.mlp.experts.81.down_proj", "model.layers.6.mlp.experts.82.down_proj", "model.layers.6.mlp.experts.83.down_proj", "model.layers.6.mlp.experts.84.down_proj", "model.layers.6.mlp.experts.85.down_proj", "model.layers.6.mlp.experts.86.down_proj", "model.layers.6.mlp.experts.87.down_proj", "model.layers.6.mlp.experts.88.down_proj", "model.layers.6.mlp.experts.89.down_proj", "model.layers.6.mlp.experts.90.down_proj", "model.layers.6.mlp.experts.91.down_proj", "model.layers.6.mlp.experts.92.down_proj", "model.layers.6.mlp.experts.93.down_proj", "model.layers.6.mlp.experts.94.down_proj", "model.layers.6.mlp.experts.95.down_proj", "model.layers.6.mlp.experts.96.down_proj", "model.layers.6.mlp.experts.97.down_proj", "model.layers.6.mlp.experts.98.down_proj", "model.layers.6.mlp.experts.99.down_proj", "model.layers.6.mlp.experts.100.down_proj", "model.layers.6.mlp.experts.101.down_proj", "model.layers.6.mlp.experts.102.down_proj", "model.layers.6.mlp.experts.103.down_proj", "model.layers.6.mlp.experts.104.down_proj", "model.layers.6.mlp.experts.105.down_proj", "model.layers.6.mlp.experts.106.down_proj", "model.layers.6.mlp.experts.107.down_proj", "model.layers.6.mlp.experts.108.down_proj", "model.layers.6.mlp.experts.109.down_proj", "model.layers.6.mlp.experts.110.down_proj", "model.layers.6.mlp.experts.111.down_proj", "model.layers.6.mlp.experts.112.down_proj", "model.layers.6.mlp.experts.113.down_proj", "model.layers.6.mlp.experts.114.down_proj", "model.layers.6.mlp.experts.115.down_proj", "model.layers.6.mlp.experts.116.down_proj", "model.layers.6.mlp.experts.117.down_proj", "model.layers.6.mlp.experts.118.down_proj", "model.layers.6.mlp.experts.119.down_proj", "model.layers.6.mlp.experts.120.down_proj", "model.layers.6.mlp.experts.121.down_proj", "model.layers.6.mlp.experts.122.down_proj", "model.layers.6.mlp.experts.123.down_proj", "model.layers.6.mlp.experts.124.down_proj", "model.layers.6.mlp.experts.125.down_proj", "model.layers.6.mlp.experts.126.down_proj", "model.layers.6.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.016651995386928325, "dbits": 2214592512 } ] }, { "idx": 20, "layers": [ "model.layers.7.self_attn.q_proj", "model.layers.7.self_attn.k_proj", "model.layers.7.self_attn.v_proj", "model.layers.7.self_attn.o_proj" ], "candidates": [ { "dkld": 0.005908014252781868, "dbits": 109051904 } ] }, { "idx": 21, "layers": [ "model.layers.7.mlp.shared_experts.gate_proj", "model.layers.7.mlp.shared_experts.up_proj", "model.layers.7.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.005133372358977789, "dbits": 17301504 } ] }, { "idx": 22, "layers": [ "model.layers.7.mlp.experts.0.gate_proj", "model.layers.7.mlp.experts.1.gate_proj", "model.layers.7.mlp.experts.2.gate_proj", "model.layers.7.mlp.experts.3.gate_proj", "model.layers.7.mlp.experts.4.gate_proj", "model.layers.7.mlp.experts.5.gate_proj", "model.layers.7.mlp.experts.6.gate_proj", "model.layers.7.mlp.experts.7.gate_proj", "model.layers.7.mlp.experts.8.gate_proj", "model.layers.7.mlp.experts.9.gate_proj", "model.layers.7.mlp.experts.10.gate_proj", "model.layers.7.mlp.experts.11.gate_proj", "model.layers.7.mlp.experts.12.gate_proj", "model.layers.7.mlp.experts.13.gate_proj", "model.layers.7.mlp.experts.14.gate_proj", "model.layers.7.mlp.experts.15.gate_proj", "model.layers.7.mlp.experts.16.gate_proj", "model.layers.7.mlp.experts.17.gate_proj", "model.layers.7.mlp.experts.18.gate_proj", "model.layers.7.mlp.experts.19.gate_proj", "model.layers.7.mlp.experts.20.gate_proj", "model.layers.7.mlp.experts.21.gate_proj", "model.layers.7.mlp.experts.22.gate_proj", "model.layers.7.mlp.experts.23.gate_proj", "model.layers.7.mlp.experts.24.gate_proj", "model.layers.7.mlp.experts.25.gate_proj", "model.layers.7.mlp.experts.26.gate_proj", "model.layers.7.mlp.experts.27.gate_proj", "model.layers.7.mlp.experts.28.gate_proj", "model.layers.7.mlp.experts.29.gate_proj", "model.layers.7.mlp.experts.30.gate_proj", "model.layers.7.mlp.experts.31.gate_proj", "model.layers.7.mlp.experts.32.gate_proj", "model.layers.7.mlp.experts.33.gate_proj", "model.layers.7.mlp.experts.34.gate_proj", "model.layers.7.mlp.experts.35.gate_proj", "model.layers.7.mlp.experts.36.gate_proj", "model.layers.7.mlp.experts.37.gate_proj", "model.layers.7.mlp.experts.38.gate_proj", "model.layers.7.mlp.experts.39.gate_proj", "model.layers.7.mlp.experts.40.gate_proj", "model.layers.7.mlp.experts.41.gate_proj", "model.layers.7.mlp.experts.42.gate_proj", "model.layers.7.mlp.experts.43.gate_proj", "model.layers.7.mlp.experts.44.gate_proj", "model.layers.7.mlp.experts.45.gate_proj", "model.layers.7.mlp.experts.46.gate_proj", "model.layers.7.mlp.experts.47.gate_proj", "model.layers.7.mlp.experts.48.gate_proj", "model.layers.7.mlp.experts.49.gate_proj", "model.layers.7.mlp.experts.50.gate_proj", "model.layers.7.mlp.experts.51.gate_proj", "model.layers.7.mlp.experts.52.gate_proj", "model.layers.7.mlp.experts.53.gate_proj", "model.layers.7.mlp.experts.54.gate_proj", "model.layers.7.mlp.experts.55.gate_proj", "model.layers.7.mlp.experts.56.gate_proj", "model.layers.7.mlp.experts.57.gate_proj", "model.layers.7.mlp.experts.58.gate_proj", "model.layers.7.mlp.experts.59.gate_proj", "model.layers.7.mlp.experts.60.gate_proj", "model.layers.7.mlp.experts.61.gate_proj", "model.layers.7.mlp.experts.62.gate_proj", "model.layers.7.mlp.experts.63.gate_proj", "model.layers.7.mlp.experts.64.gate_proj", "model.layers.7.mlp.experts.65.gate_proj", "model.layers.7.mlp.experts.66.gate_proj", "model.layers.7.mlp.experts.67.gate_proj", "model.layers.7.mlp.experts.68.gate_proj", "model.layers.7.mlp.experts.69.gate_proj", "model.layers.7.mlp.experts.70.gate_proj", "model.layers.7.mlp.experts.71.gate_proj", "model.layers.7.mlp.experts.72.gate_proj", "model.layers.7.mlp.experts.73.gate_proj", "model.layers.7.mlp.experts.74.gate_proj", "model.layers.7.mlp.experts.75.gate_proj", "model.layers.7.mlp.experts.76.gate_proj", "model.layers.7.mlp.experts.77.gate_proj", "model.layers.7.mlp.experts.78.gate_proj", "model.layers.7.mlp.experts.79.gate_proj", "model.layers.7.mlp.experts.80.gate_proj", "model.layers.7.mlp.experts.81.gate_proj", "model.layers.7.mlp.experts.82.gate_proj", "model.layers.7.mlp.experts.83.gate_proj", "model.layers.7.mlp.experts.84.gate_proj", "model.layers.7.mlp.experts.85.gate_proj", "model.layers.7.mlp.experts.86.gate_proj", "model.layers.7.mlp.experts.87.gate_proj", "model.layers.7.mlp.experts.88.gate_proj", "model.layers.7.mlp.experts.89.gate_proj", "model.layers.7.mlp.experts.90.gate_proj", "model.layers.7.mlp.experts.91.gate_proj", "model.layers.7.mlp.experts.92.gate_proj", "model.layers.7.mlp.experts.93.gate_proj", "model.layers.7.mlp.experts.94.gate_proj", "model.layers.7.mlp.experts.95.gate_proj", "model.layers.7.mlp.experts.96.gate_proj", "model.layers.7.mlp.experts.97.gate_proj", "model.layers.7.mlp.experts.98.gate_proj", "model.layers.7.mlp.experts.99.gate_proj", "model.layers.7.mlp.experts.100.gate_proj", "model.layers.7.mlp.experts.101.gate_proj", "model.layers.7.mlp.experts.102.gate_proj", "model.layers.7.mlp.experts.103.gate_proj", "model.layers.7.mlp.experts.104.gate_proj", "model.layers.7.mlp.experts.105.gate_proj", "model.layers.7.mlp.experts.106.gate_proj", "model.layers.7.mlp.experts.107.gate_proj", "model.layers.7.mlp.experts.108.gate_proj", "model.layers.7.mlp.experts.109.gate_proj", "model.layers.7.mlp.experts.110.gate_proj", "model.layers.7.mlp.experts.111.gate_proj", "model.layers.7.mlp.experts.112.gate_proj", "model.layers.7.mlp.experts.113.gate_proj", "model.layers.7.mlp.experts.114.gate_proj", "model.layers.7.mlp.experts.115.gate_proj", "model.layers.7.mlp.experts.116.gate_proj", "model.layers.7.mlp.experts.117.gate_proj", "model.layers.7.mlp.experts.118.gate_proj", "model.layers.7.mlp.experts.119.gate_proj", "model.layers.7.mlp.experts.120.gate_proj", "model.layers.7.mlp.experts.121.gate_proj", "model.layers.7.mlp.experts.122.gate_proj", "model.layers.7.mlp.experts.123.gate_proj", "model.layers.7.mlp.experts.124.gate_proj", "model.layers.7.mlp.experts.125.gate_proj", "model.layers.7.mlp.experts.126.gate_proj", "model.layers.7.mlp.experts.127.gate_proj", "model.layers.7.mlp.experts.0.up_proj", "model.layers.7.mlp.experts.1.up_proj", "model.layers.7.mlp.experts.2.up_proj", "model.layers.7.mlp.experts.3.up_proj", "model.layers.7.mlp.experts.4.up_proj", "model.layers.7.mlp.experts.5.up_proj", "model.layers.7.mlp.experts.6.up_proj", "model.layers.7.mlp.experts.7.up_proj", "model.layers.7.mlp.experts.8.up_proj", "model.layers.7.mlp.experts.9.up_proj", "model.layers.7.mlp.experts.10.up_proj", "model.layers.7.mlp.experts.11.up_proj", "model.layers.7.mlp.experts.12.up_proj", "model.layers.7.mlp.experts.13.up_proj", "model.layers.7.mlp.experts.14.up_proj", "model.layers.7.mlp.experts.15.up_proj", "model.layers.7.mlp.experts.16.up_proj", "model.layers.7.mlp.experts.17.up_proj", "model.layers.7.mlp.experts.18.up_proj", "model.layers.7.mlp.experts.19.up_proj", "model.layers.7.mlp.experts.20.up_proj", "model.layers.7.mlp.experts.21.up_proj", "model.layers.7.mlp.experts.22.up_proj", "model.layers.7.mlp.experts.23.up_proj", "model.layers.7.mlp.experts.24.up_proj", "model.layers.7.mlp.experts.25.up_proj", "model.layers.7.mlp.experts.26.up_proj", "model.layers.7.mlp.experts.27.up_proj", "model.layers.7.mlp.experts.28.up_proj", "model.layers.7.mlp.experts.29.up_proj", "model.layers.7.mlp.experts.30.up_proj", "model.layers.7.mlp.experts.31.up_proj", "model.layers.7.mlp.experts.32.up_proj", "model.layers.7.mlp.experts.33.up_proj", "model.layers.7.mlp.experts.34.up_proj", "model.layers.7.mlp.experts.35.up_proj", "model.layers.7.mlp.experts.36.up_proj", "model.layers.7.mlp.experts.37.up_proj", "model.layers.7.mlp.experts.38.up_proj", "model.layers.7.mlp.experts.39.up_proj", "model.layers.7.mlp.experts.40.up_proj", "model.layers.7.mlp.experts.41.up_proj", "model.layers.7.mlp.experts.42.up_proj", "model.layers.7.mlp.experts.43.up_proj", "model.layers.7.mlp.experts.44.up_proj", "model.layers.7.mlp.experts.45.up_proj", "model.layers.7.mlp.experts.46.up_proj", "model.layers.7.mlp.experts.47.up_proj", "model.layers.7.mlp.experts.48.up_proj", "model.layers.7.mlp.experts.49.up_proj", "model.layers.7.mlp.experts.50.up_proj", "model.layers.7.mlp.experts.51.up_proj", "model.layers.7.mlp.experts.52.up_proj", "model.layers.7.mlp.experts.53.up_proj", "model.layers.7.mlp.experts.54.up_proj", "model.layers.7.mlp.experts.55.up_proj", "model.layers.7.mlp.experts.56.up_proj", "model.layers.7.mlp.experts.57.up_proj", "model.layers.7.mlp.experts.58.up_proj", "model.layers.7.mlp.experts.59.up_proj", "model.layers.7.mlp.experts.60.up_proj", "model.layers.7.mlp.experts.61.up_proj", "model.layers.7.mlp.experts.62.up_proj", "model.layers.7.mlp.experts.63.up_proj", "model.layers.7.mlp.experts.64.up_proj", "model.layers.7.mlp.experts.65.up_proj", "model.layers.7.mlp.experts.66.up_proj", "model.layers.7.mlp.experts.67.up_proj", "model.layers.7.mlp.experts.68.up_proj", "model.layers.7.mlp.experts.69.up_proj", "model.layers.7.mlp.experts.70.up_proj", "model.layers.7.mlp.experts.71.up_proj", "model.layers.7.mlp.experts.72.up_proj", "model.layers.7.mlp.experts.73.up_proj", "model.layers.7.mlp.experts.74.up_proj", "model.layers.7.mlp.experts.75.up_proj", "model.layers.7.mlp.experts.76.up_proj", "model.layers.7.mlp.experts.77.up_proj", "model.layers.7.mlp.experts.78.up_proj", "model.layers.7.mlp.experts.79.up_proj", "model.layers.7.mlp.experts.80.up_proj", "model.layers.7.mlp.experts.81.up_proj", "model.layers.7.mlp.experts.82.up_proj", "model.layers.7.mlp.experts.83.up_proj", "model.layers.7.mlp.experts.84.up_proj", "model.layers.7.mlp.experts.85.up_proj", "model.layers.7.mlp.experts.86.up_proj", "model.layers.7.mlp.experts.87.up_proj", "model.layers.7.mlp.experts.88.up_proj", "model.layers.7.mlp.experts.89.up_proj", "model.layers.7.mlp.experts.90.up_proj", "model.layers.7.mlp.experts.91.up_proj", "model.layers.7.mlp.experts.92.up_proj", "model.layers.7.mlp.experts.93.up_proj", "model.layers.7.mlp.experts.94.up_proj", "model.layers.7.mlp.experts.95.up_proj", "model.layers.7.mlp.experts.96.up_proj", "model.layers.7.mlp.experts.97.up_proj", "model.layers.7.mlp.experts.98.up_proj", "model.layers.7.mlp.experts.99.up_proj", "model.layers.7.mlp.experts.100.up_proj", "model.layers.7.mlp.experts.101.up_proj", "model.layers.7.mlp.experts.102.up_proj", "model.layers.7.mlp.experts.103.up_proj", "model.layers.7.mlp.experts.104.up_proj", "model.layers.7.mlp.experts.105.up_proj", "model.layers.7.mlp.experts.106.up_proj", "model.layers.7.mlp.experts.107.up_proj", "model.layers.7.mlp.experts.108.up_proj", "model.layers.7.mlp.experts.109.up_proj", "model.layers.7.mlp.experts.110.up_proj", "model.layers.7.mlp.experts.111.up_proj", "model.layers.7.mlp.experts.112.up_proj", "model.layers.7.mlp.experts.113.up_proj", "model.layers.7.mlp.experts.114.up_proj", "model.layers.7.mlp.experts.115.up_proj", "model.layers.7.mlp.experts.116.up_proj", "model.layers.7.mlp.experts.117.up_proj", "model.layers.7.mlp.experts.118.up_proj", "model.layers.7.mlp.experts.119.up_proj", "model.layers.7.mlp.experts.120.up_proj", "model.layers.7.mlp.experts.121.up_proj", "model.layers.7.mlp.experts.122.up_proj", "model.layers.7.mlp.experts.123.up_proj", "model.layers.7.mlp.experts.124.up_proj", "model.layers.7.mlp.experts.125.up_proj", "model.layers.7.mlp.experts.126.up_proj", "model.layers.7.mlp.experts.127.up_proj", "model.layers.7.mlp.experts.0.down_proj", "model.layers.7.mlp.experts.1.down_proj", "model.layers.7.mlp.experts.2.down_proj", "model.layers.7.mlp.experts.3.down_proj", "model.layers.7.mlp.experts.4.down_proj", "model.layers.7.mlp.experts.5.down_proj", "model.layers.7.mlp.experts.6.down_proj", "model.layers.7.mlp.experts.7.down_proj", "model.layers.7.mlp.experts.8.down_proj", "model.layers.7.mlp.experts.9.down_proj", "model.layers.7.mlp.experts.10.down_proj", "model.layers.7.mlp.experts.11.down_proj", "model.layers.7.mlp.experts.12.down_proj", "model.layers.7.mlp.experts.13.down_proj", "model.layers.7.mlp.experts.14.down_proj", "model.layers.7.mlp.experts.15.down_proj", "model.layers.7.mlp.experts.16.down_proj", "model.layers.7.mlp.experts.17.down_proj", "model.layers.7.mlp.experts.18.down_proj", "model.layers.7.mlp.experts.19.down_proj", "model.layers.7.mlp.experts.20.down_proj", "model.layers.7.mlp.experts.21.down_proj", "model.layers.7.mlp.experts.22.down_proj", "model.layers.7.mlp.experts.23.down_proj", "model.layers.7.mlp.experts.24.down_proj", "model.layers.7.mlp.experts.25.down_proj", "model.layers.7.mlp.experts.26.down_proj", "model.layers.7.mlp.experts.27.down_proj", "model.layers.7.mlp.experts.28.down_proj", "model.layers.7.mlp.experts.29.down_proj", "model.layers.7.mlp.experts.30.down_proj", "model.layers.7.mlp.experts.31.down_proj", "model.layers.7.mlp.experts.32.down_proj", "model.layers.7.mlp.experts.33.down_proj", "model.layers.7.mlp.experts.34.down_proj", "model.layers.7.mlp.experts.35.down_proj", "model.layers.7.mlp.experts.36.down_proj", "model.layers.7.mlp.experts.37.down_proj", "model.layers.7.mlp.experts.38.down_proj", "model.layers.7.mlp.experts.39.down_proj", "model.layers.7.mlp.experts.40.down_proj", "model.layers.7.mlp.experts.41.down_proj", "model.layers.7.mlp.experts.42.down_proj", "model.layers.7.mlp.experts.43.down_proj", "model.layers.7.mlp.experts.44.down_proj", "model.layers.7.mlp.experts.45.down_proj", "model.layers.7.mlp.experts.46.down_proj", "model.layers.7.mlp.experts.47.down_proj", "model.layers.7.mlp.experts.48.down_proj", "model.layers.7.mlp.experts.49.down_proj", "model.layers.7.mlp.experts.50.down_proj", "model.layers.7.mlp.experts.51.down_proj", "model.layers.7.mlp.experts.52.down_proj", "model.layers.7.mlp.experts.53.down_proj", "model.layers.7.mlp.experts.54.down_proj", "model.layers.7.mlp.experts.55.down_proj", "model.layers.7.mlp.experts.56.down_proj", "model.layers.7.mlp.experts.57.down_proj", "model.layers.7.mlp.experts.58.down_proj", "model.layers.7.mlp.experts.59.down_proj", "model.layers.7.mlp.experts.60.down_proj", "model.layers.7.mlp.experts.61.down_proj", "model.layers.7.mlp.experts.62.down_proj", "model.layers.7.mlp.experts.63.down_proj", "model.layers.7.mlp.experts.64.down_proj", "model.layers.7.mlp.experts.65.down_proj", "model.layers.7.mlp.experts.66.down_proj", "model.layers.7.mlp.experts.67.down_proj", "model.layers.7.mlp.experts.68.down_proj", "model.layers.7.mlp.experts.69.down_proj", "model.layers.7.mlp.experts.70.down_proj", "model.layers.7.mlp.experts.71.down_proj", "model.layers.7.mlp.experts.72.down_proj", "model.layers.7.mlp.experts.73.down_proj", "model.layers.7.mlp.experts.74.down_proj", "model.layers.7.mlp.experts.75.down_proj", "model.layers.7.mlp.experts.76.down_proj", "model.layers.7.mlp.experts.77.down_proj", "model.layers.7.mlp.experts.78.down_proj", "model.layers.7.mlp.experts.79.down_proj", "model.layers.7.mlp.experts.80.down_proj", "model.layers.7.mlp.experts.81.down_proj", "model.layers.7.mlp.experts.82.down_proj", "model.layers.7.mlp.experts.83.down_proj", "model.layers.7.mlp.experts.84.down_proj", "model.layers.7.mlp.experts.85.down_proj", "model.layers.7.mlp.experts.86.down_proj", "model.layers.7.mlp.experts.87.down_proj", "model.layers.7.mlp.experts.88.down_proj", "model.layers.7.mlp.experts.89.down_proj", "model.layers.7.mlp.experts.90.down_proj", "model.layers.7.mlp.experts.91.down_proj", "model.layers.7.mlp.experts.92.down_proj", "model.layers.7.mlp.experts.93.down_proj", "model.layers.7.mlp.experts.94.down_proj", "model.layers.7.mlp.experts.95.down_proj", "model.layers.7.mlp.experts.96.down_proj", "model.layers.7.mlp.experts.97.down_proj", "model.layers.7.mlp.experts.98.down_proj", "model.layers.7.mlp.experts.99.down_proj", "model.layers.7.mlp.experts.100.down_proj", "model.layers.7.mlp.experts.101.down_proj", "model.layers.7.mlp.experts.102.down_proj", "model.layers.7.mlp.experts.103.down_proj", "model.layers.7.mlp.experts.104.down_proj", "model.layers.7.mlp.experts.105.down_proj", "model.layers.7.mlp.experts.106.down_proj", "model.layers.7.mlp.experts.107.down_proj", "model.layers.7.mlp.experts.108.down_proj", "model.layers.7.mlp.experts.109.down_proj", "model.layers.7.mlp.experts.110.down_proj", "model.layers.7.mlp.experts.111.down_proj", "model.layers.7.mlp.experts.112.down_proj", "model.layers.7.mlp.experts.113.down_proj", "model.layers.7.mlp.experts.114.down_proj", "model.layers.7.mlp.experts.115.down_proj", "model.layers.7.mlp.experts.116.down_proj", "model.layers.7.mlp.experts.117.down_proj", "model.layers.7.mlp.experts.118.down_proj", "model.layers.7.mlp.experts.119.down_proj", "model.layers.7.mlp.experts.120.down_proj", "model.layers.7.mlp.experts.121.down_proj", "model.layers.7.mlp.experts.122.down_proj", "model.layers.7.mlp.experts.123.down_proj", "model.layers.7.mlp.experts.124.down_proj", "model.layers.7.mlp.experts.125.down_proj", "model.layers.7.mlp.experts.126.down_proj", "model.layers.7.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.004972417093813408, "dbits": 2214592512 } ] }, { "idx": 23, "layers": [ "model.layers.8.self_attn.q_proj", "model.layers.8.self_attn.k_proj", "model.layers.8.self_attn.v_proj", "model.layers.8.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0001503962092101463, "dbits": 109051904 } ] }, { "idx": 24, "layers": [ "model.layers.8.mlp.shared_experts.gate_proj", "model.layers.8.mlp.shared_experts.up_proj", "model.layers.8.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.006060685776174052, "dbits": 17301504 } ] }, { "idx": 25, "layers": [ "model.layers.8.mlp.experts.0.gate_proj", "model.layers.8.mlp.experts.1.gate_proj", "model.layers.8.mlp.experts.2.gate_proj", "model.layers.8.mlp.experts.3.gate_proj", "model.layers.8.mlp.experts.4.gate_proj", "model.layers.8.mlp.experts.5.gate_proj", "model.layers.8.mlp.experts.6.gate_proj", "model.layers.8.mlp.experts.7.gate_proj", "model.layers.8.mlp.experts.8.gate_proj", "model.layers.8.mlp.experts.9.gate_proj", "model.layers.8.mlp.experts.10.gate_proj", "model.layers.8.mlp.experts.11.gate_proj", "model.layers.8.mlp.experts.12.gate_proj", "model.layers.8.mlp.experts.13.gate_proj", "model.layers.8.mlp.experts.14.gate_proj", "model.layers.8.mlp.experts.15.gate_proj", "model.layers.8.mlp.experts.16.gate_proj", "model.layers.8.mlp.experts.17.gate_proj", "model.layers.8.mlp.experts.18.gate_proj", "model.layers.8.mlp.experts.19.gate_proj", "model.layers.8.mlp.experts.20.gate_proj", "model.layers.8.mlp.experts.21.gate_proj", "model.layers.8.mlp.experts.22.gate_proj", "model.layers.8.mlp.experts.23.gate_proj", "model.layers.8.mlp.experts.24.gate_proj", "model.layers.8.mlp.experts.25.gate_proj", "model.layers.8.mlp.experts.26.gate_proj", "model.layers.8.mlp.experts.27.gate_proj", "model.layers.8.mlp.experts.28.gate_proj", "model.layers.8.mlp.experts.29.gate_proj", "model.layers.8.mlp.experts.30.gate_proj", "model.layers.8.mlp.experts.31.gate_proj", "model.layers.8.mlp.experts.32.gate_proj", "model.layers.8.mlp.experts.33.gate_proj", "model.layers.8.mlp.experts.34.gate_proj", "model.layers.8.mlp.experts.35.gate_proj", "model.layers.8.mlp.experts.36.gate_proj", "model.layers.8.mlp.experts.37.gate_proj", "model.layers.8.mlp.experts.38.gate_proj", "model.layers.8.mlp.experts.39.gate_proj", "model.layers.8.mlp.experts.40.gate_proj", "model.layers.8.mlp.experts.41.gate_proj", "model.layers.8.mlp.experts.42.gate_proj", "model.layers.8.mlp.experts.43.gate_proj", "model.layers.8.mlp.experts.44.gate_proj", "model.layers.8.mlp.experts.45.gate_proj", "model.layers.8.mlp.experts.46.gate_proj", "model.layers.8.mlp.experts.47.gate_proj", "model.layers.8.mlp.experts.48.gate_proj", "model.layers.8.mlp.experts.49.gate_proj", "model.layers.8.mlp.experts.50.gate_proj", "model.layers.8.mlp.experts.51.gate_proj", "model.layers.8.mlp.experts.52.gate_proj", "model.layers.8.mlp.experts.53.gate_proj", "model.layers.8.mlp.experts.54.gate_proj", "model.layers.8.mlp.experts.55.gate_proj", "model.layers.8.mlp.experts.56.gate_proj", "model.layers.8.mlp.experts.57.gate_proj", "model.layers.8.mlp.experts.58.gate_proj", "model.layers.8.mlp.experts.59.gate_proj", "model.layers.8.mlp.experts.60.gate_proj", "model.layers.8.mlp.experts.61.gate_proj", "model.layers.8.mlp.experts.62.gate_proj", "model.layers.8.mlp.experts.63.gate_proj", "model.layers.8.mlp.experts.64.gate_proj", "model.layers.8.mlp.experts.65.gate_proj", "model.layers.8.mlp.experts.66.gate_proj", "model.layers.8.mlp.experts.67.gate_proj", "model.layers.8.mlp.experts.68.gate_proj", "model.layers.8.mlp.experts.69.gate_proj", "model.layers.8.mlp.experts.70.gate_proj", "model.layers.8.mlp.experts.71.gate_proj", "model.layers.8.mlp.experts.72.gate_proj", "model.layers.8.mlp.experts.73.gate_proj", "model.layers.8.mlp.experts.74.gate_proj", "model.layers.8.mlp.experts.75.gate_proj", "model.layers.8.mlp.experts.76.gate_proj", "model.layers.8.mlp.experts.77.gate_proj", "model.layers.8.mlp.experts.78.gate_proj", "model.layers.8.mlp.experts.79.gate_proj", "model.layers.8.mlp.experts.80.gate_proj", "model.layers.8.mlp.experts.81.gate_proj", "model.layers.8.mlp.experts.82.gate_proj", "model.layers.8.mlp.experts.83.gate_proj", "model.layers.8.mlp.experts.84.gate_proj", "model.layers.8.mlp.experts.85.gate_proj", "model.layers.8.mlp.experts.86.gate_proj", "model.layers.8.mlp.experts.87.gate_proj", "model.layers.8.mlp.experts.88.gate_proj", "model.layers.8.mlp.experts.89.gate_proj", "model.layers.8.mlp.experts.90.gate_proj", "model.layers.8.mlp.experts.91.gate_proj", "model.layers.8.mlp.experts.92.gate_proj", "model.layers.8.mlp.experts.93.gate_proj", "model.layers.8.mlp.experts.94.gate_proj", "model.layers.8.mlp.experts.95.gate_proj", "model.layers.8.mlp.experts.96.gate_proj", "model.layers.8.mlp.experts.97.gate_proj", "model.layers.8.mlp.experts.98.gate_proj", "model.layers.8.mlp.experts.99.gate_proj", "model.layers.8.mlp.experts.100.gate_proj", "model.layers.8.mlp.experts.101.gate_proj", "model.layers.8.mlp.experts.102.gate_proj", "model.layers.8.mlp.experts.103.gate_proj", "model.layers.8.mlp.experts.104.gate_proj", "model.layers.8.mlp.experts.105.gate_proj", "model.layers.8.mlp.experts.106.gate_proj", "model.layers.8.mlp.experts.107.gate_proj", "model.layers.8.mlp.experts.108.gate_proj", "model.layers.8.mlp.experts.109.gate_proj", "model.layers.8.mlp.experts.110.gate_proj", "model.layers.8.mlp.experts.111.gate_proj", "model.layers.8.mlp.experts.112.gate_proj", "model.layers.8.mlp.experts.113.gate_proj", "model.layers.8.mlp.experts.114.gate_proj", "model.layers.8.mlp.experts.115.gate_proj", "model.layers.8.mlp.experts.116.gate_proj", "model.layers.8.mlp.experts.117.gate_proj", "model.layers.8.mlp.experts.118.gate_proj", "model.layers.8.mlp.experts.119.gate_proj", "model.layers.8.mlp.experts.120.gate_proj", "model.layers.8.mlp.experts.121.gate_proj", "model.layers.8.mlp.experts.122.gate_proj", "model.layers.8.mlp.experts.123.gate_proj", "model.layers.8.mlp.experts.124.gate_proj", "model.layers.8.mlp.experts.125.gate_proj", "model.layers.8.mlp.experts.126.gate_proj", "model.layers.8.mlp.experts.127.gate_proj", "model.layers.8.mlp.experts.0.up_proj", "model.layers.8.mlp.experts.1.up_proj", "model.layers.8.mlp.experts.2.up_proj", "model.layers.8.mlp.experts.3.up_proj", "model.layers.8.mlp.experts.4.up_proj", "model.layers.8.mlp.experts.5.up_proj", "model.layers.8.mlp.experts.6.up_proj", "model.layers.8.mlp.experts.7.up_proj", "model.layers.8.mlp.experts.8.up_proj", "model.layers.8.mlp.experts.9.up_proj", "model.layers.8.mlp.experts.10.up_proj", "model.layers.8.mlp.experts.11.up_proj", "model.layers.8.mlp.experts.12.up_proj", "model.layers.8.mlp.experts.13.up_proj", "model.layers.8.mlp.experts.14.up_proj", "model.layers.8.mlp.experts.15.up_proj", "model.layers.8.mlp.experts.16.up_proj", "model.layers.8.mlp.experts.17.up_proj", "model.layers.8.mlp.experts.18.up_proj", "model.layers.8.mlp.experts.19.up_proj", "model.layers.8.mlp.experts.20.up_proj", "model.layers.8.mlp.experts.21.up_proj", "model.layers.8.mlp.experts.22.up_proj", "model.layers.8.mlp.experts.23.up_proj", "model.layers.8.mlp.experts.24.up_proj", "model.layers.8.mlp.experts.25.up_proj", "model.layers.8.mlp.experts.26.up_proj", "model.layers.8.mlp.experts.27.up_proj", "model.layers.8.mlp.experts.28.up_proj", "model.layers.8.mlp.experts.29.up_proj", "model.layers.8.mlp.experts.30.up_proj", "model.layers.8.mlp.experts.31.up_proj", "model.layers.8.mlp.experts.32.up_proj", "model.layers.8.mlp.experts.33.up_proj", "model.layers.8.mlp.experts.34.up_proj", "model.layers.8.mlp.experts.35.up_proj", "model.layers.8.mlp.experts.36.up_proj", "model.layers.8.mlp.experts.37.up_proj", "model.layers.8.mlp.experts.38.up_proj", "model.layers.8.mlp.experts.39.up_proj", "model.layers.8.mlp.experts.40.up_proj", "model.layers.8.mlp.experts.41.up_proj", "model.layers.8.mlp.experts.42.up_proj", "model.layers.8.mlp.experts.43.up_proj", "model.layers.8.mlp.experts.44.up_proj", "model.layers.8.mlp.experts.45.up_proj", "model.layers.8.mlp.experts.46.up_proj", "model.layers.8.mlp.experts.47.up_proj", "model.layers.8.mlp.experts.48.up_proj", "model.layers.8.mlp.experts.49.up_proj", "model.layers.8.mlp.experts.50.up_proj", "model.layers.8.mlp.experts.51.up_proj", "model.layers.8.mlp.experts.52.up_proj", "model.layers.8.mlp.experts.53.up_proj", "model.layers.8.mlp.experts.54.up_proj", "model.layers.8.mlp.experts.55.up_proj", "model.layers.8.mlp.experts.56.up_proj", "model.layers.8.mlp.experts.57.up_proj", "model.layers.8.mlp.experts.58.up_proj", "model.layers.8.mlp.experts.59.up_proj", "model.layers.8.mlp.experts.60.up_proj", "model.layers.8.mlp.experts.61.up_proj", "model.layers.8.mlp.experts.62.up_proj", "model.layers.8.mlp.experts.63.up_proj", "model.layers.8.mlp.experts.64.up_proj", "model.layers.8.mlp.experts.65.up_proj", "model.layers.8.mlp.experts.66.up_proj", "model.layers.8.mlp.experts.67.up_proj", "model.layers.8.mlp.experts.68.up_proj", "model.layers.8.mlp.experts.69.up_proj", "model.layers.8.mlp.experts.70.up_proj", "model.layers.8.mlp.experts.71.up_proj", "model.layers.8.mlp.experts.72.up_proj", "model.layers.8.mlp.experts.73.up_proj", "model.layers.8.mlp.experts.74.up_proj", "model.layers.8.mlp.experts.75.up_proj", "model.layers.8.mlp.experts.76.up_proj", "model.layers.8.mlp.experts.77.up_proj", "model.layers.8.mlp.experts.78.up_proj", "model.layers.8.mlp.experts.79.up_proj", "model.layers.8.mlp.experts.80.up_proj", "model.layers.8.mlp.experts.81.up_proj", "model.layers.8.mlp.experts.82.up_proj", "model.layers.8.mlp.experts.83.up_proj", "model.layers.8.mlp.experts.84.up_proj", "model.layers.8.mlp.experts.85.up_proj", "model.layers.8.mlp.experts.86.up_proj", "model.layers.8.mlp.experts.87.up_proj", "model.layers.8.mlp.experts.88.up_proj", "model.layers.8.mlp.experts.89.up_proj", "model.layers.8.mlp.experts.90.up_proj", "model.layers.8.mlp.experts.91.up_proj", "model.layers.8.mlp.experts.92.up_proj", "model.layers.8.mlp.experts.93.up_proj", "model.layers.8.mlp.experts.94.up_proj", "model.layers.8.mlp.experts.95.up_proj", "model.layers.8.mlp.experts.96.up_proj", "model.layers.8.mlp.experts.97.up_proj", "model.layers.8.mlp.experts.98.up_proj", "model.layers.8.mlp.experts.99.up_proj", "model.layers.8.mlp.experts.100.up_proj", "model.layers.8.mlp.experts.101.up_proj", "model.layers.8.mlp.experts.102.up_proj", "model.layers.8.mlp.experts.103.up_proj", "model.layers.8.mlp.experts.104.up_proj", "model.layers.8.mlp.experts.105.up_proj", "model.layers.8.mlp.experts.106.up_proj", "model.layers.8.mlp.experts.107.up_proj", "model.layers.8.mlp.experts.108.up_proj", "model.layers.8.mlp.experts.109.up_proj", "model.layers.8.mlp.experts.110.up_proj", "model.layers.8.mlp.experts.111.up_proj", "model.layers.8.mlp.experts.112.up_proj", "model.layers.8.mlp.experts.113.up_proj", "model.layers.8.mlp.experts.114.up_proj", "model.layers.8.mlp.experts.115.up_proj", "model.layers.8.mlp.experts.116.up_proj", "model.layers.8.mlp.experts.117.up_proj", "model.layers.8.mlp.experts.118.up_proj", "model.layers.8.mlp.experts.119.up_proj", "model.layers.8.mlp.experts.120.up_proj", "model.layers.8.mlp.experts.121.up_proj", "model.layers.8.mlp.experts.122.up_proj", "model.layers.8.mlp.experts.123.up_proj", "model.layers.8.mlp.experts.124.up_proj", "model.layers.8.mlp.experts.125.up_proj", "model.layers.8.mlp.experts.126.up_proj", "model.layers.8.mlp.experts.127.up_proj", "model.layers.8.mlp.experts.0.down_proj", "model.layers.8.mlp.experts.1.down_proj", "model.layers.8.mlp.experts.2.down_proj", "model.layers.8.mlp.experts.3.down_proj", "model.layers.8.mlp.experts.4.down_proj", "model.layers.8.mlp.experts.5.down_proj", "model.layers.8.mlp.experts.6.down_proj", "model.layers.8.mlp.experts.7.down_proj", "model.layers.8.mlp.experts.8.down_proj", "model.layers.8.mlp.experts.9.down_proj", "model.layers.8.mlp.experts.10.down_proj", "model.layers.8.mlp.experts.11.down_proj", "model.layers.8.mlp.experts.12.down_proj", "model.layers.8.mlp.experts.13.down_proj", "model.layers.8.mlp.experts.14.down_proj", "model.layers.8.mlp.experts.15.down_proj", "model.layers.8.mlp.experts.16.down_proj", "model.layers.8.mlp.experts.17.down_proj", "model.layers.8.mlp.experts.18.down_proj", "model.layers.8.mlp.experts.19.down_proj", "model.layers.8.mlp.experts.20.down_proj", "model.layers.8.mlp.experts.21.down_proj", "model.layers.8.mlp.experts.22.down_proj", "model.layers.8.mlp.experts.23.down_proj", "model.layers.8.mlp.experts.24.down_proj", "model.layers.8.mlp.experts.25.down_proj", "model.layers.8.mlp.experts.26.down_proj", "model.layers.8.mlp.experts.27.down_proj", "model.layers.8.mlp.experts.28.down_proj", "model.layers.8.mlp.experts.29.down_proj", "model.layers.8.mlp.experts.30.down_proj", "model.layers.8.mlp.experts.31.down_proj", "model.layers.8.mlp.experts.32.down_proj", "model.layers.8.mlp.experts.33.down_proj", "model.layers.8.mlp.experts.34.down_proj", "model.layers.8.mlp.experts.35.down_proj", "model.layers.8.mlp.experts.36.down_proj", "model.layers.8.mlp.experts.37.down_proj", "model.layers.8.mlp.experts.38.down_proj", "model.layers.8.mlp.experts.39.down_proj", "model.layers.8.mlp.experts.40.down_proj", "model.layers.8.mlp.experts.41.down_proj", "model.layers.8.mlp.experts.42.down_proj", "model.layers.8.mlp.experts.43.down_proj", "model.layers.8.mlp.experts.44.down_proj", "model.layers.8.mlp.experts.45.down_proj", "model.layers.8.mlp.experts.46.down_proj", "model.layers.8.mlp.experts.47.down_proj", "model.layers.8.mlp.experts.48.down_proj", "model.layers.8.mlp.experts.49.down_proj", "model.layers.8.mlp.experts.50.down_proj", "model.layers.8.mlp.experts.51.down_proj", "model.layers.8.mlp.experts.52.down_proj", "model.layers.8.mlp.experts.53.down_proj", "model.layers.8.mlp.experts.54.down_proj", "model.layers.8.mlp.experts.55.down_proj", "model.layers.8.mlp.experts.56.down_proj", "model.layers.8.mlp.experts.57.down_proj", "model.layers.8.mlp.experts.58.down_proj", "model.layers.8.mlp.experts.59.down_proj", "model.layers.8.mlp.experts.60.down_proj", "model.layers.8.mlp.experts.61.down_proj", "model.layers.8.mlp.experts.62.down_proj", "model.layers.8.mlp.experts.63.down_proj", "model.layers.8.mlp.experts.64.down_proj", "model.layers.8.mlp.experts.65.down_proj", "model.layers.8.mlp.experts.66.down_proj", "model.layers.8.mlp.experts.67.down_proj", "model.layers.8.mlp.experts.68.down_proj", "model.layers.8.mlp.experts.69.down_proj", "model.layers.8.mlp.experts.70.down_proj", "model.layers.8.mlp.experts.71.down_proj", "model.layers.8.mlp.experts.72.down_proj", "model.layers.8.mlp.experts.73.down_proj", "model.layers.8.mlp.experts.74.down_proj", "model.layers.8.mlp.experts.75.down_proj", "model.layers.8.mlp.experts.76.down_proj", "model.layers.8.mlp.experts.77.down_proj", "model.layers.8.mlp.experts.78.down_proj", "model.layers.8.mlp.experts.79.down_proj", "model.layers.8.mlp.experts.80.down_proj", "model.layers.8.mlp.experts.81.down_proj", "model.layers.8.mlp.experts.82.down_proj", "model.layers.8.mlp.experts.83.down_proj", "model.layers.8.mlp.experts.84.down_proj", "model.layers.8.mlp.experts.85.down_proj", "model.layers.8.mlp.experts.86.down_proj", "model.layers.8.mlp.experts.87.down_proj", "model.layers.8.mlp.experts.88.down_proj", "model.layers.8.mlp.experts.89.down_proj", "model.layers.8.mlp.experts.90.down_proj", "model.layers.8.mlp.experts.91.down_proj", "model.layers.8.mlp.experts.92.down_proj", "model.layers.8.mlp.experts.93.down_proj", "model.layers.8.mlp.experts.94.down_proj", "model.layers.8.mlp.experts.95.down_proj", "model.layers.8.mlp.experts.96.down_proj", "model.layers.8.mlp.experts.97.down_proj", "model.layers.8.mlp.experts.98.down_proj", "model.layers.8.mlp.experts.99.down_proj", "model.layers.8.mlp.experts.100.down_proj", "model.layers.8.mlp.experts.101.down_proj", "model.layers.8.mlp.experts.102.down_proj", "model.layers.8.mlp.experts.103.down_proj", "model.layers.8.mlp.experts.104.down_proj", "model.layers.8.mlp.experts.105.down_proj", "model.layers.8.mlp.experts.106.down_proj", "model.layers.8.mlp.experts.107.down_proj", "model.layers.8.mlp.experts.108.down_proj", "model.layers.8.mlp.experts.109.down_proj", "model.layers.8.mlp.experts.110.down_proj", "model.layers.8.mlp.experts.111.down_proj", "model.layers.8.mlp.experts.112.down_proj", "model.layers.8.mlp.experts.113.down_proj", "model.layers.8.mlp.experts.114.down_proj", "model.layers.8.mlp.experts.115.down_proj", "model.layers.8.mlp.experts.116.down_proj", "model.layers.8.mlp.experts.117.down_proj", "model.layers.8.mlp.experts.118.down_proj", "model.layers.8.mlp.experts.119.down_proj", "model.layers.8.mlp.experts.120.down_proj", "model.layers.8.mlp.experts.121.down_proj", "model.layers.8.mlp.experts.122.down_proj", "model.layers.8.mlp.experts.123.down_proj", "model.layers.8.mlp.experts.124.down_proj", "model.layers.8.mlp.experts.125.down_proj", "model.layers.8.mlp.experts.126.down_proj", "model.layers.8.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.01400523418560623, "dbits": 2214592512 } ] }, { "idx": 26, "layers": [ "model.layers.9.self_attn.q_proj", "model.layers.9.self_attn.k_proj", "model.layers.9.self_attn.v_proj", "model.layers.9.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0002767906524241137, "dbits": 109051904 } ] }, { "idx": 27, "layers": [ "model.layers.9.mlp.shared_experts.gate_proj", "model.layers.9.mlp.shared_experts.up_proj", "model.layers.9.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0039191533811390455, "dbits": 17301504 } ] }, { "idx": 28, "layers": [ "model.layers.9.mlp.experts.0.gate_proj", "model.layers.9.mlp.experts.1.gate_proj", "model.layers.9.mlp.experts.2.gate_proj", "model.layers.9.mlp.experts.3.gate_proj", "model.layers.9.mlp.experts.4.gate_proj", "model.layers.9.mlp.experts.5.gate_proj", "model.layers.9.mlp.experts.6.gate_proj", "model.layers.9.mlp.experts.7.gate_proj", "model.layers.9.mlp.experts.8.gate_proj", "model.layers.9.mlp.experts.9.gate_proj", "model.layers.9.mlp.experts.10.gate_proj", "model.layers.9.mlp.experts.11.gate_proj", "model.layers.9.mlp.experts.12.gate_proj", "model.layers.9.mlp.experts.13.gate_proj", "model.layers.9.mlp.experts.14.gate_proj", "model.layers.9.mlp.experts.15.gate_proj", "model.layers.9.mlp.experts.16.gate_proj", "model.layers.9.mlp.experts.17.gate_proj", "model.layers.9.mlp.experts.18.gate_proj", "model.layers.9.mlp.experts.19.gate_proj", "model.layers.9.mlp.experts.20.gate_proj", "model.layers.9.mlp.experts.21.gate_proj", "model.layers.9.mlp.experts.22.gate_proj", "model.layers.9.mlp.experts.23.gate_proj", "model.layers.9.mlp.experts.24.gate_proj", "model.layers.9.mlp.experts.25.gate_proj", "model.layers.9.mlp.experts.26.gate_proj", "model.layers.9.mlp.experts.27.gate_proj", "model.layers.9.mlp.experts.28.gate_proj", "model.layers.9.mlp.experts.29.gate_proj", "model.layers.9.mlp.experts.30.gate_proj", "model.layers.9.mlp.experts.31.gate_proj", "model.layers.9.mlp.experts.32.gate_proj", "model.layers.9.mlp.experts.33.gate_proj", "model.layers.9.mlp.experts.34.gate_proj", "model.layers.9.mlp.experts.35.gate_proj", "model.layers.9.mlp.experts.36.gate_proj", "model.layers.9.mlp.experts.37.gate_proj", "model.layers.9.mlp.experts.38.gate_proj", "model.layers.9.mlp.experts.39.gate_proj", "model.layers.9.mlp.experts.40.gate_proj", "model.layers.9.mlp.experts.41.gate_proj", "model.layers.9.mlp.experts.42.gate_proj", "model.layers.9.mlp.experts.43.gate_proj", "model.layers.9.mlp.experts.44.gate_proj", "model.layers.9.mlp.experts.45.gate_proj", "model.layers.9.mlp.experts.46.gate_proj", "model.layers.9.mlp.experts.47.gate_proj", "model.layers.9.mlp.experts.48.gate_proj", "model.layers.9.mlp.experts.49.gate_proj", "model.layers.9.mlp.experts.50.gate_proj", "model.layers.9.mlp.experts.51.gate_proj", "model.layers.9.mlp.experts.52.gate_proj", "model.layers.9.mlp.experts.53.gate_proj", "model.layers.9.mlp.experts.54.gate_proj", "model.layers.9.mlp.experts.55.gate_proj", "model.layers.9.mlp.experts.56.gate_proj", "model.layers.9.mlp.experts.57.gate_proj", "model.layers.9.mlp.experts.58.gate_proj", "model.layers.9.mlp.experts.59.gate_proj", "model.layers.9.mlp.experts.60.gate_proj", "model.layers.9.mlp.experts.61.gate_proj", "model.layers.9.mlp.experts.62.gate_proj", "model.layers.9.mlp.experts.63.gate_proj", "model.layers.9.mlp.experts.64.gate_proj", "model.layers.9.mlp.experts.65.gate_proj", "model.layers.9.mlp.experts.66.gate_proj", "model.layers.9.mlp.experts.67.gate_proj", "model.layers.9.mlp.experts.68.gate_proj", "model.layers.9.mlp.experts.69.gate_proj", "model.layers.9.mlp.experts.70.gate_proj", "model.layers.9.mlp.experts.71.gate_proj", "model.layers.9.mlp.experts.72.gate_proj", "model.layers.9.mlp.experts.73.gate_proj", "model.layers.9.mlp.experts.74.gate_proj", "model.layers.9.mlp.experts.75.gate_proj", "model.layers.9.mlp.experts.76.gate_proj", "model.layers.9.mlp.experts.77.gate_proj", "model.layers.9.mlp.experts.78.gate_proj", "model.layers.9.mlp.experts.79.gate_proj", "model.layers.9.mlp.experts.80.gate_proj", "model.layers.9.mlp.experts.81.gate_proj", "model.layers.9.mlp.experts.82.gate_proj", "model.layers.9.mlp.experts.83.gate_proj", "model.layers.9.mlp.experts.84.gate_proj", "model.layers.9.mlp.experts.85.gate_proj", "model.layers.9.mlp.experts.86.gate_proj", "model.layers.9.mlp.experts.87.gate_proj", "model.layers.9.mlp.experts.88.gate_proj", "model.layers.9.mlp.experts.89.gate_proj", "model.layers.9.mlp.experts.90.gate_proj", "model.layers.9.mlp.experts.91.gate_proj", "model.layers.9.mlp.experts.92.gate_proj", "model.layers.9.mlp.experts.93.gate_proj", "model.layers.9.mlp.experts.94.gate_proj", "model.layers.9.mlp.experts.95.gate_proj", "model.layers.9.mlp.experts.96.gate_proj", "model.layers.9.mlp.experts.97.gate_proj", "model.layers.9.mlp.experts.98.gate_proj", "model.layers.9.mlp.experts.99.gate_proj", "model.layers.9.mlp.experts.100.gate_proj", "model.layers.9.mlp.experts.101.gate_proj", "model.layers.9.mlp.experts.102.gate_proj", "model.layers.9.mlp.experts.103.gate_proj", "model.layers.9.mlp.experts.104.gate_proj", "model.layers.9.mlp.experts.105.gate_proj", "model.layers.9.mlp.experts.106.gate_proj", "model.layers.9.mlp.experts.107.gate_proj", "model.layers.9.mlp.experts.108.gate_proj", "model.layers.9.mlp.experts.109.gate_proj", "model.layers.9.mlp.experts.110.gate_proj", "model.layers.9.mlp.experts.111.gate_proj", "model.layers.9.mlp.experts.112.gate_proj", "model.layers.9.mlp.experts.113.gate_proj", "model.layers.9.mlp.experts.114.gate_proj", "model.layers.9.mlp.experts.115.gate_proj", "model.layers.9.mlp.experts.116.gate_proj", "model.layers.9.mlp.experts.117.gate_proj", "model.layers.9.mlp.experts.118.gate_proj", "model.layers.9.mlp.experts.119.gate_proj", "model.layers.9.mlp.experts.120.gate_proj", "model.layers.9.mlp.experts.121.gate_proj", "model.layers.9.mlp.experts.122.gate_proj", "model.layers.9.mlp.experts.123.gate_proj", "model.layers.9.mlp.experts.124.gate_proj", "model.layers.9.mlp.experts.125.gate_proj", "model.layers.9.mlp.experts.126.gate_proj", "model.layers.9.mlp.experts.127.gate_proj", "model.layers.9.mlp.experts.0.up_proj", "model.layers.9.mlp.experts.1.up_proj", "model.layers.9.mlp.experts.2.up_proj", "model.layers.9.mlp.experts.3.up_proj", "model.layers.9.mlp.experts.4.up_proj", "model.layers.9.mlp.experts.5.up_proj", "model.layers.9.mlp.experts.6.up_proj", "model.layers.9.mlp.experts.7.up_proj", "model.layers.9.mlp.experts.8.up_proj", "model.layers.9.mlp.experts.9.up_proj", "model.layers.9.mlp.experts.10.up_proj", "model.layers.9.mlp.experts.11.up_proj", "model.layers.9.mlp.experts.12.up_proj", "model.layers.9.mlp.experts.13.up_proj", "model.layers.9.mlp.experts.14.up_proj", "model.layers.9.mlp.experts.15.up_proj", "model.layers.9.mlp.experts.16.up_proj", "model.layers.9.mlp.experts.17.up_proj", "model.layers.9.mlp.experts.18.up_proj", "model.layers.9.mlp.experts.19.up_proj", "model.layers.9.mlp.experts.20.up_proj", "model.layers.9.mlp.experts.21.up_proj", "model.layers.9.mlp.experts.22.up_proj", "model.layers.9.mlp.experts.23.up_proj", "model.layers.9.mlp.experts.24.up_proj", "model.layers.9.mlp.experts.25.up_proj", "model.layers.9.mlp.experts.26.up_proj", "model.layers.9.mlp.experts.27.up_proj", "model.layers.9.mlp.experts.28.up_proj", "model.layers.9.mlp.experts.29.up_proj", "model.layers.9.mlp.experts.30.up_proj", "model.layers.9.mlp.experts.31.up_proj", "model.layers.9.mlp.experts.32.up_proj", "model.layers.9.mlp.experts.33.up_proj", "model.layers.9.mlp.experts.34.up_proj", "model.layers.9.mlp.experts.35.up_proj", "model.layers.9.mlp.experts.36.up_proj", "model.layers.9.mlp.experts.37.up_proj", "model.layers.9.mlp.experts.38.up_proj", "model.layers.9.mlp.experts.39.up_proj", "model.layers.9.mlp.experts.40.up_proj", "model.layers.9.mlp.experts.41.up_proj", "model.layers.9.mlp.experts.42.up_proj", "model.layers.9.mlp.experts.43.up_proj", "model.layers.9.mlp.experts.44.up_proj", "model.layers.9.mlp.experts.45.up_proj", "model.layers.9.mlp.experts.46.up_proj", "model.layers.9.mlp.experts.47.up_proj", "model.layers.9.mlp.experts.48.up_proj", "model.layers.9.mlp.experts.49.up_proj", "model.layers.9.mlp.experts.50.up_proj", "model.layers.9.mlp.experts.51.up_proj", "model.layers.9.mlp.experts.52.up_proj", "model.layers.9.mlp.experts.53.up_proj", "model.layers.9.mlp.experts.54.up_proj", "model.layers.9.mlp.experts.55.up_proj", "model.layers.9.mlp.experts.56.up_proj", "model.layers.9.mlp.experts.57.up_proj", "model.layers.9.mlp.experts.58.up_proj", "model.layers.9.mlp.experts.59.up_proj", "model.layers.9.mlp.experts.60.up_proj", "model.layers.9.mlp.experts.61.up_proj", "model.layers.9.mlp.experts.62.up_proj", "model.layers.9.mlp.experts.63.up_proj", "model.layers.9.mlp.experts.64.up_proj", "model.layers.9.mlp.experts.65.up_proj", "model.layers.9.mlp.experts.66.up_proj", "model.layers.9.mlp.experts.67.up_proj", "model.layers.9.mlp.experts.68.up_proj", "model.layers.9.mlp.experts.69.up_proj", "model.layers.9.mlp.experts.70.up_proj", "model.layers.9.mlp.experts.71.up_proj", "model.layers.9.mlp.experts.72.up_proj", "model.layers.9.mlp.experts.73.up_proj", "model.layers.9.mlp.experts.74.up_proj", "model.layers.9.mlp.experts.75.up_proj", "model.layers.9.mlp.experts.76.up_proj", "model.layers.9.mlp.experts.77.up_proj", "model.layers.9.mlp.experts.78.up_proj", "model.layers.9.mlp.experts.79.up_proj", "model.layers.9.mlp.experts.80.up_proj", "model.layers.9.mlp.experts.81.up_proj", "model.layers.9.mlp.experts.82.up_proj", "model.layers.9.mlp.experts.83.up_proj", "model.layers.9.mlp.experts.84.up_proj", "model.layers.9.mlp.experts.85.up_proj", "model.layers.9.mlp.experts.86.up_proj", "model.layers.9.mlp.experts.87.up_proj", "model.layers.9.mlp.experts.88.up_proj", "model.layers.9.mlp.experts.89.up_proj", "model.layers.9.mlp.experts.90.up_proj", "model.layers.9.mlp.experts.91.up_proj", "model.layers.9.mlp.experts.92.up_proj", "model.layers.9.mlp.experts.93.up_proj", "model.layers.9.mlp.experts.94.up_proj", "model.layers.9.mlp.experts.95.up_proj", "model.layers.9.mlp.experts.96.up_proj", "model.layers.9.mlp.experts.97.up_proj", "model.layers.9.mlp.experts.98.up_proj", "model.layers.9.mlp.experts.99.up_proj", "model.layers.9.mlp.experts.100.up_proj", "model.layers.9.mlp.experts.101.up_proj", "model.layers.9.mlp.experts.102.up_proj", "model.layers.9.mlp.experts.103.up_proj", "model.layers.9.mlp.experts.104.up_proj", "model.layers.9.mlp.experts.105.up_proj", "model.layers.9.mlp.experts.106.up_proj", "model.layers.9.mlp.experts.107.up_proj", "model.layers.9.mlp.experts.108.up_proj", "model.layers.9.mlp.experts.109.up_proj", "model.layers.9.mlp.experts.110.up_proj", "model.layers.9.mlp.experts.111.up_proj", "model.layers.9.mlp.experts.112.up_proj", "model.layers.9.mlp.experts.113.up_proj", "model.layers.9.mlp.experts.114.up_proj", "model.layers.9.mlp.experts.115.up_proj", "model.layers.9.mlp.experts.116.up_proj", "model.layers.9.mlp.experts.117.up_proj", "model.layers.9.mlp.experts.118.up_proj", "model.layers.9.mlp.experts.119.up_proj", "model.layers.9.mlp.experts.120.up_proj", "model.layers.9.mlp.experts.121.up_proj", "model.layers.9.mlp.experts.122.up_proj", "model.layers.9.mlp.experts.123.up_proj", "model.layers.9.mlp.experts.124.up_proj", "model.layers.9.mlp.experts.125.up_proj", "model.layers.9.mlp.experts.126.up_proj", "model.layers.9.mlp.experts.127.up_proj", "model.layers.9.mlp.experts.0.down_proj", "model.layers.9.mlp.experts.1.down_proj", "model.layers.9.mlp.experts.2.down_proj", "model.layers.9.mlp.experts.3.down_proj", "model.layers.9.mlp.experts.4.down_proj", "model.layers.9.mlp.experts.5.down_proj", "model.layers.9.mlp.experts.6.down_proj", "model.layers.9.mlp.experts.7.down_proj", "model.layers.9.mlp.experts.8.down_proj", "model.layers.9.mlp.experts.9.down_proj", "model.layers.9.mlp.experts.10.down_proj", "model.layers.9.mlp.experts.11.down_proj", "model.layers.9.mlp.experts.12.down_proj", "model.layers.9.mlp.experts.13.down_proj", "model.layers.9.mlp.experts.14.down_proj", "model.layers.9.mlp.experts.15.down_proj", "model.layers.9.mlp.experts.16.down_proj", "model.layers.9.mlp.experts.17.down_proj", "model.layers.9.mlp.experts.18.down_proj", "model.layers.9.mlp.experts.19.down_proj", "model.layers.9.mlp.experts.20.down_proj", "model.layers.9.mlp.experts.21.down_proj", "model.layers.9.mlp.experts.22.down_proj", "model.layers.9.mlp.experts.23.down_proj", "model.layers.9.mlp.experts.24.down_proj", "model.layers.9.mlp.experts.25.down_proj", "model.layers.9.mlp.experts.26.down_proj", "model.layers.9.mlp.experts.27.down_proj", "model.layers.9.mlp.experts.28.down_proj", "model.layers.9.mlp.experts.29.down_proj", "model.layers.9.mlp.experts.30.down_proj", "model.layers.9.mlp.experts.31.down_proj", "model.layers.9.mlp.experts.32.down_proj", "model.layers.9.mlp.experts.33.down_proj", "model.layers.9.mlp.experts.34.down_proj", "model.layers.9.mlp.experts.35.down_proj", "model.layers.9.mlp.experts.36.down_proj", "model.layers.9.mlp.experts.37.down_proj", "model.layers.9.mlp.experts.38.down_proj", "model.layers.9.mlp.experts.39.down_proj", "model.layers.9.mlp.experts.40.down_proj", "model.layers.9.mlp.experts.41.down_proj", "model.layers.9.mlp.experts.42.down_proj", "model.layers.9.mlp.experts.43.down_proj", "model.layers.9.mlp.experts.44.down_proj", "model.layers.9.mlp.experts.45.down_proj", "model.layers.9.mlp.experts.46.down_proj", "model.layers.9.mlp.experts.47.down_proj", "model.layers.9.mlp.experts.48.down_proj", "model.layers.9.mlp.experts.49.down_proj", "model.layers.9.mlp.experts.50.down_proj", "model.layers.9.mlp.experts.51.down_proj", "model.layers.9.mlp.experts.52.down_proj", "model.layers.9.mlp.experts.53.down_proj", "model.layers.9.mlp.experts.54.down_proj", "model.layers.9.mlp.experts.55.down_proj", "model.layers.9.mlp.experts.56.down_proj", "model.layers.9.mlp.experts.57.down_proj", "model.layers.9.mlp.experts.58.down_proj", "model.layers.9.mlp.experts.59.down_proj", "model.layers.9.mlp.experts.60.down_proj", "model.layers.9.mlp.experts.61.down_proj", "model.layers.9.mlp.experts.62.down_proj", "model.layers.9.mlp.experts.63.down_proj", "model.layers.9.mlp.experts.64.down_proj", "model.layers.9.mlp.experts.65.down_proj", "model.layers.9.mlp.experts.66.down_proj", "model.layers.9.mlp.experts.67.down_proj", "model.layers.9.mlp.experts.68.down_proj", "model.layers.9.mlp.experts.69.down_proj", "model.layers.9.mlp.experts.70.down_proj", "model.layers.9.mlp.experts.71.down_proj", "model.layers.9.mlp.experts.72.down_proj", "model.layers.9.mlp.experts.73.down_proj", "model.layers.9.mlp.experts.74.down_proj", "model.layers.9.mlp.experts.75.down_proj", "model.layers.9.mlp.experts.76.down_proj", "model.layers.9.mlp.experts.77.down_proj", "model.layers.9.mlp.experts.78.down_proj", "model.layers.9.mlp.experts.79.down_proj", "model.layers.9.mlp.experts.80.down_proj", "model.layers.9.mlp.experts.81.down_proj", "model.layers.9.mlp.experts.82.down_proj", "model.layers.9.mlp.experts.83.down_proj", "model.layers.9.mlp.experts.84.down_proj", "model.layers.9.mlp.experts.85.down_proj", "model.layers.9.mlp.experts.86.down_proj", "model.layers.9.mlp.experts.87.down_proj", "model.layers.9.mlp.experts.88.down_proj", "model.layers.9.mlp.experts.89.down_proj", "model.layers.9.mlp.experts.90.down_proj", "model.layers.9.mlp.experts.91.down_proj", "model.layers.9.mlp.experts.92.down_proj", "model.layers.9.mlp.experts.93.down_proj", "model.layers.9.mlp.experts.94.down_proj", "model.layers.9.mlp.experts.95.down_proj", "model.layers.9.mlp.experts.96.down_proj", "model.layers.9.mlp.experts.97.down_proj", "model.layers.9.mlp.experts.98.down_proj", "model.layers.9.mlp.experts.99.down_proj", "model.layers.9.mlp.experts.100.down_proj", "model.layers.9.mlp.experts.101.down_proj", "model.layers.9.mlp.experts.102.down_proj", "model.layers.9.mlp.experts.103.down_proj", "model.layers.9.mlp.experts.104.down_proj", "model.layers.9.mlp.experts.105.down_proj", "model.layers.9.mlp.experts.106.down_proj", "model.layers.9.mlp.experts.107.down_proj", "model.layers.9.mlp.experts.108.down_proj", "model.layers.9.mlp.experts.109.down_proj", "model.layers.9.mlp.experts.110.down_proj", "model.layers.9.mlp.experts.111.down_proj", "model.layers.9.mlp.experts.112.down_proj", "model.layers.9.mlp.experts.113.down_proj", "model.layers.9.mlp.experts.114.down_proj", "model.layers.9.mlp.experts.115.down_proj", "model.layers.9.mlp.experts.116.down_proj", "model.layers.9.mlp.experts.117.down_proj", "model.layers.9.mlp.experts.118.down_proj", "model.layers.9.mlp.experts.119.down_proj", "model.layers.9.mlp.experts.120.down_proj", "model.layers.9.mlp.experts.121.down_proj", "model.layers.9.mlp.experts.122.down_proj", "model.layers.9.mlp.experts.123.down_proj", "model.layers.9.mlp.experts.124.down_proj", "model.layers.9.mlp.experts.125.down_proj", "model.layers.9.mlp.experts.126.down_proj", "model.layers.9.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.006745889224112039, "dbits": 2214592512 } ] }, { "idx": 29, "layers": [ "model.layers.10.self_attn.q_proj", "model.layers.10.self_attn.k_proj", "model.layers.10.self_attn.v_proj", "model.layers.10.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0001238755881786402, "dbits": 109051904 } ] }, { "idx": 30, "layers": [ "model.layers.10.mlp.shared_experts.gate_proj", "model.layers.10.mlp.shared_experts.up_proj", "model.layers.10.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.001413480844348669, "dbits": 17301504 } ] }, { "idx": 31, "layers": [ "model.layers.10.mlp.experts.0.gate_proj", "model.layers.10.mlp.experts.1.gate_proj", "model.layers.10.mlp.experts.2.gate_proj", "model.layers.10.mlp.experts.3.gate_proj", "model.layers.10.mlp.experts.4.gate_proj", "model.layers.10.mlp.experts.5.gate_proj", "model.layers.10.mlp.experts.6.gate_proj", "model.layers.10.mlp.experts.7.gate_proj", "model.layers.10.mlp.experts.8.gate_proj", "model.layers.10.mlp.experts.9.gate_proj", "model.layers.10.mlp.experts.10.gate_proj", "model.layers.10.mlp.experts.11.gate_proj", "model.layers.10.mlp.experts.12.gate_proj", "model.layers.10.mlp.experts.13.gate_proj", "model.layers.10.mlp.experts.14.gate_proj", "model.layers.10.mlp.experts.15.gate_proj", "model.layers.10.mlp.experts.16.gate_proj", "model.layers.10.mlp.experts.17.gate_proj", "model.layers.10.mlp.experts.18.gate_proj", "model.layers.10.mlp.experts.19.gate_proj", "model.layers.10.mlp.experts.20.gate_proj", "model.layers.10.mlp.experts.21.gate_proj", "model.layers.10.mlp.experts.22.gate_proj", "model.layers.10.mlp.experts.23.gate_proj", "model.layers.10.mlp.experts.24.gate_proj", "model.layers.10.mlp.experts.25.gate_proj", "model.layers.10.mlp.experts.26.gate_proj", "model.layers.10.mlp.experts.27.gate_proj", "model.layers.10.mlp.experts.28.gate_proj", "model.layers.10.mlp.experts.29.gate_proj", "model.layers.10.mlp.experts.30.gate_proj", "model.layers.10.mlp.experts.31.gate_proj", "model.layers.10.mlp.experts.32.gate_proj", "model.layers.10.mlp.experts.33.gate_proj", "model.layers.10.mlp.experts.34.gate_proj", "model.layers.10.mlp.experts.35.gate_proj", "model.layers.10.mlp.experts.36.gate_proj", "model.layers.10.mlp.experts.37.gate_proj", "model.layers.10.mlp.experts.38.gate_proj", "model.layers.10.mlp.experts.39.gate_proj", "model.layers.10.mlp.experts.40.gate_proj", "model.layers.10.mlp.experts.41.gate_proj", "model.layers.10.mlp.experts.42.gate_proj", "model.layers.10.mlp.experts.43.gate_proj", "model.layers.10.mlp.experts.44.gate_proj", "model.layers.10.mlp.experts.45.gate_proj", "model.layers.10.mlp.experts.46.gate_proj", "model.layers.10.mlp.experts.47.gate_proj", "model.layers.10.mlp.experts.48.gate_proj", "model.layers.10.mlp.experts.49.gate_proj", "model.layers.10.mlp.experts.50.gate_proj", "model.layers.10.mlp.experts.51.gate_proj", "model.layers.10.mlp.experts.52.gate_proj", "model.layers.10.mlp.experts.53.gate_proj", "model.layers.10.mlp.experts.54.gate_proj", "model.layers.10.mlp.experts.55.gate_proj", "model.layers.10.mlp.experts.56.gate_proj", "model.layers.10.mlp.experts.57.gate_proj", "model.layers.10.mlp.experts.58.gate_proj", "model.layers.10.mlp.experts.59.gate_proj", "model.layers.10.mlp.experts.60.gate_proj", "model.layers.10.mlp.experts.61.gate_proj", "model.layers.10.mlp.experts.62.gate_proj", "model.layers.10.mlp.experts.63.gate_proj", "model.layers.10.mlp.experts.64.gate_proj", "model.layers.10.mlp.experts.65.gate_proj", "model.layers.10.mlp.experts.66.gate_proj", "model.layers.10.mlp.experts.67.gate_proj", "model.layers.10.mlp.experts.68.gate_proj", "model.layers.10.mlp.experts.69.gate_proj", "model.layers.10.mlp.experts.70.gate_proj", "model.layers.10.mlp.experts.71.gate_proj", "model.layers.10.mlp.experts.72.gate_proj", "model.layers.10.mlp.experts.73.gate_proj", "model.layers.10.mlp.experts.74.gate_proj", "model.layers.10.mlp.experts.75.gate_proj", "model.layers.10.mlp.experts.76.gate_proj", "model.layers.10.mlp.experts.77.gate_proj", "model.layers.10.mlp.experts.78.gate_proj", "model.layers.10.mlp.experts.79.gate_proj", "model.layers.10.mlp.experts.80.gate_proj", "model.layers.10.mlp.experts.81.gate_proj", "model.layers.10.mlp.experts.82.gate_proj", "model.layers.10.mlp.experts.83.gate_proj", "model.layers.10.mlp.experts.84.gate_proj", "model.layers.10.mlp.experts.85.gate_proj", "model.layers.10.mlp.experts.86.gate_proj", "model.layers.10.mlp.experts.87.gate_proj", "model.layers.10.mlp.experts.88.gate_proj", "model.layers.10.mlp.experts.89.gate_proj", "model.layers.10.mlp.experts.90.gate_proj", "model.layers.10.mlp.experts.91.gate_proj", "model.layers.10.mlp.experts.92.gate_proj", "model.layers.10.mlp.experts.93.gate_proj", "model.layers.10.mlp.experts.94.gate_proj", "model.layers.10.mlp.experts.95.gate_proj", "model.layers.10.mlp.experts.96.gate_proj", "model.layers.10.mlp.experts.97.gate_proj", "model.layers.10.mlp.experts.98.gate_proj", "model.layers.10.mlp.experts.99.gate_proj", "model.layers.10.mlp.experts.100.gate_proj", "model.layers.10.mlp.experts.101.gate_proj", "model.layers.10.mlp.experts.102.gate_proj", "model.layers.10.mlp.experts.103.gate_proj", "model.layers.10.mlp.experts.104.gate_proj", "model.layers.10.mlp.experts.105.gate_proj", "model.layers.10.mlp.experts.106.gate_proj", "model.layers.10.mlp.experts.107.gate_proj", "model.layers.10.mlp.experts.108.gate_proj", "model.layers.10.mlp.experts.109.gate_proj", "model.layers.10.mlp.experts.110.gate_proj", "model.layers.10.mlp.experts.111.gate_proj", "model.layers.10.mlp.experts.112.gate_proj", "model.layers.10.mlp.experts.113.gate_proj", "model.layers.10.mlp.experts.114.gate_proj", "model.layers.10.mlp.experts.115.gate_proj", "model.layers.10.mlp.experts.116.gate_proj", "model.layers.10.mlp.experts.117.gate_proj", "model.layers.10.mlp.experts.118.gate_proj", "model.layers.10.mlp.experts.119.gate_proj", "model.layers.10.mlp.experts.120.gate_proj", "model.layers.10.mlp.experts.121.gate_proj", "model.layers.10.mlp.experts.122.gate_proj", "model.layers.10.mlp.experts.123.gate_proj", "model.layers.10.mlp.experts.124.gate_proj", "model.layers.10.mlp.experts.125.gate_proj", "model.layers.10.mlp.experts.126.gate_proj", "model.layers.10.mlp.experts.127.gate_proj", "model.layers.10.mlp.experts.0.up_proj", "model.layers.10.mlp.experts.1.up_proj", "model.layers.10.mlp.experts.2.up_proj", "model.layers.10.mlp.experts.3.up_proj", "model.layers.10.mlp.experts.4.up_proj", "model.layers.10.mlp.experts.5.up_proj", "model.layers.10.mlp.experts.6.up_proj", "model.layers.10.mlp.experts.7.up_proj", "model.layers.10.mlp.experts.8.up_proj", "model.layers.10.mlp.experts.9.up_proj", "model.layers.10.mlp.experts.10.up_proj", "model.layers.10.mlp.experts.11.up_proj", "model.layers.10.mlp.experts.12.up_proj", "model.layers.10.mlp.experts.13.up_proj", "model.layers.10.mlp.experts.14.up_proj", "model.layers.10.mlp.experts.15.up_proj", "model.layers.10.mlp.experts.16.up_proj", "model.layers.10.mlp.experts.17.up_proj", "model.layers.10.mlp.experts.18.up_proj", "model.layers.10.mlp.experts.19.up_proj", "model.layers.10.mlp.experts.20.up_proj", "model.layers.10.mlp.experts.21.up_proj", "model.layers.10.mlp.experts.22.up_proj", "model.layers.10.mlp.experts.23.up_proj", "model.layers.10.mlp.experts.24.up_proj", "model.layers.10.mlp.experts.25.up_proj", "model.layers.10.mlp.experts.26.up_proj", "model.layers.10.mlp.experts.27.up_proj", "model.layers.10.mlp.experts.28.up_proj", "model.layers.10.mlp.experts.29.up_proj", "model.layers.10.mlp.experts.30.up_proj", "model.layers.10.mlp.experts.31.up_proj", "model.layers.10.mlp.experts.32.up_proj", "model.layers.10.mlp.experts.33.up_proj", "model.layers.10.mlp.experts.34.up_proj", "model.layers.10.mlp.experts.35.up_proj", "model.layers.10.mlp.experts.36.up_proj", "model.layers.10.mlp.experts.37.up_proj", "model.layers.10.mlp.experts.38.up_proj", "model.layers.10.mlp.experts.39.up_proj", "model.layers.10.mlp.experts.40.up_proj", "model.layers.10.mlp.experts.41.up_proj", "model.layers.10.mlp.experts.42.up_proj", "model.layers.10.mlp.experts.43.up_proj", "model.layers.10.mlp.experts.44.up_proj", "model.layers.10.mlp.experts.45.up_proj", "model.layers.10.mlp.experts.46.up_proj", "model.layers.10.mlp.experts.47.up_proj", "model.layers.10.mlp.experts.48.up_proj", "model.layers.10.mlp.experts.49.up_proj", "model.layers.10.mlp.experts.50.up_proj", "model.layers.10.mlp.experts.51.up_proj", "model.layers.10.mlp.experts.52.up_proj", "model.layers.10.mlp.experts.53.up_proj", "model.layers.10.mlp.experts.54.up_proj", "model.layers.10.mlp.experts.55.up_proj", "model.layers.10.mlp.experts.56.up_proj", "model.layers.10.mlp.experts.57.up_proj", "model.layers.10.mlp.experts.58.up_proj", "model.layers.10.mlp.experts.59.up_proj", "model.layers.10.mlp.experts.60.up_proj", "model.layers.10.mlp.experts.61.up_proj", "model.layers.10.mlp.experts.62.up_proj", "model.layers.10.mlp.experts.63.up_proj", "model.layers.10.mlp.experts.64.up_proj", "model.layers.10.mlp.experts.65.up_proj", "model.layers.10.mlp.experts.66.up_proj", "model.layers.10.mlp.experts.67.up_proj", "model.layers.10.mlp.experts.68.up_proj", "model.layers.10.mlp.experts.69.up_proj", "model.layers.10.mlp.experts.70.up_proj", "model.layers.10.mlp.experts.71.up_proj", "model.layers.10.mlp.experts.72.up_proj", "model.layers.10.mlp.experts.73.up_proj", "model.layers.10.mlp.experts.74.up_proj", "model.layers.10.mlp.experts.75.up_proj", "model.layers.10.mlp.experts.76.up_proj", "model.layers.10.mlp.experts.77.up_proj", "model.layers.10.mlp.experts.78.up_proj", "model.layers.10.mlp.experts.79.up_proj", "model.layers.10.mlp.experts.80.up_proj", "model.layers.10.mlp.experts.81.up_proj", "model.layers.10.mlp.experts.82.up_proj", "model.layers.10.mlp.experts.83.up_proj", "model.layers.10.mlp.experts.84.up_proj", "model.layers.10.mlp.experts.85.up_proj", "model.layers.10.mlp.experts.86.up_proj", "model.layers.10.mlp.experts.87.up_proj", "model.layers.10.mlp.experts.88.up_proj", "model.layers.10.mlp.experts.89.up_proj", "model.layers.10.mlp.experts.90.up_proj", "model.layers.10.mlp.experts.91.up_proj", "model.layers.10.mlp.experts.92.up_proj", "model.layers.10.mlp.experts.93.up_proj", "model.layers.10.mlp.experts.94.up_proj", "model.layers.10.mlp.experts.95.up_proj", "model.layers.10.mlp.experts.96.up_proj", "model.layers.10.mlp.experts.97.up_proj", "model.layers.10.mlp.experts.98.up_proj", "model.layers.10.mlp.experts.99.up_proj", "model.layers.10.mlp.experts.100.up_proj", "model.layers.10.mlp.experts.101.up_proj", "model.layers.10.mlp.experts.102.up_proj", "model.layers.10.mlp.experts.103.up_proj", "model.layers.10.mlp.experts.104.up_proj", "model.layers.10.mlp.experts.105.up_proj", "model.layers.10.mlp.experts.106.up_proj", "model.layers.10.mlp.experts.107.up_proj", "model.layers.10.mlp.experts.108.up_proj", "model.layers.10.mlp.experts.109.up_proj", "model.layers.10.mlp.experts.110.up_proj", "model.layers.10.mlp.experts.111.up_proj", "model.layers.10.mlp.experts.112.up_proj", "model.layers.10.mlp.experts.113.up_proj", "model.layers.10.mlp.experts.114.up_proj", "model.layers.10.mlp.experts.115.up_proj", "model.layers.10.mlp.experts.116.up_proj", "model.layers.10.mlp.experts.117.up_proj", "model.layers.10.mlp.experts.118.up_proj", "model.layers.10.mlp.experts.119.up_proj", "model.layers.10.mlp.experts.120.up_proj", "model.layers.10.mlp.experts.121.up_proj", "model.layers.10.mlp.experts.122.up_proj", "model.layers.10.mlp.experts.123.up_proj", "model.layers.10.mlp.experts.124.up_proj", "model.layers.10.mlp.experts.125.up_proj", "model.layers.10.mlp.experts.126.up_proj", "model.layers.10.mlp.experts.127.up_proj", "model.layers.10.mlp.experts.0.down_proj", "model.layers.10.mlp.experts.1.down_proj", "model.layers.10.mlp.experts.2.down_proj", "model.layers.10.mlp.experts.3.down_proj", "model.layers.10.mlp.experts.4.down_proj", "model.layers.10.mlp.experts.5.down_proj", "model.layers.10.mlp.experts.6.down_proj", "model.layers.10.mlp.experts.7.down_proj", "model.layers.10.mlp.experts.8.down_proj", "model.layers.10.mlp.experts.9.down_proj", "model.layers.10.mlp.experts.10.down_proj", "model.layers.10.mlp.experts.11.down_proj", "model.layers.10.mlp.experts.12.down_proj", "model.layers.10.mlp.experts.13.down_proj", "model.layers.10.mlp.experts.14.down_proj", "model.layers.10.mlp.experts.15.down_proj", "model.layers.10.mlp.experts.16.down_proj", "model.layers.10.mlp.experts.17.down_proj", "model.layers.10.mlp.experts.18.down_proj", "model.layers.10.mlp.experts.19.down_proj", "model.layers.10.mlp.experts.20.down_proj", "model.layers.10.mlp.experts.21.down_proj", "model.layers.10.mlp.experts.22.down_proj", "model.layers.10.mlp.experts.23.down_proj", "model.layers.10.mlp.experts.24.down_proj", "model.layers.10.mlp.experts.25.down_proj", "model.layers.10.mlp.experts.26.down_proj", "model.layers.10.mlp.experts.27.down_proj", "model.layers.10.mlp.experts.28.down_proj", "model.layers.10.mlp.experts.29.down_proj", "model.layers.10.mlp.experts.30.down_proj", "model.layers.10.mlp.experts.31.down_proj", "model.layers.10.mlp.experts.32.down_proj", "model.layers.10.mlp.experts.33.down_proj", "model.layers.10.mlp.experts.34.down_proj", "model.layers.10.mlp.experts.35.down_proj", "model.layers.10.mlp.experts.36.down_proj", "model.layers.10.mlp.experts.37.down_proj", "model.layers.10.mlp.experts.38.down_proj", "model.layers.10.mlp.experts.39.down_proj", "model.layers.10.mlp.experts.40.down_proj", "model.layers.10.mlp.experts.41.down_proj", "model.layers.10.mlp.experts.42.down_proj", "model.layers.10.mlp.experts.43.down_proj", "model.layers.10.mlp.experts.44.down_proj", "model.layers.10.mlp.experts.45.down_proj", "model.layers.10.mlp.experts.46.down_proj", "model.layers.10.mlp.experts.47.down_proj", "model.layers.10.mlp.experts.48.down_proj", "model.layers.10.mlp.experts.49.down_proj", "model.layers.10.mlp.experts.50.down_proj", "model.layers.10.mlp.experts.51.down_proj", "model.layers.10.mlp.experts.52.down_proj", "model.layers.10.mlp.experts.53.down_proj", "model.layers.10.mlp.experts.54.down_proj", "model.layers.10.mlp.experts.55.down_proj", "model.layers.10.mlp.experts.56.down_proj", "model.layers.10.mlp.experts.57.down_proj", "model.layers.10.mlp.experts.58.down_proj", "model.layers.10.mlp.experts.59.down_proj", "model.layers.10.mlp.experts.60.down_proj", "model.layers.10.mlp.experts.61.down_proj", "model.layers.10.mlp.experts.62.down_proj", "model.layers.10.mlp.experts.63.down_proj", "model.layers.10.mlp.experts.64.down_proj", "model.layers.10.mlp.experts.65.down_proj", "model.layers.10.mlp.experts.66.down_proj", "model.layers.10.mlp.experts.67.down_proj", "model.layers.10.mlp.experts.68.down_proj", "model.layers.10.mlp.experts.69.down_proj", "model.layers.10.mlp.experts.70.down_proj", "model.layers.10.mlp.experts.71.down_proj", "model.layers.10.mlp.experts.72.down_proj", "model.layers.10.mlp.experts.73.down_proj", "model.layers.10.mlp.experts.74.down_proj", "model.layers.10.mlp.experts.75.down_proj", "model.layers.10.mlp.experts.76.down_proj", "model.layers.10.mlp.experts.77.down_proj", "model.layers.10.mlp.experts.78.down_proj", "model.layers.10.mlp.experts.79.down_proj", "model.layers.10.mlp.experts.80.down_proj", "model.layers.10.mlp.experts.81.down_proj", "model.layers.10.mlp.experts.82.down_proj", "model.layers.10.mlp.experts.83.down_proj", "model.layers.10.mlp.experts.84.down_proj", "model.layers.10.mlp.experts.85.down_proj", "model.layers.10.mlp.experts.86.down_proj", "model.layers.10.mlp.experts.87.down_proj", "model.layers.10.mlp.experts.88.down_proj", "model.layers.10.mlp.experts.89.down_proj", "model.layers.10.mlp.experts.90.down_proj", "model.layers.10.mlp.experts.91.down_proj", "model.layers.10.mlp.experts.92.down_proj", "model.layers.10.mlp.experts.93.down_proj", "model.layers.10.mlp.experts.94.down_proj", "model.layers.10.mlp.experts.95.down_proj", "model.layers.10.mlp.experts.96.down_proj", "model.layers.10.mlp.experts.97.down_proj", "model.layers.10.mlp.experts.98.down_proj", "model.layers.10.mlp.experts.99.down_proj", "model.layers.10.mlp.experts.100.down_proj", "model.layers.10.mlp.experts.101.down_proj", "model.layers.10.mlp.experts.102.down_proj", "model.layers.10.mlp.experts.103.down_proj", "model.layers.10.mlp.experts.104.down_proj", "model.layers.10.mlp.experts.105.down_proj", "model.layers.10.mlp.experts.106.down_proj", "model.layers.10.mlp.experts.107.down_proj", "model.layers.10.mlp.experts.108.down_proj", "model.layers.10.mlp.experts.109.down_proj", "model.layers.10.mlp.experts.110.down_proj", "model.layers.10.mlp.experts.111.down_proj", "model.layers.10.mlp.experts.112.down_proj", "model.layers.10.mlp.experts.113.down_proj", "model.layers.10.mlp.experts.114.down_proj", "model.layers.10.mlp.experts.115.down_proj", "model.layers.10.mlp.experts.116.down_proj", "model.layers.10.mlp.experts.117.down_proj", "model.layers.10.mlp.experts.118.down_proj", "model.layers.10.mlp.experts.119.down_proj", "model.layers.10.mlp.experts.120.down_proj", "model.layers.10.mlp.experts.121.down_proj", "model.layers.10.mlp.experts.122.down_proj", "model.layers.10.mlp.experts.123.down_proj", "model.layers.10.mlp.experts.124.down_proj", "model.layers.10.mlp.experts.125.down_proj", "model.layers.10.mlp.experts.126.down_proj", "model.layers.10.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.008714989759027958, "dbits": 2214592512 } ] }, { "idx": 32, "layers": [ "model.layers.11.self_attn.q_proj", "model.layers.11.self_attn.k_proj", "model.layers.11.self_attn.v_proj", "model.layers.11.self_attn.o_proj" ], "candidates": [ { "dkld": -0.002380718104541296, "dbits": 109051904 } ] }, { "idx": 33, "layers": [ "model.layers.11.mlp.shared_experts.gate_proj", "model.layers.11.mlp.shared_experts.up_proj", "model.layers.11.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0006415588781237547, "dbits": 17301504 } ] }, { "idx": 34, "layers": [ "model.layers.11.mlp.experts.0.gate_proj", "model.layers.11.mlp.experts.1.gate_proj", "model.layers.11.mlp.experts.2.gate_proj", "model.layers.11.mlp.experts.3.gate_proj", "model.layers.11.mlp.experts.4.gate_proj", "model.layers.11.mlp.experts.5.gate_proj", "model.layers.11.mlp.experts.6.gate_proj", "model.layers.11.mlp.experts.7.gate_proj", "model.layers.11.mlp.experts.8.gate_proj", "model.layers.11.mlp.experts.9.gate_proj", "model.layers.11.mlp.experts.10.gate_proj", "model.layers.11.mlp.experts.11.gate_proj", "model.layers.11.mlp.experts.12.gate_proj", "model.layers.11.mlp.experts.13.gate_proj", "model.layers.11.mlp.experts.14.gate_proj", "model.layers.11.mlp.experts.15.gate_proj", "model.layers.11.mlp.experts.16.gate_proj", "model.layers.11.mlp.experts.17.gate_proj", "model.layers.11.mlp.experts.18.gate_proj", "model.layers.11.mlp.experts.19.gate_proj", "model.layers.11.mlp.experts.20.gate_proj", "model.layers.11.mlp.experts.21.gate_proj", "model.layers.11.mlp.experts.22.gate_proj", "model.layers.11.mlp.experts.23.gate_proj", "model.layers.11.mlp.experts.24.gate_proj", "model.layers.11.mlp.experts.25.gate_proj", "model.layers.11.mlp.experts.26.gate_proj", "model.layers.11.mlp.experts.27.gate_proj", "model.layers.11.mlp.experts.28.gate_proj", "model.layers.11.mlp.experts.29.gate_proj", "model.layers.11.mlp.experts.30.gate_proj", "model.layers.11.mlp.experts.31.gate_proj", "model.layers.11.mlp.experts.32.gate_proj", "model.layers.11.mlp.experts.33.gate_proj", "model.layers.11.mlp.experts.34.gate_proj", "model.layers.11.mlp.experts.35.gate_proj", "model.layers.11.mlp.experts.36.gate_proj", "model.layers.11.mlp.experts.37.gate_proj", "model.layers.11.mlp.experts.38.gate_proj", "model.layers.11.mlp.experts.39.gate_proj", "model.layers.11.mlp.experts.40.gate_proj", "model.layers.11.mlp.experts.41.gate_proj", "model.layers.11.mlp.experts.42.gate_proj", "model.layers.11.mlp.experts.43.gate_proj", "model.layers.11.mlp.experts.44.gate_proj", "model.layers.11.mlp.experts.45.gate_proj", "model.layers.11.mlp.experts.46.gate_proj", "model.layers.11.mlp.experts.47.gate_proj", "model.layers.11.mlp.experts.48.gate_proj", "model.layers.11.mlp.experts.49.gate_proj", "model.layers.11.mlp.experts.50.gate_proj", "model.layers.11.mlp.experts.51.gate_proj", "model.layers.11.mlp.experts.52.gate_proj", "model.layers.11.mlp.experts.53.gate_proj", "model.layers.11.mlp.experts.54.gate_proj", "model.layers.11.mlp.experts.55.gate_proj", "model.layers.11.mlp.experts.56.gate_proj", "model.layers.11.mlp.experts.57.gate_proj", "model.layers.11.mlp.experts.58.gate_proj", "model.layers.11.mlp.experts.59.gate_proj", "model.layers.11.mlp.experts.60.gate_proj", "model.layers.11.mlp.experts.61.gate_proj", "model.layers.11.mlp.experts.62.gate_proj", "model.layers.11.mlp.experts.63.gate_proj", "model.layers.11.mlp.experts.64.gate_proj", "model.layers.11.mlp.experts.65.gate_proj", "model.layers.11.mlp.experts.66.gate_proj", "model.layers.11.mlp.experts.67.gate_proj", "model.layers.11.mlp.experts.68.gate_proj", "model.layers.11.mlp.experts.69.gate_proj", "model.layers.11.mlp.experts.70.gate_proj", "model.layers.11.mlp.experts.71.gate_proj", "model.layers.11.mlp.experts.72.gate_proj", "model.layers.11.mlp.experts.73.gate_proj", "model.layers.11.mlp.experts.74.gate_proj", "model.layers.11.mlp.experts.75.gate_proj", "model.layers.11.mlp.experts.76.gate_proj", "model.layers.11.mlp.experts.77.gate_proj", "model.layers.11.mlp.experts.78.gate_proj", "model.layers.11.mlp.experts.79.gate_proj", "model.layers.11.mlp.experts.80.gate_proj", "model.layers.11.mlp.experts.81.gate_proj", "model.layers.11.mlp.experts.82.gate_proj", "model.layers.11.mlp.experts.83.gate_proj", "model.layers.11.mlp.experts.84.gate_proj", "model.layers.11.mlp.experts.85.gate_proj", "model.layers.11.mlp.experts.86.gate_proj", "model.layers.11.mlp.experts.87.gate_proj", "model.layers.11.mlp.experts.88.gate_proj", "model.layers.11.mlp.experts.89.gate_proj", "model.layers.11.mlp.experts.90.gate_proj", "model.layers.11.mlp.experts.91.gate_proj", "model.layers.11.mlp.experts.92.gate_proj", "model.layers.11.mlp.experts.93.gate_proj", "model.layers.11.mlp.experts.94.gate_proj", "model.layers.11.mlp.experts.95.gate_proj", "model.layers.11.mlp.experts.96.gate_proj", "model.layers.11.mlp.experts.97.gate_proj", "model.layers.11.mlp.experts.98.gate_proj", "model.layers.11.mlp.experts.99.gate_proj", "model.layers.11.mlp.experts.100.gate_proj", "model.layers.11.mlp.experts.101.gate_proj", "model.layers.11.mlp.experts.102.gate_proj", "model.layers.11.mlp.experts.103.gate_proj", "model.layers.11.mlp.experts.104.gate_proj", "model.layers.11.mlp.experts.105.gate_proj", "model.layers.11.mlp.experts.106.gate_proj", "model.layers.11.mlp.experts.107.gate_proj", "model.layers.11.mlp.experts.108.gate_proj", "model.layers.11.mlp.experts.109.gate_proj", "model.layers.11.mlp.experts.110.gate_proj", "model.layers.11.mlp.experts.111.gate_proj", "model.layers.11.mlp.experts.112.gate_proj", "model.layers.11.mlp.experts.113.gate_proj", "model.layers.11.mlp.experts.114.gate_proj", "model.layers.11.mlp.experts.115.gate_proj", "model.layers.11.mlp.experts.116.gate_proj", "model.layers.11.mlp.experts.117.gate_proj", "model.layers.11.mlp.experts.118.gate_proj", "model.layers.11.mlp.experts.119.gate_proj", "model.layers.11.mlp.experts.120.gate_proj", "model.layers.11.mlp.experts.121.gate_proj", "model.layers.11.mlp.experts.122.gate_proj", "model.layers.11.mlp.experts.123.gate_proj", "model.layers.11.mlp.experts.124.gate_proj", "model.layers.11.mlp.experts.125.gate_proj", "model.layers.11.mlp.experts.126.gate_proj", "model.layers.11.mlp.experts.127.gate_proj", "model.layers.11.mlp.experts.0.up_proj", "model.layers.11.mlp.experts.1.up_proj", "model.layers.11.mlp.experts.2.up_proj", "model.layers.11.mlp.experts.3.up_proj", "model.layers.11.mlp.experts.4.up_proj", "model.layers.11.mlp.experts.5.up_proj", "model.layers.11.mlp.experts.6.up_proj", "model.layers.11.mlp.experts.7.up_proj", "model.layers.11.mlp.experts.8.up_proj", "model.layers.11.mlp.experts.9.up_proj", "model.layers.11.mlp.experts.10.up_proj", "model.layers.11.mlp.experts.11.up_proj", "model.layers.11.mlp.experts.12.up_proj", "model.layers.11.mlp.experts.13.up_proj", "model.layers.11.mlp.experts.14.up_proj", "model.layers.11.mlp.experts.15.up_proj", "model.layers.11.mlp.experts.16.up_proj", "model.layers.11.mlp.experts.17.up_proj", "model.layers.11.mlp.experts.18.up_proj", "model.layers.11.mlp.experts.19.up_proj", "model.layers.11.mlp.experts.20.up_proj", "model.layers.11.mlp.experts.21.up_proj", "model.layers.11.mlp.experts.22.up_proj", "model.layers.11.mlp.experts.23.up_proj", "model.layers.11.mlp.experts.24.up_proj", "model.layers.11.mlp.experts.25.up_proj", "model.layers.11.mlp.experts.26.up_proj", "model.layers.11.mlp.experts.27.up_proj", "model.layers.11.mlp.experts.28.up_proj", "model.layers.11.mlp.experts.29.up_proj", "model.layers.11.mlp.experts.30.up_proj", "model.layers.11.mlp.experts.31.up_proj", "model.layers.11.mlp.experts.32.up_proj", "model.layers.11.mlp.experts.33.up_proj", "model.layers.11.mlp.experts.34.up_proj", "model.layers.11.mlp.experts.35.up_proj", "model.layers.11.mlp.experts.36.up_proj", "model.layers.11.mlp.experts.37.up_proj", "model.layers.11.mlp.experts.38.up_proj", "model.layers.11.mlp.experts.39.up_proj", "model.layers.11.mlp.experts.40.up_proj", "model.layers.11.mlp.experts.41.up_proj", "model.layers.11.mlp.experts.42.up_proj", "model.layers.11.mlp.experts.43.up_proj", "model.layers.11.mlp.experts.44.up_proj", "model.layers.11.mlp.experts.45.up_proj", "model.layers.11.mlp.experts.46.up_proj", "model.layers.11.mlp.experts.47.up_proj", "model.layers.11.mlp.experts.48.up_proj", "model.layers.11.mlp.experts.49.up_proj", "model.layers.11.mlp.experts.50.up_proj", "model.layers.11.mlp.experts.51.up_proj", "model.layers.11.mlp.experts.52.up_proj", "model.layers.11.mlp.experts.53.up_proj", "model.layers.11.mlp.experts.54.up_proj", "model.layers.11.mlp.experts.55.up_proj", "model.layers.11.mlp.experts.56.up_proj", "model.layers.11.mlp.experts.57.up_proj", "model.layers.11.mlp.experts.58.up_proj", "model.layers.11.mlp.experts.59.up_proj", "model.layers.11.mlp.experts.60.up_proj", "model.layers.11.mlp.experts.61.up_proj", "model.layers.11.mlp.experts.62.up_proj", "model.layers.11.mlp.experts.63.up_proj", "model.layers.11.mlp.experts.64.up_proj", "model.layers.11.mlp.experts.65.up_proj", "model.layers.11.mlp.experts.66.up_proj", "model.layers.11.mlp.experts.67.up_proj", "model.layers.11.mlp.experts.68.up_proj", "model.layers.11.mlp.experts.69.up_proj", "model.layers.11.mlp.experts.70.up_proj", "model.layers.11.mlp.experts.71.up_proj", "model.layers.11.mlp.experts.72.up_proj", "model.layers.11.mlp.experts.73.up_proj", "model.layers.11.mlp.experts.74.up_proj", "model.layers.11.mlp.experts.75.up_proj", "model.layers.11.mlp.experts.76.up_proj", "model.layers.11.mlp.experts.77.up_proj", "model.layers.11.mlp.experts.78.up_proj", "model.layers.11.mlp.experts.79.up_proj", "model.layers.11.mlp.experts.80.up_proj", "model.layers.11.mlp.experts.81.up_proj", "model.layers.11.mlp.experts.82.up_proj", "model.layers.11.mlp.experts.83.up_proj", "model.layers.11.mlp.experts.84.up_proj", "model.layers.11.mlp.experts.85.up_proj", "model.layers.11.mlp.experts.86.up_proj", "model.layers.11.mlp.experts.87.up_proj", "model.layers.11.mlp.experts.88.up_proj", "model.layers.11.mlp.experts.89.up_proj", "model.layers.11.mlp.experts.90.up_proj", "model.layers.11.mlp.experts.91.up_proj", "model.layers.11.mlp.experts.92.up_proj", "model.layers.11.mlp.experts.93.up_proj", "model.layers.11.mlp.experts.94.up_proj", "model.layers.11.mlp.experts.95.up_proj", "model.layers.11.mlp.experts.96.up_proj", "model.layers.11.mlp.experts.97.up_proj", "model.layers.11.mlp.experts.98.up_proj", "model.layers.11.mlp.experts.99.up_proj", "model.layers.11.mlp.experts.100.up_proj", "model.layers.11.mlp.experts.101.up_proj", "model.layers.11.mlp.experts.102.up_proj", "model.layers.11.mlp.experts.103.up_proj", "model.layers.11.mlp.experts.104.up_proj", "model.layers.11.mlp.experts.105.up_proj", "model.layers.11.mlp.experts.106.up_proj", "model.layers.11.mlp.experts.107.up_proj", "model.layers.11.mlp.experts.108.up_proj", "model.layers.11.mlp.experts.109.up_proj", "model.layers.11.mlp.experts.110.up_proj", "model.layers.11.mlp.experts.111.up_proj", "model.layers.11.mlp.experts.112.up_proj", "model.layers.11.mlp.experts.113.up_proj", "model.layers.11.mlp.experts.114.up_proj", "model.layers.11.mlp.experts.115.up_proj", "model.layers.11.mlp.experts.116.up_proj", "model.layers.11.mlp.experts.117.up_proj", "model.layers.11.mlp.experts.118.up_proj", "model.layers.11.mlp.experts.119.up_proj", "model.layers.11.mlp.experts.120.up_proj", "model.layers.11.mlp.experts.121.up_proj", "model.layers.11.mlp.experts.122.up_proj", "model.layers.11.mlp.experts.123.up_proj", "model.layers.11.mlp.experts.124.up_proj", "model.layers.11.mlp.experts.125.up_proj", "model.layers.11.mlp.experts.126.up_proj", "model.layers.11.mlp.experts.127.up_proj", "model.layers.11.mlp.experts.0.down_proj", "model.layers.11.mlp.experts.1.down_proj", "model.layers.11.mlp.experts.2.down_proj", "model.layers.11.mlp.experts.3.down_proj", "model.layers.11.mlp.experts.4.down_proj", "model.layers.11.mlp.experts.5.down_proj", "model.layers.11.mlp.experts.6.down_proj", "model.layers.11.mlp.experts.7.down_proj", "model.layers.11.mlp.experts.8.down_proj", "model.layers.11.mlp.experts.9.down_proj", "model.layers.11.mlp.experts.10.down_proj", "model.layers.11.mlp.experts.11.down_proj", "model.layers.11.mlp.experts.12.down_proj", "model.layers.11.mlp.experts.13.down_proj", "model.layers.11.mlp.experts.14.down_proj", "model.layers.11.mlp.experts.15.down_proj", "model.layers.11.mlp.experts.16.down_proj", "model.layers.11.mlp.experts.17.down_proj", "model.layers.11.mlp.experts.18.down_proj", "model.layers.11.mlp.experts.19.down_proj", "model.layers.11.mlp.experts.20.down_proj", "model.layers.11.mlp.experts.21.down_proj", "model.layers.11.mlp.experts.22.down_proj", "model.layers.11.mlp.experts.23.down_proj", "model.layers.11.mlp.experts.24.down_proj", "model.layers.11.mlp.experts.25.down_proj", "model.layers.11.mlp.experts.26.down_proj", "model.layers.11.mlp.experts.27.down_proj", "model.layers.11.mlp.experts.28.down_proj", "model.layers.11.mlp.experts.29.down_proj", "model.layers.11.mlp.experts.30.down_proj", "model.layers.11.mlp.experts.31.down_proj", "model.layers.11.mlp.experts.32.down_proj", "model.layers.11.mlp.experts.33.down_proj", "model.layers.11.mlp.experts.34.down_proj", "model.layers.11.mlp.experts.35.down_proj", "model.layers.11.mlp.experts.36.down_proj", "model.layers.11.mlp.experts.37.down_proj", "model.layers.11.mlp.experts.38.down_proj", "model.layers.11.mlp.experts.39.down_proj", "model.layers.11.mlp.experts.40.down_proj", "model.layers.11.mlp.experts.41.down_proj", "model.layers.11.mlp.experts.42.down_proj", "model.layers.11.mlp.experts.43.down_proj", "model.layers.11.mlp.experts.44.down_proj", "model.layers.11.mlp.experts.45.down_proj", "model.layers.11.mlp.experts.46.down_proj", "model.layers.11.mlp.experts.47.down_proj", "model.layers.11.mlp.experts.48.down_proj", "model.layers.11.mlp.experts.49.down_proj", "model.layers.11.mlp.experts.50.down_proj", "model.layers.11.mlp.experts.51.down_proj", "model.layers.11.mlp.experts.52.down_proj", "model.layers.11.mlp.experts.53.down_proj", "model.layers.11.mlp.experts.54.down_proj", "model.layers.11.mlp.experts.55.down_proj", "model.layers.11.mlp.experts.56.down_proj", "model.layers.11.mlp.experts.57.down_proj", "model.layers.11.mlp.experts.58.down_proj", "model.layers.11.mlp.experts.59.down_proj", "model.layers.11.mlp.experts.60.down_proj", "model.layers.11.mlp.experts.61.down_proj", "model.layers.11.mlp.experts.62.down_proj", "model.layers.11.mlp.experts.63.down_proj", "model.layers.11.mlp.experts.64.down_proj", "model.layers.11.mlp.experts.65.down_proj", "model.layers.11.mlp.experts.66.down_proj", "model.layers.11.mlp.experts.67.down_proj", "model.layers.11.mlp.experts.68.down_proj", "model.layers.11.mlp.experts.69.down_proj", "model.layers.11.mlp.experts.70.down_proj", "model.layers.11.mlp.experts.71.down_proj", "model.layers.11.mlp.experts.72.down_proj", "model.layers.11.mlp.experts.73.down_proj", "model.layers.11.mlp.experts.74.down_proj", "model.layers.11.mlp.experts.75.down_proj", "model.layers.11.mlp.experts.76.down_proj", "model.layers.11.mlp.experts.77.down_proj", "model.layers.11.mlp.experts.78.down_proj", "model.layers.11.mlp.experts.79.down_proj", "model.layers.11.mlp.experts.80.down_proj", "model.layers.11.mlp.experts.81.down_proj", "model.layers.11.mlp.experts.82.down_proj", "model.layers.11.mlp.experts.83.down_proj", "model.layers.11.mlp.experts.84.down_proj", "model.layers.11.mlp.experts.85.down_proj", "model.layers.11.mlp.experts.86.down_proj", "model.layers.11.mlp.experts.87.down_proj", "model.layers.11.mlp.experts.88.down_proj", "model.layers.11.mlp.experts.89.down_proj", "model.layers.11.mlp.experts.90.down_proj", "model.layers.11.mlp.experts.91.down_proj", "model.layers.11.mlp.experts.92.down_proj", "model.layers.11.mlp.experts.93.down_proj", "model.layers.11.mlp.experts.94.down_proj", "model.layers.11.mlp.experts.95.down_proj", "model.layers.11.mlp.experts.96.down_proj", "model.layers.11.mlp.experts.97.down_proj", "model.layers.11.mlp.experts.98.down_proj", "model.layers.11.mlp.experts.99.down_proj", "model.layers.11.mlp.experts.100.down_proj", "model.layers.11.mlp.experts.101.down_proj", "model.layers.11.mlp.experts.102.down_proj", "model.layers.11.mlp.experts.103.down_proj", "model.layers.11.mlp.experts.104.down_proj", "model.layers.11.mlp.experts.105.down_proj", "model.layers.11.mlp.experts.106.down_proj", "model.layers.11.mlp.experts.107.down_proj", "model.layers.11.mlp.experts.108.down_proj", "model.layers.11.mlp.experts.109.down_proj", "model.layers.11.mlp.experts.110.down_proj", "model.layers.11.mlp.experts.111.down_proj", "model.layers.11.mlp.experts.112.down_proj", "model.layers.11.mlp.experts.113.down_proj", "model.layers.11.mlp.experts.114.down_proj", "model.layers.11.mlp.experts.115.down_proj", "model.layers.11.mlp.experts.116.down_proj", "model.layers.11.mlp.experts.117.down_proj", "model.layers.11.mlp.experts.118.down_proj", "model.layers.11.mlp.experts.119.down_proj", "model.layers.11.mlp.experts.120.down_proj", "model.layers.11.mlp.experts.121.down_proj", "model.layers.11.mlp.experts.122.down_proj", "model.layers.11.mlp.experts.123.down_proj", "model.layers.11.mlp.experts.124.down_proj", "model.layers.11.mlp.experts.125.down_proj", "model.layers.11.mlp.experts.126.down_proj", "model.layers.11.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.004637201968580473, "dbits": 2214592512 } ] }, { "idx": 35, "layers": [ "model.layers.12.self_attn.q_proj", "model.layers.12.self_attn.k_proj", "model.layers.12.self_attn.v_proj", "model.layers.12.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0009279369376599678, "dbits": 109051904 } ] }, { "idx": 36, "layers": [ "model.layers.12.mlp.shared_experts.gate_proj", "model.layers.12.mlp.shared_experts.up_proj", "model.layers.12.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.004387972597032802, "dbits": 17301504 } ] }, { "idx": 37, "layers": [ "model.layers.12.mlp.experts.0.gate_proj", "model.layers.12.mlp.experts.1.gate_proj", "model.layers.12.mlp.experts.2.gate_proj", "model.layers.12.mlp.experts.3.gate_proj", "model.layers.12.mlp.experts.4.gate_proj", "model.layers.12.mlp.experts.5.gate_proj", "model.layers.12.mlp.experts.6.gate_proj", "model.layers.12.mlp.experts.7.gate_proj", "model.layers.12.mlp.experts.8.gate_proj", "model.layers.12.mlp.experts.9.gate_proj", "model.layers.12.mlp.experts.10.gate_proj", "model.layers.12.mlp.experts.11.gate_proj", "model.layers.12.mlp.experts.12.gate_proj", "model.layers.12.mlp.experts.13.gate_proj", "model.layers.12.mlp.experts.14.gate_proj", "model.layers.12.mlp.experts.15.gate_proj", "model.layers.12.mlp.experts.16.gate_proj", "model.layers.12.mlp.experts.17.gate_proj", "model.layers.12.mlp.experts.18.gate_proj", "model.layers.12.mlp.experts.19.gate_proj", "model.layers.12.mlp.experts.20.gate_proj", "model.layers.12.mlp.experts.21.gate_proj", "model.layers.12.mlp.experts.22.gate_proj", "model.layers.12.mlp.experts.23.gate_proj", "model.layers.12.mlp.experts.24.gate_proj", "model.layers.12.mlp.experts.25.gate_proj", "model.layers.12.mlp.experts.26.gate_proj", "model.layers.12.mlp.experts.27.gate_proj", "model.layers.12.mlp.experts.28.gate_proj", "model.layers.12.mlp.experts.29.gate_proj", "model.layers.12.mlp.experts.30.gate_proj", "model.layers.12.mlp.experts.31.gate_proj", "model.layers.12.mlp.experts.32.gate_proj", "model.layers.12.mlp.experts.33.gate_proj", "model.layers.12.mlp.experts.34.gate_proj", "model.layers.12.mlp.experts.35.gate_proj", "model.layers.12.mlp.experts.36.gate_proj", "model.layers.12.mlp.experts.37.gate_proj", "model.layers.12.mlp.experts.38.gate_proj", "model.layers.12.mlp.experts.39.gate_proj", "model.layers.12.mlp.experts.40.gate_proj", "model.layers.12.mlp.experts.41.gate_proj", "model.layers.12.mlp.experts.42.gate_proj", "model.layers.12.mlp.experts.43.gate_proj", "model.layers.12.mlp.experts.44.gate_proj", "model.layers.12.mlp.experts.45.gate_proj", "model.layers.12.mlp.experts.46.gate_proj", "model.layers.12.mlp.experts.47.gate_proj", "model.layers.12.mlp.experts.48.gate_proj", "model.layers.12.mlp.experts.49.gate_proj", "model.layers.12.mlp.experts.50.gate_proj", "model.layers.12.mlp.experts.51.gate_proj", "model.layers.12.mlp.experts.52.gate_proj", "model.layers.12.mlp.experts.53.gate_proj", "model.layers.12.mlp.experts.54.gate_proj", "model.layers.12.mlp.experts.55.gate_proj", "model.layers.12.mlp.experts.56.gate_proj", "model.layers.12.mlp.experts.57.gate_proj", "model.layers.12.mlp.experts.58.gate_proj", "model.layers.12.mlp.experts.59.gate_proj", "model.layers.12.mlp.experts.60.gate_proj", "model.layers.12.mlp.experts.61.gate_proj", "model.layers.12.mlp.experts.62.gate_proj", "model.layers.12.mlp.experts.63.gate_proj", "model.layers.12.mlp.experts.64.gate_proj", "model.layers.12.mlp.experts.65.gate_proj", "model.layers.12.mlp.experts.66.gate_proj", "model.layers.12.mlp.experts.67.gate_proj", "model.layers.12.mlp.experts.68.gate_proj", "model.layers.12.mlp.experts.69.gate_proj", "model.layers.12.mlp.experts.70.gate_proj", "model.layers.12.mlp.experts.71.gate_proj", "model.layers.12.mlp.experts.72.gate_proj", "model.layers.12.mlp.experts.73.gate_proj", "model.layers.12.mlp.experts.74.gate_proj", "model.layers.12.mlp.experts.75.gate_proj", "model.layers.12.mlp.experts.76.gate_proj", "model.layers.12.mlp.experts.77.gate_proj", "model.layers.12.mlp.experts.78.gate_proj", "model.layers.12.mlp.experts.79.gate_proj", "model.layers.12.mlp.experts.80.gate_proj", "model.layers.12.mlp.experts.81.gate_proj", "model.layers.12.mlp.experts.82.gate_proj", "model.layers.12.mlp.experts.83.gate_proj", "model.layers.12.mlp.experts.84.gate_proj", "model.layers.12.mlp.experts.85.gate_proj", "model.layers.12.mlp.experts.86.gate_proj", "model.layers.12.mlp.experts.87.gate_proj", "model.layers.12.mlp.experts.88.gate_proj", "model.layers.12.mlp.experts.89.gate_proj", "model.layers.12.mlp.experts.90.gate_proj", "model.layers.12.mlp.experts.91.gate_proj", "model.layers.12.mlp.experts.92.gate_proj", "model.layers.12.mlp.experts.93.gate_proj", "model.layers.12.mlp.experts.94.gate_proj", "model.layers.12.mlp.experts.95.gate_proj", "model.layers.12.mlp.experts.96.gate_proj", "model.layers.12.mlp.experts.97.gate_proj", "model.layers.12.mlp.experts.98.gate_proj", "model.layers.12.mlp.experts.99.gate_proj", "model.layers.12.mlp.experts.100.gate_proj", "model.layers.12.mlp.experts.101.gate_proj", "model.layers.12.mlp.experts.102.gate_proj", "model.layers.12.mlp.experts.103.gate_proj", "model.layers.12.mlp.experts.104.gate_proj", "model.layers.12.mlp.experts.105.gate_proj", "model.layers.12.mlp.experts.106.gate_proj", "model.layers.12.mlp.experts.107.gate_proj", "model.layers.12.mlp.experts.108.gate_proj", "model.layers.12.mlp.experts.109.gate_proj", "model.layers.12.mlp.experts.110.gate_proj", "model.layers.12.mlp.experts.111.gate_proj", "model.layers.12.mlp.experts.112.gate_proj", "model.layers.12.mlp.experts.113.gate_proj", "model.layers.12.mlp.experts.114.gate_proj", "model.layers.12.mlp.experts.115.gate_proj", "model.layers.12.mlp.experts.116.gate_proj", "model.layers.12.mlp.experts.117.gate_proj", "model.layers.12.mlp.experts.118.gate_proj", "model.layers.12.mlp.experts.119.gate_proj", "model.layers.12.mlp.experts.120.gate_proj", "model.layers.12.mlp.experts.121.gate_proj", "model.layers.12.mlp.experts.122.gate_proj", "model.layers.12.mlp.experts.123.gate_proj", "model.layers.12.mlp.experts.124.gate_proj", "model.layers.12.mlp.experts.125.gate_proj", "model.layers.12.mlp.experts.126.gate_proj", "model.layers.12.mlp.experts.127.gate_proj", "model.layers.12.mlp.experts.0.up_proj", "model.layers.12.mlp.experts.1.up_proj", "model.layers.12.mlp.experts.2.up_proj", "model.layers.12.mlp.experts.3.up_proj", "model.layers.12.mlp.experts.4.up_proj", "model.layers.12.mlp.experts.5.up_proj", "model.layers.12.mlp.experts.6.up_proj", "model.layers.12.mlp.experts.7.up_proj", "model.layers.12.mlp.experts.8.up_proj", "model.layers.12.mlp.experts.9.up_proj", "model.layers.12.mlp.experts.10.up_proj", "model.layers.12.mlp.experts.11.up_proj", "model.layers.12.mlp.experts.12.up_proj", "model.layers.12.mlp.experts.13.up_proj", "model.layers.12.mlp.experts.14.up_proj", "model.layers.12.mlp.experts.15.up_proj", "model.layers.12.mlp.experts.16.up_proj", "model.layers.12.mlp.experts.17.up_proj", "model.layers.12.mlp.experts.18.up_proj", "model.layers.12.mlp.experts.19.up_proj", "model.layers.12.mlp.experts.20.up_proj", "model.layers.12.mlp.experts.21.up_proj", "model.layers.12.mlp.experts.22.up_proj", "model.layers.12.mlp.experts.23.up_proj", "model.layers.12.mlp.experts.24.up_proj", "model.layers.12.mlp.experts.25.up_proj", "model.layers.12.mlp.experts.26.up_proj", "model.layers.12.mlp.experts.27.up_proj", "model.layers.12.mlp.experts.28.up_proj", "model.layers.12.mlp.experts.29.up_proj", "model.layers.12.mlp.experts.30.up_proj", "model.layers.12.mlp.experts.31.up_proj", "model.layers.12.mlp.experts.32.up_proj", "model.layers.12.mlp.experts.33.up_proj", "model.layers.12.mlp.experts.34.up_proj", "model.layers.12.mlp.experts.35.up_proj", "model.layers.12.mlp.experts.36.up_proj", "model.layers.12.mlp.experts.37.up_proj", "model.layers.12.mlp.experts.38.up_proj", "model.layers.12.mlp.experts.39.up_proj", "model.layers.12.mlp.experts.40.up_proj", "model.layers.12.mlp.experts.41.up_proj", "model.layers.12.mlp.experts.42.up_proj", "model.layers.12.mlp.experts.43.up_proj", "model.layers.12.mlp.experts.44.up_proj", "model.layers.12.mlp.experts.45.up_proj", "model.layers.12.mlp.experts.46.up_proj", "model.layers.12.mlp.experts.47.up_proj", "model.layers.12.mlp.experts.48.up_proj", "model.layers.12.mlp.experts.49.up_proj", "model.layers.12.mlp.experts.50.up_proj", "model.layers.12.mlp.experts.51.up_proj", "model.layers.12.mlp.experts.52.up_proj", "model.layers.12.mlp.experts.53.up_proj", "model.layers.12.mlp.experts.54.up_proj", "model.layers.12.mlp.experts.55.up_proj", "model.layers.12.mlp.experts.56.up_proj", "model.layers.12.mlp.experts.57.up_proj", "model.layers.12.mlp.experts.58.up_proj", "model.layers.12.mlp.experts.59.up_proj", "model.layers.12.mlp.experts.60.up_proj", "model.layers.12.mlp.experts.61.up_proj", "model.layers.12.mlp.experts.62.up_proj", "model.layers.12.mlp.experts.63.up_proj", "model.layers.12.mlp.experts.64.up_proj", "model.layers.12.mlp.experts.65.up_proj", "model.layers.12.mlp.experts.66.up_proj", "model.layers.12.mlp.experts.67.up_proj", "model.layers.12.mlp.experts.68.up_proj", "model.layers.12.mlp.experts.69.up_proj", "model.layers.12.mlp.experts.70.up_proj", "model.layers.12.mlp.experts.71.up_proj", "model.layers.12.mlp.experts.72.up_proj", "model.layers.12.mlp.experts.73.up_proj", "model.layers.12.mlp.experts.74.up_proj", "model.layers.12.mlp.experts.75.up_proj", "model.layers.12.mlp.experts.76.up_proj", "model.layers.12.mlp.experts.77.up_proj", "model.layers.12.mlp.experts.78.up_proj", "model.layers.12.mlp.experts.79.up_proj", "model.layers.12.mlp.experts.80.up_proj", "model.layers.12.mlp.experts.81.up_proj", "model.layers.12.mlp.experts.82.up_proj", "model.layers.12.mlp.experts.83.up_proj", "model.layers.12.mlp.experts.84.up_proj", "model.layers.12.mlp.experts.85.up_proj", "model.layers.12.mlp.experts.86.up_proj", "model.layers.12.mlp.experts.87.up_proj", "model.layers.12.mlp.experts.88.up_proj", "model.layers.12.mlp.experts.89.up_proj", "model.layers.12.mlp.experts.90.up_proj", "model.layers.12.mlp.experts.91.up_proj", "model.layers.12.mlp.experts.92.up_proj", "model.layers.12.mlp.experts.93.up_proj", "model.layers.12.mlp.experts.94.up_proj", "model.layers.12.mlp.experts.95.up_proj", "model.layers.12.mlp.experts.96.up_proj", "model.layers.12.mlp.experts.97.up_proj", "model.layers.12.mlp.experts.98.up_proj", "model.layers.12.mlp.experts.99.up_proj", "model.layers.12.mlp.experts.100.up_proj", "model.layers.12.mlp.experts.101.up_proj", "model.layers.12.mlp.experts.102.up_proj", "model.layers.12.mlp.experts.103.up_proj", "model.layers.12.mlp.experts.104.up_proj", "model.layers.12.mlp.experts.105.up_proj", "model.layers.12.mlp.experts.106.up_proj", "model.layers.12.mlp.experts.107.up_proj", "model.layers.12.mlp.experts.108.up_proj", "model.layers.12.mlp.experts.109.up_proj", "model.layers.12.mlp.experts.110.up_proj", "model.layers.12.mlp.experts.111.up_proj", "model.layers.12.mlp.experts.112.up_proj", "model.layers.12.mlp.experts.113.up_proj", "model.layers.12.mlp.experts.114.up_proj", "model.layers.12.mlp.experts.115.up_proj", "model.layers.12.mlp.experts.116.up_proj", "model.layers.12.mlp.experts.117.up_proj", "model.layers.12.mlp.experts.118.up_proj", "model.layers.12.mlp.experts.119.up_proj", "model.layers.12.mlp.experts.120.up_proj", "model.layers.12.mlp.experts.121.up_proj", "model.layers.12.mlp.experts.122.up_proj", "model.layers.12.mlp.experts.123.up_proj", "model.layers.12.mlp.experts.124.up_proj", "model.layers.12.mlp.experts.125.up_proj", "model.layers.12.mlp.experts.126.up_proj", "model.layers.12.mlp.experts.127.up_proj", "model.layers.12.mlp.experts.0.down_proj", "model.layers.12.mlp.experts.1.down_proj", "model.layers.12.mlp.experts.2.down_proj", "model.layers.12.mlp.experts.3.down_proj", "model.layers.12.mlp.experts.4.down_proj", "model.layers.12.mlp.experts.5.down_proj", "model.layers.12.mlp.experts.6.down_proj", "model.layers.12.mlp.experts.7.down_proj", "model.layers.12.mlp.experts.8.down_proj", "model.layers.12.mlp.experts.9.down_proj", "model.layers.12.mlp.experts.10.down_proj", "model.layers.12.mlp.experts.11.down_proj", "model.layers.12.mlp.experts.12.down_proj", "model.layers.12.mlp.experts.13.down_proj", "model.layers.12.mlp.experts.14.down_proj", "model.layers.12.mlp.experts.15.down_proj", "model.layers.12.mlp.experts.16.down_proj", "model.layers.12.mlp.experts.17.down_proj", "model.layers.12.mlp.experts.18.down_proj", "model.layers.12.mlp.experts.19.down_proj", "model.layers.12.mlp.experts.20.down_proj", "model.layers.12.mlp.experts.21.down_proj", "model.layers.12.mlp.experts.22.down_proj", "model.layers.12.mlp.experts.23.down_proj", "model.layers.12.mlp.experts.24.down_proj", "model.layers.12.mlp.experts.25.down_proj", "model.layers.12.mlp.experts.26.down_proj", "model.layers.12.mlp.experts.27.down_proj", "model.layers.12.mlp.experts.28.down_proj", "model.layers.12.mlp.experts.29.down_proj", "model.layers.12.mlp.experts.30.down_proj", "model.layers.12.mlp.experts.31.down_proj", "model.layers.12.mlp.experts.32.down_proj", "model.layers.12.mlp.experts.33.down_proj", "model.layers.12.mlp.experts.34.down_proj", "model.layers.12.mlp.experts.35.down_proj", "model.layers.12.mlp.experts.36.down_proj", "model.layers.12.mlp.experts.37.down_proj", "model.layers.12.mlp.experts.38.down_proj", "model.layers.12.mlp.experts.39.down_proj", "model.layers.12.mlp.experts.40.down_proj", "model.layers.12.mlp.experts.41.down_proj", "model.layers.12.mlp.experts.42.down_proj", "model.layers.12.mlp.experts.43.down_proj", "model.layers.12.mlp.experts.44.down_proj", "model.layers.12.mlp.experts.45.down_proj", "model.layers.12.mlp.experts.46.down_proj", "model.layers.12.mlp.experts.47.down_proj", "model.layers.12.mlp.experts.48.down_proj", "model.layers.12.mlp.experts.49.down_proj", "model.layers.12.mlp.experts.50.down_proj", "model.layers.12.mlp.experts.51.down_proj", "model.layers.12.mlp.experts.52.down_proj", "model.layers.12.mlp.experts.53.down_proj", "model.layers.12.mlp.experts.54.down_proj", "model.layers.12.mlp.experts.55.down_proj", "model.layers.12.mlp.experts.56.down_proj", "model.layers.12.mlp.experts.57.down_proj", "model.layers.12.mlp.experts.58.down_proj", "model.layers.12.mlp.experts.59.down_proj", "model.layers.12.mlp.experts.60.down_proj", "model.layers.12.mlp.experts.61.down_proj", "model.layers.12.mlp.experts.62.down_proj", "model.layers.12.mlp.experts.63.down_proj", "model.layers.12.mlp.experts.64.down_proj", "model.layers.12.mlp.experts.65.down_proj", "model.layers.12.mlp.experts.66.down_proj", "model.layers.12.mlp.experts.67.down_proj", "model.layers.12.mlp.experts.68.down_proj", "model.layers.12.mlp.experts.69.down_proj", "model.layers.12.mlp.experts.70.down_proj", "model.layers.12.mlp.experts.71.down_proj", "model.layers.12.mlp.experts.72.down_proj", "model.layers.12.mlp.experts.73.down_proj", "model.layers.12.mlp.experts.74.down_proj", "model.layers.12.mlp.experts.75.down_proj", "model.layers.12.mlp.experts.76.down_proj", "model.layers.12.mlp.experts.77.down_proj", "model.layers.12.mlp.experts.78.down_proj", "model.layers.12.mlp.experts.79.down_proj", "model.layers.12.mlp.experts.80.down_proj", "model.layers.12.mlp.experts.81.down_proj", "model.layers.12.mlp.experts.82.down_proj", "model.layers.12.mlp.experts.83.down_proj", "model.layers.12.mlp.experts.84.down_proj", "model.layers.12.mlp.experts.85.down_proj", "model.layers.12.mlp.experts.86.down_proj", "model.layers.12.mlp.experts.87.down_proj", "model.layers.12.mlp.experts.88.down_proj", "model.layers.12.mlp.experts.89.down_proj", "model.layers.12.mlp.experts.90.down_proj", "model.layers.12.mlp.experts.91.down_proj", "model.layers.12.mlp.experts.92.down_proj", "model.layers.12.mlp.experts.93.down_proj", "model.layers.12.mlp.experts.94.down_proj", "model.layers.12.mlp.experts.95.down_proj", "model.layers.12.mlp.experts.96.down_proj", "model.layers.12.mlp.experts.97.down_proj", "model.layers.12.mlp.experts.98.down_proj", "model.layers.12.mlp.experts.99.down_proj", "model.layers.12.mlp.experts.100.down_proj", "model.layers.12.mlp.experts.101.down_proj", "model.layers.12.mlp.experts.102.down_proj", "model.layers.12.mlp.experts.103.down_proj", "model.layers.12.mlp.experts.104.down_proj", "model.layers.12.mlp.experts.105.down_proj", "model.layers.12.mlp.experts.106.down_proj", "model.layers.12.mlp.experts.107.down_proj", "model.layers.12.mlp.experts.108.down_proj", "model.layers.12.mlp.experts.109.down_proj", "model.layers.12.mlp.experts.110.down_proj", "model.layers.12.mlp.experts.111.down_proj", "model.layers.12.mlp.experts.112.down_proj", "model.layers.12.mlp.experts.113.down_proj", "model.layers.12.mlp.experts.114.down_proj", "model.layers.12.mlp.experts.115.down_proj", "model.layers.12.mlp.experts.116.down_proj", "model.layers.12.mlp.experts.117.down_proj", "model.layers.12.mlp.experts.118.down_proj", "model.layers.12.mlp.experts.119.down_proj", "model.layers.12.mlp.experts.120.down_proj", "model.layers.12.mlp.experts.121.down_proj", "model.layers.12.mlp.experts.122.down_proj", "model.layers.12.mlp.experts.123.down_proj", "model.layers.12.mlp.experts.124.down_proj", "model.layers.12.mlp.experts.125.down_proj", "model.layers.12.mlp.experts.126.down_proj", "model.layers.12.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0055915419943630695, "dbits": 2214592512 } ] }, { "idx": 38, "layers": [ "model.layers.13.self_attn.q_proj", "model.layers.13.self_attn.k_proj", "model.layers.13.self_attn.v_proj", "model.layers.13.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00029147723689673666, "dbits": 109051904 } ] }, { "idx": 39, "layers": [ "model.layers.13.mlp.shared_experts.gate_proj", "model.layers.13.mlp.shared_experts.up_proj", "model.layers.13.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0023138075135648306, "dbits": 17301504 } ] }, { "idx": 40, "layers": [ "model.layers.13.mlp.experts.0.gate_proj", "model.layers.13.mlp.experts.1.gate_proj", "model.layers.13.mlp.experts.2.gate_proj", "model.layers.13.mlp.experts.3.gate_proj", "model.layers.13.mlp.experts.4.gate_proj", "model.layers.13.mlp.experts.5.gate_proj", "model.layers.13.mlp.experts.6.gate_proj", "model.layers.13.mlp.experts.7.gate_proj", "model.layers.13.mlp.experts.8.gate_proj", "model.layers.13.mlp.experts.9.gate_proj", "model.layers.13.mlp.experts.10.gate_proj", "model.layers.13.mlp.experts.11.gate_proj", "model.layers.13.mlp.experts.12.gate_proj", "model.layers.13.mlp.experts.13.gate_proj", "model.layers.13.mlp.experts.14.gate_proj", "model.layers.13.mlp.experts.15.gate_proj", "model.layers.13.mlp.experts.16.gate_proj", "model.layers.13.mlp.experts.17.gate_proj", "model.layers.13.mlp.experts.18.gate_proj", "model.layers.13.mlp.experts.19.gate_proj", "model.layers.13.mlp.experts.20.gate_proj", "model.layers.13.mlp.experts.21.gate_proj", "model.layers.13.mlp.experts.22.gate_proj", "model.layers.13.mlp.experts.23.gate_proj", "model.layers.13.mlp.experts.24.gate_proj", "model.layers.13.mlp.experts.25.gate_proj", "model.layers.13.mlp.experts.26.gate_proj", "model.layers.13.mlp.experts.27.gate_proj", "model.layers.13.mlp.experts.28.gate_proj", "model.layers.13.mlp.experts.29.gate_proj", "model.layers.13.mlp.experts.30.gate_proj", "model.layers.13.mlp.experts.31.gate_proj", "model.layers.13.mlp.experts.32.gate_proj", "model.layers.13.mlp.experts.33.gate_proj", "model.layers.13.mlp.experts.34.gate_proj", "model.layers.13.mlp.experts.35.gate_proj", "model.layers.13.mlp.experts.36.gate_proj", "model.layers.13.mlp.experts.37.gate_proj", "model.layers.13.mlp.experts.38.gate_proj", "model.layers.13.mlp.experts.39.gate_proj", "model.layers.13.mlp.experts.40.gate_proj", "model.layers.13.mlp.experts.41.gate_proj", "model.layers.13.mlp.experts.42.gate_proj", "model.layers.13.mlp.experts.43.gate_proj", "model.layers.13.mlp.experts.44.gate_proj", "model.layers.13.mlp.experts.45.gate_proj", "model.layers.13.mlp.experts.46.gate_proj", "model.layers.13.mlp.experts.47.gate_proj", "model.layers.13.mlp.experts.48.gate_proj", "model.layers.13.mlp.experts.49.gate_proj", "model.layers.13.mlp.experts.50.gate_proj", "model.layers.13.mlp.experts.51.gate_proj", "model.layers.13.mlp.experts.52.gate_proj", "model.layers.13.mlp.experts.53.gate_proj", "model.layers.13.mlp.experts.54.gate_proj", "model.layers.13.mlp.experts.55.gate_proj", "model.layers.13.mlp.experts.56.gate_proj", "model.layers.13.mlp.experts.57.gate_proj", "model.layers.13.mlp.experts.58.gate_proj", "model.layers.13.mlp.experts.59.gate_proj", "model.layers.13.mlp.experts.60.gate_proj", "model.layers.13.mlp.experts.61.gate_proj", "model.layers.13.mlp.experts.62.gate_proj", "model.layers.13.mlp.experts.63.gate_proj", "model.layers.13.mlp.experts.64.gate_proj", "model.layers.13.mlp.experts.65.gate_proj", "model.layers.13.mlp.experts.66.gate_proj", "model.layers.13.mlp.experts.67.gate_proj", "model.layers.13.mlp.experts.68.gate_proj", "model.layers.13.mlp.experts.69.gate_proj", "model.layers.13.mlp.experts.70.gate_proj", "model.layers.13.mlp.experts.71.gate_proj", "model.layers.13.mlp.experts.72.gate_proj", "model.layers.13.mlp.experts.73.gate_proj", "model.layers.13.mlp.experts.74.gate_proj", "model.layers.13.mlp.experts.75.gate_proj", "model.layers.13.mlp.experts.76.gate_proj", "model.layers.13.mlp.experts.77.gate_proj", "model.layers.13.mlp.experts.78.gate_proj", "model.layers.13.mlp.experts.79.gate_proj", "model.layers.13.mlp.experts.80.gate_proj", "model.layers.13.mlp.experts.81.gate_proj", "model.layers.13.mlp.experts.82.gate_proj", "model.layers.13.mlp.experts.83.gate_proj", "model.layers.13.mlp.experts.84.gate_proj", "model.layers.13.mlp.experts.85.gate_proj", "model.layers.13.mlp.experts.86.gate_proj", "model.layers.13.mlp.experts.87.gate_proj", "model.layers.13.mlp.experts.88.gate_proj", "model.layers.13.mlp.experts.89.gate_proj", "model.layers.13.mlp.experts.90.gate_proj", "model.layers.13.mlp.experts.91.gate_proj", "model.layers.13.mlp.experts.92.gate_proj", "model.layers.13.mlp.experts.93.gate_proj", "model.layers.13.mlp.experts.94.gate_proj", "model.layers.13.mlp.experts.95.gate_proj", "model.layers.13.mlp.experts.96.gate_proj", "model.layers.13.mlp.experts.97.gate_proj", "model.layers.13.mlp.experts.98.gate_proj", "model.layers.13.mlp.experts.99.gate_proj", "model.layers.13.mlp.experts.100.gate_proj", "model.layers.13.mlp.experts.101.gate_proj", "model.layers.13.mlp.experts.102.gate_proj", "model.layers.13.mlp.experts.103.gate_proj", "model.layers.13.mlp.experts.104.gate_proj", "model.layers.13.mlp.experts.105.gate_proj", "model.layers.13.mlp.experts.106.gate_proj", "model.layers.13.mlp.experts.107.gate_proj", "model.layers.13.mlp.experts.108.gate_proj", "model.layers.13.mlp.experts.109.gate_proj", "model.layers.13.mlp.experts.110.gate_proj", "model.layers.13.mlp.experts.111.gate_proj", "model.layers.13.mlp.experts.112.gate_proj", "model.layers.13.mlp.experts.113.gate_proj", "model.layers.13.mlp.experts.114.gate_proj", "model.layers.13.mlp.experts.115.gate_proj", "model.layers.13.mlp.experts.116.gate_proj", "model.layers.13.mlp.experts.117.gate_proj", "model.layers.13.mlp.experts.118.gate_proj", "model.layers.13.mlp.experts.119.gate_proj", "model.layers.13.mlp.experts.120.gate_proj", "model.layers.13.mlp.experts.121.gate_proj", "model.layers.13.mlp.experts.122.gate_proj", "model.layers.13.mlp.experts.123.gate_proj", "model.layers.13.mlp.experts.124.gate_proj", "model.layers.13.mlp.experts.125.gate_proj", "model.layers.13.mlp.experts.126.gate_proj", "model.layers.13.mlp.experts.127.gate_proj", "model.layers.13.mlp.experts.0.up_proj", "model.layers.13.mlp.experts.1.up_proj", "model.layers.13.mlp.experts.2.up_proj", "model.layers.13.mlp.experts.3.up_proj", "model.layers.13.mlp.experts.4.up_proj", "model.layers.13.mlp.experts.5.up_proj", "model.layers.13.mlp.experts.6.up_proj", "model.layers.13.mlp.experts.7.up_proj", "model.layers.13.mlp.experts.8.up_proj", "model.layers.13.mlp.experts.9.up_proj", "model.layers.13.mlp.experts.10.up_proj", "model.layers.13.mlp.experts.11.up_proj", "model.layers.13.mlp.experts.12.up_proj", "model.layers.13.mlp.experts.13.up_proj", "model.layers.13.mlp.experts.14.up_proj", "model.layers.13.mlp.experts.15.up_proj", "model.layers.13.mlp.experts.16.up_proj", "model.layers.13.mlp.experts.17.up_proj", "model.layers.13.mlp.experts.18.up_proj", "model.layers.13.mlp.experts.19.up_proj", "model.layers.13.mlp.experts.20.up_proj", "model.layers.13.mlp.experts.21.up_proj", "model.layers.13.mlp.experts.22.up_proj", "model.layers.13.mlp.experts.23.up_proj", "model.layers.13.mlp.experts.24.up_proj", "model.layers.13.mlp.experts.25.up_proj", "model.layers.13.mlp.experts.26.up_proj", "model.layers.13.mlp.experts.27.up_proj", "model.layers.13.mlp.experts.28.up_proj", "model.layers.13.mlp.experts.29.up_proj", "model.layers.13.mlp.experts.30.up_proj", "model.layers.13.mlp.experts.31.up_proj", "model.layers.13.mlp.experts.32.up_proj", "model.layers.13.mlp.experts.33.up_proj", "model.layers.13.mlp.experts.34.up_proj", "model.layers.13.mlp.experts.35.up_proj", "model.layers.13.mlp.experts.36.up_proj", "model.layers.13.mlp.experts.37.up_proj", "model.layers.13.mlp.experts.38.up_proj", "model.layers.13.mlp.experts.39.up_proj", "model.layers.13.mlp.experts.40.up_proj", "model.layers.13.mlp.experts.41.up_proj", "model.layers.13.mlp.experts.42.up_proj", "model.layers.13.mlp.experts.43.up_proj", "model.layers.13.mlp.experts.44.up_proj", "model.layers.13.mlp.experts.45.up_proj", "model.layers.13.mlp.experts.46.up_proj", "model.layers.13.mlp.experts.47.up_proj", "model.layers.13.mlp.experts.48.up_proj", "model.layers.13.mlp.experts.49.up_proj", "model.layers.13.mlp.experts.50.up_proj", "model.layers.13.mlp.experts.51.up_proj", "model.layers.13.mlp.experts.52.up_proj", "model.layers.13.mlp.experts.53.up_proj", "model.layers.13.mlp.experts.54.up_proj", "model.layers.13.mlp.experts.55.up_proj", "model.layers.13.mlp.experts.56.up_proj", "model.layers.13.mlp.experts.57.up_proj", "model.layers.13.mlp.experts.58.up_proj", "model.layers.13.mlp.experts.59.up_proj", "model.layers.13.mlp.experts.60.up_proj", "model.layers.13.mlp.experts.61.up_proj", "model.layers.13.mlp.experts.62.up_proj", "model.layers.13.mlp.experts.63.up_proj", "model.layers.13.mlp.experts.64.up_proj", "model.layers.13.mlp.experts.65.up_proj", "model.layers.13.mlp.experts.66.up_proj", "model.layers.13.mlp.experts.67.up_proj", "model.layers.13.mlp.experts.68.up_proj", "model.layers.13.mlp.experts.69.up_proj", "model.layers.13.mlp.experts.70.up_proj", "model.layers.13.mlp.experts.71.up_proj", "model.layers.13.mlp.experts.72.up_proj", "model.layers.13.mlp.experts.73.up_proj", "model.layers.13.mlp.experts.74.up_proj", "model.layers.13.mlp.experts.75.up_proj", "model.layers.13.mlp.experts.76.up_proj", "model.layers.13.mlp.experts.77.up_proj", "model.layers.13.mlp.experts.78.up_proj", "model.layers.13.mlp.experts.79.up_proj", "model.layers.13.mlp.experts.80.up_proj", "model.layers.13.mlp.experts.81.up_proj", "model.layers.13.mlp.experts.82.up_proj", "model.layers.13.mlp.experts.83.up_proj", "model.layers.13.mlp.experts.84.up_proj", "model.layers.13.mlp.experts.85.up_proj", "model.layers.13.mlp.experts.86.up_proj", "model.layers.13.mlp.experts.87.up_proj", "model.layers.13.mlp.experts.88.up_proj", "model.layers.13.mlp.experts.89.up_proj", "model.layers.13.mlp.experts.90.up_proj", "model.layers.13.mlp.experts.91.up_proj", "model.layers.13.mlp.experts.92.up_proj", "model.layers.13.mlp.experts.93.up_proj", "model.layers.13.mlp.experts.94.up_proj", "model.layers.13.mlp.experts.95.up_proj", "model.layers.13.mlp.experts.96.up_proj", "model.layers.13.mlp.experts.97.up_proj", "model.layers.13.mlp.experts.98.up_proj", "model.layers.13.mlp.experts.99.up_proj", "model.layers.13.mlp.experts.100.up_proj", "model.layers.13.mlp.experts.101.up_proj", "model.layers.13.mlp.experts.102.up_proj", "model.layers.13.mlp.experts.103.up_proj", "model.layers.13.mlp.experts.104.up_proj", "model.layers.13.mlp.experts.105.up_proj", "model.layers.13.mlp.experts.106.up_proj", "model.layers.13.mlp.experts.107.up_proj", "model.layers.13.mlp.experts.108.up_proj", "model.layers.13.mlp.experts.109.up_proj", "model.layers.13.mlp.experts.110.up_proj", "model.layers.13.mlp.experts.111.up_proj", "model.layers.13.mlp.experts.112.up_proj", "model.layers.13.mlp.experts.113.up_proj", "model.layers.13.mlp.experts.114.up_proj", "model.layers.13.mlp.experts.115.up_proj", "model.layers.13.mlp.experts.116.up_proj", "model.layers.13.mlp.experts.117.up_proj", "model.layers.13.mlp.experts.118.up_proj", "model.layers.13.mlp.experts.119.up_proj", "model.layers.13.mlp.experts.120.up_proj", "model.layers.13.mlp.experts.121.up_proj", "model.layers.13.mlp.experts.122.up_proj", "model.layers.13.mlp.experts.123.up_proj", "model.layers.13.mlp.experts.124.up_proj", "model.layers.13.mlp.experts.125.up_proj", "model.layers.13.mlp.experts.126.up_proj", "model.layers.13.mlp.experts.127.up_proj", "model.layers.13.mlp.experts.0.down_proj", "model.layers.13.mlp.experts.1.down_proj", "model.layers.13.mlp.experts.2.down_proj", "model.layers.13.mlp.experts.3.down_proj", "model.layers.13.mlp.experts.4.down_proj", "model.layers.13.mlp.experts.5.down_proj", "model.layers.13.mlp.experts.6.down_proj", "model.layers.13.mlp.experts.7.down_proj", "model.layers.13.mlp.experts.8.down_proj", "model.layers.13.mlp.experts.9.down_proj", "model.layers.13.mlp.experts.10.down_proj", "model.layers.13.mlp.experts.11.down_proj", "model.layers.13.mlp.experts.12.down_proj", "model.layers.13.mlp.experts.13.down_proj", "model.layers.13.mlp.experts.14.down_proj", "model.layers.13.mlp.experts.15.down_proj", "model.layers.13.mlp.experts.16.down_proj", "model.layers.13.mlp.experts.17.down_proj", "model.layers.13.mlp.experts.18.down_proj", "model.layers.13.mlp.experts.19.down_proj", "model.layers.13.mlp.experts.20.down_proj", "model.layers.13.mlp.experts.21.down_proj", "model.layers.13.mlp.experts.22.down_proj", "model.layers.13.mlp.experts.23.down_proj", "model.layers.13.mlp.experts.24.down_proj", "model.layers.13.mlp.experts.25.down_proj", "model.layers.13.mlp.experts.26.down_proj", "model.layers.13.mlp.experts.27.down_proj", "model.layers.13.mlp.experts.28.down_proj", "model.layers.13.mlp.experts.29.down_proj", "model.layers.13.mlp.experts.30.down_proj", "model.layers.13.mlp.experts.31.down_proj", "model.layers.13.mlp.experts.32.down_proj", "model.layers.13.mlp.experts.33.down_proj", "model.layers.13.mlp.experts.34.down_proj", "model.layers.13.mlp.experts.35.down_proj", "model.layers.13.mlp.experts.36.down_proj", "model.layers.13.mlp.experts.37.down_proj", "model.layers.13.mlp.experts.38.down_proj", "model.layers.13.mlp.experts.39.down_proj", "model.layers.13.mlp.experts.40.down_proj", "model.layers.13.mlp.experts.41.down_proj", "model.layers.13.mlp.experts.42.down_proj", "model.layers.13.mlp.experts.43.down_proj", "model.layers.13.mlp.experts.44.down_proj", "model.layers.13.mlp.experts.45.down_proj", "model.layers.13.mlp.experts.46.down_proj", "model.layers.13.mlp.experts.47.down_proj", "model.layers.13.mlp.experts.48.down_proj", "model.layers.13.mlp.experts.49.down_proj", "model.layers.13.mlp.experts.50.down_proj", "model.layers.13.mlp.experts.51.down_proj", "model.layers.13.mlp.experts.52.down_proj", "model.layers.13.mlp.experts.53.down_proj", "model.layers.13.mlp.experts.54.down_proj", "model.layers.13.mlp.experts.55.down_proj", "model.layers.13.mlp.experts.56.down_proj", "model.layers.13.mlp.experts.57.down_proj", "model.layers.13.mlp.experts.58.down_proj", "model.layers.13.mlp.experts.59.down_proj", "model.layers.13.mlp.experts.60.down_proj", "model.layers.13.mlp.experts.61.down_proj", "model.layers.13.mlp.experts.62.down_proj", "model.layers.13.mlp.experts.63.down_proj", "model.layers.13.mlp.experts.64.down_proj", "model.layers.13.mlp.experts.65.down_proj", "model.layers.13.mlp.experts.66.down_proj", "model.layers.13.mlp.experts.67.down_proj", "model.layers.13.mlp.experts.68.down_proj", "model.layers.13.mlp.experts.69.down_proj", "model.layers.13.mlp.experts.70.down_proj", "model.layers.13.mlp.experts.71.down_proj", "model.layers.13.mlp.experts.72.down_proj", "model.layers.13.mlp.experts.73.down_proj", "model.layers.13.mlp.experts.74.down_proj", "model.layers.13.mlp.experts.75.down_proj", "model.layers.13.mlp.experts.76.down_proj", "model.layers.13.mlp.experts.77.down_proj", "model.layers.13.mlp.experts.78.down_proj", "model.layers.13.mlp.experts.79.down_proj", "model.layers.13.mlp.experts.80.down_proj", "model.layers.13.mlp.experts.81.down_proj", "model.layers.13.mlp.experts.82.down_proj", "model.layers.13.mlp.experts.83.down_proj", "model.layers.13.mlp.experts.84.down_proj", "model.layers.13.mlp.experts.85.down_proj", "model.layers.13.mlp.experts.86.down_proj", "model.layers.13.mlp.experts.87.down_proj", "model.layers.13.mlp.experts.88.down_proj", "model.layers.13.mlp.experts.89.down_proj", "model.layers.13.mlp.experts.90.down_proj", "model.layers.13.mlp.experts.91.down_proj", "model.layers.13.mlp.experts.92.down_proj", "model.layers.13.mlp.experts.93.down_proj", "model.layers.13.mlp.experts.94.down_proj", "model.layers.13.mlp.experts.95.down_proj", "model.layers.13.mlp.experts.96.down_proj", "model.layers.13.mlp.experts.97.down_proj", "model.layers.13.mlp.experts.98.down_proj", "model.layers.13.mlp.experts.99.down_proj", "model.layers.13.mlp.experts.100.down_proj", "model.layers.13.mlp.experts.101.down_proj", "model.layers.13.mlp.experts.102.down_proj", "model.layers.13.mlp.experts.103.down_proj", "model.layers.13.mlp.experts.104.down_proj", "model.layers.13.mlp.experts.105.down_proj", "model.layers.13.mlp.experts.106.down_proj", "model.layers.13.mlp.experts.107.down_proj", "model.layers.13.mlp.experts.108.down_proj", "model.layers.13.mlp.experts.109.down_proj", "model.layers.13.mlp.experts.110.down_proj", "model.layers.13.mlp.experts.111.down_proj", "model.layers.13.mlp.experts.112.down_proj", "model.layers.13.mlp.experts.113.down_proj", "model.layers.13.mlp.experts.114.down_proj", "model.layers.13.mlp.experts.115.down_proj", "model.layers.13.mlp.experts.116.down_proj", "model.layers.13.mlp.experts.117.down_proj", "model.layers.13.mlp.experts.118.down_proj", "model.layers.13.mlp.experts.119.down_proj", "model.layers.13.mlp.experts.120.down_proj", "model.layers.13.mlp.experts.121.down_proj", "model.layers.13.mlp.experts.122.down_proj", "model.layers.13.mlp.experts.123.down_proj", "model.layers.13.mlp.experts.124.down_proj", "model.layers.13.mlp.experts.125.down_proj", "model.layers.13.mlp.experts.126.down_proj", "model.layers.13.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.005514808930456638, "dbits": 2214592512 } ] }, { "idx": 41, "layers": [ "model.layers.14.self_attn.q_proj", "model.layers.14.self_attn.k_proj", "model.layers.14.self_attn.v_proj", "model.layers.14.self_attn.o_proj" ], "candidates": [ { "dkld": -4.204362630849667e-06, "dbits": 109051904 } ] }, { "idx": 42, "layers": [ "model.layers.14.mlp.shared_experts.gate_proj", "model.layers.14.mlp.shared_experts.up_proj", "model.layers.14.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.001004421059042221, "dbits": 17301504 } ] }, { "idx": 43, "layers": [ "model.layers.14.mlp.experts.0.gate_proj", "model.layers.14.mlp.experts.1.gate_proj", "model.layers.14.mlp.experts.2.gate_proj", "model.layers.14.mlp.experts.3.gate_proj", "model.layers.14.mlp.experts.4.gate_proj", "model.layers.14.mlp.experts.5.gate_proj", "model.layers.14.mlp.experts.6.gate_proj", "model.layers.14.mlp.experts.7.gate_proj", "model.layers.14.mlp.experts.8.gate_proj", "model.layers.14.mlp.experts.9.gate_proj", "model.layers.14.mlp.experts.10.gate_proj", "model.layers.14.mlp.experts.11.gate_proj", "model.layers.14.mlp.experts.12.gate_proj", "model.layers.14.mlp.experts.13.gate_proj", "model.layers.14.mlp.experts.14.gate_proj", "model.layers.14.mlp.experts.15.gate_proj", "model.layers.14.mlp.experts.16.gate_proj", "model.layers.14.mlp.experts.17.gate_proj", "model.layers.14.mlp.experts.18.gate_proj", "model.layers.14.mlp.experts.19.gate_proj", "model.layers.14.mlp.experts.20.gate_proj", "model.layers.14.mlp.experts.21.gate_proj", "model.layers.14.mlp.experts.22.gate_proj", "model.layers.14.mlp.experts.23.gate_proj", "model.layers.14.mlp.experts.24.gate_proj", "model.layers.14.mlp.experts.25.gate_proj", "model.layers.14.mlp.experts.26.gate_proj", "model.layers.14.mlp.experts.27.gate_proj", "model.layers.14.mlp.experts.28.gate_proj", "model.layers.14.mlp.experts.29.gate_proj", "model.layers.14.mlp.experts.30.gate_proj", "model.layers.14.mlp.experts.31.gate_proj", "model.layers.14.mlp.experts.32.gate_proj", "model.layers.14.mlp.experts.33.gate_proj", "model.layers.14.mlp.experts.34.gate_proj", "model.layers.14.mlp.experts.35.gate_proj", "model.layers.14.mlp.experts.36.gate_proj", "model.layers.14.mlp.experts.37.gate_proj", "model.layers.14.mlp.experts.38.gate_proj", "model.layers.14.mlp.experts.39.gate_proj", "model.layers.14.mlp.experts.40.gate_proj", "model.layers.14.mlp.experts.41.gate_proj", "model.layers.14.mlp.experts.42.gate_proj", "model.layers.14.mlp.experts.43.gate_proj", "model.layers.14.mlp.experts.44.gate_proj", "model.layers.14.mlp.experts.45.gate_proj", "model.layers.14.mlp.experts.46.gate_proj", "model.layers.14.mlp.experts.47.gate_proj", "model.layers.14.mlp.experts.48.gate_proj", "model.layers.14.mlp.experts.49.gate_proj", "model.layers.14.mlp.experts.50.gate_proj", "model.layers.14.mlp.experts.51.gate_proj", "model.layers.14.mlp.experts.52.gate_proj", "model.layers.14.mlp.experts.53.gate_proj", "model.layers.14.mlp.experts.54.gate_proj", "model.layers.14.mlp.experts.55.gate_proj", "model.layers.14.mlp.experts.56.gate_proj", "model.layers.14.mlp.experts.57.gate_proj", "model.layers.14.mlp.experts.58.gate_proj", "model.layers.14.mlp.experts.59.gate_proj", "model.layers.14.mlp.experts.60.gate_proj", "model.layers.14.mlp.experts.61.gate_proj", "model.layers.14.mlp.experts.62.gate_proj", "model.layers.14.mlp.experts.63.gate_proj", "model.layers.14.mlp.experts.64.gate_proj", "model.layers.14.mlp.experts.65.gate_proj", "model.layers.14.mlp.experts.66.gate_proj", "model.layers.14.mlp.experts.67.gate_proj", "model.layers.14.mlp.experts.68.gate_proj", "model.layers.14.mlp.experts.69.gate_proj", "model.layers.14.mlp.experts.70.gate_proj", "model.layers.14.mlp.experts.71.gate_proj", "model.layers.14.mlp.experts.72.gate_proj", "model.layers.14.mlp.experts.73.gate_proj", "model.layers.14.mlp.experts.74.gate_proj", "model.layers.14.mlp.experts.75.gate_proj", "model.layers.14.mlp.experts.76.gate_proj", "model.layers.14.mlp.experts.77.gate_proj", "model.layers.14.mlp.experts.78.gate_proj", "model.layers.14.mlp.experts.79.gate_proj", "model.layers.14.mlp.experts.80.gate_proj", "model.layers.14.mlp.experts.81.gate_proj", "model.layers.14.mlp.experts.82.gate_proj", "model.layers.14.mlp.experts.83.gate_proj", "model.layers.14.mlp.experts.84.gate_proj", "model.layers.14.mlp.experts.85.gate_proj", "model.layers.14.mlp.experts.86.gate_proj", "model.layers.14.mlp.experts.87.gate_proj", "model.layers.14.mlp.experts.88.gate_proj", "model.layers.14.mlp.experts.89.gate_proj", "model.layers.14.mlp.experts.90.gate_proj", "model.layers.14.mlp.experts.91.gate_proj", "model.layers.14.mlp.experts.92.gate_proj", "model.layers.14.mlp.experts.93.gate_proj", "model.layers.14.mlp.experts.94.gate_proj", "model.layers.14.mlp.experts.95.gate_proj", "model.layers.14.mlp.experts.96.gate_proj", "model.layers.14.mlp.experts.97.gate_proj", "model.layers.14.mlp.experts.98.gate_proj", "model.layers.14.mlp.experts.99.gate_proj", "model.layers.14.mlp.experts.100.gate_proj", "model.layers.14.mlp.experts.101.gate_proj", "model.layers.14.mlp.experts.102.gate_proj", "model.layers.14.mlp.experts.103.gate_proj", "model.layers.14.mlp.experts.104.gate_proj", "model.layers.14.mlp.experts.105.gate_proj", "model.layers.14.mlp.experts.106.gate_proj", "model.layers.14.mlp.experts.107.gate_proj", "model.layers.14.mlp.experts.108.gate_proj", "model.layers.14.mlp.experts.109.gate_proj", "model.layers.14.mlp.experts.110.gate_proj", "model.layers.14.mlp.experts.111.gate_proj", "model.layers.14.mlp.experts.112.gate_proj", "model.layers.14.mlp.experts.113.gate_proj", "model.layers.14.mlp.experts.114.gate_proj", "model.layers.14.mlp.experts.115.gate_proj", "model.layers.14.mlp.experts.116.gate_proj", "model.layers.14.mlp.experts.117.gate_proj", "model.layers.14.mlp.experts.118.gate_proj", "model.layers.14.mlp.experts.119.gate_proj", "model.layers.14.mlp.experts.120.gate_proj", "model.layers.14.mlp.experts.121.gate_proj", "model.layers.14.mlp.experts.122.gate_proj", "model.layers.14.mlp.experts.123.gate_proj", "model.layers.14.mlp.experts.124.gate_proj", "model.layers.14.mlp.experts.125.gate_proj", "model.layers.14.mlp.experts.126.gate_proj", "model.layers.14.mlp.experts.127.gate_proj", "model.layers.14.mlp.experts.0.up_proj", "model.layers.14.mlp.experts.1.up_proj", "model.layers.14.mlp.experts.2.up_proj", "model.layers.14.mlp.experts.3.up_proj", "model.layers.14.mlp.experts.4.up_proj", "model.layers.14.mlp.experts.5.up_proj", "model.layers.14.mlp.experts.6.up_proj", "model.layers.14.mlp.experts.7.up_proj", "model.layers.14.mlp.experts.8.up_proj", "model.layers.14.mlp.experts.9.up_proj", "model.layers.14.mlp.experts.10.up_proj", "model.layers.14.mlp.experts.11.up_proj", "model.layers.14.mlp.experts.12.up_proj", "model.layers.14.mlp.experts.13.up_proj", "model.layers.14.mlp.experts.14.up_proj", "model.layers.14.mlp.experts.15.up_proj", "model.layers.14.mlp.experts.16.up_proj", "model.layers.14.mlp.experts.17.up_proj", "model.layers.14.mlp.experts.18.up_proj", "model.layers.14.mlp.experts.19.up_proj", "model.layers.14.mlp.experts.20.up_proj", "model.layers.14.mlp.experts.21.up_proj", "model.layers.14.mlp.experts.22.up_proj", "model.layers.14.mlp.experts.23.up_proj", "model.layers.14.mlp.experts.24.up_proj", "model.layers.14.mlp.experts.25.up_proj", "model.layers.14.mlp.experts.26.up_proj", "model.layers.14.mlp.experts.27.up_proj", "model.layers.14.mlp.experts.28.up_proj", "model.layers.14.mlp.experts.29.up_proj", "model.layers.14.mlp.experts.30.up_proj", "model.layers.14.mlp.experts.31.up_proj", "model.layers.14.mlp.experts.32.up_proj", "model.layers.14.mlp.experts.33.up_proj", "model.layers.14.mlp.experts.34.up_proj", "model.layers.14.mlp.experts.35.up_proj", "model.layers.14.mlp.experts.36.up_proj", "model.layers.14.mlp.experts.37.up_proj", "model.layers.14.mlp.experts.38.up_proj", "model.layers.14.mlp.experts.39.up_proj", "model.layers.14.mlp.experts.40.up_proj", "model.layers.14.mlp.experts.41.up_proj", "model.layers.14.mlp.experts.42.up_proj", "model.layers.14.mlp.experts.43.up_proj", "model.layers.14.mlp.experts.44.up_proj", "model.layers.14.mlp.experts.45.up_proj", "model.layers.14.mlp.experts.46.up_proj", "model.layers.14.mlp.experts.47.up_proj", "model.layers.14.mlp.experts.48.up_proj", "model.layers.14.mlp.experts.49.up_proj", "model.layers.14.mlp.experts.50.up_proj", "model.layers.14.mlp.experts.51.up_proj", "model.layers.14.mlp.experts.52.up_proj", "model.layers.14.mlp.experts.53.up_proj", "model.layers.14.mlp.experts.54.up_proj", "model.layers.14.mlp.experts.55.up_proj", "model.layers.14.mlp.experts.56.up_proj", "model.layers.14.mlp.experts.57.up_proj", "model.layers.14.mlp.experts.58.up_proj", "model.layers.14.mlp.experts.59.up_proj", "model.layers.14.mlp.experts.60.up_proj", "model.layers.14.mlp.experts.61.up_proj", "model.layers.14.mlp.experts.62.up_proj", "model.layers.14.mlp.experts.63.up_proj", "model.layers.14.mlp.experts.64.up_proj", "model.layers.14.mlp.experts.65.up_proj", "model.layers.14.mlp.experts.66.up_proj", "model.layers.14.mlp.experts.67.up_proj", "model.layers.14.mlp.experts.68.up_proj", "model.layers.14.mlp.experts.69.up_proj", "model.layers.14.mlp.experts.70.up_proj", "model.layers.14.mlp.experts.71.up_proj", "model.layers.14.mlp.experts.72.up_proj", "model.layers.14.mlp.experts.73.up_proj", "model.layers.14.mlp.experts.74.up_proj", "model.layers.14.mlp.experts.75.up_proj", "model.layers.14.mlp.experts.76.up_proj", "model.layers.14.mlp.experts.77.up_proj", "model.layers.14.mlp.experts.78.up_proj", "model.layers.14.mlp.experts.79.up_proj", "model.layers.14.mlp.experts.80.up_proj", "model.layers.14.mlp.experts.81.up_proj", "model.layers.14.mlp.experts.82.up_proj", "model.layers.14.mlp.experts.83.up_proj", "model.layers.14.mlp.experts.84.up_proj", "model.layers.14.mlp.experts.85.up_proj", "model.layers.14.mlp.experts.86.up_proj", "model.layers.14.mlp.experts.87.up_proj", "model.layers.14.mlp.experts.88.up_proj", "model.layers.14.mlp.experts.89.up_proj", "model.layers.14.mlp.experts.90.up_proj", "model.layers.14.mlp.experts.91.up_proj", "model.layers.14.mlp.experts.92.up_proj", "model.layers.14.mlp.experts.93.up_proj", "model.layers.14.mlp.experts.94.up_proj", "model.layers.14.mlp.experts.95.up_proj", "model.layers.14.mlp.experts.96.up_proj", "model.layers.14.mlp.experts.97.up_proj", "model.layers.14.mlp.experts.98.up_proj", "model.layers.14.mlp.experts.99.up_proj", "model.layers.14.mlp.experts.100.up_proj", "model.layers.14.mlp.experts.101.up_proj", "model.layers.14.mlp.experts.102.up_proj", "model.layers.14.mlp.experts.103.up_proj", "model.layers.14.mlp.experts.104.up_proj", "model.layers.14.mlp.experts.105.up_proj", "model.layers.14.mlp.experts.106.up_proj", "model.layers.14.mlp.experts.107.up_proj", "model.layers.14.mlp.experts.108.up_proj", "model.layers.14.mlp.experts.109.up_proj", "model.layers.14.mlp.experts.110.up_proj", "model.layers.14.mlp.experts.111.up_proj", "model.layers.14.mlp.experts.112.up_proj", "model.layers.14.mlp.experts.113.up_proj", "model.layers.14.mlp.experts.114.up_proj", "model.layers.14.mlp.experts.115.up_proj", "model.layers.14.mlp.experts.116.up_proj", "model.layers.14.mlp.experts.117.up_proj", "model.layers.14.mlp.experts.118.up_proj", "model.layers.14.mlp.experts.119.up_proj", "model.layers.14.mlp.experts.120.up_proj", "model.layers.14.mlp.experts.121.up_proj", "model.layers.14.mlp.experts.122.up_proj", "model.layers.14.mlp.experts.123.up_proj", "model.layers.14.mlp.experts.124.up_proj", "model.layers.14.mlp.experts.125.up_proj", "model.layers.14.mlp.experts.126.up_proj", "model.layers.14.mlp.experts.127.up_proj", "model.layers.14.mlp.experts.0.down_proj", "model.layers.14.mlp.experts.1.down_proj", "model.layers.14.mlp.experts.2.down_proj", "model.layers.14.mlp.experts.3.down_proj", "model.layers.14.mlp.experts.4.down_proj", "model.layers.14.mlp.experts.5.down_proj", "model.layers.14.mlp.experts.6.down_proj", "model.layers.14.mlp.experts.7.down_proj", "model.layers.14.mlp.experts.8.down_proj", "model.layers.14.mlp.experts.9.down_proj", "model.layers.14.mlp.experts.10.down_proj", "model.layers.14.mlp.experts.11.down_proj", "model.layers.14.mlp.experts.12.down_proj", "model.layers.14.mlp.experts.13.down_proj", "model.layers.14.mlp.experts.14.down_proj", "model.layers.14.mlp.experts.15.down_proj", "model.layers.14.mlp.experts.16.down_proj", "model.layers.14.mlp.experts.17.down_proj", "model.layers.14.mlp.experts.18.down_proj", "model.layers.14.mlp.experts.19.down_proj", "model.layers.14.mlp.experts.20.down_proj", "model.layers.14.mlp.experts.21.down_proj", "model.layers.14.mlp.experts.22.down_proj", "model.layers.14.mlp.experts.23.down_proj", "model.layers.14.mlp.experts.24.down_proj", "model.layers.14.mlp.experts.25.down_proj", "model.layers.14.mlp.experts.26.down_proj", "model.layers.14.mlp.experts.27.down_proj", "model.layers.14.mlp.experts.28.down_proj", "model.layers.14.mlp.experts.29.down_proj", "model.layers.14.mlp.experts.30.down_proj", "model.layers.14.mlp.experts.31.down_proj", "model.layers.14.mlp.experts.32.down_proj", "model.layers.14.mlp.experts.33.down_proj", "model.layers.14.mlp.experts.34.down_proj", "model.layers.14.mlp.experts.35.down_proj", "model.layers.14.mlp.experts.36.down_proj", "model.layers.14.mlp.experts.37.down_proj", "model.layers.14.mlp.experts.38.down_proj", "model.layers.14.mlp.experts.39.down_proj", "model.layers.14.mlp.experts.40.down_proj", "model.layers.14.mlp.experts.41.down_proj", "model.layers.14.mlp.experts.42.down_proj", "model.layers.14.mlp.experts.43.down_proj", "model.layers.14.mlp.experts.44.down_proj", "model.layers.14.mlp.experts.45.down_proj", "model.layers.14.mlp.experts.46.down_proj", "model.layers.14.mlp.experts.47.down_proj", "model.layers.14.mlp.experts.48.down_proj", "model.layers.14.mlp.experts.49.down_proj", "model.layers.14.mlp.experts.50.down_proj", "model.layers.14.mlp.experts.51.down_proj", "model.layers.14.mlp.experts.52.down_proj", "model.layers.14.mlp.experts.53.down_proj", "model.layers.14.mlp.experts.54.down_proj", "model.layers.14.mlp.experts.55.down_proj", "model.layers.14.mlp.experts.56.down_proj", "model.layers.14.mlp.experts.57.down_proj", "model.layers.14.mlp.experts.58.down_proj", "model.layers.14.mlp.experts.59.down_proj", "model.layers.14.mlp.experts.60.down_proj", "model.layers.14.mlp.experts.61.down_proj", "model.layers.14.mlp.experts.62.down_proj", "model.layers.14.mlp.experts.63.down_proj", "model.layers.14.mlp.experts.64.down_proj", "model.layers.14.mlp.experts.65.down_proj", "model.layers.14.mlp.experts.66.down_proj", "model.layers.14.mlp.experts.67.down_proj", "model.layers.14.mlp.experts.68.down_proj", "model.layers.14.mlp.experts.69.down_proj", "model.layers.14.mlp.experts.70.down_proj", "model.layers.14.mlp.experts.71.down_proj", "model.layers.14.mlp.experts.72.down_proj", "model.layers.14.mlp.experts.73.down_proj", "model.layers.14.mlp.experts.74.down_proj", "model.layers.14.mlp.experts.75.down_proj", "model.layers.14.mlp.experts.76.down_proj", "model.layers.14.mlp.experts.77.down_proj", "model.layers.14.mlp.experts.78.down_proj", "model.layers.14.mlp.experts.79.down_proj", "model.layers.14.mlp.experts.80.down_proj", "model.layers.14.mlp.experts.81.down_proj", "model.layers.14.mlp.experts.82.down_proj", "model.layers.14.mlp.experts.83.down_proj", "model.layers.14.mlp.experts.84.down_proj", "model.layers.14.mlp.experts.85.down_proj", "model.layers.14.mlp.experts.86.down_proj", "model.layers.14.mlp.experts.87.down_proj", "model.layers.14.mlp.experts.88.down_proj", "model.layers.14.mlp.experts.89.down_proj", "model.layers.14.mlp.experts.90.down_proj", "model.layers.14.mlp.experts.91.down_proj", "model.layers.14.mlp.experts.92.down_proj", "model.layers.14.mlp.experts.93.down_proj", "model.layers.14.mlp.experts.94.down_proj", "model.layers.14.mlp.experts.95.down_proj", "model.layers.14.mlp.experts.96.down_proj", "model.layers.14.mlp.experts.97.down_proj", "model.layers.14.mlp.experts.98.down_proj", "model.layers.14.mlp.experts.99.down_proj", "model.layers.14.mlp.experts.100.down_proj", "model.layers.14.mlp.experts.101.down_proj", "model.layers.14.mlp.experts.102.down_proj", "model.layers.14.mlp.experts.103.down_proj", "model.layers.14.mlp.experts.104.down_proj", "model.layers.14.mlp.experts.105.down_proj", "model.layers.14.mlp.experts.106.down_proj", "model.layers.14.mlp.experts.107.down_proj", "model.layers.14.mlp.experts.108.down_proj", "model.layers.14.mlp.experts.109.down_proj", "model.layers.14.mlp.experts.110.down_proj", "model.layers.14.mlp.experts.111.down_proj", "model.layers.14.mlp.experts.112.down_proj", "model.layers.14.mlp.experts.113.down_proj", "model.layers.14.mlp.experts.114.down_proj", "model.layers.14.mlp.experts.115.down_proj", "model.layers.14.mlp.experts.116.down_proj", "model.layers.14.mlp.experts.117.down_proj", "model.layers.14.mlp.experts.118.down_proj", "model.layers.14.mlp.experts.119.down_proj", "model.layers.14.mlp.experts.120.down_proj", "model.layers.14.mlp.experts.121.down_proj", "model.layers.14.mlp.experts.122.down_proj", "model.layers.14.mlp.experts.123.down_proj", "model.layers.14.mlp.experts.124.down_proj", "model.layers.14.mlp.experts.125.down_proj", "model.layers.14.mlp.experts.126.down_proj", "model.layers.14.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.003592845611274237, "dbits": 2214592512 } ] }, { "idx": 44, "layers": [ "model.layers.15.self_attn.q_proj", "model.layers.15.self_attn.k_proj", "model.layers.15.self_attn.v_proj", "model.layers.15.self_attn.o_proj" ], "candidates": [ { "dkld": -0.001576653216034174, "dbits": 109051904 } ] }, { "idx": 45, "layers": [ "model.layers.15.mlp.shared_experts.gate_proj", "model.layers.15.mlp.shared_experts.up_proj", "model.layers.15.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.003047170117497444, "dbits": 17301504 } ] }, { "idx": 46, "layers": [ "model.layers.15.mlp.experts.0.gate_proj", "model.layers.15.mlp.experts.1.gate_proj", "model.layers.15.mlp.experts.2.gate_proj", "model.layers.15.mlp.experts.3.gate_proj", "model.layers.15.mlp.experts.4.gate_proj", "model.layers.15.mlp.experts.5.gate_proj", "model.layers.15.mlp.experts.6.gate_proj", "model.layers.15.mlp.experts.7.gate_proj", "model.layers.15.mlp.experts.8.gate_proj", "model.layers.15.mlp.experts.9.gate_proj", "model.layers.15.mlp.experts.10.gate_proj", "model.layers.15.mlp.experts.11.gate_proj", "model.layers.15.mlp.experts.12.gate_proj", "model.layers.15.mlp.experts.13.gate_proj", "model.layers.15.mlp.experts.14.gate_proj", "model.layers.15.mlp.experts.15.gate_proj", "model.layers.15.mlp.experts.16.gate_proj", "model.layers.15.mlp.experts.17.gate_proj", "model.layers.15.mlp.experts.18.gate_proj", "model.layers.15.mlp.experts.19.gate_proj", "model.layers.15.mlp.experts.20.gate_proj", "model.layers.15.mlp.experts.21.gate_proj", "model.layers.15.mlp.experts.22.gate_proj", "model.layers.15.mlp.experts.23.gate_proj", "model.layers.15.mlp.experts.24.gate_proj", "model.layers.15.mlp.experts.25.gate_proj", "model.layers.15.mlp.experts.26.gate_proj", "model.layers.15.mlp.experts.27.gate_proj", "model.layers.15.mlp.experts.28.gate_proj", "model.layers.15.mlp.experts.29.gate_proj", "model.layers.15.mlp.experts.30.gate_proj", "model.layers.15.mlp.experts.31.gate_proj", "model.layers.15.mlp.experts.32.gate_proj", "model.layers.15.mlp.experts.33.gate_proj", "model.layers.15.mlp.experts.34.gate_proj", "model.layers.15.mlp.experts.35.gate_proj", "model.layers.15.mlp.experts.36.gate_proj", "model.layers.15.mlp.experts.37.gate_proj", "model.layers.15.mlp.experts.38.gate_proj", "model.layers.15.mlp.experts.39.gate_proj", "model.layers.15.mlp.experts.40.gate_proj", "model.layers.15.mlp.experts.41.gate_proj", "model.layers.15.mlp.experts.42.gate_proj", "model.layers.15.mlp.experts.43.gate_proj", "model.layers.15.mlp.experts.44.gate_proj", "model.layers.15.mlp.experts.45.gate_proj", "model.layers.15.mlp.experts.46.gate_proj", "model.layers.15.mlp.experts.47.gate_proj", "model.layers.15.mlp.experts.48.gate_proj", "model.layers.15.mlp.experts.49.gate_proj", "model.layers.15.mlp.experts.50.gate_proj", "model.layers.15.mlp.experts.51.gate_proj", "model.layers.15.mlp.experts.52.gate_proj", "model.layers.15.mlp.experts.53.gate_proj", "model.layers.15.mlp.experts.54.gate_proj", "model.layers.15.mlp.experts.55.gate_proj", "model.layers.15.mlp.experts.56.gate_proj", "model.layers.15.mlp.experts.57.gate_proj", "model.layers.15.mlp.experts.58.gate_proj", "model.layers.15.mlp.experts.59.gate_proj", "model.layers.15.mlp.experts.60.gate_proj", "model.layers.15.mlp.experts.61.gate_proj", "model.layers.15.mlp.experts.62.gate_proj", "model.layers.15.mlp.experts.63.gate_proj", "model.layers.15.mlp.experts.64.gate_proj", "model.layers.15.mlp.experts.65.gate_proj", "model.layers.15.mlp.experts.66.gate_proj", "model.layers.15.mlp.experts.67.gate_proj", "model.layers.15.mlp.experts.68.gate_proj", "model.layers.15.mlp.experts.69.gate_proj", "model.layers.15.mlp.experts.70.gate_proj", "model.layers.15.mlp.experts.71.gate_proj", "model.layers.15.mlp.experts.72.gate_proj", "model.layers.15.mlp.experts.73.gate_proj", "model.layers.15.mlp.experts.74.gate_proj", "model.layers.15.mlp.experts.75.gate_proj", "model.layers.15.mlp.experts.76.gate_proj", "model.layers.15.mlp.experts.77.gate_proj", "model.layers.15.mlp.experts.78.gate_proj", "model.layers.15.mlp.experts.79.gate_proj", "model.layers.15.mlp.experts.80.gate_proj", "model.layers.15.mlp.experts.81.gate_proj", "model.layers.15.mlp.experts.82.gate_proj", "model.layers.15.mlp.experts.83.gate_proj", "model.layers.15.mlp.experts.84.gate_proj", "model.layers.15.mlp.experts.85.gate_proj", "model.layers.15.mlp.experts.86.gate_proj", "model.layers.15.mlp.experts.87.gate_proj", "model.layers.15.mlp.experts.88.gate_proj", "model.layers.15.mlp.experts.89.gate_proj", "model.layers.15.mlp.experts.90.gate_proj", "model.layers.15.mlp.experts.91.gate_proj", "model.layers.15.mlp.experts.92.gate_proj", "model.layers.15.mlp.experts.93.gate_proj", "model.layers.15.mlp.experts.94.gate_proj", "model.layers.15.mlp.experts.95.gate_proj", "model.layers.15.mlp.experts.96.gate_proj", "model.layers.15.mlp.experts.97.gate_proj", "model.layers.15.mlp.experts.98.gate_proj", "model.layers.15.mlp.experts.99.gate_proj", "model.layers.15.mlp.experts.100.gate_proj", "model.layers.15.mlp.experts.101.gate_proj", "model.layers.15.mlp.experts.102.gate_proj", "model.layers.15.mlp.experts.103.gate_proj", "model.layers.15.mlp.experts.104.gate_proj", "model.layers.15.mlp.experts.105.gate_proj", "model.layers.15.mlp.experts.106.gate_proj", "model.layers.15.mlp.experts.107.gate_proj", "model.layers.15.mlp.experts.108.gate_proj", "model.layers.15.mlp.experts.109.gate_proj", "model.layers.15.mlp.experts.110.gate_proj", "model.layers.15.mlp.experts.111.gate_proj", "model.layers.15.mlp.experts.112.gate_proj", "model.layers.15.mlp.experts.113.gate_proj", "model.layers.15.mlp.experts.114.gate_proj", "model.layers.15.mlp.experts.115.gate_proj", "model.layers.15.mlp.experts.116.gate_proj", "model.layers.15.mlp.experts.117.gate_proj", "model.layers.15.mlp.experts.118.gate_proj", "model.layers.15.mlp.experts.119.gate_proj", "model.layers.15.mlp.experts.120.gate_proj", "model.layers.15.mlp.experts.121.gate_proj", "model.layers.15.mlp.experts.122.gate_proj", "model.layers.15.mlp.experts.123.gate_proj", "model.layers.15.mlp.experts.124.gate_proj", "model.layers.15.mlp.experts.125.gate_proj", "model.layers.15.mlp.experts.126.gate_proj", "model.layers.15.mlp.experts.127.gate_proj", "model.layers.15.mlp.experts.0.up_proj", "model.layers.15.mlp.experts.1.up_proj", "model.layers.15.mlp.experts.2.up_proj", "model.layers.15.mlp.experts.3.up_proj", "model.layers.15.mlp.experts.4.up_proj", "model.layers.15.mlp.experts.5.up_proj", "model.layers.15.mlp.experts.6.up_proj", "model.layers.15.mlp.experts.7.up_proj", "model.layers.15.mlp.experts.8.up_proj", "model.layers.15.mlp.experts.9.up_proj", "model.layers.15.mlp.experts.10.up_proj", "model.layers.15.mlp.experts.11.up_proj", "model.layers.15.mlp.experts.12.up_proj", "model.layers.15.mlp.experts.13.up_proj", "model.layers.15.mlp.experts.14.up_proj", "model.layers.15.mlp.experts.15.up_proj", "model.layers.15.mlp.experts.16.up_proj", "model.layers.15.mlp.experts.17.up_proj", "model.layers.15.mlp.experts.18.up_proj", "model.layers.15.mlp.experts.19.up_proj", "model.layers.15.mlp.experts.20.up_proj", "model.layers.15.mlp.experts.21.up_proj", "model.layers.15.mlp.experts.22.up_proj", "model.layers.15.mlp.experts.23.up_proj", "model.layers.15.mlp.experts.24.up_proj", "model.layers.15.mlp.experts.25.up_proj", "model.layers.15.mlp.experts.26.up_proj", "model.layers.15.mlp.experts.27.up_proj", "model.layers.15.mlp.experts.28.up_proj", "model.layers.15.mlp.experts.29.up_proj", "model.layers.15.mlp.experts.30.up_proj", "model.layers.15.mlp.experts.31.up_proj", "model.layers.15.mlp.experts.32.up_proj", "model.layers.15.mlp.experts.33.up_proj", "model.layers.15.mlp.experts.34.up_proj", "model.layers.15.mlp.experts.35.up_proj", "model.layers.15.mlp.experts.36.up_proj", "model.layers.15.mlp.experts.37.up_proj", "model.layers.15.mlp.experts.38.up_proj", "model.layers.15.mlp.experts.39.up_proj", "model.layers.15.mlp.experts.40.up_proj", "model.layers.15.mlp.experts.41.up_proj", "model.layers.15.mlp.experts.42.up_proj", "model.layers.15.mlp.experts.43.up_proj", "model.layers.15.mlp.experts.44.up_proj", "model.layers.15.mlp.experts.45.up_proj", "model.layers.15.mlp.experts.46.up_proj", "model.layers.15.mlp.experts.47.up_proj", "model.layers.15.mlp.experts.48.up_proj", "model.layers.15.mlp.experts.49.up_proj", "model.layers.15.mlp.experts.50.up_proj", "model.layers.15.mlp.experts.51.up_proj", "model.layers.15.mlp.experts.52.up_proj", "model.layers.15.mlp.experts.53.up_proj", "model.layers.15.mlp.experts.54.up_proj", "model.layers.15.mlp.experts.55.up_proj", "model.layers.15.mlp.experts.56.up_proj", "model.layers.15.mlp.experts.57.up_proj", "model.layers.15.mlp.experts.58.up_proj", "model.layers.15.mlp.experts.59.up_proj", "model.layers.15.mlp.experts.60.up_proj", "model.layers.15.mlp.experts.61.up_proj", "model.layers.15.mlp.experts.62.up_proj", "model.layers.15.mlp.experts.63.up_proj", "model.layers.15.mlp.experts.64.up_proj", "model.layers.15.mlp.experts.65.up_proj", "model.layers.15.mlp.experts.66.up_proj", "model.layers.15.mlp.experts.67.up_proj", "model.layers.15.mlp.experts.68.up_proj", "model.layers.15.mlp.experts.69.up_proj", "model.layers.15.mlp.experts.70.up_proj", "model.layers.15.mlp.experts.71.up_proj", "model.layers.15.mlp.experts.72.up_proj", "model.layers.15.mlp.experts.73.up_proj", "model.layers.15.mlp.experts.74.up_proj", "model.layers.15.mlp.experts.75.up_proj", "model.layers.15.mlp.experts.76.up_proj", "model.layers.15.mlp.experts.77.up_proj", "model.layers.15.mlp.experts.78.up_proj", "model.layers.15.mlp.experts.79.up_proj", "model.layers.15.mlp.experts.80.up_proj", "model.layers.15.mlp.experts.81.up_proj", "model.layers.15.mlp.experts.82.up_proj", "model.layers.15.mlp.experts.83.up_proj", "model.layers.15.mlp.experts.84.up_proj", "model.layers.15.mlp.experts.85.up_proj", "model.layers.15.mlp.experts.86.up_proj", "model.layers.15.mlp.experts.87.up_proj", "model.layers.15.mlp.experts.88.up_proj", "model.layers.15.mlp.experts.89.up_proj", "model.layers.15.mlp.experts.90.up_proj", "model.layers.15.mlp.experts.91.up_proj", "model.layers.15.mlp.experts.92.up_proj", "model.layers.15.mlp.experts.93.up_proj", "model.layers.15.mlp.experts.94.up_proj", "model.layers.15.mlp.experts.95.up_proj", "model.layers.15.mlp.experts.96.up_proj", "model.layers.15.mlp.experts.97.up_proj", "model.layers.15.mlp.experts.98.up_proj", "model.layers.15.mlp.experts.99.up_proj", "model.layers.15.mlp.experts.100.up_proj", "model.layers.15.mlp.experts.101.up_proj", "model.layers.15.mlp.experts.102.up_proj", "model.layers.15.mlp.experts.103.up_proj", "model.layers.15.mlp.experts.104.up_proj", "model.layers.15.mlp.experts.105.up_proj", "model.layers.15.mlp.experts.106.up_proj", "model.layers.15.mlp.experts.107.up_proj", "model.layers.15.mlp.experts.108.up_proj", "model.layers.15.mlp.experts.109.up_proj", "model.layers.15.mlp.experts.110.up_proj", "model.layers.15.mlp.experts.111.up_proj", "model.layers.15.mlp.experts.112.up_proj", "model.layers.15.mlp.experts.113.up_proj", "model.layers.15.mlp.experts.114.up_proj", "model.layers.15.mlp.experts.115.up_proj", "model.layers.15.mlp.experts.116.up_proj", "model.layers.15.mlp.experts.117.up_proj", "model.layers.15.mlp.experts.118.up_proj", "model.layers.15.mlp.experts.119.up_proj", "model.layers.15.mlp.experts.120.up_proj", "model.layers.15.mlp.experts.121.up_proj", "model.layers.15.mlp.experts.122.up_proj", "model.layers.15.mlp.experts.123.up_proj", "model.layers.15.mlp.experts.124.up_proj", "model.layers.15.mlp.experts.125.up_proj", "model.layers.15.mlp.experts.126.up_proj", "model.layers.15.mlp.experts.127.up_proj", "model.layers.15.mlp.experts.0.down_proj", "model.layers.15.mlp.experts.1.down_proj", "model.layers.15.mlp.experts.2.down_proj", "model.layers.15.mlp.experts.3.down_proj", "model.layers.15.mlp.experts.4.down_proj", "model.layers.15.mlp.experts.5.down_proj", "model.layers.15.mlp.experts.6.down_proj", "model.layers.15.mlp.experts.7.down_proj", "model.layers.15.mlp.experts.8.down_proj", "model.layers.15.mlp.experts.9.down_proj", "model.layers.15.mlp.experts.10.down_proj", "model.layers.15.mlp.experts.11.down_proj", "model.layers.15.mlp.experts.12.down_proj", "model.layers.15.mlp.experts.13.down_proj", "model.layers.15.mlp.experts.14.down_proj", "model.layers.15.mlp.experts.15.down_proj", "model.layers.15.mlp.experts.16.down_proj", "model.layers.15.mlp.experts.17.down_proj", "model.layers.15.mlp.experts.18.down_proj", "model.layers.15.mlp.experts.19.down_proj", "model.layers.15.mlp.experts.20.down_proj", "model.layers.15.mlp.experts.21.down_proj", "model.layers.15.mlp.experts.22.down_proj", "model.layers.15.mlp.experts.23.down_proj", "model.layers.15.mlp.experts.24.down_proj", "model.layers.15.mlp.experts.25.down_proj", "model.layers.15.mlp.experts.26.down_proj", "model.layers.15.mlp.experts.27.down_proj", "model.layers.15.mlp.experts.28.down_proj", "model.layers.15.mlp.experts.29.down_proj", "model.layers.15.mlp.experts.30.down_proj", "model.layers.15.mlp.experts.31.down_proj", "model.layers.15.mlp.experts.32.down_proj", "model.layers.15.mlp.experts.33.down_proj", "model.layers.15.mlp.experts.34.down_proj", "model.layers.15.mlp.experts.35.down_proj", "model.layers.15.mlp.experts.36.down_proj", "model.layers.15.mlp.experts.37.down_proj", "model.layers.15.mlp.experts.38.down_proj", "model.layers.15.mlp.experts.39.down_proj", "model.layers.15.mlp.experts.40.down_proj", "model.layers.15.mlp.experts.41.down_proj", "model.layers.15.mlp.experts.42.down_proj", "model.layers.15.mlp.experts.43.down_proj", "model.layers.15.mlp.experts.44.down_proj", "model.layers.15.mlp.experts.45.down_proj", "model.layers.15.mlp.experts.46.down_proj", "model.layers.15.mlp.experts.47.down_proj", "model.layers.15.mlp.experts.48.down_proj", "model.layers.15.mlp.experts.49.down_proj", "model.layers.15.mlp.experts.50.down_proj", "model.layers.15.mlp.experts.51.down_proj", "model.layers.15.mlp.experts.52.down_proj", "model.layers.15.mlp.experts.53.down_proj", "model.layers.15.mlp.experts.54.down_proj", "model.layers.15.mlp.experts.55.down_proj", "model.layers.15.mlp.experts.56.down_proj", "model.layers.15.mlp.experts.57.down_proj", "model.layers.15.mlp.experts.58.down_proj", "model.layers.15.mlp.experts.59.down_proj", "model.layers.15.mlp.experts.60.down_proj", "model.layers.15.mlp.experts.61.down_proj", "model.layers.15.mlp.experts.62.down_proj", "model.layers.15.mlp.experts.63.down_proj", "model.layers.15.mlp.experts.64.down_proj", "model.layers.15.mlp.experts.65.down_proj", "model.layers.15.mlp.experts.66.down_proj", "model.layers.15.mlp.experts.67.down_proj", "model.layers.15.mlp.experts.68.down_proj", "model.layers.15.mlp.experts.69.down_proj", "model.layers.15.mlp.experts.70.down_proj", "model.layers.15.mlp.experts.71.down_proj", "model.layers.15.mlp.experts.72.down_proj", "model.layers.15.mlp.experts.73.down_proj", "model.layers.15.mlp.experts.74.down_proj", "model.layers.15.mlp.experts.75.down_proj", "model.layers.15.mlp.experts.76.down_proj", "model.layers.15.mlp.experts.77.down_proj", "model.layers.15.mlp.experts.78.down_proj", "model.layers.15.mlp.experts.79.down_proj", "model.layers.15.mlp.experts.80.down_proj", "model.layers.15.mlp.experts.81.down_proj", "model.layers.15.mlp.experts.82.down_proj", "model.layers.15.mlp.experts.83.down_proj", "model.layers.15.mlp.experts.84.down_proj", "model.layers.15.mlp.experts.85.down_proj", "model.layers.15.mlp.experts.86.down_proj", "model.layers.15.mlp.experts.87.down_proj", "model.layers.15.mlp.experts.88.down_proj", "model.layers.15.mlp.experts.89.down_proj", "model.layers.15.mlp.experts.90.down_proj", "model.layers.15.mlp.experts.91.down_proj", "model.layers.15.mlp.experts.92.down_proj", "model.layers.15.mlp.experts.93.down_proj", "model.layers.15.mlp.experts.94.down_proj", "model.layers.15.mlp.experts.95.down_proj", "model.layers.15.mlp.experts.96.down_proj", "model.layers.15.mlp.experts.97.down_proj", "model.layers.15.mlp.experts.98.down_proj", "model.layers.15.mlp.experts.99.down_proj", "model.layers.15.mlp.experts.100.down_proj", "model.layers.15.mlp.experts.101.down_proj", "model.layers.15.mlp.experts.102.down_proj", "model.layers.15.mlp.experts.103.down_proj", "model.layers.15.mlp.experts.104.down_proj", "model.layers.15.mlp.experts.105.down_proj", "model.layers.15.mlp.experts.106.down_proj", "model.layers.15.mlp.experts.107.down_proj", "model.layers.15.mlp.experts.108.down_proj", "model.layers.15.mlp.experts.109.down_proj", "model.layers.15.mlp.experts.110.down_proj", "model.layers.15.mlp.experts.111.down_proj", "model.layers.15.mlp.experts.112.down_proj", "model.layers.15.mlp.experts.113.down_proj", "model.layers.15.mlp.experts.114.down_proj", "model.layers.15.mlp.experts.115.down_proj", "model.layers.15.mlp.experts.116.down_proj", "model.layers.15.mlp.experts.117.down_proj", "model.layers.15.mlp.experts.118.down_proj", "model.layers.15.mlp.experts.119.down_proj", "model.layers.15.mlp.experts.120.down_proj", "model.layers.15.mlp.experts.121.down_proj", "model.layers.15.mlp.experts.122.down_proj", "model.layers.15.mlp.experts.123.down_proj", "model.layers.15.mlp.experts.124.down_proj", "model.layers.15.mlp.experts.125.down_proj", "model.layers.15.mlp.experts.126.down_proj", "model.layers.15.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0037107101641595253, "dbits": 2214592512 } ] }, { "idx": 47, "layers": [ "model.layers.16.self_attn.q_proj", "model.layers.16.self_attn.k_proj", "model.layers.16.self_attn.v_proj", "model.layers.16.self_attn.o_proj" ], "candidates": [ { "dkld": 0.00042323749512435116, "dbits": 109051904 } ] }, { "idx": 48, "layers": [ "model.layers.16.mlp.shared_experts.gate_proj", "model.layers.16.mlp.shared_experts.up_proj", "model.layers.16.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.003929769620299339, "dbits": 17301504 } ] }, { "idx": 49, "layers": [ "model.layers.16.mlp.experts.0.gate_proj", "model.layers.16.mlp.experts.1.gate_proj", "model.layers.16.mlp.experts.2.gate_proj", "model.layers.16.mlp.experts.3.gate_proj", "model.layers.16.mlp.experts.4.gate_proj", "model.layers.16.mlp.experts.5.gate_proj", "model.layers.16.mlp.experts.6.gate_proj", "model.layers.16.mlp.experts.7.gate_proj", "model.layers.16.mlp.experts.8.gate_proj", "model.layers.16.mlp.experts.9.gate_proj", "model.layers.16.mlp.experts.10.gate_proj", "model.layers.16.mlp.experts.11.gate_proj", "model.layers.16.mlp.experts.12.gate_proj", "model.layers.16.mlp.experts.13.gate_proj", "model.layers.16.mlp.experts.14.gate_proj", "model.layers.16.mlp.experts.15.gate_proj", "model.layers.16.mlp.experts.16.gate_proj", "model.layers.16.mlp.experts.17.gate_proj", "model.layers.16.mlp.experts.18.gate_proj", "model.layers.16.mlp.experts.19.gate_proj", "model.layers.16.mlp.experts.20.gate_proj", "model.layers.16.mlp.experts.21.gate_proj", "model.layers.16.mlp.experts.22.gate_proj", "model.layers.16.mlp.experts.23.gate_proj", "model.layers.16.mlp.experts.24.gate_proj", "model.layers.16.mlp.experts.25.gate_proj", "model.layers.16.mlp.experts.26.gate_proj", "model.layers.16.mlp.experts.27.gate_proj", "model.layers.16.mlp.experts.28.gate_proj", "model.layers.16.mlp.experts.29.gate_proj", "model.layers.16.mlp.experts.30.gate_proj", "model.layers.16.mlp.experts.31.gate_proj", "model.layers.16.mlp.experts.32.gate_proj", "model.layers.16.mlp.experts.33.gate_proj", "model.layers.16.mlp.experts.34.gate_proj", "model.layers.16.mlp.experts.35.gate_proj", "model.layers.16.mlp.experts.36.gate_proj", "model.layers.16.mlp.experts.37.gate_proj", "model.layers.16.mlp.experts.38.gate_proj", "model.layers.16.mlp.experts.39.gate_proj", "model.layers.16.mlp.experts.40.gate_proj", "model.layers.16.mlp.experts.41.gate_proj", "model.layers.16.mlp.experts.42.gate_proj", "model.layers.16.mlp.experts.43.gate_proj", "model.layers.16.mlp.experts.44.gate_proj", "model.layers.16.mlp.experts.45.gate_proj", "model.layers.16.mlp.experts.46.gate_proj", "model.layers.16.mlp.experts.47.gate_proj", "model.layers.16.mlp.experts.48.gate_proj", "model.layers.16.mlp.experts.49.gate_proj", "model.layers.16.mlp.experts.50.gate_proj", "model.layers.16.mlp.experts.51.gate_proj", "model.layers.16.mlp.experts.52.gate_proj", "model.layers.16.mlp.experts.53.gate_proj", "model.layers.16.mlp.experts.54.gate_proj", "model.layers.16.mlp.experts.55.gate_proj", "model.layers.16.mlp.experts.56.gate_proj", "model.layers.16.mlp.experts.57.gate_proj", "model.layers.16.mlp.experts.58.gate_proj", "model.layers.16.mlp.experts.59.gate_proj", "model.layers.16.mlp.experts.60.gate_proj", "model.layers.16.mlp.experts.61.gate_proj", "model.layers.16.mlp.experts.62.gate_proj", "model.layers.16.mlp.experts.63.gate_proj", "model.layers.16.mlp.experts.64.gate_proj", "model.layers.16.mlp.experts.65.gate_proj", "model.layers.16.mlp.experts.66.gate_proj", "model.layers.16.mlp.experts.67.gate_proj", "model.layers.16.mlp.experts.68.gate_proj", "model.layers.16.mlp.experts.69.gate_proj", "model.layers.16.mlp.experts.70.gate_proj", "model.layers.16.mlp.experts.71.gate_proj", "model.layers.16.mlp.experts.72.gate_proj", "model.layers.16.mlp.experts.73.gate_proj", "model.layers.16.mlp.experts.74.gate_proj", "model.layers.16.mlp.experts.75.gate_proj", "model.layers.16.mlp.experts.76.gate_proj", "model.layers.16.mlp.experts.77.gate_proj", "model.layers.16.mlp.experts.78.gate_proj", "model.layers.16.mlp.experts.79.gate_proj", "model.layers.16.mlp.experts.80.gate_proj", "model.layers.16.mlp.experts.81.gate_proj", "model.layers.16.mlp.experts.82.gate_proj", "model.layers.16.mlp.experts.83.gate_proj", "model.layers.16.mlp.experts.84.gate_proj", "model.layers.16.mlp.experts.85.gate_proj", "model.layers.16.mlp.experts.86.gate_proj", "model.layers.16.mlp.experts.87.gate_proj", "model.layers.16.mlp.experts.88.gate_proj", "model.layers.16.mlp.experts.89.gate_proj", "model.layers.16.mlp.experts.90.gate_proj", "model.layers.16.mlp.experts.91.gate_proj", "model.layers.16.mlp.experts.92.gate_proj", "model.layers.16.mlp.experts.93.gate_proj", "model.layers.16.mlp.experts.94.gate_proj", "model.layers.16.mlp.experts.95.gate_proj", "model.layers.16.mlp.experts.96.gate_proj", "model.layers.16.mlp.experts.97.gate_proj", "model.layers.16.mlp.experts.98.gate_proj", "model.layers.16.mlp.experts.99.gate_proj", "model.layers.16.mlp.experts.100.gate_proj", "model.layers.16.mlp.experts.101.gate_proj", "model.layers.16.mlp.experts.102.gate_proj", "model.layers.16.mlp.experts.103.gate_proj", "model.layers.16.mlp.experts.104.gate_proj", "model.layers.16.mlp.experts.105.gate_proj", "model.layers.16.mlp.experts.106.gate_proj", "model.layers.16.mlp.experts.107.gate_proj", "model.layers.16.mlp.experts.108.gate_proj", "model.layers.16.mlp.experts.109.gate_proj", "model.layers.16.mlp.experts.110.gate_proj", "model.layers.16.mlp.experts.111.gate_proj", "model.layers.16.mlp.experts.112.gate_proj", "model.layers.16.mlp.experts.113.gate_proj", "model.layers.16.mlp.experts.114.gate_proj", "model.layers.16.mlp.experts.115.gate_proj", "model.layers.16.mlp.experts.116.gate_proj", "model.layers.16.mlp.experts.117.gate_proj", "model.layers.16.mlp.experts.118.gate_proj", "model.layers.16.mlp.experts.119.gate_proj", "model.layers.16.mlp.experts.120.gate_proj", "model.layers.16.mlp.experts.121.gate_proj", "model.layers.16.mlp.experts.122.gate_proj", "model.layers.16.mlp.experts.123.gate_proj", "model.layers.16.mlp.experts.124.gate_proj", "model.layers.16.mlp.experts.125.gate_proj", "model.layers.16.mlp.experts.126.gate_proj", "model.layers.16.mlp.experts.127.gate_proj", "model.layers.16.mlp.experts.0.up_proj", "model.layers.16.mlp.experts.1.up_proj", "model.layers.16.mlp.experts.2.up_proj", "model.layers.16.mlp.experts.3.up_proj", "model.layers.16.mlp.experts.4.up_proj", "model.layers.16.mlp.experts.5.up_proj", "model.layers.16.mlp.experts.6.up_proj", "model.layers.16.mlp.experts.7.up_proj", "model.layers.16.mlp.experts.8.up_proj", "model.layers.16.mlp.experts.9.up_proj", "model.layers.16.mlp.experts.10.up_proj", "model.layers.16.mlp.experts.11.up_proj", "model.layers.16.mlp.experts.12.up_proj", "model.layers.16.mlp.experts.13.up_proj", "model.layers.16.mlp.experts.14.up_proj", "model.layers.16.mlp.experts.15.up_proj", "model.layers.16.mlp.experts.16.up_proj", "model.layers.16.mlp.experts.17.up_proj", "model.layers.16.mlp.experts.18.up_proj", "model.layers.16.mlp.experts.19.up_proj", "model.layers.16.mlp.experts.20.up_proj", "model.layers.16.mlp.experts.21.up_proj", "model.layers.16.mlp.experts.22.up_proj", "model.layers.16.mlp.experts.23.up_proj", "model.layers.16.mlp.experts.24.up_proj", "model.layers.16.mlp.experts.25.up_proj", "model.layers.16.mlp.experts.26.up_proj", "model.layers.16.mlp.experts.27.up_proj", "model.layers.16.mlp.experts.28.up_proj", "model.layers.16.mlp.experts.29.up_proj", "model.layers.16.mlp.experts.30.up_proj", "model.layers.16.mlp.experts.31.up_proj", "model.layers.16.mlp.experts.32.up_proj", "model.layers.16.mlp.experts.33.up_proj", "model.layers.16.mlp.experts.34.up_proj", "model.layers.16.mlp.experts.35.up_proj", "model.layers.16.mlp.experts.36.up_proj", "model.layers.16.mlp.experts.37.up_proj", "model.layers.16.mlp.experts.38.up_proj", "model.layers.16.mlp.experts.39.up_proj", "model.layers.16.mlp.experts.40.up_proj", "model.layers.16.mlp.experts.41.up_proj", "model.layers.16.mlp.experts.42.up_proj", "model.layers.16.mlp.experts.43.up_proj", "model.layers.16.mlp.experts.44.up_proj", "model.layers.16.mlp.experts.45.up_proj", "model.layers.16.mlp.experts.46.up_proj", "model.layers.16.mlp.experts.47.up_proj", "model.layers.16.mlp.experts.48.up_proj", "model.layers.16.mlp.experts.49.up_proj", "model.layers.16.mlp.experts.50.up_proj", "model.layers.16.mlp.experts.51.up_proj", "model.layers.16.mlp.experts.52.up_proj", "model.layers.16.mlp.experts.53.up_proj", "model.layers.16.mlp.experts.54.up_proj", "model.layers.16.mlp.experts.55.up_proj", "model.layers.16.mlp.experts.56.up_proj", "model.layers.16.mlp.experts.57.up_proj", "model.layers.16.mlp.experts.58.up_proj", "model.layers.16.mlp.experts.59.up_proj", "model.layers.16.mlp.experts.60.up_proj", "model.layers.16.mlp.experts.61.up_proj", "model.layers.16.mlp.experts.62.up_proj", "model.layers.16.mlp.experts.63.up_proj", "model.layers.16.mlp.experts.64.up_proj", "model.layers.16.mlp.experts.65.up_proj", "model.layers.16.mlp.experts.66.up_proj", "model.layers.16.mlp.experts.67.up_proj", "model.layers.16.mlp.experts.68.up_proj", "model.layers.16.mlp.experts.69.up_proj", "model.layers.16.mlp.experts.70.up_proj", "model.layers.16.mlp.experts.71.up_proj", "model.layers.16.mlp.experts.72.up_proj", "model.layers.16.mlp.experts.73.up_proj", "model.layers.16.mlp.experts.74.up_proj", "model.layers.16.mlp.experts.75.up_proj", "model.layers.16.mlp.experts.76.up_proj", "model.layers.16.mlp.experts.77.up_proj", "model.layers.16.mlp.experts.78.up_proj", "model.layers.16.mlp.experts.79.up_proj", "model.layers.16.mlp.experts.80.up_proj", "model.layers.16.mlp.experts.81.up_proj", "model.layers.16.mlp.experts.82.up_proj", "model.layers.16.mlp.experts.83.up_proj", "model.layers.16.mlp.experts.84.up_proj", "model.layers.16.mlp.experts.85.up_proj", "model.layers.16.mlp.experts.86.up_proj", "model.layers.16.mlp.experts.87.up_proj", "model.layers.16.mlp.experts.88.up_proj", "model.layers.16.mlp.experts.89.up_proj", "model.layers.16.mlp.experts.90.up_proj", "model.layers.16.mlp.experts.91.up_proj", "model.layers.16.mlp.experts.92.up_proj", "model.layers.16.mlp.experts.93.up_proj", "model.layers.16.mlp.experts.94.up_proj", "model.layers.16.mlp.experts.95.up_proj", "model.layers.16.mlp.experts.96.up_proj", "model.layers.16.mlp.experts.97.up_proj", "model.layers.16.mlp.experts.98.up_proj", "model.layers.16.mlp.experts.99.up_proj", "model.layers.16.mlp.experts.100.up_proj", "model.layers.16.mlp.experts.101.up_proj", "model.layers.16.mlp.experts.102.up_proj", "model.layers.16.mlp.experts.103.up_proj", "model.layers.16.mlp.experts.104.up_proj", "model.layers.16.mlp.experts.105.up_proj", "model.layers.16.mlp.experts.106.up_proj", "model.layers.16.mlp.experts.107.up_proj", "model.layers.16.mlp.experts.108.up_proj", "model.layers.16.mlp.experts.109.up_proj", "model.layers.16.mlp.experts.110.up_proj", "model.layers.16.mlp.experts.111.up_proj", "model.layers.16.mlp.experts.112.up_proj", "model.layers.16.mlp.experts.113.up_proj", "model.layers.16.mlp.experts.114.up_proj", "model.layers.16.mlp.experts.115.up_proj", "model.layers.16.mlp.experts.116.up_proj", "model.layers.16.mlp.experts.117.up_proj", "model.layers.16.mlp.experts.118.up_proj", "model.layers.16.mlp.experts.119.up_proj", "model.layers.16.mlp.experts.120.up_proj", "model.layers.16.mlp.experts.121.up_proj", "model.layers.16.mlp.experts.122.up_proj", "model.layers.16.mlp.experts.123.up_proj", "model.layers.16.mlp.experts.124.up_proj", "model.layers.16.mlp.experts.125.up_proj", "model.layers.16.mlp.experts.126.up_proj", "model.layers.16.mlp.experts.127.up_proj", "model.layers.16.mlp.experts.0.down_proj", "model.layers.16.mlp.experts.1.down_proj", "model.layers.16.mlp.experts.2.down_proj", "model.layers.16.mlp.experts.3.down_proj", "model.layers.16.mlp.experts.4.down_proj", "model.layers.16.mlp.experts.5.down_proj", "model.layers.16.mlp.experts.6.down_proj", "model.layers.16.mlp.experts.7.down_proj", "model.layers.16.mlp.experts.8.down_proj", "model.layers.16.mlp.experts.9.down_proj", "model.layers.16.mlp.experts.10.down_proj", "model.layers.16.mlp.experts.11.down_proj", "model.layers.16.mlp.experts.12.down_proj", "model.layers.16.mlp.experts.13.down_proj", "model.layers.16.mlp.experts.14.down_proj", "model.layers.16.mlp.experts.15.down_proj", "model.layers.16.mlp.experts.16.down_proj", "model.layers.16.mlp.experts.17.down_proj", "model.layers.16.mlp.experts.18.down_proj", "model.layers.16.mlp.experts.19.down_proj", "model.layers.16.mlp.experts.20.down_proj", "model.layers.16.mlp.experts.21.down_proj", "model.layers.16.mlp.experts.22.down_proj", "model.layers.16.mlp.experts.23.down_proj", "model.layers.16.mlp.experts.24.down_proj", "model.layers.16.mlp.experts.25.down_proj", "model.layers.16.mlp.experts.26.down_proj", "model.layers.16.mlp.experts.27.down_proj", "model.layers.16.mlp.experts.28.down_proj", "model.layers.16.mlp.experts.29.down_proj", "model.layers.16.mlp.experts.30.down_proj", "model.layers.16.mlp.experts.31.down_proj", "model.layers.16.mlp.experts.32.down_proj", "model.layers.16.mlp.experts.33.down_proj", "model.layers.16.mlp.experts.34.down_proj", "model.layers.16.mlp.experts.35.down_proj", "model.layers.16.mlp.experts.36.down_proj", "model.layers.16.mlp.experts.37.down_proj", "model.layers.16.mlp.experts.38.down_proj", "model.layers.16.mlp.experts.39.down_proj", "model.layers.16.mlp.experts.40.down_proj", "model.layers.16.mlp.experts.41.down_proj", "model.layers.16.mlp.experts.42.down_proj", "model.layers.16.mlp.experts.43.down_proj", "model.layers.16.mlp.experts.44.down_proj", "model.layers.16.mlp.experts.45.down_proj", "model.layers.16.mlp.experts.46.down_proj", "model.layers.16.mlp.experts.47.down_proj", "model.layers.16.mlp.experts.48.down_proj", "model.layers.16.mlp.experts.49.down_proj", "model.layers.16.mlp.experts.50.down_proj", "model.layers.16.mlp.experts.51.down_proj", "model.layers.16.mlp.experts.52.down_proj", "model.layers.16.mlp.experts.53.down_proj", "model.layers.16.mlp.experts.54.down_proj", "model.layers.16.mlp.experts.55.down_proj", "model.layers.16.mlp.experts.56.down_proj", "model.layers.16.mlp.experts.57.down_proj", "model.layers.16.mlp.experts.58.down_proj", "model.layers.16.mlp.experts.59.down_proj", "model.layers.16.mlp.experts.60.down_proj", "model.layers.16.mlp.experts.61.down_proj", "model.layers.16.mlp.experts.62.down_proj", "model.layers.16.mlp.experts.63.down_proj", "model.layers.16.mlp.experts.64.down_proj", "model.layers.16.mlp.experts.65.down_proj", "model.layers.16.mlp.experts.66.down_proj", "model.layers.16.mlp.experts.67.down_proj", "model.layers.16.mlp.experts.68.down_proj", "model.layers.16.mlp.experts.69.down_proj", "model.layers.16.mlp.experts.70.down_proj", "model.layers.16.mlp.experts.71.down_proj", "model.layers.16.mlp.experts.72.down_proj", "model.layers.16.mlp.experts.73.down_proj", "model.layers.16.mlp.experts.74.down_proj", "model.layers.16.mlp.experts.75.down_proj", "model.layers.16.mlp.experts.76.down_proj", "model.layers.16.mlp.experts.77.down_proj", "model.layers.16.mlp.experts.78.down_proj", "model.layers.16.mlp.experts.79.down_proj", "model.layers.16.mlp.experts.80.down_proj", "model.layers.16.mlp.experts.81.down_proj", "model.layers.16.mlp.experts.82.down_proj", "model.layers.16.mlp.experts.83.down_proj", "model.layers.16.mlp.experts.84.down_proj", "model.layers.16.mlp.experts.85.down_proj", "model.layers.16.mlp.experts.86.down_proj", "model.layers.16.mlp.experts.87.down_proj", "model.layers.16.mlp.experts.88.down_proj", "model.layers.16.mlp.experts.89.down_proj", "model.layers.16.mlp.experts.90.down_proj", "model.layers.16.mlp.experts.91.down_proj", "model.layers.16.mlp.experts.92.down_proj", "model.layers.16.mlp.experts.93.down_proj", "model.layers.16.mlp.experts.94.down_proj", "model.layers.16.mlp.experts.95.down_proj", "model.layers.16.mlp.experts.96.down_proj", "model.layers.16.mlp.experts.97.down_proj", "model.layers.16.mlp.experts.98.down_proj", "model.layers.16.mlp.experts.99.down_proj", "model.layers.16.mlp.experts.100.down_proj", "model.layers.16.mlp.experts.101.down_proj", "model.layers.16.mlp.experts.102.down_proj", "model.layers.16.mlp.experts.103.down_proj", "model.layers.16.mlp.experts.104.down_proj", "model.layers.16.mlp.experts.105.down_proj", "model.layers.16.mlp.experts.106.down_proj", "model.layers.16.mlp.experts.107.down_proj", "model.layers.16.mlp.experts.108.down_proj", "model.layers.16.mlp.experts.109.down_proj", "model.layers.16.mlp.experts.110.down_proj", "model.layers.16.mlp.experts.111.down_proj", "model.layers.16.mlp.experts.112.down_proj", "model.layers.16.mlp.experts.113.down_proj", "model.layers.16.mlp.experts.114.down_proj", "model.layers.16.mlp.experts.115.down_proj", "model.layers.16.mlp.experts.116.down_proj", "model.layers.16.mlp.experts.117.down_proj", "model.layers.16.mlp.experts.118.down_proj", "model.layers.16.mlp.experts.119.down_proj", "model.layers.16.mlp.experts.120.down_proj", "model.layers.16.mlp.experts.121.down_proj", "model.layers.16.mlp.experts.122.down_proj", "model.layers.16.mlp.experts.123.down_proj", "model.layers.16.mlp.experts.124.down_proj", "model.layers.16.mlp.experts.125.down_proj", "model.layers.16.mlp.experts.126.down_proj", "model.layers.16.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.00373734738677739, "dbits": 2214592512 } ] }, { "idx": 50, "layers": [ "model.layers.17.self_attn.q_proj", "model.layers.17.self_attn.k_proj", "model.layers.17.self_attn.v_proj", "model.layers.17.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0006858152337372303, "dbits": 109051904 } ] }, { "idx": 51, "layers": [ "model.layers.17.mlp.shared_experts.gate_proj", "model.layers.17.mlp.shared_experts.up_proj", "model.layers.17.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0014444967731833402, "dbits": 17301504 } ] }, { "idx": 52, "layers": [ "model.layers.17.mlp.experts.0.gate_proj", "model.layers.17.mlp.experts.1.gate_proj", "model.layers.17.mlp.experts.2.gate_proj", "model.layers.17.mlp.experts.3.gate_proj", "model.layers.17.mlp.experts.4.gate_proj", "model.layers.17.mlp.experts.5.gate_proj", "model.layers.17.mlp.experts.6.gate_proj", "model.layers.17.mlp.experts.7.gate_proj", "model.layers.17.mlp.experts.8.gate_proj", "model.layers.17.mlp.experts.9.gate_proj", "model.layers.17.mlp.experts.10.gate_proj", "model.layers.17.mlp.experts.11.gate_proj", "model.layers.17.mlp.experts.12.gate_proj", "model.layers.17.mlp.experts.13.gate_proj", "model.layers.17.mlp.experts.14.gate_proj", "model.layers.17.mlp.experts.15.gate_proj", "model.layers.17.mlp.experts.16.gate_proj", "model.layers.17.mlp.experts.17.gate_proj", "model.layers.17.mlp.experts.18.gate_proj", "model.layers.17.mlp.experts.19.gate_proj", "model.layers.17.mlp.experts.20.gate_proj", "model.layers.17.mlp.experts.21.gate_proj", "model.layers.17.mlp.experts.22.gate_proj", "model.layers.17.mlp.experts.23.gate_proj", "model.layers.17.mlp.experts.24.gate_proj", "model.layers.17.mlp.experts.25.gate_proj", "model.layers.17.mlp.experts.26.gate_proj", "model.layers.17.mlp.experts.27.gate_proj", "model.layers.17.mlp.experts.28.gate_proj", "model.layers.17.mlp.experts.29.gate_proj", "model.layers.17.mlp.experts.30.gate_proj", "model.layers.17.mlp.experts.31.gate_proj", "model.layers.17.mlp.experts.32.gate_proj", "model.layers.17.mlp.experts.33.gate_proj", "model.layers.17.mlp.experts.34.gate_proj", "model.layers.17.mlp.experts.35.gate_proj", "model.layers.17.mlp.experts.36.gate_proj", "model.layers.17.mlp.experts.37.gate_proj", "model.layers.17.mlp.experts.38.gate_proj", "model.layers.17.mlp.experts.39.gate_proj", "model.layers.17.mlp.experts.40.gate_proj", "model.layers.17.mlp.experts.41.gate_proj", "model.layers.17.mlp.experts.42.gate_proj", "model.layers.17.mlp.experts.43.gate_proj", "model.layers.17.mlp.experts.44.gate_proj", "model.layers.17.mlp.experts.45.gate_proj", "model.layers.17.mlp.experts.46.gate_proj", "model.layers.17.mlp.experts.47.gate_proj", "model.layers.17.mlp.experts.48.gate_proj", "model.layers.17.mlp.experts.49.gate_proj", "model.layers.17.mlp.experts.50.gate_proj", "model.layers.17.mlp.experts.51.gate_proj", "model.layers.17.mlp.experts.52.gate_proj", "model.layers.17.mlp.experts.53.gate_proj", "model.layers.17.mlp.experts.54.gate_proj", "model.layers.17.mlp.experts.55.gate_proj", "model.layers.17.mlp.experts.56.gate_proj", "model.layers.17.mlp.experts.57.gate_proj", "model.layers.17.mlp.experts.58.gate_proj", "model.layers.17.mlp.experts.59.gate_proj", "model.layers.17.mlp.experts.60.gate_proj", "model.layers.17.mlp.experts.61.gate_proj", "model.layers.17.mlp.experts.62.gate_proj", "model.layers.17.mlp.experts.63.gate_proj", "model.layers.17.mlp.experts.64.gate_proj", "model.layers.17.mlp.experts.65.gate_proj", "model.layers.17.mlp.experts.66.gate_proj", "model.layers.17.mlp.experts.67.gate_proj", "model.layers.17.mlp.experts.68.gate_proj", "model.layers.17.mlp.experts.69.gate_proj", "model.layers.17.mlp.experts.70.gate_proj", "model.layers.17.mlp.experts.71.gate_proj", "model.layers.17.mlp.experts.72.gate_proj", "model.layers.17.mlp.experts.73.gate_proj", "model.layers.17.mlp.experts.74.gate_proj", "model.layers.17.mlp.experts.75.gate_proj", "model.layers.17.mlp.experts.76.gate_proj", "model.layers.17.mlp.experts.77.gate_proj", "model.layers.17.mlp.experts.78.gate_proj", "model.layers.17.mlp.experts.79.gate_proj", "model.layers.17.mlp.experts.80.gate_proj", "model.layers.17.mlp.experts.81.gate_proj", "model.layers.17.mlp.experts.82.gate_proj", "model.layers.17.mlp.experts.83.gate_proj", "model.layers.17.mlp.experts.84.gate_proj", "model.layers.17.mlp.experts.85.gate_proj", "model.layers.17.mlp.experts.86.gate_proj", "model.layers.17.mlp.experts.87.gate_proj", "model.layers.17.mlp.experts.88.gate_proj", "model.layers.17.mlp.experts.89.gate_proj", "model.layers.17.mlp.experts.90.gate_proj", "model.layers.17.mlp.experts.91.gate_proj", "model.layers.17.mlp.experts.92.gate_proj", "model.layers.17.mlp.experts.93.gate_proj", "model.layers.17.mlp.experts.94.gate_proj", "model.layers.17.mlp.experts.95.gate_proj", "model.layers.17.mlp.experts.96.gate_proj", "model.layers.17.mlp.experts.97.gate_proj", "model.layers.17.mlp.experts.98.gate_proj", "model.layers.17.mlp.experts.99.gate_proj", "model.layers.17.mlp.experts.100.gate_proj", "model.layers.17.mlp.experts.101.gate_proj", "model.layers.17.mlp.experts.102.gate_proj", "model.layers.17.mlp.experts.103.gate_proj", "model.layers.17.mlp.experts.104.gate_proj", "model.layers.17.mlp.experts.105.gate_proj", "model.layers.17.mlp.experts.106.gate_proj", "model.layers.17.mlp.experts.107.gate_proj", "model.layers.17.mlp.experts.108.gate_proj", "model.layers.17.mlp.experts.109.gate_proj", "model.layers.17.mlp.experts.110.gate_proj", "model.layers.17.mlp.experts.111.gate_proj", "model.layers.17.mlp.experts.112.gate_proj", "model.layers.17.mlp.experts.113.gate_proj", "model.layers.17.mlp.experts.114.gate_proj", "model.layers.17.mlp.experts.115.gate_proj", "model.layers.17.mlp.experts.116.gate_proj", "model.layers.17.mlp.experts.117.gate_proj", "model.layers.17.mlp.experts.118.gate_proj", "model.layers.17.mlp.experts.119.gate_proj", "model.layers.17.mlp.experts.120.gate_proj", "model.layers.17.mlp.experts.121.gate_proj", "model.layers.17.mlp.experts.122.gate_proj", "model.layers.17.mlp.experts.123.gate_proj", "model.layers.17.mlp.experts.124.gate_proj", "model.layers.17.mlp.experts.125.gate_proj", "model.layers.17.mlp.experts.126.gate_proj", "model.layers.17.mlp.experts.127.gate_proj", "model.layers.17.mlp.experts.0.up_proj", "model.layers.17.mlp.experts.1.up_proj", "model.layers.17.mlp.experts.2.up_proj", "model.layers.17.mlp.experts.3.up_proj", "model.layers.17.mlp.experts.4.up_proj", "model.layers.17.mlp.experts.5.up_proj", "model.layers.17.mlp.experts.6.up_proj", "model.layers.17.mlp.experts.7.up_proj", "model.layers.17.mlp.experts.8.up_proj", "model.layers.17.mlp.experts.9.up_proj", "model.layers.17.mlp.experts.10.up_proj", "model.layers.17.mlp.experts.11.up_proj", "model.layers.17.mlp.experts.12.up_proj", "model.layers.17.mlp.experts.13.up_proj", "model.layers.17.mlp.experts.14.up_proj", "model.layers.17.mlp.experts.15.up_proj", "model.layers.17.mlp.experts.16.up_proj", "model.layers.17.mlp.experts.17.up_proj", "model.layers.17.mlp.experts.18.up_proj", "model.layers.17.mlp.experts.19.up_proj", "model.layers.17.mlp.experts.20.up_proj", "model.layers.17.mlp.experts.21.up_proj", "model.layers.17.mlp.experts.22.up_proj", "model.layers.17.mlp.experts.23.up_proj", "model.layers.17.mlp.experts.24.up_proj", "model.layers.17.mlp.experts.25.up_proj", "model.layers.17.mlp.experts.26.up_proj", "model.layers.17.mlp.experts.27.up_proj", "model.layers.17.mlp.experts.28.up_proj", "model.layers.17.mlp.experts.29.up_proj", "model.layers.17.mlp.experts.30.up_proj", "model.layers.17.mlp.experts.31.up_proj", "model.layers.17.mlp.experts.32.up_proj", "model.layers.17.mlp.experts.33.up_proj", "model.layers.17.mlp.experts.34.up_proj", "model.layers.17.mlp.experts.35.up_proj", "model.layers.17.mlp.experts.36.up_proj", "model.layers.17.mlp.experts.37.up_proj", "model.layers.17.mlp.experts.38.up_proj", "model.layers.17.mlp.experts.39.up_proj", "model.layers.17.mlp.experts.40.up_proj", "model.layers.17.mlp.experts.41.up_proj", "model.layers.17.mlp.experts.42.up_proj", "model.layers.17.mlp.experts.43.up_proj", "model.layers.17.mlp.experts.44.up_proj", "model.layers.17.mlp.experts.45.up_proj", "model.layers.17.mlp.experts.46.up_proj", "model.layers.17.mlp.experts.47.up_proj", "model.layers.17.mlp.experts.48.up_proj", "model.layers.17.mlp.experts.49.up_proj", "model.layers.17.mlp.experts.50.up_proj", "model.layers.17.mlp.experts.51.up_proj", "model.layers.17.mlp.experts.52.up_proj", "model.layers.17.mlp.experts.53.up_proj", "model.layers.17.mlp.experts.54.up_proj", "model.layers.17.mlp.experts.55.up_proj", "model.layers.17.mlp.experts.56.up_proj", "model.layers.17.mlp.experts.57.up_proj", "model.layers.17.mlp.experts.58.up_proj", "model.layers.17.mlp.experts.59.up_proj", "model.layers.17.mlp.experts.60.up_proj", "model.layers.17.mlp.experts.61.up_proj", "model.layers.17.mlp.experts.62.up_proj", "model.layers.17.mlp.experts.63.up_proj", "model.layers.17.mlp.experts.64.up_proj", "model.layers.17.mlp.experts.65.up_proj", "model.layers.17.mlp.experts.66.up_proj", "model.layers.17.mlp.experts.67.up_proj", "model.layers.17.mlp.experts.68.up_proj", "model.layers.17.mlp.experts.69.up_proj", "model.layers.17.mlp.experts.70.up_proj", "model.layers.17.mlp.experts.71.up_proj", "model.layers.17.mlp.experts.72.up_proj", "model.layers.17.mlp.experts.73.up_proj", "model.layers.17.mlp.experts.74.up_proj", "model.layers.17.mlp.experts.75.up_proj", "model.layers.17.mlp.experts.76.up_proj", "model.layers.17.mlp.experts.77.up_proj", "model.layers.17.mlp.experts.78.up_proj", "model.layers.17.mlp.experts.79.up_proj", "model.layers.17.mlp.experts.80.up_proj", "model.layers.17.mlp.experts.81.up_proj", "model.layers.17.mlp.experts.82.up_proj", "model.layers.17.mlp.experts.83.up_proj", "model.layers.17.mlp.experts.84.up_proj", "model.layers.17.mlp.experts.85.up_proj", "model.layers.17.mlp.experts.86.up_proj", "model.layers.17.mlp.experts.87.up_proj", "model.layers.17.mlp.experts.88.up_proj", "model.layers.17.mlp.experts.89.up_proj", "model.layers.17.mlp.experts.90.up_proj", "model.layers.17.mlp.experts.91.up_proj", "model.layers.17.mlp.experts.92.up_proj", "model.layers.17.mlp.experts.93.up_proj", "model.layers.17.mlp.experts.94.up_proj", "model.layers.17.mlp.experts.95.up_proj", "model.layers.17.mlp.experts.96.up_proj", "model.layers.17.mlp.experts.97.up_proj", "model.layers.17.mlp.experts.98.up_proj", "model.layers.17.mlp.experts.99.up_proj", "model.layers.17.mlp.experts.100.up_proj", "model.layers.17.mlp.experts.101.up_proj", "model.layers.17.mlp.experts.102.up_proj", "model.layers.17.mlp.experts.103.up_proj", "model.layers.17.mlp.experts.104.up_proj", "model.layers.17.mlp.experts.105.up_proj", "model.layers.17.mlp.experts.106.up_proj", "model.layers.17.mlp.experts.107.up_proj", "model.layers.17.mlp.experts.108.up_proj", "model.layers.17.mlp.experts.109.up_proj", "model.layers.17.mlp.experts.110.up_proj", "model.layers.17.mlp.experts.111.up_proj", "model.layers.17.mlp.experts.112.up_proj", "model.layers.17.mlp.experts.113.up_proj", "model.layers.17.mlp.experts.114.up_proj", "model.layers.17.mlp.experts.115.up_proj", "model.layers.17.mlp.experts.116.up_proj", "model.layers.17.mlp.experts.117.up_proj", "model.layers.17.mlp.experts.118.up_proj", "model.layers.17.mlp.experts.119.up_proj", "model.layers.17.mlp.experts.120.up_proj", "model.layers.17.mlp.experts.121.up_proj", "model.layers.17.mlp.experts.122.up_proj", "model.layers.17.mlp.experts.123.up_proj", "model.layers.17.mlp.experts.124.up_proj", "model.layers.17.mlp.experts.125.up_proj", "model.layers.17.mlp.experts.126.up_proj", "model.layers.17.mlp.experts.127.up_proj", "model.layers.17.mlp.experts.0.down_proj", "model.layers.17.mlp.experts.1.down_proj", "model.layers.17.mlp.experts.2.down_proj", "model.layers.17.mlp.experts.3.down_proj", "model.layers.17.mlp.experts.4.down_proj", "model.layers.17.mlp.experts.5.down_proj", "model.layers.17.mlp.experts.6.down_proj", "model.layers.17.mlp.experts.7.down_proj", "model.layers.17.mlp.experts.8.down_proj", "model.layers.17.mlp.experts.9.down_proj", "model.layers.17.mlp.experts.10.down_proj", "model.layers.17.mlp.experts.11.down_proj", "model.layers.17.mlp.experts.12.down_proj", "model.layers.17.mlp.experts.13.down_proj", "model.layers.17.mlp.experts.14.down_proj", "model.layers.17.mlp.experts.15.down_proj", "model.layers.17.mlp.experts.16.down_proj", "model.layers.17.mlp.experts.17.down_proj", "model.layers.17.mlp.experts.18.down_proj", "model.layers.17.mlp.experts.19.down_proj", "model.layers.17.mlp.experts.20.down_proj", "model.layers.17.mlp.experts.21.down_proj", "model.layers.17.mlp.experts.22.down_proj", "model.layers.17.mlp.experts.23.down_proj", "model.layers.17.mlp.experts.24.down_proj", "model.layers.17.mlp.experts.25.down_proj", "model.layers.17.mlp.experts.26.down_proj", "model.layers.17.mlp.experts.27.down_proj", "model.layers.17.mlp.experts.28.down_proj", "model.layers.17.mlp.experts.29.down_proj", "model.layers.17.mlp.experts.30.down_proj", "model.layers.17.mlp.experts.31.down_proj", "model.layers.17.mlp.experts.32.down_proj", "model.layers.17.mlp.experts.33.down_proj", "model.layers.17.mlp.experts.34.down_proj", "model.layers.17.mlp.experts.35.down_proj", "model.layers.17.mlp.experts.36.down_proj", "model.layers.17.mlp.experts.37.down_proj", "model.layers.17.mlp.experts.38.down_proj", "model.layers.17.mlp.experts.39.down_proj", "model.layers.17.mlp.experts.40.down_proj", "model.layers.17.mlp.experts.41.down_proj", "model.layers.17.mlp.experts.42.down_proj", "model.layers.17.mlp.experts.43.down_proj", "model.layers.17.mlp.experts.44.down_proj", "model.layers.17.mlp.experts.45.down_proj", "model.layers.17.mlp.experts.46.down_proj", "model.layers.17.mlp.experts.47.down_proj", "model.layers.17.mlp.experts.48.down_proj", "model.layers.17.mlp.experts.49.down_proj", "model.layers.17.mlp.experts.50.down_proj", "model.layers.17.mlp.experts.51.down_proj", "model.layers.17.mlp.experts.52.down_proj", "model.layers.17.mlp.experts.53.down_proj", "model.layers.17.mlp.experts.54.down_proj", "model.layers.17.mlp.experts.55.down_proj", "model.layers.17.mlp.experts.56.down_proj", "model.layers.17.mlp.experts.57.down_proj", "model.layers.17.mlp.experts.58.down_proj", "model.layers.17.mlp.experts.59.down_proj", "model.layers.17.mlp.experts.60.down_proj", "model.layers.17.mlp.experts.61.down_proj", "model.layers.17.mlp.experts.62.down_proj", "model.layers.17.mlp.experts.63.down_proj", "model.layers.17.mlp.experts.64.down_proj", "model.layers.17.mlp.experts.65.down_proj", "model.layers.17.mlp.experts.66.down_proj", "model.layers.17.mlp.experts.67.down_proj", "model.layers.17.mlp.experts.68.down_proj", "model.layers.17.mlp.experts.69.down_proj", "model.layers.17.mlp.experts.70.down_proj", "model.layers.17.mlp.experts.71.down_proj", "model.layers.17.mlp.experts.72.down_proj", "model.layers.17.mlp.experts.73.down_proj", "model.layers.17.mlp.experts.74.down_proj", "model.layers.17.mlp.experts.75.down_proj", "model.layers.17.mlp.experts.76.down_proj", "model.layers.17.mlp.experts.77.down_proj", "model.layers.17.mlp.experts.78.down_proj", "model.layers.17.mlp.experts.79.down_proj", "model.layers.17.mlp.experts.80.down_proj", "model.layers.17.mlp.experts.81.down_proj", "model.layers.17.mlp.experts.82.down_proj", "model.layers.17.mlp.experts.83.down_proj", "model.layers.17.mlp.experts.84.down_proj", "model.layers.17.mlp.experts.85.down_proj", "model.layers.17.mlp.experts.86.down_proj", "model.layers.17.mlp.experts.87.down_proj", "model.layers.17.mlp.experts.88.down_proj", "model.layers.17.mlp.experts.89.down_proj", "model.layers.17.mlp.experts.90.down_proj", "model.layers.17.mlp.experts.91.down_proj", "model.layers.17.mlp.experts.92.down_proj", "model.layers.17.mlp.experts.93.down_proj", "model.layers.17.mlp.experts.94.down_proj", "model.layers.17.mlp.experts.95.down_proj", "model.layers.17.mlp.experts.96.down_proj", "model.layers.17.mlp.experts.97.down_proj", "model.layers.17.mlp.experts.98.down_proj", "model.layers.17.mlp.experts.99.down_proj", "model.layers.17.mlp.experts.100.down_proj", "model.layers.17.mlp.experts.101.down_proj", "model.layers.17.mlp.experts.102.down_proj", "model.layers.17.mlp.experts.103.down_proj", "model.layers.17.mlp.experts.104.down_proj", "model.layers.17.mlp.experts.105.down_proj", "model.layers.17.mlp.experts.106.down_proj", "model.layers.17.mlp.experts.107.down_proj", "model.layers.17.mlp.experts.108.down_proj", "model.layers.17.mlp.experts.109.down_proj", "model.layers.17.mlp.experts.110.down_proj", "model.layers.17.mlp.experts.111.down_proj", "model.layers.17.mlp.experts.112.down_proj", "model.layers.17.mlp.experts.113.down_proj", "model.layers.17.mlp.experts.114.down_proj", "model.layers.17.mlp.experts.115.down_proj", "model.layers.17.mlp.experts.116.down_proj", "model.layers.17.mlp.experts.117.down_proj", "model.layers.17.mlp.experts.118.down_proj", "model.layers.17.mlp.experts.119.down_proj", "model.layers.17.mlp.experts.120.down_proj", "model.layers.17.mlp.experts.121.down_proj", "model.layers.17.mlp.experts.122.down_proj", "model.layers.17.mlp.experts.123.down_proj", "model.layers.17.mlp.experts.124.down_proj", "model.layers.17.mlp.experts.125.down_proj", "model.layers.17.mlp.experts.126.down_proj", "model.layers.17.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.004046099632978445, "dbits": 2214592512 } ] }, { "idx": 53, "layers": [ "model.layers.18.self_attn.q_proj", "model.layers.18.self_attn.k_proj", "model.layers.18.self_attn.v_proj", "model.layers.18.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0007603982463479098, "dbits": 109051904 } ] }, { "idx": 54, "layers": [ "model.layers.18.mlp.shared_experts.gate_proj", "model.layers.18.mlp.shared_experts.up_proj", "model.layers.18.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00032613649964333136, "dbits": 17301504 } ] }, { "idx": 55, "layers": [ "model.layers.18.mlp.experts.0.gate_proj", "model.layers.18.mlp.experts.1.gate_proj", "model.layers.18.mlp.experts.2.gate_proj", "model.layers.18.mlp.experts.3.gate_proj", "model.layers.18.mlp.experts.4.gate_proj", "model.layers.18.mlp.experts.5.gate_proj", "model.layers.18.mlp.experts.6.gate_proj", "model.layers.18.mlp.experts.7.gate_proj", "model.layers.18.mlp.experts.8.gate_proj", "model.layers.18.mlp.experts.9.gate_proj", "model.layers.18.mlp.experts.10.gate_proj", "model.layers.18.mlp.experts.11.gate_proj", "model.layers.18.mlp.experts.12.gate_proj", "model.layers.18.mlp.experts.13.gate_proj", "model.layers.18.mlp.experts.14.gate_proj", "model.layers.18.mlp.experts.15.gate_proj", "model.layers.18.mlp.experts.16.gate_proj", "model.layers.18.mlp.experts.17.gate_proj", "model.layers.18.mlp.experts.18.gate_proj", "model.layers.18.mlp.experts.19.gate_proj", "model.layers.18.mlp.experts.20.gate_proj", "model.layers.18.mlp.experts.21.gate_proj", "model.layers.18.mlp.experts.22.gate_proj", "model.layers.18.mlp.experts.23.gate_proj", "model.layers.18.mlp.experts.24.gate_proj", "model.layers.18.mlp.experts.25.gate_proj", "model.layers.18.mlp.experts.26.gate_proj", "model.layers.18.mlp.experts.27.gate_proj", "model.layers.18.mlp.experts.28.gate_proj", "model.layers.18.mlp.experts.29.gate_proj", "model.layers.18.mlp.experts.30.gate_proj", "model.layers.18.mlp.experts.31.gate_proj", "model.layers.18.mlp.experts.32.gate_proj", "model.layers.18.mlp.experts.33.gate_proj", "model.layers.18.mlp.experts.34.gate_proj", "model.layers.18.mlp.experts.35.gate_proj", "model.layers.18.mlp.experts.36.gate_proj", "model.layers.18.mlp.experts.37.gate_proj", "model.layers.18.mlp.experts.38.gate_proj", "model.layers.18.mlp.experts.39.gate_proj", "model.layers.18.mlp.experts.40.gate_proj", "model.layers.18.mlp.experts.41.gate_proj", "model.layers.18.mlp.experts.42.gate_proj", "model.layers.18.mlp.experts.43.gate_proj", "model.layers.18.mlp.experts.44.gate_proj", "model.layers.18.mlp.experts.45.gate_proj", "model.layers.18.mlp.experts.46.gate_proj", "model.layers.18.mlp.experts.47.gate_proj", "model.layers.18.mlp.experts.48.gate_proj", "model.layers.18.mlp.experts.49.gate_proj", "model.layers.18.mlp.experts.50.gate_proj", "model.layers.18.mlp.experts.51.gate_proj", "model.layers.18.mlp.experts.52.gate_proj", "model.layers.18.mlp.experts.53.gate_proj", "model.layers.18.mlp.experts.54.gate_proj", "model.layers.18.mlp.experts.55.gate_proj", "model.layers.18.mlp.experts.56.gate_proj", "model.layers.18.mlp.experts.57.gate_proj", "model.layers.18.mlp.experts.58.gate_proj", "model.layers.18.mlp.experts.59.gate_proj", "model.layers.18.mlp.experts.60.gate_proj", "model.layers.18.mlp.experts.61.gate_proj", "model.layers.18.mlp.experts.62.gate_proj", "model.layers.18.mlp.experts.63.gate_proj", "model.layers.18.mlp.experts.64.gate_proj", "model.layers.18.mlp.experts.65.gate_proj", "model.layers.18.mlp.experts.66.gate_proj", "model.layers.18.mlp.experts.67.gate_proj", "model.layers.18.mlp.experts.68.gate_proj", "model.layers.18.mlp.experts.69.gate_proj", "model.layers.18.mlp.experts.70.gate_proj", "model.layers.18.mlp.experts.71.gate_proj", "model.layers.18.mlp.experts.72.gate_proj", "model.layers.18.mlp.experts.73.gate_proj", "model.layers.18.mlp.experts.74.gate_proj", "model.layers.18.mlp.experts.75.gate_proj", "model.layers.18.mlp.experts.76.gate_proj", "model.layers.18.mlp.experts.77.gate_proj", "model.layers.18.mlp.experts.78.gate_proj", "model.layers.18.mlp.experts.79.gate_proj", "model.layers.18.mlp.experts.80.gate_proj", "model.layers.18.mlp.experts.81.gate_proj", "model.layers.18.mlp.experts.82.gate_proj", "model.layers.18.mlp.experts.83.gate_proj", "model.layers.18.mlp.experts.84.gate_proj", "model.layers.18.mlp.experts.85.gate_proj", "model.layers.18.mlp.experts.86.gate_proj", "model.layers.18.mlp.experts.87.gate_proj", "model.layers.18.mlp.experts.88.gate_proj", "model.layers.18.mlp.experts.89.gate_proj", "model.layers.18.mlp.experts.90.gate_proj", "model.layers.18.mlp.experts.91.gate_proj", "model.layers.18.mlp.experts.92.gate_proj", "model.layers.18.mlp.experts.93.gate_proj", "model.layers.18.mlp.experts.94.gate_proj", "model.layers.18.mlp.experts.95.gate_proj", "model.layers.18.mlp.experts.96.gate_proj", "model.layers.18.mlp.experts.97.gate_proj", "model.layers.18.mlp.experts.98.gate_proj", "model.layers.18.mlp.experts.99.gate_proj", "model.layers.18.mlp.experts.100.gate_proj", "model.layers.18.mlp.experts.101.gate_proj", "model.layers.18.mlp.experts.102.gate_proj", "model.layers.18.mlp.experts.103.gate_proj", "model.layers.18.mlp.experts.104.gate_proj", "model.layers.18.mlp.experts.105.gate_proj", "model.layers.18.mlp.experts.106.gate_proj", "model.layers.18.mlp.experts.107.gate_proj", "model.layers.18.mlp.experts.108.gate_proj", "model.layers.18.mlp.experts.109.gate_proj", "model.layers.18.mlp.experts.110.gate_proj", "model.layers.18.mlp.experts.111.gate_proj", "model.layers.18.mlp.experts.112.gate_proj", "model.layers.18.mlp.experts.113.gate_proj", "model.layers.18.mlp.experts.114.gate_proj", "model.layers.18.mlp.experts.115.gate_proj", "model.layers.18.mlp.experts.116.gate_proj", "model.layers.18.mlp.experts.117.gate_proj", "model.layers.18.mlp.experts.118.gate_proj", "model.layers.18.mlp.experts.119.gate_proj", "model.layers.18.mlp.experts.120.gate_proj", "model.layers.18.mlp.experts.121.gate_proj", "model.layers.18.mlp.experts.122.gate_proj", "model.layers.18.mlp.experts.123.gate_proj", "model.layers.18.mlp.experts.124.gate_proj", "model.layers.18.mlp.experts.125.gate_proj", "model.layers.18.mlp.experts.126.gate_proj", "model.layers.18.mlp.experts.127.gate_proj", "model.layers.18.mlp.experts.0.up_proj", "model.layers.18.mlp.experts.1.up_proj", "model.layers.18.mlp.experts.2.up_proj", "model.layers.18.mlp.experts.3.up_proj", "model.layers.18.mlp.experts.4.up_proj", "model.layers.18.mlp.experts.5.up_proj", "model.layers.18.mlp.experts.6.up_proj", "model.layers.18.mlp.experts.7.up_proj", "model.layers.18.mlp.experts.8.up_proj", "model.layers.18.mlp.experts.9.up_proj", "model.layers.18.mlp.experts.10.up_proj", "model.layers.18.mlp.experts.11.up_proj", "model.layers.18.mlp.experts.12.up_proj", "model.layers.18.mlp.experts.13.up_proj", "model.layers.18.mlp.experts.14.up_proj", "model.layers.18.mlp.experts.15.up_proj", "model.layers.18.mlp.experts.16.up_proj", "model.layers.18.mlp.experts.17.up_proj", "model.layers.18.mlp.experts.18.up_proj", "model.layers.18.mlp.experts.19.up_proj", "model.layers.18.mlp.experts.20.up_proj", "model.layers.18.mlp.experts.21.up_proj", "model.layers.18.mlp.experts.22.up_proj", "model.layers.18.mlp.experts.23.up_proj", "model.layers.18.mlp.experts.24.up_proj", "model.layers.18.mlp.experts.25.up_proj", "model.layers.18.mlp.experts.26.up_proj", "model.layers.18.mlp.experts.27.up_proj", "model.layers.18.mlp.experts.28.up_proj", "model.layers.18.mlp.experts.29.up_proj", "model.layers.18.mlp.experts.30.up_proj", "model.layers.18.mlp.experts.31.up_proj", "model.layers.18.mlp.experts.32.up_proj", "model.layers.18.mlp.experts.33.up_proj", "model.layers.18.mlp.experts.34.up_proj", "model.layers.18.mlp.experts.35.up_proj", "model.layers.18.mlp.experts.36.up_proj", "model.layers.18.mlp.experts.37.up_proj", "model.layers.18.mlp.experts.38.up_proj", "model.layers.18.mlp.experts.39.up_proj", "model.layers.18.mlp.experts.40.up_proj", "model.layers.18.mlp.experts.41.up_proj", "model.layers.18.mlp.experts.42.up_proj", "model.layers.18.mlp.experts.43.up_proj", "model.layers.18.mlp.experts.44.up_proj", "model.layers.18.mlp.experts.45.up_proj", "model.layers.18.mlp.experts.46.up_proj", "model.layers.18.mlp.experts.47.up_proj", "model.layers.18.mlp.experts.48.up_proj", "model.layers.18.mlp.experts.49.up_proj", "model.layers.18.mlp.experts.50.up_proj", "model.layers.18.mlp.experts.51.up_proj", "model.layers.18.mlp.experts.52.up_proj", "model.layers.18.mlp.experts.53.up_proj", "model.layers.18.mlp.experts.54.up_proj", "model.layers.18.mlp.experts.55.up_proj", "model.layers.18.mlp.experts.56.up_proj", "model.layers.18.mlp.experts.57.up_proj", "model.layers.18.mlp.experts.58.up_proj", "model.layers.18.mlp.experts.59.up_proj", "model.layers.18.mlp.experts.60.up_proj", "model.layers.18.mlp.experts.61.up_proj", "model.layers.18.mlp.experts.62.up_proj", "model.layers.18.mlp.experts.63.up_proj", "model.layers.18.mlp.experts.64.up_proj", "model.layers.18.mlp.experts.65.up_proj", "model.layers.18.mlp.experts.66.up_proj", "model.layers.18.mlp.experts.67.up_proj", "model.layers.18.mlp.experts.68.up_proj", "model.layers.18.mlp.experts.69.up_proj", "model.layers.18.mlp.experts.70.up_proj", "model.layers.18.mlp.experts.71.up_proj", "model.layers.18.mlp.experts.72.up_proj", "model.layers.18.mlp.experts.73.up_proj", "model.layers.18.mlp.experts.74.up_proj", "model.layers.18.mlp.experts.75.up_proj", "model.layers.18.mlp.experts.76.up_proj", "model.layers.18.mlp.experts.77.up_proj", "model.layers.18.mlp.experts.78.up_proj", "model.layers.18.mlp.experts.79.up_proj", "model.layers.18.mlp.experts.80.up_proj", "model.layers.18.mlp.experts.81.up_proj", "model.layers.18.mlp.experts.82.up_proj", "model.layers.18.mlp.experts.83.up_proj", "model.layers.18.mlp.experts.84.up_proj", "model.layers.18.mlp.experts.85.up_proj", "model.layers.18.mlp.experts.86.up_proj", "model.layers.18.mlp.experts.87.up_proj", "model.layers.18.mlp.experts.88.up_proj", "model.layers.18.mlp.experts.89.up_proj", "model.layers.18.mlp.experts.90.up_proj", "model.layers.18.mlp.experts.91.up_proj", "model.layers.18.mlp.experts.92.up_proj", "model.layers.18.mlp.experts.93.up_proj", "model.layers.18.mlp.experts.94.up_proj", "model.layers.18.mlp.experts.95.up_proj", "model.layers.18.mlp.experts.96.up_proj", "model.layers.18.mlp.experts.97.up_proj", "model.layers.18.mlp.experts.98.up_proj", "model.layers.18.mlp.experts.99.up_proj", "model.layers.18.mlp.experts.100.up_proj", "model.layers.18.mlp.experts.101.up_proj", "model.layers.18.mlp.experts.102.up_proj", "model.layers.18.mlp.experts.103.up_proj", "model.layers.18.mlp.experts.104.up_proj", "model.layers.18.mlp.experts.105.up_proj", "model.layers.18.mlp.experts.106.up_proj", "model.layers.18.mlp.experts.107.up_proj", "model.layers.18.mlp.experts.108.up_proj", "model.layers.18.mlp.experts.109.up_proj", "model.layers.18.mlp.experts.110.up_proj", "model.layers.18.mlp.experts.111.up_proj", "model.layers.18.mlp.experts.112.up_proj", "model.layers.18.mlp.experts.113.up_proj", "model.layers.18.mlp.experts.114.up_proj", "model.layers.18.mlp.experts.115.up_proj", "model.layers.18.mlp.experts.116.up_proj", "model.layers.18.mlp.experts.117.up_proj", "model.layers.18.mlp.experts.118.up_proj", "model.layers.18.mlp.experts.119.up_proj", "model.layers.18.mlp.experts.120.up_proj", "model.layers.18.mlp.experts.121.up_proj", "model.layers.18.mlp.experts.122.up_proj", "model.layers.18.mlp.experts.123.up_proj", "model.layers.18.mlp.experts.124.up_proj", "model.layers.18.mlp.experts.125.up_proj", "model.layers.18.mlp.experts.126.up_proj", "model.layers.18.mlp.experts.127.up_proj", "model.layers.18.mlp.experts.0.down_proj", "model.layers.18.mlp.experts.1.down_proj", "model.layers.18.mlp.experts.2.down_proj", "model.layers.18.mlp.experts.3.down_proj", "model.layers.18.mlp.experts.4.down_proj", "model.layers.18.mlp.experts.5.down_proj", "model.layers.18.mlp.experts.6.down_proj", "model.layers.18.mlp.experts.7.down_proj", "model.layers.18.mlp.experts.8.down_proj", "model.layers.18.mlp.experts.9.down_proj", "model.layers.18.mlp.experts.10.down_proj", "model.layers.18.mlp.experts.11.down_proj", "model.layers.18.mlp.experts.12.down_proj", "model.layers.18.mlp.experts.13.down_proj", "model.layers.18.mlp.experts.14.down_proj", "model.layers.18.mlp.experts.15.down_proj", "model.layers.18.mlp.experts.16.down_proj", "model.layers.18.mlp.experts.17.down_proj", "model.layers.18.mlp.experts.18.down_proj", "model.layers.18.mlp.experts.19.down_proj", "model.layers.18.mlp.experts.20.down_proj", "model.layers.18.mlp.experts.21.down_proj", "model.layers.18.mlp.experts.22.down_proj", "model.layers.18.mlp.experts.23.down_proj", "model.layers.18.mlp.experts.24.down_proj", "model.layers.18.mlp.experts.25.down_proj", "model.layers.18.mlp.experts.26.down_proj", "model.layers.18.mlp.experts.27.down_proj", "model.layers.18.mlp.experts.28.down_proj", "model.layers.18.mlp.experts.29.down_proj", "model.layers.18.mlp.experts.30.down_proj", "model.layers.18.mlp.experts.31.down_proj", "model.layers.18.mlp.experts.32.down_proj", "model.layers.18.mlp.experts.33.down_proj", "model.layers.18.mlp.experts.34.down_proj", "model.layers.18.mlp.experts.35.down_proj", "model.layers.18.mlp.experts.36.down_proj", "model.layers.18.mlp.experts.37.down_proj", "model.layers.18.mlp.experts.38.down_proj", "model.layers.18.mlp.experts.39.down_proj", "model.layers.18.mlp.experts.40.down_proj", "model.layers.18.mlp.experts.41.down_proj", "model.layers.18.mlp.experts.42.down_proj", "model.layers.18.mlp.experts.43.down_proj", "model.layers.18.mlp.experts.44.down_proj", "model.layers.18.mlp.experts.45.down_proj", "model.layers.18.mlp.experts.46.down_proj", "model.layers.18.mlp.experts.47.down_proj", "model.layers.18.mlp.experts.48.down_proj", "model.layers.18.mlp.experts.49.down_proj", "model.layers.18.mlp.experts.50.down_proj", "model.layers.18.mlp.experts.51.down_proj", "model.layers.18.mlp.experts.52.down_proj", "model.layers.18.mlp.experts.53.down_proj", "model.layers.18.mlp.experts.54.down_proj", "model.layers.18.mlp.experts.55.down_proj", "model.layers.18.mlp.experts.56.down_proj", "model.layers.18.mlp.experts.57.down_proj", "model.layers.18.mlp.experts.58.down_proj", "model.layers.18.mlp.experts.59.down_proj", "model.layers.18.mlp.experts.60.down_proj", "model.layers.18.mlp.experts.61.down_proj", "model.layers.18.mlp.experts.62.down_proj", "model.layers.18.mlp.experts.63.down_proj", "model.layers.18.mlp.experts.64.down_proj", "model.layers.18.mlp.experts.65.down_proj", "model.layers.18.mlp.experts.66.down_proj", "model.layers.18.mlp.experts.67.down_proj", "model.layers.18.mlp.experts.68.down_proj", "model.layers.18.mlp.experts.69.down_proj", "model.layers.18.mlp.experts.70.down_proj", "model.layers.18.mlp.experts.71.down_proj", "model.layers.18.mlp.experts.72.down_proj", "model.layers.18.mlp.experts.73.down_proj", "model.layers.18.mlp.experts.74.down_proj", "model.layers.18.mlp.experts.75.down_proj", "model.layers.18.mlp.experts.76.down_proj", "model.layers.18.mlp.experts.77.down_proj", "model.layers.18.mlp.experts.78.down_proj", "model.layers.18.mlp.experts.79.down_proj", "model.layers.18.mlp.experts.80.down_proj", "model.layers.18.mlp.experts.81.down_proj", "model.layers.18.mlp.experts.82.down_proj", "model.layers.18.mlp.experts.83.down_proj", "model.layers.18.mlp.experts.84.down_proj", "model.layers.18.mlp.experts.85.down_proj", "model.layers.18.mlp.experts.86.down_proj", "model.layers.18.mlp.experts.87.down_proj", "model.layers.18.mlp.experts.88.down_proj", "model.layers.18.mlp.experts.89.down_proj", "model.layers.18.mlp.experts.90.down_proj", "model.layers.18.mlp.experts.91.down_proj", "model.layers.18.mlp.experts.92.down_proj", "model.layers.18.mlp.experts.93.down_proj", "model.layers.18.mlp.experts.94.down_proj", "model.layers.18.mlp.experts.95.down_proj", "model.layers.18.mlp.experts.96.down_proj", "model.layers.18.mlp.experts.97.down_proj", "model.layers.18.mlp.experts.98.down_proj", "model.layers.18.mlp.experts.99.down_proj", "model.layers.18.mlp.experts.100.down_proj", "model.layers.18.mlp.experts.101.down_proj", "model.layers.18.mlp.experts.102.down_proj", "model.layers.18.mlp.experts.103.down_proj", "model.layers.18.mlp.experts.104.down_proj", "model.layers.18.mlp.experts.105.down_proj", "model.layers.18.mlp.experts.106.down_proj", "model.layers.18.mlp.experts.107.down_proj", "model.layers.18.mlp.experts.108.down_proj", "model.layers.18.mlp.experts.109.down_proj", "model.layers.18.mlp.experts.110.down_proj", "model.layers.18.mlp.experts.111.down_proj", "model.layers.18.mlp.experts.112.down_proj", "model.layers.18.mlp.experts.113.down_proj", "model.layers.18.mlp.experts.114.down_proj", "model.layers.18.mlp.experts.115.down_proj", "model.layers.18.mlp.experts.116.down_proj", "model.layers.18.mlp.experts.117.down_proj", "model.layers.18.mlp.experts.118.down_proj", "model.layers.18.mlp.experts.119.down_proj", "model.layers.18.mlp.experts.120.down_proj", "model.layers.18.mlp.experts.121.down_proj", "model.layers.18.mlp.experts.122.down_proj", "model.layers.18.mlp.experts.123.down_proj", "model.layers.18.mlp.experts.124.down_proj", "model.layers.18.mlp.experts.125.down_proj", "model.layers.18.mlp.experts.126.down_proj", "model.layers.18.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.003326257131993776, "dbits": 2214592512 } ] }, { "idx": 56, "layers": [ "model.layers.19.self_attn.q_proj", "model.layers.19.self_attn.k_proj", "model.layers.19.self_attn.v_proj", "model.layers.19.self_attn.o_proj" ], "candidates": [ { "dkld": -0.002125240303575998, "dbits": 109051904 } ] }, { "idx": 57, "layers": [ "model.layers.19.mlp.shared_experts.gate_proj", "model.layers.19.mlp.shared_experts.up_proj", "model.layers.19.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0005423284135758766, "dbits": 17301504 } ] }, { "idx": 58, "layers": [ "model.layers.19.mlp.experts.0.gate_proj", "model.layers.19.mlp.experts.1.gate_proj", "model.layers.19.mlp.experts.2.gate_proj", "model.layers.19.mlp.experts.3.gate_proj", "model.layers.19.mlp.experts.4.gate_proj", "model.layers.19.mlp.experts.5.gate_proj", "model.layers.19.mlp.experts.6.gate_proj", "model.layers.19.mlp.experts.7.gate_proj", "model.layers.19.mlp.experts.8.gate_proj", "model.layers.19.mlp.experts.9.gate_proj", "model.layers.19.mlp.experts.10.gate_proj", "model.layers.19.mlp.experts.11.gate_proj", "model.layers.19.mlp.experts.12.gate_proj", "model.layers.19.mlp.experts.13.gate_proj", "model.layers.19.mlp.experts.14.gate_proj", "model.layers.19.mlp.experts.15.gate_proj", "model.layers.19.mlp.experts.16.gate_proj", "model.layers.19.mlp.experts.17.gate_proj", "model.layers.19.mlp.experts.18.gate_proj", "model.layers.19.mlp.experts.19.gate_proj", "model.layers.19.mlp.experts.20.gate_proj", "model.layers.19.mlp.experts.21.gate_proj", "model.layers.19.mlp.experts.22.gate_proj", "model.layers.19.mlp.experts.23.gate_proj", "model.layers.19.mlp.experts.24.gate_proj", "model.layers.19.mlp.experts.25.gate_proj", "model.layers.19.mlp.experts.26.gate_proj", "model.layers.19.mlp.experts.27.gate_proj", "model.layers.19.mlp.experts.28.gate_proj", "model.layers.19.mlp.experts.29.gate_proj", "model.layers.19.mlp.experts.30.gate_proj", "model.layers.19.mlp.experts.31.gate_proj", "model.layers.19.mlp.experts.32.gate_proj", "model.layers.19.mlp.experts.33.gate_proj", "model.layers.19.mlp.experts.34.gate_proj", "model.layers.19.mlp.experts.35.gate_proj", "model.layers.19.mlp.experts.36.gate_proj", "model.layers.19.mlp.experts.37.gate_proj", "model.layers.19.mlp.experts.38.gate_proj", "model.layers.19.mlp.experts.39.gate_proj", "model.layers.19.mlp.experts.40.gate_proj", "model.layers.19.mlp.experts.41.gate_proj", "model.layers.19.mlp.experts.42.gate_proj", "model.layers.19.mlp.experts.43.gate_proj", "model.layers.19.mlp.experts.44.gate_proj", "model.layers.19.mlp.experts.45.gate_proj", "model.layers.19.mlp.experts.46.gate_proj", "model.layers.19.mlp.experts.47.gate_proj", "model.layers.19.mlp.experts.48.gate_proj", "model.layers.19.mlp.experts.49.gate_proj", "model.layers.19.mlp.experts.50.gate_proj", "model.layers.19.mlp.experts.51.gate_proj", "model.layers.19.mlp.experts.52.gate_proj", "model.layers.19.mlp.experts.53.gate_proj", "model.layers.19.mlp.experts.54.gate_proj", "model.layers.19.mlp.experts.55.gate_proj", "model.layers.19.mlp.experts.56.gate_proj", "model.layers.19.mlp.experts.57.gate_proj", "model.layers.19.mlp.experts.58.gate_proj", "model.layers.19.mlp.experts.59.gate_proj", "model.layers.19.mlp.experts.60.gate_proj", "model.layers.19.mlp.experts.61.gate_proj", "model.layers.19.mlp.experts.62.gate_proj", "model.layers.19.mlp.experts.63.gate_proj", "model.layers.19.mlp.experts.64.gate_proj", "model.layers.19.mlp.experts.65.gate_proj", "model.layers.19.mlp.experts.66.gate_proj", "model.layers.19.mlp.experts.67.gate_proj", "model.layers.19.mlp.experts.68.gate_proj", "model.layers.19.mlp.experts.69.gate_proj", "model.layers.19.mlp.experts.70.gate_proj", "model.layers.19.mlp.experts.71.gate_proj", "model.layers.19.mlp.experts.72.gate_proj", "model.layers.19.mlp.experts.73.gate_proj", "model.layers.19.mlp.experts.74.gate_proj", "model.layers.19.mlp.experts.75.gate_proj", "model.layers.19.mlp.experts.76.gate_proj", "model.layers.19.mlp.experts.77.gate_proj", "model.layers.19.mlp.experts.78.gate_proj", "model.layers.19.mlp.experts.79.gate_proj", "model.layers.19.mlp.experts.80.gate_proj", "model.layers.19.mlp.experts.81.gate_proj", "model.layers.19.mlp.experts.82.gate_proj", "model.layers.19.mlp.experts.83.gate_proj", "model.layers.19.mlp.experts.84.gate_proj", "model.layers.19.mlp.experts.85.gate_proj", "model.layers.19.mlp.experts.86.gate_proj", "model.layers.19.mlp.experts.87.gate_proj", "model.layers.19.mlp.experts.88.gate_proj", "model.layers.19.mlp.experts.89.gate_proj", "model.layers.19.mlp.experts.90.gate_proj", "model.layers.19.mlp.experts.91.gate_proj", "model.layers.19.mlp.experts.92.gate_proj", "model.layers.19.mlp.experts.93.gate_proj", "model.layers.19.mlp.experts.94.gate_proj", "model.layers.19.mlp.experts.95.gate_proj", "model.layers.19.mlp.experts.96.gate_proj", "model.layers.19.mlp.experts.97.gate_proj", "model.layers.19.mlp.experts.98.gate_proj", "model.layers.19.mlp.experts.99.gate_proj", "model.layers.19.mlp.experts.100.gate_proj", "model.layers.19.mlp.experts.101.gate_proj", "model.layers.19.mlp.experts.102.gate_proj", "model.layers.19.mlp.experts.103.gate_proj", "model.layers.19.mlp.experts.104.gate_proj", "model.layers.19.mlp.experts.105.gate_proj", "model.layers.19.mlp.experts.106.gate_proj", "model.layers.19.mlp.experts.107.gate_proj", "model.layers.19.mlp.experts.108.gate_proj", "model.layers.19.mlp.experts.109.gate_proj", "model.layers.19.mlp.experts.110.gate_proj", "model.layers.19.mlp.experts.111.gate_proj", "model.layers.19.mlp.experts.112.gate_proj", "model.layers.19.mlp.experts.113.gate_proj", "model.layers.19.mlp.experts.114.gate_proj", "model.layers.19.mlp.experts.115.gate_proj", "model.layers.19.mlp.experts.116.gate_proj", "model.layers.19.mlp.experts.117.gate_proj", "model.layers.19.mlp.experts.118.gate_proj", "model.layers.19.mlp.experts.119.gate_proj", "model.layers.19.mlp.experts.120.gate_proj", "model.layers.19.mlp.experts.121.gate_proj", "model.layers.19.mlp.experts.122.gate_proj", "model.layers.19.mlp.experts.123.gate_proj", "model.layers.19.mlp.experts.124.gate_proj", "model.layers.19.mlp.experts.125.gate_proj", "model.layers.19.mlp.experts.126.gate_proj", "model.layers.19.mlp.experts.127.gate_proj", "model.layers.19.mlp.experts.0.up_proj", "model.layers.19.mlp.experts.1.up_proj", "model.layers.19.mlp.experts.2.up_proj", "model.layers.19.mlp.experts.3.up_proj", "model.layers.19.mlp.experts.4.up_proj", "model.layers.19.mlp.experts.5.up_proj", "model.layers.19.mlp.experts.6.up_proj", "model.layers.19.mlp.experts.7.up_proj", "model.layers.19.mlp.experts.8.up_proj", "model.layers.19.mlp.experts.9.up_proj", "model.layers.19.mlp.experts.10.up_proj", "model.layers.19.mlp.experts.11.up_proj", "model.layers.19.mlp.experts.12.up_proj", "model.layers.19.mlp.experts.13.up_proj", "model.layers.19.mlp.experts.14.up_proj", "model.layers.19.mlp.experts.15.up_proj", "model.layers.19.mlp.experts.16.up_proj", "model.layers.19.mlp.experts.17.up_proj", "model.layers.19.mlp.experts.18.up_proj", "model.layers.19.mlp.experts.19.up_proj", "model.layers.19.mlp.experts.20.up_proj", "model.layers.19.mlp.experts.21.up_proj", "model.layers.19.mlp.experts.22.up_proj", "model.layers.19.mlp.experts.23.up_proj", "model.layers.19.mlp.experts.24.up_proj", "model.layers.19.mlp.experts.25.up_proj", "model.layers.19.mlp.experts.26.up_proj", "model.layers.19.mlp.experts.27.up_proj", "model.layers.19.mlp.experts.28.up_proj", "model.layers.19.mlp.experts.29.up_proj", "model.layers.19.mlp.experts.30.up_proj", "model.layers.19.mlp.experts.31.up_proj", "model.layers.19.mlp.experts.32.up_proj", "model.layers.19.mlp.experts.33.up_proj", "model.layers.19.mlp.experts.34.up_proj", "model.layers.19.mlp.experts.35.up_proj", "model.layers.19.mlp.experts.36.up_proj", "model.layers.19.mlp.experts.37.up_proj", "model.layers.19.mlp.experts.38.up_proj", "model.layers.19.mlp.experts.39.up_proj", "model.layers.19.mlp.experts.40.up_proj", "model.layers.19.mlp.experts.41.up_proj", "model.layers.19.mlp.experts.42.up_proj", "model.layers.19.mlp.experts.43.up_proj", "model.layers.19.mlp.experts.44.up_proj", "model.layers.19.mlp.experts.45.up_proj", "model.layers.19.mlp.experts.46.up_proj", "model.layers.19.mlp.experts.47.up_proj", "model.layers.19.mlp.experts.48.up_proj", "model.layers.19.mlp.experts.49.up_proj", "model.layers.19.mlp.experts.50.up_proj", "model.layers.19.mlp.experts.51.up_proj", "model.layers.19.mlp.experts.52.up_proj", "model.layers.19.mlp.experts.53.up_proj", "model.layers.19.mlp.experts.54.up_proj", "model.layers.19.mlp.experts.55.up_proj", "model.layers.19.mlp.experts.56.up_proj", "model.layers.19.mlp.experts.57.up_proj", "model.layers.19.mlp.experts.58.up_proj", "model.layers.19.mlp.experts.59.up_proj", "model.layers.19.mlp.experts.60.up_proj", "model.layers.19.mlp.experts.61.up_proj", "model.layers.19.mlp.experts.62.up_proj", "model.layers.19.mlp.experts.63.up_proj", "model.layers.19.mlp.experts.64.up_proj", "model.layers.19.mlp.experts.65.up_proj", "model.layers.19.mlp.experts.66.up_proj", "model.layers.19.mlp.experts.67.up_proj", "model.layers.19.mlp.experts.68.up_proj", "model.layers.19.mlp.experts.69.up_proj", "model.layers.19.mlp.experts.70.up_proj", "model.layers.19.mlp.experts.71.up_proj", "model.layers.19.mlp.experts.72.up_proj", "model.layers.19.mlp.experts.73.up_proj", "model.layers.19.mlp.experts.74.up_proj", "model.layers.19.mlp.experts.75.up_proj", "model.layers.19.mlp.experts.76.up_proj", "model.layers.19.mlp.experts.77.up_proj", "model.layers.19.mlp.experts.78.up_proj", "model.layers.19.mlp.experts.79.up_proj", "model.layers.19.mlp.experts.80.up_proj", "model.layers.19.mlp.experts.81.up_proj", "model.layers.19.mlp.experts.82.up_proj", "model.layers.19.mlp.experts.83.up_proj", "model.layers.19.mlp.experts.84.up_proj", "model.layers.19.mlp.experts.85.up_proj", "model.layers.19.mlp.experts.86.up_proj", "model.layers.19.mlp.experts.87.up_proj", "model.layers.19.mlp.experts.88.up_proj", "model.layers.19.mlp.experts.89.up_proj", "model.layers.19.mlp.experts.90.up_proj", "model.layers.19.mlp.experts.91.up_proj", "model.layers.19.mlp.experts.92.up_proj", "model.layers.19.mlp.experts.93.up_proj", "model.layers.19.mlp.experts.94.up_proj", "model.layers.19.mlp.experts.95.up_proj", "model.layers.19.mlp.experts.96.up_proj", "model.layers.19.mlp.experts.97.up_proj", "model.layers.19.mlp.experts.98.up_proj", "model.layers.19.mlp.experts.99.up_proj", "model.layers.19.mlp.experts.100.up_proj", "model.layers.19.mlp.experts.101.up_proj", "model.layers.19.mlp.experts.102.up_proj", "model.layers.19.mlp.experts.103.up_proj", "model.layers.19.mlp.experts.104.up_proj", "model.layers.19.mlp.experts.105.up_proj", "model.layers.19.mlp.experts.106.up_proj", "model.layers.19.mlp.experts.107.up_proj", "model.layers.19.mlp.experts.108.up_proj", "model.layers.19.mlp.experts.109.up_proj", "model.layers.19.mlp.experts.110.up_proj", "model.layers.19.mlp.experts.111.up_proj", "model.layers.19.mlp.experts.112.up_proj", "model.layers.19.mlp.experts.113.up_proj", "model.layers.19.mlp.experts.114.up_proj", "model.layers.19.mlp.experts.115.up_proj", "model.layers.19.mlp.experts.116.up_proj", "model.layers.19.mlp.experts.117.up_proj", "model.layers.19.mlp.experts.118.up_proj", "model.layers.19.mlp.experts.119.up_proj", "model.layers.19.mlp.experts.120.up_proj", "model.layers.19.mlp.experts.121.up_proj", "model.layers.19.mlp.experts.122.up_proj", "model.layers.19.mlp.experts.123.up_proj", "model.layers.19.mlp.experts.124.up_proj", "model.layers.19.mlp.experts.125.up_proj", "model.layers.19.mlp.experts.126.up_proj", "model.layers.19.mlp.experts.127.up_proj", "model.layers.19.mlp.experts.0.down_proj", "model.layers.19.mlp.experts.1.down_proj", "model.layers.19.mlp.experts.2.down_proj", "model.layers.19.mlp.experts.3.down_proj", "model.layers.19.mlp.experts.4.down_proj", "model.layers.19.mlp.experts.5.down_proj", "model.layers.19.mlp.experts.6.down_proj", "model.layers.19.mlp.experts.7.down_proj", "model.layers.19.mlp.experts.8.down_proj", "model.layers.19.mlp.experts.9.down_proj", "model.layers.19.mlp.experts.10.down_proj", "model.layers.19.mlp.experts.11.down_proj", "model.layers.19.mlp.experts.12.down_proj", "model.layers.19.mlp.experts.13.down_proj", "model.layers.19.mlp.experts.14.down_proj", "model.layers.19.mlp.experts.15.down_proj", "model.layers.19.mlp.experts.16.down_proj", "model.layers.19.mlp.experts.17.down_proj", "model.layers.19.mlp.experts.18.down_proj", "model.layers.19.mlp.experts.19.down_proj", "model.layers.19.mlp.experts.20.down_proj", "model.layers.19.mlp.experts.21.down_proj", "model.layers.19.mlp.experts.22.down_proj", "model.layers.19.mlp.experts.23.down_proj", "model.layers.19.mlp.experts.24.down_proj", "model.layers.19.mlp.experts.25.down_proj", "model.layers.19.mlp.experts.26.down_proj", "model.layers.19.mlp.experts.27.down_proj", "model.layers.19.mlp.experts.28.down_proj", "model.layers.19.mlp.experts.29.down_proj", "model.layers.19.mlp.experts.30.down_proj", "model.layers.19.mlp.experts.31.down_proj", "model.layers.19.mlp.experts.32.down_proj", "model.layers.19.mlp.experts.33.down_proj", "model.layers.19.mlp.experts.34.down_proj", "model.layers.19.mlp.experts.35.down_proj", "model.layers.19.mlp.experts.36.down_proj", "model.layers.19.mlp.experts.37.down_proj", "model.layers.19.mlp.experts.38.down_proj", "model.layers.19.mlp.experts.39.down_proj", "model.layers.19.mlp.experts.40.down_proj", "model.layers.19.mlp.experts.41.down_proj", "model.layers.19.mlp.experts.42.down_proj", "model.layers.19.mlp.experts.43.down_proj", "model.layers.19.mlp.experts.44.down_proj", "model.layers.19.mlp.experts.45.down_proj", "model.layers.19.mlp.experts.46.down_proj", "model.layers.19.mlp.experts.47.down_proj", "model.layers.19.mlp.experts.48.down_proj", "model.layers.19.mlp.experts.49.down_proj", "model.layers.19.mlp.experts.50.down_proj", "model.layers.19.mlp.experts.51.down_proj", "model.layers.19.mlp.experts.52.down_proj", "model.layers.19.mlp.experts.53.down_proj", "model.layers.19.mlp.experts.54.down_proj", "model.layers.19.mlp.experts.55.down_proj", "model.layers.19.mlp.experts.56.down_proj", "model.layers.19.mlp.experts.57.down_proj", "model.layers.19.mlp.experts.58.down_proj", "model.layers.19.mlp.experts.59.down_proj", "model.layers.19.mlp.experts.60.down_proj", "model.layers.19.mlp.experts.61.down_proj", "model.layers.19.mlp.experts.62.down_proj", "model.layers.19.mlp.experts.63.down_proj", "model.layers.19.mlp.experts.64.down_proj", "model.layers.19.mlp.experts.65.down_proj", "model.layers.19.mlp.experts.66.down_proj", "model.layers.19.mlp.experts.67.down_proj", "model.layers.19.mlp.experts.68.down_proj", "model.layers.19.mlp.experts.69.down_proj", "model.layers.19.mlp.experts.70.down_proj", "model.layers.19.mlp.experts.71.down_proj", "model.layers.19.mlp.experts.72.down_proj", "model.layers.19.mlp.experts.73.down_proj", "model.layers.19.mlp.experts.74.down_proj", "model.layers.19.mlp.experts.75.down_proj", "model.layers.19.mlp.experts.76.down_proj", "model.layers.19.mlp.experts.77.down_proj", "model.layers.19.mlp.experts.78.down_proj", "model.layers.19.mlp.experts.79.down_proj", "model.layers.19.mlp.experts.80.down_proj", "model.layers.19.mlp.experts.81.down_proj", "model.layers.19.mlp.experts.82.down_proj", "model.layers.19.mlp.experts.83.down_proj", "model.layers.19.mlp.experts.84.down_proj", "model.layers.19.mlp.experts.85.down_proj", "model.layers.19.mlp.experts.86.down_proj", "model.layers.19.mlp.experts.87.down_proj", "model.layers.19.mlp.experts.88.down_proj", "model.layers.19.mlp.experts.89.down_proj", "model.layers.19.mlp.experts.90.down_proj", "model.layers.19.mlp.experts.91.down_proj", "model.layers.19.mlp.experts.92.down_proj", "model.layers.19.mlp.experts.93.down_proj", "model.layers.19.mlp.experts.94.down_proj", "model.layers.19.mlp.experts.95.down_proj", "model.layers.19.mlp.experts.96.down_proj", "model.layers.19.mlp.experts.97.down_proj", "model.layers.19.mlp.experts.98.down_proj", "model.layers.19.mlp.experts.99.down_proj", "model.layers.19.mlp.experts.100.down_proj", "model.layers.19.mlp.experts.101.down_proj", "model.layers.19.mlp.experts.102.down_proj", "model.layers.19.mlp.experts.103.down_proj", "model.layers.19.mlp.experts.104.down_proj", "model.layers.19.mlp.experts.105.down_proj", "model.layers.19.mlp.experts.106.down_proj", "model.layers.19.mlp.experts.107.down_proj", "model.layers.19.mlp.experts.108.down_proj", "model.layers.19.mlp.experts.109.down_proj", "model.layers.19.mlp.experts.110.down_proj", "model.layers.19.mlp.experts.111.down_proj", "model.layers.19.mlp.experts.112.down_proj", "model.layers.19.mlp.experts.113.down_proj", "model.layers.19.mlp.experts.114.down_proj", "model.layers.19.mlp.experts.115.down_proj", "model.layers.19.mlp.experts.116.down_proj", "model.layers.19.mlp.experts.117.down_proj", "model.layers.19.mlp.experts.118.down_proj", "model.layers.19.mlp.experts.119.down_proj", "model.layers.19.mlp.experts.120.down_proj", "model.layers.19.mlp.experts.121.down_proj", "model.layers.19.mlp.experts.122.down_proj", "model.layers.19.mlp.experts.123.down_proj", "model.layers.19.mlp.experts.124.down_proj", "model.layers.19.mlp.experts.125.down_proj", "model.layers.19.mlp.experts.126.down_proj", "model.layers.19.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.00470351185649634, "dbits": 2214592512 } ] }, { "idx": 59, "layers": [ "model.layers.20.self_attn.q_proj", "model.layers.20.self_attn.k_proj", "model.layers.20.self_attn.v_proj", "model.layers.20.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0009955259971320685, "dbits": 109051904 } ] }, { "idx": 60, "layers": [ "model.layers.20.mlp.shared_experts.gate_proj", "model.layers.20.mlp.shared_experts.up_proj", "model.layers.20.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0014827062375843414, "dbits": 17301504 } ] }, { "idx": 61, "layers": [ "model.layers.20.mlp.experts.0.gate_proj", "model.layers.20.mlp.experts.1.gate_proj", "model.layers.20.mlp.experts.2.gate_proj", "model.layers.20.mlp.experts.3.gate_proj", "model.layers.20.mlp.experts.4.gate_proj", "model.layers.20.mlp.experts.5.gate_proj", "model.layers.20.mlp.experts.6.gate_proj", "model.layers.20.mlp.experts.7.gate_proj", "model.layers.20.mlp.experts.8.gate_proj", "model.layers.20.mlp.experts.9.gate_proj", "model.layers.20.mlp.experts.10.gate_proj", "model.layers.20.mlp.experts.11.gate_proj", "model.layers.20.mlp.experts.12.gate_proj", "model.layers.20.mlp.experts.13.gate_proj", "model.layers.20.mlp.experts.14.gate_proj", "model.layers.20.mlp.experts.15.gate_proj", "model.layers.20.mlp.experts.16.gate_proj", "model.layers.20.mlp.experts.17.gate_proj", "model.layers.20.mlp.experts.18.gate_proj", "model.layers.20.mlp.experts.19.gate_proj", "model.layers.20.mlp.experts.20.gate_proj", "model.layers.20.mlp.experts.21.gate_proj", "model.layers.20.mlp.experts.22.gate_proj", "model.layers.20.mlp.experts.23.gate_proj", "model.layers.20.mlp.experts.24.gate_proj", "model.layers.20.mlp.experts.25.gate_proj", "model.layers.20.mlp.experts.26.gate_proj", "model.layers.20.mlp.experts.27.gate_proj", "model.layers.20.mlp.experts.28.gate_proj", "model.layers.20.mlp.experts.29.gate_proj", "model.layers.20.mlp.experts.30.gate_proj", "model.layers.20.mlp.experts.31.gate_proj", "model.layers.20.mlp.experts.32.gate_proj", "model.layers.20.mlp.experts.33.gate_proj", "model.layers.20.mlp.experts.34.gate_proj", "model.layers.20.mlp.experts.35.gate_proj", "model.layers.20.mlp.experts.36.gate_proj", "model.layers.20.mlp.experts.37.gate_proj", "model.layers.20.mlp.experts.38.gate_proj", "model.layers.20.mlp.experts.39.gate_proj", "model.layers.20.mlp.experts.40.gate_proj", "model.layers.20.mlp.experts.41.gate_proj", "model.layers.20.mlp.experts.42.gate_proj", "model.layers.20.mlp.experts.43.gate_proj", "model.layers.20.mlp.experts.44.gate_proj", "model.layers.20.mlp.experts.45.gate_proj", "model.layers.20.mlp.experts.46.gate_proj", "model.layers.20.mlp.experts.47.gate_proj", "model.layers.20.mlp.experts.48.gate_proj", "model.layers.20.mlp.experts.49.gate_proj", "model.layers.20.mlp.experts.50.gate_proj", "model.layers.20.mlp.experts.51.gate_proj", "model.layers.20.mlp.experts.52.gate_proj", "model.layers.20.mlp.experts.53.gate_proj", "model.layers.20.mlp.experts.54.gate_proj", "model.layers.20.mlp.experts.55.gate_proj", "model.layers.20.mlp.experts.56.gate_proj", "model.layers.20.mlp.experts.57.gate_proj", "model.layers.20.mlp.experts.58.gate_proj", "model.layers.20.mlp.experts.59.gate_proj", "model.layers.20.mlp.experts.60.gate_proj", "model.layers.20.mlp.experts.61.gate_proj", "model.layers.20.mlp.experts.62.gate_proj", "model.layers.20.mlp.experts.63.gate_proj", "model.layers.20.mlp.experts.64.gate_proj", "model.layers.20.mlp.experts.65.gate_proj", "model.layers.20.mlp.experts.66.gate_proj", "model.layers.20.mlp.experts.67.gate_proj", "model.layers.20.mlp.experts.68.gate_proj", "model.layers.20.mlp.experts.69.gate_proj", "model.layers.20.mlp.experts.70.gate_proj", "model.layers.20.mlp.experts.71.gate_proj", "model.layers.20.mlp.experts.72.gate_proj", "model.layers.20.mlp.experts.73.gate_proj", "model.layers.20.mlp.experts.74.gate_proj", "model.layers.20.mlp.experts.75.gate_proj", "model.layers.20.mlp.experts.76.gate_proj", "model.layers.20.mlp.experts.77.gate_proj", "model.layers.20.mlp.experts.78.gate_proj", "model.layers.20.mlp.experts.79.gate_proj", "model.layers.20.mlp.experts.80.gate_proj", "model.layers.20.mlp.experts.81.gate_proj", "model.layers.20.mlp.experts.82.gate_proj", "model.layers.20.mlp.experts.83.gate_proj", "model.layers.20.mlp.experts.84.gate_proj", "model.layers.20.mlp.experts.85.gate_proj", "model.layers.20.mlp.experts.86.gate_proj", "model.layers.20.mlp.experts.87.gate_proj", "model.layers.20.mlp.experts.88.gate_proj", "model.layers.20.mlp.experts.89.gate_proj", "model.layers.20.mlp.experts.90.gate_proj", "model.layers.20.mlp.experts.91.gate_proj", "model.layers.20.mlp.experts.92.gate_proj", "model.layers.20.mlp.experts.93.gate_proj", "model.layers.20.mlp.experts.94.gate_proj", "model.layers.20.mlp.experts.95.gate_proj", "model.layers.20.mlp.experts.96.gate_proj", "model.layers.20.mlp.experts.97.gate_proj", "model.layers.20.mlp.experts.98.gate_proj", "model.layers.20.mlp.experts.99.gate_proj", "model.layers.20.mlp.experts.100.gate_proj", "model.layers.20.mlp.experts.101.gate_proj", "model.layers.20.mlp.experts.102.gate_proj", "model.layers.20.mlp.experts.103.gate_proj", "model.layers.20.mlp.experts.104.gate_proj", "model.layers.20.mlp.experts.105.gate_proj", "model.layers.20.mlp.experts.106.gate_proj", "model.layers.20.mlp.experts.107.gate_proj", "model.layers.20.mlp.experts.108.gate_proj", "model.layers.20.mlp.experts.109.gate_proj", "model.layers.20.mlp.experts.110.gate_proj", "model.layers.20.mlp.experts.111.gate_proj", "model.layers.20.mlp.experts.112.gate_proj", "model.layers.20.mlp.experts.113.gate_proj", "model.layers.20.mlp.experts.114.gate_proj", "model.layers.20.mlp.experts.115.gate_proj", "model.layers.20.mlp.experts.116.gate_proj", "model.layers.20.mlp.experts.117.gate_proj", "model.layers.20.mlp.experts.118.gate_proj", "model.layers.20.mlp.experts.119.gate_proj", "model.layers.20.mlp.experts.120.gate_proj", "model.layers.20.mlp.experts.121.gate_proj", "model.layers.20.mlp.experts.122.gate_proj", "model.layers.20.mlp.experts.123.gate_proj", "model.layers.20.mlp.experts.124.gate_proj", "model.layers.20.mlp.experts.125.gate_proj", "model.layers.20.mlp.experts.126.gate_proj", "model.layers.20.mlp.experts.127.gate_proj", "model.layers.20.mlp.experts.0.up_proj", "model.layers.20.mlp.experts.1.up_proj", "model.layers.20.mlp.experts.2.up_proj", "model.layers.20.mlp.experts.3.up_proj", "model.layers.20.mlp.experts.4.up_proj", "model.layers.20.mlp.experts.5.up_proj", "model.layers.20.mlp.experts.6.up_proj", "model.layers.20.mlp.experts.7.up_proj", "model.layers.20.mlp.experts.8.up_proj", "model.layers.20.mlp.experts.9.up_proj", "model.layers.20.mlp.experts.10.up_proj", "model.layers.20.mlp.experts.11.up_proj", "model.layers.20.mlp.experts.12.up_proj", "model.layers.20.mlp.experts.13.up_proj", "model.layers.20.mlp.experts.14.up_proj", "model.layers.20.mlp.experts.15.up_proj", "model.layers.20.mlp.experts.16.up_proj", "model.layers.20.mlp.experts.17.up_proj", "model.layers.20.mlp.experts.18.up_proj", "model.layers.20.mlp.experts.19.up_proj", "model.layers.20.mlp.experts.20.up_proj", "model.layers.20.mlp.experts.21.up_proj", "model.layers.20.mlp.experts.22.up_proj", "model.layers.20.mlp.experts.23.up_proj", "model.layers.20.mlp.experts.24.up_proj", "model.layers.20.mlp.experts.25.up_proj", "model.layers.20.mlp.experts.26.up_proj", "model.layers.20.mlp.experts.27.up_proj", "model.layers.20.mlp.experts.28.up_proj", "model.layers.20.mlp.experts.29.up_proj", "model.layers.20.mlp.experts.30.up_proj", "model.layers.20.mlp.experts.31.up_proj", "model.layers.20.mlp.experts.32.up_proj", "model.layers.20.mlp.experts.33.up_proj", "model.layers.20.mlp.experts.34.up_proj", "model.layers.20.mlp.experts.35.up_proj", "model.layers.20.mlp.experts.36.up_proj", "model.layers.20.mlp.experts.37.up_proj", "model.layers.20.mlp.experts.38.up_proj", "model.layers.20.mlp.experts.39.up_proj", "model.layers.20.mlp.experts.40.up_proj", "model.layers.20.mlp.experts.41.up_proj", "model.layers.20.mlp.experts.42.up_proj", "model.layers.20.mlp.experts.43.up_proj", "model.layers.20.mlp.experts.44.up_proj", "model.layers.20.mlp.experts.45.up_proj", "model.layers.20.mlp.experts.46.up_proj", "model.layers.20.mlp.experts.47.up_proj", "model.layers.20.mlp.experts.48.up_proj", "model.layers.20.mlp.experts.49.up_proj", "model.layers.20.mlp.experts.50.up_proj", "model.layers.20.mlp.experts.51.up_proj", "model.layers.20.mlp.experts.52.up_proj", "model.layers.20.mlp.experts.53.up_proj", "model.layers.20.mlp.experts.54.up_proj", "model.layers.20.mlp.experts.55.up_proj", "model.layers.20.mlp.experts.56.up_proj", "model.layers.20.mlp.experts.57.up_proj", "model.layers.20.mlp.experts.58.up_proj", "model.layers.20.mlp.experts.59.up_proj", "model.layers.20.mlp.experts.60.up_proj", "model.layers.20.mlp.experts.61.up_proj", "model.layers.20.mlp.experts.62.up_proj", "model.layers.20.mlp.experts.63.up_proj", "model.layers.20.mlp.experts.64.up_proj", "model.layers.20.mlp.experts.65.up_proj", "model.layers.20.mlp.experts.66.up_proj", "model.layers.20.mlp.experts.67.up_proj", "model.layers.20.mlp.experts.68.up_proj", "model.layers.20.mlp.experts.69.up_proj", "model.layers.20.mlp.experts.70.up_proj", "model.layers.20.mlp.experts.71.up_proj", "model.layers.20.mlp.experts.72.up_proj", "model.layers.20.mlp.experts.73.up_proj", "model.layers.20.mlp.experts.74.up_proj", "model.layers.20.mlp.experts.75.up_proj", "model.layers.20.mlp.experts.76.up_proj", "model.layers.20.mlp.experts.77.up_proj", "model.layers.20.mlp.experts.78.up_proj", "model.layers.20.mlp.experts.79.up_proj", "model.layers.20.mlp.experts.80.up_proj", "model.layers.20.mlp.experts.81.up_proj", "model.layers.20.mlp.experts.82.up_proj", "model.layers.20.mlp.experts.83.up_proj", "model.layers.20.mlp.experts.84.up_proj", "model.layers.20.mlp.experts.85.up_proj", "model.layers.20.mlp.experts.86.up_proj", "model.layers.20.mlp.experts.87.up_proj", "model.layers.20.mlp.experts.88.up_proj", "model.layers.20.mlp.experts.89.up_proj", "model.layers.20.mlp.experts.90.up_proj", "model.layers.20.mlp.experts.91.up_proj", "model.layers.20.mlp.experts.92.up_proj", "model.layers.20.mlp.experts.93.up_proj", "model.layers.20.mlp.experts.94.up_proj", "model.layers.20.mlp.experts.95.up_proj", "model.layers.20.mlp.experts.96.up_proj", "model.layers.20.mlp.experts.97.up_proj", "model.layers.20.mlp.experts.98.up_proj", "model.layers.20.mlp.experts.99.up_proj", "model.layers.20.mlp.experts.100.up_proj", "model.layers.20.mlp.experts.101.up_proj", "model.layers.20.mlp.experts.102.up_proj", "model.layers.20.mlp.experts.103.up_proj", "model.layers.20.mlp.experts.104.up_proj", "model.layers.20.mlp.experts.105.up_proj", "model.layers.20.mlp.experts.106.up_proj", "model.layers.20.mlp.experts.107.up_proj", "model.layers.20.mlp.experts.108.up_proj", "model.layers.20.mlp.experts.109.up_proj", "model.layers.20.mlp.experts.110.up_proj", "model.layers.20.mlp.experts.111.up_proj", "model.layers.20.mlp.experts.112.up_proj", "model.layers.20.mlp.experts.113.up_proj", "model.layers.20.mlp.experts.114.up_proj", "model.layers.20.mlp.experts.115.up_proj", "model.layers.20.mlp.experts.116.up_proj", "model.layers.20.mlp.experts.117.up_proj", "model.layers.20.mlp.experts.118.up_proj", "model.layers.20.mlp.experts.119.up_proj", "model.layers.20.mlp.experts.120.up_proj", "model.layers.20.mlp.experts.121.up_proj", "model.layers.20.mlp.experts.122.up_proj", "model.layers.20.mlp.experts.123.up_proj", "model.layers.20.mlp.experts.124.up_proj", "model.layers.20.mlp.experts.125.up_proj", "model.layers.20.mlp.experts.126.up_proj", "model.layers.20.mlp.experts.127.up_proj", "model.layers.20.mlp.experts.0.down_proj", "model.layers.20.mlp.experts.1.down_proj", "model.layers.20.mlp.experts.2.down_proj", "model.layers.20.mlp.experts.3.down_proj", "model.layers.20.mlp.experts.4.down_proj", "model.layers.20.mlp.experts.5.down_proj", "model.layers.20.mlp.experts.6.down_proj", "model.layers.20.mlp.experts.7.down_proj", "model.layers.20.mlp.experts.8.down_proj", "model.layers.20.mlp.experts.9.down_proj", "model.layers.20.mlp.experts.10.down_proj", "model.layers.20.mlp.experts.11.down_proj", "model.layers.20.mlp.experts.12.down_proj", "model.layers.20.mlp.experts.13.down_proj", "model.layers.20.mlp.experts.14.down_proj", "model.layers.20.mlp.experts.15.down_proj", "model.layers.20.mlp.experts.16.down_proj", "model.layers.20.mlp.experts.17.down_proj", "model.layers.20.mlp.experts.18.down_proj", "model.layers.20.mlp.experts.19.down_proj", "model.layers.20.mlp.experts.20.down_proj", "model.layers.20.mlp.experts.21.down_proj", "model.layers.20.mlp.experts.22.down_proj", "model.layers.20.mlp.experts.23.down_proj", "model.layers.20.mlp.experts.24.down_proj", "model.layers.20.mlp.experts.25.down_proj", "model.layers.20.mlp.experts.26.down_proj", "model.layers.20.mlp.experts.27.down_proj", "model.layers.20.mlp.experts.28.down_proj", "model.layers.20.mlp.experts.29.down_proj", "model.layers.20.mlp.experts.30.down_proj", "model.layers.20.mlp.experts.31.down_proj", "model.layers.20.mlp.experts.32.down_proj", "model.layers.20.mlp.experts.33.down_proj", "model.layers.20.mlp.experts.34.down_proj", "model.layers.20.mlp.experts.35.down_proj", "model.layers.20.mlp.experts.36.down_proj", "model.layers.20.mlp.experts.37.down_proj", "model.layers.20.mlp.experts.38.down_proj", "model.layers.20.mlp.experts.39.down_proj", "model.layers.20.mlp.experts.40.down_proj", "model.layers.20.mlp.experts.41.down_proj", "model.layers.20.mlp.experts.42.down_proj", "model.layers.20.mlp.experts.43.down_proj", "model.layers.20.mlp.experts.44.down_proj", "model.layers.20.mlp.experts.45.down_proj", "model.layers.20.mlp.experts.46.down_proj", "model.layers.20.mlp.experts.47.down_proj", "model.layers.20.mlp.experts.48.down_proj", "model.layers.20.mlp.experts.49.down_proj", "model.layers.20.mlp.experts.50.down_proj", "model.layers.20.mlp.experts.51.down_proj", "model.layers.20.mlp.experts.52.down_proj", "model.layers.20.mlp.experts.53.down_proj", "model.layers.20.mlp.experts.54.down_proj", "model.layers.20.mlp.experts.55.down_proj", "model.layers.20.mlp.experts.56.down_proj", "model.layers.20.mlp.experts.57.down_proj", "model.layers.20.mlp.experts.58.down_proj", "model.layers.20.mlp.experts.59.down_proj", "model.layers.20.mlp.experts.60.down_proj", "model.layers.20.mlp.experts.61.down_proj", "model.layers.20.mlp.experts.62.down_proj", "model.layers.20.mlp.experts.63.down_proj", "model.layers.20.mlp.experts.64.down_proj", "model.layers.20.mlp.experts.65.down_proj", "model.layers.20.mlp.experts.66.down_proj", "model.layers.20.mlp.experts.67.down_proj", "model.layers.20.mlp.experts.68.down_proj", "model.layers.20.mlp.experts.69.down_proj", "model.layers.20.mlp.experts.70.down_proj", "model.layers.20.mlp.experts.71.down_proj", "model.layers.20.mlp.experts.72.down_proj", "model.layers.20.mlp.experts.73.down_proj", "model.layers.20.mlp.experts.74.down_proj", "model.layers.20.mlp.experts.75.down_proj", "model.layers.20.mlp.experts.76.down_proj", "model.layers.20.mlp.experts.77.down_proj", "model.layers.20.mlp.experts.78.down_proj", "model.layers.20.mlp.experts.79.down_proj", "model.layers.20.mlp.experts.80.down_proj", "model.layers.20.mlp.experts.81.down_proj", "model.layers.20.mlp.experts.82.down_proj", "model.layers.20.mlp.experts.83.down_proj", "model.layers.20.mlp.experts.84.down_proj", "model.layers.20.mlp.experts.85.down_proj", "model.layers.20.mlp.experts.86.down_proj", "model.layers.20.mlp.experts.87.down_proj", "model.layers.20.mlp.experts.88.down_proj", "model.layers.20.mlp.experts.89.down_proj", "model.layers.20.mlp.experts.90.down_proj", "model.layers.20.mlp.experts.91.down_proj", "model.layers.20.mlp.experts.92.down_proj", "model.layers.20.mlp.experts.93.down_proj", "model.layers.20.mlp.experts.94.down_proj", "model.layers.20.mlp.experts.95.down_proj", "model.layers.20.mlp.experts.96.down_proj", "model.layers.20.mlp.experts.97.down_proj", "model.layers.20.mlp.experts.98.down_proj", "model.layers.20.mlp.experts.99.down_proj", "model.layers.20.mlp.experts.100.down_proj", "model.layers.20.mlp.experts.101.down_proj", "model.layers.20.mlp.experts.102.down_proj", "model.layers.20.mlp.experts.103.down_proj", "model.layers.20.mlp.experts.104.down_proj", "model.layers.20.mlp.experts.105.down_proj", "model.layers.20.mlp.experts.106.down_proj", "model.layers.20.mlp.experts.107.down_proj", "model.layers.20.mlp.experts.108.down_proj", "model.layers.20.mlp.experts.109.down_proj", "model.layers.20.mlp.experts.110.down_proj", "model.layers.20.mlp.experts.111.down_proj", "model.layers.20.mlp.experts.112.down_proj", "model.layers.20.mlp.experts.113.down_proj", "model.layers.20.mlp.experts.114.down_proj", "model.layers.20.mlp.experts.115.down_proj", "model.layers.20.mlp.experts.116.down_proj", "model.layers.20.mlp.experts.117.down_proj", "model.layers.20.mlp.experts.118.down_proj", "model.layers.20.mlp.experts.119.down_proj", "model.layers.20.mlp.experts.120.down_proj", "model.layers.20.mlp.experts.121.down_proj", "model.layers.20.mlp.experts.122.down_proj", "model.layers.20.mlp.experts.123.down_proj", "model.layers.20.mlp.experts.124.down_proj", "model.layers.20.mlp.experts.125.down_proj", "model.layers.20.mlp.experts.126.down_proj", "model.layers.20.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.003039991389960056, "dbits": 2214592512 } ] }, { "idx": 62, "layers": [ "model.layers.21.self_attn.q_proj", "model.layers.21.self_attn.k_proj", "model.layers.21.self_attn.v_proj", "model.layers.21.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00028614839538931847, "dbits": 109051904 } ] }, { "idx": 63, "layers": [ "model.layers.21.mlp.shared_experts.gate_proj", "model.layers.21.mlp.shared_experts.up_proj", "model.layers.21.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.00017015673220158178, "dbits": 17301504 } ] }, { "idx": 64, "layers": [ "model.layers.21.mlp.experts.0.gate_proj", "model.layers.21.mlp.experts.1.gate_proj", "model.layers.21.mlp.experts.2.gate_proj", "model.layers.21.mlp.experts.3.gate_proj", "model.layers.21.mlp.experts.4.gate_proj", "model.layers.21.mlp.experts.5.gate_proj", "model.layers.21.mlp.experts.6.gate_proj", "model.layers.21.mlp.experts.7.gate_proj", "model.layers.21.mlp.experts.8.gate_proj", "model.layers.21.mlp.experts.9.gate_proj", "model.layers.21.mlp.experts.10.gate_proj", "model.layers.21.mlp.experts.11.gate_proj", "model.layers.21.mlp.experts.12.gate_proj", "model.layers.21.mlp.experts.13.gate_proj", "model.layers.21.mlp.experts.14.gate_proj", "model.layers.21.mlp.experts.15.gate_proj", "model.layers.21.mlp.experts.16.gate_proj", "model.layers.21.mlp.experts.17.gate_proj", "model.layers.21.mlp.experts.18.gate_proj", "model.layers.21.mlp.experts.19.gate_proj", "model.layers.21.mlp.experts.20.gate_proj", "model.layers.21.mlp.experts.21.gate_proj", "model.layers.21.mlp.experts.22.gate_proj", "model.layers.21.mlp.experts.23.gate_proj", "model.layers.21.mlp.experts.24.gate_proj", "model.layers.21.mlp.experts.25.gate_proj", "model.layers.21.mlp.experts.26.gate_proj", "model.layers.21.mlp.experts.27.gate_proj", "model.layers.21.mlp.experts.28.gate_proj", "model.layers.21.mlp.experts.29.gate_proj", "model.layers.21.mlp.experts.30.gate_proj", "model.layers.21.mlp.experts.31.gate_proj", "model.layers.21.mlp.experts.32.gate_proj", "model.layers.21.mlp.experts.33.gate_proj", "model.layers.21.mlp.experts.34.gate_proj", "model.layers.21.mlp.experts.35.gate_proj", "model.layers.21.mlp.experts.36.gate_proj", "model.layers.21.mlp.experts.37.gate_proj", "model.layers.21.mlp.experts.38.gate_proj", "model.layers.21.mlp.experts.39.gate_proj", "model.layers.21.mlp.experts.40.gate_proj", "model.layers.21.mlp.experts.41.gate_proj", "model.layers.21.mlp.experts.42.gate_proj", "model.layers.21.mlp.experts.43.gate_proj", "model.layers.21.mlp.experts.44.gate_proj", "model.layers.21.mlp.experts.45.gate_proj", "model.layers.21.mlp.experts.46.gate_proj", "model.layers.21.mlp.experts.47.gate_proj", "model.layers.21.mlp.experts.48.gate_proj", "model.layers.21.mlp.experts.49.gate_proj", "model.layers.21.mlp.experts.50.gate_proj", "model.layers.21.mlp.experts.51.gate_proj", "model.layers.21.mlp.experts.52.gate_proj", "model.layers.21.mlp.experts.53.gate_proj", "model.layers.21.mlp.experts.54.gate_proj", "model.layers.21.mlp.experts.55.gate_proj", "model.layers.21.mlp.experts.56.gate_proj", "model.layers.21.mlp.experts.57.gate_proj", "model.layers.21.mlp.experts.58.gate_proj", "model.layers.21.mlp.experts.59.gate_proj", "model.layers.21.mlp.experts.60.gate_proj", "model.layers.21.mlp.experts.61.gate_proj", "model.layers.21.mlp.experts.62.gate_proj", "model.layers.21.mlp.experts.63.gate_proj", "model.layers.21.mlp.experts.64.gate_proj", "model.layers.21.mlp.experts.65.gate_proj", "model.layers.21.mlp.experts.66.gate_proj", "model.layers.21.mlp.experts.67.gate_proj", "model.layers.21.mlp.experts.68.gate_proj", "model.layers.21.mlp.experts.69.gate_proj", "model.layers.21.mlp.experts.70.gate_proj", "model.layers.21.mlp.experts.71.gate_proj", "model.layers.21.mlp.experts.72.gate_proj", "model.layers.21.mlp.experts.73.gate_proj", "model.layers.21.mlp.experts.74.gate_proj", "model.layers.21.mlp.experts.75.gate_proj", "model.layers.21.mlp.experts.76.gate_proj", "model.layers.21.mlp.experts.77.gate_proj", "model.layers.21.mlp.experts.78.gate_proj", "model.layers.21.mlp.experts.79.gate_proj", "model.layers.21.mlp.experts.80.gate_proj", "model.layers.21.mlp.experts.81.gate_proj", "model.layers.21.mlp.experts.82.gate_proj", "model.layers.21.mlp.experts.83.gate_proj", "model.layers.21.mlp.experts.84.gate_proj", "model.layers.21.mlp.experts.85.gate_proj", "model.layers.21.mlp.experts.86.gate_proj", "model.layers.21.mlp.experts.87.gate_proj", "model.layers.21.mlp.experts.88.gate_proj", "model.layers.21.mlp.experts.89.gate_proj", "model.layers.21.mlp.experts.90.gate_proj", "model.layers.21.mlp.experts.91.gate_proj", "model.layers.21.mlp.experts.92.gate_proj", "model.layers.21.mlp.experts.93.gate_proj", "model.layers.21.mlp.experts.94.gate_proj", "model.layers.21.mlp.experts.95.gate_proj", "model.layers.21.mlp.experts.96.gate_proj", "model.layers.21.mlp.experts.97.gate_proj", "model.layers.21.mlp.experts.98.gate_proj", "model.layers.21.mlp.experts.99.gate_proj", "model.layers.21.mlp.experts.100.gate_proj", "model.layers.21.mlp.experts.101.gate_proj", "model.layers.21.mlp.experts.102.gate_proj", "model.layers.21.mlp.experts.103.gate_proj", "model.layers.21.mlp.experts.104.gate_proj", "model.layers.21.mlp.experts.105.gate_proj", "model.layers.21.mlp.experts.106.gate_proj", "model.layers.21.mlp.experts.107.gate_proj", "model.layers.21.mlp.experts.108.gate_proj", "model.layers.21.mlp.experts.109.gate_proj", "model.layers.21.mlp.experts.110.gate_proj", "model.layers.21.mlp.experts.111.gate_proj", "model.layers.21.mlp.experts.112.gate_proj", "model.layers.21.mlp.experts.113.gate_proj", "model.layers.21.mlp.experts.114.gate_proj", "model.layers.21.mlp.experts.115.gate_proj", "model.layers.21.mlp.experts.116.gate_proj", "model.layers.21.mlp.experts.117.gate_proj", "model.layers.21.mlp.experts.118.gate_proj", "model.layers.21.mlp.experts.119.gate_proj", "model.layers.21.mlp.experts.120.gate_proj", "model.layers.21.mlp.experts.121.gate_proj", "model.layers.21.mlp.experts.122.gate_proj", "model.layers.21.mlp.experts.123.gate_proj", "model.layers.21.mlp.experts.124.gate_proj", "model.layers.21.mlp.experts.125.gate_proj", "model.layers.21.mlp.experts.126.gate_proj", "model.layers.21.mlp.experts.127.gate_proj", "model.layers.21.mlp.experts.0.up_proj", "model.layers.21.mlp.experts.1.up_proj", "model.layers.21.mlp.experts.2.up_proj", "model.layers.21.mlp.experts.3.up_proj", "model.layers.21.mlp.experts.4.up_proj", "model.layers.21.mlp.experts.5.up_proj", "model.layers.21.mlp.experts.6.up_proj", "model.layers.21.mlp.experts.7.up_proj", "model.layers.21.mlp.experts.8.up_proj", "model.layers.21.mlp.experts.9.up_proj", "model.layers.21.mlp.experts.10.up_proj", "model.layers.21.mlp.experts.11.up_proj", "model.layers.21.mlp.experts.12.up_proj", "model.layers.21.mlp.experts.13.up_proj", "model.layers.21.mlp.experts.14.up_proj", "model.layers.21.mlp.experts.15.up_proj", "model.layers.21.mlp.experts.16.up_proj", "model.layers.21.mlp.experts.17.up_proj", "model.layers.21.mlp.experts.18.up_proj", "model.layers.21.mlp.experts.19.up_proj", "model.layers.21.mlp.experts.20.up_proj", "model.layers.21.mlp.experts.21.up_proj", "model.layers.21.mlp.experts.22.up_proj", "model.layers.21.mlp.experts.23.up_proj", "model.layers.21.mlp.experts.24.up_proj", "model.layers.21.mlp.experts.25.up_proj", "model.layers.21.mlp.experts.26.up_proj", "model.layers.21.mlp.experts.27.up_proj", "model.layers.21.mlp.experts.28.up_proj", "model.layers.21.mlp.experts.29.up_proj", "model.layers.21.mlp.experts.30.up_proj", "model.layers.21.mlp.experts.31.up_proj", "model.layers.21.mlp.experts.32.up_proj", "model.layers.21.mlp.experts.33.up_proj", "model.layers.21.mlp.experts.34.up_proj", "model.layers.21.mlp.experts.35.up_proj", "model.layers.21.mlp.experts.36.up_proj", "model.layers.21.mlp.experts.37.up_proj", "model.layers.21.mlp.experts.38.up_proj", "model.layers.21.mlp.experts.39.up_proj", "model.layers.21.mlp.experts.40.up_proj", "model.layers.21.mlp.experts.41.up_proj", "model.layers.21.mlp.experts.42.up_proj", "model.layers.21.mlp.experts.43.up_proj", "model.layers.21.mlp.experts.44.up_proj", "model.layers.21.mlp.experts.45.up_proj", "model.layers.21.mlp.experts.46.up_proj", "model.layers.21.mlp.experts.47.up_proj", "model.layers.21.mlp.experts.48.up_proj", "model.layers.21.mlp.experts.49.up_proj", "model.layers.21.mlp.experts.50.up_proj", "model.layers.21.mlp.experts.51.up_proj", "model.layers.21.mlp.experts.52.up_proj", "model.layers.21.mlp.experts.53.up_proj", "model.layers.21.mlp.experts.54.up_proj", "model.layers.21.mlp.experts.55.up_proj", "model.layers.21.mlp.experts.56.up_proj", "model.layers.21.mlp.experts.57.up_proj", "model.layers.21.mlp.experts.58.up_proj", "model.layers.21.mlp.experts.59.up_proj", "model.layers.21.mlp.experts.60.up_proj", "model.layers.21.mlp.experts.61.up_proj", "model.layers.21.mlp.experts.62.up_proj", "model.layers.21.mlp.experts.63.up_proj", "model.layers.21.mlp.experts.64.up_proj", "model.layers.21.mlp.experts.65.up_proj", "model.layers.21.mlp.experts.66.up_proj", "model.layers.21.mlp.experts.67.up_proj", "model.layers.21.mlp.experts.68.up_proj", "model.layers.21.mlp.experts.69.up_proj", "model.layers.21.mlp.experts.70.up_proj", "model.layers.21.mlp.experts.71.up_proj", "model.layers.21.mlp.experts.72.up_proj", "model.layers.21.mlp.experts.73.up_proj", "model.layers.21.mlp.experts.74.up_proj", "model.layers.21.mlp.experts.75.up_proj", "model.layers.21.mlp.experts.76.up_proj", "model.layers.21.mlp.experts.77.up_proj", "model.layers.21.mlp.experts.78.up_proj", "model.layers.21.mlp.experts.79.up_proj", "model.layers.21.mlp.experts.80.up_proj", "model.layers.21.mlp.experts.81.up_proj", "model.layers.21.mlp.experts.82.up_proj", "model.layers.21.mlp.experts.83.up_proj", "model.layers.21.mlp.experts.84.up_proj", "model.layers.21.mlp.experts.85.up_proj", "model.layers.21.mlp.experts.86.up_proj", "model.layers.21.mlp.experts.87.up_proj", "model.layers.21.mlp.experts.88.up_proj", "model.layers.21.mlp.experts.89.up_proj", "model.layers.21.mlp.experts.90.up_proj", "model.layers.21.mlp.experts.91.up_proj", "model.layers.21.mlp.experts.92.up_proj", "model.layers.21.mlp.experts.93.up_proj", "model.layers.21.mlp.experts.94.up_proj", "model.layers.21.mlp.experts.95.up_proj", "model.layers.21.mlp.experts.96.up_proj", "model.layers.21.mlp.experts.97.up_proj", "model.layers.21.mlp.experts.98.up_proj", "model.layers.21.mlp.experts.99.up_proj", "model.layers.21.mlp.experts.100.up_proj", "model.layers.21.mlp.experts.101.up_proj", "model.layers.21.mlp.experts.102.up_proj", "model.layers.21.mlp.experts.103.up_proj", "model.layers.21.mlp.experts.104.up_proj", "model.layers.21.mlp.experts.105.up_proj", "model.layers.21.mlp.experts.106.up_proj", "model.layers.21.mlp.experts.107.up_proj", "model.layers.21.mlp.experts.108.up_proj", "model.layers.21.mlp.experts.109.up_proj", "model.layers.21.mlp.experts.110.up_proj", "model.layers.21.mlp.experts.111.up_proj", "model.layers.21.mlp.experts.112.up_proj", "model.layers.21.mlp.experts.113.up_proj", "model.layers.21.mlp.experts.114.up_proj", "model.layers.21.mlp.experts.115.up_proj", "model.layers.21.mlp.experts.116.up_proj", "model.layers.21.mlp.experts.117.up_proj", "model.layers.21.mlp.experts.118.up_proj", "model.layers.21.mlp.experts.119.up_proj", "model.layers.21.mlp.experts.120.up_proj", "model.layers.21.mlp.experts.121.up_proj", "model.layers.21.mlp.experts.122.up_proj", "model.layers.21.mlp.experts.123.up_proj", "model.layers.21.mlp.experts.124.up_proj", "model.layers.21.mlp.experts.125.up_proj", "model.layers.21.mlp.experts.126.up_proj", "model.layers.21.mlp.experts.127.up_proj", "model.layers.21.mlp.experts.0.down_proj", "model.layers.21.mlp.experts.1.down_proj", "model.layers.21.mlp.experts.2.down_proj", "model.layers.21.mlp.experts.3.down_proj", "model.layers.21.mlp.experts.4.down_proj", "model.layers.21.mlp.experts.5.down_proj", "model.layers.21.mlp.experts.6.down_proj", "model.layers.21.mlp.experts.7.down_proj", "model.layers.21.mlp.experts.8.down_proj", "model.layers.21.mlp.experts.9.down_proj", "model.layers.21.mlp.experts.10.down_proj", "model.layers.21.mlp.experts.11.down_proj", "model.layers.21.mlp.experts.12.down_proj", "model.layers.21.mlp.experts.13.down_proj", "model.layers.21.mlp.experts.14.down_proj", "model.layers.21.mlp.experts.15.down_proj", "model.layers.21.mlp.experts.16.down_proj", "model.layers.21.mlp.experts.17.down_proj", "model.layers.21.mlp.experts.18.down_proj", "model.layers.21.mlp.experts.19.down_proj", "model.layers.21.mlp.experts.20.down_proj", "model.layers.21.mlp.experts.21.down_proj", "model.layers.21.mlp.experts.22.down_proj", "model.layers.21.mlp.experts.23.down_proj", "model.layers.21.mlp.experts.24.down_proj", "model.layers.21.mlp.experts.25.down_proj", "model.layers.21.mlp.experts.26.down_proj", "model.layers.21.mlp.experts.27.down_proj", "model.layers.21.mlp.experts.28.down_proj", "model.layers.21.mlp.experts.29.down_proj", "model.layers.21.mlp.experts.30.down_proj", "model.layers.21.mlp.experts.31.down_proj", "model.layers.21.mlp.experts.32.down_proj", "model.layers.21.mlp.experts.33.down_proj", "model.layers.21.mlp.experts.34.down_proj", "model.layers.21.mlp.experts.35.down_proj", "model.layers.21.mlp.experts.36.down_proj", "model.layers.21.mlp.experts.37.down_proj", "model.layers.21.mlp.experts.38.down_proj", "model.layers.21.mlp.experts.39.down_proj", "model.layers.21.mlp.experts.40.down_proj", "model.layers.21.mlp.experts.41.down_proj", "model.layers.21.mlp.experts.42.down_proj", "model.layers.21.mlp.experts.43.down_proj", "model.layers.21.mlp.experts.44.down_proj", "model.layers.21.mlp.experts.45.down_proj", "model.layers.21.mlp.experts.46.down_proj", "model.layers.21.mlp.experts.47.down_proj", "model.layers.21.mlp.experts.48.down_proj", "model.layers.21.mlp.experts.49.down_proj", "model.layers.21.mlp.experts.50.down_proj", "model.layers.21.mlp.experts.51.down_proj", "model.layers.21.mlp.experts.52.down_proj", "model.layers.21.mlp.experts.53.down_proj", "model.layers.21.mlp.experts.54.down_proj", "model.layers.21.mlp.experts.55.down_proj", "model.layers.21.mlp.experts.56.down_proj", "model.layers.21.mlp.experts.57.down_proj", "model.layers.21.mlp.experts.58.down_proj", "model.layers.21.mlp.experts.59.down_proj", "model.layers.21.mlp.experts.60.down_proj", "model.layers.21.mlp.experts.61.down_proj", "model.layers.21.mlp.experts.62.down_proj", "model.layers.21.mlp.experts.63.down_proj", "model.layers.21.mlp.experts.64.down_proj", "model.layers.21.mlp.experts.65.down_proj", "model.layers.21.mlp.experts.66.down_proj", "model.layers.21.mlp.experts.67.down_proj", "model.layers.21.mlp.experts.68.down_proj", "model.layers.21.mlp.experts.69.down_proj", "model.layers.21.mlp.experts.70.down_proj", "model.layers.21.mlp.experts.71.down_proj", "model.layers.21.mlp.experts.72.down_proj", "model.layers.21.mlp.experts.73.down_proj", "model.layers.21.mlp.experts.74.down_proj", "model.layers.21.mlp.experts.75.down_proj", "model.layers.21.mlp.experts.76.down_proj", "model.layers.21.mlp.experts.77.down_proj", "model.layers.21.mlp.experts.78.down_proj", "model.layers.21.mlp.experts.79.down_proj", "model.layers.21.mlp.experts.80.down_proj", "model.layers.21.mlp.experts.81.down_proj", "model.layers.21.mlp.experts.82.down_proj", "model.layers.21.mlp.experts.83.down_proj", "model.layers.21.mlp.experts.84.down_proj", "model.layers.21.mlp.experts.85.down_proj", "model.layers.21.mlp.experts.86.down_proj", "model.layers.21.mlp.experts.87.down_proj", "model.layers.21.mlp.experts.88.down_proj", "model.layers.21.mlp.experts.89.down_proj", "model.layers.21.mlp.experts.90.down_proj", "model.layers.21.mlp.experts.91.down_proj", "model.layers.21.mlp.experts.92.down_proj", "model.layers.21.mlp.experts.93.down_proj", "model.layers.21.mlp.experts.94.down_proj", "model.layers.21.mlp.experts.95.down_proj", "model.layers.21.mlp.experts.96.down_proj", "model.layers.21.mlp.experts.97.down_proj", "model.layers.21.mlp.experts.98.down_proj", "model.layers.21.mlp.experts.99.down_proj", "model.layers.21.mlp.experts.100.down_proj", "model.layers.21.mlp.experts.101.down_proj", "model.layers.21.mlp.experts.102.down_proj", "model.layers.21.mlp.experts.103.down_proj", "model.layers.21.mlp.experts.104.down_proj", "model.layers.21.mlp.experts.105.down_proj", "model.layers.21.mlp.experts.106.down_proj", "model.layers.21.mlp.experts.107.down_proj", "model.layers.21.mlp.experts.108.down_proj", "model.layers.21.mlp.experts.109.down_proj", "model.layers.21.mlp.experts.110.down_proj", "model.layers.21.mlp.experts.111.down_proj", "model.layers.21.mlp.experts.112.down_proj", "model.layers.21.mlp.experts.113.down_proj", "model.layers.21.mlp.experts.114.down_proj", "model.layers.21.mlp.experts.115.down_proj", "model.layers.21.mlp.experts.116.down_proj", "model.layers.21.mlp.experts.117.down_proj", "model.layers.21.mlp.experts.118.down_proj", "model.layers.21.mlp.experts.119.down_proj", "model.layers.21.mlp.experts.120.down_proj", "model.layers.21.mlp.experts.121.down_proj", "model.layers.21.mlp.experts.122.down_proj", "model.layers.21.mlp.experts.123.down_proj", "model.layers.21.mlp.experts.124.down_proj", "model.layers.21.mlp.experts.125.down_proj", "model.layers.21.mlp.experts.126.down_proj", "model.layers.21.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0031477053649723585, "dbits": 2214592512 } ] }, { "idx": 65, "layers": [ "model.layers.22.self_attn.q_proj", "model.layers.22.self_attn.k_proj", "model.layers.22.self_attn.v_proj", "model.layers.22.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0008217070251703151, "dbits": 109051904 } ] }, { "idx": 66, "layers": [ "model.layers.22.mlp.shared_experts.gate_proj", "model.layers.22.mlp.shared_experts.up_proj", "model.layers.22.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00024694241583347876, "dbits": 17301504 } ] }, { "idx": 67, "layers": [ "model.layers.22.mlp.experts.0.gate_proj", "model.layers.22.mlp.experts.1.gate_proj", "model.layers.22.mlp.experts.2.gate_proj", "model.layers.22.mlp.experts.3.gate_proj", "model.layers.22.mlp.experts.4.gate_proj", "model.layers.22.mlp.experts.5.gate_proj", "model.layers.22.mlp.experts.6.gate_proj", "model.layers.22.mlp.experts.7.gate_proj", "model.layers.22.mlp.experts.8.gate_proj", "model.layers.22.mlp.experts.9.gate_proj", "model.layers.22.mlp.experts.10.gate_proj", "model.layers.22.mlp.experts.11.gate_proj", "model.layers.22.mlp.experts.12.gate_proj", "model.layers.22.mlp.experts.13.gate_proj", "model.layers.22.mlp.experts.14.gate_proj", "model.layers.22.mlp.experts.15.gate_proj", "model.layers.22.mlp.experts.16.gate_proj", "model.layers.22.mlp.experts.17.gate_proj", "model.layers.22.mlp.experts.18.gate_proj", "model.layers.22.mlp.experts.19.gate_proj", "model.layers.22.mlp.experts.20.gate_proj", "model.layers.22.mlp.experts.21.gate_proj", "model.layers.22.mlp.experts.22.gate_proj", "model.layers.22.mlp.experts.23.gate_proj", "model.layers.22.mlp.experts.24.gate_proj", "model.layers.22.mlp.experts.25.gate_proj", "model.layers.22.mlp.experts.26.gate_proj", "model.layers.22.mlp.experts.27.gate_proj", "model.layers.22.mlp.experts.28.gate_proj", "model.layers.22.mlp.experts.29.gate_proj", "model.layers.22.mlp.experts.30.gate_proj", "model.layers.22.mlp.experts.31.gate_proj", "model.layers.22.mlp.experts.32.gate_proj", "model.layers.22.mlp.experts.33.gate_proj", "model.layers.22.mlp.experts.34.gate_proj", "model.layers.22.mlp.experts.35.gate_proj", "model.layers.22.mlp.experts.36.gate_proj", "model.layers.22.mlp.experts.37.gate_proj", "model.layers.22.mlp.experts.38.gate_proj", "model.layers.22.mlp.experts.39.gate_proj", "model.layers.22.mlp.experts.40.gate_proj", "model.layers.22.mlp.experts.41.gate_proj", "model.layers.22.mlp.experts.42.gate_proj", "model.layers.22.mlp.experts.43.gate_proj", "model.layers.22.mlp.experts.44.gate_proj", "model.layers.22.mlp.experts.45.gate_proj", "model.layers.22.mlp.experts.46.gate_proj", "model.layers.22.mlp.experts.47.gate_proj", "model.layers.22.mlp.experts.48.gate_proj", "model.layers.22.mlp.experts.49.gate_proj", "model.layers.22.mlp.experts.50.gate_proj", "model.layers.22.mlp.experts.51.gate_proj", "model.layers.22.mlp.experts.52.gate_proj", "model.layers.22.mlp.experts.53.gate_proj", "model.layers.22.mlp.experts.54.gate_proj", "model.layers.22.mlp.experts.55.gate_proj", "model.layers.22.mlp.experts.56.gate_proj", "model.layers.22.mlp.experts.57.gate_proj", "model.layers.22.mlp.experts.58.gate_proj", "model.layers.22.mlp.experts.59.gate_proj", "model.layers.22.mlp.experts.60.gate_proj", "model.layers.22.mlp.experts.61.gate_proj", "model.layers.22.mlp.experts.62.gate_proj", "model.layers.22.mlp.experts.63.gate_proj", "model.layers.22.mlp.experts.64.gate_proj", "model.layers.22.mlp.experts.65.gate_proj", "model.layers.22.mlp.experts.66.gate_proj", "model.layers.22.mlp.experts.67.gate_proj", "model.layers.22.mlp.experts.68.gate_proj", "model.layers.22.mlp.experts.69.gate_proj", "model.layers.22.mlp.experts.70.gate_proj", "model.layers.22.mlp.experts.71.gate_proj", "model.layers.22.mlp.experts.72.gate_proj", "model.layers.22.mlp.experts.73.gate_proj", "model.layers.22.mlp.experts.74.gate_proj", "model.layers.22.mlp.experts.75.gate_proj", "model.layers.22.mlp.experts.76.gate_proj", "model.layers.22.mlp.experts.77.gate_proj", "model.layers.22.mlp.experts.78.gate_proj", "model.layers.22.mlp.experts.79.gate_proj", "model.layers.22.mlp.experts.80.gate_proj", "model.layers.22.mlp.experts.81.gate_proj", "model.layers.22.mlp.experts.82.gate_proj", "model.layers.22.mlp.experts.83.gate_proj", "model.layers.22.mlp.experts.84.gate_proj", "model.layers.22.mlp.experts.85.gate_proj", "model.layers.22.mlp.experts.86.gate_proj", "model.layers.22.mlp.experts.87.gate_proj", "model.layers.22.mlp.experts.88.gate_proj", "model.layers.22.mlp.experts.89.gate_proj", "model.layers.22.mlp.experts.90.gate_proj", "model.layers.22.mlp.experts.91.gate_proj", "model.layers.22.mlp.experts.92.gate_proj", "model.layers.22.mlp.experts.93.gate_proj", "model.layers.22.mlp.experts.94.gate_proj", "model.layers.22.mlp.experts.95.gate_proj", "model.layers.22.mlp.experts.96.gate_proj", "model.layers.22.mlp.experts.97.gate_proj", "model.layers.22.mlp.experts.98.gate_proj", "model.layers.22.mlp.experts.99.gate_proj", "model.layers.22.mlp.experts.100.gate_proj", "model.layers.22.mlp.experts.101.gate_proj", "model.layers.22.mlp.experts.102.gate_proj", "model.layers.22.mlp.experts.103.gate_proj", "model.layers.22.mlp.experts.104.gate_proj", "model.layers.22.mlp.experts.105.gate_proj", "model.layers.22.mlp.experts.106.gate_proj", "model.layers.22.mlp.experts.107.gate_proj", "model.layers.22.mlp.experts.108.gate_proj", "model.layers.22.mlp.experts.109.gate_proj", "model.layers.22.mlp.experts.110.gate_proj", "model.layers.22.mlp.experts.111.gate_proj", "model.layers.22.mlp.experts.112.gate_proj", "model.layers.22.mlp.experts.113.gate_proj", "model.layers.22.mlp.experts.114.gate_proj", "model.layers.22.mlp.experts.115.gate_proj", "model.layers.22.mlp.experts.116.gate_proj", "model.layers.22.mlp.experts.117.gate_proj", "model.layers.22.mlp.experts.118.gate_proj", "model.layers.22.mlp.experts.119.gate_proj", "model.layers.22.mlp.experts.120.gate_proj", "model.layers.22.mlp.experts.121.gate_proj", "model.layers.22.mlp.experts.122.gate_proj", "model.layers.22.mlp.experts.123.gate_proj", "model.layers.22.mlp.experts.124.gate_proj", "model.layers.22.mlp.experts.125.gate_proj", "model.layers.22.mlp.experts.126.gate_proj", "model.layers.22.mlp.experts.127.gate_proj", "model.layers.22.mlp.experts.0.up_proj", "model.layers.22.mlp.experts.1.up_proj", "model.layers.22.mlp.experts.2.up_proj", "model.layers.22.mlp.experts.3.up_proj", "model.layers.22.mlp.experts.4.up_proj", "model.layers.22.mlp.experts.5.up_proj", "model.layers.22.mlp.experts.6.up_proj", "model.layers.22.mlp.experts.7.up_proj", "model.layers.22.mlp.experts.8.up_proj", "model.layers.22.mlp.experts.9.up_proj", "model.layers.22.mlp.experts.10.up_proj", "model.layers.22.mlp.experts.11.up_proj", "model.layers.22.mlp.experts.12.up_proj", "model.layers.22.mlp.experts.13.up_proj", "model.layers.22.mlp.experts.14.up_proj", "model.layers.22.mlp.experts.15.up_proj", "model.layers.22.mlp.experts.16.up_proj", "model.layers.22.mlp.experts.17.up_proj", "model.layers.22.mlp.experts.18.up_proj", "model.layers.22.mlp.experts.19.up_proj", "model.layers.22.mlp.experts.20.up_proj", "model.layers.22.mlp.experts.21.up_proj", "model.layers.22.mlp.experts.22.up_proj", "model.layers.22.mlp.experts.23.up_proj", "model.layers.22.mlp.experts.24.up_proj", "model.layers.22.mlp.experts.25.up_proj", "model.layers.22.mlp.experts.26.up_proj", "model.layers.22.mlp.experts.27.up_proj", "model.layers.22.mlp.experts.28.up_proj", "model.layers.22.mlp.experts.29.up_proj", "model.layers.22.mlp.experts.30.up_proj", "model.layers.22.mlp.experts.31.up_proj", "model.layers.22.mlp.experts.32.up_proj", "model.layers.22.mlp.experts.33.up_proj", "model.layers.22.mlp.experts.34.up_proj", "model.layers.22.mlp.experts.35.up_proj", "model.layers.22.mlp.experts.36.up_proj", "model.layers.22.mlp.experts.37.up_proj", "model.layers.22.mlp.experts.38.up_proj", "model.layers.22.mlp.experts.39.up_proj", "model.layers.22.mlp.experts.40.up_proj", "model.layers.22.mlp.experts.41.up_proj", "model.layers.22.mlp.experts.42.up_proj", "model.layers.22.mlp.experts.43.up_proj", "model.layers.22.mlp.experts.44.up_proj", "model.layers.22.mlp.experts.45.up_proj", "model.layers.22.mlp.experts.46.up_proj", "model.layers.22.mlp.experts.47.up_proj", "model.layers.22.mlp.experts.48.up_proj", "model.layers.22.mlp.experts.49.up_proj", "model.layers.22.mlp.experts.50.up_proj", "model.layers.22.mlp.experts.51.up_proj", "model.layers.22.mlp.experts.52.up_proj", "model.layers.22.mlp.experts.53.up_proj", "model.layers.22.mlp.experts.54.up_proj", "model.layers.22.mlp.experts.55.up_proj", "model.layers.22.mlp.experts.56.up_proj", "model.layers.22.mlp.experts.57.up_proj", "model.layers.22.mlp.experts.58.up_proj", "model.layers.22.mlp.experts.59.up_proj", "model.layers.22.mlp.experts.60.up_proj", "model.layers.22.mlp.experts.61.up_proj", "model.layers.22.mlp.experts.62.up_proj", "model.layers.22.mlp.experts.63.up_proj", "model.layers.22.mlp.experts.64.up_proj", "model.layers.22.mlp.experts.65.up_proj", "model.layers.22.mlp.experts.66.up_proj", "model.layers.22.mlp.experts.67.up_proj", "model.layers.22.mlp.experts.68.up_proj", "model.layers.22.mlp.experts.69.up_proj", "model.layers.22.mlp.experts.70.up_proj", "model.layers.22.mlp.experts.71.up_proj", "model.layers.22.mlp.experts.72.up_proj", "model.layers.22.mlp.experts.73.up_proj", "model.layers.22.mlp.experts.74.up_proj", "model.layers.22.mlp.experts.75.up_proj", "model.layers.22.mlp.experts.76.up_proj", "model.layers.22.mlp.experts.77.up_proj", "model.layers.22.mlp.experts.78.up_proj", "model.layers.22.mlp.experts.79.up_proj", "model.layers.22.mlp.experts.80.up_proj", "model.layers.22.mlp.experts.81.up_proj", "model.layers.22.mlp.experts.82.up_proj", "model.layers.22.mlp.experts.83.up_proj", "model.layers.22.mlp.experts.84.up_proj", "model.layers.22.mlp.experts.85.up_proj", "model.layers.22.mlp.experts.86.up_proj", "model.layers.22.mlp.experts.87.up_proj", "model.layers.22.mlp.experts.88.up_proj", "model.layers.22.mlp.experts.89.up_proj", "model.layers.22.mlp.experts.90.up_proj", "model.layers.22.mlp.experts.91.up_proj", "model.layers.22.mlp.experts.92.up_proj", "model.layers.22.mlp.experts.93.up_proj", "model.layers.22.mlp.experts.94.up_proj", "model.layers.22.mlp.experts.95.up_proj", "model.layers.22.mlp.experts.96.up_proj", "model.layers.22.mlp.experts.97.up_proj", "model.layers.22.mlp.experts.98.up_proj", "model.layers.22.mlp.experts.99.up_proj", "model.layers.22.mlp.experts.100.up_proj", "model.layers.22.mlp.experts.101.up_proj", "model.layers.22.mlp.experts.102.up_proj", "model.layers.22.mlp.experts.103.up_proj", "model.layers.22.mlp.experts.104.up_proj", "model.layers.22.mlp.experts.105.up_proj", "model.layers.22.mlp.experts.106.up_proj", "model.layers.22.mlp.experts.107.up_proj", "model.layers.22.mlp.experts.108.up_proj", "model.layers.22.mlp.experts.109.up_proj", "model.layers.22.mlp.experts.110.up_proj", "model.layers.22.mlp.experts.111.up_proj", "model.layers.22.mlp.experts.112.up_proj", "model.layers.22.mlp.experts.113.up_proj", "model.layers.22.mlp.experts.114.up_proj", "model.layers.22.mlp.experts.115.up_proj", "model.layers.22.mlp.experts.116.up_proj", "model.layers.22.mlp.experts.117.up_proj", "model.layers.22.mlp.experts.118.up_proj", "model.layers.22.mlp.experts.119.up_proj", "model.layers.22.mlp.experts.120.up_proj", "model.layers.22.mlp.experts.121.up_proj", "model.layers.22.mlp.experts.122.up_proj", "model.layers.22.mlp.experts.123.up_proj", "model.layers.22.mlp.experts.124.up_proj", "model.layers.22.mlp.experts.125.up_proj", "model.layers.22.mlp.experts.126.up_proj", "model.layers.22.mlp.experts.127.up_proj", "model.layers.22.mlp.experts.0.down_proj", "model.layers.22.mlp.experts.1.down_proj", "model.layers.22.mlp.experts.2.down_proj", "model.layers.22.mlp.experts.3.down_proj", "model.layers.22.mlp.experts.4.down_proj", "model.layers.22.mlp.experts.5.down_proj", "model.layers.22.mlp.experts.6.down_proj", "model.layers.22.mlp.experts.7.down_proj", "model.layers.22.mlp.experts.8.down_proj", "model.layers.22.mlp.experts.9.down_proj", "model.layers.22.mlp.experts.10.down_proj", "model.layers.22.mlp.experts.11.down_proj", "model.layers.22.mlp.experts.12.down_proj", "model.layers.22.mlp.experts.13.down_proj", "model.layers.22.mlp.experts.14.down_proj", "model.layers.22.mlp.experts.15.down_proj", "model.layers.22.mlp.experts.16.down_proj", "model.layers.22.mlp.experts.17.down_proj", "model.layers.22.mlp.experts.18.down_proj", "model.layers.22.mlp.experts.19.down_proj", "model.layers.22.mlp.experts.20.down_proj", "model.layers.22.mlp.experts.21.down_proj", "model.layers.22.mlp.experts.22.down_proj", "model.layers.22.mlp.experts.23.down_proj", "model.layers.22.mlp.experts.24.down_proj", "model.layers.22.mlp.experts.25.down_proj", "model.layers.22.mlp.experts.26.down_proj", "model.layers.22.mlp.experts.27.down_proj", "model.layers.22.mlp.experts.28.down_proj", "model.layers.22.mlp.experts.29.down_proj", "model.layers.22.mlp.experts.30.down_proj", "model.layers.22.mlp.experts.31.down_proj", "model.layers.22.mlp.experts.32.down_proj", "model.layers.22.mlp.experts.33.down_proj", "model.layers.22.mlp.experts.34.down_proj", "model.layers.22.mlp.experts.35.down_proj", "model.layers.22.mlp.experts.36.down_proj", "model.layers.22.mlp.experts.37.down_proj", "model.layers.22.mlp.experts.38.down_proj", "model.layers.22.mlp.experts.39.down_proj", "model.layers.22.mlp.experts.40.down_proj", "model.layers.22.mlp.experts.41.down_proj", "model.layers.22.mlp.experts.42.down_proj", "model.layers.22.mlp.experts.43.down_proj", "model.layers.22.mlp.experts.44.down_proj", "model.layers.22.mlp.experts.45.down_proj", "model.layers.22.mlp.experts.46.down_proj", "model.layers.22.mlp.experts.47.down_proj", "model.layers.22.mlp.experts.48.down_proj", "model.layers.22.mlp.experts.49.down_proj", "model.layers.22.mlp.experts.50.down_proj", "model.layers.22.mlp.experts.51.down_proj", "model.layers.22.mlp.experts.52.down_proj", "model.layers.22.mlp.experts.53.down_proj", "model.layers.22.mlp.experts.54.down_proj", "model.layers.22.mlp.experts.55.down_proj", "model.layers.22.mlp.experts.56.down_proj", "model.layers.22.mlp.experts.57.down_proj", "model.layers.22.mlp.experts.58.down_proj", "model.layers.22.mlp.experts.59.down_proj", "model.layers.22.mlp.experts.60.down_proj", "model.layers.22.mlp.experts.61.down_proj", "model.layers.22.mlp.experts.62.down_proj", "model.layers.22.mlp.experts.63.down_proj", "model.layers.22.mlp.experts.64.down_proj", "model.layers.22.mlp.experts.65.down_proj", "model.layers.22.mlp.experts.66.down_proj", "model.layers.22.mlp.experts.67.down_proj", "model.layers.22.mlp.experts.68.down_proj", "model.layers.22.mlp.experts.69.down_proj", "model.layers.22.mlp.experts.70.down_proj", "model.layers.22.mlp.experts.71.down_proj", "model.layers.22.mlp.experts.72.down_proj", "model.layers.22.mlp.experts.73.down_proj", "model.layers.22.mlp.experts.74.down_proj", "model.layers.22.mlp.experts.75.down_proj", "model.layers.22.mlp.experts.76.down_proj", "model.layers.22.mlp.experts.77.down_proj", "model.layers.22.mlp.experts.78.down_proj", "model.layers.22.mlp.experts.79.down_proj", "model.layers.22.mlp.experts.80.down_proj", "model.layers.22.mlp.experts.81.down_proj", "model.layers.22.mlp.experts.82.down_proj", "model.layers.22.mlp.experts.83.down_proj", "model.layers.22.mlp.experts.84.down_proj", "model.layers.22.mlp.experts.85.down_proj", "model.layers.22.mlp.experts.86.down_proj", "model.layers.22.mlp.experts.87.down_proj", "model.layers.22.mlp.experts.88.down_proj", "model.layers.22.mlp.experts.89.down_proj", "model.layers.22.mlp.experts.90.down_proj", "model.layers.22.mlp.experts.91.down_proj", "model.layers.22.mlp.experts.92.down_proj", "model.layers.22.mlp.experts.93.down_proj", "model.layers.22.mlp.experts.94.down_proj", "model.layers.22.mlp.experts.95.down_proj", "model.layers.22.mlp.experts.96.down_proj", "model.layers.22.mlp.experts.97.down_proj", "model.layers.22.mlp.experts.98.down_proj", "model.layers.22.mlp.experts.99.down_proj", "model.layers.22.mlp.experts.100.down_proj", "model.layers.22.mlp.experts.101.down_proj", "model.layers.22.mlp.experts.102.down_proj", "model.layers.22.mlp.experts.103.down_proj", "model.layers.22.mlp.experts.104.down_proj", "model.layers.22.mlp.experts.105.down_proj", "model.layers.22.mlp.experts.106.down_proj", "model.layers.22.mlp.experts.107.down_proj", "model.layers.22.mlp.experts.108.down_proj", "model.layers.22.mlp.experts.109.down_proj", "model.layers.22.mlp.experts.110.down_proj", "model.layers.22.mlp.experts.111.down_proj", "model.layers.22.mlp.experts.112.down_proj", "model.layers.22.mlp.experts.113.down_proj", "model.layers.22.mlp.experts.114.down_proj", "model.layers.22.mlp.experts.115.down_proj", "model.layers.22.mlp.experts.116.down_proj", "model.layers.22.mlp.experts.117.down_proj", "model.layers.22.mlp.experts.118.down_proj", "model.layers.22.mlp.experts.119.down_proj", "model.layers.22.mlp.experts.120.down_proj", "model.layers.22.mlp.experts.121.down_proj", "model.layers.22.mlp.experts.122.down_proj", "model.layers.22.mlp.experts.123.down_proj", "model.layers.22.mlp.experts.124.down_proj", "model.layers.22.mlp.experts.125.down_proj", "model.layers.22.mlp.experts.126.down_proj", "model.layers.22.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0024681364186107946, "dbits": 2214592512 } ] }, { "idx": 68, "layers": [ "model.layers.23.self_attn.q_proj", "model.layers.23.self_attn.k_proj", "model.layers.23.self_attn.v_proj", "model.layers.23.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0011340341530740317, "dbits": 109051904 } ] }, { "idx": 69, "layers": [ "model.layers.23.mlp.shared_experts.gate_proj", "model.layers.23.mlp.shared_experts.up_proj", "model.layers.23.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0014980866573751084, "dbits": 17301504 } ] }, { "idx": 70, "layers": [ "model.layers.23.mlp.experts.0.gate_proj", "model.layers.23.mlp.experts.1.gate_proj", "model.layers.23.mlp.experts.2.gate_proj", "model.layers.23.mlp.experts.3.gate_proj", "model.layers.23.mlp.experts.4.gate_proj", "model.layers.23.mlp.experts.5.gate_proj", "model.layers.23.mlp.experts.6.gate_proj", "model.layers.23.mlp.experts.7.gate_proj", "model.layers.23.mlp.experts.8.gate_proj", "model.layers.23.mlp.experts.9.gate_proj", "model.layers.23.mlp.experts.10.gate_proj", "model.layers.23.mlp.experts.11.gate_proj", "model.layers.23.mlp.experts.12.gate_proj", "model.layers.23.mlp.experts.13.gate_proj", "model.layers.23.mlp.experts.14.gate_proj", "model.layers.23.mlp.experts.15.gate_proj", "model.layers.23.mlp.experts.16.gate_proj", "model.layers.23.mlp.experts.17.gate_proj", "model.layers.23.mlp.experts.18.gate_proj", "model.layers.23.mlp.experts.19.gate_proj", "model.layers.23.mlp.experts.20.gate_proj", "model.layers.23.mlp.experts.21.gate_proj", "model.layers.23.mlp.experts.22.gate_proj", "model.layers.23.mlp.experts.23.gate_proj", "model.layers.23.mlp.experts.24.gate_proj", "model.layers.23.mlp.experts.25.gate_proj", "model.layers.23.mlp.experts.26.gate_proj", "model.layers.23.mlp.experts.27.gate_proj", "model.layers.23.mlp.experts.28.gate_proj", "model.layers.23.mlp.experts.29.gate_proj", "model.layers.23.mlp.experts.30.gate_proj", "model.layers.23.mlp.experts.31.gate_proj", "model.layers.23.mlp.experts.32.gate_proj", "model.layers.23.mlp.experts.33.gate_proj", "model.layers.23.mlp.experts.34.gate_proj", "model.layers.23.mlp.experts.35.gate_proj", "model.layers.23.mlp.experts.36.gate_proj", "model.layers.23.mlp.experts.37.gate_proj", "model.layers.23.mlp.experts.38.gate_proj", "model.layers.23.mlp.experts.39.gate_proj", "model.layers.23.mlp.experts.40.gate_proj", "model.layers.23.mlp.experts.41.gate_proj", "model.layers.23.mlp.experts.42.gate_proj", "model.layers.23.mlp.experts.43.gate_proj", "model.layers.23.mlp.experts.44.gate_proj", "model.layers.23.mlp.experts.45.gate_proj", "model.layers.23.mlp.experts.46.gate_proj", "model.layers.23.mlp.experts.47.gate_proj", "model.layers.23.mlp.experts.48.gate_proj", "model.layers.23.mlp.experts.49.gate_proj", "model.layers.23.mlp.experts.50.gate_proj", "model.layers.23.mlp.experts.51.gate_proj", "model.layers.23.mlp.experts.52.gate_proj", "model.layers.23.mlp.experts.53.gate_proj", "model.layers.23.mlp.experts.54.gate_proj", "model.layers.23.mlp.experts.55.gate_proj", "model.layers.23.mlp.experts.56.gate_proj", "model.layers.23.mlp.experts.57.gate_proj", "model.layers.23.mlp.experts.58.gate_proj", "model.layers.23.mlp.experts.59.gate_proj", "model.layers.23.mlp.experts.60.gate_proj", "model.layers.23.mlp.experts.61.gate_proj", "model.layers.23.mlp.experts.62.gate_proj", "model.layers.23.mlp.experts.63.gate_proj", "model.layers.23.mlp.experts.64.gate_proj", "model.layers.23.mlp.experts.65.gate_proj", "model.layers.23.mlp.experts.66.gate_proj", "model.layers.23.mlp.experts.67.gate_proj", "model.layers.23.mlp.experts.68.gate_proj", "model.layers.23.mlp.experts.69.gate_proj", "model.layers.23.mlp.experts.70.gate_proj", "model.layers.23.mlp.experts.71.gate_proj", "model.layers.23.mlp.experts.72.gate_proj", "model.layers.23.mlp.experts.73.gate_proj", "model.layers.23.mlp.experts.74.gate_proj", "model.layers.23.mlp.experts.75.gate_proj", "model.layers.23.mlp.experts.76.gate_proj", "model.layers.23.mlp.experts.77.gate_proj", "model.layers.23.mlp.experts.78.gate_proj", "model.layers.23.mlp.experts.79.gate_proj", "model.layers.23.mlp.experts.80.gate_proj", "model.layers.23.mlp.experts.81.gate_proj", "model.layers.23.mlp.experts.82.gate_proj", "model.layers.23.mlp.experts.83.gate_proj", "model.layers.23.mlp.experts.84.gate_proj", "model.layers.23.mlp.experts.85.gate_proj", "model.layers.23.mlp.experts.86.gate_proj", "model.layers.23.mlp.experts.87.gate_proj", "model.layers.23.mlp.experts.88.gate_proj", "model.layers.23.mlp.experts.89.gate_proj", "model.layers.23.mlp.experts.90.gate_proj", "model.layers.23.mlp.experts.91.gate_proj", "model.layers.23.mlp.experts.92.gate_proj", "model.layers.23.mlp.experts.93.gate_proj", "model.layers.23.mlp.experts.94.gate_proj", "model.layers.23.mlp.experts.95.gate_proj", "model.layers.23.mlp.experts.96.gate_proj", "model.layers.23.mlp.experts.97.gate_proj", "model.layers.23.mlp.experts.98.gate_proj", "model.layers.23.mlp.experts.99.gate_proj", "model.layers.23.mlp.experts.100.gate_proj", "model.layers.23.mlp.experts.101.gate_proj", "model.layers.23.mlp.experts.102.gate_proj", "model.layers.23.mlp.experts.103.gate_proj", "model.layers.23.mlp.experts.104.gate_proj", "model.layers.23.mlp.experts.105.gate_proj", "model.layers.23.mlp.experts.106.gate_proj", "model.layers.23.mlp.experts.107.gate_proj", "model.layers.23.mlp.experts.108.gate_proj", "model.layers.23.mlp.experts.109.gate_proj", "model.layers.23.mlp.experts.110.gate_proj", "model.layers.23.mlp.experts.111.gate_proj", "model.layers.23.mlp.experts.112.gate_proj", "model.layers.23.mlp.experts.113.gate_proj", "model.layers.23.mlp.experts.114.gate_proj", "model.layers.23.mlp.experts.115.gate_proj", "model.layers.23.mlp.experts.116.gate_proj", "model.layers.23.mlp.experts.117.gate_proj", "model.layers.23.mlp.experts.118.gate_proj", "model.layers.23.mlp.experts.119.gate_proj", "model.layers.23.mlp.experts.120.gate_proj", "model.layers.23.mlp.experts.121.gate_proj", "model.layers.23.mlp.experts.122.gate_proj", "model.layers.23.mlp.experts.123.gate_proj", "model.layers.23.mlp.experts.124.gate_proj", "model.layers.23.mlp.experts.125.gate_proj", "model.layers.23.mlp.experts.126.gate_proj", "model.layers.23.mlp.experts.127.gate_proj", "model.layers.23.mlp.experts.0.up_proj", "model.layers.23.mlp.experts.1.up_proj", "model.layers.23.mlp.experts.2.up_proj", "model.layers.23.mlp.experts.3.up_proj", "model.layers.23.mlp.experts.4.up_proj", "model.layers.23.mlp.experts.5.up_proj", "model.layers.23.mlp.experts.6.up_proj", "model.layers.23.mlp.experts.7.up_proj", "model.layers.23.mlp.experts.8.up_proj", "model.layers.23.mlp.experts.9.up_proj", "model.layers.23.mlp.experts.10.up_proj", "model.layers.23.mlp.experts.11.up_proj", "model.layers.23.mlp.experts.12.up_proj", "model.layers.23.mlp.experts.13.up_proj", "model.layers.23.mlp.experts.14.up_proj", "model.layers.23.mlp.experts.15.up_proj", "model.layers.23.mlp.experts.16.up_proj", "model.layers.23.mlp.experts.17.up_proj", "model.layers.23.mlp.experts.18.up_proj", "model.layers.23.mlp.experts.19.up_proj", "model.layers.23.mlp.experts.20.up_proj", "model.layers.23.mlp.experts.21.up_proj", "model.layers.23.mlp.experts.22.up_proj", "model.layers.23.mlp.experts.23.up_proj", "model.layers.23.mlp.experts.24.up_proj", "model.layers.23.mlp.experts.25.up_proj", "model.layers.23.mlp.experts.26.up_proj", "model.layers.23.mlp.experts.27.up_proj", "model.layers.23.mlp.experts.28.up_proj", "model.layers.23.mlp.experts.29.up_proj", "model.layers.23.mlp.experts.30.up_proj", "model.layers.23.mlp.experts.31.up_proj", "model.layers.23.mlp.experts.32.up_proj", "model.layers.23.mlp.experts.33.up_proj", "model.layers.23.mlp.experts.34.up_proj", "model.layers.23.mlp.experts.35.up_proj", "model.layers.23.mlp.experts.36.up_proj", "model.layers.23.mlp.experts.37.up_proj", "model.layers.23.mlp.experts.38.up_proj", "model.layers.23.mlp.experts.39.up_proj", "model.layers.23.mlp.experts.40.up_proj", "model.layers.23.mlp.experts.41.up_proj", "model.layers.23.mlp.experts.42.up_proj", "model.layers.23.mlp.experts.43.up_proj", "model.layers.23.mlp.experts.44.up_proj", "model.layers.23.mlp.experts.45.up_proj", "model.layers.23.mlp.experts.46.up_proj", "model.layers.23.mlp.experts.47.up_proj", "model.layers.23.mlp.experts.48.up_proj", "model.layers.23.mlp.experts.49.up_proj", "model.layers.23.mlp.experts.50.up_proj", "model.layers.23.mlp.experts.51.up_proj", "model.layers.23.mlp.experts.52.up_proj", "model.layers.23.mlp.experts.53.up_proj", "model.layers.23.mlp.experts.54.up_proj", "model.layers.23.mlp.experts.55.up_proj", "model.layers.23.mlp.experts.56.up_proj", "model.layers.23.mlp.experts.57.up_proj", "model.layers.23.mlp.experts.58.up_proj", "model.layers.23.mlp.experts.59.up_proj", "model.layers.23.mlp.experts.60.up_proj", "model.layers.23.mlp.experts.61.up_proj", "model.layers.23.mlp.experts.62.up_proj", "model.layers.23.mlp.experts.63.up_proj", "model.layers.23.mlp.experts.64.up_proj", "model.layers.23.mlp.experts.65.up_proj", "model.layers.23.mlp.experts.66.up_proj", "model.layers.23.mlp.experts.67.up_proj", "model.layers.23.mlp.experts.68.up_proj", "model.layers.23.mlp.experts.69.up_proj", "model.layers.23.mlp.experts.70.up_proj", "model.layers.23.mlp.experts.71.up_proj", "model.layers.23.mlp.experts.72.up_proj", "model.layers.23.mlp.experts.73.up_proj", "model.layers.23.mlp.experts.74.up_proj", "model.layers.23.mlp.experts.75.up_proj", "model.layers.23.mlp.experts.76.up_proj", "model.layers.23.mlp.experts.77.up_proj", "model.layers.23.mlp.experts.78.up_proj", "model.layers.23.mlp.experts.79.up_proj", "model.layers.23.mlp.experts.80.up_proj", "model.layers.23.mlp.experts.81.up_proj", "model.layers.23.mlp.experts.82.up_proj", "model.layers.23.mlp.experts.83.up_proj", "model.layers.23.mlp.experts.84.up_proj", "model.layers.23.mlp.experts.85.up_proj", "model.layers.23.mlp.experts.86.up_proj", "model.layers.23.mlp.experts.87.up_proj", "model.layers.23.mlp.experts.88.up_proj", "model.layers.23.mlp.experts.89.up_proj", "model.layers.23.mlp.experts.90.up_proj", "model.layers.23.mlp.experts.91.up_proj", "model.layers.23.mlp.experts.92.up_proj", "model.layers.23.mlp.experts.93.up_proj", "model.layers.23.mlp.experts.94.up_proj", "model.layers.23.mlp.experts.95.up_proj", "model.layers.23.mlp.experts.96.up_proj", "model.layers.23.mlp.experts.97.up_proj", "model.layers.23.mlp.experts.98.up_proj", "model.layers.23.mlp.experts.99.up_proj", "model.layers.23.mlp.experts.100.up_proj", "model.layers.23.mlp.experts.101.up_proj", "model.layers.23.mlp.experts.102.up_proj", "model.layers.23.mlp.experts.103.up_proj", "model.layers.23.mlp.experts.104.up_proj", "model.layers.23.mlp.experts.105.up_proj", "model.layers.23.mlp.experts.106.up_proj", "model.layers.23.mlp.experts.107.up_proj", "model.layers.23.mlp.experts.108.up_proj", "model.layers.23.mlp.experts.109.up_proj", "model.layers.23.mlp.experts.110.up_proj", "model.layers.23.mlp.experts.111.up_proj", "model.layers.23.mlp.experts.112.up_proj", "model.layers.23.mlp.experts.113.up_proj", "model.layers.23.mlp.experts.114.up_proj", "model.layers.23.mlp.experts.115.up_proj", "model.layers.23.mlp.experts.116.up_proj", "model.layers.23.mlp.experts.117.up_proj", "model.layers.23.mlp.experts.118.up_proj", "model.layers.23.mlp.experts.119.up_proj", "model.layers.23.mlp.experts.120.up_proj", "model.layers.23.mlp.experts.121.up_proj", "model.layers.23.mlp.experts.122.up_proj", "model.layers.23.mlp.experts.123.up_proj", "model.layers.23.mlp.experts.124.up_proj", "model.layers.23.mlp.experts.125.up_proj", "model.layers.23.mlp.experts.126.up_proj", "model.layers.23.mlp.experts.127.up_proj", "model.layers.23.mlp.experts.0.down_proj", "model.layers.23.mlp.experts.1.down_proj", "model.layers.23.mlp.experts.2.down_proj", "model.layers.23.mlp.experts.3.down_proj", "model.layers.23.mlp.experts.4.down_proj", "model.layers.23.mlp.experts.5.down_proj", "model.layers.23.mlp.experts.6.down_proj", "model.layers.23.mlp.experts.7.down_proj", "model.layers.23.mlp.experts.8.down_proj", "model.layers.23.mlp.experts.9.down_proj", "model.layers.23.mlp.experts.10.down_proj", "model.layers.23.mlp.experts.11.down_proj", "model.layers.23.mlp.experts.12.down_proj", "model.layers.23.mlp.experts.13.down_proj", "model.layers.23.mlp.experts.14.down_proj", "model.layers.23.mlp.experts.15.down_proj", "model.layers.23.mlp.experts.16.down_proj", "model.layers.23.mlp.experts.17.down_proj", "model.layers.23.mlp.experts.18.down_proj", "model.layers.23.mlp.experts.19.down_proj", "model.layers.23.mlp.experts.20.down_proj", "model.layers.23.mlp.experts.21.down_proj", "model.layers.23.mlp.experts.22.down_proj", "model.layers.23.mlp.experts.23.down_proj", "model.layers.23.mlp.experts.24.down_proj", "model.layers.23.mlp.experts.25.down_proj", "model.layers.23.mlp.experts.26.down_proj", "model.layers.23.mlp.experts.27.down_proj", "model.layers.23.mlp.experts.28.down_proj", "model.layers.23.mlp.experts.29.down_proj", "model.layers.23.mlp.experts.30.down_proj", "model.layers.23.mlp.experts.31.down_proj", "model.layers.23.mlp.experts.32.down_proj", "model.layers.23.mlp.experts.33.down_proj", "model.layers.23.mlp.experts.34.down_proj", "model.layers.23.mlp.experts.35.down_proj", "model.layers.23.mlp.experts.36.down_proj", "model.layers.23.mlp.experts.37.down_proj", "model.layers.23.mlp.experts.38.down_proj", "model.layers.23.mlp.experts.39.down_proj", "model.layers.23.mlp.experts.40.down_proj", "model.layers.23.mlp.experts.41.down_proj", "model.layers.23.mlp.experts.42.down_proj", "model.layers.23.mlp.experts.43.down_proj", "model.layers.23.mlp.experts.44.down_proj", "model.layers.23.mlp.experts.45.down_proj", "model.layers.23.mlp.experts.46.down_proj", "model.layers.23.mlp.experts.47.down_proj", "model.layers.23.mlp.experts.48.down_proj", "model.layers.23.mlp.experts.49.down_proj", "model.layers.23.mlp.experts.50.down_proj", "model.layers.23.mlp.experts.51.down_proj", "model.layers.23.mlp.experts.52.down_proj", "model.layers.23.mlp.experts.53.down_proj", "model.layers.23.mlp.experts.54.down_proj", "model.layers.23.mlp.experts.55.down_proj", "model.layers.23.mlp.experts.56.down_proj", "model.layers.23.mlp.experts.57.down_proj", "model.layers.23.mlp.experts.58.down_proj", "model.layers.23.mlp.experts.59.down_proj", "model.layers.23.mlp.experts.60.down_proj", "model.layers.23.mlp.experts.61.down_proj", "model.layers.23.mlp.experts.62.down_proj", "model.layers.23.mlp.experts.63.down_proj", "model.layers.23.mlp.experts.64.down_proj", "model.layers.23.mlp.experts.65.down_proj", "model.layers.23.mlp.experts.66.down_proj", "model.layers.23.mlp.experts.67.down_proj", "model.layers.23.mlp.experts.68.down_proj", "model.layers.23.mlp.experts.69.down_proj", "model.layers.23.mlp.experts.70.down_proj", "model.layers.23.mlp.experts.71.down_proj", "model.layers.23.mlp.experts.72.down_proj", "model.layers.23.mlp.experts.73.down_proj", "model.layers.23.mlp.experts.74.down_proj", "model.layers.23.mlp.experts.75.down_proj", "model.layers.23.mlp.experts.76.down_proj", "model.layers.23.mlp.experts.77.down_proj", "model.layers.23.mlp.experts.78.down_proj", "model.layers.23.mlp.experts.79.down_proj", "model.layers.23.mlp.experts.80.down_proj", "model.layers.23.mlp.experts.81.down_proj", "model.layers.23.mlp.experts.82.down_proj", "model.layers.23.mlp.experts.83.down_proj", "model.layers.23.mlp.experts.84.down_proj", "model.layers.23.mlp.experts.85.down_proj", "model.layers.23.mlp.experts.86.down_proj", "model.layers.23.mlp.experts.87.down_proj", "model.layers.23.mlp.experts.88.down_proj", "model.layers.23.mlp.experts.89.down_proj", "model.layers.23.mlp.experts.90.down_proj", "model.layers.23.mlp.experts.91.down_proj", "model.layers.23.mlp.experts.92.down_proj", "model.layers.23.mlp.experts.93.down_proj", "model.layers.23.mlp.experts.94.down_proj", "model.layers.23.mlp.experts.95.down_proj", "model.layers.23.mlp.experts.96.down_proj", "model.layers.23.mlp.experts.97.down_proj", "model.layers.23.mlp.experts.98.down_proj", "model.layers.23.mlp.experts.99.down_proj", "model.layers.23.mlp.experts.100.down_proj", "model.layers.23.mlp.experts.101.down_proj", "model.layers.23.mlp.experts.102.down_proj", "model.layers.23.mlp.experts.103.down_proj", "model.layers.23.mlp.experts.104.down_proj", "model.layers.23.mlp.experts.105.down_proj", "model.layers.23.mlp.experts.106.down_proj", "model.layers.23.mlp.experts.107.down_proj", "model.layers.23.mlp.experts.108.down_proj", "model.layers.23.mlp.experts.109.down_proj", "model.layers.23.mlp.experts.110.down_proj", "model.layers.23.mlp.experts.111.down_proj", "model.layers.23.mlp.experts.112.down_proj", "model.layers.23.mlp.experts.113.down_proj", "model.layers.23.mlp.experts.114.down_proj", "model.layers.23.mlp.experts.115.down_proj", "model.layers.23.mlp.experts.116.down_proj", "model.layers.23.mlp.experts.117.down_proj", "model.layers.23.mlp.experts.118.down_proj", "model.layers.23.mlp.experts.119.down_proj", "model.layers.23.mlp.experts.120.down_proj", "model.layers.23.mlp.experts.121.down_proj", "model.layers.23.mlp.experts.122.down_proj", "model.layers.23.mlp.experts.123.down_proj", "model.layers.23.mlp.experts.124.down_proj", "model.layers.23.mlp.experts.125.down_proj", "model.layers.23.mlp.experts.126.down_proj", "model.layers.23.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0020288887433707603, "dbits": 2214592512 } ] }, { "idx": 71, "layers": [ "model.layers.24.self_attn.q_proj", "model.layers.24.self_attn.k_proj", "model.layers.24.self_attn.v_proj", "model.layers.24.self_attn.o_proj" ], "candidates": [ { "dkld": -7.927166298030142e-05, "dbits": 109051904 } ] }, { "idx": 72, "layers": [ "model.layers.24.mlp.shared_experts.gate_proj", "model.layers.24.mlp.shared_experts.up_proj", "model.layers.24.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0012568756937980652, "dbits": 17301504 } ] }, { "idx": 73, "layers": [ "model.layers.24.mlp.experts.0.gate_proj", "model.layers.24.mlp.experts.1.gate_proj", "model.layers.24.mlp.experts.2.gate_proj", "model.layers.24.mlp.experts.3.gate_proj", "model.layers.24.mlp.experts.4.gate_proj", "model.layers.24.mlp.experts.5.gate_proj", "model.layers.24.mlp.experts.6.gate_proj", "model.layers.24.mlp.experts.7.gate_proj", "model.layers.24.mlp.experts.8.gate_proj", "model.layers.24.mlp.experts.9.gate_proj", "model.layers.24.mlp.experts.10.gate_proj", "model.layers.24.mlp.experts.11.gate_proj", "model.layers.24.mlp.experts.12.gate_proj", "model.layers.24.mlp.experts.13.gate_proj", "model.layers.24.mlp.experts.14.gate_proj", "model.layers.24.mlp.experts.15.gate_proj", "model.layers.24.mlp.experts.16.gate_proj", "model.layers.24.mlp.experts.17.gate_proj", "model.layers.24.mlp.experts.18.gate_proj", "model.layers.24.mlp.experts.19.gate_proj", "model.layers.24.mlp.experts.20.gate_proj", "model.layers.24.mlp.experts.21.gate_proj", "model.layers.24.mlp.experts.22.gate_proj", "model.layers.24.mlp.experts.23.gate_proj", "model.layers.24.mlp.experts.24.gate_proj", "model.layers.24.mlp.experts.25.gate_proj", "model.layers.24.mlp.experts.26.gate_proj", "model.layers.24.mlp.experts.27.gate_proj", "model.layers.24.mlp.experts.28.gate_proj", "model.layers.24.mlp.experts.29.gate_proj", "model.layers.24.mlp.experts.30.gate_proj", "model.layers.24.mlp.experts.31.gate_proj", "model.layers.24.mlp.experts.32.gate_proj", "model.layers.24.mlp.experts.33.gate_proj", "model.layers.24.mlp.experts.34.gate_proj", "model.layers.24.mlp.experts.35.gate_proj", "model.layers.24.mlp.experts.36.gate_proj", "model.layers.24.mlp.experts.37.gate_proj", "model.layers.24.mlp.experts.38.gate_proj", "model.layers.24.mlp.experts.39.gate_proj", "model.layers.24.mlp.experts.40.gate_proj", "model.layers.24.mlp.experts.41.gate_proj", "model.layers.24.mlp.experts.42.gate_proj", "model.layers.24.mlp.experts.43.gate_proj", "model.layers.24.mlp.experts.44.gate_proj", "model.layers.24.mlp.experts.45.gate_proj", "model.layers.24.mlp.experts.46.gate_proj", "model.layers.24.mlp.experts.47.gate_proj", "model.layers.24.mlp.experts.48.gate_proj", "model.layers.24.mlp.experts.49.gate_proj", "model.layers.24.mlp.experts.50.gate_proj", "model.layers.24.mlp.experts.51.gate_proj", "model.layers.24.mlp.experts.52.gate_proj", "model.layers.24.mlp.experts.53.gate_proj", "model.layers.24.mlp.experts.54.gate_proj", "model.layers.24.mlp.experts.55.gate_proj", "model.layers.24.mlp.experts.56.gate_proj", "model.layers.24.mlp.experts.57.gate_proj", "model.layers.24.mlp.experts.58.gate_proj", "model.layers.24.mlp.experts.59.gate_proj", "model.layers.24.mlp.experts.60.gate_proj", "model.layers.24.mlp.experts.61.gate_proj", "model.layers.24.mlp.experts.62.gate_proj", "model.layers.24.mlp.experts.63.gate_proj", "model.layers.24.mlp.experts.64.gate_proj", "model.layers.24.mlp.experts.65.gate_proj", "model.layers.24.mlp.experts.66.gate_proj", "model.layers.24.mlp.experts.67.gate_proj", "model.layers.24.mlp.experts.68.gate_proj", "model.layers.24.mlp.experts.69.gate_proj", "model.layers.24.mlp.experts.70.gate_proj", "model.layers.24.mlp.experts.71.gate_proj", "model.layers.24.mlp.experts.72.gate_proj", "model.layers.24.mlp.experts.73.gate_proj", "model.layers.24.mlp.experts.74.gate_proj", "model.layers.24.mlp.experts.75.gate_proj", "model.layers.24.mlp.experts.76.gate_proj", "model.layers.24.mlp.experts.77.gate_proj", "model.layers.24.mlp.experts.78.gate_proj", "model.layers.24.mlp.experts.79.gate_proj", "model.layers.24.mlp.experts.80.gate_proj", "model.layers.24.mlp.experts.81.gate_proj", "model.layers.24.mlp.experts.82.gate_proj", "model.layers.24.mlp.experts.83.gate_proj", "model.layers.24.mlp.experts.84.gate_proj", "model.layers.24.mlp.experts.85.gate_proj", "model.layers.24.mlp.experts.86.gate_proj", "model.layers.24.mlp.experts.87.gate_proj", "model.layers.24.mlp.experts.88.gate_proj", "model.layers.24.mlp.experts.89.gate_proj", "model.layers.24.mlp.experts.90.gate_proj", "model.layers.24.mlp.experts.91.gate_proj", "model.layers.24.mlp.experts.92.gate_proj", "model.layers.24.mlp.experts.93.gate_proj", "model.layers.24.mlp.experts.94.gate_proj", "model.layers.24.mlp.experts.95.gate_proj", "model.layers.24.mlp.experts.96.gate_proj", "model.layers.24.mlp.experts.97.gate_proj", "model.layers.24.mlp.experts.98.gate_proj", "model.layers.24.mlp.experts.99.gate_proj", "model.layers.24.mlp.experts.100.gate_proj", "model.layers.24.mlp.experts.101.gate_proj", "model.layers.24.mlp.experts.102.gate_proj", "model.layers.24.mlp.experts.103.gate_proj", "model.layers.24.mlp.experts.104.gate_proj", "model.layers.24.mlp.experts.105.gate_proj", "model.layers.24.mlp.experts.106.gate_proj", "model.layers.24.mlp.experts.107.gate_proj", "model.layers.24.mlp.experts.108.gate_proj", "model.layers.24.mlp.experts.109.gate_proj", "model.layers.24.mlp.experts.110.gate_proj", "model.layers.24.mlp.experts.111.gate_proj", "model.layers.24.mlp.experts.112.gate_proj", "model.layers.24.mlp.experts.113.gate_proj", "model.layers.24.mlp.experts.114.gate_proj", "model.layers.24.mlp.experts.115.gate_proj", "model.layers.24.mlp.experts.116.gate_proj", "model.layers.24.mlp.experts.117.gate_proj", "model.layers.24.mlp.experts.118.gate_proj", "model.layers.24.mlp.experts.119.gate_proj", "model.layers.24.mlp.experts.120.gate_proj", "model.layers.24.mlp.experts.121.gate_proj", "model.layers.24.mlp.experts.122.gate_proj", "model.layers.24.mlp.experts.123.gate_proj", "model.layers.24.mlp.experts.124.gate_proj", "model.layers.24.mlp.experts.125.gate_proj", "model.layers.24.mlp.experts.126.gate_proj", "model.layers.24.mlp.experts.127.gate_proj", "model.layers.24.mlp.experts.0.up_proj", "model.layers.24.mlp.experts.1.up_proj", "model.layers.24.mlp.experts.2.up_proj", "model.layers.24.mlp.experts.3.up_proj", "model.layers.24.mlp.experts.4.up_proj", "model.layers.24.mlp.experts.5.up_proj", "model.layers.24.mlp.experts.6.up_proj", "model.layers.24.mlp.experts.7.up_proj", "model.layers.24.mlp.experts.8.up_proj", "model.layers.24.mlp.experts.9.up_proj", "model.layers.24.mlp.experts.10.up_proj", "model.layers.24.mlp.experts.11.up_proj", "model.layers.24.mlp.experts.12.up_proj", "model.layers.24.mlp.experts.13.up_proj", "model.layers.24.mlp.experts.14.up_proj", "model.layers.24.mlp.experts.15.up_proj", "model.layers.24.mlp.experts.16.up_proj", "model.layers.24.mlp.experts.17.up_proj", "model.layers.24.mlp.experts.18.up_proj", "model.layers.24.mlp.experts.19.up_proj", "model.layers.24.mlp.experts.20.up_proj", "model.layers.24.mlp.experts.21.up_proj", "model.layers.24.mlp.experts.22.up_proj", "model.layers.24.mlp.experts.23.up_proj", "model.layers.24.mlp.experts.24.up_proj", "model.layers.24.mlp.experts.25.up_proj", "model.layers.24.mlp.experts.26.up_proj", "model.layers.24.mlp.experts.27.up_proj", "model.layers.24.mlp.experts.28.up_proj", "model.layers.24.mlp.experts.29.up_proj", "model.layers.24.mlp.experts.30.up_proj", "model.layers.24.mlp.experts.31.up_proj", "model.layers.24.mlp.experts.32.up_proj", "model.layers.24.mlp.experts.33.up_proj", "model.layers.24.mlp.experts.34.up_proj", "model.layers.24.mlp.experts.35.up_proj", "model.layers.24.mlp.experts.36.up_proj", "model.layers.24.mlp.experts.37.up_proj", "model.layers.24.mlp.experts.38.up_proj", "model.layers.24.mlp.experts.39.up_proj", "model.layers.24.mlp.experts.40.up_proj", "model.layers.24.mlp.experts.41.up_proj", "model.layers.24.mlp.experts.42.up_proj", "model.layers.24.mlp.experts.43.up_proj", "model.layers.24.mlp.experts.44.up_proj", "model.layers.24.mlp.experts.45.up_proj", "model.layers.24.mlp.experts.46.up_proj", "model.layers.24.mlp.experts.47.up_proj", "model.layers.24.mlp.experts.48.up_proj", "model.layers.24.mlp.experts.49.up_proj", "model.layers.24.mlp.experts.50.up_proj", "model.layers.24.mlp.experts.51.up_proj", "model.layers.24.mlp.experts.52.up_proj", "model.layers.24.mlp.experts.53.up_proj", "model.layers.24.mlp.experts.54.up_proj", "model.layers.24.mlp.experts.55.up_proj", "model.layers.24.mlp.experts.56.up_proj", "model.layers.24.mlp.experts.57.up_proj", "model.layers.24.mlp.experts.58.up_proj", "model.layers.24.mlp.experts.59.up_proj", "model.layers.24.mlp.experts.60.up_proj", "model.layers.24.mlp.experts.61.up_proj", "model.layers.24.mlp.experts.62.up_proj", "model.layers.24.mlp.experts.63.up_proj", "model.layers.24.mlp.experts.64.up_proj", "model.layers.24.mlp.experts.65.up_proj", "model.layers.24.mlp.experts.66.up_proj", "model.layers.24.mlp.experts.67.up_proj", "model.layers.24.mlp.experts.68.up_proj", "model.layers.24.mlp.experts.69.up_proj", "model.layers.24.mlp.experts.70.up_proj", "model.layers.24.mlp.experts.71.up_proj", "model.layers.24.mlp.experts.72.up_proj", "model.layers.24.mlp.experts.73.up_proj", "model.layers.24.mlp.experts.74.up_proj", "model.layers.24.mlp.experts.75.up_proj", "model.layers.24.mlp.experts.76.up_proj", "model.layers.24.mlp.experts.77.up_proj", "model.layers.24.mlp.experts.78.up_proj", "model.layers.24.mlp.experts.79.up_proj", "model.layers.24.mlp.experts.80.up_proj", "model.layers.24.mlp.experts.81.up_proj", "model.layers.24.mlp.experts.82.up_proj", "model.layers.24.mlp.experts.83.up_proj", "model.layers.24.mlp.experts.84.up_proj", "model.layers.24.mlp.experts.85.up_proj", "model.layers.24.mlp.experts.86.up_proj", "model.layers.24.mlp.experts.87.up_proj", "model.layers.24.mlp.experts.88.up_proj", "model.layers.24.mlp.experts.89.up_proj", "model.layers.24.mlp.experts.90.up_proj", "model.layers.24.mlp.experts.91.up_proj", "model.layers.24.mlp.experts.92.up_proj", "model.layers.24.mlp.experts.93.up_proj", "model.layers.24.mlp.experts.94.up_proj", "model.layers.24.mlp.experts.95.up_proj", "model.layers.24.mlp.experts.96.up_proj", "model.layers.24.mlp.experts.97.up_proj", "model.layers.24.mlp.experts.98.up_proj", "model.layers.24.mlp.experts.99.up_proj", "model.layers.24.mlp.experts.100.up_proj", "model.layers.24.mlp.experts.101.up_proj", "model.layers.24.mlp.experts.102.up_proj", "model.layers.24.mlp.experts.103.up_proj", "model.layers.24.mlp.experts.104.up_proj", "model.layers.24.mlp.experts.105.up_proj", "model.layers.24.mlp.experts.106.up_proj", "model.layers.24.mlp.experts.107.up_proj", "model.layers.24.mlp.experts.108.up_proj", "model.layers.24.mlp.experts.109.up_proj", "model.layers.24.mlp.experts.110.up_proj", "model.layers.24.mlp.experts.111.up_proj", "model.layers.24.mlp.experts.112.up_proj", "model.layers.24.mlp.experts.113.up_proj", "model.layers.24.mlp.experts.114.up_proj", "model.layers.24.mlp.experts.115.up_proj", "model.layers.24.mlp.experts.116.up_proj", "model.layers.24.mlp.experts.117.up_proj", "model.layers.24.mlp.experts.118.up_proj", "model.layers.24.mlp.experts.119.up_proj", "model.layers.24.mlp.experts.120.up_proj", "model.layers.24.mlp.experts.121.up_proj", "model.layers.24.mlp.experts.122.up_proj", "model.layers.24.mlp.experts.123.up_proj", "model.layers.24.mlp.experts.124.up_proj", "model.layers.24.mlp.experts.125.up_proj", "model.layers.24.mlp.experts.126.up_proj", "model.layers.24.mlp.experts.127.up_proj", "model.layers.24.mlp.experts.0.down_proj", "model.layers.24.mlp.experts.1.down_proj", "model.layers.24.mlp.experts.2.down_proj", "model.layers.24.mlp.experts.3.down_proj", "model.layers.24.mlp.experts.4.down_proj", "model.layers.24.mlp.experts.5.down_proj", "model.layers.24.mlp.experts.6.down_proj", "model.layers.24.mlp.experts.7.down_proj", "model.layers.24.mlp.experts.8.down_proj", "model.layers.24.mlp.experts.9.down_proj", "model.layers.24.mlp.experts.10.down_proj", "model.layers.24.mlp.experts.11.down_proj", "model.layers.24.mlp.experts.12.down_proj", "model.layers.24.mlp.experts.13.down_proj", "model.layers.24.mlp.experts.14.down_proj", "model.layers.24.mlp.experts.15.down_proj", "model.layers.24.mlp.experts.16.down_proj", "model.layers.24.mlp.experts.17.down_proj", "model.layers.24.mlp.experts.18.down_proj", "model.layers.24.mlp.experts.19.down_proj", "model.layers.24.mlp.experts.20.down_proj", "model.layers.24.mlp.experts.21.down_proj", "model.layers.24.mlp.experts.22.down_proj", "model.layers.24.mlp.experts.23.down_proj", "model.layers.24.mlp.experts.24.down_proj", "model.layers.24.mlp.experts.25.down_proj", "model.layers.24.mlp.experts.26.down_proj", "model.layers.24.mlp.experts.27.down_proj", "model.layers.24.mlp.experts.28.down_proj", "model.layers.24.mlp.experts.29.down_proj", "model.layers.24.mlp.experts.30.down_proj", "model.layers.24.mlp.experts.31.down_proj", "model.layers.24.mlp.experts.32.down_proj", "model.layers.24.mlp.experts.33.down_proj", "model.layers.24.mlp.experts.34.down_proj", "model.layers.24.mlp.experts.35.down_proj", "model.layers.24.mlp.experts.36.down_proj", "model.layers.24.mlp.experts.37.down_proj", "model.layers.24.mlp.experts.38.down_proj", "model.layers.24.mlp.experts.39.down_proj", "model.layers.24.mlp.experts.40.down_proj", "model.layers.24.mlp.experts.41.down_proj", "model.layers.24.mlp.experts.42.down_proj", "model.layers.24.mlp.experts.43.down_proj", "model.layers.24.mlp.experts.44.down_proj", "model.layers.24.mlp.experts.45.down_proj", "model.layers.24.mlp.experts.46.down_proj", "model.layers.24.mlp.experts.47.down_proj", "model.layers.24.mlp.experts.48.down_proj", "model.layers.24.mlp.experts.49.down_proj", "model.layers.24.mlp.experts.50.down_proj", "model.layers.24.mlp.experts.51.down_proj", "model.layers.24.mlp.experts.52.down_proj", "model.layers.24.mlp.experts.53.down_proj", "model.layers.24.mlp.experts.54.down_proj", "model.layers.24.mlp.experts.55.down_proj", "model.layers.24.mlp.experts.56.down_proj", "model.layers.24.mlp.experts.57.down_proj", "model.layers.24.mlp.experts.58.down_proj", "model.layers.24.mlp.experts.59.down_proj", "model.layers.24.mlp.experts.60.down_proj", "model.layers.24.mlp.experts.61.down_proj", "model.layers.24.mlp.experts.62.down_proj", "model.layers.24.mlp.experts.63.down_proj", "model.layers.24.mlp.experts.64.down_proj", "model.layers.24.mlp.experts.65.down_proj", "model.layers.24.mlp.experts.66.down_proj", "model.layers.24.mlp.experts.67.down_proj", "model.layers.24.mlp.experts.68.down_proj", "model.layers.24.mlp.experts.69.down_proj", "model.layers.24.mlp.experts.70.down_proj", "model.layers.24.mlp.experts.71.down_proj", "model.layers.24.mlp.experts.72.down_proj", "model.layers.24.mlp.experts.73.down_proj", "model.layers.24.mlp.experts.74.down_proj", "model.layers.24.mlp.experts.75.down_proj", "model.layers.24.mlp.experts.76.down_proj", "model.layers.24.mlp.experts.77.down_proj", "model.layers.24.mlp.experts.78.down_proj", "model.layers.24.mlp.experts.79.down_proj", "model.layers.24.mlp.experts.80.down_proj", "model.layers.24.mlp.experts.81.down_proj", "model.layers.24.mlp.experts.82.down_proj", "model.layers.24.mlp.experts.83.down_proj", "model.layers.24.mlp.experts.84.down_proj", "model.layers.24.mlp.experts.85.down_proj", "model.layers.24.mlp.experts.86.down_proj", "model.layers.24.mlp.experts.87.down_proj", "model.layers.24.mlp.experts.88.down_proj", "model.layers.24.mlp.experts.89.down_proj", "model.layers.24.mlp.experts.90.down_proj", "model.layers.24.mlp.experts.91.down_proj", "model.layers.24.mlp.experts.92.down_proj", "model.layers.24.mlp.experts.93.down_proj", "model.layers.24.mlp.experts.94.down_proj", "model.layers.24.mlp.experts.95.down_proj", "model.layers.24.mlp.experts.96.down_proj", "model.layers.24.mlp.experts.97.down_proj", "model.layers.24.mlp.experts.98.down_proj", "model.layers.24.mlp.experts.99.down_proj", "model.layers.24.mlp.experts.100.down_proj", "model.layers.24.mlp.experts.101.down_proj", "model.layers.24.mlp.experts.102.down_proj", "model.layers.24.mlp.experts.103.down_proj", "model.layers.24.mlp.experts.104.down_proj", "model.layers.24.mlp.experts.105.down_proj", "model.layers.24.mlp.experts.106.down_proj", "model.layers.24.mlp.experts.107.down_proj", "model.layers.24.mlp.experts.108.down_proj", "model.layers.24.mlp.experts.109.down_proj", "model.layers.24.mlp.experts.110.down_proj", "model.layers.24.mlp.experts.111.down_proj", "model.layers.24.mlp.experts.112.down_proj", "model.layers.24.mlp.experts.113.down_proj", "model.layers.24.mlp.experts.114.down_proj", "model.layers.24.mlp.experts.115.down_proj", "model.layers.24.mlp.experts.116.down_proj", "model.layers.24.mlp.experts.117.down_proj", "model.layers.24.mlp.experts.118.down_proj", "model.layers.24.mlp.experts.119.down_proj", "model.layers.24.mlp.experts.120.down_proj", "model.layers.24.mlp.experts.121.down_proj", "model.layers.24.mlp.experts.122.down_proj", "model.layers.24.mlp.experts.123.down_proj", "model.layers.24.mlp.experts.124.down_proj", "model.layers.24.mlp.experts.125.down_proj", "model.layers.24.mlp.experts.126.down_proj", "model.layers.24.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0023426462896168065, "dbits": 2214592512 } ] }, { "idx": 74, "layers": [ "model.layers.25.self_attn.q_proj", "model.layers.25.self_attn.k_proj", "model.layers.25.self_attn.v_proj", "model.layers.25.self_attn.o_proj" ], "candidates": [ { "dkld": 6.12766481936089e-05, "dbits": 109051904 } ] }, { "idx": 75, "layers": [ "model.layers.25.mlp.shared_experts.gate_proj", "model.layers.25.mlp.shared_experts.up_proj", "model.layers.25.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0008034589700400829, "dbits": 17301504 } ] }, { "idx": 76, "layers": [ "model.layers.25.mlp.experts.0.gate_proj", "model.layers.25.mlp.experts.1.gate_proj", "model.layers.25.mlp.experts.2.gate_proj", "model.layers.25.mlp.experts.3.gate_proj", "model.layers.25.mlp.experts.4.gate_proj", "model.layers.25.mlp.experts.5.gate_proj", "model.layers.25.mlp.experts.6.gate_proj", "model.layers.25.mlp.experts.7.gate_proj", "model.layers.25.mlp.experts.8.gate_proj", "model.layers.25.mlp.experts.9.gate_proj", "model.layers.25.mlp.experts.10.gate_proj", "model.layers.25.mlp.experts.11.gate_proj", "model.layers.25.mlp.experts.12.gate_proj", "model.layers.25.mlp.experts.13.gate_proj", "model.layers.25.mlp.experts.14.gate_proj", "model.layers.25.mlp.experts.15.gate_proj", "model.layers.25.mlp.experts.16.gate_proj", "model.layers.25.mlp.experts.17.gate_proj", "model.layers.25.mlp.experts.18.gate_proj", "model.layers.25.mlp.experts.19.gate_proj", "model.layers.25.mlp.experts.20.gate_proj", "model.layers.25.mlp.experts.21.gate_proj", "model.layers.25.mlp.experts.22.gate_proj", "model.layers.25.mlp.experts.23.gate_proj", "model.layers.25.mlp.experts.24.gate_proj", "model.layers.25.mlp.experts.25.gate_proj", "model.layers.25.mlp.experts.26.gate_proj", "model.layers.25.mlp.experts.27.gate_proj", "model.layers.25.mlp.experts.28.gate_proj", "model.layers.25.mlp.experts.29.gate_proj", "model.layers.25.mlp.experts.30.gate_proj", "model.layers.25.mlp.experts.31.gate_proj", "model.layers.25.mlp.experts.32.gate_proj", "model.layers.25.mlp.experts.33.gate_proj", "model.layers.25.mlp.experts.34.gate_proj", "model.layers.25.mlp.experts.35.gate_proj", "model.layers.25.mlp.experts.36.gate_proj", "model.layers.25.mlp.experts.37.gate_proj", "model.layers.25.mlp.experts.38.gate_proj", "model.layers.25.mlp.experts.39.gate_proj", "model.layers.25.mlp.experts.40.gate_proj", "model.layers.25.mlp.experts.41.gate_proj", "model.layers.25.mlp.experts.42.gate_proj", "model.layers.25.mlp.experts.43.gate_proj", "model.layers.25.mlp.experts.44.gate_proj", "model.layers.25.mlp.experts.45.gate_proj", "model.layers.25.mlp.experts.46.gate_proj", "model.layers.25.mlp.experts.47.gate_proj", "model.layers.25.mlp.experts.48.gate_proj", "model.layers.25.mlp.experts.49.gate_proj", "model.layers.25.mlp.experts.50.gate_proj", "model.layers.25.mlp.experts.51.gate_proj", "model.layers.25.mlp.experts.52.gate_proj", "model.layers.25.mlp.experts.53.gate_proj", "model.layers.25.mlp.experts.54.gate_proj", "model.layers.25.mlp.experts.55.gate_proj", "model.layers.25.mlp.experts.56.gate_proj", "model.layers.25.mlp.experts.57.gate_proj", "model.layers.25.mlp.experts.58.gate_proj", "model.layers.25.mlp.experts.59.gate_proj", "model.layers.25.mlp.experts.60.gate_proj", "model.layers.25.mlp.experts.61.gate_proj", "model.layers.25.mlp.experts.62.gate_proj", "model.layers.25.mlp.experts.63.gate_proj", "model.layers.25.mlp.experts.64.gate_proj", "model.layers.25.mlp.experts.65.gate_proj", "model.layers.25.mlp.experts.66.gate_proj", "model.layers.25.mlp.experts.67.gate_proj", "model.layers.25.mlp.experts.68.gate_proj", "model.layers.25.mlp.experts.69.gate_proj", "model.layers.25.mlp.experts.70.gate_proj", "model.layers.25.mlp.experts.71.gate_proj", "model.layers.25.mlp.experts.72.gate_proj", "model.layers.25.mlp.experts.73.gate_proj", "model.layers.25.mlp.experts.74.gate_proj", "model.layers.25.mlp.experts.75.gate_proj", "model.layers.25.mlp.experts.76.gate_proj", "model.layers.25.mlp.experts.77.gate_proj", "model.layers.25.mlp.experts.78.gate_proj", "model.layers.25.mlp.experts.79.gate_proj", "model.layers.25.mlp.experts.80.gate_proj", "model.layers.25.mlp.experts.81.gate_proj", "model.layers.25.mlp.experts.82.gate_proj", "model.layers.25.mlp.experts.83.gate_proj", "model.layers.25.mlp.experts.84.gate_proj", "model.layers.25.mlp.experts.85.gate_proj", "model.layers.25.mlp.experts.86.gate_proj", "model.layers.25.mlp.experts.87.gate_proj", "model.layers.25.mlp.experts.88.gate_proj", "model.layers.25.mlp.experts.89.gate_proj", "model.layers.25.mlp.experts.90.gate_proj", "model.layers.25.mlp.experts.91.gate_proj", "model.layers.25.mlp.experts.92.gate_proj", "model.layers.25.mlp.experts.93.gate_proj", "model.layers.25.mlp.experts.94.gate_proj", "model.layers.25.mlp.experts.95.gate_proj", "model.layers.25.mlp.experts.96.gate_proj", "model.layers.25.mlp.experts.97.gate_proj", "model.layers.25.mlp.experts.98.gate_proj", "model.layers.25.mlp.experts.99.gate_proj", "model.layers.25.mlp.experts.100.gate_proj", "model.layers.25.mlp.experts.101.gate_proj", "model.layers.25.mlp.experts.102.gate_proj", "model.layers.25.mlp.experts.103.gate_proj", "model.layers.25.mlp.experts.104.gate_proj", "model.layers.25.mlp.experts.105.gate_proj", "model.layers.25.mlp.experts.106.gate_proj", "model.layers.25.mlp.experts.107.gate_proj", "model.layers.25.mlp.experts.108.gate_proj", "model.layers.25.mlp.experts.109.gate_proj", "model.layers.25.mlp.experts.110.gate_proj", "model.layers.25.mlp.experts.111.gate_proj", "model.layers.25.mlp.experts.112.gate_proj", "model.layers.25.mlp.experts.113.gate_proj", "model.layers.25.mlp.experts.114.gate_proj", "model.layers.25.mlp.experts.115.gate_proj", "model.layers.25.mlp.experts.116.gate_proj", "model.layers.25.mlp.experts.117.gate_proj", "model.layers.25.mlp.experts.118.gate_proj", "model.layers.25.mlp.experts.119.gate_proj", "model.layers.25.mlp.experts.120.gate_proj", "model.layers.25.mlp.experts.121.gate_proj", "model.layers.25.mlp.experts.122.gate_proj", "model.layers.25.mlp.experts.123.gate_proj", "model.layers.25.mlp.experts.124.gate_proj", "model.layers.25.mlp.experts.125.gate_proj", "model.layers.25.mlp.experts.126.gate_proj", "model.layers.25.mlp.experts.127.gate_proj", "model.layers.25.mlp.experts.0.up_proj", "model.layers.25.mlp.experts.1.up_proj", "model.layers.25.mlp.experts.2.up_proj", "model.layers.25.mlp.experts.3.up_proj", "model.layers.25.mlp.experts.4.up_proj", "model.layers.25.mlp.experts.5.up_proj", "model.layers.25.mlp.experts.6.up_proj", "model.layers.25.mlp.experts.7.up_proj", "model.layers.25.mlp.experts.8.up_proj", "model.layers.25.mlp.experts.9.up_proj", "model.layers.25.mlp.experts.10.up_proj", "model.layers.25.mlp.experts.11.up_proj", "model.layers.25.mlp.experts.12.up_proj", "model.layers.25.mlp.experts.13.up_proj", "model.layers.25.mlp.experts.14.up_proj", "model.layers.25.mlp.experts.15.up_proj", "model.layers.25.mlp.experts.16.up_proj", "model.layers.25.mlp.experts.17.up_proj", "model.layers.25.mlp.experts.18.up_proj", "model.layers.25.mlp.experts.19.up_proj", "model.layers.25.mlp.experts.20.up_proj", "model.layers.25.mlp.experts.21.up_proj", "model.layers.25.mlp.experts.22.up_proj", "model.layers.25.mlp.experts.23.up_proj", "model.layers.25.mlp.experts.24.up_proj", "model.layers.25.mlp.experts.25.up_proj", "model.layers.25.mlp.experts.26.up_proj", "model.layers.25.mlp.experts.27.up_proj", "model.layers.25.mlp.experts.28.up_proj", "model.layers.25.mlp.experts.29.up_proj", "model.layers.25.mlp.experts.30.up_proj", "model.layers.25.mlp.experts.31.up_proj", "model.layers.25.mlp.experts.32.up_proj", "model.layers.25.mlp.experts.33.up_proj", "model.layers.25.mlp.experts.34.up_proj", "model.layers.25.mlp.experts.35.up_proj", "model.layers.25.mlp.experts.36.up_proj", "model.layers.25.mlp.experts.37.up_proj", "model.layers.25.mlp.experts.38.up_proj", "model.layers.25.mlp.experts.39.up_proj", "model.layers.25.mlp.experts.40.up_proj", "model.layers.25.mlp.experts.41.up_proj", "model.layers.25.mlp.experts.42.up_proj", "model.layers.25.mlp.experts.43.up_proj", "model.layers.25.mlp.experts.44.up_proj", "model.layers.25.mlp.experts.45.up_proj", "model.layers.25.mlp.experts.46.up_proj", "model.layers.25.mlp.experts.47.up_proj", "model.layers.25.mlp.experts.48.up_proj", "model.layers.25.mlp.experts.49.up_proj", "model.layers.25.mlp.experts.50.up_proj", "model.layers.25.mlp.experts.51.up_proj", "model.layers.25.mlp.experts.52.up_proj", "model.layers.25.mlp.experts.53.up_proj", "model.layers.25.mlp.experts.54.up_proj", "model.layers.25.mlp.experts.55.up_proj", "model.layers.25.mlp.experts.56.up_proj", "model.layers.25.mlp.experts.57.up_proj", "model.layers.25.mlp.experts.58.up_proj", "model.layers.25.mlp.experts.59.up_proj", "model.layers.25.mlp.experts.60.up_proj", "model.layers.25.mlp.experts.61.up_proj", "model.layers.25.mlp.experts.62.up_proj", "model.layers.25.mlp.experts.63.up_proj", "model.layers.25.mlp.experts.64.up_proj", "model.layers.25.mlp.experts.65.up_proj", "model.layers.25.mlp.experts.66.up_proj", "model.layers.25.mlp.experts.67.up_proj", "model.layers.25.mlp.experts.68.up_proj", "model.layers.25.mlp.experts.69.up_proj", "model.layers.25.mlp.experts.70.up_proj", "model.layers.25.mlp.experts.71.up_proj", "model.layers.25.mlp.experts.72.up_proj", "model.layers.25.mlp.experts.73.up_proj", "model.layers.25.mlp.experts.74.up_proj", "model.layers.25.mlp.experts.75.up_proj", "model.layers.25.mlp.experts.76.up_proj", "model.layers.25.mlp.experts.77.up_proj", "model.layers.25.mlp.experts.78.up_proj", "model.layers.25.mlp.experts.79.up_proj", "model.layers.25.mlp.experts.80.up_proj", "model.layers.25.mlp.experts.81.up_proj", "model.layers.25.mlp.experts.82.up_proj", "model.layers.25.mlp.experts.83.up_proj", "model.layers.25.mlp.experts.84.up_proj", "model.layers.25.mlp.experts.85.up_proj", "model.layers.25.mlp.experts.86.up_proj", "model.layers.25.mlp.experts.87.up_proj", "model.layers.25.mlp.experts.88.up_proj", "model.layers.25.mlp.experts.89.up_proj", "model.layers.25.mlp.experts.90.up_proj", "model.layers.25.mlp.experts.91.up_proj", "model.layers.25.mlp.experts.92.up_proj", "model.layers.25.mlp.experts.93.up_proj", "model.layers.25.mlp.experts.94.up_proj", "model.layers.25.mlp.experts.95.up_proj", "model.layers.25.mlp.experts.96.up_proj", "model.layers.25.mlp.experts.97.up_proj", "model.layers.25.mlp.experts.98.up_proj", "model.layers.25.mlp.experts.99.up_proj", "model.layers.25.mlp.experts.100.up_proj", "model.layers.25.mlp.experts.101.up_proj", "model.layers.25.mlp.experts.102.up_proj", "model.layers.25.mlp.experts.103.up_proj", "model.layers.25.mlp.experts.104.up_proj", "model.layers.25.mlp.experts.105.up_proj", "model.layers.25.mlp.experts.106.up_proj", "model.layers.25.mlp.experts.107.up_proj", "model.layers.25.mlp.experts.108.up_proj", "model.layers.25.mlp.experts.109.up_proj", "model.layers.25.mlp.experts.110.up_proj", "model.layers.25.mlp.experts.111.up_proj", "model.layers.25.mlp.experts.112.up_proj", "model.layers.25.mlp.experts.113.up_proj", "model.layers.25.mlp.experts.114.up_proj", "model.layers.25.mlp.experts.115.up_proj", "model.layers.25.mlp.experts.116.up_proj", "model.layers.25.mlp.experts.117.up_proj", "model.layers.25.mlp.experts.118.up_proj", "model.layers.25.mlp.experts.119.up_proj", "model.layers.25.mlp.experts.120.up_proj", "model.layers.25.mlp.experts.121.up_proj", "model.layers.25.mlp.experts.122.up_proj", "model.layers.25.mlp.experts.123.up_proj", "model.layers.25.mlp.experts.124.up_proj", "model.layers.25.mlp.experts.125.up_proj", "model.layers.25.mlp.experts.126.up_proj", "model.layers.25.mlp.experts.127.up_proj", "model.layers.25.mlp.experts.0.down_proj", "model.layers.25.mlp.experts.1.down_proj", "model.layers.25.mlp.experts.2.down_proj", "model.layers.25.mlp.experts.3.down_proj", "model.layers.25.mlp.experts.4.down_proj", "model.layers.25.mlp.experts.5.down_proj", "model.layers.25.mlp.experts.6.down_proj", "model.layers.25.mlp.experts.7.down_proj", "model.layers.25.mlp.experts.8.down_proj", "model.layers.25.mlp.experts.9.down_proj", "model.layers.25.mlp.experts.10.down_proj", "model.layers.25.mlp.experts.11.down_proj", "model.layers.25.mlp.experts.12.down_proj", "model.layers.25.mlp.experts.13.down_proj", "model.layers.25.mlp.experts.14.down_proj", "model.layers.25.mlp.experts.15.down_proj", "model.layers.25.mlp.experts.16.down_proj", "model.layers.25.mlp.experts.17.down_proj", "model.layers.25.mlp.experts.18.down_proj", "model.layers.25.mlp.experts.19.down_proj", "model.layers.25.mlp.experts.20.down_proj", "model.layers.25.mlp.experts.21.down_proj", "model.layers.25.mlp.experts.22.down_proj", "model.layers.25.mlp.experts.23.down_proj", "model.layers.25.mlp.experts.24.down_proj", "model.layers.25.mlp.experts.25.down_proj", "model.layers.25.mlp.experts.26.down_proj", "model.layers.25.mlp.experts.27.down_proj", "model.layers.25.mlp.experts.28.down_proj", "model.layers.25.mlp.experts.29.down_proj", "model.layers.25.mlp.experts.30.down_proj", "model.layers.25.mlp.experts.31.down_proj", "model.layers.25.mlp.experts.32.down_proj", "model.layers.25.mlp.experts.33.down_proj", "model.layers.25.mlp.experts.34.down_proj", "model.layers.25.mlp.experts.35.down_proj", "model.layers.25.mlp.experts.36.down_proj", "model.layers.25.mlp.experts.37.down_proj", "model.layers.25.mlp.experts.38.down_proj", "model.layers.25.mlp.experts.39.down_proj", "model.layers.25.mlp.experts.40.down_proj", "model.layers.25.mlp.experts.41.down_proj", "model.layers.25.mlp.experts.42.down_proj", "model.layers.25.mlp.experts.43.down_proj", "model.layers.25.mlp.experts.44.down_proj", "model.layers.25.mlp.experts.45.down_proj", "model.layers.25.mlp.experts.46.down_proj", "model.layers.25.mlp.experts.47.down_proj", "model.layers.25.mlp.experts.48.down_proj", "model.layers.25.mlp.experts.49.down_proj", "model.layers.25.mlp.experts.50.down_proj", "model.layers.25.mlp.experts.51.down_proj", "model.layers.25.mlp.experts.52.down_proj", "model.layers.25.mlp.experts.53.down_proj", "model.layers.25.mlp.experts.54.down_proj", "model.layers.25.mlp.experts.55.down_proj", "model.layers.25.mlp.experts.56.down_proj", "model.layers.25.mlp.experts.57.down_proj", "model.layers.25.mlp.experts.58.down_proj", "model.layers.25.mlp.experts.59.down_proj", "model.layers.25.mlp.experts.60.down_proj", "model.layers.25.mlp.experts.61.down_proj", "model.layers.25.mlp.experts.62.down_proj", "model.layers.25.mlp.experts.63.down_proj", "model.layers.25.mlp.experts.64.down_proj", "model.layers.25.mlp.experts.65.down_proj", "model.layers.25.mlp.experts.66.down_proj", "model.layers.25.mlp.experts.67.down_proj", "model.layers.25.mlp.experts.68.down_proj", "model.layers.25.mlp.experts.69.down_proj", "model.layers.25.mlp.experts.70.down_proj", "model.layers.25.mlp.experts.71.down_proj", "model.layers.25.mlp.experts.72.down_proj", "model.layers.25.mlp.experts.73.down_proj", "model.layers.25.mlp.experts.74.down_proj", "model.layers.25.mlp.experts.75.down_proj", "model.layers.25.mlp.experts.76.down_proj", "model.layers.25.mlp.experts.77.down_proj", "model.layers.25.mlp.experts.78.down_proj", "model.layers.25.mlp.experts.79.down_proj", "model.layers.25.mlp.experts.80.down_proj", "model.layers.25.mlp.experts.81.down_proj", "model.layers.25.mlp.experts.82.down_proj", "model.layers.25.mlp.experts.83.down_proj", "model.layers.25.mlp.experts.84.down_proj", "model.layers.25.mlp.experts.85.down_proj", "model.layers.25.mlp.experts.86.down_proj", "model.layers.25.mlp.experts.87.down_proj", "model.layers.25.mlp.experts.88.down_proj", "model.layers.25.mlp.experts.89.down_proj", "model.layers.25.mlp.experts.90.down_proj", "model.layers.25.mlp.experts.91.down_proj", "model.layers.25.mlp.experts.92.down_proj", "model.layers.25.mlp.experts.93.down_proj", "model.layers.25.mlp.experts.94.down_proj", "model.layers.25.mlp.experts.95.down_proj", "model.layers.25.mlp.experts.96.down_proj", "model.layers.25.mlp.experts.97.down_proj", "model.layers.25.mlp.experts.98.down_proj", "model.layers.25.mlp.experts.99.down_proj", "model.layers.25.mlp.experts.100.down_proj", "model.layers.25.mlp.experts.101.down_proj", "model.layers.25.mlp.experts.102.down_proj", "model.layers.25.mlp.experts.103.down_proj", "model.layers.25.mlp.experts.104.down_proj", "model.layers.25.mlp.experts.105.down_proj", "model.layers.25.mlp.experts.106.down_proj", "model.layers.25.mlp.experts.107.down_proj", "model.layers.25.mlp.experts.108.down_proj", "model.layers.25.mlp.experts.109.down_proj", "model.layers.25.mlp.experts.110.down_proj", "model.layers.25.mlp.experts.111.down_proj", "model.layers.25.mlp.experts.112.down_proj", "model.layers.25.mlp.experts.113.down_proj", "model.layers.25.mlp.experts.114.down_proj", "model.layers.25.mlp.experts.115.down_proj", "model.layers.25.mlp.experts.116.down_proj", "model.layers.25.mlp.experts.117.down_proj", "model.layers.25.mlp.experts.118.down_proj", "model.layers.25.mlp.experts.119.down_proj", "model.layers.25.mlp.experts.120.down_proj", "model.layers.25.mlp.experts.121.down_proj", "model.layers.25.mlp.experts.122.down_proj", "model.layers.25.mlp.experts.123.down_proj", "model.layers.25.mlp.experts.124.down_proj", "model.layers.25.mlp.experts.125.down_proj", "model.layers.25.mlp.experts.126.down_proj", "model.layers.25.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0017038768157362771, "dbits": 2214592512 } ] }, { "idx": 77, "layers": [ "model.layers.26.self_attn.q_proj", "model.layers.26.self_attn.k_proj", "model.layers.26.self_attn.v_proj", "model.layers.26.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00015331823378800236, "dbits": 109051904 } ] }, { "idx": 78, "layers": [ "model.layers.26.mlp.shared_experts.gate_proj", "model.layers.26.mlp.shared_experts.up_proj", "model.layers.26.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0012375562451779898, "dbits": 17301504 } ] }, { "idx": 79, "layers": [ "model.layers.26.mlp.experts.0.gate_proj", "model.layers.26.mlp.experts.1.gate_proj", "model.layers.26.mlp.experts.2.gate_proj", "model.layers.26.mlp.experts.3.gate_proj", "model.layers.26.mlp.experts.4.gate_proj", "model.layers.26.mlp.experts.5.gate_proj", "model.layers.26.mlp.experts.6.gate_proj", "model.layers.26.mlp.experts.7.gate_proj", "model.layers.26.mlp.experts.8.gate_proj", "model.layers.26.mlp.experts.9.gate_proj", "model.layers.26.mlp.experts.10.gate_proj", "model.layers.26.mlp.experts.11.gate_proj", "model.layers.26.mlp.experts.12.gate_proj", "model.layers.26.mlp.experts.13.gate_proj", "model.layers.26.mlp.experts.14.gate_proj", "model.layers.26.mlp.experts.15.gate_proj", "model.layers.26.mlp.experts.16.gate_proj", "model.layers.26.mlp.experts.17.gate_proj", "model.layers.26.mlp.experts.18.gate_proj", "model.layers.26.mlp.experts.19.gate_proj", "model.layers.26.mlp.experts.20.gate_proj", "model.layers.26.mlp.experts.21.gate_proj", "model.layers.26.mlp.experts.22.gate_proj", "model.layers.26.mlp.experts.23.gate_proj", "model.layers.26.mlp.experts.24.gate_proj", "model.layers.26.mlp.experts.25.gate_proj", "model.layers.26.mlp.experts.26.gate_proj", "model.layers.26.mlp.experts.27.gate_proj", "model.layers.26.mlp.experts.28.gate_proj", "model.layers.26.mlp.experts.29.gate_proj", "model.layers.26.mlp.experts.30.gate_proj", "model.layers.26.mlp.experts.31.gate_proj", "model.layers.26.mlp.experts.32.gate_proj", "model.layers.26.mlp.experts.33.gate_proj", "model.layers.26.mlp.experts.34.gate_proj", "model.layers.26.mlp.experts.35.gate_proj", "model.layers.26.mlp.experts.36.gate_proj", "model.layers.26.mlp.experts.37.gate_proj", "model.layers.26.mlp.experts.38.gate_proj", "model.layers.26.mlp.experts.39.gate_proj", "model.layers.26.mlp.experts.40.gate_proj", "model.layers.26.mlp.experts.41.gate_proj", "model.layers.26.mlp.experts.42.gate_proj", "model.layers.26.mlp.experts.43.gate_proj", "model.layers.26.mlp.experts.44.gate_proj", "model.layers.26.mlp.experts.45.gate_proj", "model.layers.26.mlp.experts.46.gate_proj", "model.layers.26.mlp.experts.47.gate_proj", "model.layers.26.mlp.experts.48.gate_proj", "model.layers.26.mlp.experts.49.gate_proj", "model.layers.26.mlp.experts.50.gate_proj", "model.layers.26.mlp.experts.51.gate_proj", "model.layers.26.mlp.experts.52.gate_proj", "model.layers.26.mlp.experts.53.gate_proj", "model.layers.26.mlp.experts.54.gate_proj", "model.layers.26.mlp.experts.55.gate_proj", "model.layers.26.mlp.experts.56.gate_proj", "model.layers.26.mlp.experts.57.gate_proj", "model.layers.26.mlp.experts.58.gate_proj", "model.layers.26.mlp.experts.59.gate_proj", "model.layers.26.mlp.experts.60.gate_proj", "model.layers.26.mlp.experts.61.gate_proj", "model.layers.26.mlp.experts.62.gate_proj", "model.layers.26.mlp.experts.63.gate_proj", "model.layers.26.mlp.experts.64.gate_proj", "model.layers.26.mlp.experts.65.gate_proj", "model.layers.26.mlp.experts.66.gate_proj", "model.layers.26.mlp.experts.67.gate_proj", "model.layers.26.mlp.experts.68.gate_proj", "model.layers.26.mlp.experts.69.gate_proj", "model.layers.26.mlp.experts.70.gate_proj", "model.layers.26.mlp.experts.71.gate_proj", "model.layers.26.mlp.experts.72.gate_proj", "model.layers.26.mlp.experts.73.gate_proj", "model.layers.26.mlp.experts.74.gate_proj", "model.layers.26.mlp.experts.75.gate_proj", "model.layers.26.mlp.experts.76.gate_proj", "model.layers.26.mlp.experts.77.gate_proj", "model.layers.26.mlp.experts.78.gate_proj", "model.layers.26.mlp.experts.79.gate_proj", "model.layers.26.mlp.experts.80.gate_proj", "model.layers.26.mlp.experts.81.gate_proj", "model.layers.26.mlp.experts.82.gate_proj", "model.layers.26.mlp.experts.83.gate_proj", "model.layers.26.mlp.experts.84.gate_proj", "model.layers.26.mlp.experts.85.gate_proj", "model.layers.26.mlp.experts.86.gate_proj", "model.layers.26.mlp.experts.87.gate_proj", "model.layers.26.mlp.experts.88.gate_proj", "model.layers.26.mlp.experts.89.gate_proj", "model.layers.26.mlp.experts.90.gate_proj", "model.layers.26.mlp.experts.91.gate_proj", "model.layers.26.mlp.experts.92.gate_proj", "model.layers.26.mlp.experts.93.gate_proj", "model.layers.26.mlp.experts.94.gate_proj", "model.layers.26.mlp.experts.95.gate_proj", "model.layers.26.mlp.experts.96.gate_proj", "model.layers.26.mlp.experts.97.gate_proj", "model.layers.26.mlp.experts.98.gate_proj", "model.layers.26.mlp.experts.99.gate_proj", "model.layers.26.mlp.experts.100.gate_proj", "model.layers.26.mlp.experts.101.gate_proj", "model.layers.26.mlp.experts.102.gate_proj", "model.layers.26.mlp.experts.103.gate_proj", "model.layers.26.mlp.experts.104.gate_proj", "model.layers.26.mlp.experts.105.gate_proj", "model.layers.26.mlp.experts.106.gate_proj", "model.layers.26.mlp.experts.107.gate_proj", "model.layers.26.mlp.experts.108.gate_proj", "model.layers.26.mlp.experts.109.gate_proj", "model.layers.26.mlp.experts.110.gate_proj", "model.layers.26.mlp.experts.111.gate_proj", "model.layers.26.mlp.experts.112.gate_proj", "model.layers.26.mlp.experts.113.gate_proj", "model.layers.26.mlp.experts.114.gate_proj", "model.layers.26.mlp.experts.115.gate_proj", "model.layers.26.mlp.experts.116.gate_proj", "model.layers.26.mlp.experts.117.gate_proj", "model.layers.26.mlp.experts.118.gate_proj", "model.layers.26.mlp.experts.119.gate_proj", "model.layers.26.mlp.experts.120.gate_proj", "model.layers.26.mlp.experts.121.gate_proj", "model.layers.26.mlp.experts.122.gate_proj", "model.layers.26.mlp.experts.123.gate_proj", "model.layers.26.mlp.experts.124.gate_proj", "model.layers.26.mlp.experts.125.gate_proj", "model.layers.26.mlp.experts.126.gate_proj", "model.layers.26.mlp.experts.127.gate_proj", "model.layers.26.mlp.experts.0.up_proj", "model.layers.26.mlp.experts.1.up_proj", "model.layers.26.mlp.experts.2.up_proj", "model.layers.26.mlp.experts.3.up_proj", "model.layers.26.mlp.experts.4.up_proj", "model.layers.26.mlp.experts.5.up_proj", "model.layers.26.mlp.experts.6.up_proj", "model.layers.26.mlp.experts.7.up_proj", "model.layers.26.mlp.experts.8.up_proj", "model.layers.26.mlp.experts.9.up_proj", "model.layers.26.mlp.experts.10.up_proj", "model.layers.26.mlp.experts.11.up_proj", "model.layers.26.mlp.experts.12.up_proj", "model.layers.26.mlp.experts.13.up_proj", "model.layers.26.mlp.experts.14.up_proj", "model.layers.26.mlp.experts.15.up_proj", "model.layers.26.mlp.experts.16.up_proj", "model.layers.26.mlp.experts.17.up_proj", "model.layers.26.mlp.experts.18.up_proj", "model.layers.26.mlp.experts.19.up_proj", "model.layers.26.mlp.experts.20.up_proj", "model.layers.26.mlp.experts.21.up_proj", "model.layers.26.mlp.experts.22.up_proj", "model.layers.26.mlp.experts.23.up_proj", "model.layers.26.mlp.experts.24.up_proj", "model.layers.26.mlp.experts.25.up_proj", "model.layers.26.mlp.experts.26.up_proj", "model.layers.26.mlp.experts.27.up_proj", "model.layers.26.mlp.experts.28.up_proj", "model.layers.26.mlp.experts.29.up_proj", "model.layers.26.mlp.experts.30.up_proj", "model.layers.26.mlp.experts.31.up_proj", "model.layers.26.mlp.experts.32.up_proj", "model.layers.26.mlp.experts.33.up_proj", "model.layers.26.mlp.experts.34.up_proj", "model.layers.26.mlp.experts.35.up_proj", "model.layers.26.mlp.experts.36.up_proj", "model.layers.26.mlp.experts.37.up_proj", "model.layers.26.mlp.experts.38.up_proj", "model.layers.26.mlp.experts.39.up_proj", "model.layers.26.mlp.experts.40.up_proj", "model.layers.26.mlp.experts.41.up_proj", "model.layers.26.mlp.experts.42.up_proj", "model.layers.26.mlp.experts.43.up_proj", "model.layers.26.mlp.experts.44.up_proj", "model.layers.26.mlp.experts.45.up_proj", "model.layers.26.mlp.experts.46.up_proj", "model.layers.26.mlp.experts.47.up_proj", "model.layers.26.mlp.experts.48.up_proj", "model.layers.26.mlp.experts.49.up_proj", "model.layers.26.mlp.experts.50.up_proj", "model.layers.26.mlp.experts.51.up_proj", "model.layers.26.mlp.experts.52.up_proj", "model.layers.26.mlp.experts.53.up_proj", "model.layers.26.mlp.experts.54.up_proj", "model.layers.26.mlp.experts.55.up_proj", "model.layers.26.mlp.experts.56.up_proj", "model.layers.26.mlp.experts.57.up_proj", "model.layers.26.mlp.experts.58.up_proj", "model.layers.26.mlp.experts.59.up_proj", "model.layers.26.mlp.experts.60.up_proj", "model.layers.26.mlp.experts.61.up_proj", "model.layers.26.mlp.experts.62.up_proj", "model.layers.26.mlp.experts.63.up_proj", "model.layers.26.mlp.experts.64.up_proj", "model.layers.26.mlp.experts.65.up_proj", "model.layers.26.mlp.experts.66.up_proj", "model.layers.26.mlp.experts.67.up_proj", "model.layers.26.mlp.experts.68.up_proj", "model.layers.26.mlp.experts.69.up_proj", "model.layers.26.mlp.experts.70.up_proj", "model.layers.26.mlp.experts.71.up_proj", "model.layers.26.mlp.experts.72.up_proj", "model.layers.26.mlp.experts.73.up_proj", "model.layers.26.mlp.experts.74.up_proj", "model.layers.26.mlp.experts.75.up_proj", "model.layers.26.mlp.experts.76.up_proj", "model.layers.26.mlp.experts.77.up_proj", "model.layers.26.mlp.experts.78.up_proj", "model.layers.26.mlp.experts.79.up_proj", "model.layers.26.mlp.experts.80.up_proj", "model.layers.26.mlp.experts.81.up_proj", "model.layers.26.mlp.experts.82.up_proj", "model.layers.26.mlp.experts.83.up_proj", "model.layers.26.mlp.experts.84.up_proj", "model.layers.26.mlp.experts.85.up_proj", "model.layers.26.mlp.experts.86.up_proj", "model.layers.26.mlp.experts.87.up_proj", "model.layers.26.mlp.experts.88.up_proj", "model.layers.26.mlp.experts.89.up_proj", "model.layers.26.mlp.experts.90.up_proj", "model.layers.26.mlp.experts.91.up_proj", "model.layers.26.mlp.experts.92.up_proj", "model.layers.26.mlp.experts.93.up_proj", "model.layers.26.mlp.experts.94.up_proj", "model.layers.26.mlp.experts.95.up_proj", "model.layers.26.mlp.experts.96.up_proj", "model.layers.26.mlp.experts.97.up_proj", "model.layers.26.mlp.experts.98.up_proj", "model.layers.26.mlp.experts.99.up_proj", "model.layers.26.mlp.experts.100.up_proj", "model.layers.26.mlp.experts.101.up_proj", "model.layers.26.mlp.experts.102.up_proj", "model.layers.26.mlp.experts.103.up_proj", "model.layers.26.mlp.experts.104.up_proj", "model.layers.26.mlp.experts.105.up_proj", "model.layers.26.mlp.experts.106.up_proj", "model.layers.26.mlp.experts.107.up_proj", "model.layers.26.mlp.experts.108.up_proj", "model.layers.26.mlp.experts.109.up_proj", "model.layers.26.mlp.experts.110.up_proj", "model.layers.26.mlp.experts.111.up_proj", "model.layers.26.mlp.experts.112.up_proj", "model.layers.26.mlp.experts.113.up_proj", "model.layers.26.mlp.experts.114.up_proj", "model.layers.26.mlp.experts.115.up_proj", "model.layers.26.mlp.experts.116.up_proj", "model.layers.26.mlp.experts.117.up_proj", "model.layers.26.mlp.experts.118.up_proj", "model.layers.26.mlp.experts.119.up_proj", "model.layers.26.mlp.experts.120.up_proj", "model.layers.26.mlp.experts.121.up_proj", "model.layers.26.mlp.experts.122.up_proj", "model.layers.26.mlp.experts.123.up_proj", "model.layers.26.mlp.experts.124.up_proj", "model.layers.26.mlp.experts.125.up_proj", "model.layers.26.mlp.experts.126.up_proj", "model.layers.26.mlp.experts.127.up_proj", "model.layers.26.mlp.experts.0.down_proj", "model.layers.26.mlp.experts.1.down_proj", "model.layers.26.mlp.experts.2.down_proj", "model.layers.26.mlp.experts.3.down_proj", "model.layers.26.mlp.experts.4.down_proj", "model.layers.26.mlp.experts.5.down_proj", "model.layers.26.mlp.experts.6.down_proj", "model.layers.26.mlp.experts.7.down_proj", "model.layers.26.mlp.experts.8.down_proj", "model.layers.26.mlp.experts.9.down_proj", "model.layers.26.mlp.experts.10.down_proj", "model.layers.26.mlp.experts.11.down_proj", "model.layers.26.mlp.experts.12.down_proj", "model.layers.26.mlp.experts.13.down_proj", "model.layers.26.mlp.experts.14.down_proj", "model.layers.26.mlp.experts.15.down_proj", "model.layers.26.mlp.experts.16.down_proj", "model.layers.26.mlp.experts.17.down_proj", "model.layers.26.mlp.experts.18.down_proj", "model.layers.26.mlp.experts.19.down_proj", "model.layers.26.mlp.experts.20.down_proj", "model.layers.26.mlp.experts.21.down_proj", "model.layers.26.mlp.experts.22.down_proj", "model.layers.26.mlp.experts.23.down_proj", "model.layers.26.mlp.experts.24.down_proj", "model.layers.26.mlp.experts.25.down_proj", "model.layers.26.mlp.experts.26.down_proj", "model.layers.26.mlp.experts.27.down_proj", "model.layers.26.mlp.experts.28.down_proj", "model.layers.26.mlp.experts.29.down_proj", "model.layers.26.mlp.experts.30.down_proj", "model.layers.26.mlp.experts.31.down_proj", "model.layers.26.mlp.experts.32.down_proj", "model.layers.26.mlp.experts.33.down_proj", "model.layers.26.mlp.experts.34.down_proj", "model.layers.26.mlp.experts.35.down_proj", "model.layers.26.mlp.experts.36.down_proj", "model.layers.26.mlp.experts.37.down_proj", "model.layers.26.mlp.experts.38.down_proj", "model.layers.26.mlp.experts.39.down_proj", "model.layers.26.mlp.experts.40.down_proj", "model.layers.26.mlp.experts.41.down_proj", "model.layers.26.mlp.experts.42.down_proj", "model.layers.26.mlp.experts.43.down_proj", "model.layers.26.mlp.experts.44.down_proj", "model.layers.26.mlp.experts.45.down_proj", "model.layers.26.mlp.experts.46.down_proj", "model.layers.26.mlp.experts.47.down_proj", "model.layers.26.mlp.experts.48.down_proj", "model.layers.26.mlp.experts.49.down_proj", "model.layers.26.mlp.experts.50.down_proj", "model.layers.26.mlp.experts.51.down_proj", "model.layers.26.mlp.experts.52.down_proj", "model.layers.26.mlp.experts.53.down_proj", "model.layers.26.mlp.experts.54.down_proj", "model.layers.26.mlp.experts.55.down_proj", "model.layers.26.mlp.experts.56.down_proj", "model.layers.26.mlp.experts.57.down_proj", "model.layers.26.mlp.experts.58.down_proj", "model.layers.26.mlp.experts.59.down_proj", "model.layers.26.mlp.experts.60.down_proj", "model.layers.26.mlp.experts.61.down_proj", "model.layers.26.mlp.experts.62.down_proj", "model.layers.26.mlp.experts.63.down_proj", "model.layers.26.mlp.experts.64.down_proj", "model.layers.26.mlp.experts.65.down_proj", "model.layers.26.mlp.experts.66.down_proj", "model.layers.26.mlp.experts.67.down_proj", "model.layers.26.mlp.experts.68.down_proj", "model.layers.26.mlp.experts.69.down_proj", "model.layers.26.mlp.experts.70.down_proj", "model.layers.26.mlp.experts.71.down_proj", "model.layers.26.mlp.experts.72.down_proj", "model.layers.26.mlp.experts.73.down_proj", "model.layers.26.mlp.experts.74.down_proj", "model.layers.26.mlp.experts.75.down_proj", "model.layers.26.mlp.experts.76.down_proj", "model.layers.26.mlp.experts.77.down_proj", "model.layers.26.mlp.experts.78.down_proj", "model.layers.26.mlp.experts.79.down_proj", "model.layers.26.mlp.experts.80.down_proj", "model.layers.26.mlp.experts.81.down_proj", "model.layers.26.mlp.experts.82.down_proj", "model.layers.26.mlp.experts.83.down_proj", "model.layers.26.mlp.experts.84.down_proj", "model.layers.26.mlp.experts.85.down_proj", "model.layers.26.mlp.experts.86.down_proj", "model.layers.26.mlp.experts.87.down_proj", "model.layers.26.mlp.experts.88.down_proj", "model.layers.26.mlp.experts.89.down_proj", "model.layers.26.mlp.experts.90.down_proj", "model.layers.26.mlp.experts.91.down_proj", "model.layers.26.mlp.experts.92.down_proj", "model.layers.26.mlp.experts.93.down_proj", "model.layers.26.mlp.experts.94.down_proj", "model.layers.26.mlp.experts.95.down_proj", "model.layers.26.mlp.experts.96.down_proj", "model.layers.26.mlp.experts.97.down_proj", "model.layers.26.mlp.experts.98.down_proj", "model.layers.26.mlp.experts.99.down_proj", "model.layers.26.mlp.experts.100.down_proj", "model.layers.26.mlp.experts.101.down_proj", "model.layers.26.mlp.experts.102.down_proj", "model.layers.26.mlp.experts.103.down_proj", "model.layers.26.mlp.experts.104.down_proj", "model.layers.26.mlp.experts.105.down_proj", "model.layers.26.mlp.experts.106.down_proj", "model.layers.26.mlp.experts.107.down_proj", "model.layers.26.mlp.experts.108.down_proj", "model.layers.26.mlp.experts.109.down_proj", "model.layers.26.mlp.experts.110.down_proj", "model.layers.26.mlp.experts.111.down_proj", "model.layers.26.mlp.experts.112.down_proj", "model.layers.26.mlp.experts.113.down_proj", "model.layers.26.mlp.experts.114.down_proj", "model.layers.26.mlp.experts.115.down_proj", "model.layers.26.mlp.experts.116.down_proj", "model.layers.26.mlp.experts.117.down_proj", "model.layers.26.mlp.experts.118.down_proj", "model.layers.26.mlp.experts.119.down_proj", "model.layers.26.mlp.experts.120.down_proj", "model.layers.26.mlp.experts.121.down_proj", "model.layers.26.mlp.experts.122.down_proj", "model.layers.26.mlp.experts.123.down_proj", "model.layers.26.mlp.experts.124.down_proj", "model.layers.26.mlp.experts.125.down_proj", "model.layers.26.mlp.experts.126.down_proj", "model.layers.26.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0014330691657960248, "dbits": 2214592512 } ] }, { "idx": 80, "layers": [ "model.layers.27.self_attn.q_proj", "model.layers.27.self_attn.k_proj", "model.layers.27.self_attn.v_proj", "model.layers.27.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0006313558667898067, "dbits": 109051904 } ] }, { "idx": 81, "layers": [ "model.layers.27.mlp.shared_experts.gate_proj", "model.layers.27.mlp.shared_experts.up_proj", "model.layers.27.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0005240221507847309, "dbits": 17301504 } ] }, { "idx": 82, "layers": [ "model.layers.27.mlp.experts.0.gate_proj", "model.layers.27.mlp.experts.1.gate_proj", "model.layers.27.mlp.experts.2.gate_proj", "model.layers.27.mlp.experts.3.gate_proj", "model.layers.27.mlp.experts.4.gate_proj", "model.layers.27.mlp.experts.5.gate_proj", "model.layers.27.mlp.experts.6.gate_proj", "model.layers.27.mlp.experts.7.gate_proj", "model.layers.27.mlp.experts.8.gate_proj", "model.layers.27.mlp.experts.9.gate_proj", "model.layers.27.mlp.experts.10.gate_proj", "model.layers.27.mlp.experts.11.gate_proj", "model.layers.27.mlp.experts.12.gate_proj", "model.layers.27.mlp.experts.13.gate_proj", "model.layers.27.mlp.experts.14.gate_proj", "model.layers.27.mlp.experts.15.gate_proj", "model.layers.27.mlp.experts.16.gate_proj", "model.layers.27.mlp.experts.17.gate_proj", "model.layers.27.mlp.experts.18.gate_proj", "model.layers.27.mlp.experts.19.gate_proj", "model.layers.27.mlp.experts.20.gate_proj", "model.layers.27.mlp.experts.21.gate_proj", "model.layers.27.mlp.experts.22.gate_proj", "model.layers.27.mlp.experts.23.gate_proj", "model.layers.27.mlp.experts.24.gate_proj", "model.layers.27.mlp.experts.25.gate_proj", "model.layers.27.mlp.experts.26.gate_proj", "model.layers.27.mlp.experts.27.gate_proj", "model.layers.27.mlp.experts.28.gate_proj", "model.layers.27.mlp.experts.29.gate_proj", "model.layers.27.mlp.experts.30.gate_proj", "model.layers.27.mlp.experts.31.gate_proj", "model.layers.27.mlp.experts.32.gate_proj", "model.layers.27.mlp.experts.33.gate_proj", "model.layers.27.mlp.experts.34.gate_proj", "model.layers.27.mlp.experts.35.gate_proj", "model.layers.27.mlp.experts.36.gate_proj", "model.layers.27.mlp.experts.37.gate_proj", "model.layers.27.mlp.experts.38.gate_proj", "model.layers.27.mlp.experts.39.gate_proj", "model.layers.27.mlp.experts.40.gate_proj", "model.layers.27.mlp.experts.41.gate_proj", "model.layers.27.mlp.experts.42.gate_proj", "model.layers.27.mlp.experts.43.gate_proj", "model.layers.27.mlp.experts.44.gate_proj", "model.layers.27.mlp.experts.45.gate_proj", "model.layers.27.mlp.experts.46.gate_proj", "model.layers.27.mlp.experts.47.gate_proj", "model.layers.27.mlp.experts.48.gate_proj", "model.layers.27.mlp.experts.49.gate_proj", "model.layers.27.mlp.experts.50.gate_proj", "model.layers.27.mlp.experts.51.gate_proj", "model.layers.27.mlp.experts.52.gate_proj", "model.layers.27.mlp.experts.53.gate_proj", "model.layers.27.mlp.experts.54.gate_proj", "model.layers.27.mlp.experts.55.gate_proj", "model.layers.27.mlp.experts.56.gate_proj", "model.layers.27.mlp.experts.57.gate_proj", "model.layers.27.mlp.experts.58.gate_proj", "model.layers.27.mlp.experts.59.gate_proj", "model.layers.27.mlp.experts.60.gate_proj", "model.layers.27.mlp.experts.61.gate_proj", "model.layers.27.mlp.experts.62.gate_proj", "model.layers.27.mlp.experts.63.gate_proj", "model.layers.27.mlp.experts.64.gate_proj", "model.layers.27.mlp.experts.65.gate_proj", "model.layers.27.mlp.experts.66.gate_proj", "model.layers.27.mlp.experts.67.gate_proj", "model.layers.27.mlp.experts.68.gate_proj", "model.layers.27.mlp.experts.69.gate_proj", "model.layers.27.mlp.experts.70.gate_proj", "model.layers.27.mlp.experts.71.gate_proj", "model.layers.27.mlp.experts.72.gate_proj", "model.layers.27.mlp.experts.73.gate_proj", "model.layers.27.mlp.experts.74.gate_proj", "model.layers.27.mlp.experts.75.gate_proj", "model.layers.27.mlp.experts.76.gate_proj", "model.layers.27.mlp.experts.77.gate_proj", "model.layers.27.mlp.experts.78.gate_proj", "model.layers.27.mlp.experts.79.gate_proj", "model.layers.27.mlp.experts.80.gate_proj", "model.layers.27.mlp.experts.81.gate_proj", "model.layers.27.mlp.experts.82.gate_proj", "model.layers.27.mlp.experts.83.gate_proj", "model.layers.27.mlp.experts.84.gate_proj", "model.layers.27.mlp.experts.85.gate_proj", "model.layers.27.mlp.experts.86.gate_proj", "model.layers.27.mlp.experts.87.gate_proj", "model.layers.27.mlp.experts.88.gate_proj", "model.layers.27.mlp.experts.89.gate_proj", "model.layers.27.mlp.experts.90.gate_proj", "model.layers.27.mlp.experts.91.gate_proj", "model.layers.27.mlp.experts.92.gate_proj", "model.layers.27.mlp.experts.93.gate_proj", "model.layers.27.mlp.experts.94.gate_proj", "model.layers.27.mlp.experts.95.gate_proj", "model.layers.27.mlp.experts.96.gate_proj", "model.layers.27.mlp.experts.97.gate_proj", "model.layers.27.mlp.experts.98.gate_proj", "model.layers.27.mlp.experts.99.gate_proj", "model.layers.27.mlp.experts.100.gate_proj", "model.layers.27.mlp.experts.101.gate_proj", "model.layers.27.mlp.experts.102.gate_proj", "model.layers.27.mlp.experts.103.gate_proj", "model.layers.27.mlp.experts.104.gate_proj", "model.layers.27.mlp.experts.105.gate_proj", "model.layers.27.mlp.experts.106.gate_proj", "model.layers.27.mlp.experts.107.gate_proj", "model.layers.27.mlp.experts.108.gate_proj", "model.layers.27.mlp.experts.109.gate_proj", "model.layers.27.mlp.experts.110.gate_proj", "model.layers.27.mlp.experts.111.gate_proj", "model.layers.27.mlp.experts.112.gate_proj", "model.layers.27.mlp.experts.113.gate_proj", "model.layers.27.mlp.experts.114.gate_proj", "model.layers.27.mlp.experts.115.gate_proj", "model.layers.27.mlp.experts.116.gate_proj", "model.layers.27.mlp.experts.117.gate_proj", "model.layers.27.mlp.experts.118.gate_proj", "model.layers.27.mlp.experts.119.gate_proj", "model.layers.27.mlp.experts.120.gate_proj", "model.layers.27.mlp.experts.121.gate_proj", "model.layers.27.mlp.experts.122.gate_proj", "model.layers.27.mlp.experts.123.gate_proj", "model.layers.27.mlp.experts.124.gate_proj", "model.layers.27.mlp.experts.125.gate_proj", "model.layers.27.mlp.experts.126.gate_proj", "model.layers.27.mlp.experts.127.gate_proj", "model.layers.27.mlp.experts.0.up_proj", "model.layers.27.mlp.experts.1.up_proj", "model.layers.27.mlp.experts.2.up_proj", "model.layers.27.mlp.experts.3.up_proj", "model.layers.27.mlp.experts.4.up_proj", "model.layers.27.mlp.experts.5.up_proj", "model.layers.27.mlp.experts.6.up_proj", "model.layers.27.mlp.experts.7.up_proj", "model.layers.27.mlp.experts.8.up_proj", "model.layers.27.mlp.experts.9.up_proj", "model.layers.27.mlp.experts.10.up_proj", "model.layers.27.mlp.experts.11.up_proj", "model.layers.27.mlp.experts.12.up_proj", "model.layers.27.mlp.experts.13.up_proj", "model.layers.27.mlp.experts.14.up_proj", "model.layers.27.mlp.experts.15.up_proj", "model.layers.27.mlp.experts.16.up_proj", "model.layers.27.mlp.experts.17.up_proj", "model.layers.27.mlp.experts.18.up_proj", "model.layers.27.mlp.experts.19.up_proj", "model.layers.27.mlp.experts.20.up_proj", "model.layers.27.mlp.experts.21.up_proj", "model.layers.27.mlp.experts.22.up_proj", "model.layers.27.mlp.experts.23.up_proj", "model.layers.27.mlp.experts.24.up_proj", "model.layers.27.mlp.experts.25.up_proj", "model.layers.27.mlp.experts.26.up_proj", "model.layers.27.mlp.experts.27.up_proj", "model.layers.27.mlp.experts.28.up_proj", "model.layers.27.mlp.experts.29.up_proj", "model.layers.27.mlp.experts.30.up_proj", "model.layers.27.mlp.experts.31.up_proj", "model.layers.27.mlp.experts.32.up_proj", "model.layers.27.mlp.experts.33.up_proj", "model.layers.27.mlp.experts.34.up_proj", "model.layers.27.mlp.experts.35.up_proj", "model.layers.27.mlp.experts.36.up_proj", "model.layers.27.mlp.experts.37.up_proj", "model.layers.27.mlp.experts.38.up_proj", "model.layers.27.mlp.experts.39.up_proj", "model.layers.27.mlp.experts.40.up_proj", "model.layers.27.mlp.experts.41.up_proj", "model.layers.27.mlp.experts.42.up_proj", "model.layers.27.mlp.experts.43.up_proj", "model.layers.27.mlp.experts.44.up_proj", "model.layers.27.mlp.experts.45.up_proj", "model.layers.27.mlp.experts.46.up_proj", "model.layers.27.mlp.experts.47.up_proj", "model.layers.27.mlp.experts.48.up_proj", "model.layers.27.mlp.experts.49.up_proj", "model.layers.27.mlp.experts.50.up_proj", "model.layers.27.mlp.experts.51.up_proj", "model.layers.27.mlp.experts.52.up_proj", "model.layers.27.mlp.experts.53.up_proj", "model.layers.27.mlp.experts.54.up_proj", "model.layers.27.mlp.experts.55.up_proj", "model.layers.27.mlp.experts.56.up_proj", "model.layers.27.mlp.experts.57.up_proj", "model.layers.27.mlp.experts.58.up_proj", "model.layers.27.mlp.experts.59.up_proj", "model.layers.27.mlp.experts.60.up_proj", "model.layers.27.mlp.experts.61.up_proj", "model.layers.27.mlp.experts.62.up_proj", "model.layers.27.mlp.experts.63.up_proj", "model.layers.27.mlp.experts.64.up_proj", "model.layers.27.mlp.experts.65.up_proj", "model.layers.27.mlp.experts.66.up_proj", "model.layers.27.mlp.experts.67.up_proj", "model.layers.27.mlp.experts.68.up_proj", "model.layers.27.mlp.experts.69.up_proj", "model.layers.27.mlp.experts.70.up_proj", "model.layers.27.mlp.experts.71.up_proj", "model.layers.27.mlp.experts.72.up_proj", "model.layers.27.mlp.experts.73.up_proj", "model.layers.27.mlp.experts.74.up_proj", "model.layers.27.mlp.experts.75.up_proj", "model.layers.27.mlp.experts.76.up_proj", "model.layers.27.mlp.experts.77.up_proj", "model.layers.27.mlp.experts.78.up_proj", "model.layers.27.mlp.experts.79.up_proj", "model.layers.27.mlp.experts.80.up_proj", "model.layers.27.mlp.experts.81.up_proj", "model.layers.27.mlp.experts.82.up_proj", "model.layers.27.mlp.experts.83.up_proj", "model.layers.27.mlp.experts.84.up_proj", "model.layers.27.mlp.experts.85.up_proj", "model.layers.27.mlp.experts.86.up_proj", "model.layers.27.mlp.experts.87.up_proj", "model.layers.27.mlp.experts.88.up_proj", "model.layers.27.mlp.experts.89.up_proj", "model.layers.27.mlp.experts.90.up_proj", "model.layers.27.mlp.experts.91.up_proj", "model.layers.27.mlp.experts.92.up_proj", "model.layers.27.mlp.experts.93.up_proj", "model.layers.27.mlp.experts.94.up_proj", "model.layers.27.mlp.experts.95.up_proj", "model.layers.27.mlp.experts.96.up_proj", "model.layers.27.mlp.experts.97.up_proj", "model.layers.27.mlp.experts.98.up_proj", "model.layers.27.mlp.experts.99.up_proj", "model.layers.27.mlp.experts.100.up_proj", "model.layers.27.mlp.experts.101.up_proj", "model.layers.27.mlp.experts.102.up_proj", "model.layers.27.mlp.experts.103.up_proj", "model.layers.27.mlp.experts.104.up_proj", "model.layers.27.mlp.experts.105.up_proj", "model.layers.27.mlp.experts.106.up_proj", "model.layers.27.mlp.experts.107.up_proj", "model.layers.27.mlp.experts.108.up_proj", "model.layers.27.mlp.experts.109.up_proj", "model.layers.27.mlp.experts.110.up_proj", "model.layers.27.mlp.experts.111.up_proj", "model.layers.27.mlp.experts.112.up_proj", "model.layers.27.mlp.experts.113.up_proj", "model.layers.27.mlp.experts.114.up_proj", "model.layers.27.mlp.experts.115.up_proj", "model.layers.27.mlp.experts.116.up_proj", "model.layers.27.mlp.experts.117.up_proj", "model.layers.27.mlp.experts.118.up_proj", "model.layers.27.mlp.experts.119.up_proj", "model.layers.27.mlp.experts.120.up_proj", "model.layers.27.mlp.experts.121.up_proj", "model.layers.27.mlp.experts.122.up_proj", "model.layers.27.mlp.experts.123.up_proj", "model.layers.27.mlp.experts.124.up_proj", "model.layers.27.mlp.experts.125.up_proj", "model.layers.27.mlp.experts.126.up_proj", "model.layers.27.mlp.experts.127.up_proj", "model.layers.27.mlp.experts.0.down_proj", "model.layers.27.mlp.experts.1.down_proj", "model.layers.27.mlp.experts.2.down_proj", "model.layers.27.mlp.experts.3.down_proj", "model.layers.27.mlp.experts.4.down_proj", "model.layers.27.mlp.experts.5.down_proj", "model.layers.27.mlp.experts.6.down_proj", "model.layers.27.mlp.experts.7.down_proj", "model.layers.27.mlp.experts.8.down_proj", "model.layers.27.mlp.experts.9.down_proj", "model.layers.27.mlp.experts.10.down_proj", "model.layers.27.mlp.experts.11.down_proj", "model.layers.27.mlp.experts.12.down_proj", "model.layers.27.mlp.experts.13.down_proj", "model.layers.27.mlp.experts.14.down_proj", "model.layers.27.mlp.experts.15.down_proj", "model.layers.27.mlp.experts.16.down_proj", "model.layers.27.mlp.experts.17.down_proj", "model.layers.27.mlp.experts.18.down_proj", "model.layers.27.mlp.experts.19.down_proj", "model.layers.27.mlp.experts.20.down_proj", "model.layers.27.mlp.experts.21.down_proj", "model.layers.27.mlp.experts.22.down_proj", "model.layers.27.mlp.experts.23.down_proj", "model.layers.27.mlp.experts.24.down_proj", "model.layers.27.mlp.experts.25.down_proj", "model.layers.27.mlp.experts.26.down_proj", "model.layers.27.mlp.experts.27.down_proj", "model.layers.27.mlp.experts.28.down_proj", "model.layers.27.mlp.experts.29.down_proj", "model.layers.27.mlp.experts.30.down_proj", "model.layers.27.mlp.experts.31.down_proj", "model.layers.27.mlp.experts.32.down_proj", "model.layers.27.mlp.experts.33.down_proj", "model.layers.27.mlp.experts.34.down_proj", "model.layers.27.mlp.experts.35.down_proj", "model.layers.27.mlp.experts.36.down_proj", "model.layers.27.mlp.experts.37.down_proj", "model.layers.27.mlp.experts.38.down_proj", "model.layers.27.mlp.experts.39.down_proj", "model.layers.27.mlp.experts.40.down_proj", "model.layers.27.mlp.experts.41.down_proj", "model.layers.27.mlp.experts.42.down_proj", "model.layers.27.mlp.experts.43.down_proj", "model.layers.27.mlp.experts.44.down_proj", "model.layers.27.mlp.experts.45.down_proj", "model.layers.27.mlp.experts.46.down_proj", "model.layers.27.mlp.experts.47.down_proj", "model.layers.27.mlp.experts.48.down_proj", "model.layers.27.mlp.experts.49.down_proj", "model.layers.27.mlp.experts.50.down_proj", "model.layers.27.mlp.experts.51.down_proj", "model.layers.27.mlp.experts.52.down_proj", "model.layers.27.mlp.experts.53.down_proj", "model.layers.27.mlp.experts.54.down_proj", "model.layers.27.mlp.experts.55.down_proj", "model.layers.27.mlp.experts.56.down_proj", "model.layers.27.mlp.experts.57.down_proj", "model.layers.27.mlp.experts.58.down_proj", "model.layers.27.mlp.experts.59.down_proj", "model.layers.27.mlp.experts.60.down_proj", "model.layers.27.mlp.experts.61.down_proj", "model.layers.27.mlp.experts.62.down_proj", "model.layers.27.mlp.experts.63.down_proj", "model.layers.27.mlp.experts.64.down_proj", "model.layers.27.mlp.experts.65.down_proj", "model.layers.27.mlp.experts.66.down_proj", "model.layers.27.mlp.experts.67.down_proj", "model.layers.27.mlp.experts.68.down_proj", "model.layers.27.mlp.experts.69.down_proj", "model.layers.27.mlp.experts.70.down_proj", "model.layers.27.mlp.experts.71.down_proj", "model.layers.27.mlp.experts.72.down_proj", "model.layers.27.mlp.experts.73.down_proj", "model.layers.27.mlp.experts.74.down_proj", "model.layers.27.mlp.experts.75.down_proj", "model.layers.27.mlp.experts.76.down_proj", "model.layers.27.mlp.experts.77.down_proj", "model.layers.27.mlp.experts.78.down_proj", "model.layers.27.mlp.experts.79.down_proj", "model.layers.27.mlp.experts.80.down_proj", "model.layers.27.mlp.experts.81.down_proj", "model.layers.27.mlp.experts.82.down_proj", "model.layers.27.mlp.experts.83.down_proj", "model.layers.27.mlp.experts.84.down_proj", "model.layers.27.mlp.experts.85.down_proj", "model.layers.27.mlp.experts.86.down_proj", "model.layers.27.mlp.experts.87.down_proj", "model.layers.27.mlp.experts.88.down_proj", "model.layers.27.mlp.experts.89.down_proj", "model.layers.27.mlp.experts.90.down_proj", "model.layers.27.mlp.experts.91.down_proj", "model.layers.27.mlp.experts.92.down_proj", "model.layers.27.mlp.experts.93.down_proj", "model.layers.27.mlp.experts.94.down_proj", "model.layers.27.mlp.experts.95.down_proj", "model.layers.27.mlp.experts.96.down_proj", "model.layers.27.mlp.experts.97.down_proj", "model.layers.27.mlp.experts.98.down_proj", "model.layers.27.mlp.experts.99.down_proj", "model.layers.27.mlp.experts.100.down_proj", "model.layers.27.mlp.experts.101.down_proj", "model.layers.27.mlp.experts.102.down_proj", "model.layers.27.mlp.experts.103.down_proj", "model.layers.27.mlp.experts.104.down_proj", "model.layers.27.mlp.experts.105.down_proj", "model.layers.27.mlp.experts.106.down_proj", "model.layers.27.mlp.experts.107.down_proj", "model.layers.27.mlp.experts.108.down_proj", "model.layers.27.mlp.experts.109.down_proj", "model.layers.27.mlp.experts.110.down_proj", "model.layers.27.mlp.experts.111.down_proj", "model.layers.27.mlp.experts.112.down_proj", "model.layers.27.mlp.experts.113.down_proj", "model.layers.27.mlp.experts.114.down_proj", "model.layers.27.mlp.experts.115.down_proj", "model.layers.27.mlp.experts.116.down_proj", "model.layers.27.mlp.experts.117.down_proj", "model.layers.27.mlp.experts.118.down_proj", "model.layers.27.mlp.experts.119.down_proj", "model.layers.27.mlp.experts.120.down_proj", "model.layers.27.mlp.experts.121.down_proj", "model.layers.27.mlp.experts.122.down_proj", "model.layers.27.mlp.experts.123.down_proj", "model.layers.27.mlp.experts.124.down_proj", "model.layers.27.mlp.experts.125.down_proj", "model.layers.27.mlp.experts.126.down_proj", "model.layers.27.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.001143829990178341, "dbits": 2214592512 } ] }, { "idx": 83, "layers": [ "model.layers.28.self_attn.q_proj", "model.layers.28.self_attn.k_proj", "model.layers.28.self_attn.v_proj", "model.layers.28.self_attn.o_proj" ], "candidates": [ { "dkld": 0.0007798749022185913, "dbits": 109051904 } ] }, { "idx": 84, "layers": [ "model.layers.28.mlp.shared_experts.gate_proj", "model.layers.28.mlp.shared_experts.up_proj", "model.layers.28.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0008378526195883806, "dbits": 17301504 } ] }, { "idx": 85, "layers": [ "model.layers.28.mlp.experts.0.gate_proj", "model.layers.28.mlp.experts.1.gate_proj", "model.layers.28.mlp.experts.2.gate_proj", "model.layers.28.mlp.experts.3.gate_proj", "model.layers.28.mlp.experts.4.gate_proj", "model.layers.28.mlp.experts.5.gate_proj", "model.layers.28.mlp.experts.6.gate_proj", "model.layers.28.mlp.experts.7.gate_proj", "model.layers.28.mlp.experts.8.gate_proj", "model.layers.28.mlp.experts.9.gate_proj", "model.layers.28.mlp.experts.10.gate_proj", "model.layers.28.mlp.experts.11.gate_proj", "model.layers.28.mlp.experts.12.gate_proj", "model.layers.28.mlp.experts.13.gate_proj", "model.layers.28.mlp.experts.14.gate_proj", "model.layers.28.mlp.experts.15.gate_proj", "model.layers.28.mlp.experts.16.gate_proj", "model.layers.28.mlp.experts.17.gate_proj", "model.layers.28.mlp.experts.18.gate_proj", "model.layers.28.mlp.experts.19.gate_proj", "model.layers.28.mlp.experts.20.gate_proj", "model.layers.28.mlp.experts.21.gate_proj", "model.layers.28.mlp.experts.22.gate_proj", "model.layers.28.mlp.experts.23.gate_proj", "model.layers.28.mlp.experts.24.gate_proj", "model.layers.28.mlp.experts.25.gate_proj", "model.layers.28.mlp.experts.26.gate_proj", "model.layers.28.mlp.experts.27.gate_proj", "model.layers.28.mlp.experts.28.gate_proj", "model.layers.28.mlp.experts.29.gate_proj", "model.layers.28.mlp.experts.30.gate_proj", "model.layers.28.mlp.experts.31.gate_proj", "model.layers.28.mlp.experts.32.gate_proj", "model.layers.28.mlp.experts.33.gate_proj", "model.layers.28.mlp.experts.34.gate_proj", "model.layers.28.mlp.experts.35.gate_proj", "model.layers.28.mlp.experts.36.gate_proj", "model.layers.28.mlp.experts.37.gate_proj", "model.layers.28.mlp.experts.38.gate_proj", "model.layers.28.mlp.experts.39.gate_proj", "model.layers.28.mlp.experts.40.gate_proj", "model.layers.28.mlp.experts.41.gate_proj", "model.layers.28.mlp.experts.42.gate_proj", "model.layers.28.mlp.experts.43.gate_proj", "model.layers.28.mlp.experts.44.gate_proj", "model.layers.28.mlp.experts.45.gate_proj", "model.layers.28.mlp.experts.46.gate_proj", "model.layers.28.mlp.experts.47.gate_proj", "model.layers.28.mlp.experts.48.gate_proj", "model.layers.28.mlp.experts.49.gate_proj", "model.layers.28.mlp.experts.50.gate_proj", "model.layers.28.mlp.experts.51.gate_proj", "model.layers.28.mlp.experts.52.gate_proj", "model.layers.28.mlp.experts.53.gate_proj", "model.layers.28.mlp.experts.54.gate_proj", "model.layers.28.mlp.experts.55.gate_proj", "model.layers.28.mlp.experts.56.gate_proj", "model.layers.28.mlp.experts.57.gate_proj", "model.layers.28.mlp.experts.58.gate_proj", "model.layers.28.mlp.experts.59.gate_proj", "model.layers.28.mlp.experts.60.gate_proj", "model.layers.28.mlp.experts.61.gate_proj", "model.layers.28.mlp.experts.62.gate_proj", "model.layers.28.mlp.experts.63.gate_proj", "model.layers.28.mlp.experts.64.gate_proj", "model.layers.28.mlp.experts.65.gate_proj", "model.layers.28.mlp.experts.66.gate_proj", "model.layers.28.mlp.experts.67.gate_proj", "model.layers.28.mlp.experts.68.gate_proj", "model.layers.28.mlp.experts.69.gate_proj", "model.layers.28.mlp.experts.70.gate_proj", "model.layers.28.mlp.experts.71.gate_proj", "model.layers.28.mlp.experts.72.gate_proj", "model.layers.28.mlp.experts.73.gate_proj", "model.layers.28.mlp.experts.74.gate_proj", "model.layers.28.mlp.experts.75.gate_proj", "model.layers.28.mlp.experts.76.gate_proj", "model.layers.28.mlp.experts.77.gate_proj", "model.layers.28.mlp.experts.78.gate_proj", "model.layers.28.mlp.experts.79.gate_proj", "model.layers.28.mlp.experts.80.gate_proj", "model.layers.28.mlp.experts.81.gate_proj", "model.layers.28.mlp.experts.82.gate_proj", "model.layers.28.mlp.experts.83.gate_proj", "model.layers.28.mlp.experts.84.gate_proj", "model.layers.28.mlp.experts.85.gate_proj", "model.layers.28.mlp.experts.86.gate_proj", "model.layers.28.mlp.experts.87.gate_proj", "model.layers.28.mlp.experts.88.gate_proj", "model.layers.28.mlp.experts.89.gate_proj", "model.layers.28.mlp.experts.90.gate_proj", "model.layers.28.mlp.experts.91.gate_proj", "model.layers.28.mlp.experts.92.gate_proj", "model.layers.28.mlp.experts.93.gate_proj", "model.layers.28.mlp.experts.94.gate_proj", "model.layers.28.mlp.experts.95.gate_proj", "model.layers.28.mlp.experts.96.gate_proj", "model.layers.28.mlp.experts.97.gate_proj", "model.layers.28.mlp.experts.98.gate_proj", "model.layers.28.mlp.experts.99.gate_proj", "model.layers.28.mlp.experts.100.gate_proj", "model.layers.28.mlp.experts.101.gate_proj", "model.layers.28.mlp.experts.102.gate_proj", "model.layers.28.mlp.experts.103.gate_proj", "model.layers.28.mlp.experts.104.gate_proj", "model.layers.28.mlp.experts.105.gate_proj", "model.layers.28.mlp.experts.106.gate_proj", "model.layers.28.mlp.experts.107.gate_proj", "model.layers.28.mlp.experts.108.gate_proj", "model.layers.28.mlp.experts.109.gate_proj", "model.layers.28.mlp.experts.110.gate_proj", "model.layers.28.mlp.experts.111.gate_proj", "model.layers.28.mlp.experts.112.gate_proj", "model.layers.28.mlp.experts.113.gate_proj", "model.layers.28.mlp.experts.114.gate_proj", "model.layers.28.mlp.experts.115.gate_proj", "model.layers.28.mlp.experts.116.gate_proj", "model.layers.28.mlp.experts.117.gate_proj", "model.layers.28.mlp.experts.118.gate_proj", "model.layers.28.mlp.experts.119.gate_proj", "model.layers.28.mlp.experts.120.gate_proj", "model.layers.28.mlp.experts.121.gate_proj", "model.layers.28.mlp.experts.122.gate_proj", "model.layers.28.mlp.experts.123.gate_proj", "model.layers.28.mlp.experts.124.gate_proj", "model.layers.28.mlp.experts.125.gate_proj", "model.layers.28.mlp.experts.126.gate_proj", "model.layers.28.mlp.experts.127.gate_proj", "model.layers.28.mlp.experts.0.up_proj", "model.layers.28.mlp.experts.1.up_proj", "model.layers.28.mlp.experts.2.up_proj", "model.layers.28.mlp.experts.3.up_proj", "model.layers.28.mlp.experts.4.up_proj", "model.layers.28.mlp.experts.5.up_proj", "model.layers.28.mlp.experts.6.up_proj", "model.layers.28.mlp.experts.7.up_proj", "model.layers.28.mlp.experts.8.up_proj", "model.layers.28.mlp.experts.9.up_proj", "model.layers.28.mlp.experts.10.up_proj", "model.layers.28.mlp.experts.11.up_proj", "model.layers.28.mlp.experts.12.up_proj", "model.layers.28.mlp.experts.13.up_proj", "model.layers.28.mlp.experts.14.up_proj", "model.layers.28.mlp.experts.15.up_proj", "model.layers.28.mlp.experts.16.up_proj", "model.layers.28.mlp.experts.17.up_proj", "model.layers.28.mlp.experts.18.up_proj", "model.layers.28.mlp.experts.19.up_proj", "model.layers.28.mlp.experts.20.up_proj", "model.layers.28.mlp.experts.21.up_proj", "model.layers.28.mlp.experts.22.up_proj", "model.layers.28.mlp.experts.23.up_proj", "model.layers.28.mlp.experts.24.up_proj", "model.layers.28.mlp.experts.25.up_proj", "model.layers.28.mlp.experts.26.up_proj", "model.layers.28.mlp.experts.27.up_proj", "model.layers.28.mlp.experts.28.up_proj", "model.layers.28.mlp.experts.29.up_proj", "model.layers.28.mlp.experts.30.up_proj", "model.layers.28.mlp.experts.31.up_proj", "model.layers.28.mlp.experts.32.up_proj", "model.layers.28.mlp.experts.33.up_proj", "model.layers.28.mlp.experts.34.up_proj", "model.layers.28.mlp.experts.35.up_proj", "model.layers.28.mlp.experts.36.up_proj", "model.layers.28.mlp.experts.37.up_proj", "model.layers.28.mlp.experts.38.up_proj", "model.layers.28.mlp.experts.39.up_proj", "model.layers.28.mlp.experts.40.up_proj", "model.layers.28.mlp.experts.41.up_proj", "model.layers.28.mlp.experts.42.up_proj", "model.layers.28.mlp.experts.43.up_proj", "model.layers.28.mlp.experts.44.up_proj", "model.layers.28.mlp.experts.45.up_proj", "model.layers.28.mlp.experts.46.up_proj", "model.layers.28.mlp.experts.47.up_proj", "model.layers.28.mlp.experts.48.up_proj", "model.layers.28.mlp.experts.49.up_proj", "model.layers.28.mlp.experts.50.up_proj", "model.layers.28.mlp.experts.51.up_proj", "model.layers.28.mlp.experts.52.up_proj", "model.layers.28.mlp.experts.53.up_proj", "model.layers.28.mlp.experts.54.up_proj", "model.layers.28.mlp.experts.55.up_proj", "model.layers.28.mlp.experts.56.up_proj", "model.layers.28.mlp.experts.57.up_proj", "model.layers.28.mlp.experts.58.up_proj", "model.layers.28.mlp.experts.59.up_proj", "model.layers.28.mlp.experts.60.up_proj", "model.layers.28.mlp.experts.61.up_proj", "model.layers.28.mlp.experts.62.up_proj", "model.layers.28.mlp.experts.63.up_proj", "model.layers.28.mlp.experts.64.up_proj", "model.layers.28.mlp.experts.65.up_proj", "model.layers.28.mlp.experts.66.up_proj", "model.layers.28.mlp.experts.67.up_proj", "model.layers.28.mlp.experts.68.up_proj", "model.layers.28.mlp.experts.69.up_proj", "model.layers.28.mlp.experts.70.up_proj", "model.layers.28.mlp.experts.71.up_proj", "model.layers.28.mlp.experts.72.up_proj", "model.layers.28.mlp.experts.73.up_proj", "model.layers.28.mlp.experts.74.up_proj", "model.layers.28.mlp.experts.75.up_proj", "model.layers.28.mlp.experts.76.up_proj", "model.layers.28.mlp.experts.77.up_proj", "model.layers.28.mlp.experts.78.up_proj", "model.layers.28.mlp.experts.79.up_proj", "model.layers.28.mlp.experts.80.up_proj", "model.layers.28.mlp.experts.81.up_proj", "model.layers.28.mlp.experts.82.up_proj", "model.layers.28.mlp.experts.83.up_proj", "model.layers.28.mlp.experts.84.up_proj", "model.layers.28.mlp.experts.85.up_proj", "model.layers.28.mlp.experts.86.up_proj", "model.layers.28.mlp.experts.87.up_proj", "model.layers.28.mlp.experts.88.up_proj", "model.layers.28.mlp.experts.89.up_proj", "model.layers.28.mlp.experts.90.up_proj", "model.layers.28.mlp.experts.91.up_proj", "model.layers.28.mlp.experts.92.up_proj", "model.layers.28.mlp.experts.93.up_proj", "model.layers.28.mlp.experts.94.up_proj", "model.layers.28.mlp.experts.95.up_proj", "model.layers.28.mlp.experts.96.up_proj", "model.layers.28.mlp.experts.97.up_proj", "model.layers.28.mlp.experts.98.up_proj", "model.layers.28.mlp.experts.99.up_proj", "model.layers.28.mlp.experts.100.up_proj", "model.layers.28.mlp.experts.101.up_proj", "model.layers.28.mlp.experts.102.up_proj", "model.layers.28.mlp.experts.103.up_proj", "model.layers.28.mlp.experts.104.up_proj", "model.layers.28.mlp.experts.105.up_proj", "model.layers.28.mlp.experts.106.up_proj", "model.layers.28.mlp.experts.107.up_proj", "model.layers.28.mlp.experts.108.up_proj", "model.layers.28.mlp.experts.109.up_proj", "model.layers.28.mlp.experts.110.up_proj", "model.layers.28.mlp.experts.111.up_proj", "model.layers.28.mlp.experts.112.up_proj", "model.layers.28.mlp.experts.113.up_proj", "model.layers.28.mlp.experts.114.up_proj", "model.layers.28.mlp.experts.115.up_proj", "model.layers.28.mlp.experts.116.up_proj", "model.layers.28.mlp.experts.117.up_proj", "model.layers.28.mlp.experts.118.up_proj", "model.layers.28.mlp.experts.119.up_proj", "model.layers.28.mlp.experts.120.up_proj", "model.layers.28.mlp.experts.121.up_proj", "model.layers.28.mlp.experts.122.up_proj", "model.layers.28.mlp.experts.123.up_proj", "model.layers.28.mlp.experts.124.up_proj", "model.layers.28.mlp.experts.125.up_proj", "model.layers.28.mlp.experts.126.up_proj", "model.layers.28.mlp.experts.127.up_proj", "model.layers.28.mlp.experts.0.down_proj", "model.layers.28.mlp.experts.1.down_proj", "model.layers.28.mlp.experts.2.down_proj", "model.layers.28.mlp.experts.3.down_proj", "model.layers.28.mlp.experts.4.down_proj", "model.layers.28.mlp.experts.5.down_proj", "model.layers.28.mlp.experts.6.down_proj", "model.layers.28.mlp.experts.7.down_proj", "model.layers.28.mlp.experts.8.down_proj", "model.layers.28.mlp.experts.9.down_proj", "model.layers.28.mlp.experts.10.down_proj", "model.layers.28.mlp.experts.11.down_proj", "model.layers.28.mlp.experts.12.down_proj", "model.layers.28.mlp.experts.13.down_proj", "model.layers.28.mlp.experts.14.down_proj", "model.layers.28.mlp.experts.15.down_proj", "model.layers.28.mlp.experts.16.down_proj", "model.layers.28.mlp.experts.17.down_proj", "model.layers.28.mlp.experts.18.down_proj", "model.layers.28.mlp.experts.19.down_proj", "model.layers.28.mlp.experts.20.down_proj", "model.layers.28.mlp.experts.21.down_proj", "model.layers.28.mlp.experts.22.down_proj", "model.layers.28.mlp.experts.23.down_proj", "model.layers.28.mlp.experts.24.down_proj", "model.layers.28.mlp.experts.25.down_proj", "model.layers.28.mlp.experts.26.down_proj", "model.layers.28.mlp.experts.27.down_proj", "model.layers.28.mlp.experts.28.down_proj", "model.layers.28.mlp.experts.29.down_proj", "model.layers.28.mlp.experts.30.down_proj", "model.layers.28.mlp.experts.31.down_proj", "model.layers.28.mlp.experts.32.down_proj", "model.layers.28.mlp.experts.33.down_proj", "model.layers.28.mlp.experts.34.down_proj", "model.layers.28.mlp.experts.35.down_proj", "model.layers.28.mlp.experts.36.down_proj", "model.layers.28.mlp.experts.37.down_proj", "model.layers.28.mlp.experts.38.down_proj", "model.layers.28.mlp.experts.39.down_proj", "model.layers.28.mlp.experts.40.down_proj", "model.layers.28.mlp.experts.41.down_proj", "model.layers.28.mlp.experts.42.down_proj", "model.layers.28.mlp.experts.43.down_proj", "model.layers.28.mlp.experts.44.down_proj", "model.layers.28.mlp.experts.45.down_proj", "model.layers.28.mlp.experts.46.down_proj", "model.layers.28.mlp.experts.47.down_proj", "model.layers.28.mlp.experts.48.down_proj", "model.layers.28.mlp.experts.49.down_proj", "model.layers.28.mlp.experts.50.down_proj", "model.layers.28.mlp.experts.51.down_proj", "model.layers.28.mlp.experts.52.down_proj", "model.layers.28.mlp.experts.53.down_proj", "model.layers.28.mlp.experts.54.down_proj", "model.layers.28.mlp.experts.55.down_proj", "model.layers.28.mlp.experts.56.down_proj", "model.layers.28.mlp.experts.57.down_proj", "model.layers.28.mlp.experts.58.down_proj", "model.layers.28.mlp.experts.59.down_proj", "model.layers.28.mlp.experts.60.down_proj", "model.layers.28.mlp.experts.61.down_proj", "model.layers.28.mlp.experts.62.down_proj", "model.layers.28.mlp.experts.63.down_proj", "model.layers.28.mlp.experts.64.down_proj", "model.layers.28.mlp.experts.65.down_proj", "model.layers.28.mlp.experts.66.down_proj", "model.layers.28.mlp.experts.67.down_proj", "model.layers.28.mlp.experts.68.down_proj", "model.layers.28.mlp.experts.69.down_proj", "model.layers.28.mlp.experts.70.down_proj", "model.layers.28.mlp.experts.71.down_proj", "model.layers.28.mlp.experts.72.down_proj", "model.layers.28.mlp.experts.73.down_proj", "model.layers.28.mlp.experts.74.down_proj", "model.layers.28.mlp.experts.75.down_proj", "model.layers.28.mlp.experts.76.down_proj", "model.layers.28.mlp.experts.77.down_proj", "model.layers.28.mlp.experts.78.down_proj", "model.layers.28.mlp.experts.79.down_proj", "model.layers.28.mlp.experts.80.down_proj", "model.layers.28.mlp.experts.81.down_proj", "model.layers.28.mlp.experts.82.down_proj", "model.layers.28.mlp.experts.83.down_proj", "model.layers.28.mlp.experts.84.down_proj", "model.layers.28.mlp.experts.85.down_proj", "model.layers.28.mlp.experts.86.down_proj", "model.layers.28.mlp.experts.87.down_proj", "model.layers.28.mlp.experts.88.down_proj", "model.layers.28.mlp.experts.89.down_proj", "model.layers.28.mlp.experts.90.down_proj", "model.layers.28.mlp.experts.91.down_proj", "model.layers.28.mlp.experts.92.down_proj", "model.layers.28.mlp.experts.93.down_proj", "model.layers.28.mlp.experts.94.down_proj", "model.layers.28.mlp.experts.95.down_proj", "model.layers.28.mlp.experts.96.down_proj", "model.layers.28.mlp.experts.97.down_proj", "model.layers.28.mlp.experts.98.down_proj", "model.layers.28.mlp.experts.99.down_proj", "model.layers.28.mlp.experts.100.down_proj", "model.layers.28.mlp.experts.101.down_proj", "model.layers.28.mlp.experts.102.down_proj", "model.layers.28.mlp.experts.103.down_proj", "model.layers.28.mlp.experts.104.down_proj", "model.layers.28.mlp.experts.105.down_proj", "model.layers.28.mlp.experts.106.down_proj", "model.layers.28.mlp.experts.107.down_proj", "model.layers.28.mlp.experts.108.down_proj", "model.layers.28.mlp.experts.109.down_proj", "model.layers.28.mlp.experts.110.down_proj", "model.layers.28.mlp.experts.111.down_proj", "model.layers.28.mlp.experts.112.down_proj", "model.layers.28.mlp.experts.113.down_proj", "model.layers.28.mlp.experts.114.down_proj", "model.layers.28.mlp.experts.115.down_proj", "model.layers.28.mlp.experts.116.down_proj", "model.layers.28.mlp.experts.117.down_proj", "model.layers.28.mlp.experts.118.down_proj", "model.layers.28.mlp.experts.119.down_proj", "model.layers.28.mlp.experts.120.down_proj", "model.layers.28.mlp.experts.121.down_proj", "model.layers.28.mlp.experts.122.down_proj", "model.layers.28.mlp.experts.123.down_proj", "model.layers.28.mlp.experts.124.down_proj", "model.layers.28.mlp.experts.125.down_proj", "model.layers.28.mlp.experts.126.down_proj", "model.layers.28.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.002316563948988909, "dbits": 2214592512 } ] }, { "idx": 86, "layers": [ "model.layers.29.self_attn.q_proj", "model.layers.29.self_attn.k_proj", "model.layers.29.self_attn.v_proj", "model.layers.29.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0008686421439051684, "dbits": 109051904 } ] }, { "idx": 87, "layers": [ "model.layers.29.mlp.shared_experts.gate_proj", "model.layers.29.mlp.shared_experts.up_proj", "model.layers.29.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0006850133650004864, "dbits": 17301504 } ] }, { "idx": 88, "layers": [ "model.layers.29.mlp.experts.0.gate_proj", "model.layers.29.mlp.experts.1.gate_proj", "model.layers.29.mlp.experts.2.gate_proj", "model.layers.29.mlp.experts.3.gate_proj", "model.layers.29.mlp.experts.4.gate_proj", "model.layers.29.mlp.experts.5.gate_proj", "model.layers.29.mlp.experts.6.gate_proj", "model.layers.29.mlp.experts.7.gate_proj", "model.layers.29.mlp.experts.8.gate_proj", "model.layers.29.mlp.experts.9.gate_proj", "model.layers.29.mlp.experts.10.gate_proj", "model.layers.29.mlp.experts.11.gate_proj", "model.layers.29.mlp.experts.12.gate_proj", "model.layers.29.mlp.experts.13.gate_proj", "model.layers.29.mlp.experts.14.gate_proj", "model.layers.29.mlp.experts.15.gate_proj", "model.layers.29.mlp.experts.16.gate_proj", "model.layers.29.mlp.experts.17.gate_proj", "model.layers.29.mlp.experts.18.gate_proj", "model.layers.29.mlp.experts.19.gate_proj", "model.layers.29.mlp.experts.20.gate_proj", "model.layers.29.mlp.experts.21.gate_proj", "model.layers.29.mlp.experts.22.gate_proj", "model.layers.29.mlp.experts.23.gate_proj", "model.layers.29.mlp.experts.24.gate_proj", "model.layers.29.mlp.experts.25.gate_proj", "model.layers.29.mlp.experts.26.gate_proj", "model.layers.29.mlp.experts.27.gate_proj", "model.layers.29.mlp.experts.28.gate_proj", "model.layers.29.mlp.experts.29.gate_proj", "model.layers.29.mlp.experts.30.gate_proj", "model.layers.29.mlp.experts.31.gate_proj", "model.layers.29.mlp.experts.32.gate_proj", "model.layers.29.mlp.experts.33.gate_proj", "model.layers.29.mlp.experts.34.gate_proj", "model.layers.29.mlp.experts.35.gate_proj", "model.layers.29.mlp.experts.36.gate_proj", "model.layers.29.mlp.experts.37.gate_proj", "model.layers.29.mlp.experts.38.gate_proj", "model.layers.29.mlp.experts.39.gate_proj", "model.layers.29.mlp.experts.40.gate_proj", "model.layers.29.mlp.experts.41.gate_proj", "model.layers.29.mlp.experts.42.gate_proj", "model.layers.29.mlp.experts.43.gate_proj", "model.layers.29.mlp.experts.44.gate_proj", "model.layers.29.mlp.experts.45.gate_proj", "model.layers.29.mlp.experts.46.gate_proj", "model.layers.29.mlp.experts.47.gate_proj", "model.layers.29.mlp.experts.48.gate_proj", "model.layers.29.mlp.experts.49.gate_proj", "model.layers.29.mlp.experts.50.gate_proj", "model.layers.29.mlp.experts.51.gate_proj", "model.layers.29.mlp.experts.52.gate_proj", "model.layers.29.mlp.experts.53.gate_proj", "model.layers.29.mlp.experts.54.gate_proj", "model.layers.29.mlp.experts.55.gate_proj", "model.layers.29.mlp.experts.56.gate_proj", "model.layers.29.mlp.experts.57.gate_proj", "model.layers.29.mlp.experts.58.gate_proj", "model.layers.29.mlp.experts.59.gate_proj", "model.layers.29.mlp.experts.60.gate_proj", "model.layers.29.mlp.experts.61.gate_proj", "model.layers.29.mlp.experts.62.gate_proj", "model.layers.29.mlp.experts.63.gate_proj", "model.layers.29.mlp.experts.64.gate_proj", "model.layers.29.mlp.experts.65.gate_proj", "model.layers.29.mlp.experts.66.gate_proj", "model.layers.29.mlp.experts.67.gate_proj", "model.layers.29.mlp.experts.68.gate_proj", "model.layers.29.mlp.experts.69.gate_proj", "model.layers.29.mlp.experts.70.gate_proj", "model.layers.29.mlp.experts.71.gate_proj", "model.layers.29.mlp.experts.72.gate_proj", "model.layers.29.mlp.experts.73.gate_proj", "model.layers.29.mlp.experts.74.gate_proj", "model.layers.29.mlp.experts.75.gate_proj", "model.layers.29.mlp.experts.76.gate_proj", "model.layers.29.mlp.experts.77.gate_proj", "model.layers.29.mlp.experts.78.gate_proj", "model.layers.29.mlp.experts.79.gate_proj", "model.layers.29.mlp.experts.80.gate_proj", "model.layers.29.mlp.experts.81.gate_proj", "model.layers.29.mlp.experts.82.gate_proj", "model.layers.29.mlp.experts.83.gate_proj", "model.layers.29.mlp.experts.84.gate_proj", "model.layers.29.mlp.experts.85.gate_proj", "model.layers.29.mlp.experts.86.gate_proj", "model.layers.29.mlp.experts.87.gate_proj", "model.layers.29.mlp.experts.88.gate_proj", "model.layers.29.mlp.experts.89.gate_proj", "model.layers.29.mlp.experts.90.gate_proj", "model.layers.29.mlp.experts.91.gate_proj", "model.layers.29.mlp.experts.92.gate_proj", "model.layers.29.mlp.experts.93.gate_proj", "model.layers.29.mlp.experts.94.gate_proj", "model.layers.29.mlp.experts.95.gate_proj", "model.layers.29.mlp.experts.96.gate_proj", "model.layers.29.mlp.experts.97.gate_proj", "model.layers.29.mlp.experts.98.gate_proj", "model.layers.29.mlp.experts.99.gate_proj", "model.layers.29.mlp.experts.100.gate_proj", "model.layers.29.mlp.experts.101.gate_proj", "model.layers.29.mlp.experts.102.gate_proj", "model.layers.29.mlp.experts.103.gate_proj", "model.layers.29.mlp.experts.104.gate_proj", "model.layers.29.mlp.experts.105.gate_proj", "model.layers.29.mlp.experts.106.gate_proj", "model.layers.29.mlp.experts.107.gate_proj", "model.layers.29.mlp.experts.108.gate_proj", "model.layers.29.mlp.experts.109.gate_proj", "model.layers.29.mlp.experts.110.gate_proj", "model.layers.29.mlp.experts.111.gate_proj", "model.layers.29.mlp.experts.112.gate_proj", "model.layers.29.mlp.experts.113.gate_proj", "model.layers.29.mlp.experts.114.gate_proj", "model.layers.29.mlp.experts.115.gate_proj", "model.layers.29.mlp.experts.116.gate_proj", "model.layers.29.mlp.experts.117.gate_proj", "model.layers.29.mlp.experts.118.gate_proj", "model.layers.29.mlp.experts.119.gate_proj", "model.layers.29.mlp.experts.120.gate_proj", "model.layers.29.mlp.experts.121.gate_proj", "model.layers.29.mlp.experts.122.gate_proj", "model.layers.29.mlp.experts.123.gate_proj", "model.layers.29.mlp.experts.124.gate_proj", "model.layers.29.mlp.experts.125.gate_proj", "model.layers.29.mlp.experts.126.gate_proj", "model.layers.29.mlp.experts.127.gate_proj", "model.layers.29.mlp.experts.0.up_proj", "model.layers.29.mlp.experts.1.up_proj", "model.layers.29.mlp.experts.2.up_proj", "model.layers.29.mlp.experts.3.up_proj", "model.layers.29.mlp.experts.4.up_proj", "model.layers.29.mlp.experts.5.up_proj", "model.layers.29.mlp.experts.6.up_proj", "model.layers.29.mlp.experts.7.up_proj", "model.layers.29.mlp.experts.8.up_proj", "model.layers.29.mlp.experts.9.up_proj", "model.layers.29.mlp.experts.10.up_proj", "model.layers.29.mlp.experts.11.up_proj", "model.layers.29.mlp.experts.12.up_proj", "model.layers.29.mlp.experts.13.up_proj", "model.layers.29.mlp.experts.14.up_proj", "model.layers.29.mlp.experts.15.up_proj", "model.layers.29.mlp.experts.16.up_proj", "model.layers.29.mlp.experts.17.up_proj", "model.layers.29.mlp.experts.18.up_proj", "model.layers.29.mlp.experts.19.up_proj", "model.layers.29.mlp.experts.20.up_proj", "model.layers.29.mlp.experts.21.up_proj", "model.layers.29.mlp.experts.22.up_proj", "model.layers.29.mlp.experts.23.up_proj", "model.layers.29.mlp.experts.24.up_proj", "model.layers.29.mlp.experts.25.up_proj", "model.layers.29.mlp.experts.26.up_proj", "model.layers.29.mlp.experts.27.up_proj", "model.layers.29.mlp.experts.28.up_proj", "model.layers.29.mlp.experts.29.up_proj", "model.layers.29.mlp.experts.30.up_proj", "model.layers.29.mlp.experts.31.up_proj", "model.layers.29.mlp.experts.32.up_proj", "model.layers.29.mlp.experts.33.up_proj", "model.layers.29.mlp.experts.34.up_proj", "model.layers.29.mlp.experts.35.up_proj", "model.layers.29.mlp.experts.36.up_proj", "model.layers.29.mlp.experts.37.up_proj", "model.layers.29.mlp.experts.38.up_proj", "model.layers.29.mlp.experts.39.up_proj", "model.layers.29.mlp.experts.40.up_proj", "model.layers.29.mlp.experts.41.up_proj", "model.layers.29.mlp.experts.42.up_proj", "model.layers.29.mlp.experts.43.up_proj", "model.layers.29.mlp.experts.44.up_proj", "model.layers.29.mlp.experts.45.up_proj", "model.layers.29.mlp.experts.46.up_proj", "model.layers.29.mlp.experts.47.up_proj", "model.layers.29.mlp.experts.48.up_proj", "model.layers.29.mlp.experts.49.up_proj", "model.layers.29.mlp.experts.50.up_proj", "model.layers.29.mlp.experts.51.up_proj", "model.layers.29.mlp.experts.52.up_proj", "model.layers.29.mlp.experts.53.up_proj", "model.layers.29.mlp.experts.54.up_proj", "model.layers.29.mlp.experts.55.up_proj", "model.layers.29.mlp.experts.56.up_proj", "model.layers.29.mlp.experts.57.up_proj", "model.layers.29.mlp.experts.58.up_proj", "model.layers.29.mlp.experts.59.up_proj", "model.layers.29.mlp.experts.60.up_proj", "model.layers.29.mlp.experts.61.up_proj", "model.layers.29.mlp.experts.62.up_proj", "model.layers.29.mlp.experts.63.up_proj", "model.layers.29.mlp.experts.64.up_proj", "model.layers.29.mlp.experts.65.up_proj", "model.layers.29.mlp.experts.66.up_proj", "model.layers.29.mlp.experts.67.up_proj", "model.layers.29.mlp.experts.68.up_proj", "model.layers.29.mlp.experts.69.up_proj", "model.layers.29.mlp.experts.70.up_proj", "model.layers.29.mlp.experts.71.up_proj", "model.layers.29.mlp.experts.72.up_proj", "model.layers.29.mlp.experts.73.up_proj", "model.layers.29.mlp.experts.74.up_proj", "model.layers.29.mlp.experts.75.up_proj", "model.layers.29.mlp.experts.76.up_proj", "model.layers.29.mlp.experts.77.up_proj", "model.layers.29.mlp.experts.78.up_proj", "model.layers.29.mlp.experts.79.up_proj", "model.layers.29.mlp.experts.80.up_proj", "model.layers.29.mlp.experts.81.up_proj", "model.layers.29.mlp.experts.82.up_proj", "model.layers.29.mlp.experts.83.up_proj", "model.layers.29.mlp.experts.84.up_proj", "model.layers.29.mlp.experts.85.up_proj", "model.layers.29.mlp.experts.86.up_proj", "model.layers.29.mlp.experts.87.up_proj", "model.layers.29.mlp.experts.88.up_proj", "model.layers.29.mlp.experts.89.up_proj", "model.layers.29.mlp.experts.90.up_proj", "model.layers.29.mlp.experts.91.up_proj", "model.layers.29.mlp.experts.92.up_proj", "model.layers.29.mlp.experts.93.up_proj", "model.layers.29.mlp.experts.94.up_proj", "model.layers.29.mlp.experts.95.up_proj", "model.layers.29.mlp.experts.96.up_proj", "model.layers.29.mlp.experts.97.up_proj", "model.layers.29.mlp.experts.98.up_proj", "model.layers.29.mlp.experts.99.up_proj", "model.layers.29.mlp.experts.100.up_proj", "model.layers.29.mlp.experts.101.up_proj", "model.layers.29.mlp.experts.102.up_proj", "model.layers.29.mlp.experts.103.up_proj", "model.layers.29.mlp.experts.104.up_proj", "model.layers.29.mlp.experts.105.up_proj", "model.layers.29.mlp.experts.106.up_proj", "model.layers.29.mlp.experts.107.up_proj", "model.layers.29.mlp.experts.108.up_proj", "model.layers.29.mlp.experts.109.up_proj", "model.layers.29.mlp.experts.110.up_proj", "model.layers.29.mlp.experts.111.up_proj", "model.layers.29.mlp.experts.112.up_proj", "model.layers.29.mlp.experts.113.up_proj", "model.layers.29.mlp.experts.114.up_proj", "model.layers.29.mlp.experts.115.up_proj", "model.layers.29.mlp.experts.116.up_proj", "model.layers.29.mlp.experts.117.up_proj", "model.layers.29.mlp.experts.118.up_proj", "model.layers.29.mlp.experts.119.up_proj", "model.layers.29.mlp.experts.120.up_proj", "model.layers.29.mlp.experts.121.up_proj", "model.layers.29.mlp.experts.122.up_proj", "model.layers.29.mlp.experts.123.up_proj", "model.layers.29.mlp.experts.124.up_proj", "model.layers.29.mlp.experts.125.up_proj", "model.layers.29.mlp.experts.126.up_proj", "model.layers.29.mlp.experts.127.up_proj", "model.layers.29.mlp.experts.0.down_proj", "model.layers.29.mlp.experts.1.down_proj", "model.layers.29.mlp.experts.2.down_proj", "model.layers.29.mlp.experts.3.down_proj", "model.layers.29.mlp.experts.4.down_proj", "model.layers.29.mlp.experts.5.down_proj", "model.layers.29.mlp.experts.6.down_proj", "model.layers.29.mlp.experts.7.down_proj", "model.layers.29.mlp.experts.8.down_proj", "model.layers.29.mlp.experts.9.down_proj", "model.layers.29.mlp.experts.10.down_proj", "model.layers.29.mlp.experts.11.down_proj", "model.layers.29.mlp.experts.12.down_proj", "model.layers.29.mlp.experts.13.down_proj", "model.layers.29.mlp.experts.14.down_proj", "model.layers.29.mlp.experts.15.down_proj", "model.layers.29.mlp.experts.16.down_proj", "model.layers.29.mlp.experts.17.down_proj", "model.layers.29.mlp.experts.18.down_proj", "model.layers.29.mlp.experts.19.down_proj", "model.layers.29.mlp.experts.20.down_proj", "model.layers.29.mlp.experts.21.down_proj", "model.layers.29.mlp.experts.22.down_proj", "model.layers.29.mlp.experts.23.down_proj", "model.layers.29.mlp.experts.24.down_proj", "model.layers.29.mlp.experts.25.down_proj", "model.layers.29.mlp.experts.26.down_proj", "model.layers.29.mlp.experts.27.down_proj", "model.layers.29.mlp.experts.28.down_proj", "model.layers.29.mlp.experts.29.down_proj", "model.layers.29.mlp.experts.30.down_proj", "model.layers.29.mlp.experts.31.down_proj", "model.layers.29.mlp.experts.32.down_proj", "model.layers.29.mlp.experts.33.down_proj", "model.layers.29.mlp.experts.34.down_proj", "model.layers.29.mlp.experts.35.down_proj", "model.layers.29.mlp.experts.36.down_proj", "model.layers.29.mlp.experts.37.down_proj", "model.layers.29.mlp.experts.38.down_proj", "model.layers.29.mlp.experts.39.down_proj", "model.layers.29.mlp.experts.40.down_proj", "model.layers.29.mlp.experts.41.down_proj", "model.layers.29.mlp.experts.42.down_proj", "model.layers.29.mlp.experts.43.down_proj", "model.layers.29.mlp.experts.44.down_proj", "model.layers.29.mlp.experts.45.down_proj", "model.layers.29.mlp.experts.46.down_proj", "model.layers.29.mlp.experts.47.down_proj", "model.layers.29.mlp.experts.48.down_proj", "model.layers.29.mlp.experts.49.down_proj", "model.layers.29.mlp.experts.50.down_proj", "model.layers.29.mlp.experts.51.down_proj", "model.layers.29.mlp.experts.52.down_proj", "model.layers.29.mlp.experts.53.down_proj", "model.layers.29.mlp.experts.54.down_proj", "model.layers.29.mlp.experts.55.down_proj", "model.layers.29.mlp.experts.56.down_proj", "model.layers.29.mlp.experts.57.down_proj", "model.layers.29.mlp.experts.58.down_proj", "model.layers.29.mlp.experts.59.down_proj", "model.layers.29.mlp.experts.60.down_proj", "model.layers.29.mlp.experts.61.down_proj", "model.layers.29.mlp.experts.62.down_proj", "model.layers.29.mlp.experts.63.down_proj", "model.layers.29.mlp.experts.64.down_proj", "model.layers.29.mlp.experts.65.down_proj", "model.layers.29.mlp.experts.66.down_proj", "model.layers.29.mlp.experts.67.down_proj", "model.layers.29.mlp.experts.68.down_proj", "model.layers.29.mlp.experts.69.down_proj", "model.layers.29.mlp.experts.70.down_proj", "model.layers.29.mlp.experts.71.down_proj", "model.layers.29.mlp.experts.72.down_proj", "model.layers.29.mlp.experts.73.down_proj", "model.layers.29.mlp.experts.74.down_proj", "model.layers.29.mlp.experts.75.down_proj", "model.layers.29.mlp.experts.76.down_proj", "model.layers.29.mlp.experts.77.down_proj", "model.layers.29.mlp.experts.78.down_proj", "model.layers.29.mlp.experts.79.down_proj", "model.layers.29.mlp.experts.80.down_proj", "model.layers.29.mlp.experts.81.down_proj", "model.layers.29.mlp.experts.82.down_proj", "model.layers.29.mlp.experts.83.down_proj", "model.layers.29.mlp.experts.84.down_proj", "model.layers.29.mlp.experts.85.down_proj", "model.layers.29.mlp.experts.86.down_proj", "model.layers.29.mlp.experts.87.down_proj", "model.layers.29.mlp.experts.88.down_proj", "model.layers.29.mlp.experts.89.down_proj", "model.layers.29.mlp.experts.90.down_proj", "model.layers.29.mlp.experts.91.down_proj", "model.layers.29.mlp.experts.92.down_proj", "model.layers.29.mlp.experts.93.down_proj", "model.layers.29.mlp.experts.94.down_proj", "model.layers.29.mlp.experts.95.down_proj", "model.layers.29.mlp.experts.96.down_proj", "model.layers.29.mlp.experts.97.down_proj", "model.layers.29.mlp.experts.98.down_proj", "model.layers.29.mlp.experts.99.down_proj", "model.layers.29.mlp.experts.100.down_proj", "model.layers.29.mlp.experts.101.down_proj", "model.layers.29.mlp.experts.102.down_proj", "model.layers.29.mlp.experts.103.down_proj", "model.layers.29.mlp.experts.104.down_proj", "model.layers.29.mlp.experts.105.down_proj", "model.layers.29.mlp.experts.106.down_proj", "model.layers.29.mlp.experts.107.down_proj", "model.layers.29.mlp.experts.108.down_proj", "model.layers.29.mlp.experts.109.down_proj", "model.layers.29.mlp.experts.110.down_proj", "model.layers.29.mlp.experts.111.down_proj", "model.layers.29.mlp.experts.112.down_proj", "model.layers.29.mlp.experts.113.down_proj", "model.layers.29.mlp.experts.114.down_proj", "model.layers.29.mlp.experts.115.down_proj", "model.layers.29.mlp.experts.116.down_proj", "model.layers.29.mlp.experts.117.down_proj", "model.layers.29.mlp.experts.118.down_proj", "model.layers.29.mlp.experts.119.down_proj", "model.layers.29.mlp.experts.120.down_proj", "model.layers.29.mlp.experts.121.down_proj", "model.layers.29.mlp.experts.122.down_proj", "model.layers.29.mlp.experts.123.down_proj", "model.layers.29.mlp.experts.124.down_proj", "model.layers.29.mlp.experts.125.down_proj", "model.layers.29.mlp.experts.126.down_proj", "model.layers.29.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.001679317094385624, "dbits": 2214592512 } ] }, { "idx": 89, "layers": [ "model.layers.30.self_attn.q_proj", "model.layers.30.self_attn.k_proj", "model.layers.30.self_attn.v_proj", "model.layers.30.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0007982864044606686, "dbits": 109051904 } ] }, { "idx": 90, "layers": [ "model.layers.30.mlp.shared_experts.gate_proj", "model.layers.30.mlp.shared_experts.up_proj", "model.layers.30.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.00015183528885245878, "dbits": 17301504 } ] }, { "idx": 91, "layers": [ "model.layers.30.mlp.experts.0.gate_proj", "model.layers.30.mlp.experts.1.gate_proj", "model.layers.30.mlp.experts.2.gate_proj", "model.layers.30.mlp.experts.3.gate_proj", "model.layers.30.mlp.experts.4.gate_proj", "model.layers.30.mlp.experts.5.gate_proj", "model.layers.30.mlp.experts.6.gate_proj", "model.layers.30.mlp.experts.7.gate_proj", "model.layers.30.mlp.experts.8.gate_proj", "model.layers.30.mlp.experts.9.gate_proj", "model.layers.30.mlp.experts.10.gate_proj", "model.layers.30.mlp.experts.11.gate_proj", "model.layers.30.mlp.experts.12.gate_proj", "model.layers.30.mlp.experts.13.gate_proj", "model.layers.30.mlp.experts.14.gate_proj", "model.layers.30.mlp.experts.15.gate_proj", "model.layers.30.mlp.experts.16.gate_proj", "model.layers.30.mlp.experts.17.gate_proj", "model.layers.30.mlp.experts.18.gate_proj", "model.layers.30.mlp.experts.19.gate_proj", "model.layers.30.mlp.experts.20.gate_proj", "model.layers.30.mlp.experts.21.gate_proj", "model.layers.30.mlp.experts.22.gate_proj", "model.layers.30.mlp.experts.23.gate_proj", "model.layers.30.mlp.experts.24.gate_proj", "model.layers.30.mlp.experts.25.gate_proj", "model.layers.30.mlp.experts.26.gate_proj", "model.layers.30.mlp.experts.27.gate_proj", "model.layers.30.mlp.experts.28.gate_proj", "model.layers.30.mlp.experts.29.gate_proj", "model.layers.30.mlp.experts.30.gate_proj", "model.layers.30.mlp.experts.31.gate_proj", "model.layers.30.mlp.experts.32.gate_proj", "model.layers.30.mlp.experts.33.gate_proj", "model.layers.30.mlp.experts.34.gate_proj", "model.layers.30.mlp.experts.35.gate_proj", "model.layers.30.mlp.experts.36.gate_proj", "model.layers.30.mlp.experts.37.gate_proj", "model.layers.30.mlp.experts.38.gate_proj", "model.layers.30.mlp.experts.39.gate_proj", "model.layers.30.mlp.experts.40.gate_proj", "model.layers.30.mlp.experts.41.gate_proj", "model.layers.30.mlp.experts.42.gate_proj", "model.layers.30.mlp.experts.43.gate_proj", "model.layers.30.mlp.experts.44.gate_proj", "model.layers.30.mlp.experts.45.gate_proj", "model.layers.30.mlp.experts.46.gate_proj", "model.layers.30.mlp.experts.47.gate_proj", "model.layers.30.mlp.experts.48.gate_proj", "model.layers.30.mlp.experts.49.gate_proj", "model.layers.30.mlp.experts.50.gate_proj", "model.layers.30.mlp.experts.51.gate_proj", "model.layers.30.mlp.experts.52.gate_proj", "model.layers.30.mlp.experts.53.gate_proj", "model.layers.30.mlp.experts.54.gate_proj", "model.layers.30.mlp.experts.55.gate_proj", "model.layers.30.mlp.experts.56.gate_proj", "model.layers.30.mlp.experts.57.gate_proj", "model.layers.30.mlp.experts.58.gate_proj", "model.layers.30.mlp.experts.59.gate_proj", "model.layers.30.mlp.experts.60.gate_proj", "model.layers.30.mlp.experts.61.gate_proj", "model.layers.30.mlp.experts.62.gate_proj", "model.layers.30.mlp.experts.63.gate_proj", "model.layers.30.mlp.experts.64.gate_proj", "model.layers.30.mlp.experts.65.gate_proj", "model.layers.30.mlp.experts.66.gate_proj", "model.layers.30.mlp.experts.67.gate_proj", "model.layers.30.mlp.experts.68.gate_proj", "model.layers.30.mlp.experts.69.gate_proj", "model.layers.30.mlp.experts.70.gate_proj", "model.layers.30.mlp.experts.71.gate_proj", "model.layers.30.mlp.experts.72.gate_proj", "model.layers.30.mlp.experts.73.gate_proj", "model.layers.30.mlp.experts.74.gate_proj", "model.layers.30.mlp.experts.75.gate_proj", "model.layers.30.mlp.experts.76.gate_proj", "model.layers.30.mlp.experts.77.gate_proj", "model.layers.30.mlp.experts.78.gate_proj", "model.layers.30.mlp.experts.79.gate_proj", "model.layers.30.mlp.experts.80.gate_proj", "model.layers.30.mlp.experts.81.gate_proj", "model.layers.30.mlp.experts.82.gate_proj", "model.layers.30.mlp.experts.83.gate_proj", "model.layers.30.mlp.experts.84.gate_proj", "model.layers.30.mlp.experts.85.gate_proj", "model.layers.30.mlp.experts.86.gate_proj", "model.layers.30.mlp.experts.87.gate_proj", "model.layers.30.mlp.experts.88.gate_proj", "model.layers.30.mlp.experts.89.gate_proj", "model.layers.30.mlp.experts.90.gate_proj", "model.layers.30.mlp.experts.91.gate_proj", "model.layers.30.mlp.experts.92.gate_proj", "model.layers.30.mlp.experts.93.gate_proj", "model.layers.30.mlp.experts.94.gate_proj", "model.layers.30.mlp.experts.95.gate_proj", "model.layers.30.mlp.experts.96.gate_proj", "model.layers.30.mlp.experts.97.gate_proj", "model.layers.30.mlp.experts.98.gate_proj", "model.layers.30.mlp.experts.99.gate_proj", "model.layers.30.mlp.experts.100.gate_proj", "model.layers.30.mlp.experts.101.gate_proj", "model.layers.30.mlp.experts.102.gate_proj", "model.layers.30.mlp.experts.103.gate_proj", "model.layers.30.mlp.experts.104.gate_proj", "model.layers.30.mlp.experts.105.gate_proj", "model.layers.30.mlp.experts.106.gate_proj", "model.layers.30.mlp.experts.107.gate_proj", "model.layers.30.mlp.experts.108.gate_proj", "model.layers.30.mlp.experts.109.gate_proj", "model.layers.30.mlp.experts.110.gate_proj", "model.layers.30.mlp.experts.111.gate_proj", "model.layers.30.mlp.experts.112.gate_proj", "model.layers.30.mlp.experts.113.gate_proj", "model.layers.30.mlp.experts.114.gate_proj", "model.layers.30.mlp.experts.115.gate_proj", "model.layers.30.mlp.experts.116.gate_proj", "model.layers.30.mlp.experts.117.gate_proj", "model.layers.30.mlp.experts.118.gate_proj", "model.layers.30.mlp.experts.119.gate_proj", "model.layers.30.mlp.experts.120.gate_proj", "model.layers.30.mlp.experts.121.gate_proj", "model.layers.30.mlp.experts.122.gate_proj", "model.layers.30.mlp.experts.123.gate_proj", "model.layers.30.mlp.experts.124.gate_proj", "model.layers.30.mlp.experts.125.gate_proj", "model.layers.30.mlp.experts.126.gate_proj", "model.layers.30.mlp.experts.127.gate_proj", "model.layers.30.mlp.experts.0.up_proj", "model.layers.30.mlp.experts.1.up_proj", "model.layers.30.mlp.experts.2.up_proj", "model.layers.30.mlp.experts.3.up_proj", "model.layers.30.mlp.experts.4.up_proj", "model.layers.30.mlp.experts.5.up_proj", "model.layers.30.mlp.experts.6.up_proj", "model.layers.30.mlp.experts.7.up_proj", "model.layers.30.mlp.experts.8.up_proj", "model.layers.30.mlp.experts.9.up_proj", "model.layers.30.mlp.experts.10.up_proj", "model.layers.30.mlp.experts.11.up_proj", "model.layers.30.mlp.experts.12.up_proj", "model.layers.30.mlp.experts.13.up_proj", "model.layers.30.mlp.experts.14.up_proj", "model.layers.30.mlp.experts.15.up_proj", "model.layers.30.mlp.experts.16.up_proj", "model.layers.30.mlp.experts.17.up_proj", "model.layers.30.mlp.experts.18.up_proj", "model.layers.30.mlp.experts.19.up_proj", "model.layers.30.mlp.experts.20.up_proj", "model.layers.30.mlp.experts.21.up_proj", "model.layers.30.mlp.experts.22.up_proj", "model.layers.30.mlp.experts.23.up_proj", "model.layers.30.mlp.experts.24.up_proj", "model.layers.30.mlp.experts.25.up_proj", "model.layers.30.mlp.experts.26.up_proj", "model.layers.30.mlp.experts.27.up_proj", "model.layers.30.mlp.experts.28.up_proj", "model.layers.30.mlp.experts.29.up_proj", "model.layers.30.mlp.experts.30.up_proj", "model.layers.30.mlp.experts.31.up_proj", "model.layers.30.mlp.experts.32.up_proj", "model.layers.30.mlp.experts.33.up_proj", "model.layers.30.mlp.experts.34.up_proj", "model.layers.30.mlp.experts.35.up_proj", "model.layers.30.mlp.experts.36.up_proj", "model.layers.30.mlp.experts.37.up_proj", "model.layers.30.mlp.experts.38.up_proj", "model.layers.30.mlp.experts.39.up_proj", "model.layers.30.mlp.experts.40.up_proj", "model.layers.30.mlp.experts.41.up_proj", "model.layers.30.mlp.experts.42.up_proj", "model.layers.30.mlp.experts.43.up_proj", "model.layers.30.mlp.experts.44.up_proj", "model.layers.30.mlp.experts.45.up_proj", "model.layers.30.mlp.experts.46.up_proj", "model.layers.30.mlp.experts.47.up_proj", "model.layers.30.mlp.experts.48.up_proj", "model.layers.30.mlp.experts.49.up_proj", "model.layers.30.mlp.experts.50.up_proj", "model.layers.30.mlp.experts.51.up_proj", "model.layers.30.mlp.experts.52.up_proj", "model.layers.30.mlp.experts.53.up_proj", "model.layers.30.mlp.experts.54.up_proj", "model.layers.30.mlp.experts.55.up_proj", "model.layers.30.mlp.experts.56.up_proj", "model.layers.30.mlp.experts.57.up_proj", "model.layers.30.mlp.experts.58.up_proj", "model.layers.30.mlp.experts.59.up_proj", "model.layers.30.mlp.experts.60.up_proj", "model.layers.30.mlp.experts.61.up_proj", "model.layers.30.mlp.experts.62.up_proj", "model.layers.30.mlp.experts.63.up_proj", "model.layers.30.mlp.experts.64.up_proj", "model.layers.30.mlp.experts.65.up_proj", "model.layers.30.mlp.experts.66.up_proj", "model.layers.30.mlp.experts.67.up_proj", "model.layers.30.mlp.experts.68.up_proj", "model.layers.30.mlp.experts.69.up_proj", "model.layers.30.mlp.experts.70.up_proj", "model.layers.30.mlp.experts.71.up_proj", "model.layers.30.mlp.experts.72.up_proj", "model.layers.30.mlp.experts.73.up_proj", "model.layers.30.mlp.experts.74.up_proj", "model.layers.30.mlp.experts.75.up_proj", "model.layers.30.mlp.experts.76.up_proj", "model.layers.30.mlp.experts.77.up_proj", "model.layers.30.mlp.experts.78.up_proj", "model.layers.30.mlp.experts.79.up_proj", "model.layers.30.mlp.experts.80.up_proj", "model.layers.30.mlp.experts.81.up_proj", "model.layers.30.mlp.experts.82.up_proj", "model.layers.30.mlp.experts.83.up_proj", "model.layers.30.mlp.experts.84.up_proj", "model.layers.30.mlp.experts.85.up_proj", "model.layers.30.mlp.experts.86.up_proj", "model.layers.30.mlp.experts.87.up_proj", "model.layers.30.mlp.experts.88.up_proj", "model.layers.30.mlp.experts.89.up_proj", "model.layers.30.mlp.experts.90.up_proj", "model.layers.30.mlp.experts.91.up_proj", "model.layers.30.mlp.experts.92.up_proj", "model.layers.30.mlp.experts.93.up_proj", "model.layers.30.mlp.experts.94.up_proj", "model.layers.30.mlp.experts.95.up_proj", "model.layers.30.mlp.experts.96.up_proj", "model.layers.30.mlp.experts.97.up_proj", "model.layers.30.mlp.experts.98.up_proj", "model.layers.30.mlp.experts.99.up_proj", "model.layers.30.mlp.experts.100.up_proj", "model.layers.30.mlp.experts.101.up_proj", "model.layers.30.mlp.experts.102.up_proj", "model.layers.30.mlp.experts.103.up_proj", "model.layers.30.mlp.experts.104.up_proj", "model.layers.30.mlp.experts.105.up_proj", "model.layers.30.mlp.experts.106.up_proj", "model.layers.30.mlp.experts.107.up_proj", "model.layers.30.mlp.experts.108.up_proj", "model.layers.30.mlp.experts.109.up_proj", "model.layers.30.mlp.experts.110.up_proj", "model.layers.30.mlp.experts.111.up_proj", "model.layers.30.mlp.experts.112.up_proj", "model.layers.30.mlp.experts.113.up_proj", "model.layers.30.mlp.experts.114.up_proj", "model.layers.30.mlp.experts.115.up_proj", "model.layers.30.mlp.experts.116.up_proj", "model.layers.30.mlp.experts.117.up_proj", "model.layers.30.mlp.experts.118.up_proj", "model.layers.30.mlp.experts.119.up_proj", "model.layers.30.mlp.experts.120.up_proj", "model.layers.30.mlp.experts.121.up_proj", "model.layers.30.mlp.experts.122.up_proj", "model.layers.30.mlp.experts.123.up_proj", "model.layers.30.mlp.experts.124.up_proj", "model.layers.30.mlp.experts.125.up_proj", "model.layers.30.mlp.experts.126.up_proj", "model.layers.30.mlp.experts.127.up_proj", "model.layers.30.mlp.experts.0.down_proj", "model.layers.30.mlp.experts.1.down_proj", "model.layers.30.mlp.experts.2.down_proj", "model.layers.30.mlp.experts.3.down_proj", "model.layers.30.mlp.experts.4.down_proj", "model.layers.30.mlp.experts.5.down_proj", "model.layers.30.mlp.experts.6.down_proj", "model.layers.30.mlp.experts.7.down_proj", "model.layers.30.mlp.experts.8.down_proj", "model.layers.30.mlp.experts.9.down_proj", "model.layers.30.mlp.experts.10.down_proj", "model.layers.30.mlp.experts.11.down_proj", "model.layers.30.mlp.experts.12.down_proj", "model.layers.30.mlp.experts.13.down_proj", "model.layers.30.mlp.experts.14.down_proj", "model.layers.30.mlp.experts.15.down_proj", "model.layers.30.mlp.experts.16.down_proj", "model.layers.30.mlp.experts.17.down_proj", "model.layers.30.mlp.experts.18.down_proj", "model.layers.30.mlp.experts.19.down_proj", "model.layers.30.mlp.experts.20.down_proj", "model.layers.30.mlp.experts.21.down_proj", "model.layers.30.mlp.experts.22.down_proj", "model.layers.30.mlp.experts.23.down_proj", "model.layers.30.mlp.experts.24.down_proj", "model.layers.30.mlp.experts.25.down_proj", "model.layers.30.mlp.experts.26.down_proj", "model.layers.30.mlp.experts.27.down_proj", "model.layers.30.mlp.experts.28.down_proj", "model.layers.30.mlp.experts.29.down_proj", "model.layers.30.mlp.experts.30.down_proj", "model.layers.30.mlp.experts.31.down_proj", "model.layers.30.mlp.experts.32.down_proj", "model.layers.30.mlp.experts.33.down_proj", "model.layers.30.mlp.experts.34.down_proj", "model.layers.30.mlp.experts.35.down_proj", "model.layers.30.mlp.experts.36.down_proj", "model.layers.30.mlp.experts.37.down_proj", "model.layers.30.mlp.experts.38.down_proj", "model.layers.30.mlp.experts.39.down_proj", "model.layers.30.mlp.experts.40.down_proj", "model.layers.30.mlp.experts.41.down_proj", "model.layers.30.mlp.experts.42.down_proj", "model.layers.30.mlp.experts.43.down_proj", "model.layers.30.mlp.experts.44.down_proj", "model.layers.30.mlp.experts.45.down_proj", "model.layers.30.mlp.experts.46.down_proj", "model.layers.30.mlp.experts.47.down_proj", "model.layers.30.mlp.experts.48.down_proj", "model.layers.30.mlp.experts.49.down_proj", "model.layers.30.mlp.experts.50.down_proj", "model.layers.30.mlp.experts.51.down_proj", "model.layers.30.mlp.experts.52.down_proj", "model.layers.30.mlp.experts.53.down_proj", "model.layers.30.mlp.experts.54.down_proj", "model.layers.30.mlp.experts.55.down_proj", "model.layers.30.mlp.experts.56.down_proj", "model.layers.30.mlp.experts.57.down_proj", "model.layers.30.mlp.experts.58.down_proj", "model.layers.30.mlp.experts.59.down_proj", "model.layers.30.mlp.experts.60.down_proj", "model.layers.30.mlp.experts.61.down_proj", "model.layers.30.mlp.experts.62.down_proj", "model.layers.30.mlp.experts.63.down_proj", "model.layers.30.mlp.experts.64.down_proj", "model.layers.30.mlp.experts.65.down_proj", "model.layers.30.mlp.experts.66.down_proj", "model.layers.30.mlp.experts.67.down_proj", "model.layers.30.mlp.experts.68.down_proj", "model.layers.30.mlp.experts.69.down_proj", "model.layers.30.mlp.experts.70.down_proj", "model.layers.30.mlp.experts.71.down_proj", "model.layers.30.mlp.experts.72.down_proj", "model.layers.30.mlp.experts.73.down_proj", "model.layers.30.mlp.experts.74.down_proj", "model.layers.30.mlp.experts.75.down_proj", "model.layers.30.mlp.experts.76.down_proj", "model.layers.30.mlp.experts.77.down_proj", "model.layers.30.mlp.experts.78.down_proj", "model.layers.30.mlp.experts.79.down_proj", "model.layers.30.mlp.experts.80.down_proj", "model.layers.30.mlp.experts.81.down_proj", "model.layers.30.mlp.experts.82.down_proj", "model.layers.30.mlp.experts.83.down_proj", "model.layers.30.mlp.experts.84.down_proj", "model.layers.30.mlp.experts.85.down_proj", "model.layers.30.mlp.experts.86.down_proj", "model.layers.30.mlp.experts.87.down_proj", "model.layers.30.mlp.experts.88.down_proj", "model.layers.30.mlp.experts.89.down_proj", "model.layers.30.mlp.experts.90.down_proj", "model.layers.30.mlp.experts.91.down_proj", "model.layers.30.mlp.experts.92.down_proj", "model.layers.30.mlp.experts.93.down_proj", "model.layers.30.mlp.experts.94.down_proj", "model.layers.30.mlp.experts.95.down_proj", "model.layers.30.mlp.experts.96.down_proj", "model.layers.30.mlp.experts.97.down_proj", "model.layers.30.mlp.experts.98.down_proj", "model.layers.30.mlp.experts.99.down_proj", "model.layers.30.mlp.experts.100.down_proj", "model.layers.30.mlp.experts.101.down_proj", "model.layers.30.mlp.experts.102.down_proj", "model.layers.30.mlp.experts.103.down_proj", "model.layers.30.mlp.experts.104.down_proj", "model.layers.30.mlp.experts.105.down_proj", "model.layers.30.mlp.experts.106.down_proj", "model.layers.30.mlp.experts.107.down_proj", "model.layers.30.mlp.experts.108.down_proj", "model.layers.30.mlp.experts.109.down_proj", "model.layers.30.mlp.experts.110.down_proj", "model.layers.30.mlp.experts.111.down_proj", "model.layers.30.mlp.experts.112.down_proj", "model.layers.30.mlp.experts.113.down_proj", "model.layers.30.mlp.experts.114.down_proj", "model.layers.30.mlp.experts.115.down_proj", "model.layers.30.mlp.experts.116.down_proj", "model.layers.30.mlp.experts.117.down_proj", "model.layers.30.mlp.experts.118.down_proj", "model.layers.30.mlp.experts.119.down_proj", "model.layers.30.mlp.experts.120.down_proj", "model.layers.30.mlp.experts.121.down_proj", "model.layers.30.mlp.experts.122.down_proj", "model.layers.30.mlp.experts.123.down_proj", "model.layers.30.mlp.experts.124.down_proj", "model.layers.30.mlp.experts.125.down_proj", "model.layers.30.mlp.experts.126.down_proj", "model.layers.30.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0013751215301454067, "dbits": 2214592512 } ] }, { "idx": 92, "layers": [ "model.layers.31.self_attn.q_proj", "model.layers.31.self_attn.k_proj", "model.layers.31.self_attn.v_proj", "model.layers.31.self_attn.o_proj" ], "candidates": [ { "dkld": -0.001155925914645195, "dbits": 109051904 } ] }, { "idx": 93, "layers": [ "model.layers.31.mlp.shared_experts.gate_proj", "model.layers.31.mlp.shared_experts.up_proj", "model.layers.31.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0007207431830465738, "dbits": 17301504 } ] }, { "idx": 94, "layers": [ "model.layers.31.mlp.experts.0.gate_proj", "model.layers.31.mlp.experts.1.gate_proj", "model.layers.31.mlp.experts.2.gate_proj", "model.layers.31.mlp.experts.3.gate_proj", "model.layers.31.mlp.experts.4.gate_proj", "model.layers.31.mlp.experts.5.gate_proj", "model.layers.31.mlp.experts.6.gate_proj", "model.layers.31.mlp.experts.7.gate_proj", "model.layers.31.mlp.experts.8.gate_proj", "model.layers.31.mlp.experts.9.gate_proj", "model.layers.31.mlp.experts.10.gate_proj", "model.layers.31.mlp.experts.11.gate_proj", "model.layers.31.mlp.experts.12.gate_proj", "model.layers.31.mlp.experts.13.gate_proj", "model.layers.31.mlp.experts.14.gate_proj", "model.layers.31.mlp.experts.15.gate_proj", "model.layers.31.mlp.experts.16.gate_proj", "model.layers.31.mlp.experts.17.gate_proj", "model.layers.31.mlp.experts.18.gate_proj", "model.layers.31.mlp.experts.19.gate_proj", "model.layers.31.mlp.experts.20.gate_proj", "model.layers.31.mlp.experts.21.gate_proj", "model.layers.31.mlp.experts.22.gate_proj", "model.layers.31.mlp.experts.23.gate_proj", "model.layers.31.mlp.experts.24.gate_proj", "model.layers.31.mlp.experts.25.gate_proj", "model.layers.31.mlp.experts.26.gate_proj", "model.layers.31.mlp.experts.27.gate_proj", "model.layers.31.mlp.experts.28.gate_proj", "model.layers.31.mlp.experts.29.gate_proj", "model.layers.31.mlp.experts.30.gate_proj", "model.layers.31.mlp.experts.31.gate_proj", "model.layers.31.mlp.experts.32.gate_proj", "model.layers.31.mlp.experts.33.gate_proj", "model.layers.31.mlp.experts.34.gate_proj", "model.layers.31.mlp.experts.35.gate_proj", "model.layers.31.mlp.experts.36.gate_proj", "model.layers.31.mlp.experts.37.gate_proj", "model.layers.31.mlp.experts.38.gate_proj", "model.layers.31.mlp.experts.39.gate_proj", "model.layers.31.mlp.experts.40.gate_proj", "model.layers.31.mlp.experts.41.gate_proj", "model.layers.31.mlp.experts.42.gate_proj", "model.layers.31.mlp.experts.43.gate_proj", "model.layers.31.mlp.experts.44.gate_proj", "model.layers.31.mlp.experts.45.gate_proj", "model.layers.31.mlp.experts.46.gate_proj", "model.layers.31.mlp.experts.47.gate_proj", "model.layers.31.mlp.experts.48.gate_proj", "model.layers.31.mlp.experts.49.gate_proj", "model.layers.31.mlp.experts.50.gate_proj", "model.layers.31.mlp.experts.51.gate_proj", "model.layers.31.mlp.experts.52.gate_proj", "model.layers.31.mlp.experts.53.gate_proj", "model.layers.31.mlp.experts.54.gate_proj", "model.layers.31.mlp.experts.55.gate_proj", "model.layers.31.mlp.experts.56.gate_proj", "model.layers.31.mlp.experts.57.gate_proj", "model.layers.31.mlp.experts.58.gate_proj", "model.layers.31.mlp.experts.59.gate_proj", "model.layers.31.mlp.experts.60.gate_proj", "model.layers.31.mlp.experts.61.gate_proj", "model.layers.31.mlp.experts.62.gate_proj", "model.layers.31.mlp.experts.63.gate_proj", "model.layers.31.mlp.experts.64.gate_proj", "model.layers.31.mlp.experts.65.gate_proj", "model.layers.31.mlp.experts.66.gate_proj", "model.layers.31.mlp.experts.67.gate_proj", "model.layers.31.mlp.experts.68.gate_proj", "model.layers.31.mlp.experts.69.gate_proj", "model.layers.31.mlp.experts.70.gate_proj", "model.layers.31.mlp.experts.71.gate_proj", "model.layers.31.mlp.experts.72.gate_proj", "model.layers.31.mlp.experts.73.gate_proj", "model.layers.31.mlp.experts.74.gate_proj", "model.layers.31.mlp.experts.75.gate_proj", "model.layers.31.mlp.experts.76.gate_proj", "model.layers.31.mlp.experts.77.gate_proj", "model.layers.31.mlp.experts.78.gate_proj", "model.layers.31.mlp.experts.79.gate_proj", "model.layers.31.mlp.experts.80.gate_proj", "model.layers.31.mlp.experts.81.gate_proj", "model.layers.31.mlp.experts.82.gate_proj", "model.layers.31.mlp.experts.83.gate_proj", "model.layers.31.mlp.experts.84.gate_proj", "model.layers.31.mlp.experts.85.gate_proj", "model.layers.31.mlp.experts.86.gate_proj", "model.layers.31.mlp.experts.87.gate_proj", "model.layers.31.mlp.experts.88.gate_proj", "model.layers.31.mlp.experts.89.gate_proj", "model.layers.31.mlp.experts.90.gate_proj", "model.layers.31.mlp.experts.91.gate_proj", "model.layers.31.mlp.experts.92.gate_proj", "model.layers.31.mlp.experts.93.gate_proj", "model.layers.31.mlp.experts.94.gate_proj", "model.layers.31.mlp.experts.95.gate_proj", "model.layers.31.mlp.experts.96.gate_proj", "model.layers.31.mlp.experts.97.gate_proj", "model.layers.31.mlp.experts.98.gate_proj", "model.layers.31.mlp.experts.99.gate_proj", "model.layers.31.mlp.experts.100.gate_proj", "model.layers.31.mlp.experts.101.gate_proj", "model.layers.31.mlp.experts.102.gate_proj", "model.layers.31.mlp.experts.103.gate_proj", "model.layers.31.mlp.experts.104.gate_proj", "model.layers.31.mlp.experts.105.gate_proj", "model.layers.31.mlp.experts.106.gate_proj", "model.layers.31.mlp.experts.107.gate_proj", "model.layers.31.mlp.experts.108.gate_proj", "model.layers.31.mlp.experts.109.gate_proj", "model.layers.31.mlp.experts.110.gate_proj", "model.layers.31.mlp.experts.111.gate_proj", "model.layers.31.mlp.experts.112.gate_proj", "model.layers.31.mlp.experts.113.gate_proj", "model.layers.31.mlp.experts.114.gate_proj", "model.layers.31.mlp.experts.115.gate_proj", "model.layers.31.mlp.experts.116.gate_proj", "model.layers.31.mlp.experts.117.gate_proj", "model.layers.31.mlp.experts.118.gate_proj", "model.layers.31.mlp.experts.119.gate_proj", "model.layers.31.mlp.experts.120.gate_proj", "model.layers.31.mlp.experts.121.gate_proj", "model.layers.31.mlp.experts.122.gate_proj", "model.layers.31.mlp.experts.123.gate_proj", "model.layers.31.mlp.experts.124.gate_proj", "model.layers.31.mlp.experts.125.gate_proj", "model.layers.31.mlp.experts.126.gate_proj", "model.layers.31.mlp.experts.127.gate_proj", "model.layers.31.mlp.experts.0.up_proj", "model.layers.31.mlp.experts.1.up_proj", "model.layers.31.mlp.experts.2.up_proj", "model.layers.31.mlp.experts.3.up_proj", "model.layers.31.mlp.experts.4.up_proj", "model.layers.31.mlp.experts.5.up_proj", "model.layers.31.mlp.experts.6.up_proj", "model.layers.31.mlp.experts.7.up_proj", "model.layers.31.mlp.experts.8.up_proj", "model.layers.31.mlp.experts.9.up_proj", "model.layers.31.mlp.experts.10.up_proj", "model.layers.31.mlp.experts.11.up_proj", "model.layers.31.mlp.experts.12.up_proj", "model.layers.31.mlp.experts.13.up_proj", "model.layers.31.mlp.experts.14.up_proj", "model.layers.31.mlp.experts.15.up_proj", "model.layers.31.mlp.experts.16.up_proj", "model.layers.31.mlp.experts.17.up_proj", "model.layers.31.mlp.experts.18.up_proj", "model.layers.31.mlp.experts.19.up_proj", "model.layers.31.mlp.experts.20.up_proj", "model.layers.31.mlp.experts.21.up_proj", "model.layers.31.mlp.experts.22.up_proj", "model.layers.31.mlp.experts.23.up_proj", "model.layers.31.mlp.experts.24.up_proj", "model.layers.31.mlp.experts.25.up_proj", "model.layers.31.mlp.experts.26.up_proj", "model.layers.31.mlp.experts.27.up_proj", "model.layers.31.mlp.experts.28.up_proj", "model.layers.31.mlp.experts.29.up_proj", "model.layers.31.mlp.experts.30.up_proj", "model.layers.31.mlp.experts.31.up_proj", "model.layers.31.mlp.experts.32.up_proj", "model.layers.31.mlp.experts.33.up_proj", "model.layers.31.mlp.experts.34.up_proj", "model.layers.31.mlp.experts.35.up_proj", "model.layers.31.mlp.experts.36.up_proj", "model.layers.31.mlp.experts.37.up_proj", "model.layers.31.mlp.experts.38.up_proj", "model.layers.31.mlp.experts.39.up_proj", "model.layers.31.mlp.experts.40.up_proj", "model.layers.31.mlp.experts.41.up_proj", "model.layers.31.mlp.experts.42.up_proj", "model.layers.31.mlp.experts.43.up_proj", "model.layers.31.mlp.experts.44.up_proj", "model.layers.31.mlp.experts.45.up_proj", "model.layers.31.mlp.experts.46.up_proj", "model.layers.31.mlp.experts.47.up_proj", "model.layers.31.mlp.experts.48.up_proj", "model.layers.31.mlp.experts.49.up_proj", "model.layers.31.mlp.experts.50.up_proj", "model.layers.31.mlp.experts.51.up_proj", "model.layers.31.mlp.experts.52.up_proj", "model.layers.31.mlp.experts.53.up_proj", "model.layers.31.mlp.experts.54.up_proj", "model.layers.31.mlp.experts.55.up_proj", "model.layers.31.mlp.experts.56.up_proj", "model.layers.31.mlp.experts.57.up_proj", "model.layers.31.mlp.experts.58.up_proj", "model.layers.31.mlp.experts.59.up_proj", "model.layers.31.mlp.experts.60.up_proj", "model.layers.31.mlp.experts.61.up_proj", "model.layers.31.mlp.experts.62.up_proj", "model.layers.31.mlp.experts.63.up_proj", "model.layers.31.mlp.experts.64.up_proj", "model.layers.31.mlp.experts.65.up_proj", "model.layers.31.mlp.experts.66.up_proj", "model.layers.31.mlp.experts.67.up_proj", "model.layers.31.mlp.experts.68.up_proj", "model.layers.31.mlp.experts.69.up_proj", "model.layers.31.mlp.experts.70.up_proj", "model.layers.31.mlp.experts.71.up_proj", "model.layers.31.mlp.experts.72.up_proj", "model.layers.31.mlp.experts.73.up_proj", "model.layers.31.mlp.experts.74.up_proj", "model.layers.31.mlp.experts.75.up_proj", "model.layers.31.mlp.experts.76.up_proj", "model.layers.31.mlp.experts.77.up_proj", "model.layers.31.mlp.experts.78.up_proj", "model.layers.31.mlp.experts.79.up_proj", "model.layers.31.mlp.experts.80.up_proj", "model.layers.31.mlp.experts.81.up_proj", "model.layers.31.mlp.experts.82.up_proj", "model.layers.31.mlp.experts.83.up_proj", "model.layers.31.mlp.experts.84.up_proj", "model.layers.31.mlp.experts.85.up_proj", "model.layers.31.mlp.experts.86.up_proj", "model.layers.31.mlp.experts.87.up_proj", "model.layers.31.mlp.experts.88.up_proj", "model.layers.31.mlp.experts.89.up_proj", "model.layers.31.mlp.experts.90.up_proj", "model.layers.31.mlp.experts.91.up_proj", "model.layers.31.mlp.experts.92.up_proj", "model.layers.31.mlp.experts.93.up_proj", "model.layers.31.mlp.experts.94.up_proj", "model.layers.31.mlp.experts.95.up_proj", "model.layers.31.mlp.experts.96.up_proj", "model.layers.31.mlp.experts.97.up_proj", "model.layers.31.mlp.experts.98.up_proj", "model.layers.31.mlp.experts.99.up_proj", "model.layers.31.mlp.experts.100.up_proj", "model.layers.31.mlp.experts.101.up_proj", "model.layers.31.mlp.experts.102.up_proj", "model.layers.31.mlp.experts.103.up_proj", "model.layers.31.mlp.experts.104.up_proj", "model.layers.31.mlp.experts.105.up_proj", "model.layers.31.mlp.experts.106.up_proj", "model.layers.31.mlp.experts.107.up_proj", "model.layers.31.mlp.experts.108.up_proj", "model.layers.31.mlp.experts.109.up_proj", "model.layers.31.mlp.experts.110.up_proj", "model.layers.31.mlp.experts.111.up_proj", "model.layers.31.mlp.experts.112.up_proj", "model.layers.31.mlp.experts.113.up_proj", "model.layers.31.mlp.experts.114.up_proj", "model.layers.31.mlp.experts.115.up_proj", "model.layers.31.mlp.experts.116.up_proj", "model.layers.31.mlp.experts.117.up_proj", "model.layers.31.mlp.experts.118.up_proj", "model.layers.31.mlp.experts.119.up_proj", "model.layers.31.mlp.experts.120.up_proj", "model.layers.31.mlp.experts.121.up_proj", "model.layers.31.mlp.experts.122.up_proj", "model.layers.31.mlp.experts.123.up_proj", "model.layers.31.mlp.experts.124.up_proj", "model.layers.31.mlp.experts.125.up_proj", "model.layers.31.mlp.experts.126.up_proj", "model.layers.31.mlp.experts.127.up_proj", "model.layers.31.mlp.experts.0.down_proj", "model.layers.31.mlp.experts.1.down_proj", "model.layers.31.mlp.experts.2.down_proj", "model.layers.31.mlp.experts.3.down_proj", "model.layers.31.mlp.experts.4.down_proj", "model.layers.31.mlp.experts.5.down_proj", "model.layers.31.mlp.experts.6.down_proj", "model.layers.31.mlp.experts.7.down_proj", "model.layers.31.mlp.experts.8.down_proj", "model.layers.31.mlp.experts.9.down_proj", "model.layers.31.mlp.experts.10.down_proj", "model.layers.31.mlp.experts.11.down_proj", "model.layers.31.mlp.experts.12.down_proj", "model.layers.31.mlp.experts.13.down_proj", "model.layers.31.mlp.experts.14.down_proj", "model.layers.31.mlp.experts.15.down_proj", "model.layers.31.mlp.experts.16.down_proj", "model.layers.31.mlp.experts.17.down_proj", "model.layers.31.mlp.experts.18.down_proj", "model.layers.31.mlp.experts.19.down_proj", "model.layers.31.mlp.experts.20.down_proj", "model.layers.31.mlp.experts.21.down_proj", "model.layers.31.mlp.experts.22.down_proj", "model.layers.31.mlp.experts.23.down_proj", "model.layers.31.mlp.experts.24.down_proj", "model.layers.31.mlp.experts.25.down_proj", "model.layers.31.mlp.experts.26.down_proj", "model.layers.31.mlp.experts.27.down_proj", "model.layers.31.mlp.experts.28.down_proj", "model.layers.31.mlp.experts.29.down_proj", "model.layers.31.mlp.experts.30.down_proj", "model.layers.31.mlp.experts.31.down_proj", "model.layers.31.mlp.experts.32.down_proj", "model.layers.31.mlp.experts.33.down_proj", "model.layers.31.mlp.experts.34.down_proj", "model.layers.31.mlp.experts.35.down_proj", "model.layers.31.mlp.experts.36.down_proj", "model.layers.31.mlp.experts.37.down_proj", "model.layers.31.mlp.experts.38.down_proj", "model.layers.31.mlp.experts.39.down_proj", "model.layers.31.mlp.experts.40.down_proj", "model.layers.31.mlp.experts.41.down_proj", "model.layers.31.mlp.experts.42.down_proj", "model.layers.31.mlp.experts.43.down_proj", "model.layers.31.mlp.experts.44.down_proj", "model.layers.31.mlp.experts.45.down_proj", "model.layers.31.mlp.experts.46.down_proj", "model.layers.31.mlp.experts.47.down_proj", "model.layers.31.mlp.experts.48.down_proj", "model.layers.31.mlp.experts.49.down_proj", "model.layers.31.mlp.experts.50.down_proj", "model.layers.31.mlp.experts.51.down_proj", "model.layers.31.mlp.experts.52.down_proj", "model.layers.31.mlp.experts.53.down_proj", "model.layers.31.mlp.experts.54.down_proj", "model.layers.31.mlp.experts.55.down_proj", "model.layers.31.mlp.experts.56.down_proj", "model.layers.31.mlp.experts.57.down_proj", "model.layers.31.mlp.experts.58.down_proj", "model.layers.31.mlp.experts.59.down_proj", "model.layers.31.mlp.experts.60.down_proj", "model.layers.31.mlp.experts.61.down_proj", "model.layers.31.mlp.experts.62.down_proj", "model.layers.31.mlp.experts.63.down_proj", "model.layers.31.mlp.experts.64.down_proj", "model.layers.31.mlp.experts.65.down_proj", "model.layers.31.mlp.experts.66.down_proj", "model.layers.31.mlp.experts.67.down_proj", "model.layers.31.mlp.experts.68.down_proj", "model.layers.31.mlp.experts.69.down_proj", "model.layers.31.mlp.experts.70.down_proj", "model.layers.31.mlp.experts.71.down_proj", "model.layers.31.mlp.experts.72.down_proj", "model.layers.31.mlp.experts.73.down_proj", "model.layers.31.mlp.experts.74.down_proj", "model.layers.31.mlp.experts.75.down_proj", "model.layers.31.mlp.experts.76.down_proj", "model.layers.31.mlp.experts.77.down_proj", "model.layers.31.mlp.experts.78.down_proj", "model.layers.31.mlp.experts.79.down_proj", "model.layers.31.mlp.experts.80.down_proj", "model.layers.31.mlp.experts.81.down_proj", "model.layers.31.mlp.experts.82.down_proj", "model.layers.31.mlp.experts.83.down_proj", "model.layers.31.mlp.experts.84.down_proj", "model.layers.31.mlp.experts.85.down_proj", "model.layers.31.mlp.experts.86.down_proj", "model.layers.31.mlp.experts.87.down_proj", "model.layers.31.mlp.experts.88.down_proj", "model.layers.31.mlp.experts.89.down_proj", "model.layers.31.mlp.experts.90.down_proj", "model.layers.31.mlp.experts.91.down_proj", "model.layers.31.mlp.experts.92.down_proj", "model.layers.31.mlp.experts.93.down_proj", "model.layers.31.mlp.experts.94.down_proj", "model.layers.31.mlp.experts.95.down_proj", "model.layers.31.mlp.experts.96.down_proj", "model.layers.31.mlp.experts.97.down_proj", "model.layers.31.mlp.experts.98.down_proj", "model.layers.31.mlp.experts.99.down_proj", "model.layers.31.mlp.experts.100.down_proj", "model.layers.31.mlp.experts.101.down_proj", "model.layers.31.mlp.experts.102.down_proj", "model.layers.31.mlp.experts.103.down_proj", "model.layers.31.mlp.experts.104.down_proj", "model.layers.31.mlp.experts.105.down_proj", "model.layers.31.mlp.experts.106.down_proj", "model.layers.31.mlp.experts.107.down_proj", "model.layers.31.mlp.experts.108.down_proj", "model.layers.31.mlp.experts.109.down_proj", "model.layers.31.mlp.experts.110.down_proj", "model.layers.31.mlp.experts.111.down_proj", "model.layers.31.mlp.experts.112.down_proj", "model.layers.31.mlp.experts.113.down_proj", "model.layers.31.mlp.experts.114.down_proj", "model.layers.31.mlp.experts.115.down_proj", "model.layers.31.mlp.experts.116.down_proj", "model.layers.31.mlp.experts.117.down_proj", "model.layers.31.mlp.experts.118.down_proj", "model.layers.31.mlp.experts.119.down_proj", "model.layers.31.mlp.experts.120.down_proj", "model.layers.31.mlp.experts.121.down_proj", "model.layers.31.mlp.experts.122.down_proj", "model.layers.31.mlp.experts.123.down_proj", "model.layers.31.mlp.experts.124.down_proj", "model.layers.31.mlp.experts.125.down_proj", "model.layers.31.mlp.experts.126.down_proj", "model.layers.31.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.000952167622745026, "dbits": 2214592512 } ] }, { "idx": 95, "layers": [ "model.layers.32.self_attn.q_proj", "model.layers.32.self_attn.k_proj", "model.layers.32.self_attn.v_proj", "model.layers.32.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00035283332690597136, "dbits": 109051904 } ] }, { "idx": 96, "layers": [ "model.layers.32.mlp.shared_experts.gate_proj", "model.layers.32.mlp.shared_experts.up_proj", "model.layers.32.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 2.030683681369938e-05, "dbits": 17301504 } ] }, { "idx": 97, "layers": [ "model.layers.32.mlp.experts.0.gate_proj", "model.layers.32.mlp.experts.1.gate_proj", "model.layers.32.mlp.experts.2.gate_proj", "model.layers.32.mlp.experts.3.gate_proj", "model.layers.32.mlp.experts.4.gate_proj", "model.layers.32.mlp.experts.5.gate_proj", "model.layers.32.mlp.experts.6.gate_proj", "model.layers.32.mlp.experts.7.gate_proj", "model.layers.32.mlp.experts.8.gate_proj", "model.layers.32.mlp.experts.9.gate_proj", "model.layers.32.mlp.experts.10.gate_proj", "model.layers.32.mlp.experts.11.gate_proj", "model.layers.32.mlp.experts.12.gate_proj", "model.layers.32.mlp.experts.13.gate_proj", "model.layers.32.mlp.experts.14.gate_proj", "model.layers.32.mlp.experts.15.gate_proj", "model.layers.32.mlp.experts.16.gate_proj", "model.layers.32.mlp.experts.17.gate_proj", "model.layers.32.mlp.experts.18.gate_proj", "model.layers.32.mlp.experts.19.gate_proj", "model.layers.32.mlp.experts.20.gate_proj", "model.layers.32.mlp.experts.21.gate_proj", "model.layers.32.mlp.experts.22.gate_proj", "model.layers.32.mlp.experts.23.gate_proj", "model.layers.32.mlp.experts.24.gate_proj", "model.layers.32.mlp.experts.25.gate_proj", "model.layers.32.mlp.experts.26.gate_proj", "model.layers.32.mlp.experts.27.gate_proj", "model.layers.32.mlp.experts.28.gate_proj", "model.layers.32.mlp.experts.29.gate_proj", "model.layers.32.mlp.experts.30.gate_proj", "model.layers.32.mlp.experts.31.gate_proj", "model.layers.32.mlp.experts.32.gate_proj", "model.layers.32.mlp.experts.33.gate_proj", "model.layers.32.mlp.experts.34.gate_proj", "model.layers.32.mlp.experts.35.gate_proj", "model.layers.32.mlp.experts.36.gate_proj", "model.layers.32.mlp.experts.37.gate_proj", "model.layers.32.mlp.experts.38.gate_proj", "model.layers.32.mlp.experts.39.gate_proj", "model.layers.32.mlp.experts.40.gate_proj", "model.layers.32.mlp.experts.41.gate_proj", "model.layers.32.mlp.experts.42.gate_proj", "model.layers.32.mlp.experts.43.gate_proj", "model.layers.32.mlp.experts.44.gate_proj", "model.layers.32.mlp.experts.45.gate_proj", "model.layers.32.mlp.experts.46.gate_proj", "model.layers.32.mlp.experts.47.gate_proj", "model.layers.32.mlp.experts.48.gate_proj", "model.layers.32.mlp.experts.49.gate_proj", "model.layers.32.mlp.experts.50.gate_proj", "model.layers.32.mlp.experts.51.gate_proj", "model.layers.32.mlp.experts.52.gate_proj", "model.layers.32.mlp.experts.53.gate_proj", "model.layers.32.mlp.experts.54.gate_proj", "model.layers.32.mlp.experts.55.gate_proj", "model.layers.32.mlp.experts.56.gate_proj", "model.layers.32.mlp.experts.57.gate_proj", "model.layers.32.mlp.experts.58.gate_proj", "model.layers.32.mlp.experts.59.gate_proj", "model.layers.32.mlp.experts.60.gate_proj", "model.layers.32.mlp.experts.61.gate_proj", "model.layers.32.mlp.experts.62.gate_proj", "model.layers.32.mlp.experts.63.gate_proj", "model.layers.32.mlp.experts.64.gate_proj", "model.layers.32.mlp.experts.65.gate_proj", "model.layers.32.mlp.experts.66.gate_proj", "model.layers.32.mlp.experts.67.gate_proj", "model.layers.32.mlp.experts.68.gate_proj", "model.layers.32.mlp.experts.69.gate_proj", "model.layers.32.mlp.experts.70.gate_proj", "model.layers.32.mlp.experts.71.gate_proj", "model.layers.32.mlp.experts.72.gate_proj", "model.layers.32.mlp.experts.73.gate_proj", "model.layers.32.mlp.experts.74.gate_proj", "model.layers.32.mlp.experts.75.gate_proj", "model.layers.32.mlp.experts.76.gate_proj", "model.layers.32.mlp.experts.77.gate_proj", "model.layers.32.mlp.experts.78.gate_proj", "model.layers.32.mlp.experts.79.gate_proj", "model.layers.32.mlp.experts.80.gate_proj", "model.layers.32.mlp.experts.81.gate_proj", "model.layers.32.mlp.experts.82.gate_proj", "model.layers.32.mlp.experts.83.gate_proj", "model.layers.32.mlp.experts.84.gate_proj", "model.layers.32.mlp.experts.85.gate_proj", "model.layers.32.mlp.experts.86.gate_proj", "model.layers.32.mlp.experts.87.gate_proj", "model.layers.32.mlp.experts.88.gate_proj", "model.layers.32.mlp.experts.89.gate_proj", "model.layers.32.mlp.experts.90.gate_proj", "model.layers.32.mlp.experts.91.gate_proj", "model.layers.32.mlp.experts.92.gate_proj", "model.layers.32.mlp.experts.93.gate_proj", "model.layers.32.mlp.experts.94.gate_proj", "model.layers.32.mlp.experts.95.gate_proj", "model.layers.32.mlp.experts.96.gate_proj", "model.layers.32.mlp.experts.97.gate_proj", "model.layers.32.mlp.experts.98.gate_proj", "model.layers.32.mlp.experts.99.gate_proj", "model.layers.32.mlp.experts.100.gate_proj", "model.layers.32.mlp.experts.101.gate_proj", "model.layers.32.mlp.experts.102.gate_proj", "model.layers.32.mlp.experts.103.gate_proj", "model.layers.32.mlp.experts.104.gate_proj", "model.layers.32.mlp.experts.105.gate_proj", "model.layers.32.mlp.experts.106.gate_proj", "model.layers.32.mlp.experts.107.gate_proj", "model.layers.32.mlp.experts.108.gate_proj", "model.layers.32.mlp.experts.109.gate_proj", "model.layers.32.mlp.experts.110.gate_proj", "model.layers.32.mlp.experts.111.gate_proj", "model.layers.32.mlp.experts.112.gate_proj", "model.layers.32.mlp.experts.113.gate_proj", "model.layers.32.mlp.experts.114.gate_proj", "model.layers.32.mlp.experts.115.gate_proj", "model.layers.32.mlp.experts.116.gate_proj", "model.layers.32.mlp.experts.117.gate_proj", "model.layers.32.mlp.experts.118.gate_proj", "model.layers.32.mlp.experts.119.gate_proj", "model.layers.32.mlp.experts.120.gate_proj", "model.layers.32.mlp.experts.121.gate_proj", "model.layers.32.mlp.experts.122.gate_proj", "model.layers.32.mlp.experts.123.gate_proj", "model.layers.32.mlp.experts.124.gate_proj", "model.layers.32.mlp.experts.125.gate_proj", "model.layers.32.mlp.experts.126.gate_proj", "model.layers.32.mlp.experts.127.gate_proj", "model.layers.32.mlp.experts.0.up_proj", "model.layers.32.mlp.experts.1.up_proj", "model.layers.32.mlp.experts.2.up_proj", "model.layers.32.mlp.experts.3.up_proj", "model.layers.32.mlp.experts.4.up_proj", "model.layers.32.mlp.experts.5.up_proj", "model.layers.32.mlp.experts.6.up_proj", "model.layers.32.mlp.experts.7.up_proj", "model.layers.32.mlp.experts.8.up_proj", "model.layers.32.mlp.experts.9.up_proj", "model.layers.32.mlp.experts.10.up_proj", "model.layers.32.mlp.experts.11.up_proj", "model.layers.32.mlp.experts.12.up_proj", "model.layers.32.mlp.experts.13.up_proj", "model.layers.32.mlp.experts.14.up_proj", "model.layers.32.mlp.experts.15.up_proj", "model.layers.32.mlp.experts.16.up_proj", "model.layers.32.mlp.experts.17.up_proj", "model.layers.32.mlp.experts.18.up_proj", "model.layers.32.mlp.experts.19.up_proj", "model.layers.32.mlp.experts.20.up_proj", "model.layers.32.mlp.experts.21.up_proj", "model.layers.32.mlp.experts.22.up_proj", "model.layers.32.mlp.experts.23.up_proj", "model.layers.32.mlp.experts.24.up_proj", "model.layers.32.mlp.experts.25.up_proj", "model.layers.32.mlp.experts.26.up_proj", "model.layers.32.mlp.experts.27.up_proj", "model.layers.32.mlp.experts.28.up_proj", "model.layers.32.mlp.experts.29.up_proj", "model.layers.32.mlp.experts.30.up_proj", "model.layers.32.mlp.experts.31.up_proj", "model.layers.32.mlp.experts.32.up_proj", "model.layers.32.mlp.experts.33.up_proj", "model.layers.32.mlp.experts.34.up_proj", "model.layers.32.mlp.experts.35.up_proj", "model.layers.32.mlp.experts.36.up_proj", "model.layers.32.mlp.experts.37.up_proj", "model.layers.32.mlp.experts.38.up_proj", "model.layers.32.mlp.experts.39.up_proj", "model.layers.32.mlp.experts.40.up_proj", "model.layers.32.mlp.experts.41.up_proj", "model.layers.32.mlp.experts.42.up_proj", "model.layers.32.mlp.experts.43.up_proj", "model.layers.32.mlp.experts.44.up_proj", "model.layers.32.mlp.experts.45.up_proj", "model.layers.32.mlp.experts.46.up_proj", "model.layers.32.mlp.experts.47.up_proj", "model.layers.32.mlp.experts.48.up_proj", "model.layers.32.mlp.experts.49.up_proj", "model.layers.32.mlp.experts.50.up_proj", "model.layers.32.mlp.experts.51.up_proj", "model.layers.32.mlp.experts.52.up_proj", "model.layers.32.mlp.experts.53.up_proj", "model.layers.32.mlp.experts.54.up_proj", "model.layers.32.mlp.experts.55.up_proj", "model.layers.32.mlp.experts.56.up_proj", "model.layers.32.mlp.experts.57.up_proj", "model.layers.32.mlp.experts.58.up_proj", "model.layers.32.mlp.experts.59.up_proj", "model.layers.32.mlp.experts.60.up_proj", "model.layers.32.mlp.experts.61.up_proj", "model.layers.32.mlp.experts.62.up_proj", "model.layers.32.mlp.experts.63.up_proj", "model.layers.32.mlp.experts.64.up_proj", "model.layers.32.mlp.experts.65.up_proj", "model.layers.32.mlp.experts.66.up_proj", "model.layers.32.mlp.experts.67.up_proj", "model.layers.32.mlp.experts.68.up_proj", "model.layers.32.mlp.experts.69.up_proj", "model.layers.32.mlp.experts.70.up_proj", "model.layers.32.mlp.experts.71.up_proj", "model.layers.32.mlp.experts.72.up_proj", "model.layers.32.mlp.experts.73.up_proj", "model.layers.32.mlp.experts.74.up_proj", "model.layers.32.mlp.experts.75.up_proj", "model.layers.32.mlp.experts.76.up_proj", "model.layers.32.mlp.experts.77.up_proj", "model.layers.32.mlp.experts.78.up_proj", "model.layers.32.mlp.experts.79.up_proj", "model.layers.32.mlp.experts.80.up_proj", "model.layers.32.mlp.experts.81.up_proj", "model.layers.32.mlp.experts.82.up_proj", "model.layers.32.mlp.experts.83.up_proj", "model.layers.32.mlp.experts.84.up_proj", "model.layers.32.mlp.experts.85.up_proj", "model.layers.32.mlp.experts.86.up_proj", "model.layers.32.mlp.experts.87.up_proj", "model.layers.32.mlp.experts.88.up_proj", "model.layers.32.mlp.experts.89.up_proj", "model.layers.32.mlp.experts.90.up_proj", "model.layers.32.mlp.experts.91.up_proj", "model.layers.32.mlp.experts.92.up_proj", "model.layers.32.mlp.experts.93.up_proj", "model.layers.32.mlp.experts.94.up_proj", "model.layers.32.mlp.experts.95.up_proj", "model.layers.32.mlp.experts.96.up_proj", "model.layers.32.mlp.experts.97.up_proj", "model.layers.32.mlp.experts.98.up_proj", "model.layers.32.mlp.experts.99.up_proj", "model.layers.32.mlp.experts.100.up_proj", "model.layers.32.mlp.experts.101.up_proj", "model.layers.32.mlp.experts.102.up_proj", "model.layers.32.mlp.experts.103.up_proj", "model.layers.32.mlp.experts.104.up_proj", "model.layers.32.mlp.experts.105.up_proj", "model.layers.32.mlp.experts.106.up_proj", "model.layers.32.mlp.experts.107.up_proj", "model.layers.32.mlp.experts.108.up_proj", "model.layers.32.mlp.experts.109.up_proj", "model.layers.32.mlp.experts.110.up_proj", "model.layers.32.mlp.experts.111.up_proj", "model.layers.32.mlp.experts.112.up_proj", "model.layers.32.mlp.experts.113.up_proj", "model.layers.32.mlp.experts.114.up_proj", "model.layers.32.mlp.experts.115.up_proj", "model.layers.32.mlp.experts.116.up_proj", "model.layers.32.mlp.experts.117.up_proj", "model.layers.32.mlp.experts.118.up_proj", "model.layers.32.mlp.experts.119.up_proj", "model.layers.32.mlp.experts.120.up_proj", "model.layers.32.mlp.experts.121.up_proj", "model.layers.32.mlp.experts.122.up_proj", "model.layers.32.mlp.experts.123.up_proj", "model.layers.32.mlp.experts.124.up_proj", "model.layers.32.mlp.experts.125.up_proj", "model.layers.32.mlp.experts.126.up_proj", "model.layers.32.mlp.experts.127.up_proj", "model.layers.32.mlp.experts.0.down_proj", "model.layers.32.mlp.experts.1.down_proj", "model.layers.32.mlp.experts.2.down_proj", "model.layers.32.mlp.experts.3.down_proj", "model.layers.32.mlp.experts.4.down_proj", "model.layers.32.mlp.experts.5.down_proj", "model.layers.32.mlp.experts.6.down_proj", "model.layers.32.mlp.experts.7.down_proj", "model.layers.32.mlp.experts.8.down_proj", "model.layers.32.mlp.experts.9.down_proj", "model.layers.32.mlp.experts.10.down_proj", "model.layers.32.mlp.experts.11.down_proj", "model.layers.32.mlp.experts.12.down_proj", "model.layers.32.mlp.experts.13.down_proj", "model.layers.32.mlp.experts.14.down_proj", "model.layers.32.mlp.experts.15.down_proj", "model.layers.32.mlp.experts.16.down_proj", "model.layers.32.mlp.experts.17.down_proj", "model.layers.32.mlp.experts.18.down_proj", "model.layers.32.mlp.experts.19.down_proj", "model.layers.32.mlp.experts.20.down_proj", "model.layers.32.mlp.experts.21.down_proj", "model.layers.32.mlp.experts.22.down_proj", "model.layers.32.mlp.experts.23.down_proj", "model.layers.32.mlp.experts.24.down_proj", "model.layers.32.mlp.experts.25.down_proj", "model.layers.32.mlp.experts.26.down_proj", "model.layers.32.mlp.experts.27.down_proj", "model.layers.32.mlp.experts.28.down_proj", "model.layers.32.mlp.experts.29.down_proj", "model.layers.32.mlp.experts.30.down_proj", "model.layers.32.mlp.experts.31.down_proj", "model.layers.32.mlp.experts.32.down_proj", "model.layers.32.mlp.experts.33.down_proj", "model.layers.32.mlp.experts.34.down_proj", "model.layers.32.mlp.experts.35.down_proj", "model.layers.32.mlp.experts.36.down_proj", "model.layers.32.mlp.experts.37.down_proj", "model.layers.32.mlp.experts.38.down_proj", "model.layers.32.mlp.experts.39.down_proj", "model.layers.32.mlp.experts.40.down_proj", "model.layers.32.mlp.experts.41.down_proj", "model.layers.32.mlp.experts.42.down_proj", "model.layers.32.mlp.experts.43.down_proj", "model.layers.32.mlp.experts.44.down_proj", "model.layers.32.mlp.experts.45.down_proj", "model.layers.32.mlp.experts.46.down_proj", "model.layers.32.mlp.experts.47.down_proj", "model.layers.32.mlp.experts.48.down_proj", "model.layers.32.mlp.experts.49.down_proj", "model.layers.32.mlp.experts.50.down_proj", "model.layers.32.mlp.experts.51.down_proj", "model.layers.32.mlp.experts.52.down_proj", "model.layers.32.mlp.experts.53.down_proj", "model.layers.32.mlp.experts.54.down_proj", "model.layers.32.mlp.experts.55.down_proj", "model.layers.32.mlp.experts.56.down_proj", "model.layers.32.mlp.experts.57.down_proj", "model.layers.32.mlp.experts.58.down_proj", "model.layers.32.mlp.experts.59.down_proj", "model.layers.32.mlp.experts.60.down_proj", "model.layers.32.mlp.experts.61.down_proj", "model.layers.32.mlp.experts.62.down_proj", "model.layers.32.mlp.experts.63.down_proj", "model.layers.32.mlp.experts.64.down_proj", "model.layers.32.mlp.experts.65.down_proj", "model.layers.32.mlp.experts.66.down_proj", "model.layers.32.mlp.experts.67.down_proj", "model.layers.32.mlp.experts.68.down_proj", "model.layers.32.mlp.experts.69.down_proj", "model.layers.32.mlp.experts.70.down_proj", "model.layers.32.mlp.experts.71.down_proj", "model.layers.32.mlp.experts.72.down_proj", "model.layers.32.mlp.experts.73.down_proj", "model.layers.32.mlp.experts.74.down_proj", "model.layers.32.mlp.experts.75.down_proj", "model.layers.32.mlp.experts.76.down_proj", "model.layers.32.mlp.experts.77.down_proj", "model.layers.32.mlp.experts.78.down_proj", "model.layers.32.mlp.experts.79.down_proj", "model.layers.32.mlp.experts.80.down_proj", "model.layers.32.mlp.experts.81.down_proj", "model.layers.32.mlp.experts.82.down_proj", "model.layers.32.mlp.experts.83.down_proj", "model.layers.32.mlp.experts.84.down_proj", "model.layers.32.mlp.experts.85.down_proj", "model.layers.32.mlp.experts.86.down_proj", "model.layers.32.mlp.experts.87.down_proj", "model.layers.32.mlp.experts.88.down_proj", "model.layers.32.mlp.experts.89.down_proj", "model.layers.32.mlp.experts.90.down_proj", "model.layers.32.mlp.experts.91.down_proj", "model.layers.32.mlp.experts.92.down_proj", "model.layers.32.mlp.experts.93.down_proj", "model.layers.32.mlp.experts.94.down_proj", "model.layers.32.mlp.experts.95.down_proj", "model.layers.32.mlp.experts.96.down_proj", "model.layers.32.mlp.experts.97.down_proj", "model.layers.32.mlp.experts.98.down_proj", "model.layers.32.mlp.experts.99.down_proj", "model.layers.32.mlp.experts.100.down_proj", "model.layers.32.mlp.experts.101.down_proj", "model.layers.32.mlp.experts.102.down_proj", "model.layers.32.mlp.experts.103.down_proj", "model.layers.32.mlp.experts.104.down_proj", "model.layers.32.mlp.experts.105.down_proj", "model.layers.32.mlp.experts.106.down_proj", "model.layers.32.mlp.experts.107.down_proj", "model.layers.32.mlp.experts.108.down_proj", "model.layers.32.mlp.experts.109.down_proj", "model.layers.32.mlp.experts.110.down_proj", "model.layers.32.mlp.experts.111.down_proj", "model.layers.32.mlp.experts.112.down_proj", "model.layers.32.mlp.experts.113.down_proj", "model.layers.32.mlp.experts.114.down_proj", "model.layers.32.mlp.experts.115.down_proj", "model.layers.32.mlp.experts.116.down_proj", "model.layers.32.mlp.experts.117.down_proj", "model.layers.32.mlp.experts.118.down_proj", "model.layers.32.mlp.experts.119.down_proj", "model.layers.32.mlp.experts.120.down_proj", "model.layers.32.mlp.experts.121.down_proj", "model.layers.32.mlp.experts.122.down_proj", "model.layers.32.mlp.experts.123.down_proj", "model.layers.32.mlp.experts.124.down_proj", "model.layers.32.mlp.experts.125.down_proj", "model.layers.32.mlp.experts.126.down_proj", "model.layers.32.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0010028410702943857, "dbits": 2214592512 } ] }, { "idx": 98, "layers": [ "model.layers.33.self_attn.q_proj", "model.layers.33.self_attn.k_proj", "model.layers.33.self_attn.v_proj", "model.layers.33.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0025051156058907564, "dbits": 109051904 } ] }, { "idx": 99, "layers": [ "model.layers.33.mlp.shared_experts.gate_proj", "model.layers.33.mlp.shared_experts.up_proj", "model.layers.33.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0009112244471907449, "dbits": 17301504 } ] }, { "idx": 100, "layers": [ "model.layers.33.mlp.experts.0.gate_proj", "model.layers.33.mlp.experts.1.gate_proj", "model.layers.33.mlp.experts.2.gate_proj", "model.layers.33.mlp.experts.3.gate_proj", "model.layers.33.mlp.experts.4.gate_proj", "model.layers.33.mlp.experts.5.gate_proj", "model.layers.33.mlp.experts.6.gate_proj", "model.layers.33.mlp.experts.7.gate_proj", "model.layers.33.mlp.experts.8.gate_proj", "model.layers.33.mlp.experts.9.gate_proj", "model.layers.33.mlp.experts.10.gate_proj", "model.layers.33.mlp.experts.11.gate_proj", "model.layers.33.mlp.experts.12.gate_proj", "model.layers.33.mlp.experts.13.gate_proj", "model.layers.33.mlp.experts.14.gate_proj", "model.layers.33.mlp.experts.15.gate_proj", "model.layers.33.mlp.experts.16.gate_proj", "model.layers.33.mlp.experts.17.gate_proj", "model.layers.33.mlp.experts.18.gate_proj", "model.layers.33.mlp.experts.19.gate_proj", "model.layers.33.mlp.experts.20.gate_proj", "model.layers.33.mlp.experts.21.gate_proj", "model.layers.33.mlp.experts.22.gate_proj", "model.layers.33.mlp.experts.23.gate_proj", "model.layers.33.mlp.experts.24.gate_proj", "model.layers.33.mlp.experts.25.gate_proj", "model.layers.33.mlp.experts.26.gate_proj", "model.layers.33.mlp.experts.27.gate_proj", "model.layers.33.mlp.experts.28.gate_proj", "model.layers.33.mlp.experts.29.gate_proj", "model.layers.33.mlp.experts.30.gate_proj", "model.layers.33.mlp.experts.31.gate_proj", "model.layers.33.mlp.experts.32.gate_proj", "model.layers.33.mlp.experts.33.gate_proj", "model.layers.33.mlp.experts.34.gate_proj", "model.layers.33.mlp.experts.35.gate_proj", "model.layers.33.mlp.experts.36.gate_proj", "model.layers.33.mlp.experts.37.gate_proj", "model.layers.33.mlp.experts.38.gate_proj", "model.layers.33.mlp.experts.39.gate_proj", "model.layers.33.mlp.experts.40.gate_proj", "model.layers.33.mlp.experts.41.gate_proj", "model.layers.33.mlp.experts.42.gate_proj", "model.layers.33.mlp.experts.43.gate_proj", "model.layers.33.mlp.experts.44.gate_proj", "model.layers.33.mlp.experts.45.gate_proj", "model.layers.33.mlp.experts.46.gate_proj", "model.layers.33.mlp.experts.47.gate_proj", "model.layers.33.mlp.experts.48.gate_proj", "model.layers.33.mlp.experts.49.gate_proj", "model.layers.33.mlp.experts.50.gate_proj", "model.layers.33.mlp.experts.51.gate_proj", "model.layers.33.mlp.experts.52.gate_proj", "model.layers.33.mlp.experts.53.gate_proj", "model.layers.33.mlp.experts.54.gate_proj", "model.layers.33.mlp.experts.55.gate_proj", "model.layers.33.mlp.experts.56.gate_proj", "model.layers.33.mlp.experts.57.gate_proj", "model.layers.33.mlp.experts.58.gate_proj", "model.layers.33.mlp.experts.59.gate_proj", "model.layers.33.mlp.experts.60.gate_proj", "model.layers.33.mlp.experts.61.gate_proj", "model.layers.33.mlp.experts.62.gate_proj", "model.layers.33.mlp.experts.63.gate_proj", "model.layers.33.mlp.experts.64.gate_proj", "model.layers.33.mlp.experts.65.gate_proj", "model.layers.33.mlp.experts.66.gate_proj", "model.layers.33.mlp.experts.67.gate_proj", "model.layers.33.mlp.experts.68.gate_proj", "model.layers.33.mlp.experts.69.gate_proj", "model.layers.33.mlp.experts.70.gate_proj", "model.layers.33.mlp.experts.71.gate_proj", "model.layers.33.mlp.experts.72.gate_proj", "model.layers.33.mlp.experts.73.gate_proj", "model.layers.33.mlp.experts.74.gate_proj", "model.layers.33.mlp.experts.75.gate_proj", "model.layers.33.mlp.experts.76.gate_proj", "model.layers.33.mlp.experts.77.gate_proj", "model.layers.33.mlp.experts.78.gate_proj", "model.layers.33.mlp.experts.79.gate_proj", "model.layers.33.mlp.experts.80.gate_proj", "model.layers.33.mlp.experts.81.gate_proj", "model.layers.33.mlp.experts.82.gate_proj", "model.layers.33.mlp.experts.83.gate_proj", "model.layers.33.mlp.experts.84.gate_proj", "model.layers.33.mlp.experts.85.gate_proj", "model.layers.33.mlp.experts.86.gate_proj", "model.layers.33.mlp.experts.87.gate_proj", "model.layers.33.mlp.experts.88.gate_proj", "model.layers.33.mlp.experts.89.gate_proj", "model.layers.33.mlp.experts.90.gate_proj", "model.layers.33.mlp.experts.91.gate_proj", "model.layers.33.mlp.experts.92.gate_proj", "model.layers.33.mlp.experts.93.gate_proj", "model.layers.33.mlp.experts.94.gate_proj", "model.layers.33.mlp.experts.95.gate_proj", "model.layers.33.mlp.experts.96.gate_proj", "model.layers.33.mlp.experts.97.gate_proj", "model.layers.33.mlp.experts.98.gate_proj", "model.layers.33.mlp.experts.99.gate_proj", "model.layers.33.mlp.experts.100.gate_proj", "model.layers.33.mlp.experts.101.gate_proj", "model.layers.33.mlp.experts.102.gate_proj", "model.layers.33.mlp.experts.103.gate_proj", "model.layers.33.mlp.experts.104.gate_proj", "model.layers.33.mlp.experts.105.gate_proj", "model.layers.33.mlp.experts.106.gate_proj", "model.layers.33.mlp.experts.107.gate_proj", "model.layers.33.mlp.experts.108.gate_proj", "model.layers.33.mlp.experts.109.gate_proj", "model.layers.33.mlp.experts.110.gate_proj", "model.layers.33.mlp.experts.111.gate_proj", "model.layers.33.mlp.experts.112.gate_proj", "model.layers.33.mlp.experts.113.gate_proj", "model.layers.33.mlp.experts.114.gate_proj", "model.layers.33.mlp.experts.115.gate_proj", "model.layers.33.mlp.experts.116.gate_proj", "model.layers.33.mlp.experts.117.gate_proj", "model.layers.33.mlp.experts.118.gate_proj", "model.layers.33.mlp.experts.119.gate_proj", "model.layers.33.mlp.experts.120.gate_proj", "model.layers.33.mlp.experts.121.gate_proj", "model.layers.33.mlp.experts.122.gate_proj", "model.layers.33.mlp.experts.123.gate_proj", "model.layers.33.mlp.experts.124.gate_proj", "model.layers.33.mlp.experts.125.gate_proj", "model.layers.33.mlp.experts.126.gate_proj", "model.layers.33.mlp.experts.127.gate_proj", "model.layers.33.mlp.experts.0.up_proj", "model.layers.33.mlp.experts.1.up_proj", "model.layers.33.mlp.experts.2.up_proj", "model.layers.33.mlp.experts.3.up_proj", "model.layers.33.mlp.experts.4.up_proj", "model.layers.33.mlp.experts.5.up_proj", "model.layers.33.mlp.experts.6.up_proj", "model.layers.33.mlp.experts.7.up_proj", "model.layers.33.mlp.experts.8.up_proj", "model.layers.33.mlp.experts.9.up_proj", "model.layers.33.mlp.experts.10.up_proj", "model.layers.33.mlp.experts.11.up_proj", "model.layers.33.mlp.experts.12.up_proj", "model.layers.33.mlp.experts.13.up_proj", "model.layers.33.mlp.experts.14.up_proj", "model.layers.33.mlp.experts.15.up_proj", "model.layers.33.mlp.experts.16.up_proj", "model.layers.33.mlp.experts.17.up_proj", "model.layers.33.mlp.experts.18.up_proj", "model.layers.33.mlp.experts.19.up_proj", "model.layers.33.mlp.experts.20.up_proj", "model.layers.33.mlp.experts.21.up_proj", "model.layers.33.mlp.experts.22.up_proj", "model.layers.33.mlp.experts.23.up_proj", "model.layers.33.mlp.experts.24.up_proj", "model.layers.33.mlp.experts.25.up_proj", "model.layers.33.mlp.experts.26.up_proj", "model.layers.33.mlp.experts.27.up_proj", "model.layers.33.mlp.experts.28.up_proj", "model.layers.33.mlp.experts.29.up_proj", "model.layers.33.mlp.experts.30.up_proj", "model.layers.33.mlp.experts.31.up_proj", "model.layers.33.mlp.experts.32.up_proj", "model.layers.33.mlp.experts.33.up_proj", "model.layers.33.mlp.experts.34.up_proj", "model.layers.33.mlp.experts.35.up_proj", "model.layers.33.mlp.experts.36.up_proj", "model.layers.33.mlp.experts.37.up_proj", "model.layers.33.mlp.experts.38.up_proj", "model.layers.33.mlp.experts.39.up_proj", "model.layers.33.mlp.experts.40.up_proj", "model.layers.33.mlp.experts.41.up_proj", "model.layers.33.mlp.experts.42.up_proj", "model.layers.33.mlp.experts.43.up_proj", "model.layers.33.mlp.experts.44.up_proj", "model.layers.33.mlp.experts.45.up_proj", "model.layers.33.mlp.experts.46.up_proj", "model.layers.33.mlp.experts.47.up_proj", "model.layers.33.mlp.experts.48.up_proj", "model.layers.33.mlp.experts.49.up_proj", "model.layers.33.mlp.experts.50.up_proj", "model.layers.33.mlp.experts.51.up_proj", "model.layers.33.mlp.experts.52.up_proj", "model.layers.33.mlp.experts.53.up_proj", "model.layers.33.mlp.experts.54.up_proj", "model.layers.33.mlp.experts.55.up_proj", "model.layers.33.mlp.experts.56.up_proj", "model.layers.33.mlp.experts.57.up_proj", "model.layers.33.mlp.experts.58.up_proj", "model.layers.33.mlp.experts.59.up_proj", "model.layers.33.mlp.experts.60.up_proj", "model.layers.33.mlp.experts.61.up_proj", "model.layers.33.mlp.experts.62.up_proj", "model.layers.33.mlp.experts.63.up_proj", "model.layers.33.mlp.experts.64.up_proj", "model.layers.33.mlp.experts.65.up_proj", "model.layers.33.mlp.experts.66.up_proj", "model.layers.33.mlp.experts.67.up_proj", "model.layers.33.mlp.experts.68.up_proj", "model.layers.33.mlp.experts.69.up_proj", "model.layers.33.mlp.experts.70.up_proj", "model.layers.33.mlp.experts.71.up_proj", "model.layers.33.mlp.experts.72.up_proj", "model.layers.33.mlp.experts.73.up_proj", "model.layers.33.mlp.experts.74.up_proj", "model.layers.33.mlp.experts.75.up_proj", "model.layers.33.mlp.experts.76.up_proj", "model.layers.33.mlp.experts.77.up_proj", "model.layers.33.mlp.experts.78.up_proj", "model.layers.33.mlp.experts.79.up_proj", "model.layers.33.mlp.experts.80.up_proj", "model.layers.33.mlp.experts.81.up_proj", "model.layers.33.mlp.experts.82.up_proj", "model.layers.33.mlp.experts.83.up_proj", "model.layers.33.mlp.experts.84.up_proj", "model.layers.33.mlp.experts.85.up_proj", "model.layers.33.mlp.experts.86.up_proj", "model.layers.33.mlp.experts.87.up_proj", "model.layers.33.mlp.experts.88.up_proj", "model.layers.33.mlp.experts.89.up_proj", "model.layers.33.mlp.experts.90.up_proj", "model.layers.33.mlp.experts.91.up_proj", "model.layers.33.mlp.experts.92.up_proj", "model.layers.33.mlp.experts.93.up_proj", "model.layers.33.mlp.experts.94.up_proj", "model.layers.33.mlp.experts.95.up_proj", "model.layers.33.mlp.experts.96.up_proj", "model.layers.33.mlp.experts.97.up_proj", "model.layers.33.mlp.experts.98.up_proj", "model.layers.33.mlp.experts.99.up_proj", "model.layers.33.mlp.experts.100.up_proj", "model.layers.33.mlp.experts.101.up_proj", "model.layers.33.mlp.experts.102.up_proj", "model.layers.33.mlp.experts.103.up_proj", "model.layers.33.mlp.experts.104.up_proj", "model.layers.33.mlp.experts.105.up_proj", "model.layers.33.mlp.experts.106.up_proj", "model.layers.33.mlp.experts.107.up_proj", "model.layers.33.mlp.experts.108.up_proj", "model.layers.33.mlp.experts.109.up_proj", "model.layers.33.mlp.experts.110.up_proj", "model.layers.33.mlp.experts.111.up_proj", "model.layers.33.mlp.experts.112.up_proj", "model.layers.33.mlp.experts.113.up_proj", "model.layers.33.mlp.experts.114.up_proj", "model.layers.33.mlp.experts.115.up_proj", "model.layers.33.mlp.experts.116.up_proj", "model.layers.33.mlp.experts.117.up_proj", "model.layers.33.mlp.experts.118.up_proj", "model.layers.33.mlp.experts.119.up_proj", "model.layers.33.mlp.experts.120.up_proj", "model.layers.33.mlp.experts.121.up_proj", "model.layers.33.mlp.experts.122.up_proj", "model.layers.33.mlp.experts.123.up_proj", "model.layers.33.mlp.experts.124.up_proj", "model.layers.33.mlp.experts.125.up_proj", "model.layers.33.mlp.experts.126.up_proj", "model.layers.33.mlp.experts.127.up_proj", "model.layers.33.mlp.experts.0.down_proj", "model.layers.33.mlp.experts.1.down_proj", "model.layers.33.mlp.experts.2.down_proj", "model.layers.33.mlp.experts.3.down_proj", "model.layers.33.mlp.experts.4.down_proj", "model.layers.33.mlp.experts.5.down_proj", "model.layers.33.mlp.experts.6.down_proj", "model.layers.33.mlp.experts.7.down_proj", "model.layers.33.mlp.experts.8.down_proj", "model.layers.33.mlp.experts.9.down_proj", "model.layers.33.mlp.experts.10.down_proj", "model.layers.33.mlp.experts.11.down_proj", "model.layers.33.mlp.experts.12.down_proj", "model.layers.33.mlp.experts.13.down_proj", "model.layers.33.mlp.experts.14.down_proj", "model.layers.33.mlp.experts.15.down_proj", "model.layers.33.mlp.experts.16.down_proj", "model.layers.33.mlp.experts.17.down_proj", "model.layers.33.mlp.experts.18.down_proj", "model.layers.33.mlp.experts.19.down_proj", "model.layers.33.mlp.experts.20.down_proj", "model.layers.33.mlp.experts.21.down_proj", "model.layers.33.mlp.experts.22.down_proj", "model.layers.33.mlp.experts.23.down_proj", "model.layers.33.mlp.experts.24.down_proj", "model.layers.33.mlp.experts.25.down_proj", "model.layers.33.mlp.experts.26.down_proj", "model.layers.33.mlp.experts.27.down_proj", "model.layers.33.mlp.experts.28.down_proj", "model.layers.33.mlp.experts.29.down_proj", "model.layers.33.mlp.experts.30.down_proj", "model.layers.33.mlp.experts.31.down_proj", "model.layers.33.mlp.experts.32.down_proj", "model.layers.33.mlp.experts.33.down_proj", "model.layers.33.mlp.experts.34.down_proj", "model.layers.33.mlp.experts.35.down_proj", "model.layers.33.mlp.experts.36.down_proj", "model.layers.33.mlp.experts.37.down_proj", "model.layers.33.mlp.experts.38.down_proj", "model.layers.33.mlp.experts.39.down_proj", "model.layers.33.mlp.experts.40.down_proj", "model.layers.33.mlp.experts.41.down_proj", "model.layers.33.mlp.experts.42.down_proj", "model.layers.33.mlp.experts.43.down_proj", "model.layers.33.mlp.experts.44.down_proj", "model.layers.33.mlp.experts.45.down_proj", "model.layers.33.mlp.experts.46.down_proj", "model.layers.33.mlp.experts.47.down_proj", "model.layers.33.mlp.experts.48.down_proj", "model.layers.33.mlp.experts.49.down_proj", "model.layers.33.mlp.experts.50.down_proj", "model.layers.33.mlp.experts.51.down_proj", "model.layers.33.mlp.experts.52.down_proj", "model.layers.33.mlp.experts.53.down_proj", "model.layers.33.mlp.experts.54.down_proj", "model.layers.33.mlp.experts.55.down_proj", "model.layers.33.mlp.experts.56.down_proj", "model.layers.33.mlp.experts.57.down_proj", "model.layers.33.mlp.experts.58.down_proj", "model.layers.33.mlp.experts.59.down_proj", "model.layers.33.mlp.experts.60.down_proj", "model.layers.33.mlp.experts.61.down_proj", "model.layers.33.mlp.experts.62.down_proj", "model.layers.33.mlp.experts.63.down_proj", "model.layers.33.mlp.experts.64.down_proj", "model.layers.33.mlp.experts.65.down_proj", "model.layers.33.mlp.experts.66.down_proj", "model.layers.33.mlp.experts.67.down_proj", "model.layers.33.mlp.experts.68.down_proj", "model.layers.33.mlp.experts.69.down_proj", "model.layers.33.mlp.experts.70.down_proj", "model.layers.33.mlp.experts.71.down_proj", "model.layers.33.mlp.experts.72.down_proj", "model.layers.33.mlp.experts.73.down_proj", "model.layers.33.mlp.experts.74.down_proj", "model.layers.33.mlp.experts.75.down_proj", "model.layers.33.mlp.experts.76.down_proj", "model.layers.33.mlp.experts.77.down_proj", "model.layers.33.mlp.experts.78.down_proj", "model.layers.33.mlp.experts.79.down_proj", "model.layers.33.mlp.experts.80.down_proj", "model.layers.33.mlp.experts.81.down_proj", "model.layers.33.mlp.experts.82.down_proj", "model.layers.33.mlp.experts.83.down_proj", "model.layers.33.mlp.experts.84.down_proj", "model.layers.33.mlp.experts.85.down_proj", "model.layers.33.mlp.experts.86.down_proj", "model.layers.33.mlp.experts.87.down_proj", "model.layers.33.mlp.experts.88.down_proj", "model.layers.33.mlp.experts.89.down_proj", "model.layers.33.mlp.experts.90.down_proj", "model.layers.33.mlp.experts.91.down_proj", "model.layers.33.mlp.experts.92.down_proj", "model.layers.33.mlp.experts.93.down_proj", "model.layers.33.mlp.experts.94.down_proj", "model.layers.33.mlp.experts.95.down_proj", "model.layers.33.mlp.experts.96.down_proj", "model.layers.33.mlp.experts.97.down_proj", "model.layers.33.mlp.experts.98.down_proj", "model.layers.33.mlp.experts.99.down_proj", "model.layers.33.mlp.experts.100.down_proj", "model.layers.33.mlp.experts.101.down_proj", "model.layers.33.mlp.experts.102.down_proj", "model.layers.33.mlp.experts.103.down_proj", "model.layers.33.mlp.experts.104.down_proj", "model.layers.33.mlp.experts.105.down_proj", "model.layers.33.mlp.experts.106.down_proj", "model.layers.33.mlp.experts.107.down_proj", "model.layers.33.mlp.experts.108.down_proj", "model.layers.33.mlp.experts.109.down_proj", "model.layers.33.mlp.experts.110.down_proj", "model.layers.33.mlp.experts.111.down_proj", "model.layers.33.mlp.experts.112.down_proj", "model.layers.33.mlp.experts.113.down_proj", "model.layers.33.mlp.experts.114.down_proj", "model.layers.33.mlp.experts.115.down_proj", "model.layers.33.mlp.experts.116.down_proj", "model.layers.33.mlp.experts.117.down_proj", "model.layers.33.mlp.experts.118.down_proj", "model.layers.33.mlp.experts.119.down_proj", "model.layers.33.mlp.experts.120.down_proj", "model.layers.33.mlp.experts.121.down_proj", "model.layers.33.mlp.experts.122.down_proj", "model.layers.33.mlp.experts.123.down_proj", "model.layers.33.mlp.experts.124.down_proj", "model.layers.33.mlp.experts.125.down_proj", "model.layers.33.mlp.experts.126.down_proj", "model.layers.33.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0010661426000297125, "dbits": 2214592512 } ] }, { "idx": 101, "layers": [ "model.layers.34.self_attn.q_proj", "model.layers.34.self_attn.k_proj", "model.layers.34.self_attn.v_proj", "model.layers.34.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0004848079755902235, "dbits": 109051904 } ] }, { "idx": 102, "layers": [ "model.layers.34.mlp.shared_experts.gate_proj", "model.layers.34.mlp.shared_experts.up_proj", "model.layers.34.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 7.479954510927755e-05, "dbits": 17301504 } ] }, { "idx": 103, "layers": [ "model.layers.34.mlp.experts.0.gate_proj", "model.layers.34.mlp.experts.1.gate_proj", "model.layers.34.mlp.experts.2.gate_proj", "model.layers.34.mlp.experts.3.gate_proj", "model.layers.34.mlp.experts.4.gate_proj", "model.layers.34.mlp.experts.5.gate_proj", "model.layers.34.mlp.experts.6.gate_proj", "model.layers.34.mlp.experts.7.gate_proj", "model.layers.34.mlp.experts.8.gate_proj", "model.layers.34.mlp.experts.9.gate_proj", "model.layers.34.mlp.experts.10.gate_proj", "model.layers.34.mlp.experts.11.gate_proj", "model.layers.34.mlp.experts.12.gate_proj", "model.layers.34.mlp.experts.13.gate_proj", "model.layers.34.mlp.experts.14.gate_proj", "model.layers.34.mlp.experts.15.gate_proj", "model.layers.34.mlp.experts.16.gate_proj", "model.layers.34.mlp.experts.17.gate_proj", "model.layers.34.mlp.experts.18.gate_proj", "model.layers.34.mlp.experts.19.gate_proj", "model.layers.34.mlp.experts.20.gate_proj", "model.layers.34.mlp.experts.21.gate_proj", "model.layers.34.mlp.experts.22.gate_proj", "model.layers.34.mlp.experts.23.gate_proj", "model.layers.34.mlp.experts.24.gate_proj", "model.layers.34.mlp.experts.25.gate_proj", "model.layers.34.mlp.experts.26.gate_proj", "model.layers.34.mlp.experts.27.gate_proj", "model.layers.34.mlp.experts.28.gate_proj", "model.layers.34.mlp.experts.29.gate_proj", "model.layers.34.mlp.experts.30.gate_proj", "model.layers.34.mlp.experts.31.gate_proj", "model.layers.34.mlp.experts.32.gate_proj", "model.layers.34.mlp.experts.33.gate_proj", "model.layers.34.mlp.experts.34.gate_proj", "model.layers.34.mlp.experts.35.gate_proj", "model.layers.34.mlp.experts.36.gate_proj", "model.layers.34.mlp.experts.37.gate_proj", "model.layers.34.mlp.experts.38.gate_proj", "model.layers.34.mlp.experts.39.gate_proj", "model.layers.34.mlp.experts.40.gate_proj", "model.layers.34.mlp.experts.41.gate_proj", "model.layers.34.mlp.experts.42.gate_proj", "model.layers.34.mlp.experts.43.gate_proj", "model.layers.34.mlp.experts.44.gate_proj", "model.layers.34.mlp.experts.45.gate_proj", "model.layers.34.mlp.experts.46.gate_proj", "model.layers.34.mlp.experts.47.gate_proj", "model.layers.34.mlp.experts.48.gate_proj", "model.layers.34.mlp.experts.49.gate_proj", "model.layers.34.mlp.experts.50.gate_proj", "model.layers.34.mlp.experts.51.gate_proj", "model.layers.34.mlp.experts.52.gate_proj", "model.layers.34.mlp.experts.53.gate_proj", "model.layers.34.mlp.experts.54.gate_proj", "model.layers.34.mlp.experts.55.gate_proj", "model.layers.34.mlp.experts.56.gate_proj", "model.layers.34.mlp.experts.57.gate_proj", "model.layers.34.mlp.experts.58.gate_proj", "model.layers.34.mlp.experts.59.gate_proj", "model.layers.34.mlp.experts.60.gate_proj", "model.layers.34.mlp.experts.61.gate_proj", "model.layers.34.mlp.experts.62.gate_proj", "model.layers.34.mlp.experts.63.gate_proj", "model.layers.34.mlp.experts.64.gate_proj", "model.layers.34.mlp.experts.65.gate_proj", "model.layers.34.mlp.experts.66.gate_proj", "model.layers.34.mlp.experts.67.gate_proj", "model.layers.34.mlp.experts.68.gate_proj", "model.layers.34.mlp.experts.69.gate_proj", "model.layers.34.mlp.experts.70.gate_proj", "model.layers.34.mlp.experts.71.gate_proj", "model.layers.34.mlp.experts.72.gate_proj", "model.layers.34.mlp.experts.73.gate_proj", "model.layers.34.mlp.experts.74.gate_proj", "model.layers.34.mlp.experts.75.gate_proj", "model.layers.34.mlp.experts.76.gate_proj", "model.layers.34.mlp.experts.77.gate_proj", "model.layers.34.mlp.experts.78.gate_proj", "model.layers.34.mlp.experts.79.gate_proj", "model.layers.34.mlp.experts.80.gate_proj", "model.layers.34.mlp.experts.81.gate_proj", "model.layers.34.mlp.experts.82.gate_proj", "model.layers.34.mlp.experts.83.gate_proj", "model.layers.34.mlp.experts.84.gate_proj", "model.layers.34.mlp.experts.85.gate_proj", "model.layers.34.mlp.experts.86.gate_proj", "model.layers.34.mlp.experts.87.gate_proj", "model.layers.34.mlp.experts.88.gate_proj", "model.layers.34.mlp.experts.89.gate_proj", "model.layers.34.mlp.experts.90.gate_proj", "model.layers.34.mlp.experts.91.gate_proj", "model.layers.34.mlp.experts.92.gate_proj", "model.layers.34.mlp.experts.93.gate_proj", "model.layers.34.mlp.experts.94.gate_proj", "model.layers.34.mlp.experts.95.gate_proj", "model.layers.34.mlp.experts.96.gate_proj", "model.layers.34.mlp.experts.97.gate_proj", "model.layers.34.mlp.experts.98.gate_proj", "model.layers.34.mlp.experts.99.gate_proj", "model.layers.34.mlp.experts.100.gate_proj", "model.layers.34.mlp.experts.101.gate_proj", "model.layers.34.mlp.experts.102.gate_proj", "model.layers.34.mlp.experts.103.gate_proj", "model.layers.34.mlp.experts.104.gate_proj", "model.layers.34.mlp.experts.105.gate_proj", "model.layers.34.mlp.experts.106.gate_proj", "model.layers.34.mlp.experts.107.gate_proj", "model.layers.34.mlp.experts.108.gate_proj", "model.layers.34.mlp.experts.109.gate_proj", "model.layers.34.mlp.experts.110.gate_proj", "model.layers.34.mlp.experts.111.gate_proj", "model.layers.34.mlp.experts.112.gate_proj", "model.layers.34.mlp.experts.113.gate_proj", "model.layers.34.mlp.experts.114.gate_proj", "model.layers.34.mlp.experts.115.gate_proj", "model.layers.34.mlp.experts.116.gate_proj", "model.layers.34.mlp.experts.117.gate_proj", "model.layers.34.mlp.experts.118.gate_proj", "model.layers.34.mlp.experts.119.gate_proj", "model.layers.34.mlp.experts.120.gate_proj", "model.layers.34.mlp.experts.121.gate_proj", "model.layers.34.mlp.experts.122.gate_proj", "model.layers.34.mlp.experts.123.gate_proj", "model.layers.34.mlp.experts.124.gate_proj", "model.layers.34.mlp.experts.125.gate_proj", "model.layers.34.mlp.experts.126.gate_proj", "model.layers.34.mlp.experts.127.gate_proj", "model.layers.34.mlp.experts.0.up_proj", "model.layers.34.mlp.experts.1.up_proj", "model.layers.34.mlp.experts.2.up_proj", "model.layers.34.mlp.experts.3.up_proj", "model.layers.34.mlp.experts.4.up_proj", "model.layers.34.mlp.experts.5.up_proj", "model.layers.34.mlp.experts.6.up_proj", "model.layers.34.mlp.experts.7.up_proj", "model.layers.34.mlp.experts.8.up_proj", "model.layers.34.mlp.experts.9.up_proj", "model.layers.34.mlp.experts.10.up_proj", "model.layers.34.mlp.experts.11.up_proj", "model.layers.34.mlp.experts.12.up_proj", "model.layers.34.mlp.experts.13.up_proj", "model.layers.34.mlp.experts.14.up_proj", "model.layers.34.mlp.experts.15.up_proj", "model.layers.34.mlp.experts.16.up_proj", "model.layers.34.mlp.experts.17.up_proj", "model.layers.34.mlp.experts.18.up_proj", "model.layers.34.mlp.experts.19.up_proj", "model.layers.34.mlp.experts.20.up_proj", "model.layers.34.mlp.experts.21.up_proj", "model.layers.34.mlp.experts.22.up_proj", "model.layers.34.mlp.experts.23.up_proj", "model.layers.34.mlp.experts.24.up_proj", "model.layers.34.mlp.experts.25.up_proj", "model.layers.34.mlp.experts.26.up_proj", "model.layers.34.mlp.experts.27.up_proj", "model.layers.34.mlp.experts.28.up_proj", "model.layers.34.mlp.experts.29.up_proj", "model.layers.34.mlp.experts.30.up_proj", "model.layers.34.mlp.experts.31.up_proj", "model.layers.34.mlp.experts.32.up_proj", "model.layers.34.mlp.experts.33.up_proj", "model.layers.34.mlp.experts.34.up_proj", "model.layers.34.mlp.experts.35.up_proj", "model.layers.34.mlp.experts.36.up_proj", "model.layers.34.mlp.experts.37.up_proj", "model.layers.34.mlp.experts.38.up_proj", "model.layers.34.mlp.experts.39.up_proj", "model.layers.34.mlp.experts.40.up_proj", "model.layers.34.mlp.experts.41.up_proj", "model.layers.34.mlp.experts.42.up_proj", "model.layers.34.mlp.experts.43.up_proj", "model.layers.34.mlp.experts.44.up_proj", "model.layers.34.mlp.experts.45.up_proj", "model.layers.34.mlp.experts.46.up_proj", "model.layers.34.mlp.experts.47.up_proj", "model.layers.34.mlp.experts.48.up_proj", "model.layers.34.mlp.experts.49.up_proj", "model.layers.34.mlp.experts.50.up_proj", "model.layers.34.mlp.experts.51.up_proj", "model.layers.34.mlp.experts.52.up_proj", "model.layers.34.mlp.experts.53.up_proj", "model.layers.34.mlp.experts.54.up_proj", "model.layers.34.mlp.experts.55.up_proj", "model.layers.34.mlp.experts.56.up_proj", "model.layers.34.mlp.experts.57.up_proj", "model.layers.34.mlp.experts.58.up_proj", "model.layers.34.mlp.experts.59.up_proj", "model.layers.34.mlp.experts.60.up_proj", "model.layers.34.mlp.experts.61.up_proj", "model.layers.34.mlp.experts.62.up_proj", "model.layers.34.mlp.experts.63.up_proj", "model.layers.34.mlp.experts.64.up_proj", "model.layers.34.mlp.experts.65.up_proj", "model.layers.34.mlp.experts.66.up_proj", "model.layers.34.mlp.experts.67.up_proj", "model.layers.34.mlp.experts.68.up_proj", "model.layers.34.mlp.experts.69.up_proj", "model.layers.34.mlp.experts.70.up_proj", "model.layers.34.mlp.experts.71.up_proj", "model.layers.34.mlp.experts.72.up_proj", "model.layers.34.mlp.experts.73.up_proj", "model.layers.34.mlp.experts.74.up_proj", "model.layers.34.mlp.experts.75.up_proj", "model.layers.34.mlp.experts.76.up_proj", "model.layers.34.mlp.experts.77.up_proj", "model.layers.34.mlp.experts.78.up_proj", "model.layers.34.mlp.experts.79.up_proj", "model.layers.34.mlp.experts.80.up_proj", "model.layers.34.mlp.experts.81.up_proj", "model.layers.34.mlp.experts.82.up_proj", "model.layers.34.mlp.experts.83.up_proj", "model.layers.34.mlp.experts.84.up_proj", "model.layers.34.mlp.experts.85.up_proj", "model.layers.34.mlp.experts.86.up_proj", "model.layers.34.mlp.experts.87.up_proj", "model.layers.34.mlp.experts.88.up_proj", "model.layers.34.mlp.experts.89.up_proj", "model.layers.34.mlp.experts.90.up_proj", "model.layers.34.mlp.experts.91.up_proj", "model.layers.34.mlp.experts.92.up_proj", "model.layers.34.mlp.experts.93.up_proj", "model.layers.34.mlp.experts.94.up_proj", "model.layers.34.mlp.experts.95.up_proj", "model.layers.34.mlp.experts.96.up_proj", "model.layers.34.mlp.experts.97.up_proj", "model.layers.34.mlp.experts.98.up_proj", "model.layers.34.mlp.experts.99.up_proj", "model.layers.34.mlp.experts.100.up_proj", "model.layers.34.mlp.experts.101.up_proj", "model.layers.34.mlp.experts.102.up_proj", "model.layers.34.mlp.experts.103.up_proj", "model.layers.34.mlp.experts.104.up_proj", "model.layers.34.mlp.experts.105.up_proj", "model.layers.34.mlp.experts.106.up_proj", "model.layers.34.mlp.experts.107.up_proj", "model.layers.34.mlp.experts.108.up_proj", "model.layers.34.mlp.experts.109.up_proj", "model.layers.34.mlp.experts.110.up_proj", "model.layers.34.mlp.experts.111.up_proj", "model.layers.34.mlp.experts.112.up_proj", "model.layers.34.mlp.experts.113.up_proj", "model.layers.34.mlp.experts.114.up_proj", "model.layers.34.mlp.experts.115.up_proj", "model.layers.34.mlp.experts.116.up_proj", "model.layers.34.mlp.experts.117.up_proj", "model.layers.34.mlp.experts.118.up_proj", "model.layers.34.mlp.experts.119.up_proj", "model.layers.34.mlp.experts.120.up_proj", "model.layers.34.mlp.experts.121.up_proj", "model.layers.34.mlp.experts.122.up_proj", "model.layers.34.mlp.experts.123.up_proj", "model.layers.34.mlp.experts.124.up_proj", "model.layers.34.mlp.experts.125.up_proj", "model.layers.34.mlp.experts.126.up_proj", "model.layers.34.mlp.experts.127.up_proj", "model.layers.34.mlp.experts.0.down_proj", "model.layers.34.mlp.experts.1.down_proj", "model.layers.34.mlp.experts.2.down_proj", "model.layers.34.mlp.experts.3.down_proj", "model.layers.34.mlp.experts.4.down_proj", "model.layers.34.mlp.experts.5.down_proj", "model.layers.34.mlp.experts.6.down_proj", "model.layers.34.mlp.experts.7.down_proj", "model.layers.34.mlp.experts.8.down_proj", "model.layers.34.mlp.experts.9.down_proj", "model.layers.34.mlp.experts.10.down_proj", "model.layers.34.mlp.experts.11.down_proj", "model.layers.34.mlp.experts.12.down_proj", "model.layers.34.mlp.experts.13.down_proj", "model.layers.34.mlp.experts.14.down_proj", "model.layers.34.mlp.experts.15.down_proj", "model.layers.34.mlp.experts.16.down_proj", "model.layers.34.mlp.experts.17.down_proj", "model.layers.34.mlp.experts.18.down_proj", "model.layers.34.mlp.experts.19.down_proj", "model.layers.34.mlp.experts.20.down_proj", "model.layers.34.mlp.experts.21.down_proj", "model.layers.34.mlp.experts.22.down_proj", "model.layers.34.mlp.experts.23.down_proj", "model.layers.34.mlp.experts.24.down_proj", "model.layers.34.mlp.experts.25.down_proj", "model.layers.34.mlp.experts.26.down_proj", "model.layers.34.mlp.experts.27.down_proj", "model.layers.34.mlp.experts.28.down_proj", "model.layers.34.mlp.experts.29.down_proj", "model.layers.34.mlp.experts.30.down_proj", "model.layers.34.mlp.experts.31.down_proj", "model.layers.34.mlp.experts.32.down_proj", "model.layers.34.mlp.experts.33.down_proj", "model.layers.34.mlp.experts.34.down_proj", "model.layers.34.mlp.experts.35.down_proj", "model.layers.34.mlp.experts.36.down_proj", "model.layers.34.mlp.experts.37.down_proj", "model.layers.34.mlp.experts.38.down_proj", "model.layers.34.mlp.experts.39.down_proj", "model.layers.34.mlp.experts.40.down_proj", "model.layers.34.mlp.experts.41.down_proj", "model.layers.34.mlp.experts.42.down_proj", "model.layers.34.mlp.experts.43.down_proj", "model.layers.34.mlp.experts.44.down_proj", "model.layers.34.mlp.experts.45.down_proj", "model.layers.34.mlp.experts.46.down_proj", "model.layers.34.mlp.experts.47.down_proj", "model.layers.34.mlp.experts.48.down_proj", "model.layers.34.mlp.experts.49.down_proj", "model.layers.34.mlp.experts.50.down_proj", "model.layers.34.mlp.experts.51.down_proj", "model.layers.34.mlp.experts.52.down_proj", "model.layers.34.mlp.experts.53.down_proj", "model.layers.34.mlp.experts.54.down_proj", "model.layers.34.mlp.experts.55.down_proj", "model.layers.34.mlp.experts.56.down_proj", "model.layers.34.mlp.experts.57.down_proj", "model.layers.34.mlp.experts.58.down_proj", "model.layers.34.mlp.experts.59.down_proj", "model.layers.34.mlp.experts.60.down_proj", "model.layers.34.mlp.experts.61.down_proj", "model.layers.34.mlp.experts.62.down_proj", "model.layers.34.mlp.experts.63.down_proj", "model.layers.34.mlp.experts.64.down_proj", "model.layers.34.mlp.experts.65.down_proj", "model.layers.34.mlp.experts.66.down_proj", "model.layers.34.mlp.experts.67.down_proj", "model.layers.34.mlp.experts.68.down_proj", "model.layers.34.mlp.experts.69.down_proj", "model.layers.34.mlp.experts.70.down_proj", "model.layers.34.mlp.experts.71.down_proj", "model.layers.34.mlp.experts.72.down_proj", "model.layers.34.mlp.experts.73.down_proj", "model.layers.34.mlp.experts.74.down_proj", "model.layers.34.mlp.experts.75.down_proj", "model.layers.34.mlp.experts.76.down_proj", "model.layers.34.mlp.experts.77.down_proj", "model.layers.34.mlp.experts.78.down_proj", "model.layers.34.mlp.experts.79.down_proj", "model.layers.34.mlp.experts.80.down_proj", "model.layers.34.mlp.experts.81.down_proj", "model.layers.34.mlp.experts.82.down_proj", "model.layers.34.mlp.experts.83.down_proj", "model.layers.34.mlp.experts.84.down_proj", "model.layers.34.mlp.experts.85.down_proj", "model.layers.34.mlp.experts.86.down_proj", "model.layers.34.mlp.experts.87.down_proj", "model.layers.34.mlp.experts.88.down_proj", "model.layers.34.mlp.experts.89.down_proj", "model.layers.34.mlp.experts.90.down_proj", "model.layers.34.mlp.experts.91.down_proj", "model.layers.34.mlp.experts.92.down_proj", "model.layers.34.mlp.experts.93.down_proj", "model.layers.34.mlp.experts.94.down_proj", "model.layers.34.mlp.experts.95.down_proj", "model.layers.34.mlp.experts.96.down_proj", "model.layers.34.mlp.experts.97.down_proj", "model.layers.34.mlp.experts.98.down_proj", "model.layers.34.mlp.experts.99.down_proj", "model.layers.34.mlp.experts.100.down_proj", "model.layers.34.mlp.experts.101.down_proj", "model.layers.34.mlp.experts.102.down_proj", "model.layers.34.mlp.experts.103.down_proj", "model.layers.34.mlp.experts.104.down_proj", "model.layers.34.mlp.experts.105.down_proj", "model.layers.34.mlp.experts.106.down_proj", "model.layers.34.mlp.experts.107.down_proj", "model.layers.34.mlp.experts.108.down_proj", "model.layers.34.mlp.experts.109.down_proj", "model.layers.34.mlp.experts.110.down_proj", "model.layers.34.mlp.experts.111.down_proj", "model.layers.34.mlp.experts.112.down_proj", "model.layers.34.mlp.experts.113.down_proj", "model.layers.34.mlp.experts.114.down_proj", "model.layers.34.mlp.experts.115.down_proj", "model.layers.34.mlp.experts.116.down_proj", "model.layers.34.mlp.experts.117.down_proj", "model.layers.34.mlp.experts.118.down_proj", "model.layers.34.mlp.experts.119.down_proj", "model.layers.34.mlp.experts.120.down_proj", "model.layers.34.mlp.experts.121.down_proj", "model.layers.34.mlp.experts.122.down_proj", "model.layers.34.mlp.experts.123.down_proj", "model.layers.34.mlp.experts.124.down_proj", "model.layers.34.mlp.experts.125.down_proj", "model.layers.34.mlp.experts.126.down_proj", "model.layers.34.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0006685939617454839, "dbits": 2214592512 } ] }, { "idx": 104, "layers": [ "model.layers.35.self_attn.q_proj", "model.layers.35.self_attn.k_proj", "model.layers.35.self_attn.v_proj", "model.layers.35.self_attn.o_proj" ], "candidates": [ { "dkld": 0.00043079126626252573, "dbits": 109051904 } ] }, { "idx": 105, "layers": [ "model.layers.35.mlp.shared_experts.gate_proj", "model.layers.35.mlp.shared_experts.up_proj", "model.layers.35.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00046997377648949623, "dbits": 17301504 } ] }, { "idx": 106, "layers": [ "model.layers.35.mlp.experts.0.gate_proj", "model.layers.35.mlp.experts.1.gate_proj", "model.layers.35.mlp.experts.2.gate_proj", "model.layers.35.mlp.experts.3.gate_proj", "model.layers.35.mlp.experts.4.gate_proj", "model.layers.35.mlp.experts.5.gate_proj", "model.layers.35.mlp.experts.6.gate_proj", "model.layers.35.mlp.experts.7.gate_proj", "model.layers.35.mlp.experts.8.gate_proj", "model.layers.35.mlp.experts.9.gate_proj", "model.layers.35.mlp.experts.10.gate_proj", "model.layers.35.mlp.experts.11.gate_proj", "model.layers.35.mlp.experts.12.gate_proj", "model.layers.35.mlp.experts.13.gate_proj", "model.layers.35.mlp.experts.14.gate_proj", "model.layers.35.mlp.experts.15.gate_proj", "model.layers.35.mlp.experts.16.gate_proj", "model.layers.35.mlp.experts.17.gate_proj", "model.layers.35.mlp.experts.18.gate_proj", "model.layers.35.mlp.experts.19.gate_proj", "model.layers.35.mlp.experts.20.gate_proj", "model.layers.35.mlp.experts.21.gate_proj", "model.layers.35.mlp.experts.22.gate_proj", "model.layers.35.mlp.experts.23.gate_proj", "model.layers.35.mlp.experts.24.gate_proj", "model.layers.35.mlp.experts.25.gate_proj", "model.layers.35.mlp.experts.26.gate_proj", "model.layers.35.mlp.experts.27.gate_proj", "model.layers.35.mlp.experts.28.gate_proj", "model.layers.35.mlp.experts.29.gate_proj", "model.layers.35.mlp.experts.30.gate_proj", "model.layers.35.mlp.experts.31.gate_proj", "model.layers.35.mlp.experts.32.gate_proj", "model.layers.35.mlp.experts.33.gate_proj", "model.layers.35.mlp.experts.34.gate_proj", "model.layers.35.mlp.experts.35.gate_proj", "model.layers.35.mlp.experts.36.gate_proj", "model.layers.35.mlp.experts.37.gate_proj", "model.layers.35.mlp.experts.38.gate_proj", "model.layers.35.mlp.experts.39.gate_proj", "model.layers.35.mlp.experts.40.gate_proj", "model.layers.35.mlp.experts.41.gate_proj", "model.layers.35.mlp.experts.42.gate_proj", "model.layers.35.mlp.experts.43.gate_proj", "model.layers.35.mlp.experts.44.gate_proj", "model.layers.35.mlp.experts.45.gate_proj", "model.layers.35.mlp.experts.46.gate_proj", "model.layers.35.mlp.experts.47.gate_proj", "model.layers.35.mlp.experts.48.gate_proj", "model.layers.35.mlp.experts.49.gate_proj", "model.layers.35.mlp.experts.50.gate_proj", "model.layers.35.mlp.experts.51.gate_proj", "model.layers.35.mlp.experts.52.gate_proj", "model.layers.35.mlp.experts.53.gate_proj", "model.layers.35.mlp.experts.54.gate_proj", "model.layers.35.mlp.experts.55.gate_proj", "model.layers.35.mlp.experts.56.gate_proj", "model.layers.35.mlp.experts.57.gate_proj", "model.layers.35.mlp.experts.58.gate_proj", "model.layers.35.mlp.experts.59.gate_proj", "model.layers.35.mlp.experts.60.gate_proj", "model.layers.35.mlp.experts.61.gate_proj", "model.layers.35.mlp.experts.62.gate_proj", "model.layers.35.mlp.experts.63.gate_proj", "model.layers.35.mlp.experts.64.gate_proj", "model.layers.35.mlp.experts.65.gate_proj", "model.layers.35.mlp.experts.66.gate_proj", "model.layers.35.mlp.experts.67.gate_proj", "model.layers.35.mlp.experts.68.gate_proj", "model.layers.35.mlp.experts.69.gate_proj", "model.layers.35.mlp.experts.70.gate_proj", "model.layers.35.mlp.experts.71.gate_proj", "model.layers.35.mlp.experts.72.gate_proj", "model.layers.35.mlp.experts.73.gate_proj", "model.layers.35.mlp.experts.74.gate_proj", "model.layers.35.mlp.experts.75.gate_proj", "model.layers.35.mlp.experts.76.gate_proj", "model.layers.35.mlp.experts.77.gate_proj", "model.layers.35.mlp.experts.78.gate_proj", "model.layers.35.mlp.experts.79.gate_proj", "model.layers.35.mlp.experts.80.gate_proj", "model.layers.35.mlp.experts.81.gate_proj", "model.layers.35.mlp.experts.82.gate_proj", "model.layers.35.mlp.experts.83.gate_proj", "model.layers.35.mlp.experts.84.gate_proj", "model.layers.35.mlp.experts.85.gate_proj", "model.layers.35.mlp.experts.86.gate_proj", "model.layers.35.mlp.experts.87.gate_proj", "model.layers.35.mlp.experts.88.gate_proj", "model.layers.35.mlp.experts.89.gate_proj", "model.layers.35.mlp.experts.90.gate_proj", "model.layers.35.mlp.experts.91.gate_proj", "model.layers.35.mlp.experts.92.gate_proj", "model.layers.35.mlp.experts.93.gate_proj", "model.layers.35.mlp.experts.94.gate_proj", "model.layers.35.mlp.experts.95.gate_proj", "model.layers.35.mlp.experts.96.gate_proj", "model.layers.35.mlp.experts.97.gate_proj", "model.layers.35.mlp.experts.98.gate_proj", "model.layers.35.mlp.experts.99.gate_proj", "model.layers.35.mlp.experts.100.gate_proj", "model.layers.35.mlp.experts.101.gate_proj", "model.layers.35.mlp.experts.102.gate_proj", "model.layers.35.mlp.experts.103.gate_proj", "model.layers.35.mlp.experts.104.gate_proj", "model.layers.35.mlp.experts.105.gate_proj", "model.layers.35.mlp.experts.106.gate_proj", "model.layers.35.mlp.experts.107.gate_proj", "model.layers.35.mlp.experts.108.gate_proj", "model.layers.35.mlp.experts.109.gate_proj", "model.layers.35.mlp.experts.110.gate_proj", "model.layers.35.mlp.experts.111.gate_proj", "model.layers.35.mlp.experts.112.gate_proj", "model.layers.35.mlp.experts.113.gate_proj", "model.layers.35.mlp.experts.114.gate_proj", "model.layers.35.mlp.experts.115.gate_proj", "model.layers.35.mlp.experts.116.gate_proj", "model.layers.35.mlp.experts.117.gate_proj", "model.layers.35.mlp.experts.118.gate_proj", "model.layers.35.mlp.experts.119.gate_proj", "model.layers.35.mlp.experts.120.gate_proj", "model.layers.35.mlp.experts.121.gate_proj", "model.layers.35.mlp.experts.122.gate_proj", "model.layers.35.mlp.experts.123.gate_proj", "model.layers.35.mlp.experts.124.gate_proj", "model.layers.35.mlp.experts.125.gate_proj", "model.layers.35.mlp.experts.126.gate_proj", "model.layers.35.mlp.experts.127.gate_proj", "model.layers.35.mlp.experts.0.up_proj", "model.layers.35.mlp.experts.1.up_proj", "model.layers.35.mlp.experts.2.up_proj", "model.layers.35.mlp.experts.3.up_proj", "model.layers.35.mlp.experts.4.up_proj", "model.layers.35.mlp.experts.5.up_proj", "model.layers.35.mlp.experts.6.up_proj", "model.layers.35.mlp.experts.7.up_proj", "model.layers.35.mlp.experts.8.up_proj", "model.layers.35.mlp.experts.9.up_proj", "model.layers.35.mlp.experts.10.up_proj", "model.layers.35.mlp.experts.11.up_proj", "model.layers.35.mlp.experts.12.up_proj", "model.layers.35.mlp.experts.13.up_proj", "model.layers.35.mlp.experts.14.up_proj", "model.layers.35.mlp.experts.15.up_proj", "model.layers.35.mlp.experts.16.up_proj", "model.layers.35.mlp.experts.17.up_proj", "model.layers.35.mlp.experts.18.up_proj", "model.layers.35.mlp.experts.19.up_proj", "model.layers.35.mlp.experts.20.up_proj", "model.layers.35.mlp.experts.21.up_proj", "model.layers.35.mlp.experts.22.up_proj", "model.layers.35.mlp.experts.23.up_proj", "model.layers.35.mlp.experts.24.up_proj", "model.layers.35.mlp.experts.25.up_proj", "model.layers.35.mlp.experts.26.up_proj", "model.layers.35.mlp.experts.27.up_proj", "model.layers.35.mlp.experts.28.up_proj", "model.layers.35.mlp.experts.29.up_proj", "model.layers.35.mlp.experts.30.up_proj", "model.layers.35.mlp.experts.31.up_proj", "model.layers.35.mlp.experts.32.up_proj", "model.layers.35.mlp.experts.33.up_proj", "model.layers.35.mlp.experts.34.up_proj", "model.layers.35.mlp.experts.35.up_proj", "model.layers.35.mlp.experts.36.up_proj", "model.layers.35.mlp.experts.37.up_proj", "model.layers.35.mlp.experts.38.up_proj", "model.layers.35.mlp.experts.39.up_proj", "model.layers.35.mlp.experts.40.up_proj", "model.layers.35.mlp.experts.41.up_proj", "model.layers.35.mlp.experts.42.up_proj", "model.layers.35.mlp.experts.43.up_proj", "model.layers.35.mlp.experts.44.up_proj", "model.layers.35.mlp.experts.45.up_proj", "model.layers.35.mlp.experts.46.up_proj", "model.layers.35.mlp.experts.47.up_proj", "model.layers.35.mlp.experts.48.up_proj", "model.layers.35.mlp.experts.49.up_proj", "model.layers.35.mlp.experts.50.up_proj", "model.layers.35.mlp.experts.51.up_proj", "model.layers.35.mlp.experts.52.up_proj", "model.layers.35.mlp.experts.53.up_proj", "model.layers.35.mlp.experts.54.up_proj", "model.layers.35.mlp.experts.55.up_proj", "model.layers.35.mlp.experts.56.up_proj", "model.layers.35.mlp.experts.57.up_proj", "model.layers.35.mlp.experts.58.up_proj", "model.layers.35.mlp.experts.59.up_proj", "model.layers.35.mlp.experts.60.up_proj", "model.layers.35.mlp.experts.61.up_proj", "model.layers.35.mlp.experts.62.up_proj", "model.layers.35.mlp.experts.63.up_proj", "model.layers.35.mlp.experts.64.up_proj", "model.layers.35.mlp.experts.65.up_proj", "model.layers.35.mlp.experts.66.up_proj", "model.layers.35.mlp.experts.67.up_proj", "model.layers.35.mlp.experts.68.up_proj", "model.layers.35.mlp.experts.69.up_proj", "model.layers.35.mlp.experts.70.up_proj", "model.layers.35.mlp.experts.71.up_proj", "model.layers.35.mlp.experts.72.up_proj", "model.layers.35.mlp.experts.73.up_proj", "model.layers.35.mlp.experts.74.up_proj", "model.layers.35.mlp.experts.75.up_proj", "model.layers.35.mlp.experts.76.up_proj", "model.layers.35.mlp.experts.77.up_proj", "model.layers.35.mlp.experts.78.up_proj", "model.layers.35.mlp.experts.79.up_proj", "model.layers.35.mlp.experts.80.up_proj", "model.layers.35.mlp.experts.81.up_proj", "model.layers.35.mlp.experts.82.up_proj", "model.layers.35.mlp.experts.83.up_proj", "model.layers.35.mlp.experts.84.up_proj", "model.layers.35.mlp.experts.85.up_proj", "model.layers.35.mlp.experts.86.up_proj", "model.layers.35.mlp.experts.87.up_proj", "model.layers.35.mlp.experts.88.up_proj", "model.layers.35.mlp.experts.89.up_proj", "model.layers.35.mlp.experts.90.up_proj", "model.layers.35.mlp.experts.91.up_proj", "model.layers.35.mlp.experts.92.up_proj", "model.layers.35.mlp.experts.93.up_proj", "model.layers.35.mlp.experts.94.up_proj", "model.layers.35.mlp.experts.95.up_proj", "model.layers.35.mlp.experts.96.up_proj", "model.layers.35.mlp.experts.97.up_proj", "model.layers.35.mlp.experts.98.up_proj", "model.layers.35.mlp.experts.99.up_proj", "model.layers.35.mlp.experts.100.up_proj", "model.layers.35.mlp.experts.101.up_proj", "model.layers.35.mlp.experts.102.up_proj", "model.layers.35.mlp.experts.103.up_proj", "model.layers.35.mlp.experts.104.up_proj", "model.layers.35.mlp.experts.105.up_proj", "model.layers.35.mlp.experts.106.up_proj", "model.layers.35.mlp.experts.107.up_proj", "model.layers.35.mlp.experts.108.up_proj", "model.layers.35.mlp.experts.109.up_proj", "model.layers.35.mlp.experts.110.up_proj", "model.layers.35.mlp.experts.111.up_proj", "model.layers.35.mlp.experts.112.up_proj", "model.layers.35.mlp.experts.113.up_proj", "model.layers.35.mlp.experts.114.up_proj", "model.layers.35.mlp.experts.115.up_proj", "model.layers.35.mlp.experts.116.up_proj", "model.layers.35.mlp.experts.117.up_proj", "model.layers.35.mlp.experts.118.up_proj", "model.layers.35.mlp.experts.119.up_proj", "model.layers.35.mlp.experts.120.up_proj", "model.layers.35.mlp.experts.121.up_proj", "model.layers.35.mlp.experts.122.up_proj", "model.layers.35.mlp.experts.123.up_proj", "model.layers.35.mlp.experts.124.up_proj", "model.layers.35.mlp.experts.125.up_proj", "model.layers.35.mlp.experts.126.up_proj", "model.layers.35.mlp.experts.127.up_proj", "model.layers.35.mlp.experts.0.down_proj", "model.layers.35.mlp.experts.1.down_proj", "model.layers.35.mlp.experts.2.down_proj", "model.layers.35.mlp.experts.3.down_proj", "model.layers.35.mlp.experts.4.down_proj", "model.layers.35.mlp.experts.5.down_proj", "model.layers.35.mlp.experts.6.down_proj", "model.layers.35.mlp.experts.7.down_proj", "model.layers.35.mlp.experts.8.down_proj", "model.layers.35.mlp.experts.9.down_proj", "model.layers.35.mlp.experts.10.down_proj", "model.layers.35.mlp.experts.11.down_proj", "model.layers.35.mlp.experts.12.down_proj", "model.layers.35.mlp.experts.13.down_proj", "model.layers.35.mlp.experts.14.down_proj", "model.layers.35.mlp.experts.15.down_proj", "model.layers.35.mlp.experts.16.down_proj", "model.layers.35.mlp.experts.17.down_proj", "model.layers.35.mlp.experts.18.down_proj", "model.layers.35.mlp.experts.19.down_proj", "model.layers.35.mlp.experts.20.down_proj", "model.layers.35.mlp.experts.21.down_proj", "model.layers.35.mlp.experts.22.down_proj", "model.layers.35.mlp.experts.23.down_proj", "model.layers.35.mlp.experts.24.down_proj", "model.layers.35.mlp.experts.25.down_proj", "model.layers.35.mlp.experts.26.down_proj", "model.layers.35.mlp.experts.27.down_proj", "model.layers.35.mlp.experts.28.down_proj", "model.layers.35.mlp.experts.29.down_proj", "model.layers.35.mlp.experts.30.down_proj", "model.layers.35.mlp.experts.31.down_proj", "model.layers.35.mlp.experts.32.down_proj", "model.layers.35.mlp.experts.33.down_proj", "model.layers.35.mlp.experts.34.down_proj", "model.layers.35.mlp.experts.35.down_proj", "model.layers.35.mlp.experts.36.down_proj", "model.layers.35.mlp.experts.37.down_proj", "model.layers.35.mlp.experts.38.down_proj", "model.layers.35.mlp.experts.39.down_proj", "model.layers.35.mlp.experts.40.down_proj", "model.layers.35.mlp.experts.41.down_proj", "model.layers.35.mlp.experts.42.down_proj", "model.layers.35.mlp.experts.43.down_proj", "model.layers.35.mlp.experts.44.down_proj", "model.layers.35.mlp.experts.45.down_proj", "model.layers.35.mlp.experts.46.down_proj", "model.layers.35.mlp.experts.47.down_proj", "model.layers.35.mlp.experts.48.down_proj", "model.layers.35.mlp.experts.49.down_proj", "model.layers.35.mlp.experts.50.down_proj", "model.layers.35.mlp.experts.51.down_proj", "model.layers.35.mlp.experts.52.down_proj", "model.layers.35.mlp.experts.53.down_proj", "model.layers.35.mlp.experts.54.down_proj", "model.layers.35.mlp.experts.55.down_proj", "model.layers.35.mlp.experts.56.down_proj", "model.layers.35.mlp.experts.57.down_proj", "model.layers.35.mlp.experts.58.down_proj", "model.layers.35.mlp.experts.59.down_proj", "model.layers.35.mlp.experts.60.down_proj", "model.layers.35.mlp.experts.61.down_proj", "model.layers.35.mlp.experts.62.down_proj", "model.layers.35.mlp.experts.63.down_proj", "model.layers.35.mlp.experts.64.down_proj", "model.layers.35.mlp.experts.65.down_proj", "model.layers.35.mlp.experts.66.down_proj", "model.layers.35.mlp.experts.67.down_proj", "model.layers.35.mlp.experts.68.down_proj", "model.layers.35.mlp.experts.69.down_proj", "model.layers.35.mlp.experts.70.down_proj", "model.layers.35.mlp.experts.71.down_proj", "model.layers.35.mlp.experts.72.down_proj", "model.layers.35.mlp.experts.73.down_proj", "model.layers.35.mlp.experts.74.down_proj", "model.layers.35.mlp.experts.75.down_proj", "model.layers.35.mlp.experts.76.down_proj", "model.layers.35.mlp.experts.77.down_proj", "model.layers.35.mlp.experts.78.down_proj", "model.layers.35.mlp.experts.79.down_proj", "model.layers.35.mlp.experts.80.down_proj", "model.layers.35.mlp.experts.81.down_proj", "model.layers.35.mlp.experts.82.down_proj", "model.layers.35.mlp.experts.83.down_proj", "model.layers.35.mlp.experts.84.down_proj", "model.layers.35.mlp.experts.85.down_proj", "model.layers.35.mlp.experts.86.down_proj", "model.layers.35.mlp.experts.87.down_proj", "model.layers.35.mlp.experts.88.down_proj", "model.layers.35.mlp.experts.89.down_proj", "model.layers.35.mlp.experts.90.down_proj", "model.layers.35.mlp.experts.91.down_proj", "model.layers.35.mlp.experts.92.down_proj", "model.layers.35.mlp.experts.93.down_proj", "model.layers.35.mlp.experts.94.down_proj", "model.layers.35.mlp.experts.95.down_proj", "model.layers.35.mlp.experts.96.down_proj", "model.layers.35.mlp.experts.97.down_proj", "model.layers.35.mlp.experts.98.down_proj", "model.layers.35.mlp.experts.99.down_proj", "model.layers.35.mlp.experts.100.down_proj", "model.layers.35.mlp.experts.101.down_proj", "model.layers.35.mlp.experts.102.down_proj", "model.layers.35.mlp.experts.103.down_proj", "model.layers.35.mlp.experts.104.down_proj", "model.layers.35.mlp.experts.105.down_proj", "model.layers.35.mlp.experts.106.down_proj", "model.layers.35.mlp.experts.107.down_proj", "model.layers.35.mlp.experts.108.down_proj", "model.layers.35.mlp.experts.109.down_proj", "model.layers.35.mlp.experts.110.down_proj", "model.layers.35.mlp.experts.111.down_proj", "model.layers.35.mlp.experts.112.down_proj", "model.layers.35.mlp.experts.113.down_proj", "model.layers.35.mlp.experts.114.down_proj", "model.layers.35.mlp.experts.115.down_proj", "model.layers.35.mlp.experts.116.down_proj", "model.layers.35.mlp.experts.117.down_proj", "model.layers.35.mlp.experts.118.down_proj", "model.layers.35.mlp.experts.119.down_proj", "model.layers.35.mlp.experts.120.down_proj", "model.layers.35.mlp.experts.121.down_proj", "model.layers.35.mlp.experts.122.down_proj", "model.layers.35.mlp.experts.123.down_proj", "model.layers.35.mlp.experts.124.down_proj", "model.layers.35.mlp.experts.125.down_proj", "model.layers.35.mlp.experts.126.down_proj", "model.layers.35.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0008682697080075741, "dbits": 2214592512 } ] }, { "idx": 107, "layers": [ "model.layers.36.self_attn.q_proj", "model.layers.36.self_attn.k_proj", "model.layers.36.self_attn.v_proj", "model.layers.36.self_attn.o_proj" ], "candidates": [ { "dkld": -9.471327066420954e-05, "dbits": 109051904 } ] }, { "idx": 108, "layers": [ "model.layers.36.mlp.shared_experts.gate_proj", "model.layers.36.mlp.shared_experts.up_proj", "model.layers.36.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.000421301089227194, "dbits": 17301504 } ] }, { "idx": 109, "layers": [ "model.layers.36.mlp.experts.0.gate_proj", "model.layers.36.mlp.experts.1.gate_proj", "model.layers.36.mlp.experts.2.gate_proj", "model.layers.36.mlp.experts.3.gate_proj", "model.layers.36.mlp.experts.4.gate_proj", "model.layers.36.mlp.experts.5.gate_proj", "model.layers.36.mlp.experts.6.gate_proj", "model.layers.36.mlp.experts.7.gate_proj", "model.layers.36.mlp.experts.8.gate_proj", "model.layers.36.mlp.experts.9.gate_proj", "model.layers.36.mlp.experts.10.gate_proj", "model.layers.36.mlp.experts.11.gate_proj", "model.layers.36.mlp.experts.12.gate_proj", "model.layers.36.mlp.experts.13.gate_proj", "model.layers.36.mlp.experts.14.gate_proj", "model.layers.36.mlp.experts.15.gate_proj", "model.layers.36.mlp.experts.16.gate_proj", "model.layers.36.mlp.experts.17.gate_proj", "model.layers.36.mlp.experts.18.gate_proj", "model.layers.36.mlp.experts.19.gate_proj", "model.layers.36.mlp.experts.20.gate_proj", "model.layers.36.mlp.experts.21.gate_proj", "model.layers.36.mlp.experts.22.gate_proj", "model.layers.36.mlp.experts.23.gate_proj", "model.layers.36.mlp.experts.24.gate_proj", "model.layers.36.mlp.experts.25.gate_proj", "model.layers.36.mlp.experts.26.gate_proj", "model.layers.36.mlp.experts.27.gate_proj", "model.layers.36.mlp.experts.28.gate_proj", "model.layers.36.mlp.experts.29.gate_proj", "model.layers.36.mlp.experts.30.gate_proj", "model.layers.36.mlp.experts.31.gate_proj", "model.layers.36.mlp.experts.32.gate_proj", "model.layers.36.mlp.experts.33.gate_proj", "model.layers.36.mlp.experts.34.gate_proj", "model.layers.36.mlp.experts.35.gate_proj", "model.layers.36.mlp.experts.36.gate_proj", "model.layers.36.mlp.experts.37.gate_proj", "model.layers.36.mlp.experts.38.gate_proj", "model.layers.36.mlp.experts.39.gate_proj", "model.layers.36.mlp.experts.40.gate_proj", "model.layers.36.mlp.experts.41.gate_proj", "model.layers.36.mlp.experts.42.gate_proj", "model.layers.36.mlp.experts.43.gate_proj", "model.layers.36.mlp.experts.44.gate_proj", "model.layers.36.mlp.experts.45.gate_proj", "model.layers.36.mlp.experts.46.gate_proj", "model.layers.36.mlp.experts.47.gate_proj", "model.layers.36.mlp.experts.48.gate_proj", "model.layers.36.mlp.experts.49.gate_proj", "model.layers.36.mlp.experts.50.gate_proj", "model.layers.36.mlp.experts.51.gate_proj", "model.layers.36.mlp.experts.52.gate_proj", "model.layers.36.mlp.experts.53.gate_proj", "model.layers.36.mlp.experts.54.gate_proj", "model.layers.36.mlp.experts.55.gate_proj", "model.layers.36.mlp.experts.56.gate_proj", "model.layers.36.mlp.experts.57.gate_proj", "model.layers.36.mlp.experts.58.gate_proj", "model.layers.36.mlp.experts.59.gate_proj", "model.layers.36.mlp.experts.60.gate_proj", "model.layers.36.mlp.experts.61.gate_proj", "model.layers.36.mlp.experts.62.gate_proj", "model.layers.36.mlp.experts.63.gate_proj", "model.layers.36.mlp.experts.64.gate_proj", "model.layers.36.mlp.experts.65.gate_proj", "model.layers.36.mlp.experts.66.gate_proj", "model.layers.36.mlp.experts.67.gate_proj", "model.layers.36.mlp.experts.68.gate_proj", "model.layers.36.mlp.experts.69.gate_proj", "model.layers.36.mlp.experts.70.gate_proj", "model.layers.36.mlp.experts.71.gate_proj", "model.layers.36.mlp.experts.72.gate_proj", "model.layers.36.mlp.experts.73.gate_proj", "model.layers.36.mlp.experts.74.gate_proj", "model.layers.36.mlp.experts.75.gate_proj", "model.layers.36.mlp.experts.76.gate_proj", "model.layers.36.mlp.experts.77.gate_proj", "model.layers.36.mlp.experts.78.gate_proj", "model.layers.36.mlp.experts.79.gate_proj", "model.layers.36.mlp.experts.80.gate_proj", "model.layers.36.mlp.experts.81.gate_proj", "model.layers.36.mlp.experts.82.gate_proj", "model.layers.36.mlp.experts.83.gate_proj", "model.layers.36.mlp.experts.84.gate_proj", "model.layers.36.mlp.experts.85.gate_proj", "model.layers.36.mlp.experts.86.gate_proj", "model.layers.36.mlp.experts.87.gate_proj", "model.layers.36.mlp.experts.88.gate_proj", "model.layers.36.mlp.experts.89.gate_proj", "model.layers.36.mlp.experts.90.gate_proj", "model.layers.36.mlp.experts.91.gate_proj", "model.layers.36.mlp.experts.92.gate_proj", "model.layers.36.mlp.experts.93.gate_proj", "model.layers.36.mlp.experts.94.gate_proj", "model.layers.36.mlp.experts.95.gate_proj", "model.layers.36.mlp.experts.96.gate_proj", "model.layers.36.mlp.experts.97.gate_proj", "model.layers.36.mlp.experts.98.gate_proj", "model.layers.36.mlp.experts.99.gate_proj", "model.layers.36.mlp.experts.100.gate_proj", "model.layers.36.mlp.experts.101.gate_proj", "model.layers.36.mlp.experts.102.gate_proj", "model.layers.36.mlp.experts.103.gate_proj", "model.layers.36.mlp.experts.104.gate_proj", "model.layers.36.mlp.experts.105.gate_proj", "model.layers.36.mlp.experts.106.gate_proj", "model.layers.36.mlp.experts.107.gate_proj", "model.layers.36.mlp.experts.108.gate_proj", "model.layers.36.mlp.experts.109.gate_proj", "model.layers.36.mlp.experts.110.gate_proj", "model.layers.36.mlp.experts.111.gate_proj", "model.layers.36.mlp.experts.112.gate_proj", "model.layers.36.mlp.experts.113.gate_proj", "model.layers.36.mlp.experts.114.gate_proj", "model.layers.36.mlp.experts.115.gate_proj", "model.layers.36.mlp.experts.116.gate_proj", "model.layers.36.mlp.experts.117.gate_proj", "model.layers.36.mlp.experts.118.gate_proj", "model.layers.36.mlp.experts.119.gate_proj", "model.layers.36.mlp.experts.120.gate_proj", "model.layers.36.mlp.experts.121.gate_proj", "model.layers.36.mlp.experts.122.gate_proj", "model.layers.36.mlp.experts.123.gate_proj", "model.layers.36.mlp.experts.124.gate_proj", "model.layers.36.mlp.experts.125.gate_proj", "model.layers.36.mlp.experts.126.gate_proj", "model.layers.36.mlp.experts.127.gate_proj", "model.layers.36.mlp.experts.0.up_proj", "model.layers.36.mlp.experts.1.up_proj", "model.layers.36.mlp.experts.2.up_proj", "model.layers.36.mlp.experts.3.up_proj", "model.layers.36.mlp.experts.4.up_proj", "model.layers.36.mlp.experts.5.up_proj", "model.layers.36.mlp.experts.6.up_proj", "model.layers.36.mlp.experts.7.up_proj", "model.layers.36.mlp.experts.8.up_proj", "model.layers.36.mlp.experts.9.up_proj", "model.layers.36.mlp.experts.10.up_proj", "model.layers.36.mlp.experts.11.up_proj", "model.layers.36.mlp.experts.12.up_proj", "model.layers.36.mlp.experts.13.up_proj", "model.layers.36.mlp.experts.14.up_proj", "model.layers.36.mlp.experts.15.up_proj", "model.layers.36.mlp.experts.16.up_proj", "model.layers.36.mlp.experts.17.up_proj", "model.layers.36.mlp.experts.18.up_proj", "model.layers.36.mlp.experts.19.up_proj", "model.layers.36.mlp.experts.20.up_proj", "model.layers.36.mlp.experts.21.up_proj", "model.layers.36.mlp.experts.22.up_proj", "model.layers.36.mlp.experts.23.up_proj", "model.layers.36.mlp.experts.24.up_proj", "model.layers.36.mlp.experts.25.up_proj", "model.layers.36.mlp.experts.26.up_proj", "model.layers.36.mlp.experts.27.up_proj", "model.layers.36.mlp.experts.28.up_proj", "model.layers.36.mlp.experts.29.up_proj", "model.layers.36.mlp.experts.30.up_proj", "model.layers.36.mlp.experts.31.up_proj", "model.layers.36.mlp.experts.32.up_proj", "model.layers.36.mlp.experts.33.up_proj", "model.layers.36.mlp.experts.34.up_proj", "model.layers.36.mlp.experts.35.up_proj", "model.layers.36.mlp.experts.36.up_proj", "model.layers.36.mlp.experts.37.up_proj", "model.layers.36.mlp.experts.38.up_proj", "model.layers.36.mlp.experts.39.up_proj", "model.layers.36.mlp.experts.40.up_proj", "model.layers.36.mlp.experts.41.up_proj", "model.layers.36.mlp.experts.42.up_proj", "model.layers.36.mlp.experts.43.up_proj", "model.layers.36.mlp.experts.44.up_proj", "model.layers.36.mlp.experts.45.up_proj", "model.layers.36.mlp.experts.46.up_proj", "model.layers.36.mlp.experts.47.up_proj", "model.layers.36.mlp.experts.48.up_proj", "model.layers.36.mlp.experts.49.up_proj", "model.layers.36.mlp.experts.50.up_proj", "model.layers.36.mlp.experts.51.up_proj", "model.layers.36.mlp.experts.52.up_proj", "model.layers.36.mlp.experts.53.up_proj", "model.layers.36.mlp.experts.54.up_proj", "model.layers.36.mlp.experts.55.up_proj", "model.layers.36.mlp.experts.56.up_proj", "model.layers.36.mlp.experts.57.up_proj", "model.layers.36.mlp.experts.58.up_proj", "model.layers.36.mlp.experts.59.up_proj", "model.layers.36.mlp.experts.60.up_proj", "model.layers.36.mlp.experts.61.up_proj", "model.layers.36.mlp.experts.62.up_proj", "model.layers.36.mlp.experts.63.up_proj", "model.layers.36.mlp.experts.64.up_proj", "model.layers.36.mlp.experts.65.up_proj", "model.layers.36.mlp.experts.66.up_proj", "model.layers.36.mlp.experts.67.up_proj", "model.layers.36.mlp.experts.68.up_proj", "model.layers.36.mlp.experts.69.up_proj", "model.layers.36.mlp.experts.70.up_proj", "model.layers.36.mlp.experts.71.up_proj", "model.layers.36.mlp.experts.72.up_proj", "model.layers.36.mlp.experts.73.up_proj", "model.layers.36.mlp.experts.74.up_proj", "model.layers.36.mlp.experts.75.up_proj", "model.layers.36.mlp.experts.76.up_proj", "model.layers.36.mlp.experts.77.up_proj", "model.layers.36.mlp.experts.78.up_proj", "model.layers.36.mlp.experts.79.up_proj", "model.layers.36.mlp.experts.80.up_proj", "model.layers.36.mlp.experts.81.up_proj", "model.layers.36.mlp.experts.82.up_proj", "model.layers.36.mlp.experts.83.up_proj", "model.layers.36.mlp.experts.84.up_proj", "model.layers.36.mlp.experts.85.up_proj", "model.layers.36.mlp.experts.86.up_proj", "model.layers.36.mlp.experts.87.up_proj", "model.layers.36.mlp.experts.88.up_proj", "model.layers.36.mlp.experts.89.up_proj", "model.layers.36.mlp.experts.90.up_proj", "model.layers.36.mlp.experts.91.up_proj", "model.layers.36.mlp.experts.92.up_proj", "model.layers.36.mlp.experts.93.up_proj", "model.layers.36.mlp.experts.94.up_proj", "model.layers.36.mlp.experts.95.up_proj", "model.layers.36.mlp.experts.96.up_proj", "model.layers.36.mlp.experts.97.up_proj", "model.layers.36.mlp.experts.98.up_proj", "model.layers.36.mlp.experts.99.up_proj", "model.layers.36.mlp.experts.100.up_proj", "model.layers.36.mlp.experts.101.up_proj", "model.layers.36.mlp.experts.102.up_proj", "model.layers.36.mlp.experts.103.up_proj", "model.layers.36.mlp.experts.104.up_proj", "model.layers.36.mlp.experts.105.up_proj", "model.layers.36.mlp.experts.106.up_proj", "model.layers.36.mlp.experts.107.up_proj", "model.layers.36.mlp.experts.108.up_proj", "model.layers.36.mlp.experts.109.up_proj", "model.layers.36.mlp.experts.110.up_proj", "model.layers.36.mlp.experts.111.up_proj", "model.layers.36.mlp.experts.112.up_proj", "model.layers.36.mlp.experts.113.up_proj", "model.layers.36.mlp.experts.114.up_proj", "model.layers.36.mlp.experts.115.up_proj", "model.layers.36.mlp.experts.116.up_proj", "model.layers.36.mlp.experts.117.up_proj", "model.layers.36.mlp.experts.118.up_proj", "model.layers.36.mlp.experts.119.up_proj", "model.layers.36.mlp.experts.120.up_proj", "model.layers.36.mlp.experts.121.up_proj", "model.layers.36.mlp.experts.122.up_proj", "model.layers.36.mlp.experts.123.up_proj", "model.layers.36.mlp.experts.124.up_proj", "model.layers.36.mlp.experts.125.up_proj", "model.layers.36.mlp.experts.126.up_proj", "model.layers.36.mlp.experts.127.up_proj", "model.layers.36.mlp.experts.0.down_proj", "model.layers.36.mlp.experts.1.down_proj", "model.layers.36.mlp.experts.2.down_proj", "model.layers.36.mlp.experts.3.down_proj", "model.layers.36.mlp.experts.4.down_proj", "model.layers.36.mlp.experts.5.down_proj", "model.layers.36.mlp.experts.6.down_proj", "model.layers.36.mlp.experts.7.down_proj", "model.layers.36.mlp.experts.8.down_proj", "model.layers.36.mlp.experts.9.down_proj", "model.layers.36.mlp.experts.10.down_proj", "model.layers.36.mlp.experts.11.down_proj", "model.layers.36.mlp.experts.12.down_proj", "model.layers.36.mlp.experts.13.down_proj", "model.layers.36.mlp.experts.14.down_proj", "model.layers.36.mlp.experts.15.down_proj", "model.layers.36.mlp.experts.16.down_proj", "model.layers.36.mlp.experts.17.down_proj", "model.layers.36.mlp.experts.18.down_proj", "model.layers.36.mlp.experts.19.down_proj", "model.layers.36.mlp.experts.20.down_proj", "model.layers.36.mlp.experts.21.down_proj", "model.layers.36.mlp.experts.22.down_proj", "model.layers.36.mlp.experts.23.down_proj", "model.layers.36.mlp.experts.24.down_proj", "model.layers.36.mlp.experts.25.down_proj", "model.layers.36.mlp.experts.26.down_proj", "model.layers.36.mlp.experts.27.down_proj", "model.layers.36.mlp.experts.28.down_proj", "model.layers.36.mlp.experts.29.down_proj", "model.layers.36.mlp.experts.30.down_proj", "model.layers.36.mlp.experts.31.down_proj", "model.layers.36.mlp.experts.32.down_proj", "model.layers.36.mlp.experts.33.down_proj", "model.layers.36.mlp.experts.34.down_proj", "model.layers.36.mlp.experts.35.down_proj", "model.layers.36.mlp.experts.36.down_proj", "model.layers.36.mlp.experts.37.down_proj", "model.layers.36.mlp.experts.38.down_proj", "model.layers.36.mlp.experts.39.down_proj", "model.layers.36.mlp.experts.40.down_proj", "model.layers.36.mlp.experts.41.down_proj", "model.layers.36.mlp.experts.42.down_proj", "model.layers.36.mlp.experts.43.down_proj", "model.layers.36.mlp.experts.44.down_proj", "model.layers.36.mlp.experts.45.down_proj", "model.layers.36.mlp.experts.46.down_proj", "model.layers.36.mlp.experts.47.down_proj", "model.layers.36.mlp.experts.48.down_proj", "model.layers.36.mlp.experts.49.down_proj", "model.layers.36.mlp.experts.50.down_proj", "model.layers.36.mlp.experts.51.down_proj", "model.layers.36.mlp.experts.52.down_proj", "model.layers.36.mlp.experts.53.down_proj", "model.layers.36.mlp.experts.54.down_proj", "model.layers.36.mlp.experts.55.down_proj", "model.layers.36.mlp.experts.56.down_proj", "model.layers.36.mlp.experts.57.down_proj", "model.layers.36.mlp.experts.58.down_proj", "model.layers.36.mlp.experts.59.down_proj", "model.layers.36.mlp.experts.60.down_proj", "model.layers.36.mlp.experts.61.down_proj", "model.layers.36.mlp.experts.62.down_proj", "model.layers.36.mlp.experts.63.down_proj", "model.layers.36.mlp.experts.64.down_proj", "model.layers.36.mlp.experts.65.down_proj", "model.layers.36.mlp.experts.66.down_proj", "model.layers.36.mlp.experts.67.down_proj", "model.layers.36.mlp.experts.68.down_proj", "model.layers.36.mlp.experts.69.down_proj", "model.layers.36.mlp.experts.70.down_proj", "model.layers.36.mlp.experts.71.down_proj", "model.layers.36.mlp.experts.72.down_proj", "model.layers.36.mlp.experts.73.down_proj", "model.layers.36.mlp.experts.74.down_proj", "model.layers.36.mlp.experts.75.down_proj", "model.layers.36.mlp.experts.76.down_proj", "model.layers.36.mlp.experts.77.down_proj", "model.layers.36.mlp.experts.78.down_proj", "model.layers.36.mlp.experts.79.down_proj", "model.layers.36.mlp.experts.80.down_proj", "model.layers.36.mlp.experts.81.down_proj", "model.layers.36.mlp.experts.82.down_proj", "model.layers.36.mlp.experts.83.down_proj", "model.layers.36.mlp.experts.84.down_proj", "model.layers.36.mlp.experts.85.down_proj", "model.layers.36.mlp.experts.86.down_proj", "model.layers.36.mlp.experts.87.down_proj", "model.layers.36.mlp.experts.88.down_proj", "model.layers.36.mlp.experts.89.down_proj", "model.layers.36.mlp.experts.90.down_proj", "model.layers.36.mlp.experts.91.down_proj", "model.layers.36.mlp.experts.92.down_proj", "model.layers.36.mlp.experts.93.down_proj", "model.layers.36.mlp.experts.94.down_proj", "model.layers.36.mlp.experts.95.down_proj", "model.layers.36.mlp.experts.96.down_proj", "model.layers.36.mlp.experts.97.down_proj", "model.layers.36.mlp.experts.98.down_proj", "model.layers.36.mlp.experts.99.down_proj", "model.layers.36.mlp.experts.100.down_proj", "model.layers.36.mlp.experts.101.down_proj", "model.layers.36.mlp.experts.102.down_proj", "model.layers.36.mlp.experts.103.down_proj", "model.layers.36.mlp.experts.104.down_proj", "model.layers.36.mlp.experts.105.down_proj", "model.layers.36.mlp.experts.106.down_proj", "model.layers.36.mlp.experts.107.down_proj", "model.layers.36.mlp.experts.108.down_proj", "model.layers.36.mlp.experts.109.down_proj", "model.layers.36.mlp.experts.110.down_proj", "model.layers.36.mlp.experts.111.down_proj", "model.layers.36.mlp.experts.112.down_proj", "model.layers.36.mlp.experts.113.down_proj", "model.layers.36.mlp.experts.114.down_proj", "model.layers.36.mlp.experts.115.down_proj", "model.layers.36.mlp.experts.116.down_proj", "model.layers.36.mlp.experts.117.down_proj", "model.layers.36.mlp.experts.118.down_proj", "model.layers.36.mlp.experts.119.down_proj", "model.layers.36.mlp.experts.120.down_proj", "model.layers.36.mlp.experts.121.down_proj", "model.layers.36.mlp.experts.122.down_proj", "model.layers.36.mlp.experts.123.down_proj", "model.layers.36.mlp.experts.124.down_proj", "model.layers.36.mlp.experts.125.down_proj", "model.layers.36.mlp.experts.126.down_proj", "model.layers.36.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0004705564118921701, "dbits": 2214592512 } ] }, { "idx": 110, "layers": [ "model.layers.37.self_attn.q_proj", "model.layers.37.self_attn.k_proj", "model.layers.37.self_attn.v_proj", "model.layers.37.self_attn.o_proj" ], "candidates": [ { "dkld": 3.474764525834839e-07, "dbits": 109051904 } ] }, { "idx": 111, "layers": [ "model.layers.37.mlp.shared_experts.gate_proj", "model.layers.37.mlp.shared_experts.up_proj", "model.layers.37.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0004672292619943563, "dbits": 17301504 } ] }, { "idx": 112, "layers": [ "model.layers.37.mlp.experts.0.gate_proj", "model.layers.37.mlp.experts.1.gate_proj", "model.layers.37.mlp.experts.2.gate_proj", "model.layers.37.mlp.experts.3.gate_proj", "model.layers.37.mlp.experts.4.gate_proj", "model.layers.37.mlp.experts.5.gate_proj", "model.layers.37.mlp.experts.6.gate_proj", "model.layers.37.mlp.experts.7.gate_proj", "model.layers.37.mlp.experts.8.gate_proj", "model.layers.37.mlp.experts.9.gate_proj", "model.layers.37.mlp.experts.10.gate_proj", "model.layers.37.mlp.experts.11.gate_proj", "model.layers.37.mlp.experts.12.gate_proj", "model.layers.37.mlp.experts.13.gate_proj", "model.layers.37.mlp.experts.14.gate_proj", "model.layers.37.mlp.experts.15.gate_proj", "model.layers.37.mlp.experts.16.gate_proj", "model.layers.37.mlp.experts.17.gate_proj", "model.layers.37.mlp.experts.18.gate_proj", "model.layers.37.mlp.experts.19.gate_proj", "model.layers.37.mlp.experts.20.gate_proj", "model.layers.37.mlp.experts.21.gate_proj", "model.layers.37.mlp.experts.22.gate_proj", "model.layers.37.mlp.experts.23.gate_proj", "model.layers.37.mlp.experts.24.gate_proj", "model.layers.37.mlp.experts.25.gate_proj", "model.layers.37.mlp.experts.26.gate_proj", "model.layers.37.mlp.experts.27.gate_proj", "model.layers.37.mlp.experts.28.gate_proj", "model.layers.37.mlp.experts.29.gate_proj", "model.layers.37.mlp.experts.30.gate_proj", "model.layers.37.mlp.experts.31.gate_proj", "model.layers.37.mlp.experts.32.gate_proj", "model.layers.37.mlp.experts.33.gate_proj", "model.layers.37.mlp.experts.34.gate_proj", "model.layers.37.mlp.experts.35.gate_proj", "model.layers.37.mlp.experts.36.gate_proj", "model.layers.37.mlp.experts.37.gate_proj", "model.layers.37.mlp.experts.38.gate_proj", "model.layers.37.mlp.experts.39.gate_proj", "model.layers.37.mlp.experts.40.gate_proj", "model.layers.37.mlp.experts.41.gate_proj", "model.layers.37.mlp.experts.42.gate_proj", "model.layers.37.mlp.experts.43.gate_proj", "model.layers.37.mlp.experts.44.gate_proj", "model.layers.37.mlp.experts.45.gate_proj", "model.layers.37.mlp.experts.46.gate_proj", "model.layers.37.mlp.experts.47.gate_proj", "model.layers.37.mlp.experts.48.gate_proj", "model.layers.37.mlp.experts.49.gate_proj", "model.layers.37.mlp.experts.50.gate_proj", "model.layers.37.mlp.experts.51.gate_proj", "model.layers.37.mlp.experts.52.gate_proj", "model.layers.37.mlp.experts.53.gate_proj", "model.layers.37.mlp.experts.54.gate_proj", "model.layers.37.mlp.experts.55.gate_proj", "model.layers.37.mlp.experts.56.gate_proj", "model.layers.37.mlp.experts.57.gate_proj", "model.layers.37.mlp.experts.58.gate_proj", "model.layers.37.mlp.experts.59.gate_proj", "model.layers.37.mlp.experts.60.gate_proj", "model.layers.37.mlp.experts.61.gate_proj", "model.layers.37.mlp.experts.62.gate_proj", "model.layers.37.mlp.experts.63.gate_proj", "model.layers.37.mlp.experts.64.gate_proj", "model.layers.37.mlp.experts.65.gate_proj", "model.layers.37.mlp.experts.66.gate_proj", "model.layers.37.mlp.experts.67.gate_proj", "model.layers.37.mlp.experts.68.gate_proj", "model.layers.37.mlp.experts.69.gate_proj", "model.layers.37.mlp.experts.70.gate_proj", "model.layers.37.mlp.experts.71.gate_proj", "model.layers.37.mlp.experts.72.gate_proj", "model.layers.37.mlp.experts.73.gate_proj", "model.layers.37.mlp.experts.74.gate_proj", "model.layers.37.mlp.experts.75.gate_proj", "model.layers.37.mlp.experts.76.gate_proj", "model.layers.37.mlp.experts.77.gate_proj", "model.layers.37.mlp.experts.78.gate_proj", "model.layers.37.mlp.experts.79.gate_proj", "model.layers.37.mlp.experts.80.gate_proj", "model.layers.37.mlp.experts.81.gate_proj", "model.layers.37.mlp.experts.82.gate_proj", "model.layers.37.mlp.experts.83.gate_proj", "model.layers.37.mlp.experts.84.gate_proj", "model.layers.37.mlp.experts.85.gate_proj", "model.layers.37.mlp.experts.86.gate_proj", "model.layers.37.mlp.experts.87.gate_proj", "model.layers.37.mlp.experts.88.gate_proj", "model.layers.37.mlp.experts.89.gate_proj", "model.layers.37.mlp.experts.90.gate_proj", "model.layers.37.mlp.experts.91.gate_proj", "model.layers.37.mlp.experts.92.gate_proj", "model.layers.37.mlp.experts.93.gate_proj", "model.layers.37.mlp.experts.94.gate_proj", "model.layers.37.mlp.experts.95.gate_proj", "model.layers.37.mlp.experts.96.gate_proj", "model.layers.37.mlp.experts.97.gate_proj", "model.layers.37.mlp.experts.98.gate_proj", "model.layers.37.mlp.experts.99.gate_proj", "model.layers.37.mlp.experts.100.gate_proj", "model.layers.37.mlp.experts.101.gate_proj", "model.layers.37.mlp.experts.102.gate_proj", "model.layers.37.mlp.experts.103.gate_proj", "model.layers.37.mlp.experts.104.gate_proj", "model.layers.37.mlp.experts.105.gate_proj", "model.layers.37.mlp.experts.106.gate_proj", "model.layers.37.mlp.experts.107.gate_proj", "model.layers.37.mlp.experts.108.gate_proj", "model.layers.37.mlp.experts.109.gate_proj", "model.layers.37.mlp.experts.110.gate_proj", "model.layers.37.mlp.experts.111.gate_proj", "model.layers.37.mlp.experts.112.gate_proj", "model.layers.37.mlp.experts.113.gate_proj", "model.layers.37.mlp.experts.114.gate_proj", "model.layers.37.mlp.experts.115.gate_proj", "model.layers.37.mlp.experts.116.gate_proj", "model.layers.37.mlp.experts.117.gate_proj", "model.layers.37.mlp.experts.118.gate_proj", "model.layers.37.mlp.experts.119.gate_proj", "model.layers.37.mlp.experts.120.gate_proj", "model.layers.37.mlp.experts.121.gate_proj", "model.layers.37.mlp.experts.122.gate_proj", "model.layers.37.mlp.experts.123.gate_proj", "model.layers.37.mlp.experts.124.gate_proj", "model.layers.37.mlp.experts.125.gate_proj", "model.layers.37.mlp.experts.126.gate_proj", "model.layers.37.mlp.experts.127.gate_proj", "model.layers.37.mlp.experts.0.up_proj", "model.layers.37.mlp.experts.1.up_proj", "model.layers.37.mlp.experts.2.up_proj", "model.layers.37.mlp.experts.3.up_proj", "model.layers.37.mlp.experts.4.up_proj", "model.layers.37.mlp.experts.5.up_proj", "model.layers.37.mlp.experts.6.up_proj", "model.layers.37.mlp.experts.7.up_proj", "model.layers.37.mlp.experts.8.up_proj", "model.layers.37.mlp.experts.9.up_proj", "model.layers.37.mlp.experts.10.up_proj", "model.layers.37.mlp.experts.11.up_proj", "model.layers.37.mlp.experts.12.up_proj", "model.layers.37.mlp.experts.13.up_proj", "model.layers.37.mlp.experts.14.up_proj", "model.layers.37.mlp.experts.15.up_proj", "model.layers.37.mlp.experts.16.up_proj", "model.layers.37.mlp.experts.17.up_proj", "model.layers.37.mlp.experts.18.up_proj", "model.layers.37.mlp.experts.19.up_proj", "model.layers.37.mlp.experts.20.up_proj", "model.layers.37.mlp.experts.21.up_proj", "model.layers.37.mlp.experts.22.up_proj", "model.layers.37.mlp.experts.23.up_proj", "model.layers.37.mlp.experts.24.up_proj", "model.layers.37.mlp.experts.25.up_proj", "model.layers.37.mlp.experts.26.up_proj", "model.layers.37.mlp.experts.27.up_proj", "model.layers.37.mlp.experts.28.up_proj", "model.layers.37.mlp.experts.29.up_proj", "model.layers.37.mlp.experts.30.up_proj", "model.layers.37.mlp.experts.31.up_proj", "model.layers.37.mlp.experts.32.up_proj", "model.layers.37.mlp.experts.33.up_proj", "model.layers.37.mlp.experts.34.up_proj", "model.layers.37.mlp.experts.35.up_proj", "model.layers.37.mlp.experts.36.up_proj", "model.layers.37.mlp.experts.37.up_proj", "model.layers.37.mlp.experts.38.up_proj", "model.layers.37.mlp.experts.39.up_proj", "model.layers.37.mlp.experts.40.up_proj", "model.layers.37.mlp.experts.41.up_proj", "model.layers.37.mlp.experts.42.up_proj", "model.layers.37.mlp.experts.43.up_proj", "model.layers.37.mlp.experts.44.up_proj", "model.layers.37.mlp.experts.45.up_proj", "model.layers.37.mlp.experts.46.up_proj", "model.layers.37.mlp.experts.47.up_proj", "model.layers.37.mlp.experts.48.up_proj", "model.layers.37.mlp.experts.49.up_proj", "model.layers.37.mlp.experts.50.up_proj", "model.layers.37.mlp.experts.51.up_proj", "model.layers.37.mlp.experts.52.up_proj", "model.layers.37.mlp.experts.53.up_proj", "model.layers.37.mlp.experts.54.up_proj", "model.layers.37.mlp.experts.55.up_proj", "model.layers.37.mlp.experts.56.up_proj", "model.layers.37.mlp.experts.57.up_proj", "model.layers.37.mlp.experts.58.up_proj", "model.layers.37.mlp.experts.59.up_proj", "model.layers.37.mlp.experts.60.up_proj", "model.layers.37.mlp.experts.61.up_proj", "model.layers.37.mlp.experts.62.up_proj", "model.layers.37.mlp.experts.63.up_proj", "model.layers.37.mlp.experts.64.up_proj", "model.layers.37.mlp.experts.65.up_proj", "model.layers.37.mlp.experts.66.up_proj", "model.layers.37.mlp.experts.67.up_proj", "model.layers.37.mlp.experts.68.up_proj", "model.layers.37.mlp.experts.69.up_proj", "model.layers.37.mlp.experts.70.up_proj", "model.layers.37.mlp.experts.71.up_proj", "model.layers.37.mlp.experts.72.up_proj", "model.layers.37.mlp.experts.73.up_proj", "model.layers.37.mlp.experts.74.up_proj", "model.layers.37.mlp.experts.75.up_proj", "model.layers.37.mlp.experts.76.up_proj", "model.layers.37.mlp.experts.77.up_proj", "model.layers.37.mlp.experts.78.up_proj", "model.layers.37.mlp.experts.79.up_proj", "model.layers.37.mlp.experts.80.up_proj", "model.layers.37.mlp.experts.81.up_proj", "model.layers.37.mlp.experts.82.up_proj", "model.layers.37.mlp.experts.83.up_proj", "model.layers.37.mlp.experts.84.up_proj", "model.layers.37.mlp.experts.85.up_proj", "model.layers.37.mlp.experts.86.up_proj", "model.layers.37.mlp.experts.87.up_proj", "model.layers.37.mlp.experts.88.up_proj", "model.layers.37.mlp.experts.89.up_proj", "model.layers.37.mlp.experts.90.up_proj", "model.layers.37.mlp.experts.91.up_proj", "model.layers.37.mlp.experts.92.up_proj", "model.layers.37.mlp.experts.93.up_proj", "model.layers.37.mlp.experts.94.up_proj", "model.layers.37.mlp.experts.95.up_proj", "model.layers.37.mlp.experts.96.up_proj", "model.layers.37.mlp.experts.97.up_proj", "model.layers.37.mlp.experts.98.up_proj", "model.layers.37.mlp.experts.99.up_proj", "model.layers.37.mlp.experts.100.up_proj", "model.layers.37.mlp.experts.101.up_proj", "model.layers.37.mlp.experts.102.up_proj", "model.layers.37.mlp.experts.103.up_proj", "model.layers.37.mlp.experts.104.up_proj", "model.layers.37.mlp.experts.105.up_proj", "model.layers.37.mlp.experts.106.up_proj", "model.layers.37.mlp.experts.107.up_proj", "model.layers.37.mlp.experts.108.up_proj", "model.layers.37.mlp.experts.109.up_proj", "model.layers.37.mlp.experts.110.up_proj", "model.layers.37.mlp.experts.111.up_proj", "model.layers.37.mlp.experts.112.up_proj", "model.layers.37.mlp.experts.113.up_proj", "model.layers.37.mlp.experts.114.up_proj", "model.layers.37.mlp.experts.115.up_proj", "model.layers.37.mlp.experts.116.up_proj", "model.layers.37.mlp.experts.117.up_proj", "model.layers.37.mlp.experts.118.up_proj", "model.layers.37.mlp.experts.119.up_proj", "model.layers.37.mlp.experts.120.up_proj", "model.layers.37.mlp.experts.121.up_proj", "model.layers.37.mlp.experts.122.up_proj", "model.layers.37.mlp.experts.123.up_proj", "model.layers.37.mlp.experts.124.up_proj", "model.layers.37.mlp.experts.125.up_proj", "model.layers.37.mlp.experts.126.up_proj", "model.layers.37.mlp.experts.127.up_proj", "model.layers.37.mlp.experts.0.down_proj", "model.layers.37.mlp.experts.1.down_proj", "model.layers.37.mlp.experts.2.down_proj", "model.layers.37.mlp.experts.3.down_proj", "model.layers.37.mlp.experts.4.down_proj", "model.layers.37.mlp.experts.5.down_proj", "model.layers.37.mlp.experts.6.down_proj", "model.layers.37.mlp.experts.7.down_proj", "model.layers.37.mlp.experts.8.down_proj", "model.layers.37.mlp.experts.9.down_proj", "model.layers.37.mlp.experts.10.down_proj", "model.layers.37.mlp.experts.11.down_proj", "model.layers.37.mlp.experts.12.down_proj", "model.layers.37.mlp.experts.13.down_proj", "model.layers.37.mlp.experts.14.down_proj", "model.layers.37.mlp.experts.15.down_proj", "model.layers.37.mlp.experts.16.down_proj", "model.layers.37.mlp.experts.17.down_proj", "model.layers.37.mlp.experts.18.down_proj", "model.layers.37.mlp.experts.19.down_proj", "model.layers.37.mlp.experts.20.down_proj", "model.layers.37.mlp.experts.21.down_proj", "model.layers.37.mlp.experts.22.down_proj", "model.layers.37.mlp.experts.23.down_proj", "model.layers.37.mlp.experts.24.down_proj", "model.layers.37.mlp.experts.25.down_proj", "model.layers.37.mlp.experts.26.down_proj", "model.layers.37.mlp.experts.27.down_proj", "model.layers.37.mlp.experts.28.down_proj", "model.layers.37.mlp.experts.29.down_proj", "model.layers.37.mlp.experts.30.down_proj", "model.layers.37.mlp.experts.31.down_proj", "model.layers.37.mlp.experts.32.down_proj", "model.layers.37.mlp.experts.33.down_proj", "model.layers.37.mlp.experts.34.down_proj", "model.layers.37.mlp.experts.35.down_proj", "model.layers.37.mlp.experts.36.down_proj", "model.layers.37.mlp.experts.37.down_proj", "model.layers.37.mlp.experts.38.down_proj", "model.layers.37.mlp.experts.39.down_proj", "model.layers.37.mlp.experts.40.down_proj", "model.layers.37.mlp.experts.41.down_proj", "model.layers.37.mlp.experts.42.down_proj", "model.layers.37.mlp.experts.43.down_proj", "model.layers.37.mlp.experts.44.down_proj", "model.layers.37.mlp.experts.45.down_proj", "model.layers.37.mlp.experts.46.down_proj", "model.layers.37.mlp.experts.47.down_proj", "model.layers.37.mlp.experts.48.down_proj", "model.layers.37.mlp.experts.49.down_proj", "model.layers.37.mlp.experts.50.down_proj", "model.layers.37.mlp.experts.51.down_proj", "model.layers.37.mlp.experts.52.down_proj", "model.layers.37.mlp.experts.53.down_proj", "model.layers.37.mlp.experts.54.down_proj", "model.layers.37.mlp.experts.55.down_proj", "model.layers.37.mlp.experts.56.down_proj", "model.layers.37.mlp.experts.57.down_proj", "model.layers.37.mlp.experts.58.down_proj", "model.layers.37.mlp.experts.59.down_proj", "model.layers.37.mlp.experts.60.down_proj", "model.layers.37.mlp.experts.61.down_proj", "model.layers.37.mlp.experts.62.down_proj", "model.layers.37.mlp.experts.63.down_proj", "model.layers.37.mlp.experts.64.down_proj", "model.layers.37.mlp.experts.65.down_proj", "model.layers.37.mlp.experts.66.down_proj", "model.layers.37.mlp.experts.67.down_proj", "model.layers.37.mlp.experts.68.down_proj", "model.layers.37.mlp.experts.69.down_proj", "model.layers.37.mlp.experts.70.down_proj", "model.layers.37.mlp.experts.71.down_proj", "model.layers.37.mlp.experts.72.down_proj", "model.layers.37.mlp.experts.73.down_proj", "model.layers.37.mlp.experts.74.down_proj", "model.layers.37.mlp.experts.75.down_proj", "model.layers.37.mlp.experts.76.down_proj", "model.layers.37.mlp.experts.77.down_proj", "model.layers.37.mlp.experts.78.down_proj", "model.layers.37.mlp.experts.79.down_proj", "model.layers.37.mlp.experts.80.down_proj", "model.layers.37.mlp.experts.81.down_proj", "model.layers.37.mlp.experts.82.down_proj", "model.layers.37.mlp.experts.83.down_proj", "model.layers.37.mlp.experts.84.down_proj", "model.layers.37.mlp.experts.85.down_proj", "model.layers.37.mlp.experts.86.down_proj", "model.layers.37.mlp.experts.87.down_proj", "model.layers.37.mlp.experts.88.down_proj", "model.layers.37.mlp.experts.89.down_proj", "model.layers.37.mlp.experts.90.down_proj", "model.layers.37.mlp.experts.91.down_proj", "model.layers.37.mlp.experts.92.down_proj", "model.layers.37.mlp.experts.93.down_proj", "model.layers.37.mlp.experts.94.down_proj", "model.layers.37.mlp.experts.95.down_proj", "model.layers.37.mlp.experts.96.down_proj", "model.layers.37.mlp.experts.97.down_proj", "model.layers.37.mlp.experts.98.down_proj", "model.layers.37.mlp.experts.99.down_proj", "model.layers.37.mlp.experts.100.down_proj", "model.layers.37.mlp.experts.101.down_proj", "model.layers.37.mlp.experts.102.down_proj", "model.layers.37.mlp.experts.103.down_proj", "model.layers.37.mlp.experts.104.down_proj", "model.layers.37.mlp.experts.105.down_proj", "model.layers.37.mlp.experts.106.down_proj", "model.layers.37.mlp.experts.107.down_proj", "model.layers.37.mlp.experts.108.down_proj", "model.layers.37.mlp.experts.109.down_proj", "model.layers.37.mlp.experts.110.down_proj", "model.layers.37.mlp.experts.111.down_proj", "model.layers.37.mlp.experts.112.down_proj", "model.layers.37.mlp.experts.113.down_proj", "model.layers.37.mlp.experts.114.down_proj", "model.layers.37.mlp.experts.115.down_proj", "model.layers.37.mlp.experts.116.down_proj", "model.layers.37.mlp.experts.117.down_proj", "model.layers.37.mlp.experts.118.down_proj", "model.layers.37.mlp.experts.119.down_proj", "model.layers.37.mlp.experts.120.down_proj", "model.layers.37.mlp.experts.121.down_proj", "model.layers.37.mlp.experts.122.down_proj", "model.layers.37.mlp.experts.123.down_proj", "model.layers.37.mlp.experts.124.down_proj", "model.layers.37.mlp.experts.125.down_proj", "model.layers.37.mlp.experts.126.down_proj", "model.layers.37.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0012277360074222088, "dbits": 2214592512 } ] }, { "idx": 113, "layers": [ "model.layers.38.self_attn.q_proj", "model.layers.38.self_attn.k_proj", "model.layers.38.self_attn.v_proj", "model.layers.38.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00020617507398126844, "dbits": 109051904 } ] }, { "idx": 114, "layers": [ "model.layers.38.mlp.shared_experts.gate_proj", "model.layers.38.mlp.shared_experts.up_proj", "model.layers.38.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00034947516396641176, "dbits": 17301504 } ] }, { "idx": 115, "layers": [ "model.layers.38.mlp.experts.0.gate_proj", "model.layers.38.mlp.experts.1.gate_proj", "model.layers.38.mlp.experts.2.gate_proj", "model.layers.38.mlp.experts.3.gate_proj", "model.layers.38.mlp.experts.4.gate_proj", "model.layers.38.mlp.experts.5.gate_proj", "model.layers.38.mlp.experts.6.gate_proj", "model.layers.38.mlp.experts.7.gate_proj", "model.layers.38.mlp.experts.8.gate_proj", "model.layers.38.mlp.experts.9.gate_proj", "model.layers.38.mlp.experts.10.gate_proj", "model.layers.38.mlp.experts.11.gate_proj", "model.layers.38.mlp.experts.12.gate_proj", "model.layers.38.mlp.experts.13.gate_proj", "model.layers.38.mlp.experts.14.gate_proj", "model.layers.38.mlp.experts.15.gate_proj", "model.layers.38.mlp.experts.16.gate_proj", "model.layers.38.mlp.experts.17.gate_proj", "model.layers.38.mlp.experts.18.gate_proj", "model.layers.38.mlp.experts.19.gate_proj", "model.layers.38.mlp.experts.20.gate_proj", "model.layers.38.mlp.experts.21.gate_proj", "model.layers.38.mlp.experts.22.gate_proj", "model.layers.38.mlp.experts.23.gate_proj", "model.layers.38.mlp.experts.24.gate_proj", "model.layers.38.mlp.experts.25.gate_proj", "model.layers.38.mlp.experts.26.gate_proj", "model.layers.38.mlp.experts.27.gate_proj", "model.layers.38.mlp.experts.28.gate_proj", "model.layers.38.mlp.experts.29.gate_proj", "model.layers.38.mlp.experts.30.gate_proj", "model.layers.38.mlp.experts.31.gate_proj", "model.layers.38.mlp.experts.32.gate_proj", "model.layers.38.mlp.experts.33.gate_proj", "model.layers.38.mlp.experts.34.gate_proj", "model.layers.38.mlp.experts.35.gate_proj", "model.layers.38.mlp.experts.36.gate_proj", "model.layers.38.mlp.experts.37.gate_proj", "model.layers.38.mlp.experts.38.gate_proj", "model.layers.38.mlp.experts.39.gate_proj", "model.layers.38.mlp.experts.40.gate_proj", "model.layers.38.mlp.experts.41.gate_proj", "model.layers.38.mlp.experts.42.gate_proj", "model.layers.38.mlp.experts.43.gate_proj", "model.layers.38.mlp.experts.44.gate_proj", "model.layers.38.mlp.experts.45.gate_proj", "model.layers.38.mlp.experts.46.gate_proj", "model.layers.38.mlp.experts.47.gate_proj", "model.layers.38.mlp.experts.48.gate_proj", "model.layers.38.mlp.experts.49.gate_proj", "model.layers.38.mlp.experts.50.gate_proj", "model.layers.38.mlp.experts.51.gate_proj", "model.layers.38.mlp.experts.52.gate_proj", "model.layers.38.mlp.experts.53.gate_proj", "model.layers.38.mlp.experts.54.gate_proj", "model.layers.38.mlp.experts.55.gate_proj", "model.layers.38.mlp.experts.56.gate_proj", "model.layers.38.mlp.experts.57.gate_proj", "model.layers.38.mlp.experts.58.gate_proj", "model.layers.38.mlp.experts.59.gate_proj", "model.layers.38.mlp.experts.60.gate_proj", "model.layers.38.mlp.experts.61.gate_proj", "model.layers.38.mlp.experts.62.gate_proj", "model.layers.38.mlp.experts.63.gate_proj", "model.layers.38.mlp.experts.64.gate_proj", "model.layers.38.mlp.experts.65.gate_proj", "model.layers.38.mlp.experts.66.gate_proj", "model.layers.38.mlp.experts.67.gate_proj", "model.layers.38.mlp.experts.68.gate_proj", "model.layers.38.mlp.experts.69.gate_proj", "model.layers.38.mlp.experts.70.gate_proj", "model.layers.38.mlp.experts.71.gate_proj", "model.layers.38.mlp.experts.72.gate_proj", "model.layers.38.mlp.experts.73.gate_proj", "model.layers.38.mlp.experts.74.gate_proj", "model.layers.38.mlp.experts.75.gate_proj", "model.layers.38.mlp.experts.76.gate_proj", "model.layers.38.mlp.experts.77.gate_proj", "model.layers.38.mlp.experts.78.gate_proj", "model.layers.38.mlp.experts.79.gate_proj", "model.layers.38.mlp.experts.80.gate_proj", "model.layers.38.mlp.experts.81.gate_proj", "model.layers.38.mlp.experts.82.gate_proj", "model.layers.38.mlp.experts.83.gate_proj", "model.layers.38.mlp.experts.84.gate_proj", "model.layers.38.mlp.experts.85.gate_proj", "model.layers.38.mlp.experts.86.gate_proj", "model.layers.38.mlp.experts.87.gate_proj", "model.layers.38.mlp.experts.88.gate_proj", "model.layers.38.mlp.experts.89.gate_proj", "model.layers.38.mlp.experts.90.gate_proj", "model.layers.38.mlp.experts.91.gate_proj", "model.layers.38.mlp.experts.92.gate_proj", "model.layers.38.mlp.experts.93.gate_proj", "model.layers.38.mlp.experts.94.gate_proj", "model.layers.38.mlp.experts.95.gate_proj", "model.layers.38.mlp.experts.96.gate_proj", "model.layers.38.mlp.experts.97.gate_proj", "model.layers.38.mlp.experts.98.gate_proj", "model.layers.38.mlp.experts.99.gate_proj", "model.layers.38.mlp.experts.100.gate_proj", "model.layers.38.mlp.experts.101.gate_proj", "model.layers.38.mlp.experts.102.gate_proj", "model.layers.38.mlp.experts.103.gate_proj", "model.layers.38.mlp.experts.104.gate_proj", "model.layers.38.mlp.experts.105.gate_proj", "model.layers.38.mlp.experts.106.gate_proj", "model.layers.38.mlp.experts.107.gate_proj", "model.layers.38.mlp.experts.108.gate_proj", "model.layers.38.mlp.experts.109.gate_proj", "model.layers.38.mlp.experts.110.gate_proj", "model.layers.38.mlp.experts.111.gate_proj", "model.layers.38.mlp.experts.112.gate_proj", "model.layers.38.mlp.experts.113.gate_proj", "model.layers.38.mlp.experts.114.gate_proj", "model.layers.38.mlp.experts.115.gate_proj", "model.layers.38.mlp.experts.116.gate_proj", "model.layers.38.mlp.experts.117.gate_proj", "model.layers.38.mlp.experts.118.gate_proj", "model.layers.38.mlp.experts.119.gate_proj", "model.layers.38.mlp.experts.120.gate_proj", "model.layers.38.mlp.experts.121.gate_proj", "model.layers.38.mlp.experts.122.gate_proj", "model.layers.38.mlp.experts.123.gate_proj", "model.layers.38.mlp.experts.124.gate_proj", "model.layers.38.mlp.experts.125.gate_proj", "model.layers.38.mlp.experts.126.gate_proj", "model.layers.38.mlp.experts.127.gate_proj", "model.layers.38.mlp.experts.0.up_proj", "model.layers.38.mlp.experts.1.up_proj", "model.layers.38.mlp.experts.2.up_proj", "model.layers.38.mlp.experts.3.up_proj", "model.layers.38.mlp.experts.4.up_proj", "model.layers.38.mlp.experts.5.up_proj", "model.layers.38.mlp.experts.6.up_proj", "model.layers.38.mlp.experts.7.up_proj", "model.layers.38.mlp.experts.8.up_proj", "model.layers.38.mlp.experts.9.up_proj", "model.layers.38.mlp.experts.10.up_proj", "model.layers.38.mlp.experts.11.up_proj", "model.layers.38.mlp.experts.12.up_proj", "model.layers.38.mlp.experts.13.up_proj", "model.layers.38.mlp.experts.14.up_proj", "model.layers.38.mlp.experts.15.up_proj", "model.layers.38.mlp.experts.16.up_proj", "model.layers.38.mlp.experts.17.up_proj", "model.layers.38.mlp.experts.18.up_proj", "model.layers.38.mlp.experts.19.up_proj", "model.layers.38.mlp.experts.20.up_proj", "model.layers.38.mlp.experts.21.up_proj", "model.layers.38.mlp.experts.22.up_proj", "model.layers.38.mlp.experts.23.up_proj", "model.layers.38.mlp.experts.24.up_proj", "model.layers.38.mlp.experts.25.up_proj", "model.layers.38.mlp.experts.26.up_proj", "model.layers.38.mlp.experts.27.up_proj", "model.layers.38.mlp.experts.28.up_proj", "model.layers.38.mlp.experts.29.up_proj", "model.layers.38.mlp.experts.30.up_proj", "model.layers.38.mlp.experts.31.up_proj", "model.layers.38.mlp.experts.32.up_proj", "model.layers.38.mlp.experts.33.up_proj", "model.layers.38.mlp.experts.34.up_proj", "model.layers.38.mlp.experts.35.up_proj", "model.layers.38.mlp.experts.36.up_proj", "model.layers.38.mlp.experts.37.up_proj", "model.layers.38.mlp.experts.38.up_proj", "model.layers.38.mlp.experts.39.up_proj", "model.layers.38.mlp.experts.40.up_proj", "model.layers.38.mlp.experts.41.up_proj", "model.layers.38.mlp.experts.42.up_proj", "model.layers.38.mlp.experts.43.up_proj", "model.layers.38.mlp.experts.44.up_proj", "model.layers.38.mlp.experts.45.up_proj", "model.layers.38.mlp.experts.46.up_proj", "model.layers.38.mlp.experts.47.up_proj", "model.layers.38.mlp.experts.48.up_proj", "model.layers.38.mlp.experts.49.up_proj", "model.layers.38.mlp.experts.50.up_proj", "model.layers.38.mlp.experts.51.up_proj", "model.layers.38.mlp.experts.52.up_proj", "model.layers.38.mlp.experts.53.up_proj", "model.layers.38.mlp.experts.54.up_proj", "model.layers.38.mlp.experts.55.up_proj", "model.layers.38.mlp.experts.56.up_proj", "model.layers.38.mlp.experts.57.up_proj", "model.layers.38.mlp.experts.58.up_proj", "model.layers.38.mlp.experts.59.up_proj", "model.layers.38.mlp.experts.60.up_proj", "model.layers.38.mlp.experts.61.up_proj", "model.layers.38.mlp.experts.62.up_proj", "model.layers.38.mlp.experts.63.up_proj", "model.layers.38.mlp.experts.64.up_proj", "model.layers.38.mlp.experts.65.up_proj", "model.layers.38.mlp.experts.66.up_proj", "model.layers.38.mlp.experts.67.up_proj", "model.layers.38.mlp.experts.68.up_proj", "model.layers.38.mlp.experts.69.up_proj", "model.layers.38.mlp.experts.70.up_proj", "model.layers.38.mlp.experts.71.up_proj", "model.layers.38.mlp.experts.72.up_proj", "model.layers.38.mlp.experts.73.up_proj", "model.layers.38.mlp.experts.74.up_proj", "model.layers.38.mlp.experts.75.up_proj", "model.layers.38.mlp.experts.76.up_proj", "model.layers.38.mlp.experts.77.up_proj", "model.layers.38.mlp.experts.78.up_proj", "model.layers.38.mlp.experts.79.up_proj", "model.layers.38.mlp.experts.80.up_proj", "model.layers.38.mlp.experts.81.up_proj", "model.layers.38.mlp.experts.82.up_proj", "model.layers.38.mlp.experts.83.up_proj", "model.layers.38.mlp.experts.84.up_proj", "model.layers.38.mlp.experts.85.up_proj", "model.layers.38.mlp.experts.86.up_proj", "model.layers.38.mlp.experts.87.up_proj", "model.layers.38.mlp.experts.88.up_proj", "model.layers.38.mlp.experts.89.up_proj", "model.layers.38.mlp.experts.90.up_proj", "model.layers.38.mlp.experts.91.up_proj", "model.layers.38.mlp.experts.92.up_proj", "model.layers.38.mlp.experts.93.up_proj", "model.layers.38.mlp.experts.94.up_proj", "model.layers.38.mlp.experts.95.up_proj", "model.layers.38.mlp.experts.96.up_proj", "model.layers.38.mlp.experts.97.up_proj", "model.layers.38.mlp.experts.98.up_proj", "model.layers.38.mlp.experts.99.up_proj", "model.layers.38.mlp.experts.100.up_proj", "model.layers.38.mlp.experts.101.up_proj", "model.layers.38.mlp.experts.102.up_proj", "model.layers.38.mlp.experts.103.up_proj", "model.layers.38.mlp.experts.104.up_proj", "model.layers.38.mlp.experts.105.up_proj", "model.layers.38.mlp.experts.106.up_proj", "model.layers.38.mlp.experts.107.up_proj", "model.layers.38.mlp.experts.108.up_proj", "model.layers.38.mlp.experts.109.up_proj", "model.layers.38.mlp.experts.110.up_proj", "model.layers.38.mlp.experts.111.up_proj", "model.layers.38.mlp.experts.112.up_proj", "model.layers.38.mlp.experts.113.up_proj", "model.layers.38.mlp.experts.114.up_proj", "model.layers.38.mlp.experts.115.up_proj", "model.layers.38.mlp.experts.116.up_proj", "model.layers.38.mlp.experts.117.up_proj", "model.layers.38.mlp.experts.118.up_proj", "model.layers.38.mlp.experts.119.up_proj", "model.layers.38.mlp.experts.120.up_proj", "model.layers.38.mlp.experts.121.up_proj", "model.layers.38.mlp.experts.122.up_proj", "model.layers.38.mlp.experts.123.up_proj", "model.layers.38.mlp.experts.124.up_proj", "model.layers.38.mlp.experts.125.up_proj", "model.layers.38.mlp.experts.126.up_proj", "model.layers.38.mlp.experts.127.up_proj", "model.layers.38.mlp.experts.0.down_proj", "model.layers.38.mlp.experts.1.down_proj", "model.layers.38.mlp.experts.2.down_proj", "model.layers.38.mlp.experts.3.down_proj", "model.layers.38.mlp.experts.4.down_proj", "model.layers.38.mlp.experts.5.down_proj", "model.layers.38.mlp.experts.6.down_proj", "model.layers.38.mlp.experts.7.down_proj", "model.layers.38.mlp.experts.8.down_proj", "model.layers.38.mlp.experts.9.down_proj", "model.layers.38.mlp.experts.10.down_proj", "model.layers.38.mlp.experts.11.down_proj", "model.layers.38.mlp.experts.12.down_proj", "model.layers.38.mlp.experts.13.down_proj", "model.layers.38.mlp.experts.14.down_proj", "model.layers.38.mlp.experts.15.down_proj", "model.layers.38.mlp.experts.16.down_proj", "model.layers.38.mlp.experts.17.down_proj", "model.layers.38.mlp.experts.18.down_proj", "model.layers.38.mlp.experts.19.down_proj", "model.layers.38.mlp.experts.20.down_proj", "model.layers.38.mlp.experts.21.down_proj", "model.layers.38.mlp.experts.22.down_proj", "model.layers.38.mlp.experts.23.down_proj", "model.layers.38.mlp.experts.24.down_proj", "model.layers.38.mlp.experts.25.down_proj", "model.layers.38.mlp.experts.26.down_proj", "model.layers.38.mlp.experts.27.down_proj", "model.layers.38.mlp.experts.28.down_proj", "model.layers.38.mlp.experts.29.down_proj", "model.layers.38.mlp.experts.30.down_proj", "model.layers.38.mlp.experts.31.down_proj", "model.layers.38.mlp.experts.32.down_proj", "model.layers.38.mlp.experts.33.down_proj", "model.layers.38.mlp.experts.34.down_proj", "model.layers.38.mlp.experts.35.down_proj", "model.layers.38.mlp.experts.36.down_proj", "model.layers.38.mlp.experts.37.down_proj", "model.layers.38.mlp.experts.38.down_proj", "model.layers.38.mlp.experts.39.down_proj", "model.layers.38.mlp.experts.40.down_proj", "model.layers.38.mlp.experts.41.down_proj", "model.layers.38.mlp.experts.42.down_proj", "model.layers.38.mlp.experts.43.down_proj", "model.layers.38.mlp.experts.44.down_proj", "model.layers.38.mlp.experts.45.down_proj", "model.layers.38.mlp.experts.46.down_proj", "model.layers.38.mlp.experts.47.down_proj", "model.layers.38.mlp.experts.48.down_proj", "model.layers.38.mlp.experts.49.down_proj", "model.layers.38.mlp.experts.50.down_proj", "model.layers.38.mlp.experts.51.down_proj", "model.layers.38.mlp.experts.52.down_proj", "model.layers.38.mlp.experts.53.down_proj", "model.layers.38.mlp.experts.54.down_proj", "model.layers.38.mlp.experts.55.down_proj", "model.layers.38.mlp.experts.56.down_proj", "model.layers.38.mlp.experts.57.down_proj", "model.layers.38.mlp.experts.58.down_proj", "model.layers.38.mlp.experts.59.down_proj", "model.layers.38.mlp.experts.60.down_proj", "model.layers.38.mlp.experts.61.down_proj", "model.layers.38.mlp.experts.62.down_proj", "model.layers.38.mlp.experts.63.down_proj", "model.layers.38.mlp.experts.64.down_proj", "model.layers.38.mlp.experts.65.down_proj", "model.layers.38.mlp.experts.66.down_proj", "model.layers.38.mlp.experts.67.down_proj", "model.layers.38.mlp.experts.68.down_proj", "model.layers.38.mlp.experts.69.down_proj", "model.layers.38.mlp.experts.70.down_proj", "model.layers.38.mlp.experts.71.down_proj", "model.layers.38.mlp.experts.72.down_proj", "model.layers.38.mlp.experts.73.down_proj", "model.layers.38.mlp.experts.74.down_proj", "model.layers.38.mlp.experts.75.down_proj", "model.layers.38.mlp.experts.76.down_proj", "model.layers.38.mlp.experts.77.down_proj", "model.layers.38.mlp.experts.78.down_proj", "model.layers.38.mlp.experts.79.down_proj", "model.layers.38.mlp.experts.80.down_proj", "model.layers.38.mlp.experts.81.down_proj", "model.layers.38.mlp.experts.82.down_proj", "model.layers.38.mlp.experts.83.down_proj", "model.layers.38.mlp.experts.84.down_proj", "model.layers.38.mlp.experts.85.down_proj", "model.layers.38.mlp.experts.86.down_proj", "model.layers.38.mlp.experts.87.down_proj", "model.layers.38.mlp.experts.88.down_proj", "model.layers.38.mlp.experts.89.down_proj", "model.layers.38.mlp.experts.90.down_proj", "model.layers.38.mlp.experts.91.down_proj", "model.layers.38.mlp.experts.92.down_proj", "model.layers.38.mlp.experts.93.down_proj", "model.layers.38.mlp.experts.94.down_proj", "model.layers.38.mlp.experts.95.down_proj", "model.layers.38.mlp.experts.96.down_proj", "model.layers.38.mlp.experts.97.down_proj", "model.layers.38.mlp.experts.98.down_proj", "model.layers.38.mlp.experts.99.down_proj", "model.layers.38.mlp.experts.100.down_proj", "model.layers.38.mlp.experts.101.down_proj", "model.layers.38.mlp.experts.102.down_proj", "model.layers.38.mlp.experts.103.down_proj", "model.layers.38.mlp.experts.104.down_proj", "model.layers.38.mlp.experts.105.down_proj", "model.layers.38.mlp.experts.106.down_proj", "model.layers.38.mlp.experts.107.down_proj", "model.layers.38.mlp.experts.108.down_proj", "model.layers.38.mlp.experts.109.down_proj", "model.layers.38.mlp.experts.110.down_proj", "model.layers.38.mlp.experts.111.down_proj", "model.layers.38.mlp.experts.112.down_proj", "model.layers.38.mlp.experts.113.down_proj", "model.layers.38.mlp.experts.114.down_proj", "model.layers.38.mlp.experts.115.down_proj", "model.layers.38.mlp.experts.116.down_proj", "model.layers.38.mlp.experts.117.down_proj", "model.layers.38.mlp.experts.118.down_proj", "model.layers.38.mlp.experts.119.down_proj", "model.layers.38.mlp.experts.120.down_proj", "model.layers.38.mlp.experts.121.down_proj", "model.layers.38.mlp.experts.122.down_proj", "model.layers.38.mlp.experts.123.down_proj", "model.layers.38.mlp.experts.124.down_proj", "model.layers.38.mlp.experts.125.down_proj", "model.layers.38.mlp.experts.126.down_proj", "model.layers.38.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0007538830861449075, "dbits": 2214592512 } ] }, { "idx": 116, "layers": [ "model.layers.39.self_attn.q_proj", "model.layers.39.self_attn.k_proj", "model.layers.39.self_attn.v_proj", "model.layers.39.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0004628580994903986, "dbits": 109051904 } ] }, { "idx": 117, "layers": [ "model.layers.39.mlp.shared_experts.gate_proj", "model.layers.39.mlp.shared_experts.up_proj", "model.layers.39.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": 0.0002931061200797669, "dbits": 17301504 } ] }, { "idx": 118, "layers": [ "model.layers.39.mlp.experts.0.gate_proj", "model.layers.39.mlp.experts.1.gate_proj", "model.layers.39.mlp.experts.2.gate_proj", "model.layers.39.mlp.experts.3.gate_proj", "model.layers.39.mlp.experts.4.gate_proj", "model.layers.39.mlp.experts.5.gate_proj", "model.layers.39.mlp.experts.6.gate_proj", "model.layers.39.mlp.experts.7.gate_proj", "model.layers.39.mlp.experts.8.gate_proj", "model.layers.39.mlp.experts.9.gate_proj", "model.layers.39.mlp.experts.10.gate_proj", "model.layers.39.mlp.experts.11.gate_proj", "model.layers.39.mlp.experts.12.gate_proj", "model.layers.39.mlp.experts.13.gate_proj", "model.layers.39.mlp.experts.14.gate_proj", "model.layers.39.mlp.experts.15.gate_proj", "model.layers.39.mlp.experts.16.gate_proj", "model.layers.39.mlp.experts.17.gate_proj", "model.layers.39.mlp.experts.18.gate_proj", "model.layers.39.mlp.experts.19.gate_proj", "model.layers.39.mlp.experts.20.gate_proj", "model.layers.39.mlp.experts.21.gate_proj", "model.layers.39.mlp.experts.22.gate_proj", "model.layers.39.mlp.experts.23.gate_proj", "model.layers.39.mlp.experts.24.gate_proj", "model.layers.39.mlp.experts.25.gate_proj", "model.layers.39.mlp.experts.26.gate_proj", "model.layers.39.mlp.experts.27.gate_proj", "model.layers.39.mlp.experts.28.gate_proj", "model.layers.39.mlp.experts.29.gate_proj", "model.layers.39.mlp.experts.30.gate_proj", "model.layers.39.mlp.experts.31.gate_proj", "model.layers.39.mlp.experts.32.gate_proj", "model.layers.39.mlp.experts.33.gate_proj", "model.layers.39.mlp.experts.34.gate_proj", "model.layers.39.mlp.experts.35.gate_proj", "model.layers.39.mlp.experts.36.gate_proj", "model.layers.39.mlp.experts.37.gate_proj", "model.layers.39.mlp.experts.38.gate_proj", "model.layers.39.mlp.experts.39.gate_proj", "model.layers.39.mlp.experts.40.gate_proj", "model.layers.39.mlp.experts.41.gate_proj", "model.layers.39.mlp.experts.42.gate_proj", "model.layers.39.mlp.experts.43.gate_proj", "model.layers.39.mlp.experts.44.gate_proj", "model.layers.39.mlp.experts.45.gate_proj", "model.layers.39.mlp.experts.46.gate_proj", "model.layers.39.mlp.experts.47.gate_proj", "model.layers.39.mlp.experts.48.gate_proj", "model.layers.39.mlp.experts.49.gate_proj", "model.layers.39.mlp.experts.50.gate_proj", "model.layers.39.mlp.experts.51.gate_proj", "model.layers.39.mlp.experts.52.gate_proj", "model.layers.39.mlp.experts.53.gate_proj", "model.layers.39.mlp.experts.54.gate_proj", "model.layers.39.mlp.experts.55.gate_proj", "model.layers.39.mlp.experts.56.gate_proj", "model.layers.39.mlp.experts.57.gate_proj", "model.layers.39.mlp.experts.58.gate_proj", "model.layers.39.mlp.experts.59.gate_proj", "model.layers.39.mlp.experts.60.gate_proj", "model.layers.39.mlp.experts.61.gate_proj", "model.layers.39.mlp.experts.62.gate_proj", "model.layers.39.mlp.experts.63.gate_proj", "model.layers.39.mlp.experts.64.gate_proj", "model.layers.39.mlp.experts.65.gate_proj", "model.layers.39.mlp.experts.66.gate_proj", "model.layers.39.mlp.experts.67.gate_proj", "model.layers.39.mlp.experts.68.gate_proj", "model.layers.39.mlp.experts.69.gate_proj", "model.layers.39.mlp.experts.70.gate_proj", "model.layers.39.mlp.experts.71.gate_proj", "model.layers.39.mlp.experts.72.gate_proj", "model.layers.39.mlp.experts.73.gate_proj", "model.layers.39.mlp.experts.74.gate_proj", "model.layers.39.mlp.experts.75.gate_proj", "model.layers.39.mlp.experts.76.gate_proj", "model.layers.39.mlp.experts.77.gate_proj", "model.layers.39.mlp.experts.78.gate_proj", "model.layers.39.mlp.experts.79.gate_proj", "model.layers.39.mlp.experts.80.gate_proj", "model.layers.39.mlp.experts.81.gate_proj", "model.layers.39.mlp.experts.82.gate_proj", "model.layers.39.mlp.experts.83.gate_proj", "model.layers.39.mlp.experts.84.gate_proj", "model.layers.39.mlp.experts.85.gate_proj", "model.layers.39.mlp.experts.86.gate_proj", "model.layers.39.mlp.experts.87.gate_proj", "model.layers.39.mlp.experts.88.gate_proj", "model.layers.39.mlp.experts.89.gate_proj", "model.layers.39.mlp.experts.90.gate_proj", "model.layers.39.mlp.experts.91.gate_proj", "model.layers.39.mlp.experts.92.gate_proj", "model.layers.39.mlp.experts.93.gate_proj", "model.layers.39.mlp.experts.94.gate_proj", "model.layers.39.mlp.experts.95.gate_proj", "model.layers.39.mlp.experts.96.gate_proj", "model.layers.39.mlp.experts.97.gate_proj", "model.layers.39.mlp.experts.98.gate_proj", "model.layers.39.mlp.experts.99.gate_proj", "model.layers.39.mlp.experts.100.gate_proj", "model.layers.39.mlp.experts.101.gate_proj", "model.layers.39.mlp.experts.102.gate_proj", "model.layers.39.mlp.experts.103.gate_proj", "model.layers.39.mlp.experts.104.gate_proj", "model.layers.39.mlp.experts.105.gate_proj", "model.layers.39.mlp.experts.106.gate_proj", "model.layers.39.mlp.experts.107.gate_proj", "model.layers.39.mlp.experts.108.gate_proj", "model.layers.39.mlp.experts.109.gate_proj", "model.layers.39.mlp.experts.110.gate_proj", "model.layers.39.mlp.experts.111.gate_proj", "model.layers.39.mlp.experts.112.gate_proj", "model.layers.39.mlp.experts.113.gate_proj", "model.layers.39.mlp.experts.114.gate_proj", "model.layers.39.mlp.experts.115.gate_proj", "model.layers.39.mlp.experts.116.gate_proj", "model.layers.39.mlp.experts.117.gate_proj", "model.layers.39.mlp.experts.118.gate_proj", "model.layers.39.mlp.experts.119.gate_proj", "model.layers.39.mlp.experts.120.gate_proj", "model.layers.39.mlp.experts.121.gate_proj", "model.layers.39.mlp.experts.122.gate_proj", "model.layers.39.mlp.experts.123.gate_proj", "model.layers.39.mlp.experts.124.gate_proj", "model.layers.39.mlp.experts.125.gate_proj", "model.layers.39.mlp.experts.126.gate_proj", "model.layers.39.mlp.experts.127.gate_proj", "model.layers.39.mlp.experts.0.up_proj", "model.layers.39.mlp.experts.1.up_proj", "model.layers.39.mlp.experts.2.up_proj", "model.layers.39.mlp.experts.3.up_proj", "model.layers.39.mlp.experts.4.up_proj", "model.layers.39.mlp.experts.5.up_proj", "model.layers.39.mlp.experts.6.up_proj", "model.layers.39.mlp.experts.7.up_proj", "model.layers.39.mlp.experts.8.up_proj", "model.layers.39.mlp.experts.9.up_proj", "model.layers.39.mlp.experts.10.up_proj", "model.layers.39.mlp.experts.11.up_proj", "model.layers.39.mlp.experts.12.up_proj", "model.layers.39.mlp.experts.13.up_proj", "model.layers.39.mlp.experts.14.up_proj", "model.layers.39.mlp.experts.15.up_proj", "model.layers.39.mlp.experts.16.up_proj", "model.layers.39.mlp.experts.17.up_proj", "model.layers.39.mlp.experts.18.up_proj", "model.layers.39.mlp.experts.19.up_proj", "model.layers.39.mlp.experts.20.up_proj", "model.layers.39.mlp.experts.21.up_proj", "model.layers.39.mlp.experts.22.up_proj", "model.layers.39.mlp.experts.23.up_proj", "model.layers.39.mlp.experts.24.up_proj", "model.layers.39.mlp.experts.25.up_proj", "model.layers.39.mlp.experts.26.up_proj", "model.layers.39.mlp.experts.27.up_proj", "model.layers.39.mlp.experts.28.up_proj", "model.layers.39.mlp.experts.29.up_proj", "model.layers.39.mlp.experts.30.up_proj", "model.layers.39.mlp.experts.31.up_proj", "model.layers.39.mlp.experts.32.up_proj", "model.layers.39.mlp.experts.33.up_proj", "model.layers.39.mlp.experts.34.up_proj", "model.layers.39.mlp.experts.35.up_proj", "model.layers.39.mlp.experts.36.up_proj", "model.layers.39.mlp.experts.37.up_proj", "model.layers.39.mlp.experts.38.up_proj", "model.layers.39.mlp.experts.39.up_proj", "model.layers.39.mlp.experts.40.up_proj", "model.layers.39.mlp.experts.41.up_proj", "model.layers.39.mlp.experts.42.up_proj", "model.layers.39.mlp.experts.43.up_proj", "model.layers.39.mlp.experts.44.up_proj", "model.layers.39.mlp.experts.45.up_proj", "model.layers.39.mlp.experts.46.up_proj", "model.layers.39.mlp.experts.47.up_proj", "model.layers.39.mlp.experts.48.up_proj", "model.layers.39.mlp.experts.49.up_proj", "model.layers.39.mlp.experts.50.up_proj", "model.layers.39.mlp.experts.51.up_proj", "model.layers.39.mlp.experts.52.up_proj", "model.layers.39.mlp.experts.53.up_proj", "model.layers.39.mlp.experts.54.up_proj", "model.layers.39.mlp.experts.55.up_proj", "model.layers.39.mlp.experts.56.up_proj", "model.layers.39.mlp.experts.57.up_proj", "model.layers.39.mlp.experts.58.up_proj", "model.layers.39.mlp.experts.59.up_proj", "model.layers.39.mlp.experts.60.up_proj", "model.layers.39.mlp.experts.61.up_proj", "model.layers.39.mlp.experts.62.up_proj", "model.layers.39.mlp.experts.63.up_proj", "model.layers.39.mlp.experts.64.up_proj", "model.layers.39.mlp.experts.65.up_proj", "model.layers.39.mlp.experts.66.up_proj", "model.layers.39.mlp.experts.67.up_proj", "model.layers.39.mlp.experts.68.up_proj", "model.layers.39.mlp.experts.69.up_proj", "model.layers.39.mlp.experts.70.up_proj", "model.layers.39.mlp.experts.71.up_proj", "model.layers.39.mlp.experts.72.up_proj", "model.layers.39.mlp.experts.73.up_proj", "model.layers.39.mlp.experts.74.up_proj", "model.layers.39.mlp.experts.75.up_proj", "model.layers.39.mlp.experts.76.up_proj", "model.layers.39.mlp.experts.77.up_proj", "model.layers.39.mlp.experts.78.up_proj", "model.layers.39.mlp.experts.79.up_proj", "model.layers.39.mlp.experts.80.up_proj", "model.layers.39.mlp.experts.81.up_proj", "model.layers.39.mlp.experts.82.up_proj", "model.layers.39.mlp.experts.83.up_proj", "model.layers.39.mlp.experts.84.up_proj", "model.layers.39.mlp.experts.85.up_proj", "model.layers.39.mlp.experts.86.up_proj", "model.layers.39.mlp.experts.87.up_proj", "model.layers.39.mlp.experts.88.up_proj", "model.layers.39.mlp.experts.89.up_proj", "model.layers.39.mlp.experts.90.up_proj", "model.layers.39.mlp.experts.91.up_proj", "model.layers.39.mlp.experts.92.up_proj", "model.layers.39.mlp.experts.93.up_proj", "model.layers.39.mlp.experts.94.up_proj", "model.layers.39.mlp.experts.95.up_proj", "model.layers.39.mlp.experts.96.up_proj", "model.layers.39.mlp.experts.97.up_proj", "model.layers.39.mlp.experts.98.up_proj", "model.layers.39.mlp.experts.99.up_proj", "model.layers.39.mlp.experts.100.up_proj", "model.layers.39.mlp.experts.101.up_proj", "model.layers.39.mlp.experts.102.up_proj", "model.layers.39.mlp.experts.103.up_proj", "model.layers.39.mlp.experts.104.up_proj", "model.layers.39.mlp.experts.105.up_proj", "model.layers.39.mlp.experts.106.up_proj", "model.layers.39.mlp.experts.107.up_proj", "model.layers.39.mlp.experts.108.up_proj", "model.layers.39.mlp.experts.109.up_proj", "model.layers.39.mlp.experts.110.up_proj", "model.layers.39.mlp.experts.111.up_proj", "model.layers.39.mlp.experts.112.up_proj", "model.layers.39.mlp.experts.113.up_proj", "model.layers.39.mlp.experts.114.up_proj", "model.layers.39.mlp.experts.115.up_proj", "model.layers.39.mlp.experts.116.up_proj", "model.layers.39.mlp.experts.117.up_proj", "model.layers.39.mlp.experts.118.up_proj", "model.layers.39.mlp.experts.119.up_proj", "model.layers.39.mlp.experts.120.up_proj", "model.layers.39.mlp.experts.121.up_proj", "model.layers.39.mlp.experts.122.up_proj", "model.layers.39.mlp.experts.123.up_proj", "model.layers.39.mlp.experts.124.up_proj", "model.layers.39.mlp.experts.125.up_proj", "model.layers.39.mlp.experts.126.up_proj", "model.layers.39.mlp.experts.127.up_proj", "model.layers.39.mlp.experts.0.down_proj", "model.layers.39.mlp.experts.1.down_proj", "model.layers.39.mlp.experts.2.down_proj", "model.layers.39.mlp.experts.3.down_proj", "model.layers.39.mlp.experts.4.down_proj", "model.layers.39.mlp.experts.5.down_proj", "model.layers.39.mlp.experts.6.down_proj", "model.layers.39.mlp.experts.7.down_proj", "model.layers.39.mlp.experts.8.down_proj", "model.layers.39.mlp.experts.9.down_proj", "model.layers.39.mlp.experts.10.down_proj", "model.layers.39.mlp.experts.11.down_proj", "model.layers.39.mlp.experts.12.down_proj", "model.layers.39.mlp.experts.13.down_proj", "model.layers.39.mlp.experts.14.down_proj", "model.layers.39.mlp.experts.15.down_proj", "model.layers.39.mlp.experts.16.down_proj", "model.layers.39.mlp.experts.17.down_proj", "model.layers.39.mlp.experts.18.down_proj", "model.layers.39.mlp.experts.19.down_proj", "model.layers.39.mlp.experts.20.down_proj", "model.layers.39.mlp.experts.21.down_proj", "model.layers.39.mlp.experts.22.down_proj", "model.layers.39.mlp.experts.23.down_proj", "model.layers.39.mlp.experts.24.down_proj", "model.layers.39.mlp.experts.25.down_proj", "model.layers.39.mlp.experts.26.down_proj", "model.layers.39.mlp.experts.27.down_proj", "model.layers.39.mlp.experts.28.down_proj", "model.layers.39.mlp.experts.29.down_proj", "model.layers.39.mlp.experts.30.down_proj", "model.layers.39.mlp.experts.31.down_proj", "model.layers.39.mlp.experts.32.down_proj", "model.layers.39.mlp.experts.33.down_proj", "model.layers.39.mlp.experts.34.down_proj", "model.layers.39.mlp.experts.35.down_proj", "model.layers.39.mlp.experts.36.down_proj", "model.layers.39.mlp.experts.37.down_proj", "model.layers.39.mlp.experts.38.down_proj", "model.layers.39.mlp.experts.39.down_proj", "model.layers.39.mlp.experts.40.down_proj", "model.layers.39.mlp.experts.41.down_proj", "model.layers.39.mlp.experts.42.down_proj", "model.layers.39.mlp.experts.43.down_proj", "model.layers.39.mlp.experts.44.down_proj", "model.layers.39.mlp.experts.45.down_proj", "model.layers.39.mlp.experts.46.down_proj", "model.layers.39.mlp.experts.47.down_proj", "model.layers.39.mlp.experts.48.down_proj", "model.layers.39.mlp.experts.49.down_proj", "model.layers.39.mlp.experts.50.down_proj", "model.layers.39.mlp.experts.51.down_proj", "model.layers.39.mlp.experts.52.down_proj", "model.layers.39.mlp.experts.53.down_proj", "model.layers.39.mlp.experts.54.down_proj", "model.layers.39.mlp.experts.55.down_proj", "model.layers.39.mlp.experts.56.down_proj", "model.layers.39.mlp.experts.57.down_proj", "model.layers.39.mlp.experts.58.down_proj", "model.layers.39.mlp.experts.59.down_proj", "model.layers.39.mlp.experts.60.down_proj", "model.layers.39.mlp.experts.61.down_proj", "model.layers.39.mlp.experts.62.down_proj", "model.layers.39.mlp.experts.63.down_proj", "model.layers.39.mlp.experts.64.down_proj", "model.layers.39.mlp.experts.65.down_proj", "model.layers.39.mlp.experts.66.down_proj", "model.layers.39.mlp.experts.67.down_proj", "model.layers.39.mlp.experts.68.down_proj", "model.layers.39.mlp.experts.69.down_proj", "model.layers.39.mlp.experts.70.down_proj", "model.layers.39.mlp.experts.71.down_proj", "model.layers.39.mlp.experts.72.down_proj", "model.layers.39.mlp.experts.73.down_proj", "model.layers.39.mlp.experts.74.down_proj", "model.layers.39.mlp.experts.75.down_proj", "model.layers.39.mlp.experts.76.down_proj", "model.layers.39.mlp.experts.77.down_proj", "model.layers.39.mlp.experts.78.down_proj", "model.layers.39.mlp.experts.79.down_proj", "model.layers.39.mlp.experts.80.down_proj", "model.layers.39.mlp.experts.81.down_proj", "model.layers.39.mlp.experts.82.down_proj", "model.layers.39.mlp.experts.83.down_proj", "model.layers.39.mlp.experts.84.down_proj", "model.layers.39.mlp.experts.85.down_proj", "model.layers.39.mlp.experts.86.down_proj", "model.layers.39.mlp.experts.87.down_proj", "model.layers.39.mlp.experts.88.down_proj", "model.layers.39.mlp.experts.89.down_proj", "model.layers.39.mlp.experts.90.down_proj", "model.layers.39.mlp.experts.91.down_proj", "model.layers.39.mlp.experts.92.down_proj", "model.layers.39.mlp.experts.93.down_proj", "model.layers.39.mlp.experts.94.down_proj", "model.layers.39.mlp.experts.95.down_proj", "model.layers.39.mlp.experts.96.down_proj", "model.layers.39.mlp.experts.97.down_proj", "model.layers.39.mlp.experts.98.down_proj", "model.layers.39.mlp.experts.99.down_proj", "model.layers.39.mlp.experts.100.down_proj", "model.layers.39.mlp.experts.101.down_proj", "model.layers.39.mlp.experts.102.down_proj", "model.layers.39.mlp.experts.103.down_proj", "model.layers.39.mlp.experts.104.down_proj", "model.layers.39.mlp.experts.105.down_proj", "model.layers.39.mlp.experts.106.down_proj", "model.layers.39.mlp.experts.107.down_proj", "model.layers.39.mlp.experts.108.down_proj", "model.layers.39.mlp.experts.109.down_proj", "model.layers.39.mlp.experts.110.down_proj", "model.layers.39.mlp.experts.111.down_proj", "model.layers.39.mlp.experts.112.down_proj", "model.layers.39.mlp.experts.113.down_proj", "model.layers.39.mlp.experts.114.down_proj", "model.layers.39.mlp.experts.115.down_proj", "model.layers.39.mlp.experts.116.down_proj", "model.layers.39.mlp.experts.117.down_proj", "model.layers.39.mlp.experts.118.down_proj", "model.layers.39.mlp.experts.119.down_proj", "model.layers.39.mlp.experts.120.down_proj", "model.layers.39.mlp.experts.121.down_proj", "model.layers.39.mlp.experts.122.down_proj", "model.layers.39.mlp.experts.123.down_proj", "model.layers.39.mlp.experts.124.down_proj", "model.layers.39.mlp.experts.125.down_proj", "model.layers.39.mlp.experts.126.down_proj", "model.layers.39.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0006150427274405901, "dbits": 2214592512 } ] }, { "idx": 119, "layers": [ "model.layers.40.self_attn.q_proj", "model.layers.40.self_attn.k_proj", "model.layers.40.self_attn.v_proj", "model.layers.40.self_attn.o_proj" ], "candidates": [ { "dkld": 2.3698247969161645e-05, "dbits": 109051904 } ] }, { "idx": 120, "layers": [ "model.layers.40.mlp.shared_experts.gate_proj", "model.layers.40.mlp.shared_experts.up_proj", "model.layers.40.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -6.519276648758732e-05, "dbits": 17301504 } ] }, { "idx": 121, "layers": [ "model.layers.40.mlp.experts.0.gate_proj", "model.layers.40.mlp.experts.1.gate_proj", "model.layers.40.mlp.experts.2.gate_proj", "model.layers.40.mlp.experts.3.gate_proj", "model.layers.40.mlp.experts.4.gate_proj", "model.layers.40.mlp.experts.5.gate_proj", "model.layers.40.mlp.experts.6.gate_proj", "model.layers.40.mlp.experts.7.gate_proj", "model.layers.40.mlp.experts.8.gate_proj", "model.layers.40.mlp.experts.9.gate_proj", "model.layers.40.mlp.experts.10.gate_proj", "model.layers.40.mlp.experts.11.gate_proj", "model.layers.40.mlp.experts.12.gate_proj", "model.layers.40.mlp.experts.13.gate_proj", "model.layers.40.mlp.experts.14.gate_proj", "model.layers.40.mlp.experts.15.gate_proj", "model.layers.40.mlp.experts.16.gate_proj", "model.layers.40.mlp.experts.17.gate_proj", "model.layers.40.mlp.experts.18.gate_proj", "model.layers.40.mlp.experts.19.gate_proj", "model.layers.40.mlp.experts.20.gate_proj", "model.layers.40.mlp.experts.21.gate_proj", "model.layers.40.mlp.experts.22.gate_proj", "model.layers.40.mlp.experts.23.gate_proj", "model.layers.40.mlp.experts.24.gate_proj", "model.layers.40.mlp.experts.25.gate_proj", "model.layers.40.mlp.experts.26.gate_proj", "model.layers.40.mlp.experts.27.gate_proj", "model.layers.40.mlp.experts.28.gate_proj", "model.layers.40.mlp.experts.29.gate_proj", "model.layers.40.mlp.experts.30.gate_proj", "model.layers.40.mlp.experts.31.gate_proj", "model.layers.40.mlp.experts.32.gate_proj", "model.layers.40.mlp.experts.33.gate_proj", "model.layers.40.mlp.experts.34.gate_proj", "model.layers.40.mlp.experts.35.gate_proj", "model.layers.40.mlp.experts.36.gate_proj", "model.layers.40.mlp.experts.37.gate_proj", "model.layers.40.mlp.experts.38.gate_proj", "model.layers.40.mlp.experts.39.gate_proj", "model.layers.40.mlp.experts.40.gate_proj", "model.layers.40.mlp.experts.41.gate_proj", "model.layers.40.mlp.experts.42.gate_proj", "model.layers.40.mlp.experts.43.gate_proj", "model.layers.40.mlp.experts.44.gate_proj", "model.layers.40.mlp.experts.45.gate_proj", "model.layers.40.mlp.experts.46.gate_proj", "model.layers.40.mlp.experts.47.gate_proj", "model.layers.40.mlp.experts.48.gate_proj", "model.layers.40.mlp.experts.49.gate_proj", "model.layers.40.mlp.experts.50.gate_proj", "model.layers.40.mlp.experts.51.gate_proj", "model.layers.40.mlp.experts.52.gate_proj", "model.layers.40.mlp.experts.53.gate_proj", "model.layers.40.mlp.experts.54.gate_proj", "model.layers.40.mlp.experts.55.gate_proj", "model.layers.40.mlp.experts.56.gate_proj", "model.layers.40.mlp.experts.57.gate_proj", "model.layers.40.mlp.experts.58.gate_proj", "model.layers.40.mlp.experts.59.gate_proj", "model.layers.40.mlp.experts.60.gate_proj", "model.layers.40.mlp.experts.61.gate_proj", "model.layers.40.mlp.experts.62.gate_proj", "model.layers.40.mlp.experts.63.gate_proj", "model.layers.40.mlp.experts.64.gate_proj", "model.layers.40.mlp.experts.65.gate_proj", "model.layers.40.mlp.experts.66.gate_proj", "model.layers.40.mlp.experts.67.gate_proj", "model.layers.40.mlp.experts.68.gate_proj", "model.layers.40.mlp.experts.69.gate_proj", "model.layers.40.mlp.experts.70.gate_proj", "model.layers.40.mlp.experts.71.gate_proj", "model.layers.40.mlp.experts.72.gate_proj", "model.layers.40.mlp.experts.73.gate_proj", "model.layers.40.mlp.experts.74.gate_proj", "model.layers.40.mlp.experts.75.gate_proj", "model.layers.40.mlp.experts.76.gate_proj", "model.layers.40.mlp.experts.77.gate_proj", "model.layers.40.mlp.experts.78.gate_proj", "model.layers.40.mlp.experts.79.gate_proj", "model.layers.40.mlp.experts.80.gate_proj", "model.layers.40.mlp.experts.81.gate_proj", "model.layers.40.mlp.experts.82.gate_proj", "model.layers.40.mlp.experts.83.gate_proj", "model.layers.40.mlp.experts.84.gate_proj", "model.layers.40.mlp.experts.85.gate_proj", "model.layers.40.mlp.experts.86.gate_proj", "model.layers.40.mlp.experts.87.gate_proj", "model.layers.40.mlp.experts.88.gate_proj", "model.layers.40.mlp.experts.89.gate_proj", "model.layers.40.mlp.experts.90.gate_proj", "model.layers.40.mlp.experts.91.gate_proj", "model.layers.40.mlp.experts.92.gate_proj", "model.layers.40.mlp.experts.93.gate_proj", "model.layers.40.mlp.experts.94.gate_proj", "model.layers.40.mlp.experts.95.gate_proj", "model.layers.40.mlp.experts.96.gate_proj", "model.layers.40.mlp.experts.97.gate_proj", "model.layers.40.mlp.experts.98.gate_proj", "model.layers.40.mlp.experts.99.gate_proj", "model.layers.40.mlp.experts.100.gate_proj", "model.layers.40.mlp.experts.101.gate_proj", "model.layers.40.mlp.experts.102.gate_proj", "model.layers.40.mlp.experts.103.gate_proj", "model.layers.40.mlp.experts.104.gate_proj", "model.layers.40.mlp.experts.105.gate_proj", "model.layers.40.mlp.experts.106.gate_proj", "model.layers.40.mlp.experts.107.gate_proj", "model.layers.40.mlp.experts.108.gate_proj", "model.layers.40.mlp.experts.109.gate_proj", "model.layers.40.mlp.experts.110.gate_proj", "model.layers.40.mlp.experts.111.gate_proj", "model.layers.40.mlp.experts.112.gate_proj", "model.layers.40.mlp.experts.113.gate_proj", "model.layers.40.mlp.experts.114.gate_proj", "model.layers.40.mlp.experts.115.gate_proj", "model.layers.40.mlp.experts.116.gate_proj", "model.layers.40.mlp.experts.117.gate_proj", "model.layers.40.mlp.experts.118.gate_proj", "model.layers.40.mlp.experts.119.gate_proj", "model.layers.40.mlp.experts.120.gate_proj", "model.layers.40.mlp.experts.121.gate_proj", "model.layers.40.mlp.experts.122.gate_proj", "model.layers.40.mlp.experts.123.gate_proj", "model.layers.40.mlp.experts.124.gate_proj", "model.layers.40.mlp.experts.125.gate_proj", "model.layers.40.mlp.experts.126.gate_proj", "model.layers.40.mlp.experts.127.gate_proj", "model.layers.40.mlp.experts.0.up_proj", "model.layers.40.mlp.experts.1.up_proj", "model.layers.40.mlp.experts.2.up_proj", "model.layers.40.mlp.experts.3.up_proj", "model.layers.40.mlp.experts.4.up_proj", "model.layers.40.mlp.experts.5.up_proj", "model.layers.40.mlp.experts.6.up_proj", "model.layers.40.mlp.experts.7.up_proj", "model.layers.40.mlp.experts.8.up_proj", "model.layers.40.mlp.experts.9.up_proj", "model.layers.40.mlp.experts.10.up_proj", "model.layers.40.mlp.experts.11.up_proj", "model.layers.40.mlp.experts.12.up_proj", "model.layers.40.mlp.experts.13.up_proj", "model.layers.40.mlp.experts.14.up_proj", "model.layers.40.mlp.experts.15.up_proj", "model.layers.40.mlp.experts.16.up_proj", "model.layers.40.mlp.experts.17.up_proj", "model.layers.40.mlp.experts.18.up_proj", "model.layers.40.mlp.experts.19.up_proj", "model.layers.40.mlp.experts.20.up_proj", "model.layers.40.mlp.experts.21.up_proj", "model.layers.40.mlp.experts.22.up_proj", "model.layers.40.mlp.experts.23.up_proj", "model.layers.40.mlp.experts.24.up_proj", "model.layers.40.mlp.experts.25.up_proj", "model.layers.40.mlp.experts.26.up_proj", "model.layers.40.mlp.experts.27.up_proj", "model.layers.40.mlp.experts.28.up_proj", "model.layers.40.mlp.experts.29.up_proj", "model.layers.40.mlp.experts.30.up_proj", "model.layers.40.mlp.experts.31.up_proj", "model.layers.40.mlp.experts.32.up_proj", "model.layers.40.mlp.experts.33.up_proj", "model.layers.40.mlp.experts.34.up_proj", "model.layers.40.mlp.experts.35.up_proj", "model.layers.40.mlp.experts.36.up_proj", "model.layers.40.mlp.experts.37.up_proj", "model.layers.40.mlp.experts.38.up_proj", "model.layers.40.mlp.experts.39.up_proj", "model.layers.40.mlp.experts.40.up_proj", "model.layers.40.mlp.experts.41.up_proj", "model.layers.40.mlp.experts.42.up_proj", "model.layers.40.mlp.experts.43.up_proj", "model.layers.40.mlp.experts.44.up_proj", "model.layers.40.mlp.experts.45.up_proj", "model.layers.40.mlp.experts.46.up_proj", "model.layers.40.mlp.experts.47.up_proj", "model.layers.40.mlp.experts.48.up_proj", "model.layers.40.mlp.experts.49.up_proj", "model.layers.40.mlp.experts.50.up_proj", "model.layers.40.mlp.experts.51.up_proj", "model.layers.40.mlp.experts.52.up_proj", "model.layers.40.mlp.experts.53.up_proj", "model.layers.40.mlp.experts.54.up_proj", "model.layers.40.mlp.experts.55.up_proj", "model.layers.40.mlp.experts.56.up_proj", "model.layers.40.mlp.experts.57.up_proj", "model.layers.40.mlp.experts.58.up_proj", "model.layers.40.mlp.experts.59.up_proj", "model.layers.40.mlp.experts.60.up_proj", "model.layers.40.mlp.experts.61.up_proj", "model.layers.40.mlp.experts.62.up_proj", "model.layers.40.mlp.experts.63.up_proj", "model.layers.40.mlp.experts.64.up_proj", "model.layers.40.mlp.experts.65.up_proj", "model.layers.40.mlp.experts.66.up_proj", "model.layers.40.mlp.experts.67.up_proj", "model.layers.40.mlp.experts.68.up_proj", "model.layers.40.mlp.experts.69.up_proj", "model.layers.40.mlp.experts.70.up_proj", "model.layers.40.mlp.experts.71.up_proj", "model.layers.40.mlp.experts.72.up_proj", "model.layers.40.mlp.experts.73.up_proj", "model.layers.40.mlp.experts.74.up_proj", "model.layers.40.mlp.experts.75.up_proj", "model.layers.40.mlp.experts.76.up_proj", "model.layers.40.mlp.experts.77.up_proj", "model.layers.40.mlp.experts.78.up_proj", "model.layers.40.mlp.experts.79.up_proj", "model.layers.40.mlp.experts.80.up_proj", "model.layers.40.mlp.experts.81.up_proj", "model.layers.40.mlp.experts.82.up_proj", "model.layers.40.mlp.experts.83.up_proj", "model.layers.40.mlp.experts.84.up_proj", "model.layers.40.mlp.experts.85.up_proj", "model.layers.40.mlp.experts.86.up_proj", "model.layers.40.mlp.experts.87.up_proj", "model.layers.40.mlp.experts.88.up_proj", "model.layers.40.mlp.experts.89.up_proj", "model.layers.40.mlp.experts.90.up_proj", "model.layers.40.mlp.experts.91.up_proj", "model.layers.40.mlp.experts.92.up_proj", "model.layers.40.mlp.experts.93.up_proj", "model.layers.40.mlp.experts.94.up_proj", "model.layers.40.mlp.experts.95.up_proj", "model.layers.40.mlp.experts.96.up_proj", "model.layers.40.mlp.experts.97.up_proj", "model.layers.40.mlp.experts.98.up_proj", "model.layers.40.mlp.experts.99.up_proj", "model.layers.40.mlp.experts.100.up_proj", "model.layers.40.mlp.experts.101.up_proj", "model.layers.40.mlp.experts.102.up_proj", "model.layers.40.mlp.experts.103.up_proj", "model.layers.40.mlp.experts.104.up_proj", "model.layers.40.mlp.experts.105.up_proj", "model.layers.40.mlp.experts.106.up_proj", "model.layers.40.mlp.experts.107.up_proj", "model.layers.40.mlp.experts.108.up_proj", "model.layers.40.mlp.experts.109.up_proj", "model.layers.40.mlp.experts.110.up_proj", "model.layers.40.mlp.experts.111.up_proj", "model.layers.40.mlp.experts.112.up_proj", "model.layers.40.mlp.experts.113.up_proj", "model.layers.40.mlp.experts.114.up_proj", "model.layers.40.mlp.experts.115.up_proj", "model.layers.40.mlp.experts.116.up_proj", "model.layers.40.mlp.experts.117.up_proj", "model.layers.40.mlp.experts.118.up_proj", "model.layers.40.mlp.experts.119.up_proj", "model.layers.40.mlp.experts.120.up_proj", "model.layers.40.mlp.experts.121.up_proj", "model.layers.40.mlp.experts.122.up_proj", "model.layers.40.mlp.experts.123.up_proj", "model.layers.40.mlp.experts.124.up_proj", "model.layers.40.mlp.experts.125.up_proj", "model.layers.40.mlp.experts.126.up_proj", "model.layers.40.mlp.experts.127.up_proj", "model.layers.40.mlp.experts.0.down_proj", "model.layers.40.mlp.experts.1.down_proj", "model.layers.40.mlp.experts.2.down_proj", "model.layers.40.mlp.experts.3.down_proj", "model.layers.40.mlp.experts.4.down_proj", "model.layers.40.mlp.experts.5.down_proj", "model.layers.40.mlp.experts.6.down_proj", "model.layers.40.mlp.experts.7.down_proj", "model.layers.40.mlp.experts.8.down_proj", "model.layers.40.mlp.experts.9.down_proj", "model.layers.40.mlp.experts.10.down_proj", "model.layers.40.mlp.experts.11.down_proj", "model.layers.40.mlp.experts.12.down_proj", "model.layers.40.mlp.experts.13.down_proj", "model.layers.40.mlp.experts.14.down_proj", "model.layers.40.mlp.experts.15.down_proj", "model.layers.40.mlp.experts.16.down_proj", "model.layers.40.mlp.experts.17.down_proj", "model.layers.40.mlp.experts.18.down_proj", "model.layers.40.mlp.experts.19.down_proj", "model.layers.40.mlp.experts.20.down_proj", "model.layers.40.mlp.experts.21.down_proj", "model.layers.40.mlp.experts.22.down_proj", "model.layers.40.mlp.experts.23.down_proj", "model.layers.40.mlp.experts.24.down_proj", "model.layers.40.mlp.experts.25.down_proj", "model.layers.40.mlp.experts.26.down_proj", "model.layers.40.mlp.experts.27.down_proj", "model.layers.40.mlp.experts.28.down_proj", "model.layers.40.mlp.experts.29.down_proj", "model.layers.40.mlp.experts.30.down_proj", "model.layers.40.mlp.experts.31.down_proj", "model.layers.40.mlp.experts.32.down_proj", "model.layers.40.mlp.experts.33.down_proj", "model.layers.40.mlp.experts.34.down_proj", "model.layers.40.mlp.experts.35.down_proj", "model.layers.40.mlp.experts.36.down_proj", "model.layers.40.mlp.experts.37.down_proj", "model.layers.40.mlp.experts.38.down_proj", "model.layers.40.mlp.experts.39.down_proj", "model.layers.40.mlp.experts.40.down_proj", "model.layers.40.mlp.experts.41.down_proj", "model.layers.40.mlp.experts.42.down_proj", "model.layers.40.mlp.experts.43.down_proj", "model.layers.40.mlp.experts.44.down_proj", "model.layers.40.mlp.experts.45.down_proj", "model.layers.40.mlp.experts.46.down_proj", "model.layers.40.mlp.experts.47.down_proj", "model.layers.40.mlp.experts.48.down_proj", "model.layers.40.mlp.experts.49.down_proj", "model.layers.40.mlp.experts.50.down_proj", "model.layers.40.mlp.experts.51.down_proj", "model.layers.40.mlp.experts.52.down_proj", "model.layers.40.mlp.experts.53.down_proj", "model.layers.40.mlp.experts.54.down_proj", "model.layers.40.mlp.experts.55.down_proj", "model.layers.40.mlp.experts.56.down_proj", "model.layers.40.mlp.experts.57.down_proj", "model.layers.40.mlp.experts.58.down_proj", "model.layers.40.mlp.experts.59.down_proj", "model.layers.40.mlp.experts.60.down_proj", "model.layers.40.mlp.experts.61.down_proj", "model.layers.40.mlp.experts.62.down_proj", "model.layers.40.mlp.experts.63.down_proj", "model.layers.40.mlp.experts.64.down_proj", "model.layers.40.mlp.experts.65.down_proj", "model.layers.40.mlp.experts.66.down_proj", "model.layers.40.mlp.experts.67.down_proj", "model.layers.40.mlp.experts.68.down_proj", "model.layers.40.mlp.experts.69.down_proj", "model.layers.40.mlp.experts.70.down_proj", "model.layers.40.mlp.experts.71.down_proj", "model.layers.40.mlp.experts.72.down_proj", "model.layers.40.mlp.experts.73.down_proj", "model.layers.40.mlp.experts.74.down_proj", "model.layers.40.mlp.experts.75.down_proj", "model.layers.40.mlp.experts.76.down_proj", "model.layers.40.mlp.experts.77.down_proj", "model.layers.40.mlp.experts.78.down_proj", "model.layers.40.mlp.experts.79.down_proj", "model.layers.40.mlp.experts.80.down_proj", "model.layers.40.mlp.experts.81.down_proj", "model.layers.40.mlp.experts.82.down_proj", "model.layers.40.mlp.experts.83.down_proj", "model.layers.40.mlp.experts.84.down_proj", "model.layers.40.mlp.experts.85.down_proj", "model.layers.40.mlp.experts.86.down_proj", "model.layers.40.mlp.experts.87.down_proj", "model.layers.40.mlp.experts.88.down_proj", "model.layers.40.mlp.experts.89.down_proj", "model.layers.40.mlp.experts.90.down_proj", "model.layers.40.mlp.experts.91.down_proj", "model.layers.40.mlp.experts.92.down_proj", "model.layers.40.mlp.experts.93.down_proj", "model.layers.40.mlp.experts.94.down_proj", "model.layers.40.mlp.experts.95.down_proj", "model.layers.40.mlp.experts.96.down_proj", "model.layers.40.mlp.experts.97.down_proj", "model.layers.40.mlp.experts.98.down_proj", "model.layers.40.mlp.experts.99.down_proj", "model.layers.40.mlp.experts.100.down_proj", "model.layers.40.mlp.experts.101.down_proj", "model.layers.40.mlp.experts.102.down_proj", "model.layers.40.mlp.experts.103.down_proj", "model.layers.40.mlp.experts.104.down_proj", "model.layers.40.mlp.experts.105.down_proj", "model.layers.40.mlp.experts.106.down_proj", "model.layers.40.mlp.experts.107.down_proj", "model.layers.40.mlp.experts.108.down_proj", "model.layers.40.mlp.experts.109.down_proj", "model.layers.40.mlp.experts.110.down_proj", "model.layers.40.mlp.experts.111.down_proj", "model.layers.40.mlp.experts.112.down_proj", "model.layers.40.mlp.experts.113.down_proj", "model.layers.40.mlp.experts.114.down_proj", "model.layers.40.mlp.experts.115.down_proj", "model.layers.40.mlp.experts.116.down_proj", "model.layers.40.mlp.experts.117.down_proj", "model.layers.40.mlp.experts.118.down_proj", "model.layers.40.mlp.experts.119.down_proj", "model.layers.40.mlp.experts.120.down_proj", "model.layers.40.mlp.experts.121.down_proj", "model.layers.40.mlp.experts.122.down_proj", "model.layers.40.mlp.experts.123.down_proj", "model.layers.40.mlp.experts.124.down_proj", "model.layers.40.mlp.experts.125.down_proj", "model.layers.40.mlp.experts.126.down_proj", "model.layers.40.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0003608790226280745, "dbits": 2214592512 } ] }, { "idx": 122, "layers": [ "model.layers.41.self_attn.q_proj", "model.layers.41.self_attn.k_proj", "model.layers.41.self_attn.v_proj", "model.layers.41.self_attn.o_proj" ], "candidates": [ { "dkld": -0.00015305979177354656, "dbits": 109051904 } ] }, { "idx": 123, "layers": [ "model.layers.41.mlp.shared_experts.gate_proj", "model.layers.41.mlp.shared_experts.up_proj", "model.layers.41.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0005174450576305445, "dbits": 17301504 } ] }, { "idx": 124, "layers": [ "model.layers.41.mlp.experts.0.gate_proj", "model.layers.41.mlp.experts.1.gate_proj", "model.layers.41.mlp.experts.2.gate_proj", "model.layers.41.mlp.experts.3.gate_proj", "model.layers.41.mlp.experts.4.gate_proj", "model.layers.41.mlp.experts.5.gate_proj", "model.layers.41.mlp.experts.6.gate_proj", "model.layers.41.mlp.experts.7.gate_proj", "model.layers.41.mlp.experts.8.gate_proj", "model.layers.41.mlp.experts.9.gate_proj", "model.layers.41.mlp.experts.10.gate_proj", "model.layers.41.mlp.experts.11.gate_proj", "model.layers.41.mlp.experts.12.gate_proj", "model.layers.41.mlp.experts.13.gate_proj", "model.layers.41.mlp.experts.14.gate_proj", "model.layers.41.mlp.experts.15.gate_proj", "model.layers.41.mlp.experts.16.gate_proj", "model.layers.41.mlp.experts.17.gate_proj", "model.layers.41.mlp.experts.18.gate_proj", "model.layers.41.mlp.experts.19.gate_proj", "model.layers.41.mlp.experts.20.gate_proj", "model.layers.41.mlp.experts.21.gate_proj", "model.layers.41.mlp.experts.22.gate_proj", "model.layers.41.mlp.experts.23.gate_proj", "model.layers.41.mlp.experts.24.gate_proj", "model.layers.41.mlp.experts.25.gate_proj", "model.layers.41.mlp.experts.26.gate_proj", "model.layers.41.mlp.experts.27.gate_proj", "model.layers.41.mlp.experts.28.gate_proj", "model.layers.41.mlp.experts.29.gate_proj", "model.layers.41.mlp.experts.30.gate_proj", "model.layers.41.mlp.experts.31.gate_proj", "model.layers.41.mlp.experts.32.gate_proj", "model.layers.41.mlp.experts.33.gate_proj", "model.layers.41.mlp.experts.34.gate_proj", "model.layers.41.mlp.experts.35.gate_proj", "model.layers.41.mlp.experts.36.gate_proj", "model.layers.41.mlp.experts.37.gate_proj", "model.layers.41.mlp.experts.38.gate_proj", "model.layers.41.mlp.experts.39.gate_proj", "model.layers.41.mlp.experts.40.gate_proj", "model.layers.41.mlp.experts.41.gate_proj", "model.layers.41.mlp.experts.42.gate_proj", "model.layers.41.mlp.experts.43.gate_proj", "model.layers.41.mlp.experts.44.gate_proj", "model.layers.41.mlp.experts.45.gate_proj", "model.layers.41.mlp.experts.46.gate_proj", "model.layers.41.mlp.experts.47.gate_proj", "model.layers.41.mlp.experts.48.gate_proj", "model.layers.41.mlp.experts.49.gate_proj", "model.layers.41.mlp.experts.50.gate_proj", "model.layers.41.mlp.experts.51.gate_proj", "model.layers.41.mlp.experts.52.gate_proj", "model.layers.41.mlp.experts.53.gate_proj", "model.layers.41.mlp.experts.54.gate_proj", "model.layers.41.mlp.experts.55.gate_proj", "model.layers.41.mlp.experts.56.gate_proj", "model.layers.41.mlp.experts.57.gate_proj", "model.layers.41.mlp.experts.58.gate_proj", "model.layers.41.mlp.experts.59.gate_proj", "model.layers.41.mlp.experts.60.gate_proj", "model.layers.41.mlp.experts.61.gate_proj", "model.layers.41.mlp.experts.62.gate_proj", "model.layers.41.mlp.experts.63.gate_proj", "model.layers.41.mlp.experts.64.gate_proj", "model.layers.41.mlp.experts.65.gate_proj", "model.layers.41.mlp.experts.66.gate_proj", "model.layers.41.mlp.experts.67.gate_proj", "model.layers.41.mlp.experts.68.gate_proj", "model.layers.41.mlp.experts.69.gate_proj", "model.layers.41.mlp.experts.70.gate_proj", "model.layers.41.mlp.experts.71.gate_proj", "model.layers.41.mlp.experts.72.gate_proj", "model.layers.41.mlp.experts.73.gate_proj", "model.layers.41.mlp.experts.74.gate_proj", "model.layers.41.mlp.experts.75.gate_proj", "model.layers.41.mlp.experts.76.gate_proj", "model.layers.41.mlp.experts.77.gate_proj", "model.layers.41.mlp.experts.78.gate_proj", "model.layers.41.mlp.experts.79.gate_proj", "model.layers.41.mlp.experts.80.gate_proj", "model.layers.41.mlp.experts.81.gate_proj", "model.layers.41.mlp.experts.82.gate_proj", "model.layers.41.mlp.experts.83.gate_proj", "model.layers.41.mlp.experts.84.gate_proj", "model.layers.41.mlp.experts.85.gate_proj", "model.layers.41.mlp.experts.86.gate_proj", "model.layers.41.mlp.experts.87.gate_proj", "model.layers.41.mlp.experts.88.gate_proj", "model.layers.41.mlp.experts.89.gate_proj", "model.layers.41.mlp.experts.90.gate_proj", "model.layers.41.mlp.experts.91.gate_proj", "model.layers.41.mlp.experts.92.gate_proj", "model.layers.41.mlp.experts.93.gate_proj", "model.layers.41.mlp.experts.94.gate_proj", "model.layers.41.mlp.experts.95.gate_proj", "model.layers.41.mlp.experts.96.gate_proj", "model.layers.41.mlp.experts.97.gate_proj", "model.layers.41.mlp.experts.98.gate_proj", "model.layers.41.mlp.experts.99.gate_proj", "model.layers.41.mlp.experts.100.gate_proj", "model.layers.41.mlp.experts.101.gate_proj", "model.layers.41.mlp.experts.102.gate_proj", "model.layers.41.mlp.experts.103.gate_proj", "model.layers.41.mlp.experts.104.gate_proj", "model.layers.41.mlp.experts.105.gate_proj", "model.layers.41.mlp.experts.106.gate_proj", "model.layers.41.mlp.experts.107.gate_proj", "model.layers.41.mlp.experts.108.gate_proj", "model.layers.41.mlp.experts.109.gate_proj", "model.layers.41.mlp.experts.110.gate_proj", "model.layers.41.mlp.experts.111.gate_proj", "model.layers.41.mlp.experts.112.gate_proj", "model.layers.41.mlp.experts.113.gate_proj", "model.layers.41.mlp.experts.114.gate_proj", "model.layers.41.mlp.experts.115.gate_proj", "model.layers.41.mlp.experts.116.gate_proj", "model.layers.41.mlp.experts.117.gate_proj", "model.layers.41.mlp.experts.118.gate_proj", "model.layers.41.mlp.experts.119.gate_proj", "model.layers.41.mlp.experts.120.gate_proj", "model.layers.41.mlp.experts.121.gate_proj", "model.layers.41.mlp.experts.122.gate_proj", "model.layers.41.mlp.experts.123.gate_proj", "model.layers.41.mlp.experts.124.gate_proj", "model.layers.41.mlp.experts.125.gate_proj", "model.layers.41.mlp.experts.126.gate_proj", "model.layers.41.mlp.experts.127.gate_proj", "model.layers.41.mlp.experts.0.up_proj", "model.layers.41.mlp.experts.1.up_proj", "model.layers.41.mlp.experts.2.up_proj", "model.layers.41.mlp.experts.3.up_proj", "model.layers.41.mlp.experts.4.up_proj", "model.layers.41.mlp.experts.5.up_proj", "model.layers.41.mlp.experts.6.up_proj", "model.layers.41.mlp.experts.7.up_proj", "model.layers.41.mlp.experts.8.up_proj", "model.layers.41.mlp.experts.9.up_proj", "model.layers.41.mlp.experts.10.up_proj", "model.layers.41.mlp.experts.11.up_proj", "model.layers.41.mlp.experts.12.up_proj", "model.layers.41.mlp.experts.13.up_proj", "model.layers.41.mlp.experts.14.up_proj", "model.layers.41.mlp.experts.15.up_proj", "model.layers.41.mlp.experts.16.up_proj", "model.layers.41.mlp.experts.17.up_proj", "model.layers.41.mlp.experts.18.up_proj", "model.layers.41.mlp.experts.19.up_proj", "model.layers.41.mlp.experts.20.up_proj", "model.layers.41.mlp.experts.21.up_proj", "model.layers.41.mlp.experts.22.up_proj", "model.layers.41.mlp.experts.23.up_proj", "model.layers.41.mlp.experts.24.up_proj", "model.layers.41.mlp.experts.25.up_proj", "model.layers.41.mlp.experts.26.up_proj", "model.layers.41.mlp.experts.27.up_proj", "model.layers.41.mlp.experts.28.up_proj", "model.layers.41.mlp.experts.29.up_proj", "model.layers.41.mlp.experts.30.up_proj", "model.layers.41.mlp.experts.31.up_proj", "model.layers.41.mlp.experts.32.up_proj", "model.layers.41.mlp.experts.33.up_proj", "model.layers.41.mlp.experts.34.up_proj", "model.layers.41.mlp.experts.35.up_proj", "model.layers.41.mlp.experts.36.up_proj", "model.layers.41.mlp.experts.37.up_proj", "model.layers.41.mlp.experts.38.up_proj", "model.layers.41.mlp.experts.39.up_proj", "model.layers.41.mlp.experts.40.up_proj", "model.layers.41.mlp.experts.41.up_proj", "model.layers.41.mlp.experts.42.up_proj", "model.layers.41.mlp.experts.43.up_proj", "model.layers.41.mlp.experts.44.up_proj", "model.layers.41.mlp.experts.45.up_proj", "model.layers.41.mlp.experts.46.up_proj", "model.layers.41.mlp.experts.47.up_proj", "model.layers.41.mlp.experts.48.up_proj", "model.layers.41.mlp.experts.49.up_proj", "model.layers.41.mlp.experts.50.up_proj", "model.layers.41.mlp.experts.51.up_proj", "model.layers.41.mlp.experts.52.up_proj", "model.layers.41.mlp.experts.53.up_proj", "model.layers.41.mlp.experts.54.up_proj", "model.layers.41.mlp.experts.55.up_proj", "model.layers.41.mlp.experts.56.up_proj", "model.layers.41.mlp.experts.57.up_proj", "model.layers.41.mlp.experts.58.up_proj", "model.layers.41.mlp.experts.59.up_proj", "model.layers.41.mlp.experts.60.up_proj", "model.layers.41.mlp.experts.61.up_proj", "model.layers.41.mlp.experts.62.up_proj", "model.layers.41.mlp.experts.63.up_proj", "model.layers.41.mlp.experts.64.up_proj", "model.layers.41.mlp.experts.65.up_proj", "model.layers.41.mlp.experts.66.up_proj", "model.layers.41.mlp.experts.67.up_proj", "model.layers.41.mlp.experts.68.up_proj", "model.layers.41.mlp.experts.69.up_proj", "model.layers.41.mlp.experts.70.up_proj", "model.layers.41.mlp.experts.71.up_proj", "model.layers.41.mlp.experts.72.up_proj", "model.layers.41.mlp.experts.73.up_proj", "model.layers.41.mlp.experts.74.up_proj", "model.layers.41.mlp.experts.75.up_proj", "model.layers.41.mlp.experts.76.up_proj", "model.layers.41.mlp.experts.77.up_proj", "model.layers.41.mlp.experts.78.up_proj", "model.layers.41.mlp.experts.79.up_proj", "model.layers.41.mlp.experts.80.up_proj", "model.layers.41.mlp.experts.81.up_proj", "model.layers.41.mlp.experts.82.up_proj", "model.layers.41.mlp.experts.83.up_proj", "model.layers.41.mlp.experts.84.up_proj", "model.layers.41.mlp.experts.85.up_proj", "model.layers.41.mlp.experts.86.up_proj", "model.layers.41.mlp.experts.87.up_proj", "model.layers.41.mlp.experts.88.up_proj", "model.layers.41.mlp.experts.89.up_proj", "model.layers.41.mlp.experts.90.up_proj", "model.layers.41.mlp.experts.91.up_proj", "model.layers.41.mlp.experts.92.up_proj", "model.layers.41.mlp.experts.93.up_proj", "model.layers.41.mlp.experts.94.up_proj", "model.layers.41.mlp.experts.95.up_proj", "model.layers.41.mlp.experts.96.up_proj", "model.layers.41.mlp.experts.97.up_proj", "model.layers.41.mlp.experts.98.up_proj", "model.layers.41.mlp.experts.99.up_proj", "model.layers.41.mlp.experts.100.up_proj", "model.layers.41.mlp.experts.101.up_proj", "model.layers.41.mlp.experts.102.up_proj", "model.layers.41.mlp.experts.103.up_proj", "model.layers.41.mlp.experts.104.up_proj", "model.layers.41.mlp.experts.105.up_proj", "model.layers.41.mlp.experts.106.up_proj", "model.layers.41.mlp.experts.107.up_proj", "model.layers.41.mlp.experts.108.up_proj", "model.layers.41.mlp.experts.109.up_proj", "model.layers.41.mlp.experts.110.up_proj", "model.layers.41.mlp.experts.111.up_proj", "model.layers.41.mlp.experts.112.up_proj", "model.layers.41.mlp.experts.113.up_proj", "model.layers.41.mlp.experts.114.up_proj", "model.layers.41.mlp.experts.115.up_proj", "model.layers.41.mlp.experts.116.up_proj", "model.layers.41.mlp.experts.117.up_proj", "model.layers.41.mlp.experts.118.up_proj", "model.layers.41.mlp.experts.119.up_proj", "model.layers.41.mlp.experts.120.up_proj", "model.layers.41.mlp.experts.121.up_proj", "model.layers.41.mlp.experts.122.up_proj", "model.layers.41.mlp.experts.123.up_proj", "model.layers.41.mlp.experts.124.up_proj", "model.layers.41.mlp.experts.125.up_proj", "model.layers.41.mlp.experts.126.up_proj", "model.layers.41.mlp.experts.127.up_proj", "model.layers.41.mlp.experts.0.down_proj", "model.layers.41.mlp.experts.1.down_proj", "model.layers.41.mlp.experts.2.down_proj", "model.layers.41.mlp.experts.3.down_proj", "model.layers.41.mlp.experts.4.down_proj", "model.layers.41.mlp.experts.5.down_proj", "model.layers.41.mlp.experts.6.down_proj", "model.layers.41.mlp.experts.7.down_proj", "model.layers.41.mlp.experts.8.down_proj", "model.layers.41.mlp.experts.9.down_proj", "model.layers.41.mlp.experts.10.down_proj", "model.layers.41.mlp.experts.11.down_proj", "model.layers.41.mlp.experts.12.down_proj", "model.layers.41.mlp.experts.13.down_proj", "model.layers.41.mlp.experts.14.down_proj", "model.layers.41.mlp.experts.15.down_proj", "model.layers.41.mlp.experts.16.down_proj", "model.layers.41.mlp.experts.17.down_proj", "model.layers.41.mlp.experts.18.down_proj", "model.layers.41.mlp.experts.19.down_proj", "model.layers.41.mlp.experts.20.down_proj", "model.layers.41.mlp.experts.21.down_proj", "model.layers.41.mlp.experts.22.down_proj", "model.layers.41.mlp.experts.23.down_proj", "model.layers.41.mlp.experts.24.down_proj", "model.layers.41.mlp.experts.25.down_proj", "model.layers.41.mlp.experts.26.down_proj", "model.layers.41.mlp.experts.27.down_proj", "model.layers.41.mlp.experts.28.down_proj", "model.layers.41.mlp.experts.29.down_proj", "model.layers.41.mlp.experts.30.down_proj", "model.layers.41.mlp.experts.31.down_proj", "model.layers.41.mlp.experts.32.down_proj", "model.layers.41.mlp.experts.33.down_proj", "model.layers.41.mlp.experts.34.down_proj", "model.layers.41.mlp.experts.35.down_proj", "model.layers.41.mlp.experts.36.down_proj", "model.layers.41.mlp.experts.37.down_proj", "model.layers.41.mlp.experts.38.down_proj", "model.layers.41.mlp.experts.39.down_proj", "model.layers.41.mlp.experts.40.down_proj", "model.layers.41.mlp.experts.41.down_proj", "model.layers.41.mlp.experts.42.down_proj", "model.layers.41.mlp.experts.43.down_proj", "model.layers.41.mlp.experts.44.down_proj", "model.layers.41.mlp.experts.45.down_proj", "model.layers.41.mlp.experts.46.down_proj", "model.layers.41.mlp.experts.47.down_proj", "model.layers.41.mlp.experts.48.down_proj", "model.layers.41.mlp.experts.49.down_proj", "model.layers.41.mlp.experts.50.down_proj", "model.layers.41.mlp.experts.51.down_proj", "model.layers.41.mlp.experts.52.down_proj", "model.layers.41.mlp.experts.53.down_proj", "model.layers.41.mlp.experts.54.down_proj", "model.layers.41.mlp.experts.55.down_proj", "model.layers.41.mlp.experts.56.down_proj", "model.layers.41.mlp.experts.57.down_proj", "model.layers.41.mlp.experts.58.down_proj", "model.layers.41.mlp.experts.59.down_proj", "model.layers.41.mlp.experts.60.down_proj", "model.layers.41.mlp.experts.61.down_proj", "model.layers.41.mlp.experts.62.down_proj", "model.layers.41.mlp.experts.63.down_proj", "model.layers.41.mlp.experts.64.down_proj", "model.layers.41.mlp.experts.65.down_proj", "model.layers.41.mlp.experts.66.down_proj", "model.layers.41.mlp.experts.67.down_proj", "model.layers.41.mlp.experts.68.down_proj", "model.layers.41.mlp.experts.69.down_proj", "model.layers.41.mlp.experts.70.down_proj", "model.layers.41.mlp.experts.71.down_proj", "model.layers.41.mlp.experts.72.down_proj", "model.layers.41.mlp.experts.73.down_proj", "model.layers.41.mlp.experts.74.down_proj", "model.layers.41.mlp.experts.75.down_proj", "model.layers.41.mlp.experts.76.down_proj", "model.layers.41.mlp.experts.77.down_proj", "model.layers.41.mlp.experts.78.down_proj", "model.layers.41.mlp.experts.79.down_proj", "model.layers.41.mlp.experts.80.down_proj", "model.layers.41.mlp.experts.81.down_proj", "model.layers.41.mlp.experts.82.down_proj", "model.layers.41.mlp.experts.83.down_proj", "model.layers.41.mlp.experts.84.down_proj", "model.layers.41.mlp.experts.85.down_proj", "model.layers.41.mlp.experts.86.down_proj", "model.layers.41.mlp.experts.87.down_proj", "model.layers.41.mlp.experts.88.down_proj", "model.layers.41.mlp.experts.89.down_proj", "model.layers.41.mlp.experts.90.down_proj", "model.layers.41.mlp.experts.91.down_proj", "model.layers.41.mlp.experts.92.down_proj", "model.layers.41.mlp.experts.93.down_proj", "model.layers.41.mlp.experts.94.down_proj", "model.layers.41.mlp.experts.95.down_proj", "model.layers.41.mlp.experts.96.down_proj", "model.layers.41.mlp.experts.97.down_proj", "model.layers.41.mlp.experts.98.down_proj", "model.layers.41.mlp.experts.99.down_proj", "model.layers.41.mlp.experts.100.down_proj", "model.layers.41.mlp.experts.101.down_proj", "model.layers.41.mlp.experts.102.down_proj", "model.layers.41.mlp.experts.103.down_proj", "model.layers.41.mlp.experts.104.down_proj", "model.layers.41.mlp.experts.105.down_proj", "model.layers.41.mlp.experts.106.down_proj", "model.layers.41.mlp.experts.107.down_proj", "model.layers.41.mlp.experts.108.down_proj", "model.layers.41.mlp.experts.109.down_proj", "model.layers.41.mlp.experts.110.down_proj", "model.layers.41.mlp.experts.111.down_proj", "model.layers.41.mlp.experts.112.down_proj", "model.layers.41.mlp.experts.113.down_proj", "model.layers.41.mlp.experts.114.down_proj", "model.layers.41.mlp.experts.115.down_proj", "model.layers.41.mlp.experts.116.down_proj", "model.layers.41.mlp.experts.117.down_proj", "model.layers.41.mlp.experts.118.down_proj", "model.layers.41.mlp.experts.119.down_proj", "model.layers.41.mlp.experts.120.down_proj", "model.layers.41.mlp.experts.121.down_proj", "model.layers.41.mlp.experts.122.down_proj", "model.layers.41.mlp.experts.123.down_proj", "model.layers.41.mlp.experts.124.down_proj", "model.layers.41.mlp.experts.125.down_proj", "model.layers.41.mlp.experts.126.down_proj", "model.layers.41.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.00041011953726410866, "dbits": 2214592512 } ] }, { "idx": 125, "layers": [ "model.layers.42.self_attn.q_proj", "model.layers.42.self_attn.k_proj", "model.layers.42.self_attn.v_proj", "model.layers.42.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0003753677941858713, "dbits": 109051904 } ] }, { "idx": 126, "layers": [ "model.layers.42.mlp.shared_experts.gate_proj", "model.layers.42.mlp.shared_experts.up_proj", "model.layers.42.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -6.975717842577778e-05, "dbits": 17301504 } ] }, { "idx": 127, "layers": [ "model.layers.42.mlp.experts.0.gate_proj", "model.layers.42.mlp.experts.1.gate_proj", "model.layers.42.mlp.experts.2.gate_proj", "model.layers.42.mlp.experts.3.gate_proj", "model.layers.42.mlp.experts.4.gate_proj", "model.layers.42.mlp.experts.5.gate_proj", "model.layers.42.mlp.experts.6.gate_proj", "model.layers.42.mlp.experts.7.gate_proj", "model.layers.42.mlp.experts.8.gate_proj", "model.layers.42.mlp.experts.9.gate_proj", "model.layers.42.mlp.experts.10.gate_proj", "model.layers.42.mlp.experts.11.gate_proj", "model.layers.42.mlp.experts.12.gate_proj", "model.layers.42.mlp.experts.13.gate_proj", "model.layers.42.mlp.experts.14.gate_proj", "model.layers.42.mlp.experts.15.gate_proj", "model.layers.42.mlp.experts.16.gate_proj", "model.layers.42.mlp.experts.17.gate_proj", "model.layers.42.mlp.experts.18.gate_proj", "model.layers.42.mlp.experts.19.gate_proj", "model.layers.42.mlp.experts.20.gate_proj", "model.layers.42.mlp.experts.21.gate_proj", "model.layers.42.mlp.experts.22.gate_proj", "model.layers.42.mlp.experts.23.gate_proj", "model.layers.42.mlp.experts.24.gate_proj", "model.layers.42.mlp.experts.25.gate_proj", "model.layers.42.mlp.experts.26.gate_proj", "model.layers.42.mlp.experts.27.gate_proj", "model.layers.42.mlp.experts.28.gate_proj", "model.layers.42.mlp.experts.29.gate_proj", "model.layers.42.mlp.experts.30.gate_proj", "model.layers.42.mlp.experts.31.gate_proj", "model.layers.42.mlp.experts.32.gate_proj", "model.layers.42.mlp.experts.33.gate_proj", "model.layers.42.mlp.experts.34.gate_proj", "model.layers.42.mlp.experts.35.gate_proj", "model.layers.42.mlp.experts.36.gate_proj", "model.layers.42.mlp.experts.37.gate_proj", "model.layers.42.mlp.experts.38.gate_proj", "model.layers.42.mlp.experts.39.gate_proj", "model.layers.42.mlp.experts.40.gate_proj", "model.layers.42.mlp.experts.41.gate_proj", "model.layers.42.mlp.experts.42.gate_proj", "model.layers.42.mlp.experts.43.gate_proj", "model.layers.42.mlp.experts.44.gate_proj", "model.layers.42.mlp.experts.45.gate_proj", "model.layers.42.mlp.experts.46.gate_proj", "model.layers.42.mlp.experts.47.gate_proj", "model.layers.42.mlp.experts.48.gate_proj", "model.layers.42.mlp.experts.49.gate_proj", "model.layers.42.mlp.experts.50.gate_proj", "model.layers.42.mlp.experts.51.gate_proj", "model.layers.42.mlp.experts.52.gate_proj", "model.layers.42.mlp.experts.53.gate_proj", "model.layers.42.mlp.experts.54.gate_proj", "model.layers.42.mlp.experts.55.gate_proj", "model.layers.42.mlp.experts.56.gate_proj", "model.layers.42.mlp.experts.57.gate_proj", "model.layers.42.mlp.experts.58.gate_proj", "model.layers.42.mlp.experts.59.gate_proj", "model.layers.42.mlp.experts.60.gate_proj", "model.layers.42.mlp.experts.61.gate_proj", "model.layers.42.mlp.experts.62.gate_proj", "model.layers.42.mlp.experts.63.gate_proj", "model.layers.42.mlp.experts.64.gate_proj", "model.layers.42.mlp.experts.65.gate_proj", "model.layers.42.mlp.experts.66.gate_proj", "model.layers.42.mlp.experts.67.gate_proj", "model.layers.42.mlp.experts.68.gate_proj", "model.layers.42.mlp.experts.69.gate_proj", "model.layers.42.mlp.experts.70.gate_proj", "model.layers.42.mlp.experts.71.gate_proj", "model.layers.42.mlp.experts.72.gate_proj", "model.layers.42.mlp.experts.73.gate_proj", "model.layers.42.mlp.experts.74.gate_proj", "model.layers.42.mlp.experts.75.gate_proj", "model.layers.42.mlp.experts.76.gate_proj", "model.layers.42.mlp.experts.77.gate_proj", "model.layers.42.mlp.experts.78.gate_proj", "model.layers.42.mlp.experts.79.gate_proj", "model.layers.42.mlp.experts.80.gate_proj", "model.layers.42.mlp.experts.81.gate_proj", "model.layers.42.mlp.experts.82.gate_proj", "model.layers.42.mlp.experts.83.gate_proj", "model.layers.42.mlp.experts.84.gate_proj", "model.layers.42.mlp.experts.85.gate_proj", "model.layers.42.mlp.experts.86.gate_proj", "model.layers.42.mlp.experts.87.gate_proj", "model.layers.42.mlp.experts.88.gate_proj", "model.layers.42.mlp.experts.89.gate_proj", "model.layers.42.mlp.experts.90.gate_proj", "model.layers.42.mlp.experts.91.gate_proj", "model.layers.42.mlp.experts.92.gate_proj", "model.layers.42.mlp.experts.93.gate_proj", "model.layers.42.mlp.experts.94.gate_proj", "model.layers.42.mlp.experts.95.gate_proj", "model.layers.42.mlp.experts.96.gate_proj", "model.layers.42.mlp.experts.97.gate_proj", "model.layers.42.mlp.experts.98.gate_proj", "model.layers.42.mlp.experts.99.gate_proj", "model.layers.42.mlp.experts.100.gate_proj", "model.layers.42.mlp.experts.101.gate_proj", "model.layers.42.mlp.experts.102.gate_proj", "model.layers.42.mlp.experts.103.gate_proj", "model.layers.42.mlp.experts.104.gate_proj", "model.layers.42.mlp.experts.105.gate_proj", "model.layers.42.mlp.experts.106.gate_proj", "model.layers.42.mlp.experts.107.gate_proj", "model.layers.42.mlp.experts.108.gate_proj", "model.layers.42.mlp.experts.109.gate_proj", "model.layers.42.mlp.experts.110.gate_proj", "model.layers.42.mlp.experts.111.gate_proj", "model.layers.42.mlp.experts.112.gate_proj", "model.layers.42.mlp.experts.113.gate_proj", "model.layers.42.mlp.experts.114.gate_proj", "model.layers.42.mlp.experts.115.gate_proj", "model.layers.42.mlp.experts.116.gate_proj", "model.layers.42.mlp.experts.117.gate_proj", "model.layers.42.mlp.experts.118.gate_proj", "model.layers.42.mlp.experts.119.gate_proj", "model.layers.42.mlp.experts.120.gate_proj", "model.layers.42.mlp.experts.121.gate_proj", "model.layers.42.mlp.experts.122.gate_proj", "model.layers.42.mlp.experts.123.gate_proj", "model.layers.42.mlp.experts.124.gate_proj", "model.layers.42.mlp.experts.125.gate_proj", "model.layers.42.mlp.experts.126.gate_proj", "model.layers.42.mlp.experts.127.gate_proj", "model.layers.42.mlp.experts.0.up_proj", "model.layers.42.mlp.experts.1.up_proj", "model.layers.42.mlp.experts.2.up_proj", "model.layers.42.mlp.experts.3.up_proj", "model.layers.42.mlp.experts.4.up_proj", "model.layers.42.mlp.experts.5.up_proj", "model.layers.42.mlp.experts.6.up_proj", "model.layers.42.mlp.experts.7.up_proj", "model.layers.42.mlp.experts.8.up_proj", "model.layers.42.mlp.experts.9.up_proj", "model.layers.42.mlp.experts.10.up_proj", "model.layers.42.mlp.experts.11.up_proj", "model.layers.42.mlp.experts.12.up_proj", "model.layers.42.mlp.experts.13.up_proj", "model.layers.42.mlp.experts.14.up_proj", "model.layers.42.mlp.experts.15.up_proj", "model.layers.42.mlp.experts.16.up_proj", "model.layers.42.mlp.experts.17.up_proj", "model.layers.42.mlp.experts.18.up_proj", "model.layers.42.mlp.experts.19.up_proj", "model.layers.42.mlp.experts.20.up_proj", "model.layers.42.mlp.experts.21.up_proj", "model.layers.42.mlp.experts.22.up_proj", "model.layers.42.mlp.experts.23.up_proj", "model.layers.42.mlp.experts.24.up_proj", "model.layers.42.mlp.experts.25.up_proj", "model.layers.42.mlp.experts.26.up_proj", "model.layers.42.mlp.experts.27.up_proj", "model.layers.42.mlp.experts.28.up_proj", "model.layers.42.mlp.experts.29.up_proj", "model.layers.42.mlp.experts.30.up_proj", "model.layers.42.mlp.experts.31.up_proj", "model.layers.42.mlp.experts.32.up_proj", "model.layers.42.mlp.experts.33.up_proj", "model.layers.42.mlp.experts.34.up_proj", "model.layers.42.mlp.experts.35.up_proj", "model.layers.42.mlp.experts.36.up_proj", "model.layers.42.mlp.experts.37.up_proj", "model.layers.42.mlp.experts.38.up_proj", "model.layers.42.mlp.experts.39.up_proj", "model.layers.42.mlp.experts.40.up_proj", "model.layers.42.mlp.experts.41.up_proj", "model.layers.42.mlp.experts.42.up_proj", "model.layers.42.mlp.experts.43.up_proj", "model.layers.42.mlp.experts.44.up_proj", "model.layers.42.mlp.experts.45.up_proj", "model.layers.42.mlp.experts.46.up_proj", "model.layers.42.mlp.experts.47.up_proj", "model.layers.42.mlp.experts.48.up_proj", "model.layers.42.mlp.experts.49.up_proj", "model.layers.42.mlp.experts.50.up_proj", "model.layers.42.mlp.experts.51.up_proj", "model.layers.42.mlp.experts.52.up_proj", "model.layers.42.mlp.experts.53.up_proj", "model.layers.42.mlp.experts.54.up_proj", "model.layers.42.mlp.experts.55.up_proj", "model.layers.42.mlp.experts.56.up_proj", "model.layers.42.mlp.experts.57.up_proj", "model.layers.42.mlp.experts.58.up_proj", "model.layers.42.mlp.experts.59.up_proj", "model.layers.42.mlp.experts.60.up_proj", "model.layers.42.mlp.experts.61.up_proj", "model.layers.42.mlp.experts.62.up_proj", "model.layers.42.mlp.experts.63.up_proj", "model.layers.42.mlp.experts.64.up_proj", "model.layers.42.mlp.experts.65.up_proj", "model.layers.42.mlp.experts.66.up_proj", "model.layers.42.mlp.experts.67.up_proj", "model.layers.42.mlp.experts.68.up_proj", "model.layers.42.mlp.experts.69.up_proj", "model.layers.42.mlp.experts.70.up_proj", "model.layers.42.mlp.experts.71.up_proj", "model.layers.42.mlp.experts.72.up_proj", "model.layers.42.mlp.experts.73.up_proj", "model.layers.42.mlp.experts.74.up_proj", "model.layers.42.mlp.experts.75.up_proj", "model.layers.42.mlp.experts.76.up_proj", "model.layers.42.mlp.experts.77.up_proj", "model.layers.42.mlp.experts.78.up_proj", "model.layers.42.mlp.experts.79.up_proj", "model.layers.42.mlp.experts.80.up_proj", "model.layers.42.mlp.experts.81.up_proj", "model.layers.42.mlp.experts.82.up_proj", "model.layers.42.mlp.experts.83.up_proj", "model.layers.42.mlp.experts.84.up_proj", "model.layers.42.mlp.experts.85.up_proj", "model.layers.42.mlp.experts.86.up_proj", "model.layers.42.mlp.experts.87.up_proj", "model.layers.42.mlp.experts.88.up_proj", "model.layers.42.mlp.experts.89.up_proj", "model.layers.42.mlp.experts.90.up_proj", "model.layers.42.mlp.experts.91.up_proj", "model.layers.42.mlp.experts.92.up_proj", "model.layers.42.mlp.experts.93.up_proj", "model.layers.42.mlp.experts.94.up_proj", "model.layers.42.mlp.experts.95.up_proj", "model.layers.42.mlp.experts.96.up_proj", "model.layers.42.mlp.experts.97.up_proj", "model.layers.42.mlp.experts.98.up_proj", "model.layers.42.mlp.experts.99.up_proj", "model.layers.42.mlp.experts.100.up_proj", "model.layers.42.mlp.experts.101.up_proj", "model.layers.42.mlp.experts.102.up_proj", "model.layers.42.mlp.experts.103.up_proj", "model.layers.42.mlp.experts.104.up_proj", "model.layers.42.mlp.experts.105.up_proj", "model.layers.42.mlp.experts.106.up_proj", "model.layers.42.mlp.experts.107.up_proj", "model.layers.42.mlp.experts.108.up_proj", "model.layers.42.mlp.experts.109.up_proj", "model.layers.42.mlp.experts.110.up_proj", "model.layers.42.mlp.experts.111.up_proj", "model.layers.42.mlp.experts.112.up_proj", "model.layers.42.mlp.experts.113.up_proj", "model.layers.42.mlp.experts.114.up_proj", "model.layers.42.mlp.experts.115.up_proj", "model.layers.42.mlp.experts.116.up_proj", "model.layers.42.mlp.experts.117.up_proj", "model.layers.42.mlp.experts.118.up_proj", "model.layers.42.mlp.experts.119.up_proj", "model.layers.42.mlp.experts.120.up_proj", "model.layers.42.mlp.experts.121.up_proj", "model.layers.42.mlp.experts.122.up_proj", "model.layers.42.mlp.experts.123.up_proj", "model.layers.42.mlp.experts.124.up_proj", "model.layers.42.mlp.experts.125.up_proj", "model.layers.42.mlp.experts.126.up_proj", "model.layers.42.mlp.experts.127.up_proj", "model.layers.42.mlp.experts.0.down_proj", "model.layers.42.mlp.experts.1.down_proj", "model.layers.42.mlp.experts.2.down_proj", "model.layers.42.mlp.experts.3.down_proj", "model.layers.42.mlp.experts.4.down_proj", "model.layers.42.mlp.experts.5.down_proj", "model.layers.42.mlp.experts.6.down_proj", "model.layers.42.mlp.experts.7.down_proj", "model.layers.42.mlp.experts.8.down_proj", "model.layers.42.mlp.experts.9.down_proj", "model.layers.42.mlp.experts.10.down_proj", "model.layers.42.mlp.experts.11.down_proj", "model.layers.42.mlp.experts.12.down_proj", "model.layers.42.mlp.experts.13.down_proj", "model.layers.42.mlp.experts.14.down_proj", "model.layers.42.mlp.experts.15.down_proj", "model.layers.42.mlp.experts.16.down_proj", "model.layers.42.mlp.experts.17.down_proj", "model.layers.42.mlp.experts.18.down_proj", "model.layers.42.mlp.experts.19.down_proj", "model.layers.42.mlp.experts.20.down_proj", "model.layers.42.mlp.experts.21.down_proj", "model.layers.42.mlp.experts.22.down_proj", "model.layers.42.mlp.experts.23.down_proj", "model.layers.42.mlp.experts.24.down_proj", "model.layers.42.mlp.experts.25.down_proj", "model.layers.42.mlp.experts.26.down_proj", "model.layers.42.mlp.experts.27.down_proj", "model.layers.42.mlp.experts.28.down_proj", "model.layers.42.mlp.experts.29.down_proj", "model.layers.42.mlp.experts.30.down_proj", "model.layers.42.mlp.experts.31.down_proj", "model.layers.42.mlp.experts.32.down_proj", "model.layers.42.mlp.experts.33.down_proj", "model.layers.42.mlp.experts.34.down_proj", "model.layers.42.mlp.experts.35.down_proj", "model.layers.42.mlp.experts.36.down_proj", "model.layers.42.mlp.experts.37.down_proj", "model.layers.42.mlp.experts.38.down_proj", "model.layers.42.mlp.experts.39.down_proj", "model.layers.42.mlp.experts.40.down_proj", "model.layers.42.mlp.experts.41.down_proj", "model.layers.42.mlp.experts.42.down_proj", "model.layers.42.mlp.experts.43.down_proj", "model.layers.42.mlp.experts.44.down_proj", "model.layers.42.mlp.experts.45.down_proj", "model.layers.42.mlp.experts.46.down_proj", "model.layers.42.mlp.experts.47.down_proj", "model.layers.42.mlp.experts.48.down_proj", "model.layers.42.mlp.experts.49.down_proj", "model.layers.42.mlp.experts.50.down_proj", "model.layers.42.mlp.experts.51.down_proj", "model.layers.42.mlp.experts.52.down_proj", "model.layers.42.mlp.experts.53.down_proj", "model.layers.42.mlp.experts.54.down_proj", "model.layers.42.mlp.experts.55.down_proj", "model.layers.42.mlp.experts.56.down_proj", "model.layers.42.mlp.experts.57.down_proj", "model.layers.42.mlp.experts.58.down_proj", "model.layers.42.mlp.experts.59.down_proj", "model.layers.42.mlp.experts.60.down_proj", "model.layers.42.mlp.experts.61.down_proj", "model.layers.42.mlp.experts.62.down_proj", "model.layers.42.mlp.experts.63.down_proj", "model.layers.42.mlp.experts.64.down_proj", "model.layers.42.mlp.experts.65.down_proj", "model.layers.42.mlp.experts.66.down_proj", "model.layers.42.mlp.experts.67.down_proj", "model.layers.42.mlp.experts.68.down_proj", "model.layers.42.mlp.experts.69.down_proj", "model.layers.42.mlp.experts.70.down_proj", "model.layers.42.mlp.experts.71.down_proj", "model.layers.42.mlp.experts.72.down_proj", "model.layers.42.mlp.experts.73.down_proj", "model.layers.42.mlp.experts.74.down_proj", "model.layers.42.mlp.experts.75.down_proj", "model.layers.42.mlp.experts.76.down_proj", "model.layers.42.mlp.experts.77.down_proj", "model.layers.42.mlp.experts.78.down_proj", "model.layers.42.mlp.experts.79.down_proj", "model.layers.42.mlp.experts.80.down_proj", "model.layers.42.mlp.experts.81.down_proj", "model.layers.42.mlp.experts.82.down_proj", "model.layers.42.mlp.experts.83.down_proj", "model.layers.42.mlp.experts.84.down_proj", "model.layers.42.mlp.experts.85.down_proj", "model.layers.42.mlp.experts.86.down_proj", "model.layers.42.mlp.experts.87.down_proj", "model.layers.42.mlp.experts.88.down_proj", "model.layers.42.mlp.experts.89.down_proj", "model.layers.42.mlp.experts.90.down_proj", "model.layers.42.mlp.experts.91.down_proj", "model.layers.42.mlp.experts.92.down_proj", "model.layers.42.mlp.experts.93.down_proj", "model.layers.42.mlp.experts.94.down_proj", "model.layers.42.mlp.experts.95.down_proj", "model.layers.42.mlp.experts.96.down_proj", "model.layers.42.mlp.experts.97.down_proj", "model.layers.42.mlp.experts.98.down_proj", "model.layers.42.mlp.experts.99.down_proj", "model.layers.42.mlp.experts.100.down_proj", "model.layers.42.mlp.experts.101.down_proj", "model.layers.42.mlp.experts.102.down_proj", "model.layers.42.mlp.experts.103.down_proj", "model.layers.42.mlp.experts.104.down_proj", "model.layers.42.mlp.experts.105.down_proj", "model.layers.42.mlp.experts.106.down_proj", "model.layers.42.mlp.experts.107.down_proj", "model.layers.42.mlp.experts.108.down_proj", "model.layers.42.mlp.experts.109.down_proj", "model.layers.42.mlp.experts.110.down_proj", "model.layers.42.mlp.experts.111.down_proj", "model.layers.42.mlp.experts.112.down_proj", "model.layers.42.mlp.experts.113.down_proj", "model.layers.42.mlp.experts.114.down_proj", "model.layers.42.mlp.experts.115.down_proj", "model.layers.42.mlp.experts.116.down_proj", "model.layers.42.mlp.experts.117.down_proj", "model.layers.42.mlp.experts.118.down_proj", "model.layers.42.mlp.experts.119.down_proj", "model.layers.42.mlp.experts.120.down_proj", "model.layers.42.mlp.experts.121.down_proj", "model.layers.42.mlp.experts.122.down_proj", "model.layers.42.mlp.experts.123.down_proj", "model.layers.42.mlp.experts.124.down_proj", "model.layers.42.mlp.experts.125.down_proj", "model.layers.42.mlp.experts.126.down_proj", "model.layers.42.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.00048411590978503227, "dbits": 2214592512 } ] }, { "idx": 128, "layers": [ "model.layers.43.self_attn.q_proj", "model.layers.43.self_attn.k_proj", "model.layers.43.self_attn.v_proj", "model.layers.43.self_attn.o_proj" ], "candidates": [ { "dkld": 7.921569049360033e-05, "dbits": 109051904 } ] }, { "idx": 129, "layers": [ "model.layers.43.mlp.shared_experts.gate_proj", "model.layers.43.mlp.shared_experts.up_proj", "model.layers.43.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.00041398163884878714, "dbits": 17301504 } ] }, { "idx": 130, "layers": [ "model.layers.43.mlp.experts.0.gate_proj", "model.layers.43.mlp.experts.1.gate_proj", "model.layers.43.mlp.experts.2.gate_proj", "model.layers.43.mlp.experts.3.gate_proj", "model.layers.43.mlp.experts.4.gate_proj", "model.layers.43.mlp.experts.5.gate_proj", "model.layers.43.mlp.experts.6.gate_proj", "model.layers.43.mlp.experts.7.gate_proj", "model.layers.43.mlp.experts.8.gate_proj", "model.layers.43.mlp.experts.9.gate_proj", "model.layers.43.mlp.experts.10.gate_proj", "model.layers.43.mlp.experts.11.gate_proj", "model.layers.43.mlp.experts.12.gate_proj", "model.layers.43.mlp.experts.13.gate_proj", "model.layers.43.mlp.experts.14.gate_proj", "model.layers.43.mlp.experts.15.gate_proj", "model.layers.43.mlp.experts.16.gate_proj", "model.layers.43.mlp.experts.17.gate_proj", "model.layers.43.mlp.experts.18.gate_proj", "model.layers.43.mlp.experts.19.gate_proj", "model.layers.43.mlp.experts.20.gate_proj", "model.layers.43.mlp.experts.21.gate_proj", "model.layers.43.mlp.experts.22.gate_proj", "model.layers.43.mlp.experts.23.gate_proj", "model.layers.43.mlp.experts.24.gate_proj", "model.layers.43.mlp.experts.25.gate_proj", "model.layers.43.mlp.experts.26.gate_proj", "model.layers.43.mlp.experts.27.gate_proj", "model.layers.43.mlp.experts.28.gate_proj", "model.layers.43.mlp.experts.29.gate_proj", "model.layers.43.mlp.experts.30.gate_proj", "model.layers.43.mlp.experts.31.gate_proj", "model.layers.43.mlp.experts.32.gate_proj", "model.layers.43.mlp.experts.33.gate_proj", "model.layers.43.mlp.experts.34.gate_proj", "model.layers.43.mlp.experts.35.gate_proj", "model.layers.43.mlp.experts.36.gate_proj", "model.layers.43.mlp.experts.37.gate_proj", "model.layers.43.mlp.experts.38.gate_proj", "model.layers.43.mlp.experts.39.gate_proj", "model.layers.43.mlp.experts.40.gate_proj", "model.layers.43.mlp.experts.41.gate_proj", "model.layers.43.mlp.experts.42.gate_proj", "model.layers.43.mlp.experts.43.gate_proj", "model.layers.43.mlp.experts.44.gate_proj", "model.layers.43.mlp.experts.45.gate_proj", "model.layers.43.mlp.experts.46.gate_proj", "model.layers.43.mlp.experts.47.gate_proj", "model.layers.43.mlp.experts.48.gate_proj", "model.layers.43.mlp.experts.49.gate_proj", "model.layers.43.mlp.experts.50.gate_proj", "model.layers.43.mlp.experts.51.gate_proj", "model.layers.43.mlp.experts.52.gate_proj", "model.layers.43.mlp.experts.53.gate_proj", "model.layers.43.mlp.experts.54.gate_proj", "model.layers.43.mlp.experts.55.gate_proj", "model.layers.43.mlp.experts.56.gate_proj", "model.layers.43.mlp.experts.57.gate_proj", "model.layers.43.mlp.experts.58.gate_proj", "model.layers.43.mlp.experts.59.gate_proj", "model.layers.43.mlp.experts.60.gate_proj", "model.layers.43.mlp.experts.61.gate_proj", "model.layers.43.mlp.experts.62.gate_proj", "model.layers.43.mlp.experts.63.gate_proj", "model.layers.43.mlp.experts.64.gate_proj", "model.layers.43.mlp.experts.65.gate_proj", "model.layers.43.mlp.experts.66.gate_proj", "model.layers.43.mlp.experts.67.gate_proj", "model.layers.43.mlp.experts.68.gate_proj", "model.layers.43.mlp.experts.69.gate_proj", "model.layers.43.mlp.experts.70.gate_proj", "model.layers.43.mlp.experts.71.gate_proj", "model.layers.43.mlp.experts.72.gate_proj", "model.layers.43.mlp.experts.73.gate_proj", "model.layers.43.mlp.experts.74.gate_proj", "model.layers.43.mlp.experts.75.gate_proj", "model.layers.43.mlp.experts.76.gate_proj", "model.layers.43.mlp.experts.77.gate_proj", "model.layers.43.mlp.experts.78.gate_proj", "model.layers.43.mlp.experts.79.gate_proj", "model.layers.43.mlp.experts.80.gate_proj", "model.layers.43.mlp.experts.81.gate_proj", "model.layers.43.mlp.experts.82.gate_proj", "model.layers.43.mlp.experts.83.gate_proj", "model.layers.43.mlp.experts.84.gate_proj", "model.layers.43.mlp.experts.85.gate_proj", "model.layers.43.mlp.experts.86.gate_proj", "model.layers.43.mlp.experts.87.gate_proj", "model.layers.43.mlp.experts.88.gate_proj", "model.layers.43.mlp.experts.89.gate_proj", "model.layers.43.mlp.experts.90.gate_proj", "model.layers.43.mlp.experts.91.gate_proj", "model.layers.43.mlp.experts.92.gate_proj", "model.layers.43.mlp.experts.93.gate_proj", "model.layers.43.mlp.experts.94.gate_proj", "model.layers.43.mlp.experts.95.gate_proj", "model.layers.43.mlp.experts.96.gate_proj", "model.layers.43.mlp.experts.97.gate_proj", "model.layers.43.mlp.experts.98.gate_proj", "model.layers.43.mlp.experts.99.gate_proj", "model.layers.43.mlp.experts.100.gate_proj", "model.layers.43.mlp.experts.101.gate_proj", "model.layers.43.mlp.experts.102.gate_proj", "model.layers.43.mlp.experts.103.gate_proj", "model.layers.43.mlp.experts.104.gate_proj", "model.layers.43.mlp.experts.105.gate_proj", "model.layers.43.mlp.experts.106.gate_proj", "model.layers.43.mlp.experts.107.gate_proj", "model.layers.43.mlp.experts.108.gate_proj", "model.layers.43.mlp.experts.109.gate_proj", "model.layers.43.mlp.experts.110.gate_proj", "model.layers.43.mlp.experts.111.gate_proj", "model.layers.43.mlp.experts.112.gate_proj", "model.layers.43.mlp.experts.113.gate_proj", "model.layers.43.mlp.experts.114.gate_proj", "model.layers.43.mlp.experts.115.gate_proj", "model.layers.43.mlp.experts.116.gate_proj", "model.layers.43.mlp.experts.117.gate_proj", "model.layers.43.mlp.experts.118.gate_proj", "model.layers.43.mlp.experts.119.gate_proj", "model.layers.43.mlp.experts.120.gate_proj", "model.layers.43.mlp.experts.121.gate_proj", "model.layers.43.mlp.experts.122.gate_proj", "model.layers.43.mlp.experts.123.gate_proj", "model.layers.43.mlp.experts.124.gate_proj", "model.layers.43.mlp.experts.125.gate_proj", "model.layers.43.mlp.experts.126.gate_proj", "model.layers.43.mlp.experts.127.gate_proj", "model.layers.43.mlp.experts.0.up_proj", "model.layers.43.mlp.experts.1.up_proj", "model.layers.43.mlp.experts.2.up_proj", "model.layers.43.mlp.experts.3.up_proj", "model.layers.43.mlp.experts.4.up_proj", "model.layers.43.mlp.experts.5.up_proj", "model.layers.43.mlp.experts.6.up_proj", "model.layers.43.mlp.experts.7.up_proj", "model.layers.43.mlp.experts.8.up_proj", "model.layers.43.mlp.experts.9.up_proj", "model.layers.43.mlp.experts.10.up_proj", "model.layers.43.mlp.experts.11.up_proj", "model.layers.43.mlp.experts.12.up_proj", "model.layers.43.mlp.experts.13.up_proj", "model.layers.43.mlp.experts.14.up_proj", "model.layers.43.mlp.experts.15.up_proj", "model.layers.43.mlp.experts.16.up_proj", "model.layers.43.mlp.experts.17.up_proj", "model.layers.43.mlp.experts.18.up_proj", "model.layers.43.mlp.experts.19.up_proj", "model.layers.43.mlp.experts.20.up_proj", "model.layers.43.mlp.experts.21.up_proj", "model.layers.43.mlp.experts.22.up_proj", "model.layers.43.mlp.experts.23.up_proj", "model.layers.43.mlp.experts.24.up_proj", "model.layers.43.mlp.experts.25.up_proj", "model.layers.43.mlp.experts.26.up_proj", "model.layers.43.mlp.experts.27.up_proj", "model.layers.43.mlp.experts.28.up_proj", "model.layers.43.mlp.experts.29.up_proj", "model.layers.43.mlp.experts.30.up_proj", "model.layers.43.mlp.experts.31.up_proj", "model.layers.43.mlp.experts.32.up_proj", "model.layers.43.mlp.experts.33.up_proj", "model.layers.43.mlp.experts.34.up_proj", "model.layers.43.mlp.experts.35.up_proj", "model.layers.43.mlp.experts.36.up_proj", "model.layers.43.mlp.experts.37.up_proj", "model.layers.43.mlp.experts.38.up_proj", "model.layers.43.mlp.experts.39.up_proj", "model.layers.43.mlp.experts.40.up_proj", "model.layers.43.mlp.experts.41.up_proj", "model.layers.43.mlp.experts.42.up_proj", "model.layers.43.mlp.experts.43.up_proj", "model.layers.43.mlp.experts.44.up_proj", "model.layers.43.mlp.experts.45.up_proj", "model.layers.43.mlp.experts.46.up_proj", "model.layers.43.mlp.experts.47.up_proj", "model.layers.43.mlp.experts.48.up_proj", "model.layers.43.mlp.experts.49.up_proj", "model.layers.43.mlp.experts.50.up_proj", "model.layers.43.mlp.experts.51.up_proj", "model.layers.43.mlp.experts.52.up_proj", "model.layers.43.mlp.experts.53.up_proj", "model.layers.43.mlp.experts.54.up_proj", "model.layers.43.mlp.experts.55.up_proj", "model.layers.43.mlp.experts.56.up_proj", "model.layers.43.mlp.experts.57.up_proj", "model.layers.43.mlp.experts.58.up_proj", "model.layers.43.mlp.experts.59.up_proj", "model.layers.43.mlp.experts.60.up_proj", "model.layers.43.mlp.experts.61.up_proj", "model.layers.43.mlp.experts.62.up_proj", "model.layers.43.mlp.experts.63.up_proj", "model.layers.43.mlp.experts.64.up_proj", "model.layers.43.mlp.experts.65.up_proj", "model.layers.43.mlp.experts.66.up_proj", "model.layers.43.mlp.experts.67.up_proj", "model.layers.43.mlp.experts.68.up_proj", "model.layers.43.mlp.experts.69.up_proj", "model.layers.43.mlp.experts.70.up_proj", "model.layers.43.mlp.experts.71.up_proj", "model.layers.43.mlp.experts.72.up_proj", "model.layers.43.mlp.experts.73.up_proj", "model.layers.43.mlp.experts.74.up_proj", "model.layers.43.mlp.experts.75.up_proj", "model.layers.43.mlp.experts.76.up_proj", "model.layers.43.mlp.experts.77.up_proj", "model.layers.43.mlp.experts.78.up_proj", "model.layers.43.mlp.experts.79.up_proj", "model.layers.43.mlp.experts.80.up_proj", "model.layers.43.mlp.experts.81.up_proj", "model.layers.43.mlp.experts.82.up_proj", "model.layers.43.mlp.experts.83.up_proj", "model.layers.43.mlp.experts.84.up_proj", "model.layers.43.mlp.experts.85.up_proj", "model.layers.43.mlp.experts.86.up_proj", "model.layers.43.mlp.experts.87.up_proj", "model.layers.43.mlp.experts.88.up_proj", "model.layers.43.mlp.experts.89.up_proj", "model.layers.43.mlp.experts.90.up_proj", "model.layers.43.mlp.experts.91.up_proj", "model.layers.43.mlp.experts.92.up_proj", "model.layers.43.mlp.experts.93.up_proj", "model.layers.43.mlp.experts.94.up_proj", "model.layers.43.mlp.experts.95.up_proj", "model.layers.43.mlp.experts.96.up_proj", "model.layers.43.mlp.experts.97.up_proj", "model.layers.43.mlp.experts.98.up_proj", "model.layers.43.mlp.experts.99.up_proj", "model.layers.43.mlp.experts.100.up_proj", "model.layers.43.mlp.experts.101.up_proj", "model.layers.43.mlp.experts.102.up_proj", "model.layers.43.mlp.experts.103.up_proj", "model.layers.43.mlp.experts.104.up_proj", "model.layers.43.mlp.experts.105.up_proj", "model.layers.43.mlp.experts.106.up_proj", "model.layers.43.mlp.experts.107.up_proj", "model.layers.43.mlp.experts.108.up_proj", "model.layers.43.mlp.experts.109.up_proj", "model.layers.43.mlp.experts.110.up_proj", "model.layers.43.mlp.experts.111.up_proj", "model.layers.43.mlp.experts.112.up_proj", "model.layers.43.mlp.experts.113.up_proj", "model.layers.43.mlp.experts.114.up_proj", "model.layers.43.mlp.experts.115.up_proj", "model.layers.43.mlp.experts.116.up_proj", "model.layers.43.mlp.experts.117.up_proj", "model.layers.43.mlp.experts.118.up_proj", "model.layers.43.mlp.experts.119.up_proj", "model.layers.43.mlp.experts.120.up_proj", "model.layers.43.mlp.experts.121.up_proj", "model.layers.43.mlp.experts.122.up_proj", "model.layers.43.mlp.experts.123.up_proj", "model.layers.43.mlp.experts.124.up_proj", "model.layers.43.mlp.experts.125.up_proj", "model.layers.43.mlp.experts.126.up_proj", "model.layers.43.mlp.experts.127.up_proj", "model.layers.43.mlp.experts.0.down_proj", "model.layers.43.mlp.experts.1.down_proj", "model.layers.43.mlp.experts.2.down_proj", "model.layers.43.mlp.experts.3.down_proj", "model.layers.43.mlp.experts.4.down_proj", "model.layers.43.mlp.experts.5.down_proj", "model.layers.43.mlp.experts.6.down_proj", "model.layers.43.mlp.experts.7.down_proj", "model.layers.43.mlp.experts.8.down_proj", "model.layers.43.mlp.experts.9.down_proj", "model.layers.43.mlp.experts.10.down_proj", "model.layers.43.mlp.experts.11.down_proj", "model.layers.43.mlp.experts.12.down_proj", "model.layers.43.mlp.experts.13.down_proj", "model.layers.43.mlp.experts.14.down_proj", "model.layers.43.mlp.experts.15.down_proj", "model.layers.43.mlp.experts.16.down_proj", "model.layers.43.mlp.experts.17.down_proj", "model.layers.43.mlp.experts.18.down_proj", "model.layers.43.mlp.experts.19.down_proj", "model.layers.43.mlp.experts.20.down_proj", "model.layers.43.mlp.experts.21.down_proj", "model.layers.43.mlp.experts.22.down_proj", "model.layers.43.mlp.experts.23.down_proj", "model.layers.43.mlp.experts.24.down_proj", "model.layers.43.mlp.experts.25.down_proj", "model.layers.43.mlp.experts.26.down_proj", "model.layers.43.mlp.experts.27.down_proj", "model.layers.43.mlp.experts.28.down_proj", "model.layers.43.mlp.experts.29.down_proj", "model.layers.43.mlp.experts.30.down_proj", "model.layers.43.mlp.experts.31.down_proj", "model.layers.43.mlp.experts.32.down_proj", "model.layers.43.mlp.experts.33.down_proj", "model.layers.43.mlp.experts.34.down_proj", "model.layers.43.mlp.experts.35.down_proj", "model.layers.43.mlp.experts.36.down_proj", "model.layers.43.mlp.experts.37.down_proj", "model.layers.43.mlp.experts.38.down_proj", "model.layers.43.mlp.experts.39.down_proj", "model.layers.43.mlp.experts.40.down_proj", "model.layers.43.mlp.experts.41.down_proj", "model.layers.43.mlp.experts.42.down_proj", "model.layers.43.mlp.experts.43.down_proj", "model.layers.43.mlp.experts.44.down_proj", "model.layers.43.mlp.experts.45.down_proj", "model.layers.43.mlp.experts.46.down_proj", "model.layers.43.mlp.experts.47.down_proj", "model.layers.43.mlp.experts.48.down_proj", "model.layers.43.mlp.experts.49.down_proj", "model.layers.43.mlp.experts.50.down_proj", "model.layers.43.mlp.experts.51.down_proj", "model.layers.43.mlp.experts.52.down_proj", "model.layers.43.mlp.experts.53.down_proj", "model.layers.43.mlp.experts.54.down_proj", "model.layers.43.mlp.experts.55.down_proj", "model.layers.43.mlp.experts.56.down_proj", "model.layers.43.mlp.experts.57.down_proj", "model.layers.43.mlp.experts.58.down_proj", "model.layers.43.mlp.experts.59.down_proj", "model.layers.43.mlp.experts.60.down_proj", "model.layers.43.mlp.experts.61.down_proj", "model.layers.43.mlp.experts.62.down_proj", "model.layers.43.mlp.experts.63.down_proj", "model.layers.43.mlp.experts.64.down_proj", "model.layers.43.mlp.experts.65.down_proj", "model.layers.43.mlp.experts.66.down_proj", "model.layers.43.mlp.experts.67.down_proj", "model.layers.43.mlp.experts.68.down_proj", "model.layers.43.mlp.experts.69.down_proj", "model.layers.43.mlp.experts.70.down_proj", "model.layers.43.mlp.experts.71.down_proj", "model.layers.43.mlp.experts.72.down_proj", "model.layers.43.mlp.experts.73.down_proj", "model.layers.43.mlp.experts.74.down_proj", "model.layers.43.mlp.experts.75.down_proj", "model.layers.43.mlp.experts.76.down_proj", "model.layers.43.mlp.experts.77.down_proj", "model.layers.43.mlp.experts.78.down_proj", "model.layers.43.mlp.experts.79.down_proj", "model.layers.43.mlp.experts.80.down_proj", "model.layers.43.mlp.experts.81.down_proj", "model.layers.43.mlp.experts.82.down_proj", "model.layers.43.mlp.experts.83.down_proj", "model.layers.43.mlp.experts.84.down_proj", "model.layers.43.mlp.experts.85.down_proj", "model.layers.43.mlp.experts.86.down_proj", "model.layers.43.mlp.experts.87.down_proj", "model.layers.43.mlp.experts.88.down_proj", "model.layers.43.mlp.experts.89.down_proj", "model.layers.43.mlp.experts.90.down_proj", "model.layers.43.mlp.experts.91.down_proj", "model.layers.43.mlp.experts.92.down_proj", "model.layers.43.mlp.experts.93.down_proj", "model.layers.43.mlp.experts.94.down_proj", "model.layers.43.mlp.experts.95.down_proj", "model.layers.43.mlp.experts.96.down_proj", "model.layers.43.mlp.experts.97.down_proj", "model.layers.43.mlp.experts.98.down_proj", "model.layers.43.mlp.experts.99.down_proj", "model.layers.43.mlp.experts.100.down_proj", "model.layers.43.mlp.experts.101.down_proj", "model.layers.43.mlp.experts.102.down_proj", "model.layers.43.mlp.experts.103.down_proj", "model.layers.43.mlp.experts.104.down_proj", "model.layers.43.mlp.experts.105.down_proj", "model.layers.43.mlp.experts.106.down_proj", "model.layers.43.mlp.experts.107.down_proj", "model.layers.43.mlp.experts.108.down_proj", "model.layers.43.mlp.experts.109.down_proj", "model.layers.43.mlp.experts.110.down_proj", "model.layers.43.mlp.experts.111.down_proj", "model.layers.43.mlp.experts.112.down_proj", "model.layers.43.mlp.experts.113.down_proj", "model.layers.43.mlp.experts.114.down_proj", "model.layers.43.mlp.experts.115.down_proj", "model.layers.43.mlp.experts.116.down_proj", "model.layers.43.mlp.experts.117.down_proj", "model.layers.43.mlp.experts.118.down_proj", "model.layers.43.mlp.experts.119.down_proj", "model.layers.43.mlp.experts.120.down_proj", "model.layers.43.mlp.experts.121.down_proj", "model.layers.43.mlp.experts.122.down_proj", "model.layers.43.mlp.experts.123.down_proj", "model.layers.43.mlp.experts.124.down_proj", "model.layers.43.mlp.experts.125.down_proj", "model.layers.43.mlp.experts.126.down_proj", "model.layers.43.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.002862455788999796, "dbits": 2214592512 } ] }, { "idx": 131, "layers": [ "model.layers.44.self_attn.q_proj", "model.layers.44.self_attn.k_proj", "model.layers.44.self_attn.v_proj", "model.layers.44.self_attn.o_proj" ], "candidates": [ { "dkld": -0.0002312533557415064, "dbits": 109051904 } ] }, { "idx": 132, "layers": [ "model.layers.44.mlp.shared_experts.gate_proj", "model.layers.44.mlp.shared_experts.up_proj", "model.layers.44.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.001161700766533602, "dbits": 17301504 } ] }, { "idx": 133, "layers": [ "model.layers.44.mlp.experts.0.gate_proj", "model.layers.44.mlp.experts.1.gate_proj", "model.layers.44.mlp.experts.2.gate_proj", "model.layers.44.mlp.experts.3.gate_proj", "model.layers.44.mlp.experts.4.gate_proj", "model.layers.44.mlp.experts.5.gate_proj", "model.layers.44.mlp.experts.6.gate_proj", "model.layers.44.mlp.experts.7.gate_proj", "model.layers.44.mlp.experts.8.gate_proj", "model.layers.44.mlp.experts.9.gate_proj", "model.layers.44.mlp.experts.10.gate_proj", "model.layers.44.mlp.experts.11.gate_proj", "model.layers.44.mlp.experts.12.gate_proj", "model.layers.44.mlp.experts.13.gate_proj", "model.layers.44.mlp.experts.14.gate_proj", "model.layers.44.mlp.experts.15.gate_proj", "model.layers.44.mlp.experts.16.gate_proj", "model.layers.44.mlp.experts.17.gate_proj", "model.layers.44.mlp.experts.18.gate_proj", "model.layers.44.mlp.experts.19.gate_proj", "model.layers.44.mlp.experts.20.gate_proj", "model.layers.44.mlp.experts.21.gate_proj", "model.layers.44.mlp.experts.22.gate_proj", "model.layers.44.mlp.experts.23.gate_proj", "model.layers.44.mlp.experts.24.gate_proj", "model.layers.44.mlp.experts.25.gate_proj", "model.layers.44.mlp.experts.26.gate_proj", "model.layers.44.mlp.experts.27.gate_proj", "model.layers.44.mlp.experts.28.gate_proj", "model.layers.44.mlp.experts.29.gate_proj", "model.layers.44.mlp.experts.30.gate_proj", "model.layers.44.mlp.experts.31.gate_proj", "model.layers.44.mlp.experts.32.gate_proj", "model.layers.44.mlp.experts.33.gate_proj", "model.layers.44.mlp.experts.34.gate_proj", "model.layers.44.mlp.experts.35.gate_proj", "model.layers.44.mlp.experts.36.gate_proj", "model.layers.44.mlp.experts.37.gate_proj", "model.layers.44.mlp.experts.38.gate_proj", "model.layers.44.mlp.experts.39.gate_proj", "model.layers.44.mlp.experts.40.gate_proj", "model.layers.44.mlp.experts.41.gate_proj", "model.layers.44.mlp.experts.42.gate_proj", "model.layers.44.mlp.experts.43.gate_proj", "model.layers.44.mlp.experts.44.gate_proj", "model.layers.44.mlp.experts.45.gate_proj", "model.layers.44.mlp.experts.46.gate_proj", "model.layers.44.mlp.experts.47.gate_proj", "model.layers.44.mlp.experts.48.gate_proj", "model.layers.44.mlp.experts.49.gate_proj", "model.layers.44.mlp.experts.50.gate_proj", "model.layers.44.mlp.experts.51.gate_proj", "model.layers.44.mlp.experts.52.gate_proj", "model.layers.44.mlp.experts.53.gate_proj", "model.layers.44.mlp.experts.54.gate_proj", "model.layers.44.mlp.experts.55.gate_proj", "model.layers.44.mlp.experts.56.gate_proj", "model.layers.44.mlp.experts.57.gate_proj", "model.layers.44.mlp.experts.58.gate_proj", "model.layers.44.mlp.experts.59.gate_proj", "model.layers.44.mlp.experts.60.gate_proj", "model.layers.44.mlp.experts.61.gate_proj", "model.layers.44.mlp.experts.62.gate_proj", "model.layers.44.mlp.experts.63.gate_proj", "model.layers.44.mlp.experts.64.gate_proj", "model.layers.44.mlp.experts.65.gate_proj", "model.layers.44.mlp.experts.66.gate_proj", "model.layers.44.mlp.experts.67.gate_proj", "model.layers.44.mlp.experts.68.gate_proj", "model.layers.44.mlp.experts.69.gate_proj", "model.layers.44.mlp.experts.70.gate_proj", "model.layers.44.mlp.experts.71.gate_proj", "model.layers.44.mlp.experts.72.gate_proj", "model.layers.44.mlp.experts.73.gate_proj", "model.layers.44.mlp.experts.74.gate_proj", "model.layers.44.mlp.experts.75.gate_proj", "model.layers.44.mlp.experts.76.gate_proj", "model.layers.44.mlp.experts.77.gate_proj", "model.layers.44.mlp.experts.78.gate_proj", "model.layers.44.mlp.experts.79.gate_proj", "model.layers.44.mlp.experts.80.gate_proj", "model.layers.44.mlp.experts.81.gate_proj", "model.layers.44.mlp.experts.82.gate_proj", "model.layers.44.mlp.experts.83.gate_proj", "model.layers.44.mlp.experts.84.gate_proj", "model.layers.44.mlp.experts.85.gate_proj", "model.layers.44.mlp.experts.86.gate_proj", "model.layers.44.mlp.experts.87.gate_proj", "model.layers.44.mlp.experts.88.gate_proj", "model.layers.44.mlp.experts.89.gate_proj", "model.layers.44.mlp.experts.90.gate_proj", "model.layers.44.mlp.experts.91.gate_proj", "model.layers.44.mlp.experts.92.gate_proj", "model.layers.44.mlp.experts.93.gate_proj", "model.layers.44.mlp.experts.94.gate_proj", "model.layers.44.mlp.experts.95.gate_proj", "model.layers.44.mlp.experts.96.gate_proj", "model.layers.44.mlp.experts.97.gate_proj", "model.layers.44.mlp.experts.98.gate_proj", "model.layers.44.mlp.experts.99.gate_proj", "model.layers.44.mlp.experts.100.gate_proj", "model.layers.44.mlp.experts.101.gate_proj", "model.layers.44.mlp.experts.102.gate_proj", "model.layers.44.mlp.experts.103.gate_proj", "model.layers.44.mlp.experts.104.gate_proj", "model.layers.44.mlp.experts.105.gate_proj", "model.layers.44.mlp.experts.106.gate_proj", "model.layers.44.mlp.experts.107.gate_proj", "model.layers.44.mlp.experts.108.gate_proj", "model.layers.44.mlp.experts.109.gate_proj", "model.layers.44.mlp.experts.110.gate_proj", "model.layers.44.mlp.experts.111.gate_proj", "model.layers.44.mlp.experts.112.gate_proj", "model.layers.44.mlp.experts.113.gate_proj", "model.layers.44.mlp.experts.114.gate_proj", "model.layers.44.mlp.experts.115.gate_proj", "model.layers.44.mlp.experts.116.gate_proj", "model.layers.44.mlp.experts.117.gate_proj", "model.layers.44.mlp.experts.118.gate_proj", "model.layers.44.mlp.experts.119.gate_proj", "model.layers.44.mlp.experts.120.gate_proj", "model.layers.44.mlp.experts.121.gate_proj", "model.layers.44.mlp.experts.122.gate_proj", "model.layers.44.mlp.experts.123.gate_proj", "model.layers.44.mlp.experts.124.gate_proj", "model.layers.44.mlp.experts.125.gate_proj", "model.layers.44.mlp.experts.126.gate_proj", "model.layers.44.mlp.experts.127.gate_proj", "model.layers.44.mlp.experts.0.up_proj", "model.layers.44.mlp.experts.1.up_proj", "model.layers.44.mlp.experts.2.up_proj", "model.layers.44.mlp.experts.3.up_proj", "model.layers.44.mlp.experts.4.up_proj", "model.layers.44.mlp.experts.5.up_proj", "model.layers.44.mlp.experts.6.up_proj", "model.layers.44.mlp.experts.7.up_proj", "model.layers.44.mlp.experts.8.up_proj", "model.layers.44.mlp.experts.9.up_proj", "model.layers.44.mlp.experts.10.up_proj", "model.layers.44.mlp.experts.11.up_proj", "model.layers.44.mlp.experts.12.up_proj", "model.layers.44.mlp.experts.13.up_proj", "model.layers.44.mlp.experts.14.up_proj", "model.layers.44.mlp.experts.15.up_proj", "model.layers.44.mlp.experts.16.up_proj", "model.layers.44.mlp.experts.17.up_proj", "model.layers.44.mlp.experts.18.up_proj", "model.layers.44.mlp.experts.19.up_proj", "model.layers.44.mlp.experts.20.up_proj", "model.layers.44.mlp.experts.21.up_proj", "model.layers.44.mlp.experts.22.up_proj", "model.layers.44.mlp.experts.23.up_proj", "model.layers.44.mlp.experts.24.up_proj", "model.layers.44.mlp.experts.25.up_proj", "model.layers.44.mlp.experts.26.up_proj", "model.layers.44.mlp.experts.27.up_proj", "model.layers.44.mlp.experts.28.up_proj", "model.layers.44.mlp.experts.29.up_proj", "model.layers.44.mlp.experts.30.up_proj", "model.layers.44.mlp.experts.31.up_proj", "model.layers.44.mlp.experts.32.up_proj", "model.layers.44.mlp.experts.33.up_proj", "model.layers.44.mlp.experts.34.up_proj", "model.layers.44.mlp.experts.35.up_proj", "model.layers.44.mlp.experts.36.up_proj", "model.layers.44.mlp.experts.37.up_proj", "model.layers.44.mlp.experts.38.up_proj", "model.layers.44.mlp.experts.39.up_proj", "model.layers.44.mlp.experts.40.up_proj", "model.layers.44.mlp.experts.41.up_proj", "model.layers.44.mlp.experts.42.up_proj", "model.layers.44.mlp.experts.43.up_proj", "model.layers.44.mlp.experts.44.up_proj", "model.layers.44.mlp.experts.45.up_proj", "model.layers.44.mlp.experts.46.up_proj", "model.layers.44.mlp.experts.47.up_proj", "model.layers.44.mlp.experts.48.up_proj", "model.layers.44.mlp.experts.49.up_proj", "model.layers.44.mlp.experts.50.up_proj", "model.layers.44.mlp.experts.51.up_proj", "model.layers.44.mlp.experts.52.up_proj", "model.layers.44.mlp.experts.53.up_proj", "model.layers.44.mlp.experts.54.up_proj", "model.layers.44.mlp.experts.55.up_proj", "model.layers.44.mlp.experts.56.up_proj", "model.layers.44.mlp.experts.57.up_proj", "model.layers.44.mlp.experts.58.up_proj", "model.layers.44.mlp.experts.59.up_proj", "model.layers.44.mlp.experts.60.up_proj", "model.layers.44.mlp.experts.61.up_proj", "model.layers.44.mlp.experts.62.up_proj", "model.layers.44.mlp.experts.63.up_proj", "model.layers.44.mlp.experts.64.up_proj", "model.layers.44.mlp.experts.65.up_proj", "model.layers.44.mlp.experts.66.up_proj", "model.layers.44.mlp.experts.67.up_proj", "model.layers.44.mlp.experts.68.up_proj", "model.layers.44.mlp.experts.69.up_proj", "model.layers.44.mlp.experts.70.up_proj", "model.layers.44.mlp.experts.71.up_proj", "model.layers.44.mlp.experts.72.up_proj", "model.layers.44.mlp.experts.73.up_proj", "model.layers.44.mlp.experts.74.up_proj", "model.layers.44.mlp.experts.75.up_proj", "model.layers.44.mlp.experts.76.up_proj", "model.layers.44.mlp.experts.77.up_proj", "model.layers.44.mlp.experts.78.up_proj", "model.layers.44.mlp.experts.79.up_proj", "model.layers.44.mlp.experts.80.up_proj", "model.layers.44.mlp.experts.81.up_proj", "model.layers.44.mlp.experts.82.up_proj", "model.layers.44.mlp.experts.83.up_proj", "model.layers.44.mlp.experts.84.up_proj", "model.layers.44.mlp.experts.85.up_proj", "model.layers.44.mlp.experts.86.up_proj", "model.layers.44.mlp.experts.87.up_proj", "model.layers.44.mlp.experts.88.up_proj", "model.layers.44.mlp.experts.89.up_proj", "model.layers.44.mlp.experts.90.up_proj", "model.layers.44.mlp.experts.91.up_proj", "model.layers.44.mlp.experts.92.up_proj", "model.layers.44.mlp.experts.93.up_proj", "model.layers.44.mlp.experts.94.up_proj", "model.layers.44.mlp.experts.95.up_proj", "model.layers.44.mlp.experts.96.up_proj", "model.layers.44.mlp.experts.97.up_proj", "model.layers.44.mlp.experts.98.up_proj", "model.layers.44.mlp.experts.99.up_proj", "model.layers.44.mlp.experts.100.up_proj", "model.layers.44.mlp.experts.101.up_proj", "model.layers.44.mlp.experts.102.up_proj", "model.layers.44.mlp.experts.103.up_proj", "model.layers.44.mlp.experts.104.up_proj", "model.layers.44.mlp.experts.105.up_proj", "model.layers.44.mlp.experts.106.up_proj", "model.layers.44.mlp.experts.107.up_proj", "model.layers.44.mlp.experts.108.up_proj", "model.layers.44.mlp.experts.109.up_proj", "model.layers.44.mlp.experts.110.up_proj", "model.layers.44.mlp.experts.111.up_proj", "model.layers.44.mlp.experts.112.up_proj", "model.layers.44.mlp.experts.113.up_proj", "model.layers.44.mlp.experts.114.up_proj", "model.layers.44.mlp.experts.115.up_proj", "model.layers.44.mlp.experts.116.up_proj", "model.layers.44.mlp.experts.117.up_proj", "model.layers.44.mlp.experts.118.up_proj", "model.layers.44.mlp.experts.119.up_proj", "model.layers.44.mlp.experts.120.up_proj", "model.layers.44.mlp.experts.121.up_proj", "model.layers.44.mlp.experts.122.up_proj", "model.layers.44.mlp.experts.123.up_proj", "model.layers.44.mlp.experts.124.up_proj", "model.layers.44.mlp.experts.125.up_proj", "model.layers.44.mlp.experts.126.up_proj", "model.layers.44.mlp.experts.127.up_proj", "model.layers.44.mlp.experts.0.down_proj", "model.layers.44.mlp.experts.1.down_proj", "model.layers.44.mlp.experts.2.down_proj", "model.layers.44.mlp.experts.3.down_proj", "model.layers.44.mlp.experts.4.down_proj", "model.layers.44.mlp.experts.5.down_proj", "model.layers.44.mlp.experts.6.down_proj", "model.layers.44.mlp.experts.7.down_proj", "model.layers.44.mlp.experts.8.down_proj", "model.layers.44.mlp.experts.9.down_proj", "model.layers.44.mlp.experts.10.down_proj", "model.layers.44.mlp.experts.11.down_proj", "model.layers.44.mlp.experts.12.down_proj", "model.layers.44.mlp.experts.13.down_proj", "model.layers.44.mlp.experts.14.down_proj", "model.layers.44.mlp.experts.15.down_proj", "model.layers.44.mlp.experts.16.down_proj", "model.layers.44.mlp.experts.17.down_proj", "model.layers.44.mlp.experts.18.down_proj", "model.layers.44.mlp.experts.19.down_proj", "model.layers.44.mlp.experts.20.down_proj", "model.layers.44.mlp.experts.21.down_proj", "model.layers.44.mlp.experts.22.down_proj", "model.layers.44.mlp.experts.23.down_proj", "model.layers.44.mlp.experts.24.down_proj", "model.layers.44.mlp.experts.25.down_proj", "model.layers.44.mlp.experts.26.down_proj", "model.layers.44.mlp.experts.27.down_proj", "model.layers.44.mlp.experts.28.down_proj", "model.layers.44.mlp.experts.29.down_proj", "model.layers.44.mlp.experts.30.down_proj", "model.layers.44.mlp.experts.31.down_proj", "model.layers.44.mlp.experts.32.down_proj", "model.layers.44.mlp.experts.33.down_proj", "model.layers.44.mlp.experts.34.down_proj", "model.layers.44.mlp.experts.35.down_proj", "model.layers.44.mlp.experts.36.down_proj", "model.layers.44.mlp.experts.37.down_proj", "model.layers.44.mlp.experts.38.down_proj", "model.layers.44.mlp.experts.39.down_proj", "model.layers.44.mlp.experts.40.down_proj", "model.layers.44.mlp.experts.41.down_proj", "model.layers.44.mlp.experts.42.down_proj", "model.layers.44.mlp.experts.43.down_proj", "model.layers.44.mlp.experts.44.down_proj", "model.layers.44.mlp.experts.45.down_proj", "model.layers.44.mlp.experts.46.down_proj", "model.layers.44.mlp.experts.47.down_proj", "model.layers.44.mlp.experts.48.down_proj", "model.layers.44.mlp.experts.49.down_proj", "model.layers.44.mlp.experts.50.down_proj", "model.layers.44.mlp.experts.51.down_proj", "model.layers.44.mlp.experts.52.down_proj", "model.layers.44.mlp.experts.53.down_proj", "model.layers.44.mlp.experts.54.down_proj", "model.layers.44.mlp.experts.55.down_proj", "model.layers.44.mlp.experts.56.down_proj", "model.layers.44.mlp.experts.57.down_proj", "model.layers.44.mlp.experts.58.down_proj", "model.layers.44.mlp.experts.59.down_proj", "model.layers.44.mlp.experts.60.down_proj", "model.layers.44.mlp.experts.61.down_proj", "model.layers.44.mlp.experts.62.down_proj", "model.layers.44.mlp.experts.63.down_proj", "model.layers.44.mlp.experts.64.down_proj", "model.layers.44.mlp.experts.65.down_proj", "model.layers.44.mlp.experts.66.down_proj", "model.layers.44.mlp.experts.67.down_proj", "model.layers.44.mlp.experts.68.down_proj", "model.layers.44.mlp.experts.69.down_proj", "model.layers.44.mlp.experts.70.down_proj", "model.layers.44.mlp.experts.71.down_proj", "model.layers.44.mlp.experts.72.down_proj", "model.layers.44.mlp.experts.73.down_proj", "model.layers.44.mlp.experts.74.down_proj", "model.layers.44.mlp.experts.75.down_proj", "model.layers.44.mlp.experts.76.down_proj", "model.layers.44.mlp.experts.77.down_proj", "model.layers.44.mlp.experts.78.down_proj", "model.layers.44.mlp.experts.79.down_proj", "model.layers.44.mlp.experts.80.down_proj", "model.layers.44.mlp.experts.81.down_proj", "model.layers.44.mlp.experts.82.down_proj", "model.layers.44.mlp.experts.83.down_proj", "model.layers.44.mlp.experts.84.down_proj", "model.layers.44.mlp.experts.85.down_proj", "model.layers.44.mlp.experts.86.down_proj", "model.layers.44.mlp.experts.87.down_proj", "model.layers.44.mlp.experts.88.down_proj", "model.layers.44.mlp.experts.89.down_proj", "model.layers.44.mlp.experts.90.down_proj", "model.layers.44.mlp.experts.91.down_proj", "model.layers.44.mlp.experts.92.down_proj", "model.layers.44.mlp.experts.93.down_proj", "model.layers.44.mlp.experts.94.down_proj", "model.layers.44.mlp.experts.95.down_proj", "model.layers.44.mlp.experts.96.down_proj", "model.layers.44.mlp.experts.97.down_proj", "model.layers.44.mlp.experts.98.down_proj", "model.layers.44.mlp.experts.99.down_proj", "model.layers.44.mlp.experts.100.down_proj", "model.layers.44.mlp.experts.101.down_proj", "model.layers.44.mlp.experts.102.down_proj", "model.layers.44.mlp.experts.103.down_proj", "model.layers.44.mlp.experts.104.down_proj", "model.layers.44.mlp.experts.105.down_proj", "model.layers.44.mlp.experts.106.down_proj", "model.layers.44.mlp.experts.107.down_proj", "model.layers.44.mlp.experts.108.down_proj", "model.layers.44.mlp.experts.109.down_proj", "model.layers.44.mlp.experts.110.down_proj", "model.layers.44.mlp.experts.111.down_proj", "model.layers.44.mlp.experts.112.down_proj", "model.layers.44.mlp.experts.113.down_proj", "model.layers.44.mlp.experts.114.down_proj", "model.layers.44.mlp.experts.115.down_proj", "model.layers.44.mlp.experts.116.down_proj", "model.layers.44.mlp.experts.117.down_proj", "model.layers.44.mlp.experts.118.down_proj", "model.layers.44.mlp.experts.119.down_proj", "model.layers.44.mlp.experts.120.down_proj", "model.layers.44.mlp.experts.121.down_proj", "model.layers.44.mlp.experts.122.down_proj", "model.layers.44.mlp.experts.123.down_proj", "model.layers.44.mlp.experts.124.down_proj", "model.layers.44.mlp.experts.125.down_proj", "model.layers.44.mlp.experts.126.down_proj", "model.layers.44.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0008512580767273958, "dbits": 2214592512 } ] }, { "idx": 134, "layers": [ "model.layers.45.self_attn.q_proj", "model.layers.45.self_attn.k_proj", "model.layers.45.self_attn.v_proj", "model.layers.45.self_attn.o_proj" ], "candidates": [ { "dkld": 0.00020999256521464504, "dbits": 109051904 } ] }, { "idx": 135, "layers": [ "model.layers.45.mlp.shared_experts.gate_proj", "model.layers.45.mlp.shared_experts.up_proj", "model.layers.45.mlp.shared_experts.down_proj" ], "candidates": [ { "dkld": -0.0015674366615712476, "dbits": 17301504 } ] }, { "idx": 136, "layers": [ "model.layers.45.mlp.experts.0.gate_proj", "model.layers.45.mlp.experts.1.gate_proj", "model.layers.45.mlp.experts.2.gate_proj", "model.layers.45.mlp.experts.3.gate_proj", "model.layers.45.mlp.experts.4.gate_proj", "model.layers.45.mlp.experts.5.gate_proj", "model.layers.45.mlp.experts.6.gate_proj", "model.layers.45.mlp.experts.7.gate_proj", "model.layers.45.mlp.experts.8.gate_proj", "model.layers.45.mlp.experts.9.gate_proj", "model.layers.45.mlp.experts.10.gate_proj", "model.layers.45.mlp.experts.11.gate_proj", "model.layers.45.mlp.experts.12.gate_proj", "model.layers.45.mlp.experts.13.gate_proj", "model.layers.45.mlp.experts.14.gate_proj", "model.layers.45.mlp.experts.15.gate_proj", "model.layers.45.mlp.experts.16.gate_proj", "model.layers.45.mlp.experts.17.gate_proj", "model.layers.45.mlp.experts.18.gate_proj", "model.layers.45.mlp.experts.19.gate_proj", "model.layers.45.mlp.experts.20.gate_proj", "model.layers.45.mlp.experts.21.gate_proj", "model.layers.45.mlp.experts.22.gate_proj", "model.layers.45.mlp.experts.23.gate_proj", "model.layers.45.mlp.experts.24.gate_proj", "model.layers.45.mlp.experts.25.gate_proj", "model.layers.45.mlp.experts.26.gate_proj", "model.layers.45.mlp.experts.27.gate_proj", "model.layers.45.mlp.experts.28.gate_proj", "model.layers.45.mlp.experts.29.gate_proj", "model.layers.45.mlp.experts.30.gate_proj", "model.layers.45.mlp.experts.31.gate_proj", "model.layers.45.mlp.experts.32.gate_proj", "model.layers.45.mlp.experts.33.gate_proj", "model.layers.45.mlp.experts.34.gate_proj", "model.layers.45.mlp.experts.35.gate_proj", "model.layers.45.mlp.experts.36.gate_proj", "model.layers.45.mlp.experts.37.gate_proj", "model.layers.45.mlp.experts.38.gate_proj", "model.layers.45.mlp.experts.39.gate_proj", "model.layers.45.mlp.experts.40.gate_proj", "model.layers.45.mlp.experts.41.gate_proj", "model.layers.45.mlp.experts.42.gate_proj", "model.layers.45.mlp.experts.43.gate_proj", "model.layers.45.mlp.experts.44.gate_proj", "model.layers.45.mlp.experts.45.gate_proj", "model.layers.45.mlp.experts.46.gate_proj", "model.layers.45.mlp.experts.47.gate_proj", "model.layers.45.mlp.experts.48.gate_proj", "model.layers.45.mlp.experts.49.gate_proj", "model.layers.45.mlp.experts.50.gate_proj", "model.layers.45.mlp.experts.51.gate_proj", "model.layers.45.mlp.experts.52.gate_proj", "model.layers.45.mlp.experts.53.gate_proj", "model.layers.45.mlp.experts.54.gate_proj", "model.layers.45.mlp.experts.55.gate_proj", "model.layers.45.mlp.experts.56.gate_proj", "model.layers.45.mlp.experts.57.gate_proj", "model.layers.45.mlp.experts.58.gate_proj", "model.layers.45.mlp.experts.59.gate_proj", "model.layers.45.mlp.experts.60.gate_proj", "model.layers.45.mlp.experts.61.gate_proj", "model.layers.45.mlp.experts.62.gate_proj", "model.layers.45.mlp.experts.63.gate_proj", "model.layers.45.mlp.experts.64.gate_proj", "model.layers.45.mlp.experts.65.gate_proj", "model.layers.45.mlp.experts.66.gate_proj", "model.layers.45.mlp.experts.67.gate_proj", "model.layers.45.mlp.experts.68.gate_proj", "model.layers.45.mlp.experts.69.gate_proj", "model.layers.45.mlp.experts.70.gate_proj", "model.layers.45.mlp.experts.71.gate_proj", "model.layers.45.mlp.experts.72.gate_proj", "model.layers.45.mlp.experts.73.gate_proj", "model.layers.45.mlp.experts.74.gate_proj", "model.layers.45.mlp.experts.75.gate_proj", "model.layers.45.mlp.experts.76.gate_proj", "model.layers.45.mlp.experts.77.gate_proj", "model.layers.45.mlp.experts.78.gate_proj", "model.layers.45.mlp.experts.79.gate_proj", "model.layers.45.mlp.experts.80.gate_proj", "model.layers.45.mlp.experts.81.gate_proj", "model.layers.45.mlp.experts.82.gate_proj", "model.layers.45.mlp.experts.83.gate_proj", "model.layers.45.mlp.experts.84.gate_proj", "model.layers.45.mlp.experts.85.gate_proj", "model.layers.45.mlp.experts.86.gate_proj", "model.layers.45.mlp.experts.87.gate_proj", "model.layers.45.mlp.experts.88.gate_proj", "model.layers.45.mlp.experts.89.gate_proj", "model.layers.45.mlp.experts.90.gate_proj", "model.layers.45.mlp.experts.91.gate_proj", "model.layers.45.mlp.experts.92.gate_proj", "model.layers.45.mlp.experts.93.gate_proj", "model.layers.45.mlp.experts.94.gate_proj", "model.layers.45.mlp.experts.95.gate_proj", "model.layers.45.mlp.experts.96.gate_proj", "model.layers.45.mlp.experts.97.gate_proj", "model.layers.45.mlp.experts.98.gate_proj", "model.layers.45.mlp.experts.99.gate_proj", "model.layers.45.mlp.experts.100.gate_proj", "model.layers.45.mlp.experts.101.gate_proj", "model.layers.45.mlp.experts.102.gate_proj", "model.layers.45.mlp.experts.103.gate_proj", "model.layers.45.mlp.experts.104.gate_proj", "model.layers.45.mlp.experts.105.gate_proj", "model.layers.45.mlp.experts.106.gate_proj", "model.layers.45.mlp.experts.107.gate_proj", "model.layers.45.mlp.experts.108.gate_proj", "model.layers.45.mlp.experts.109.gate_proj", "model.layers.45.mlp.experts.110.gate_proj", "model.layers.45.mlp.experts.111.gate_proj", "model.layers.45.mlp.experts.112.gate_proj", "model.layers.45.mlp.experts.113.gate_proj", "model.layers.45.mlp.experts.114.gate_proj", "model.layers.45.mlp.experts.115.gate_proj", "model.layers.45.mlp.experts.116.gate_proj", "model.layers.45.mlp.experts.117.gate_proj", "model.layers.45.mlp.experts.118.gate_proj", "model.layers.45.mlp.experts.119.gate_proj", "model.layers.45.mlp.experts.120.gate_proj", "model.layers.45.mlp.experts.121.gate_proj", "model.layers.45.mlp.experts.122.gate_proj", "model.layers.45.mlp.experts.123.gate_proj", "model.layers.45.mlp.experts.124.gate_proj", "model.layers.45.mlp.experts.125.gate_proj", "model.layers.45.mlp.experts.126.gate_proj", "model.layers.45.mlp.experts.127.gate_proj", "model.layers.45.mlp.experts.0.up_proj", "model.layers.45.mlp.experts.1.up_proj", "model.layers.45.mlp.experts.2.up_proj", "model.layers.45.mlp.experts.3.up_proj", "model.layers.45.mlp.experts.4.up_proj", "model.layers.45.mlp.experts.5.up_proj", "model.layers.45.mlp.experts.6.up_proj", "model.layers.45.mlp.experts.7.up_proj", "model.layers.45.mlp.experts.8.up_proj", "model.layers.45.mlp.experts.9.up_proj", "model.layers.45.mlp.experts.10.up_proj", "model.layers.45.mlp.experts.11.up_proj", "model.layers.45.mlp.experts.12.up_proj", "model.layers.45.mlp.experts.13.up_proj", "model.layers.45.mlp.experts.14.up_proj", "model.layers.45.mlp.experts.15.up_proj", "model.layers.45.mlp.experts.16.up_proj", "model.layers.45.mlp.experts.17.up_proj", "model.layers.45.mlp.experts.18.up_proj", "model.layers.45.mlp.experts.19.up_proj", "model.layers.45.mlp.experts.20.up_proj", "model.layers.45.mlp.experts.21.up_proj", "model.layers.45.mlp.experts.22.up_proj", "model.layers.45.mlp.experts.23.up_proj", "model.layers.45.mlp.experts.24.up_proj", "model.layers.45.mlp.experts.25.up_proj", "model.layers.45.mlp.experts.26.up_proj", "model.layers.45.mlp.experts.27.up_proj", "model.layers.45.mlp.experts.28.up_proj", "model.layers.45.mlp.experts.29.up_proj", "model.layers.45.mlp.experts.30.up_proj", "model.layers.45.mlp.experts.31.up_proj", "model.layers.45.mlp.experts.32.up_proj", "model.layers.45.mlp.experts.33.up_proj", "model.layers.45.mlp.experts.34.up_proj", "model.layers.45.mlp.experts.35.up_proj", "model.layers.45.mlp.experts.36.up_proj", "model.layers.45.mlp.experts.37.up_proj", "model.layers.45.mlp.experts.38.up_proj", "model.layers.45.mlp.experts.39.up_proj", "model.layers.45.mlp.experts.40.up_proj", "model.layers.45.mlp.experts.41.up_proj", "model.layers.45.mlp.experts.42.up_proj", "model.layers.45.mlp.experts.43.up_proj", "model.layers.45.mlp.experts.44.up_proj", "model.layers.45.mlp.experts.45.up_proj", "model.layers.45.mlp.experts.46.up_proj", "model.layers.45.mlp.experts.47.up_proj", "model.layers.45.mlp.experts.48.up_proj", "model.layers.45.mlp.experts.49.up_proj", "model.layers.45.mlp.experts.50.up_proj", "model.layers.45.mlp.experts.51.up_proj", "model.layers.45.mlp.experts.52.up_proj", "model.layers.45.mlp.experts.53.up_proj", "model.layers.45.mlp.experts.54.up_proj", "model.layers.45.mlp.experts.55.up_proj", "model.layers.45.mlp.experts.56.up_proj", "model.layers.45.mlp.experts.57.up_proj", "model.layers.45.mlp.experts.58.up_proj", "model.layers.45.mlp.experts.59.up_proj", "model.layers.45.mlp.experts.60.up_proj", "model.layers.45.mlp.experts.61.up_proj", "model.layers.45.mlp.experts.62.up_proj", "model.layers.45.mlp.experts.63.up_proj", "model.layers.45.mlp.experts.64.up_proj", "model.layers.45.mlp.experts.65.up_proj", "model.layers.45.mlp.experts.66.up_proj", "model.layers.45.mlp.experts.67.up_proj", "model.layers.45.mlp.experts.68.up_proj", "model.layers.45.mlp.experts.69.up_proj", "model.layers.45.mlp.experts.70.up_proj", "model.layers.45.mlp.experts.71.up_proj", "model.layers.45.mlp.experts.72.up_proj", "model.layers.45.mlp.experts.73.up_proj", "model.layers.45.mlp.experts.74.up_proj", "model.layers.45.mlp.experts.75.up_proj", "model.layers.45.mlp.experts.76.up_proj", "model.layers.45.mlp.experts.77.up_proj", "model.layers.45.mlp.experts.78.up_proj", "model.layers.45.mlp.experts.79.up_proj", "model.layers.45.mlp.experts.80.up_proj", "model.layers.45.mlp.experts.81.up_proj", "model.layers.45.mlp.experts.82.up_proj", "model.layers.45.mlp.experts.83.up_proj", "model.layers.45.mlp.experts.84.up_proj", "model.layers.45.mlp.experts.85.up_proj", "model.layers.45.mlp.experts.86.up_proj", "model.layers.45.mlp.experts.87.up_proj", "model.layers.45.mlp.experts.88.up_proj", "model.layers.45.mlp.experts.89.up_proj", "model.layers.45.mlp.experts.90.up_proj", "model.layers.45.mlp.experts.91.up_proj", "model.layers.45.mlp.experts.92.up_proj", "model.layers.45.mlp.experts.93.up_proj", "model.layers.45.mlp.experts.94.up_proj", "model.layers.45.mlp.experts.95.up_proj", "model.layers.45.mlp.experts.96.up_proj", "model.layers.45.mlp.experts.97.up_proj", "model.layers.45.mlp.experts.98.up_proj", "model.layers.45.mlp.experts.99.up_proj", "model.layers.45.mlp.experts.100.up_proj", "model.layers.45.mlp.experts.101.up_proj", "model.layers.45.mlp.experts.102.up_proj", "model.layers.45.mlp.experts.103.up_proj", "model.layers.45.mlp.experts.104.up_proj", "model.layers.45.mlp.experts.105.up_proj", "model.layers.45.mlp.experts.106.up_proj", "model.layers.45.mlp.experts.107.up_proj", "model.layers.45.mlp.experts.108.up_proj", "model.layers.45.mlp.experts.109.up_proj", "model.layers.45.mlp.experts.110.up_proj", "model.layers.45.mlp.experts.111.up_proj", "model.layers.45.mlp.experts.112.up_proj", "model.layers.45.mlp.experts.113.up_proj", "model.layers.45.mlp.experts.114.up_proj", "model.layers.45.mlp.experts.115.up_proj", "model.layers.45.mlp.experts.116.up_proj", "model.layers.45.mlp.experts.117.up_proj", "model.layers.45.mlp.experts.118.up_proj", "model.layers.45.mlp.experts.119.up_proj", "model.layers.45.mlp.experts.120.up_proj", "model.layers.45.mlp.experts.121.up_proj", "model.layers.45.mlp.experts.122.up_proj", "model.layers.45.mlp.experts.123.up_proj", "model.layers.45.mlp.experts.124.up_proj", "model.layers.45.mlp.experts.125.up_proj", "model.layers.45.mlp.experts.126.up_proj", "model.layers.45.mlp.experts.127.up_proj", "model.layers.45.mlp.experts.0.down_proj", "model.layers.45.mlp.experts.1.down_proj", "model.layers.45.mlp.experts.2.down_proj", "model.layers.45.mlp.experts.3.down_proj", "model.layers.45.mlp.experts.4.down_proj", "model.layers.45.mlp.experts.5.down_proj", "model.layers.45.mlp.experts.6.down_proj", "model.layers.45.mlp.experts.7.down_proj", "model.layers.45.mlp.experts.8.down_proj", "model.layers.45.mlp.experts.9.down_proj", "model.layers.45.mlp.experts.10.down_proj", "model.layers.45.mlp.experts.11.down_proj", "model.layers.45.mlp.experts.12.down_proj", "model.layers.45.mlp.experts.13.down_proj", "model.layers.45.mlp.experts.14.down_proj", "model.layers.45.mlp.experts.15.down_proj", "model.layers.45.mlp.experts.16.down_proj", "model.layers.45.mlp.experts.17.down_proj", "model.layers.45.mlp.experts.18.down_proj", "model.layers.45.mlp.experts.19.down_proj", "model.layers.45.mlp.experts.20.down_proj", "model.layers.45.mlp.experts.21.down_proj", "model.layers.45.mlp.experts.22.down_proj", "model.layers.45.mlp.experts.23.down_proj", "model.layers.45.mlp.experts.24.down_proj", "model.layers.45.mlp.experts.25.down_proj", "model.layers.45.mlp.experts.26.down_proj", "model.layers.45.mlp.experts.27.down_proj", "model.layers.45.mlp.experts.28.down_proj", "model.layers.45.mlp.experts.29.down_proj", "model.layers.45.mlp.experts.30.down_proj", "model.layers.45.mlp.experts.31.down_proj", "model.layers.45.mlp.experts.32.down_proj", "model.layers.45.mlp.experts.33.down_proj", "model.layers.45.mlp.experts.34.down_proj", "model.layers.45.mlp.experts.35.down_proj", "model.layers.45.mlp.experts.36.down_proj", "model.layers.45.mlp.experts.37.down_proj", "model.layers.45.mlp.experts.38.down_proj", "model.layers.45.mlp.experts.39.down_proj", "model.layers.45.mlp.experts.40.down_proj", "model.layers.45.mlp.experts.41.down_proj", "model.layers.45.mlp.experts.42.down_proj", "model.layers.45.mlp.experts.43.down_proj", "model.layers.45.mlp.experts.44.down_proj", "model.layers.45.mlp.experts.45.down_proj", "model.layers.45.mlp.experts.46.down_proj", "model.layers.45.mlp.experts.47.down_proj", "model.layers.45.mlp.experts.48.down_proj", "model.layers.45.mlp.experts.49.down_proj", "model.layers.45.mlp.experts.50.down_proj", "model.layers.45.mlp.experts.51.down_proj", "model.layers.45.mlp.experts.52.down_proj", "model.layers.45.mlp.experts.53.down_proj", "model.layers.45.mlp.experts.54.down_proj", "model.layers.45.mlp.experts.55.down_proj", "model.layers.45.mlp.experts.56.down_proj", "model.layers.45.mlp.experts.57.down_proj", "model.layers.45.mlp.experts.58.down_proj", "model.layers.45.mlp.experts.59.down_proj", "model.layers.45.mlp.experts.60.down_proj", "model.layers.45.mlp.experts.61.down_proj", "model.layers.45.mlp.experts.62.down_proj", "model.layers.45.mlp.experts.63.down_proj", "model.layers.45.mlp.experts.64.down_proj", "model.layers.45.mlp.experts.65.down_proj", "model.layers.45.mlp.experts.66.down_proj", "model.layers.45.mlp.experts.67.down_proj", "model.layers.45.mlp.experts.68.down_proj", "model.layers.45.mlp.experts.69.down_proj", "model.layers.45.mlp.experts.70.down_proj", "model.layers.45.mlp.experts.71.down_proj", "model.layers.45.mlp.experts.72.down_proj", "model.layers.45.mlp.experts.73.down_proj", "model.layers.45.mlp.experts.74.down_proj", "model.layers.45.mlp.experts.75.down_proj", "model.layers.45.mlp.experts.76.down_proj", "model.layers.45.mlp.experts.77.down_proj", "model.layers.45.mlp.experts.78.down_proj", "model.layers.45.mlp.experts.79.down_proj", "model.layers.45.mlp.experts.80.down_proj", "model.layers.45.mlp.experts.81.down_proj", "model.layers.45.mlp.experts.82.down_proj", "model.layers.45.mlp.experts.83.down_proj", "model.layers.45.mlp.experts.84.down_proj", "model.layers.45.mlp.experts.85.down_proj", "model.layers.45.mlp.experts.86.down_proj", "model.layers.45.mlp.experts.87.down_proj", "model.layers.45.mlp.experts.88.down_proj", "model.layers.45.mlp.experts.89.down_proj", "model.layers.45.mlp.experts.90.down_proj", "model.layers.45.mlp.experts.91.down_proj", "model.layers.45.mlp.experts.92.down_proj", "model.layers.45.mlp.experts.93.down_proj", "model.layers.45.mlp.experts.94.down_proj", "model.layers.45.mlp.experts.95.down_proj", "model.layers.45.mlp.experts.96.down_proj", "model.layers.45.mlp.experts.97.down_proj", "model.layers.45.mlp.experts.98.down_proj", "model.layers.45.mlp.experts.99.down_proj", "model.layers.45.mlp.experts.100.down_proj", "model.layers.45.mlp.experts.101.down_proj", "model.layers.45.mlp.experts.102.down_proj", "model.layers.45.mlp.experts.103.down_proj", "model.layers.45.mlp.experts.104.down_proj", "model.layers.45.mlp.experts.105.down_proj", "model.layers.45.mlp.experts.106.down_proj", "model.layers.45.mlp.experts.107.down_proj", "model.layers.45.mlp.experts.108.down_proj", "model.layers.45.mlp.experts.109.down_proj", "model.layers.45.mlp.experts.110.down_proj", "model.layers.45.mlp.experts.111.down_proj", "model.layers.45.mlp.experts.112.down_proj", "model.layers.45.mlp.experts.113.down_proj", "model.layers.45.mlp.experts.114.down_proj", "model.layers.45.mlp.experts.115.down_proj", "model.layers.45.mlp.experts.116.down_proj", "model.layers.45.mlp.experts.117.down_proj", "model.layers.45.mlp.experts.118.down_proj", "model.layers.45.mlp.experts.119.down_proj", "model.layers.45.mlp.experts.120.down_proj", "model.layers.45.mlp.experts.121.down_proj", "model.layers.45.mlp.experts.122.down_proj", "model.layers.45.mlp.experts.123.down_proj", "model.layers.45.mlp.experts.124.down_proj", "model.layers.45.mlp.experts.125.down_proj", "model.layers.45.mlp.experts.126.down_proj", "model.layers.45.mlp.experts.127.down_proj" ], "candidates": [ { "dkld": -0.0009057318791746982, "dbits": 2214592512 } ] } ], "base_kld": 0.21893304968252777, "arch_string": "Glm4MoeForCausalLM" }