Add files

.gitattributes

@@ -30,3 +30,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step1750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step2250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step2750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+xp3capmixlossseq_global_step1250/tokenizer.json filter=lfs diff=lfs merge=lfs -text

xp3capmixlossseq_global_step1000/config.json

@@ -0,0 +1,30 @@
+{
+  "apply_residual_connection_post_layernorm": false,
+  "attention_dropout": 0.0,
+  "attention_softmax_in_fp32": true,
+  "bias_dropout_fusion": true,
+  "bos_token_id": 1,
+  "architectures": [
+    "BloomModel"
+  ],
+  "eos_token_id": 2,
+  "pad_token_id": 3,
+  "unk_token_id": 0,
+  "hidden_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "masked_softmax_fusion": true,
+  "model_type": "bloom",
+  "n_embed": 1536,
+  "n_inner": null,
+  "n_layer": 24,
+  "num_attention_heads": 16,
+  "offset_alibi": 100,
+  "pretraining_tp": 1,
+  "seq_length": 2048,
+  "skip_bias_add": true,
+  "skip_bias_add_qkv": false,
+  "transformers_version": "4.20.0",
+  "use_cache": true,
+  "vocab_size": 250880
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/en/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4378
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/en/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.3906
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/en/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "en",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.2342
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/es/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.3964
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/es/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.3776
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/es/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "es",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.2242
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='es', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/fr/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.4174
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/fr/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.4
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/fr/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "fr",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.2478
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/zh/prompt_body_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_body_title_to_star",
+  "evaluation": {
+    "accuracy": 0.2882
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_body_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/zh/prompt_review_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_review_to_star",
+  "evaluation": {
+    "accuracy": 0.2864
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_review_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/amazon_reviews_multi/zh/prompt_title_to_star/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "amazon_reviews_multi",
+  "dataset_config_name": "zh",
+  "template_name": "prompt_title_to_star",
+  "evaluation": {
+    "accuracy": 0.2104
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='amazon_reviews_multi', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='prompt_title_to_star', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/aqua_rat/raw/Answer_questions_from_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "Answer questions from options",
+  "evaluation": {
+    "accuracy": 0.23228346456692914
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='Answer questions from options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/aqua_rat/raw/answer_quiz/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "answer_quiz",
+  "evaluation": {
+    "accuracy": 0.2440944881889764
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_quiz', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/aqua_rat/raw/select_the_best_option/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "aqua_rat",
+  "dataset_config_name": "raw",
+  "template_name": "select_the_best_option",
+  "evaluation": {
+    "accuracy": 0.24015748031496062
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='raw', dataset_name='aqua_rat', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='select_the_best_option', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/art/choose_hypothesis/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis",
+  "evaluation": {
+    "accuracy": 0.5110966057441253
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/art/choose_hypothesis_believable/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_believable",
+  "evaluation": {
+    "accuracy": 0.5117493472584856
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_believable', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/art/choose_hypothesis_desc/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_desc",
+  "evaluation": {
+    "accuracy": 0.5215404699738904
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_desc', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/art/choose_hypothesis_likely/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_likely",
+  "evaluation": {
+    "accuracy": 0.49477806788511747
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_likely', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/art/choose_hypothesis_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "art",
+  "dataset_config_name": null,
+  "template_name": "choose_hypothesis_options",
+  "evaluation": {
+    "accuracy": 0.5130548302872062
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='art', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='choose_hypothesis_options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/banking77/direct_to_which_department/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "direct_to_which_department",
+  "evaluation": {
+    "accuracy": 0.17012987012987013
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='direct_to_which_department', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/banking77/help_page_topic/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "help_page_topic",
+  "evaluation": {
+    "accuracy": 0.16525974025974027
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='help_page_topic', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/banking77/rephrase_as_banking_term/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "banking77",
+  "dataset_config_name": null,
+  "template_name": "rephrase_as_banking_term",
+  "evaluation": {
+    "accuracy": 0.22337662337662337
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='banking77', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='rephrase_as_banking_term', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blbooksgenre/title_genre_classifiction/classify/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "classify",
+  "evaluation": {
+    "accuracy": 0.2528801843317972
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='classify', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blbooksgenre/title_genre_classifiction/multi-choice/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "multi-choice",
+  "evaluation": {
+    "accuracy": 0.25057603686635943
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='multi-choice', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blbooksgenre/title_genre_classifiction/premise_context_first/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blbooksgenre",
+  "dataset_config_name": "title_genre_classifiction",
+  "template_name": "premise_context_first",
+  "evaluation": {
+    "accuracy": 0.7206221198156681
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='title_genre_classifiction', dataset_name='blbooksgenre', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='premise_context_first', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blimp/adjunct_island/grammatical_between_1_2/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_between_1_2",
+  "evaluation": {
+    "accuracy": 0.621
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_1_2', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blimp/adjunct_island/grammatical_between_A_B/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_between_A_B",
+  "evaluation": {
+    "accuracy": 0.595
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_between_A_B', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blimp/adjunct_island/grammatical_which_one_1_2/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "grammatical_which_one_1_2",
+  "evaluation": {
+    "accuracy": 0.709
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='grammatical_which_one_1_2', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blimp/adjunct_island/single_sentence_bad_yes_no/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "single_sentence_bad_yes_no",
+  "evaluation": {
+    "accuracy": 0.511
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_bad_yes_no', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/blimp/adjunct_island/single_sentence_good_yes_no/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "blimp",
+  "dataset_config_name": "adjunct_island",
+  "template_name": "single_sentence_good_yes_no",
+  "evaluation": {
+    "accuracy": 0.496
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='adjunct_island', dataset_name='blimp', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='single_sentence_good_yes_no', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/climate_fever/claim_and_all_supporting_evidences/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "claim_and_all_supporting_evidences",
+  "evaluation": {
+    "accuracy": 0.31009771986970686
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='claim_and_all_supporting_evidences', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/climate_fever/fifth_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "fifth_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.11596091205211727
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='fifth_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/climate_fever/first_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "first_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.09771986970684039
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='first_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/climate_fever/second_evidence_and_claim_itemization/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "second_evidence_and_claim_itemization",
+  "evaluation": {
+    "accuracy": 0.10618892508143322
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='second_evidence_and_claim_itemization', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/climate_fever/third_evidence_claim_pair/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "climate_fever",
+  "dataset_config_name": null,
+  "template_name": "third_evidence_claim_pair",
+  "evaluation": {
+    "accuracy": 0.098371335504886
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='climate_fever', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='test', target_max_length=256, template_config_name=None, template_name='third_evidence_claim_pair', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/codah/codah/affirmative_instruction_after_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "affirmative_instruction_after_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.32096541786743515
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/codah/codah/affirmative_instruction_before_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "affirmative_instruction_before_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.3346541786743516
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='affirmative_instruction_before_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/codah/codah/interrogative_instruction_after_sentence_and_choices/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "codah",
+  "dataset_config_name": "codah",
+  "template_name": "interrogative_instruction_after_sentence_and_choices",
+  "evaluation": {
+    "accuracy": 0.32672910662824206
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='codah', dataset_name='codah', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='train', target_max_length=256, template_config_name=None, template_name='interrogative_instruction_after_sentence_and_choices', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/commonsense_qa/answer_given_question_without_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "answer_given_question_without_options",
+  "evaluation": {
+    "accuracy": 0.47092547092547093
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='answer_given_question_without_options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/commonsense_qa/most_suitable_answer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "most_suitable_answer",
+  "evaluation": {
+    "accuracy": 0.5192465192465192
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='most_suitable_answer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/commonsense_qa/question_answering/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "commonsense_qa",
+  "dataset_config_name": null,
+  "template_name": "question_answering",
+  "evaluation": {
+    "accuracy": 0.5004095004095004
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='commonsense_qa', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='question_answering', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/conv_ai_3/ambiguous/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "ambiguous",
+  "evaluation": {
+    "accuracy": 0.39040207522697795
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='ambiguous', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/conv_ai_3/clarification_needed/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "clarification_needed",
+  "evaluation": {
+    "accuracy": 0.41461305663640297
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='clarification_needed', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/conv_ai_3/directly_answer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "directly_answer",
+  "evaluation": {
+    "accuracy": 0.6095979247730221
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='directly_answer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/conv_ai_3/score_give_number/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "score_give_number",
+  "evaluation": {
+    "accuracy": 0.12926934716817987
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_give_number', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/conv_ai_3/score_how_much/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "conv_ai_3",
+  "dataset_config_name": null,
+  "template_name": "score_how_much",
+  "evaluation": {
+    "accuracy": 0.21444012105490703
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='conv_ai_3', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='score_how_much', tokenizer_name=None, use_slow_tokenizer=False)"
+}

xp3capmixlossseq_global_step1000/evaluation/craigslist_bargains/best_deal/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "craigslist_bargains",
+  "dataset_config_name": null,
+  "template_name": "best deal",
+  "evaluation": {
+    "accuracy": 0.5008375209380235
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name=None, dataset_name='craigslist_bargains', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000', output_dir='/gpfsscratch/rech/six/commun/commun/experiments/muennighoff/bloomckpt/760mt0/xp3capmixlossseq_global_step1000/evaluation', pad_to_max_length=False, per_device_eval_batch_size=4, prefixlm=False, split='validation', target_max_length=256, template_config_name=None, template_name='best deal', tokenizer_name=None, use_slow_tokenizer=False)"
+}