add perplexity

app.py

@@ -246,7 +246,7 @@ with demo:
                 interactive=False,
                 visible=True,
                 # column_widths=["2%", "33%"]
-                height=900
+                height=800
             )
 
             # Dummy leaderboard for handling the case when the user uses backspace key

src/about.py

@@ -20,6 +20,7 @@ class Tasks(Enum):
     task6 = Task("polemo2_out_multiple_choice", "acc,none", "polemo2-out_mc", "multiple_choice")
     task7 = Task("polish_8tags_multiple_choice", "acc,none", "8tags_mc", "multiple_choice")
     task8 = Task("polish_8tags_regex", "exact_match,score-first", "8tags_g", "generate_until")
+    #task9a = Task("polish_belebele_mc", "acc,none", "belebele_mc", "multiple_choice")
     task9 = Task("polish_belebele_regex", "exact_match,score-first", "belebele_g", "generate_until")
     task10 = Task("polish_dyk_multiple_choice", "f1,none", "dyk_mc", "multiple_choice")
     task11 = Task("polish_dyk_regex", "f1,score-first", "dyk_g", "generate_until")
@@ -31,6 +32,7 @@ class Tasks(Enum):
     task17 = Task("polish_cbd_regex", "f1,score-first", "cbd_g", "generate_until")
     task18 = Task("polish_klej_ner_multiple_choice", "acc,none", "klej_ner_mc", "multiple_choice")
     task19 = Task("polish_klej_ner_regex", "exact_match,score-first", "klej_ner_g", "generate_until")
+    task20 = Task("polish_poleval2018_task3_test_10k", "word_perplexity,none", "polish_poleval2018_task3_test_10k", "other")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
@@ -72,6 +74,7 @@ or join our [Discord SpeakLeash](https://discord.gg/3G9DVM39)
 * add metadata for models (e.g. #Params)
 * add more tasks
 * use model templates
+* fix scrolling on Firefox
 
 ## Tasks
 

src/display/utils.py

@@ -26,6 +26,7 @@ auto_eval_column_dict = []
 # Init
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["lang", ColumnContent, ColumnContent("Lang", "str", True)])
 auto_eval_column_dict.append(["n_shot", ColumnContent, ColumnContent("n_shot", "str", True)])
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
@@ -39,7 +40,6 @@ auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Arch
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["lang", ColumnContent, ColumnContent("Lang", "str", True)])
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", True)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])

src/leaderboard/read_evals.py

@@ -33,6 +33,7 @@ class EvalResult:
     date: str = "" # submission date of request file
     still_on_hub: bool = False
     n_shot: NShotType = NShotType.n0
+    org_and_model: str = ""
 
     @classmethod
     def init_from_json_file(self, json_filepath, n_shot_num):
@@ -48,6 +49,7 @@ class EvalResult:
 
         # Get model and org
         org_and_model = config.get("model_name", config.get("model_args", None))
+        orig_org_and_model = org_and_model
         SPICHLERZ_ORG = "speakleash/"
 
         if re.match(r"^pretrained=(.*/(plgkwrobel|plggspkl)/)(models/)?", org_and_model):
@@ -91,7 +93,10 @@ class EvalResult:
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
-            mean_acc = np.mean(accs) * 100.0
+            if 'perplexity' in task.metric:
+                mean_acc = np.mean(accs)
+            else:
+                mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc
 
         return self(
@@ -104,7 +109,8 @@ class EvalResult:
             revision= config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture,
-            n_shot=NShotType.from_str(n_shot_num)
+            n_shot=NShotType.from_str(n_shot_num),
+            org_and_model=orig_org_and_model
         )
 
     def update_with_metadata(self, metadata):
@@ -139,10 +145,10 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         g_tasks = [task.value.benchmark for task in Tasks if task.value.type == "generate_until"]
         mc_tasks = [task.value.benchmark for task in Tasks if task.value.type == "multiple_choice"]
-
+        all_tasks = g_tasks + mc_tasks
+        average = sum([v for task,v in self.results.items() if v is not None and task in all_tasks]) / len(all_tasks)
         average_g = sum([v for task,v in self.results.items() if v is not None and task in g_tasks]) / len(g_tasks)
         average_mc = sum([v for task,v in self.results.items() if v is not None and task in mc_tasks]) / len(mc_tasks)
 
@@ -352,4 +358,21 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
             print(f"not all eval values present {v.eval_name} {v.full_model}")
             continue
 
+    missing_results_for_task = {}
+    for v in eval_results.values():
+        r = v.to_dict()
+        for task in Tasks:
+            if r[task.value.col_name] is None:
+                task_name = f"{r['n_shot']}|{task.value.benchmark}"
+                if task_name in missing_results_for_task:
+                    missing_results_for_task[task_name].append(f"{v.full_model}|{v.org_and_model}")
+                else:
+                    missing_results_for_task[task_name] = [f"{v.full_model}|{v.org_and_model}"]
+
+    # print('missing_results_for_task', missing_results_for_task)
+    for task, models in missing_results_for_task.items():
+        print(f"Missing results for {task} for {len(models)} models")
+        print(" ".join(models))
+
+
     return results