change 'proprietary' models to 'external' models and added news models

proprietary_models_results.json → external_models_results.json

@@ -6,6 +6,7 @@
         "date": "2024-04-12",
         "status": "full",
         "main_language": "Portuguese",
+        "model_type": "proprietary",
         "result_metrics": {
             "enem_challenge": 0.7172848145556333,
             "bluex": 0.5549374130737135,
@@ -27,6 +28,7 @@
         "date": "2024-04-13",
         "status": "full",
         "main_language": "Portuguese",
+        "model_type": "proprietary",
         "result_metrics": {
             "enem_challenge": 0.8180545836249126,
             "bluex": 0.717663421418637,
@@ -48,6 +50,7 @@
         "date": "2024-03-08",
         "status": "full",
         "main_language": "English",
+        "model_type": "proprietary",
         "result_metrics": {
             "enem_challenge": 0.7214835549335199,
             "bluex": 0.6244784422809457,
@@ -69,6 +72,7 @@
         "date": "2024-04-13",
         "status": "full",
         "main_language": "English",
+        "model_type": "proprietary",
         "result_metrics": {
             "enem_challenge": 0.7718684394681595,
             "bluex": 0.6662030598052852,
@@ -90,6 +94,7 @@
         "date": "2024-03-08",
         "status": "full",
         "main_language": "English",
+        "model_type": "proprietary",
         "result_metrics": {
             "enem_challenge": 0.7130860741777467,
             "bluex": 0.5869262865090403,
@@ -111,6 +116,7 @@
         "date": "2024-04-15",
         "status": "full",
         "main_language": "English",
+        "model_type": "proprietary",
         "result_metrics": {
             "enem_challenge": 0.8509447165850245,
             "bluex": 0.7719054242002782,
@@ -132,6 +138,7 @@
         "date": "2024-05-18",
         "status": "full",
         "main_language": "English",
+        "model_type": "proprietary",
         "result_metrics": {
             "enem_challenge": 0.7844646606018194,
             "bluex": 0.6954102920723226,
@@ -153,6 +160,7 @@
         "date": "2024-05-18",
         "status": "full",
         "main_language": "English",
+        "model_type": "proprietary",
         "result_metrics": {
             "enem_challenge": 0.8264520643806857,
             "bluex": 0.7482614742698191,
@@ -166,5 +174,72 @@
         },
         "result_metrics_average": 0.7914657682594597,
         "result_metrics_npm": 0.6834036936130392
+    },
+    {
+        "model": "gemini-1.5-flash",
+        "name": "Gemini 1.5 Flash",
+        "link": "https://cloud.google.com/vertex-ai",
+        "date": "2024-08-09",
+        "status": "full",
+        "main_language": "English",
+        "model_type": "proprietary",
+        "result_metrics": {
+            "enem_challenge": 0.8306508047585724,
+            "bluex": 0.7579972183588317,
+            "oab_exams": 0.6446469248291572,
+            "assin2_sts": 0.838806085610371,
+            "assin2_rte": 0.9366169973822607,
+            "faquad_nli": 0.7963910785668922,
+            "hatebr_offensive": 0.9092078461170015,
+            "portuguese_hate_speech": 0.6932563987219857,
+            "tweetsentbr": 0.7312948963367732
+        },
+        "result_metrics_average": 0.7932075834090939,
+        "result_metrics_npm": 0.6855338135928848
+    },
+    {
+        "model": "gpt-4o-mini-2024-07-18",
+        "name": "GPT 4o Mini (2024-07-18)",
+        "link": "https://www.openai.com/",
+        "date": "2024-07-25",
+        "status": "full",
+        "main_language": "English",
+        "model_type": "proprietary",
+        "result_metrics": {
+            "enem_challenge": 0.7669699090272918,
+            "bluex": 0.6842837273991655,
+            "oab_exams": 0.6013667425968109,
+            "assin2_sts": 0.7259038954527597,
+            "assin2_rte": 0.942809846745341,
+            "faquad_nli": 0.819807735300693,
+            "hatebr_offensive": 0.8682357029532165,
+            "portuguese_hate_speech": 0.7501413502853012,
+            "tweetsentbr": 0.7509303825869922
+        },
+        "result_metrics_average": 0.7678276991497301,
+        "result_metrics_npm": 0.6595966999910003
+    },
+    {
+        "model": "nemotron-4-340b-instruct",
+        "name": "nvidia/Nemotron-4-340B-Instruct (Nvidia API)",
+        "link": "https://build.nvidia.com/nvidia/nemotron-4-340b-instruct",
+        "date": "2024-06-30",
+        "status": "full",
+        "main_language": "English",
+        "model_type": "chat",
+        "params": 340.0,
+        "result_metrics": {
+            "enem_challenge": 0.6648005598320503,
+            "bluex": 0.6578581363004172,
+            "oab_exams": 0.7020501138952164,
+            "assin2_sts": 0.7857731021403329,
+            "assin2_rte": 0.9489354458928496,
+            "faquad_nli": 0.8194444444444444,
+            "hatebr_offensive": 0.8641580001234928,
+            "portuguese_hate_speech": 0.7761835184102864,
+            "tweetsentbr": 0.780880021326841
+        },
+        "result_metrics_average": 0.7777870380406591,
+        "result_metrics_npm": 0.6740728488043128
     }
 ]

src/display/utils.py

@@ -166,24 +166,30 @@ human_baseline_row[AutoEvalColumn.npm.name] = round(sum(npm) / len(npm), 2)
 if GET_ORIGINAL_HF_LEADERBOARD_EVAL_RESULTS:
     human_baseline_row["🤗 Leaderboard Average"] = None
 
-#Proprietary models
-proprietary_rows = []
-if os.path.exists('proprietary_models_results.json'):
-    with open('proprietary_models_results.json', 'r', encoding='utf8') as f:
+#External models
+external_rows = []
+if os.path.exists('external_models_results.json'):
+    with open('external_models_results.json', 'r', encoding='utf8') as f:
         all_models = json.load(f)
     for model_data in all_models:
         model_row = deepcopy(baseline_row)
         model_row[AutoEvalColumn.model.name] = f'<a target="_blank" href="{model_data["link"]}" style="color: var(--text-color); text-decoration: underline;text-decoration-style: dotted;">{model_data["name"]} [{model_data["date"]}]</a>'
         model_row[AutoEvalColumn.dummy.name] = model_data['model']
-        model_row[AutoEvalColumn.license.name] = "Proprietary"
         for task in Tasks:
             model_row[task.value.col_name] = round(model_data['result_metrics'][task.value.benchmark]*100, 2)
         model_row[AutoEvalColumn.average.name] = round(model_data['result_metrics_average']*100, 2)
         model_row[AutoEvalColumn.npm.name] = round(model_data['result_metrics_npm']*100, 2)
-        model_row[AutoEvalColumn.model_type.name] = "proprietary models (closed)"
-        model_row[AutoEvalColumn.model_type_symbol.name] = "🔒"
+        
+        model_type = ModelType.from_str(model_data['model_type'])
+        model_row[AutoEvalColumn.model_type.name] = model_type.name
+        model_row[AutoEvalColumn.model_type_symbol.name] = model_type.symbol
+        if model_type == ModelType.proprietary:
+            model_row[AutoEvalColumn.license.name] = "Proprietary"
+        if 'params' in model_data:
+            model_row[AutoEvalColumn.params.name] = model_data['params']
+
         model_row[AutoEvalColumn.main_language.name] = model_data['main_language']
-        proprietary_rows.append(model_row)
+        external_rows.append(model_row)
 
 @dataclass
 class ModelDetails:

src/populate.py

@@ -5,7 +5,7 @@ import copy
 import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_requests_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn, baseline_row, proprietary_rows
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, baseline_row, external_rows
 from src.leaderboard.filter_models import filter_models_flags
 from src.leaderboard.read_evals import get_raw_eval_results
 
@@ -14,8 +14,8 @@ def get_leaderboard_df(results_path: str, requests_path: str, dynamic_path: str,
     raw_data = get_raw_eval_results(results_path=results_path, requests_path=requests_path, dynamic_path=dynamic_path)
     all_data_json = [v.to_dict() for v in raw_data]
     all_data_json.append(baseline_row)
-    for proprietary_row in proprietary_rows:
-        all_data_json.append(proprietary_row)
+    for external_row in external_rows:
+        all_data_json.append(external_row)
     filter_models_flags(all_data_json)
 
     df = pd.DataFrame.from_records(all_data_json)