solved grouped dataset bug

.gitignore

@@ -18,4 +18,5 @@ src/assets/model_counts.html
 test
 env
 a.py
-testing.py
+testing.py
+frontend

app.py

@@ -60,6 +60,14 @@ try:
         force_download=True,
         token=TOKEN
     )
+    snapshot_download(
+        repo_id=RESULTS_GROUP_REPO, 
+        local_dir=EVAL_RESULTS_GROUP_PATH, 
+        repo_type="dataset", 
+        tqdm_class=None, 
+        etag_timeout=30, 
+        force_download=True,
+        token=TOKEN)
 except Exception:
     restart_space()
 
@@ -392,6 +400,6 @@ with demo:
             pass
     
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=300)
+scheduler.add_job(restart_space, "interval", seconds=1000)
 scheduler.start()
 demo.queue(default_concurrency_limit=40).launch()

eval-results-group/.gitattributes

@@ -0,0 +1,58 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+# Video files - compressed
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.webm filter=lfs diff=lfs merge=lfs -text

eval-results-group/gpt-4o-mini/results_2024-10-25T17:52:35.json

@@ -0,0 +1 @@
+{"config": {"submitted_time": "2024-10-25T17:52:35", "model_name": "OpenAI/gpt-4o-mini"}, "results": {"MMLU": {"metric_name": 0.7}, "ARC": {"metric_name": 1.0}, "GSM8K": {"metric_name": 1.0}, "CQA": {"metric_name": 0.4168853395851045}, "Banking": {"metric_name": 0.7717097630831459}}}

src/envs.py

@@ -15,6 +15,6 @@ CACHE_PATH=os.getenv("HF_HOME", ".")
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_RESULTS_GROUP_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_RESULTS_GROUP_PATH = os.path.join(CACHE_PATH, "eval-results-group")
 
 API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -212,11 +212,13 @@ def get_group_eval_results(results_path: str) -> list[EvalResultGroup]:
         eval_results[eval_name] = eval_result
 
     results = []
+    print(eval_results)
     for v in eval_results.values():
         try:
             v.to_dict() # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
+            print("key error")
             continue
 
     return results

src/populate.py

@@ -24,7 +24,6 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
 def get_leaderboard_group_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
     raw_data = get_group_eval_results(results_path)
     all_data_json = [v.to_dict() for v in raw_data]
-
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumnGroup.average.name], ascending=False)
     df = df[cols].round(decimals=2)