interim update in parser

app.py

@@ -1,4 +1,3 @@
-
 import gradio as gr
 import pandas as pd
 from pathlib import Path
@@ -11,7 +10,7 @@ from data_loader import (
     clear_cache,
     search_model_across_leaderboards,
     get_model_suggestions_fast,
-    DATA_DIR
+    DATA_DIR,
 )
 from ui_components import (
     get_theme,
@@ -29,12 +28,12 @@ PAGE_SIZE = 50
 def get_leaderboard_data(selected_leaderboard, progress=gr.Progress()):
     if not selected_leaderboard:
         return pd.DataFrame(), {}
-    
+
     metadata = get_eval_metadata(selected_leaderboard)
-    
+
     def progress_callback(value, desc):
         progress(value, desc=desc)
-    
+
     df = build_leaderboard_table(selected_leaderboard, "", progress_callback)
     return df, metadata
 
@@ -42,27 +41,32 @@ def get_leaderboard_data(selected_leaderboard, progress=gr.Progress()):
 def filter_and_paginate(df, search_query, sort_column, selected_columns, current_page):
     if df.empty:
         return df.copy(), 1, 1
-    
+
     df = df.copy()
     all_columns = list(df.columns)
-    
+
     if selected_columns:
-        cols = ["Model"] + [c for c in all_columns if c in selected_columns and c != "Model"]
+        cols = ["Model"] + [
+            c for c in all_columns if c in selected_columns and c != "Model"
+        ]
         df = df[cols]
-    
+
     if search_query:
-        mask = df.astype(str).apply(lambda row: row.str.contains(search_query, case=False, na=False).any(), axis=1)
+        mask = df.astype(str).apply(
+            lambda row: row.str.contains(search_query, case=False, na=False).any(),
+            axis=1,
+        )
         df = df[mask]
-    
+
     if sort_column and sort_column in df.columns:
-        df = df.sort_values(by=sort_column, ascending=False, na_position='last')
-    
+        df = df.sort_values(by=sort_column, ascending=False, na_position="last")
+
     total_rows = len(df)
     total_pages = max(1, (total_rows + PAGE_SIZE - 1) // PAGE_SIZE)
     current_page = max(1, min(current_page, total_pages))
     start = (current_page - 1) * PAGE_SIZE
     end = start + PAGE_SIZE
-    
+
     return df.iloc[start:end], current_page, total_pages
 
 
@@ -74,9 +78,9 @@ def search_model(model_query):
             <p>Enter a model name to see its benchmarks across all leaderboards</p>
         </div>
         """
-    
+
     results, _ = search_model_across_leaderboards(model_query)
-    
+
     if not results:
         return f"""
         <div class="no-results">
@@ -84,74 +88,88 @@ def search_model(model_query):
             <p>Try a different model name or check the spelling</p>
         </div>
         """
-    
+
     model_name = list(results.keys())[0]
     model_data = results[model_name]
-    
+
     return format_model_card(model_name, model_data)
 
 
 def compare_models(selected_models):
     if not selected_models:
-        return """
+        return (
+            """
         <div class="no-results">
             <h3>Select models to compare</h3>
             <p>Choose multiple models from the dropdown to see a side-by-side comparison</p>
         </div>
-        """, None
-    
+        """,
+            None,
+        )
+
     all_results = {}
     for model_name in selected_models:
         results, _ = search_model_across_leaderboards(model_name)
         if results:
             matched_model = list(results.keys())[0]
             all_results[matched_model] = results[matched_model]
-            
+
     plot = create_radar_plot(list(all_results.keys()), all_results)
-    
+
     if len(all_results) == 1:
         model_name = list(all_results.keys())[0]
         return format_model_card(model_name, all_results[model_name]), plot
     elif len(all_results) > 1:
         return format_model_comparison(list(all_results.keys()), all_results), plot
     else:
-        return """
+        return (
+            """
         <div class="no-results">
             <h3>No results found</h3>
             <p>Try selecting different models</p>
         </div>
-        """, None
+        """,
+            None,
+        )
 
 
 def get_model_suggestions(value):
     query = value or ""
     if not query or len(query) < 2:
         return gr.update(choices=[], value=[])
-    
+
     matches = get_model_suggestions_fast(query, limit=10)
     return gr.update(choices=matches, value=[])
 
 
-def export_leaderboard_to_csv(full_df, selected_leaderboard, search_query, selected_columns):
+def export_leaderboard_to_csv(
+    full_df, selected_leaderboard, search_query, selected_columns
+):
     """Export the current leaderboard view to CSV."""
     if full_df.empty:
         return None
-    
+
     df = full_df.copy()
-    
+
     # Apply column selection
     if selected_columns:
-        cols = ["Model"] + [c for c in df.columns if c in selected_columns and c != "Model"]
+        cols = ["Model"] + [
+            c for c in df.columns if c in selected_columns and c != "Model"
+        ]
         df = df[cols]
-    
+
     # Apply search filter
     if search_query:
-        mask = df.astype(str).apply(lambda row: row.str.contains(search_query, case=False, na=False).any(), axis=1)
+        mask = df.astype(str).apply(
+            lambda row: row.str.contains(search_query, case=False, na=False).any(),
+            axis=1,
+        )
         df = df[mask]
-    
+
     # Save to CSV with absolute path
     from pathlib import Path
     import tempfile
+
     temp_dir = Path(tempfile.gettempdir())
     filename = temp_dir / f"{selected_leaderboard.replace(' ', '_')}_leaderboard.csv"
     df.to_csv(filename, index=False)
@@ -162,17 +180,17 @@ def export_comparison_to_csv(selected_models):
     """Export model comparison to CSV."""
     if not selected_models:
         return None
-    
+
     all_results = {}
     for model_name in selected_models:
         results, _ = search_model_across_leaderboards(model_name)
         if results:
             matched_model = list(results.keys())[0]
             all_results[matched_model] = results[matched_model]
-    
+
     if not all_results:
         return None
-    
+
     # Build comparison table
     rows = []
     for model_name, model_data in all_results.items():
@@ -184,14 +202,15 @@ def export_comparison_to_csv(selected_models):
                 "Developer": data.get("developer"),
                 "Params (B)": data.get("params"),
                 "Architecture": data.get("architecture"),
-                "Precision": data.get("precision")
+                "Precision": data.get("precision"),
             }
             row.update(results)
             rows.append(row)
-    
+
     df = pd.DataFrame(rows)
     from pathlib import Path
     import tempfile
+
     temp_dir = Path(tempfile.gettempdir())
     filename = temp_dir / "model_comparison.csv"
     df.to_csv(filename, index=False)
@@ -205,8 +224,12 @@ initial_leaderboard = initial_leaderboards[0] if initial_leaderboards else None
 
 if initial_leaderboard:
     _init_df, _init_metadata = get_leaderboard_data(initial_leaderboard)
-    _init_columns = [c for c in _init_df.columns if c != "Model"] if not _init_df.empty else []
-    _init_df_display, _, _init_total_pages = filter_and_paginate(_init_df, "", "Average", None, 1)
+    _init_columns = (
+        [c for c in _init_df.columns if c != "Model"] if not _init_df.empty else []
+    )
+    _init_df_display, _, _init_total_pages = filter_and_paginate(
+        _init_df, "", "Average", None, 1
+    )
 else:
     _init_df = pd.DataFrame()
     _init_metadata = {}
@@ -214,12 +237,13 @@ else:
     _init_df_display = pd.DataFrame()
     _init_total_pages = 1
 
-with gr.Blocks(title="Every Eval Ever", theme=get_theme(), css=get_custom_css()) as demo:
-    
+with gr.Blocks(
+    title="Every Eval Ever", theme=get_theme(), css=get_custom_css()
+) as demo:
     full_df_state = gr.State(value=_init_df)
     metadata_state = gr.State(value=_init_metadata)
     current_page_state = gr.State(value=1)
-    
+
     gr.HTML("""
         <div class="app-header">
             <div class="logo-mark">E³</div>
@@ -232,7 +256,7 @@ with gr.Blocks(title="Every Eval Ever", theme=get_theme(), css=get_custom_css())
             </div>
         </div>
     """)
-    
+
     with gr.Tabs():
         with gr.TabItem("Leaderboards"):
             with gr.Column(elem_classes="controls-bar"):
@@ -242,21 +266,25 @@ with gr.Blocks(title="Every Eval Ever", theme=get_theme(), css=get_custom_css())
                             choices=initial_leaderboards,
                             value=initial_leaderboard,
                             label="Leaderboard",
-                            interactive=True
+                            interactive=True,
                         )
                     with gr.Column(scale=1, min_width=120):
-                        refresh_btn = gr.Button("↻ Refresh", variant="secondary", size="sm")
+                        refresh_btn = gr.Button(
+                            "↻ Refresh", variant="secondary", size="sm"
+                        )
                     with gr.Column(scale=1, min_width=120):
-                        export_btn = gr.DownloadButton("📥 Export CSV", variant="secondary", size="sm")
+                        export_btn = gr.DownloadButton(
+                            "📥 Export CSV", variant="secondary", size="sm"
+                        )
 
                 search_box = gr.Textbox(
-                    label="Filter",
-                    placeholder="Filter models...",
-                    show_label=True
+                    label="Filter", placeholder="Filter models...", show_label=True
                 )
-            
-            header_view = gr.HTML(value=format_leaderboard_header(initial_leaderboard, _init_metadata))
-            
+
+            header_view = gr.HTML(
+                value=format_leaderboard_header(initial_leaderboard, _init_metadata)
+            )
+
             with gr.Row(elem_classes="column-selector-bar"):
                 with gr.Column(scale=5, min_width=320):
                     column_selector = gr.Dropdown(
@@ -265,9 +293,9 @@ with gr.Blocks(title="Every Eval Ever", theme=get_theme(), css=get_custom_css())
                         label="Columns to Display",
                         multiselect=True,
                         interactive=True,
-                        elem_classes="column-selector-dropdown"
+                        elem_classes="column-selector-dropdown",
                     )
-        
+
             leaderboard_table = gr.Dataframe(
                 value=_init_df_display,
                 label=None,
@@ -275,17 +303,21 @@ with gr.Blocks(title="Every Eval Ever", theme=get_theme(), css=get_custom_css())
                 wrap=False,
                 elem_classes="dataframe",
             )
-            
+
             with gr.Row(elem_classes="pagination-bar"):
                 prev_btn = gr.Button("←", variant="secondary", size="sm", min_width=60)
-                page_info = gr.Markdown(value=f"1 / {_init_total_pages}", elem_classes="page-info")
+                page_info = gr.Markdown(
+                    value=f"1 / {_init_total_pages}", elem_classes="page-info"
+                )
                 next_btn = gr.Button("→", variant="secondary", size="sm", min_width=60)
-            
-            metrics_view = gr.HTML(value=format_metric_details(initial_leaderboard, _init_metadata))
-        
+
+            metrics_view = gr.HTML(
+                value=format_metric_details(initial_leaderboard, _init_metadata)
+            )
+
         with gr.TabItem("🔍 Model Lookup"):
             gr.Markdown("### Find and compare models across all leaderboards")
-            
+
             selected_models_state = gr.State(value=[])
             default_compare_html = """
                 <div class="no-results">
@@ -293,7 +325,7 @@ with gr.Blocks(title="Every Eval Ever", theme=get_theme(), css=get_custom_css())
                     <p>Type in the dropdown to search, then select a model to add it</p>
                 </div>
             """
-            
+
             with gr.Row(elem_classes="controls-bar"):
                 with gr.Column(scale=3):
                     model_search_box = gr.Textbox(
@@ -312,25 +344,29 @@ with gr.Blocks(title="Every Eval Ever", theme=get_theme(), css=get_custom_css())
                         elem_classes=["match-pills"],
                     )
                 with gr.Column(scale=1, min_width=80):
-                    clear_models_btn = gr.Button("Clear", variant="secondary", size="sm")
-            
+                    clear_models_btn = gr.Button(
+                        "Clear", variant="secondary", size="sm"
+                    )
+
             selected_models_group = gr.CheckboxGroup(
                 choices=[],
                 value=[],
                 label="Selected Models (click to remove)",
                 interactive=True,
-                elem_classes="selected-models-group"
+                elem_classes="selected-models-group",
             )
-            
+
             with gr.Row():
                 with gr.Column(scale=4):
                     pass
                 with gr.Column(scale=1, min_width=120):
-                    export_comparison_btn = gr.DownloadButton("📥 Export CSV", variant="secondary", size="sm")
-            
+                    export_comparison_btn = gr.DownloadButton(
+                        "📥 Export CSV", variant="secondary", size="sm"
+                    )
+
             radar_view = gr.Plot(label="Radar Comparison")
             model_card_view = gr.HTML(value=default_compare_html)
-    
+
     with gr.Accordion("📤 How to Submit Data", open=False):
         gr.Markdown("""
 Submit via GitHub Pull Request:
@@ -341,12 +377,12 @@ Submit via GitHub Pull Request:
 
 [Submission Guide](https://github.com/evaleval/every_eval_ever#contributor-guide) - [JSON Schema](https://github.com/evaleval/every_eval_ever/blob/main/eval.schema.json)
         """)
-    
+
     def load_leaderboard(leaderboard_name):
         df, metadata = get_leaderboard_data(leaderboard_name)
         columns = [c for c in df.columns if c != "Model"] if not df.empty else []
         df_display, page, total_pages = filter_and_paginate(df, "", "Average", None, 1)
-        
+
         return (
             df,  # full_df_state
             metadata,  # metadata_state
@@ -357,83 +393,103 @@ Submit via GitHub Pull Request:
             gr.update(choices=columns, value=columns),  # column_selector
             f"1 / {total_pages}",  # page_info
         )
-    
+
     def update_table(full_df, search_query, selected_columns, current_page):
         df_display, page, total_pages = filter_and_paginate(
             full_df, search_query, "Average", selected_columns, current_page
         )
         return df_display, f"{page} / {total_pages}", page
-    
+
     def go_page(full_df, search_query, selected_columns, current_page, delta):
         new_page = max(1, current_page + delta)
         df_display, page, total_pages = filter_and_paginate(
             full_df, search_query, "Average", selected_columns, new_page
         )
         return df_display, f"{page} / {total_pages}", page
-    
+
     leaderboard_selector.change(
         fn=load_leaderboard,
         inputs=[leaderboard_selector],
-        outputs=[full_df_state, metadata_state, current_page_state, leaderboard_table, header_view, metrics_view, column_selector, page_info]
+        outputs=[
+            full_df_state,
+            metadata_state,
+            current_page_state,
+            leaderboard_table,
+            header_view,
+            metrics_view,
+            column_selector,
+            page_info,
+        ],
     )
-    
+
     search_box.input(
         fn=lambda df, q, cols: update_table(df, q, cols, 1),
         inputs=[full_df_state, search_box, column_selector],
-        outputs=[leaderboard_table, page_info, current_page_state]
+        outputs=[leaderboard_table, page_info, current_page_state],
     )
-    
+
     def on_column_change(df, q, cols):
         if not cols:
             cols = [c for c in df.columns if c != "Model"]
         return update_table(df, q, cols, 1)
-    
+
     column_selector.change(
         fn=on_column_change,
         inputs=[full_df_state, search_box, column_selector],
-        outputs=[leaderboard_table, page_info, current_page_state]
+        outputs=[leaderboard_table, page_info, current_page_state],
     )
-    
+
     prev_btn.click(
         fn=lambda df, q, cols, p: go_page(df, q, cols, p, -1),
         inputs=[full_df_state, search_box, column_selector, current_page_state],
-        outputs=[leaderboard_table, page_info, current_page_state]
+        outputs=[leaderboard_table, page_info, current_page_state],
     )
-    
+
     next_btn.click(
         fn=lambda df, q, cols, p: go_page(df, q, cols, p, 1),
         inputs=[full_df_state, search_box, column_selector, current_page_state],
-        outputs=[leaderboard_table, page_info, current_page_state]
+        outputs=[leaderboard_table, page_info, current_page_state],
     )
-    
+
     refresh_btn.click(
         fn=lambda: (clear_cache(), gr.update(choices=get_available_leaderboards()))[1],
-        outputs=[leaderboard_selector]
+        outputs=[leaderboard_selector],
     )
-    
+
     export_btn.click(
         fn=export_leaderboard_to_csv,
         inputs=[full_df_state, leaderboard_selector, search_box, column_selector],
-        outputs=[export_btn]
+        outputs=[export_btn],
     )
-    
+
     def add_models_from_search(selected_from_results, current_selected):
         selected_from_results = selected_from_results or []
         current_selected = current_selected or []
         merged = list(dict.fromkeys(current_selected + selected_from_results))
-        comparison_html, plot = compare_models(merged) if merged else (default_compare_html, None)
+        comparison_html, plot = (
+            compare_models(merged) if merged else (default_compare_html, None)
+        )
         return (
             merged,
             gr.update(choices=[], value=[]),
             gr.update(choices=merged, value=merged),
             comparison_html,
-            plot
+            plot,
         )
-    
+
     def update_selection(selected_list):
-        comparison_html, plot = compare_models(selected_list) if selected_list else (default_compare_html, None)
-        return selected_list, gr.update(choices=selected_list, value=selected_list), comparison_html, plot
-    
+        comparison_html, plot = (
+            compare_models(selected_list)
+            if selected_list
+            else (default_compare_html, None)
+        )
+        return (
+            selected_list,
+            gr.update(choices=selected_list, value=selected_list),
+            comparison_html,
+            plot,
+        )
+
     def clear_all_models():
         return (
             [],
@@ -441,9 +497,9 @@ Submit via GitHub Pull Request:
             gr.update(choices=[], value=[]),
             gr.update(choices=[], value=[]),
             default_compare_html,
-            None
+            None,
         )
-    
+
     search_button.click(
         fn=get_model_suggestions,
         inputs=[model_search_box],
@@ -456,30 +512,48 @@ Submit via GitHub Pull Request:
         outputs=[search_results],
         queue=False,
     )
-    
+
     search_results.change(
         fn=add_models_from_search,
         inputs=[search_results, selected_models_state],
-        outputs=[selected_models_state, search_results, selected_models_group, model_card_view, radar_view],
+        outputs=[
+            selected_models_state,
+            search_results,
+            selected_models_group,
+            model_card_view,
+            radar_view,
+        ],
     )
-    
+
     selected_models_group.change(
         fn=update_selection,
         inputs=[selected_models_group],
-        outputs=[selected_models_state, selected_models_group, model_card_view, radar_view]
+        outputs=[
+            selected_models_state,
+            selected_models_group,
+            model_card_view,
+            radar_view,
+        ],
     )
-    
+
     clear_models_btn.click(
         fn=clear_all_models,
-        outputs=[selected_models_state, model_search_box, search_results, selected_models_group, model_card_view, radar_view]
+        outputs=[
+            selected_models_state,
+            model_search_box,
+            search_results,
+            selected_models_group,
+            model_card_view,
+            radar_view,
+        ],
     )
-    
+
     export_comparison_btn.click(
         fn=export_comparison_to_csv,
         inputs=[selected_models_state],
-        outputs=[export_comparison_btn]
+        outputs=[export_comparison_btn],
     )
-    
+
     DATA_DIR.mkdir(exist_ok=True)
 
 if __name__ == "__main__":

data_loader.py

@@ -1,6 +1,7 @@
 """
 Data Loader: Load from HuggingFace, parse JSON files, and build tables.
 """
+
 import json
 import pandas as pd
 from pathlib import Path
@@ -26,70 +27,133 @@ def load_hf_dataset_on_startup():
     print("Loading dataset from HuggingFace...")
     try:
         dataset = load_dataset("evaleval/every_eval_ever")
-        
+
         for split_name, split_data in dataset.items():
             print(f"Loading split: {split_name} ({len(split_data)} rows)")
-            
+
             df = split_data.to_pandas()
             parsed_items = []
-            
+
             for _, row in df.iterrows():
-                evaluation_results = json.loads(row['evaluation_results'])
-                
+                # New schema (v0.1.0) stores complex fields as JSON strings in parquet
+                # and uses unified top-level keys that mirror eval.schema.json.
+                #
+                # We keep this flexible so it works with both the old dataset
+                # (flattened columns) and the new one (JSON columns).
+                def _safe_json_load(value):
+                    if isinstance(value, str):
+                        try:
+                            return json.loads(value)
+                        except Exception:
+                            return value
+                    return value
+
+                # --- Core structured fields ---
+                evaluation_results = (
+                    _safe_json_load(row.get("evaluation_results", "[]")) or []
+                )
+                source_metadata = (
+                    _safe_json_load(row.get("source_metadata", "{}")) or {}
+                )
+                source_data = _safe_json_load(row.get("source_data", "[]")) or []
+                model_info = _safe_json_load(row.get("model_info", "{}")) or {}
+
+                # Some older parquet versions had flattened columns instead of JSON blobs.
+                # We transparently patch those into the new structure if present.
+                if not source_metadata:
+                    # Old columns: source_organization_name, evaluator_relationship,
+                    # source_organization_url, source_organization_logo_url, evaluation_source_name, evaluation_source_type
+                    sm = {}
+                    if pd.notna(row.get("evaluation_source_name", None)):
+                        sm["source_name"] = row["evaluation_source_name"]
+                    if pd.notna(row.get("evaluation_source_type", None)):
+                        sm["source_type"] = row["evaluation_source_type"]
+                    if pd.notna(row.get("source_organization_name", None)):
+                        sm["source_organization_name"] = row["source_organization_name"]
+                    if pd.notna(row.get("source_organization_url", None)):
+                        sm["source_organization_url"] = row["source_organization_url"]
+                    if pd.notna(row.get("source_organization_logo_url", None)):
+                        sm["source_organization_logo_url"] = row[
+                            "source_organization_logo_url"
+                        ]
+                    if pd.notna(row.get("evaluator_relationship", None)):
+                        sm["evaluator_relationship"] = row["evaluator_relationship"]
+                    source_metadata = sm
+
+                if not source_data:
+                    # Old schema used `source_data` as list of URLs already; if we see a
+                    # plain string, wrap it into a list for consistency.
+                    raw_sd = row.get("source_data")
+                    if isinstance(raw_sd, str) and raw_sd:
+                        source_data = [raw_sd]
+
+                if not model_info:
+                    # Old flattened model columns
+                    mi = {}
+                    if pd.notna(row.get("model_name", None)):
+                        mi["name"] = row["model_name"]
+                    if pd.notna(row.get("model_id", None)):
+                        mi["id"] = row["model_id"]
+                    if pd.notna(row.get("model_developer", None)):
+                        mi["developer"] = row["model_developer"]
+                    if pd.notna(row.get("model_inference_platform", None)):
+                        mi["inference_platform"] = row["model_inference_platform"]
+                    model_info = mi
+
+                additional_details = {}
+                # New schema: additional_details lives inside model_info
+                if isinstance(model_info, dict):
+                    additional_details = model_info.get("additional_details") or {}
+
+                # Old schema sometimes had an `additional_details` top-level column
+                # with JSON, we still honour that as a source of params/precision/arch.
+                if not additional_details and pd.notna(
+                    row.get("additional_details", None)
+                ):
+                    additional_details = (
+                        _safe_json_load(row["additional_details"]) or {}
+                    )
+
                 results = {}
                 for eval_result in evaluation_results:
                     eval_name = eval_result.get("evaluation_name")
                     score = eval_result.get("score_details", {}).get("score")
                     if eval_name and score is not None:
                         results[eval_name] = score
-                
-                additional_details = {}
-                if pd.notna(row.get('additional_details')):
-                    additional_details = json.loads(row['additional_details'])
-                
+
                 parsed_item = {
-                    "leaderboard": row['_leaderboard'],
-                    "provider": row['source_organization_name'],
-                    "model": row['model_id'],
-                    "developer": row['model_developer'],
-                    "params": additional_details.get('params_billions'),
-                    "architecture": additional_details.get('architecture', 'Unknown'),
-                    "precision": additional_details.get('precision', 'Unknown'),
+                    "leaderboard": row.get("_leaderboard", "unknown_leaderboard"),
+                    # Provider is the organization owning the source/leaderboard
+                    "provider": source_metadata.get(
+                        "source_organization_name", "Unknown Provider"
+                    ),
+                    # Prefer the canonical model id from the new schema; fall back to old columns
+                    "model": model_info.get("id")
+                    or row.get("_model")
+                    or row.get("model_id", "Unknown Model"),
+                    "developer": model_info.get("developer")
+                    or row.get("_developer")
+                    or row.get("model_developer", "Unknown Developer"),
+                    "params": additional_details.get("params_billions"),
+                    "architecture": additional_details.get("architecture", "Unknown"),
+                    "precision": additional_details.get("precision", "Unknown"),
                     "results": results,
                     "raw_data": {
-                        "schema_version": row['schema_version'],
-                        "evaluation_id": row['evaluation_id'],
-                        "retrieved_timestamp": row['retrieved_timestamp'],
-                        "source_data": json.loads(row['source_data']),
-                        "evaluation_source": {
-                            "evaluation_source_name": row['evaluation_source_name'],
-                            "evaluation_source_type": row['evaluation_source_type']
-                        },
-                        "source_metadata": {
-                            "source_organization_name": row['source_organization_name'],
-                            "evaluator_relationship": row['evaluator_relationship'],
-                        },
-                        "model_info": {
-                            "name": row['model_name'],
-                            "id": row['model_id'],
-                            "developer": row['model_developer'],
-                        },
+                        "schema_version": row.get("schema_version"),
+                        "evaluation_id": row.get("evaluation_id"),
+                        "retrieved_timestamp": row.get("retrieved_timestamp"),
+                        "source_data": source_data,
+                        "source_metadata": source_metadata,
+                        "model_info": model_info,
                         "evaluation_results": evaluation_results,
-                        "additional_details": additional_details
-                    }
+                        "additional_details": additional_details,
+                    },
                 }
-                
-                if pd.notna(row.get('source_organization_url')):
-                    parsed_item["raw_data"]["source_metadata"]["source_organization_url"] = row['source_organization_url']
-                if pd.notna(row.get('source_organization_logo_url')):
-                    parsed_item["raw_data"]["source_metadata"]["source_organization_logo_url"] = row['source_organization_logo_url']
-                if pd.notna(row.get('model_inference_platform')):
-                    parsed_item["raw_data"]["model_info"]["inference_platform"] = row['model_inference_platform']
-                
+
                 parsed_items.append(parsed_item)
-            
+
             HF_DATASET_CACHE[split_name] = parsed_items
-            
+
         print(f"Loaded {len(HF_DATASET_CACHE)} leaderboard(s) from HuggingFace")
         _build_search_index()
         return True
@@ -102,20 +166,39 @@ def load_hf_dataset_on_startup():
 def parse_eval_json(file_path):
     """Parses a single JSON file to extract model, provider, and results."""
     try:
-        with open(file_path, 'r') as f:
+        with open(file_path, "r") as f:
             data = json.load(f)
-        
-        leaderboard_name = data.get("evaluation_source", {}).get("evaluation_source_name", "Unknown Leaderboard")
-        provider_name = data.get("source_metadata", {}).get("source_organization_name", "Unknown Provider")
-        model_id = data.get("model_info", {}).get("id", "Unknown Model")
-        developer_name = data.get("model_info", {}).get("developer", "Unknown Developer")
-        
-        params = data.get("model_info", {}).get("params_billions", None)
-        architecture = data.get("model_info", {}).get("architecture", "Unknown")
-        precision = data.get("additional_details", {}).get("precision", "Unknown")
+
+        # New schema (v0.1.0) removes `evaluation_source` and moves most
+        # metadata into `source_metadata` and `model_info.additional_details`.
+        source_meta = data.get("source_metadata", {}) or {}
+        model_info = data.get("model_info", {}) or {}
+
+        # Leaderboard name:
+        #   - new schema: source_metadata.source_name
+        #   - old schema: evaluation_source.evaluation_source_name
+        leaderboard_name = source_meta.get("source_name")
+        if not leaderboard_name:
+            leaderboard_name = data.get("evaluation_source", {}).get(
+                "evaluation_source_name", "Unknown Leaderboard"
+            )
+
+        provider_name = source_meta.get("source_organization_name", "Unknown Provider")
+        model_id = model_info.get("id", "Unknown Model")
+        developer_name = model_info.get("developer", "Unknown Developer")
+
+        # Model-level details:
+        additional_details = model_info.get("additional_details") or {}
+        # Backwards compatibility with old layout
+        if not additional_details:
+            additional_details = data.get("additional_details", {}) or {}
+
+        params = additional_details.get("params_billions")
+        architecture = additional_details.get("architecture", "Unknown")
+        precision = additional_details.get("precision", "Unknown")
         if precision == "Unknown":
-             precision = data.get("model_info", {}).get("precision", "Unknown")
-             
+            precision = model_info.get("precision", "Unknown")
+
         results = {}
         if "evaluation_results" in data:
             for res in data["evaluation_results"]:
@@ -123,7 +206,7 @@ def parse_eval_json(file_path):
                 score = res.get("score_details", {}).get("score", None)
                 if score is not None:
                     results[eval_name] = score
-                    
+
         return {
             "leaderboard": leaderboard_name,
             "provider": provider_name,
@@ -133,7 +216,7 @@ def parse_eval_json(file_path):
             "architecture": architecture,
             "precision": precision,
             "results": results,
-            "raw_data": data
+            "raw_data": data,
         }
     except Exception as e:
         print(f"Error parsing {file_path}: {e}")
@@ -144,7 +227,7 @@ def get_available_leaderboards():
     """Returns available leaderboards from HF cache or local directory."""
     if HF_DATASET_CACHE:
         return list(HF_DATASET_CACHE.keys())
-    
+
     if not DATA_DIR.exists():
         return []
     return [d.name for d in DATA_DIR.iterdir() if d.is_dir()]
@@ -162,102 +245,135 @@ def get_eval_metadata(selected_leaderboard):
     """Extracts evaluation metadata from the leaderboard data."""
     if not selected_leaderboard:
         return {}
-    
+
     eval_metadata = {"evals": {}, "source_info": {}}
-    
+
     if selected_leaderboard in HF_DATASET_CACHE:
         parsed_items = HF_DATASET_CACHE[selected_leaderboard]
         if parsed_items:
             parsed = parsed_items[0]
-            
-            source_meta = parsed["raw_data"].get("source_metadata", {})
-            source_data_list = parsed["raw_data"].get("source_data", [])
-            url = source_data_list[0] if isinstance(source_data_list, list) and source_data_list else "#"
-            
+
+            source_meta = parsed["raw_data"].get("source_metadata", {}) or {}
+            source_data_val = parsed["raw_data"].get("source_data", [])
+
+            # source_data can be either:
+            #   - list[str] (URLs) OR
+            #   - object describing a HF dataset. For the latter, we skip the URL.
+            url = "#"
+            if isinstance(source_data_val, list) and source_data_val:
+                url = source_data_val[0]
+
             eval_metadata["source_info"] = {
                 "organization": source_meta.get("source_organization_name", "Unknown"),
                 "relationship": source_meta.get("evaluator_relationship", "Unknown"),
-                "url": url
+                "url": url,
             }
-            
+
             if "evaluation_results" in parsed["raw_data"]:
                 for res in parsed["raw_data"]["evaluation_results"]:
                     eval_name = res.get("evaluation_name", "Unknown Metric")
                     if eval_name not in eval_metadata["evals"]:
                         metric_config = res.get("metric_config", {})
                         eval_metadata["evals"][eval_name] = {
-                            "description": metric_config.get("evaluation_description", "No description available"),
+                            "description": metric_config.get(
+                                "evaluation_description", "No description available"
+                            ),
                             "score_type": metric_config.get("score_type", "unknown"),
-                            "lower_is_better": metric_config.get("lower_is_better", False),
+                            "lower_is_better": metric_config.get(
+                                "lower_is_better", False
+                            ),
                             "min_score": metric_config.get("min_score"),
                             "max_score": metric_config.get("max_score"),
                             "level_names": metric_config.get("level_names", []),
                             "level_metadata": metric_config.get("level_metadata", []),
-                            "has_unknown_level": metric_config.get("has_unknown_level", False)
+                            "has_unknown_level": metric_config.get(
+                                "has_unknown_level", False
+                            ),
                         }
         return eval_metadata
-    
+
     # Fall back to file system
     for json_file in walk_eval_files(selected_leaderboard):
         parsed = parse_eval_json(json_file)
         if parsed:
             if not eval_metadata["source_info"]:
-                 source_meta = parsed["raw_data"].get("source_metadata", {})
-                 source_data_list = parsed["raw_data"].get("source_data", [])
-                 url = source_data_list[0] if isinstance(source_data_list, list) and source_data_list else "#"
-                 
-                 eval_metadata["source_info"] = {
-                     "organization": source_meta.get("source_organization_name", "Unknown"),
-                     "relationship": source_meta.get("evaluator_relationship", "Unknown"),
-                     "url": url
-                 }
-            
+                source_meta = parsed["raw_data"].get("source_metadata", {})
+                source_data_list = parsed["raw_data"].get("source_data", [])
+                url = (
+                    source_data_list[0]
+                    if isinstance(source_data_list, list) and source_data_list
+                    else "#"
+                )
+
+                eval_metadata["source_info"] = {
+                    "organization": source_meta.get(
+                        "source_organization_name", "Unknown"
+                    ),
+                    "relationship": source_meta.get(
+                        "evaluator_relationship", "Unknown"
+                    ),
+                    "url": url,
+                }
+
             if "evaluation_results" in parsed["raw_data"]:
                 for res in parsed["raw_data"]["evaluation_results"]:
                     eval_name = res.get("evaluation_name", "Unknown Metric")
                     if eval_name not in eval_metadata["evals"]:
                         metric_config = res.get("metric_config", {})
                         eval_metadata["evals"][eval_name] = {
-                            "description": metric_config.get("evaluation_description", "No description available"),
+                            "description": metric_config.get(
+                                "evaluation_description", "No description available"
+                            ),
                             "score_type": metric_config.get("score_type", "unknown"),
-                            "lower_is_better": metric_config.get("lower_is_better", False),
+                            "lower_is_better": metric_config.get(
+                                "lower_is_better", False
+                            ),
                             "min_score": metric_config.get("min_score"),
                             "max_score": metric_config.get("max_score"),
                             "level_names": metric_config.get("level_names", []),
                             "level_metadata": metric_config.get("level_metadata", []),
-                            "has_unknown_level": metric_config.get("has_unknown_level", False)
+                            "has_unknown_level": metric_config.get(
+                                "has_unknown_level", False
+                            ),
                         }
             break
-    
+
     return eval_metadata
 
 
-def build_leaderboard_table(selected_leaderboard, search_query="", progress_callback=None):
+def build_leaderboard_table(
+    selected_leaderboard, search_query="", progress_callback=None
+):
     """Builds the leaderboard DataFrame from cache or files."""
     if not selected_leaderboard:
         return pd.DataFrame()
-    
+
     if selected_leaderboard in LEADERBOARD_CACHE:
         df, _ = LEADERBOARD_CACHE[selected_leaderboard]
     else:
         rows = []
-        
+
         if selected_leaderboard in HF_DATASET_CACHE:
             if progress_callback:
-                progress_callback(0, desc=f"Loading {selected_leaderboard} from cache...")
-            
+                progress_callback(
+                    0, desc=f"Loading {selected_leaderboard} from cache..."
+                )
+
             parsed_items = HF_DATASET_CACHE[selected_leaderboard]
-            
+
             for i, parsed in enumerate(parsed_items):
                 if i % 100 == 0 and progress_callback:
-                    progress_callback((i / len(parsed_items)), desc=f"Processing {selected_leaderboard}...")
-                
+                    progress_callback(
+                        (i / len(parsed_items)),
+                        desc=f"Processing {selected_leaderboard}...",
+                    )
+
                 row = {
-                    "Model": parsed["model"], 
+                    "Model": parsed["model"],
                     "Developer": parsed["developer"],
                     "Params (B)": parsed["params"],
                     "Arch": parsed["architecture"],
-                    "Precision": parsed["precision"]
+                    "Precision": parsed["precision"],
                 }
                 row.update(parsed["results"])
                 rows.append(row)
@@ -265,41 +381,45 @@ def build_leaderboard_table(selected_leaderboard, search_query="", progress_call
             # Fall back to file system
             if progress_callback:
                 progress_callback(0, desc=f"Scanning {selected_leaderboard}...")
-            
+
             all_files = list(walk_eval_files(selected_leaderboard))
             total_files = len(all_files)
-            
+
             for i, json_file in enumerate(all_files):
                 if i % 100 == 0 and progress_callback:
-                     progress_callback((i / total_files), desc=f"Loading {selected_leaderboard}...")
-                
+                    progress_callback(
+                        (i / total_files), desc=f"Loading {selected_leaderboard}..."
+                    )
+
                 parsed = parse_eval_json(json_file)
                 if parsed:
                     row = {
-                        "Model": parsed["model"], 
+                        "Model": parsed["model"],
                         "Developer": parsed["developer"],
                         "Params (B)": parsed["params"],
                         "Arch": parsed["architecture"],
-                        "Precision": parsed["precision"]
+                        "Precision": parsed["precision"],
                     }
                     row.update(parsed["results"])
                     rows.append(row)
-        
+
         if not rows:
-            df = pd.DataFrame(columns=["Model", "Developer", "Params (B)", "Arch", "Precision"])
+            df = pd.DataFrame(
+                columns=["Model", "Developer", "Params (B)", "Arch", "Precision"]
+            )
             LEADERBOARD_CACHE[selected_leaderboard] = (df, None)
             return df
-        
+
         df = pd.DataFrame(rows)
-        df = df.dropna(axis=1, how='all')
-        
+        df = df.dropna(axis=1, how="all")
+
         if df.empty:
-             LEADERBOARD_CACHE[selected_leaderboard] = (df, None)
-             return df
+            LEADERBOARD_CACHE[selected_leaderboard] = (df, None)
+            return df
 
-        numeric_cols = df.select_dtypes(include=['float', 'int']).columns
+        numeric_cols = df.select_dtypes(include=["float", "int"]).columns
         df[numeric_cols] = df[numeric_cols].round(2)
-        
+
         # Add Average Score
         eval_only_cols = [c for c in numeric_cols if c not in ["Params (B)"]]
         if len(eval_only_cols) > 0:
@@ -310,18 +430,20 @@ def build_leaderboard_table(selected_leaderboard, search_query="", progress_call
         # Model detail columns: Arch, Precision (moved to end)
         base_cols = ["Model", "Developer", "Params (B)", "Average"]
         model_detail_cols = ["Arch", "Precision"]
-        eval_cols = [c for c in df.columns if c not in base_cols and c not in model_detail_cols]
+        eval_cols = [
+            c for c in df.columns if c not in base_cols and c not in model_detail_cols
+        ]
         base_cols = [c for c in base_cols if c in df.columns]
         model_detail_cols = [c for c in model_detail_cols if c in df.columns]
-        
+
         final_cols = base_cols + sorted(eval_cols) + model_detail_cols
         df = df[final_cols]
-        
+
         if "Average" in df.columns:
             df = df.sort_values("Average", ascending=False)
-        
+
         LEADERBOARD_CACHE[selected_leaderboard] = (df, None)
-    
+
     return df
 
 
@@ -357,10 +479,10 @@ def get_model_suggestions_fast(query, limit=15):
     global LAST_QUERY, LAST_RESULTS
     if not query or len(query) < 2 or not MODEL_SEARCH_INDEX:
         return []
-    
+
     query_lower = query.lower()
     results = []
-    
+
     # Incremental reuse: if user keeps typing the same prefix, reuse last pool
     base_pool = None
     if LAST_QUERY and query_lower.startswith(LAST_QUERY) and LAST_RESULTS:
@@ -368,7 +490,7 @@ def get_model_suggestions_fast(query, limit=15):
     else:
         prefix_key = query_lower[:2]
         base_pool = MODEL_PREFIX_MAP.get(prefix_key, MODEL_SEARCH_INDEX)
-    
+
     # 1) Prefix match on model names
     if base_pool is MODEL_SEARCH_INDEX:
         idx = bisect_left(MODEL_SEARCH_INDEX, (query_lower,))
@@ -385,7 +507,7 @@ def get_model_suggestions_fast(query, limit=15):
                 results.append((0, len(name_lower), name_orig))
                 if len(results) >= limit:
                     break
-    
+
     # 2) Substring fallback on the narrowed pool
     if len(results) < limit:
         seen = {r[2] for r in results}
@@ -402,13 +524,13 @@ def get_model_suggestions_fast(query, limit=15):
                 results.append((pos, len(name_lower), name_orig))
                 if len(results) >= limit * 2:
                     break
-    
+
     results.sort(key=lambda x: (x[0], x[1]))
-    
+
     # Update incremental cache
     LAST_QUERY = query_lower
     LAST_RESULTS = base_pool if base_pool is not None else MODEL_SEARCH_INDEX
-    
+
     return [r[2] for r in results[:limit]]
 
 
@@ -416,10 +538,10 @@ def search_model_across_leaderboards(model_query):
     """Search for a model across all leaderboards and return aggregated results."""
     if not model_query or not HF_DATASET_CACHE:
         return {}, []
-    
+
     # Use fast fuzzy search for suggestions
     matches = get_model_suggestions_fast(model_query, limit=20)
-    
+
     # Get detailed results only for matched models
     results = {}
     for leaderboard_name, parsed_items in HF_DATASET_CACHE.items():
@@ -433,9 +555,9 @@ def search_model_across_leaderboards(model_query):
                     "params": item.get("params"),
                     "architecture": item.get("architecture"),
                     "precision": item.get("precision"),
-                    "results": item.get("results", {})
+                    "results": item.get("results", {}),
                 }
-    
+
     return results, matches
 
 
@@ -443,11 +565,10 @@ def get_all_model_names():
     """Get all unique model names across all leaderboards."""
     if not HF_DATASET_CACHE:
         return []
-    
+
     models = set()
     for parsed_items in HF_DATASET_CACHE.values():
         for item in parsed_items:
             models.add(item.get("model", ""))
-    
-    return sorted(models)
 
+    return sorted(models)

eval.schema.json

@@ -1,18 +1,18 @@
 {
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "version": "0.0.1",
+    "version": "0.1.0",
     "type": "object",
     "description": "Schema for storing and validating LLMs evaluation data, including model configuration, prompts, instances, Output, and evaluation metrics",
     "required": [
         "schema_version",
         "evaluation_id",
-        "evaluation_source",
         "retrieved_timestamp",
         "source_data",
         "source_metadata",
         "model_info",
         "evaluation_results"
     ],
+    "additionalProperties": false,
     "properties": {
         "schema_version": {
             "type": "string",
@@ -20,49 +20,82 @@
         },
         "evaluation_id": {
             "type": "string",
-            "description": "Unique identifier for this specific evaluation run. Use org_name/eval_name/retrieved_timestamp format"
+            "description": "Unique identifier for this specific evaluation run. Use eval_name/model_id/retrieved_timestamp format"
         },
         "retrieved_timestamp": {
             "type": "string",
-            "description": "Timestamp for when this record was created"
+            "description": "Timestamp for when this record was created - using Unix Epoch time format"
         },
         "source_data": {
-            "type": "array",
-            "description": "URLs for the source of the evaluation data",
-            "items": {
-                "type": "string"
-            }
-        },
-        "evaluation_source": {
-            "type": "object",
-            "description": "Details about evaluation origin. There are options that evaluations come from leaderboards (e.g. Live Code Bench Pro) or evaluation platforms (e.g. lm-eval, inspect ai, HELM...).",
-            "required": [
-                "evaluation_source_name",
-                "evaluation_source_type"
-            ],
-            "properties": {
-                "evaluation_source_name": {
-                    "type": "string",
-                    "description": "Name of the source (e.g. title of the source leaderboard or name of the platform used for the evaluation."
+            "description": "Source of dataset used for evaluation. There are two options supported: HuggingFace dataset or url for other data source.",
+            "oneOf": [
+                {
+                    "type": "array",
+                    "description": "URLs for the source of the evaluation data",
+                    "items": {
+                        "type": "string"
+                    }
                 },
-                "evaluation_source_type": {
-                    "type": "string",
-                    "enum": [
-                        "leaderboard", 
-                        "evaluation_platform"
+                {
+                    "type": "object",
+                    "description": "Details about HuggingFace dataset used for evaluation",
+                    "required": [
+                        "dataset_name"
                     ],
-                    "description": "Type of evaluation source, e.g., leaderboard or evaluation platform"
+                    "properties": {
+                        "dataset_name": {
+                            "type": "string",
+                            "description": "Name of the source dataset"
+                        },
+                        "hf_repo": {
+                            "type": "string",
+                            "description": "HuggingFace repository identifier"
+                        },
+                        "hf_split": {
+                            "type": "string",
+                            "description": "One of train, val or test."
+                        },
+                        "samples_number": {
+                            "type": "integer",
+                            "description": "Number of samples in the dataset"
+                        },
+                        "sample_ids": {
+                            "type": "array",
+                            "description": "Array of sample ids used for evaluation",
+                            "items": {
+                                "type": ["integer", "string"]
+                            }
+                        },
+                        "additional_details": {
+                            "type": "object",
+                            "description": "Additional dataset info parameters",
+                            "additionalProperties": true
+                        }
+                    }
                 }
-            }
+            ]
         },
         "source_metadata": {
             "type": "object",
             "description": "Metadata about the source of the leaderboard data",
             "required": [
+                "source_type",
                 "source_organization_name",
                 "evaluator_relationship"
             ],
             "properties": {
+                "source_name": {
+                    "type": "string",
+                    "description": "Name of the source (e.g. title of the source leaderboard or name of the platform used for the evaluation)."
+                },
+                "source_type": {
+                    "type": "string",
+                    "enum": [
+                        "documentation",
+                        "evaluation_run"
+                    ],
+                    "description": "Whether the data comes from a direct evaluation run or from documentation"
+                },
                 "source_organization_name": {
                     "type": "string",
                     "description": "Name of the organization that provides the data"
@@ -101,7 +134,7 @@
                 },
                 "id": {
                     "type": "string",
-                    "description": "Model name standarized to HuggingFace format (e.g. meta-llama/Llama-3.1-8B-Instruct)"
+                    "description": "Model name in HuggingFace format (e.g. meta-llama/Llama-3.1-8B-Instruct for models available on HuggingFace or openai/azure/gpt-4o-mini-2024-07-18 for closed API models)"
                 },
                 "developer": {
                     "type": "string",
@@ -109,7 +142,16 @@
                 },
                 "inference_platform": {
                     "type": "string",
-                    "description": "Description of platform used to run the evaluations (e.g. local machine, Bedrock)"
+                    "description": "Name of inference platform which provides an access to models by API to run the evaluations or provides models weights to run them locally (e.g. HuggingFace, Bedrock, Together AI)"
+                },
+                "inference_engine": {
+                    "type": "string",
+                    "description": "Name of inference engine which provides an access to optimized models to use them for local evaluations (e.g. vLLM, Ollama)."
+                },
+                "additional_details": {
+                    "type": "object",
+                    "description": "Additional model configuration parameters",
+                    "additionalProperties": true
                 }
             }
         },
@@ -213,7 +255,6 @@
                         }
                     },
                     "score_details": {
-                        "type": "object",
                         "description": "The score for the evaluation and related details",
                         "required": [
                             "score"
@@ -237,37 +278,45 @@
                     "generation_config": {
                         "type": "object",
                         "generation_args": {
-                                "type": "object",
-                                "description": "Parameters used to generate results - properties may vary by model type",
-                                "properties": {
-                                    "temperature": {
-                                        "type": [
-                                            "null",
-                                            "number"
-                                        ],
-                                        "description": "Sampling temperature"
-                                    },
-                                    "top_p": {
-                                        "type": [
-                                            "null",
-                                            "number"
-                                        ],
-                                        "description": "Nucleus sampling parameter"
-                                    },
-                                    "top_k": {
-                                        "type": [
-                                            "null",
-                                            "number"
-                                        ],
-                                        "description": "Top-k sampling parameter"
-                                    },
-                                    "max_tokens": {
-                                        "type": "integer",
-                                        "minimum": 1,
-                                        "description": "Maximum number of tokens to generate"
-                                    }
+                            "type": "object",
+                            "description": "Parameters used to generate results - properties may vary by model type",
+                            "properties": {
+                                "temperature": {
+                                    "type": [
+                                        "null",
+                                        "number"
+                                    ],
+                                    "description": "Sampling temperature"
                                 },
-                                "additionalProperties": true
+                                "top_p": {
+                                    "type": [
+                                        "null",
+                                        "number"
+                                    ],
+                                    "description": "Nucleus sampling parameter"
+                                },
+                                "top_k": {
+                                    "type": [
+                                        "null",
+                                        "number"
+                                    ],
+                                    "description": "Top-k sampling parameter"
+                                },
+                                "max_tokens": {
+                                    "type": "integer",
+                                    "minimum": 1,
+                                    "description": "Maximum number of tokens to generate"
+                                },
+                                "execution_command": {
+                                    "type": "string",
+                                    "description": "Command used to run the model to generate results"
+                                },
+                                "reasoning": {
+                                    "type": "boolean",
+                                    "description": "Whether reasoning orchain-of-thought was used to generate results"
+                                }
+                            },
+                            "additionalProperties": true
                         },
                         "additional_details": {
                             "type": "string",
@@ -276,7 +325,116 @@
                     }
                 }
             }
-
+        },
+        "detailed_evaluation_results_per_samples": {
+            "description": "Detailed eval results for all individual samples in the evaluation. This can be provided as source link or list of DetailedEvaluationResultsPerSample objects.",
+            "anyOf": [
+                {
+                    "type": "string",
+                    "description": "Link to detailed evaluation data for all samples"
+                },
+                {
+                    "type": "array",
+                    "description": "Array of evaluation results",
+                    "items": {
+                        "type": "object",
+                        "required": [
+                            "sample_id",
+                            "input",
+                            "ground_truth",
+                            "response"
+                        ],
+                        "properties": {
+                            "sample_id": {
+                                "type": "string",
+                                "description": "Simple sample ID"
+                            },
+                            "input": {
+                                "type": "string",
+                                "description": "Raw input for the model"
+                            },
+                            "prompt": {
+                                "type": "string",
+                                "description": "Full prompt for the model"
+                            },
+                            "ground_truth": {
+                                "description": "Target response that may include one or multiple correct answers.",
+                                "oneOf": [
+                                    {
+                                        "type": "string"
+                                    },
+                                    {
+                                        "type": "array",
+                                        "items": {
+                                            "type": "string"
+                                        }
+                                    }
+                                ]
+                            },
+                            "response": {
+                                "type": "string",
+                                "description": "Response from the model"
+                            },
+                            "choices": {
+                                "description": "Either an array of possible responses (list of strings) or an array of string pairs [choice, response].",
+                                "oneOf": [
+                                    {
+                                        "type": "array",
+                                        "items": {
+                                            "type": "string"
+                                        }
+                                    },
+                                    {
+                                        "type": "array",
+                                        "items": {
+                                            "type": "array",
+                                            "items": [
+                                                {
+                                                    "type": "string"
+                                                },
+                                                {
+                                                    "type": "string"
+                                                }
+                                            ],
+                                            "minItems": 2,
+                                            "maxItems": 2
+                                        }
+                                    }
+                                ]
+                            },
+                            "full_logprobs": {
+                                "type": "array",
+                                "description": "Full log probabilities generated for this sample",
+                                "items": {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "object",
+                                        "required": [
+                                            "token_id",
+                                            "logprob",
+                                            "decoded_token"
+                                        ],
+                                        "properties": {
+                                            "token_id": {
+                                                "type": "number",
+                                                "description": "Id of token for which we keep its logprob"
+                                            },
+                                            "logprob": {
+                                                "type": "number",
+                                                "description": "Log probability of the token"
+                                            },
+                                            "decoded_token": {
+                                                "type": "string",
+                                                "description": "The decoded string representation of the token"
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            ]
         }
     }
-}
+}

hf_operations.py

@@ -1,6 +1,7 @@
 """
 HuggingFace Operations: Upload data, create PRs, validate schemas.
 """
+
 from huggingface_hub import HfApi, login
 import pandas as pd
 import json
@@ -10,17 +11,17 @@ from jsonschema import validate, ValidationError, Draft7Validator
 
 # Load schema once at module level
 SCHEMA_PATH = Path(__file__).parent / "eval.schema.json"
-with open(SCHEMA_PATH, 'r') as f:
+with open(SCHEMA_PATH, "r") as f:
     EVAL_SCHEMA = json.load(f)
-    
+
 
 def validate_json_against_schema(json_data):
     """
     Validate a JSON object against eval.schema.json.
-    
+
     Args:
         json_data: Dict containing the evaluation data
-        
+
     Returns:
         (bool, str): (is_valid, error_message)
     """
@@ -38,7 +39,7 @@ def validate_json_against_schema(json_data):
 def upload_to_hf_dataset(parquet_file, split_name, repo_id):
     """
     Upload a parquet file as a new split to the HF dataset.
-    
+
     Args:
         parquet_file: Path to parquet file
         split_name: Name of the split (leaderboard name)
@@ -51,14 +52,14 @@ def upload_to_hf_dataset(parquet_file, split_name, repo_id):
 def check_hf_authentication():
     """
     Check if user is authenticated with HuggingFace.
-    
+
     Returns:
         (bool, str): (is_authenticated, username or error_message)
     """
     try:
         api = HfApi()
         user_info = api.whoami()
-        return True, user_info['name']
+        return True, user_info["name"]
     except Exception as e:
         return False, "Not authenticated. Run: huggingface-cli login"
 
@@ -66,18 +67,18 @@ def check_hf_authentication():
 def check_duplicate_pr_exists(leaderboard_name, repo_id):
     """
     Check if a PR already exists for this leaderboard.
-    
+
     Args:
         leaderboard_name: Name of the leaderboard
         repo_id: HuggingFace dataset repository ID
-        
+
     Returns:
         (bool, str or None): (exists, pr_url if exists)
     """
     try:
         api = HfApi()
         discussions = api.get_repo_discussions(repo_id=repo_id, repo_type="dataset")
-        
+
         # Check for open PRs with matching title
         pr_title_pattern = f"add new leaderboard: {leaderboard_name.lower()}"
         for discussion in discussions:
@@ -85,7 +86,7 @@ def check_duplicate_pr_exists(leaderboard_name, repo_id):
                 if pr_title_pattern in discussion.title.lower():
                     pr_url = f"https://huggingface.co/datasets/{repo_id}/discussions/{discussion.num}"
                     return True, pr_url
-        
+
         return False, None
     except Exception as e:
         # If we can't check, assume no duplicate (fail open)
@@ -96,12 +97,12 @@ def check_duplicate_pr_exists(leaderboard_name, repo_id):
 def create_pr_for_new_leaderboard(leaderboard_name, parquet_file, repo_id):
     """
     Create a pull request to add a new leaderboard split.
-    
+
     Args:
         leaderboard_name: Name of the new leaderboard
         parquet_file: Path to parquet file
         repo_id: HuggingFace dataset repository ID
-        
+
     Returns:
         (success, pr_url or error_message)
     """
@@ -109,28 +110,28 @@ def create_pr_for_new_leaderboard(leaderboard_name, parquet_file, repo_id):
     is_auth, auth_result = check_hf_authentication()
     if not is_auth:
         return False, f"❌ {auth_result}"
-    
+
     # 2. Check for duplicate PR
     has_duplicate, duplicate_url = check_duplicate_pr_exists(leaderboard_name, repo_id)
     if has_duplicate:
         return False, f"⚠️ PR already exists: {duplicate_url}"
-    
+
     # 3. Validate parquet file exists and has data
     parquet_path = Path(parquet_file)
     if not parquet_path.exists():
         return False, "❌ Parquet file not found"
-    
+
     df = pd.read_parquet(parquet_file)
     if len(df) == 0:
         return False, "❌ Parquet file is empty"
-    
+
     # 4. Create PR
     try:
         api = HfApi()
-        
+
         # Upload the parquet file to the branch
         commit_message = f"Add new leaderboard: {leaderboard_name}"
-        
+
         # Upload file and create PR
         commit_info = api.upload_file(
             path_or_fileobj=parquet_file,
@@ -140,12 +141,16 @@ def create_pr_for_new_leaderboard(leaderboard_name, parquet_file, repo_id):
             commit_message=commit_message,
             create_pr=True,
         )
-        
+
         # Extract PR URL from commit info
-        pr_url = commit_info.pr_url if hasattr(commit_info, 'pr_url') else f"https://huggingface.co/datasets/{repo_id}/discussions"
-        
+        pr_url = (
+            commit_info.pr_url
+            if hasattr(commit_info, "pr_url")
+            else f"https://huggingface.co/datasets/{repo_id}/discussions"
+        )
+
         return True, f"PR created ({len(df)} rows): {pr_url}"
-        
+
     except Exception as e:
         return False, f"❌ Failed to create PR: {str(e)}"
 
@@ -153,37 +158,50 @@ def create_pr_for_new_leaderboard(leaderboard_name, parquet_file, repo_id):
 def validate_schema(parquet_file):
     """
     Validate that a parquet file matches the expected schema.
-    
+
     Args:
         parquet_file: Path to parquet file to validate
-        
+
     Returns:
         (bool, str): (is_valid, error_message)
     """
     try:
         df = pd.read_parquet(parquet_file)
-        
+
         # Required columns
         required_cols = [
-            '_leaderboard', '_developer', '_model', '_uuid',
-            'schema_version', 'evaluation_id', 'retrieved_timestamp',
-            'source_data', 'evaluation_source_name', 'evaluation_source_type',
-            'source_organization_name', 'evaluator_relationship',
-            'model_name', 'model_id', 'model_developer',
-            'evaluation_results'
+            "_leaderboard",
+            "_developer",
+            "_model",
+            "_uuid",
+            "schema_version",
+            "evaluation_id",
+            "retrieved_timestamp",
+            "source_data",
+            "evaluation_source_name",
+            "evaluation_source_type",
+            "source_organization_name",
+            "evaluator_relationship",
+            "model_name",
+            "model_id",
+            "model_developer",
+            "evaluation_results",
         ]
-        
+
         missing = [col for col in required_cols if col not in df.columns]
         if missing:
             return False, f"Missing required columns: {', '.join(missing)}"
-        
+
         # Check data types (all should be strings)
         for col in df.columns:
-            if df[col].dtype not in ['object', 'string']:
-                return False, f"Column '{col}' has wrong type: {df[col].dtype} (expected string)"
-        
+            if df[col].dtype not in ["object", "string"]:
+                return (
+                    False,
+                    f"Column '{col}' has wrong type: {df[col].dtype} (expected string)",
+                )
+
         return True, "Schema validation passed"
-    
+
     except Exception as e:
         return False, f"Validation error: {str(e)}"
 
@@ -192,11 +210,11 @@ def export_to_json(parquet_file, output_dir):
     """
     Export parquet data back to JSON files.
     Uses the parquet_to_folder function from json_to_parquet.py
-    
+
     Args:
         parquet_file: Path to parquet file
         output_dir: Directory to write JSON files to
     """
     from json_to_parquet import parquet_to_folder
-    parquet_to_folder(parquet_file, output_dir)
 
+    parquet_to_folder(parquet_file, output_dir)

ui_components.py

@@ -752,17 +752,19 @@ table tr:hover td {
 def format_leaderboard_header(selected_leaderboard, metadata):
     if not selected_leaderboard:
         return '<div style="text-align: center; padding: 2rem; color: #525252;">Select a leaderboard to explore</div>'
-    
+
     if not metadata or not metadata.get("evals"):
         return f'<div class="info-banner"><h3>{selected_leaderboard}</h3></div>'
-    
+
     source_info = metadata.get("source_info", {})
     org = source_info.get("organization", "Unknown")
     url = source_info.get("url", "#")
     eval_names = sorted(list(metadata["evals"].keys()))
-    
-    eval_tags = "".join([f'<span class="eval-tag">{name}</span>' for name in eval_names])
-    
+
+    eval_tags = "".join(
+        [f'<span class="eval-tag">{name}</span>' for name in eval_names]
+    )
+
     return f'''
     <div class="info-banner">
         <div class="leaderboard-header">
@@ -785,14 +787,16 @@ def format_metric_details(selected_leaderboard, metadata):
 
     cards_html = ""
     for i, (eval_name, info) in enumerate(evals.items()):
-        score_type = info.get('score_type', '').upper() or "—"
-        direction = "Lower is better" if info.get('lower_is_better') else "Higher is better"
-        arrow = "↓" if info.get('lower_is_better') else "↑"
+        score_type = info.get("score_type", "").upper() or "—"
+        direction = (
+            "Lower is better" if info.get("lower_is_better") else "Higher is better"
+        )
+        arrow = "↓" if info.get("lower_is_better") else "↑"
 
         details = ""
-        if info.get('score_type') == "continuous" and info.get('min_score') is not None:
+        if info.get("score_type") == "continuous" and info.get("min_score") is not None:
             details = f"Range: [{info['min_score']} – {info['max_score']}]"
-        elif info.get('score_type') == "levels" and info.get('level_names'):
+        elif info.get("score_type") == "levels" and info.get("level_names"):
             details = f"Levels: {', '.join(str(l) for l in info['level_names'])}"
 
         card_id = f"mc{i}"
@@ -804,7 +808,7 @@ def format_metric_details(selected_leaderboard, metadata):
                 <span class="metric-card-direction"><span class="arrow">{arrow}</span> {direction}</span>
             </label>
             <div class="metric-card-body">
-                <div>{info.get('description', 'No description')}</div>
+                <div>{info.get("description", "No description")}</div>
                 <div style="display: flex; justify-content: space-between; align-items: center; margin-top: 0.5rem;">
                     <span style="font-size: 0.75rem; color: #525252;">{details}</span>
                     <span class="metric-type-badge">{score_type}</span>
@@ -813,25 +817,25 @@ def format_metric_details(selected_leaderboard, metadata):
         </div>
         '''
 
-    return f'''
+    return f"""
     <div class="metrics-section">
         <h3>Metric Reference</h3>
         <div class="metrics-grid">{cards_html}</div>
     </div>
-    '''
+    """
 
 
 def format_model_card(model_name, model_data):
     if not model_data:
         return '<div class="no-results"><h3>No results found</h3><p>Try a different model name</p></div>'
-    
+
     first = list(model_data.values())[0]
     developer = first.get("developer", "Unknown")
     params = first.get("params")
     arch = first.get("architecture", "Unknown")
     params_str = f"{params}B" if params else "—"
-    
-    html = f'''
+
+    html = f"""
     <div style="padding: 1rem; background: #ffffff; border-radius: 10px; border: 1px solid #e5e5e5;">
         <h2 style="margin: 0 0 0.5rem 0; color: #0a0a0a;">{model_name}</h2>
         <div style="color: #525252; margin-bottom: 1rem;">
@@ -839,80 +843,86 @@ def format_model_card(model_name, model_data):
             <span>Params: {params_str}</span> · 
             <span>Arch: {arch}</span>
         </div>
-    '''
-    
+    """
+
     for leaderboard_name, data in model_data.items():
         results = data.get("results", {})
         if not results:
             continue
-        
+
         scores = [v for v in results.values() if v is not None]
         avg = sum(scores) / len(scores) if scores else None
         avg_str = f"{avg:.2f}" if avg else "—"
-        
+
         html += f'<div style="margin-bottom: 1rem;"><h4 style="color: #0a0a0a;">{leaderboard_name} <span style="color: #525252;">(avg: {avg_str})</span></h4>'
         html += '<div style="display: flex; flex-wrap: wrap; gap: 0.5rem;">'
-        
-        for metric_name, score in sorted(results.items(), key=lambda x: x[1] if x[1] else 0, reverse=True):
+
+        for metric_name, score in sorted(
+            results.items(), key=lambda x: x[1] if x[1] else 0, reverse=True
+        ):
             score_display = f"{score:.2f}" if score is not None else "—"
             html += f'<div style="padding: 0.4rem 0.8rem; border-radius: 6px; background: #f5f5f5; border: 1px solid #e5e5e5;"><span style="color: #525252;">{metric_name}:</span> <strong style="color: #0a0a0a;">{score_display}</strong></div>'
-        
-        html += '</div></div>'
-    
-    html += '</div>'
+
+        html += "</div></div>"
+
+    html += "</div>"
     return html
 
 
 def format_model_comparison(selected_models, all_results):
     if not selected_models or not all_results:
         return '<div class="no-results"><h3>Select models to compare</h3><p>Choose models from the dropdown</p></div>'
-    
+
     all_leaderboards = set()
     model_data_dict = {}
-    
+
     for model_name in selected_models:
         if model_name in all_results:
             model_data_dict[model_name] = all_results[model_name]
             for lb in all_results[model_name].keys():
                 all_leaderboards.add(lb)
-    
+
     if not model_data_dict:
         return '<div class="no-results"><h3>No data found</h3></div>'
-    
+
     all_leaderboards = sorted(all_leaderboards)
-    
+
     html = '<div style="padding: 1rem; background: #ffffff; border-radius: 10px; border: 1px solid #e5e5e5;">'
-    
+
     for leaderboard_name in all_leaderboards:
         metrics = set()
         for md in model_data_dict.values():
             if leaderboard_name in md:
                 metrics.update(md[leaderboard_name].get("results", {}).keys())
-        
+
         metrics = sorted(metrics)
         if not metrics:
             continue
-        
+
         html += f'<h3 style="margin: 1rem 0 0.5rem; color: #0a0a0a;">{leaderboard_name}</h3>'
         html += '<div style="overflow-x: auto;"><table class="heatmap-table"><thead><tr><th>Metric</th>'
-        
+
         for model_name in selected_models:
             short = model_name[:20] + "…" if len(model_name) > 20 else model_name
             html += f'<th title="{model_name}">{short}</th>'
-        html += '</tr></thead><tbody>'
-        
+        html += "</tr></thead><tbody>"
+
         for metric_name in metrics:
             html += f'<tr><td class="metric-name">{metric_name}</td>'
-            
+
             scores = {}
             for m in selected_models:
                 if m in model_data_dict and leaderboard_name in model_data_dict[m]:
-                    scores[m] = model_data_dict[m][leaderboard_name].get("results", {}).get(metric_name)
-            
+                    scores[m] = (
+                        model_data_dict[m][leaderboard_name]
+                        .get("results", {})
+                        .get(metric_name)
+                    )
+
             valid = [v for v in scores.values() if v is not None]
             max_s = max(valid) if valid else None
             min_s = min(valid) if valid else None
-            
+
             for model_name in selected_models:
                 score = scores.get(model_name)
                 if score is not None:
@@ -921,7 +931,15 @@ def format_model_comparison(selected_models, all_results):
                             cls = "best"
                         elif max_s > min_s:
                             pct = (score - min_s) / (max_s - min_s)
-                            cls = "good" if pct >= 0.75 else "mid" if pct >= 0.5 else "low" if pct >= 0.25 else "worst"
+                            cls = (
+                                "good"
+                                if pct >= 0.75
+                                else "mid"
+                                if pct >= 0.5
+                                else "low"
+                                if pct >= 0.25
+                                else "worst"
+                            )
                         else:
                             cls = ""
                     else:
@@ -929,11 +947,11 @@ def format_model_comparison(selected_models, all_results):
                     html += f'<td class="score-cell {cls}">{score:.2f}</td>'
                 else:
                     html += '<td class="score-cell na">—</td>'
-            html += '</tr>'
-        
-        html += '</tbody></table></div>'
-    
-    html += '</div>'
+            html += "</tr>"
+
+        html += "</tbody></table></div>"
+
+    html += "</div>"
     return html
 
 
@@ -947,13 +965,14 @@ def create_radar_plot(selected_models, all_results):
     for model in selected_models:
         if model not in all_results:
             continue
-        
+
         model_data = all_results[model]
         for lb_name, lb_data in model_data.items():
             leaderboards_involved.add(lb_name)
             results = lb_data.get("results", {})
             for metric, score in results.items():
-                if score is None: continue
+                if score is None:
+                    continue
                 key = f"{lb_name}: {metric}"
                 if key not in metric_data:
                     metric_data[key] = {}
@@ -967,17 +986,17 @@ def create_radar_plot(selected_models, all_results):
         meta_cache[lb] = get_eval_metadata(lb)
 
     fig = go.Figure()
-    
+
     categories = sorted(metric_data.keys())
-    
+
     for model in selected_models:
         r_values = []
         theta_values = []
         hover_texts = []
-        
+
         for cat in categories:
             lb_name, metric_name = cat.split(": ", 1)
-            
+
             val = metric_data[cat].get(model)
             if val is None:
                 r_values.append(None)
@@ -987,15 +1006,15 @@ def create_radar_plot(selected_models, all_results):
                 meta = meta_cache.get(lb_name, {}).get("evals", {}).get(metric_name, {})
                 min_s = meta.get("min_score")
                 max_s = meta.get("max_score")
-                
+
                 observed_vals = []
                 for m in selected_models:
                     v = metric_data[cat].get(m)
                     if v is not None:
                         observed_vals.append(v)
-                
+
                 observed_max = max(observed_vals) if observed_vals else 1.0
-                
+
                 if min_s is None:
                     min_s = 0
                 if max_s is None:
@@ -1004,42 +1023,39 @@ def create_radar_plot(selected_models, all_results):
                     else:
                         max_s = 1
                     max_s = max(max_s, observed_max)
-                
+
                 if max_s == min_s:
                     norm_val = 1.0
                 else:
                     norm_val = (val - min_s) / (max_s - min_s)
-                
+
                 norm_val = max(0.0, min(1.0, norm_val))
-                
+
                 r_values.append(norm_val)
                 theta_values.append(cat)
                 hover_texts.append(f"{cat}<br>Score: {val:.2f} (Norm: {norm_val:.2f})")
-            
+
         if r_values:
             r_values.append(r_values[0])
             theta_values.append(theta_values[0])
             hover_texts.append(hover_texts[0])
-        
-        fig.add_trace(go.Scatterpolar(
-            r=r_values,
-            theta=theta_values,
-            name=model,
-            hovertext=hover_texts,
-            hoverinfo="text",
-            fill='toself'
-        ))
 
-    fig.update_layout(
-        polar=dict(
-            radialaxis=dict(
-                visible=True,
-                range=[0, 1]
+        fig.add_trace(
+            go.Scatterpolar(
+                r=r_values,
+                theta=theta_values,
+                name=model,
+                hovertext=hover_texts,
+                hoverinfo="text",
+                fill="toself",
             )
-        ),
+        )
+
+    fig.update_layout(
+        polar=dict(radialaxis=dict(visible=True, range=[0, 1])),
         showlegend=True,
         margin=dict(l=80, r=80, t=20, b=20),
-        title="Model Comparison Radar (Normalized Scores)"
+        title="Model Comparison Radar (Normalized Scores)",
     )
-    
+
     return fig