GSK-2498-suggest-a-dataset-for-model

app_leaderboard.py

@@ -7,6 +7,7 @@ from fetch_utils import (check_dataset_and_get_config,
                          check_dataset_and_get_split)
 from text_classification_ui_helpers import LEADERBOARD
 
+import leaderboard
 
 def get_records_from_dataset_repo(dataset_id):
     dataset_config = check_dataset_and_get_config(dataset_id)
@@ -74,7 +75,8 @@ def get_display_df(df):
 
 
 def get_demo():
-    records = get_records_from_dataset_repo(LEADERBOARD)
+    leaderboard.records = get_records_from_dataset_repo(LEADERBOARD)
+    records = leaderboard.records
 
     model_ids = get_model_ids(records)
     dataset_ids = get_dataset_ids(records)
@@ -124,6 +126,7 @@ def get_demo():
         outputs=[leaderboard_df],
     )
     def filter_table(model_id, dataset_id, columns, task):
+        records = leaderboard.records
         # filter the table based on task
         df = records[(records["task"] == task)]
         # filter the table based on the model_id and dataset_id

app_text_classification.py

@@ -4,6 +4,7 @@ import gradio as gr
 
 from io_utils import get_logs_file, read_scanners, write_scanners
 from text_classification_ui_helpers import (
+    get_related_datasets_from_leaderboard,
     align_columns_and_show_prediction,
     check_dataset,
     deselect_run_inference,
@@ -18,7 +19,6 @@ MAX_LABELS = 40
 MAX_FEATURES = 20
 
 EXAMPLE_MODEL_ID = "cardiffnlp/twitter-roberta-base-sentiment-latest"
-EXAMPLE_DATA_ID = "tweet_eval"
 CONFIG_PATH = "./config.yaml"
 
 
@@ -34,10 +34,13 @@ def get_demo():
             placeholder=EXAMPLE_MODEL_ID + " (press enter to confirm)",
         )
 
-        dataset_id_input = gr.Textbox(
-            label="Hugging Face Dataset id",
-            placeholder=EXAMPLE_DATA_ID + " (press enter to confirm)",
-        )
+        with gr.Column():
+            dataset_id_input = gr.Dropdown(
+                choices=[],
+                value="",
+                allow_custom_value=True,
+                label="Hugging Face Dataset id",
+            )
 
     with gr.Row():
         dataset_config_input = gr.Dropdown(label="Dataset Config", visible=False, allow_custom_value=True)
@@ -77,15 +80,16 @@ def get_demo():
                     for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
                         column_mappings.append(gr.Dropdown(visible=False))
 
-    with gr.Accordion(label="Model Wrap Advance Config (optional)", open=False):
-        run_local = gr.Checkbox(value=True, label="Run in this Space")
-        run_inference = gr.Checkbox(value=False, label="Run with Inference API")
+    with gr.Accordion(label="Model Wrap Advance Config", open=True):
+        run_inference = gr.Checkbox(value=True, label="Run with Inference API")
         inference_token = gr.Textbox(
             value="",
             label="HF Token for Inference API",
-            visible=False,
+            visible=True,
             interactive=True,
         )
+        run_local = gr.Checkbox(value=False, label="Run Locally with Pipeline [Slow]")
+
 
     with gr.Accordion(label="Scanner Advance Config (optional)", open=False):
         scanners = gr.CheckboxGroup(label="Scan Settings", visible=True)
@@ -149,6 +153,13 @@ def get_demo():
         outputs=[inference_token, run_inference],
     )
 
+    gr.on(
+        triggers=[model_id_input.change],
+        fn=get_related_datasets_from_leaderboard,
+        inputs=[model_id_input],
+        outputs=[dataset_id_input],
+    )
+
     gr.on(
         triggers=[label.change for label in column_mappings],
         fn=write_column_mapping_to_config,
@@ -196,6 +207,8 @@ def get_demo():
             dataset_config_input,
             dataset_split_input,
             uid_label,
+            run_inference,
+            inference_token,
         ],
         outputs=[
             example_input,
@@ -225,7 +238,11 @@ def get_demo():
         outputs=[run_btn, logs, uid_label],
     )
 
-    def enable_run_btn():
+    def enable_run_btn(run_inference, inference_token, model_id, dataset_id, dataset_config, dataset_split):
+        if run_inference and inference_token == "":
+            return gr.update(interactive=False)
+        if model_id == "" or dataset_id == "" or dataset_config == "" or dataset_split == "":
+            return gr.update(interactive=False)
         return gr.update(interactive=True)
 
     gr.on(
@@ -236,13 +253,27 @@ def get_demo():
             scanners.input,
         ],
         fn=enable_run_btn,
-        inputs=None,
+        inputs=[
+            run_inference, 
+            inference_token, 
+            model_id_input, 
+            dataset_id_input, 
+            dataset_config_input, 
+            dataset_split_input
+        ],
         outputs=[run_btn],
     )
 
     gr.on(
         triggers=[label.input for label in column_mappings],
         fn=enable_run_btn,
-        inputs=None,  # FIXME
+        inputs=[
+            run_inference, 
+            inference_token, 
+            model_id_input, 
+            dataset_id_input, 
+            dataset_config_input, 
+            dataset_split_input
+        ],  # FIXME
         outputs=[run_btn],
     )

leaderboard.py

@@ -0,0 +1,3 @@
+import pandas as pd
+
+records = pd.DataFrame()

text_classification_ui_helpers.py

@@ -4,6 +4,7 @@ import logging
 import os
 import threading
 import uuid
+import leaderboard
 
 import datasets
 import gradio as gr
@@ -42,6 +43,15 @@ HF_GSK_HUB_HF_TOKEN = "GSK_HF_TOKEN"
 HF_GSK_HUB_UNLOCK_TOKEN = "GSK_HUB_UNLOCK_TOKEN"
 
 LEADERBOARD = "giskard-bot/evaluator-leaderboard"
+def get_related_datasets_from_leaderboard(model_id):
+    records = leaderboard.records
+    model_records = records[records["model_id"] == model_id]
+    datasets_unique = model_records["dataset_id"].unique()
+    if len(datasets_unique) == 0:
+        all_unique_datasets = list(records["dataset_id"].unique())
+        print(type(all_unique_datasets), all_unique_datasets)
+        return gr.update(choices=all_unique_datasets, value="")
+    return gr.update(choices=datasets_unique, value=datasets_unique[0])
 
 
 logger = logging.getLogger(__file__)
@@ -207,7 +217,7 @@ def precheck_model_ds_enable_example_btn(
 
 
 def align_columns_and_show_prediction(
-    model_id, dataset_id, dataset_config, dataset_split, uid
+    model_id, dataset_id, dataset_config, dataset_split, uid, run_inference, inference_token
 ):
     ppl = check_model(model_id)
     if ppl is None or not isinstance(ppl, TextClassificationPipeline):
@@ -268,7 +278,7 @@ def align_columns_and_show_prediction(
             gr.update(value=MAPPING_STYLED_ERROR_WARNING, visible=True),
             gr.update(visible=False),
             gr.update(visible=True, open=True),
-            gr.update(interactive=True),
+            gr.update(interactive=(run_inference and inference_token != "")),
             "",
             *column_mappings,
         )
@@ -280,7 +290,7 @@ def align_columns_and_show_prediction(
         gr.update(value=get_styled_input(prediction_input), visible=True),
         gr.update(value=prediction_output, visible=True),
         gr.update(visible=True, open=False),
-        gr.update(interactive=True),
+        gr.update(interactive=(run_inference and inference_token != "")),
         "",
         *column_mappings,
     )