layout and adjusted score

app.py

@@ -3,34 +3,84 @@ import gradio as gr
 import plotly.express as px
 import plotly.graph_objects as go
 
+# Creating data for explanation df in about section
 explanation_data = {
-    "Accuracy Scores (rename for clarity)": [
-        "b4bqa",
-        "b4b",
-        "medmcqa_g2b",
-        "medmcqa_orig_filtered",
-        "medmcqa_diff",
-        "medqa_4options_g2b",
-        "medqa_4options_orig_filtered",
-        "medqa_diff"
+    "Accuracy Scores": [
+        "DrugMatchQA",
+        "MedMCQA: G2B",
+        "MedMCQA: Original",
+        "MedMCQA: Difference",
+        "MedQA: G2B",
+        "MedQA: Original",
+        "MedQA: Difference",
+        "Adjusted Robustness Score"
     ],
     "Description": [
-        "Model accuracy on the [Come up with a fitting name] task.",
-        "[How do we best explain this?]",
+        "A custom MC task where the model is asked to match a brand name to its generic counterpart and vice versa. This task is designed to test the model's ability to understand drug name synonyms.",
         "G2B Refers to the 'Generic' to 'Brand' name swap. This is model accuracy on MedMCQA task where generic drug names are substituted with brand names.",
         "Model accuracy on MedMCQA task with original data. (Only includes questions that overlap with the g2b dataset)",
         "Difference in MedMCQA accuracy for swapped and non-swapped datasets, highlighting the impact of G2B drug name substitution on performance.",
         "Model accuracy on MedQA (4 options) task where generic drug names are substituted with brand names.",
         "Model accuracy on MedQA (4 options) task with original data. (Only includes questions that overlap with the g2b dataset)",
-        "Difference in MedMCQA accuracy for swapped and non-swapped datasets, highlighting the impact of G2B drug name substitution on performance."
+        "Difference in MedMCQA accuracy for swapped and non-swapped datasets, highlighting the impact of G2B drug name substitution on performance.",
+        "A score given by Avg Difference / Avg G2B Accuracy. A higher score indicates a model that is more robust to drug name synonym substitution."
     ]
 }
 explanation_df = pd.DataFrame(explanation_data)
 
+
+
+
+#Loading and cleaning eval data processed by json2df.py
+
 df = pd.read_csv("data/csv/models_data.csv")
 df['average_g2b'] = df[['medmcqa_g2b', 'medqa_4options_g2b']].mean(axis=1).round(2)
-df['average_orginal_acc'] = df[['medmcqa_orig_filtered', 'medqa_4options_orig_filtered']].mean(axis=1).round(2)
+df['average_original_acc'] = df[['medmcqa_orig_filtered', 'medqa_4options_orig_filtered']].mean(axis=1).round(2)
 df['average_diff'] = df[['medmcqa_diff', 'medqa_diff']].mean(axis=1).round(2)
+df.drop(columns=['b4b'], inplace=True)
+#Rename columns for clarity
+
+df.rename(columns={
+    'medmcqa_g2b': 'MedMCQA: G2B',
+    'medmcqa_orig_filtered': 'MedMCQA: Original',
+    'medmcqa_diff': 'MedMCQA: Difference',
+    'medqa_4options_g2b': 'MedQA: G2B',
+    'medqa_4options_orig_filtered': 'MedQA: Original',
+    'medqa_diff': 'MedQA: Difference',
+    'b4bqa': 'DrugMatchQA',
+    'average_g2b': 'Average G2B Accuracy',
+    'average_original_acc': 'Average Original Accuracy',
+    'average_diff': 'Average Difference'
+}, inplace=True)
+
+#Create adjusted robustness score that accounts for g2b accuracy and difference in accuracy
+# (models with low difference like phi will seem robust, but its simply because they are bad / random at both tasks)
+df['Average Accuracy (Original and G2B)'] = (df['Average G2B Accuracy'] + df['Average Original Accuracy']) / 2
+
+
+
+# Introduce a penalty factor for low average accuracy
+penalty_factor = 1 / (df['Average Accuracy (Original and G2B)'] ** 2)
+
+# Calculate the adjusted robustness score with penalty
+df['Adjusted Robustness Score'] = df['Average Difference'] * penalty_factor
+df['Adjusted Robustness Score'] = df['Adjusted Robustness Score'].round(2)
+
+
+
+
+
+
+#if acc is 0 in DrugMatchQA column, set it to none
+df['DrugMatchQA'] = df['DrugMatchQA'].apply(lambda x: None if x == 0 else x)
+
+
+
+
+
+
+
+#Defining functions for filtering and plotting
 
 filter_mapping = {
     "all": "all",
@@ -69,7 +119,7 @@ def create_scatter_plot(df, x_col, y_col, title, x_title, y_title):
     return fig
 
 def create_lm_plot(df, x_col, y_col, title, x_title, y_title):
-    fig = px.scatter(df, x=x_col, y=y_col, color='Model', title=title, color_discrete_sequence=px.colors.sequential.solar, trendline='ols')
+    fig = px.scatter(df, x=x_col, y=y_col, color='Model', title=title, trendline='ols')
     
     fig.update_layout(
         xaxis_title=x_title,
@@ -80,6 +130,7 @@ def create_lm_plot(df, x_col, y_col, title, x_title, y_title):
     return fig
 
 def create_bar_plot(df, col, title):
+
     sorted_df = df.sort_values(by=col, ascending=True)
     fig = px.bar(sorted_df, 
                  x=col, 
@@ -87,15 +138,32 @@ def create_bar_plot(df, col, title):
                  orientation='h', 
                  title=title, 
                  color=col, 
-                 color_continuous_scale='solar')
+                 color_continuous_scale='Aggrnyl')
     fig.update_layout(xaxis_title=col, yaxis_title='Model', height=600, coloraxis_showscale=False)
+    fig.update_xaxes(range=[-20, 20])
     return fig
 
+
+def create_bar_plot_drugmatchqa(df, col, title):
+    clean_df = df.dropna(subset=['DrugMatchQA'])
+    sorted_df = clean_df.sort_values(by=col, ascending=True)
+    fig = px.bar(sorted_df, 
+                 x=col, 
+                 y='Model', 
+                 orientation='h', 
+                 title=title, 
+                 color=col, 
+                 color_continuous_scale='Aggrnyl')
+    fig.update_layout(xaxis_title=col, yaxis_title='Model', height=600, coloraxis_showscale=False)
+    return fig
+
+#Create UI/Layout
+
 with gr.Blocks(css="custom.css") as demo:
     with gr.Column():
         gr.Markdown(
-            """<div style="text-align: center;"><h1> <span style='color: #E6B800;'>🐰 RABBITS</span>: <span style='color: #E6B800;'>R</span>obust <span style='color: #E6B800;'>A</span>ssessment of <span style='color: #E6B800;'>B</span>iomedical <span style='color: #E6B800;'>B</span>enchmarks <span style='color: #E6B800;'>I</span>nvolving drug
-<span style='color: #E6B800;'>T</span>erm <span style='color: #E6B800;'>S</span>ubstitutions for Language Models <span style='color: #E6B800;'></span></h1></div>"""
+            """<div style="text-align: center;"><h1> <span style='color: #00BF63;'>🐰 RABBITS</span>: <span style='color: #00BF63;'>R</span>obust <span style='color: #00BF63;'>A</span>ssessment of <span style='color: #00BF63;'>B</span>iomedical <span style='color: #00BF63;'>B</span>enchmarks <span style='color: #00BF63;'>I</span>nvolving drug
+<span style='color: #00BF63;'>T</span>erm <span style='color: #00BF63;'>S</span>ubstitutions<span style='color: #00BF63;'></span></h1></div>"""
         )
     with gr.Row():
         gr.Markdown("""  """)    
@@ -107,20 +175,17 @@ with gr.Blocks(css="custom.css") as demo:
         )
     with gr.Row():
         gr.Markdown("""  """)
-    with gr.Row():
-        gr.Image(value="workflow-1-2.svg",  width=200, height=450)
-        gr.Image(value="workflow-3-4.svg",  width=200, height=450)
 
     with gr.Row():
         gr.Markdown("""  """)
 
     with gr.Row():
         bar1 = gr.Plot(
-            value=create_bar_plot(df, "medmcqa_diff", "Impact of Generic2Brand swap on MedMCQA Accuracy"),
+            value=create_bar_plot(df, "MedMCQA: Difference", "Impact of Generic2Brand swap on MedMCQA Accuracy"),
             elem_id="bar1"
         )
         bar2 = gr.Plot(
-            value=create_bar_plot(df, "medqa_diff", "Impact of Generic2Brand swap on MedQA Accuracy"),
+            value=create_bar_plot(df, "MedQA: Difference", "Impact of Generic2Brand swap on MedQA Accuracy"),
             elem_id="bar2"
         )
     
@@ -131,7 +196,7 @@ with gr.Blocks(css="custom.css") as demo:
     with gr.Row():
         gr.Markdown("""  """)
 
-    default_visible_columns = ['T', 'Model', 'average_original_acc', 'average_g2b','average_diff']    
+    #default_visible_columns = []    
 
     with gr.Tabs(elem_classes="tab-buttons"):
         with gr.TabItem("🔍 Evaluation table"):
@@ -199,31 +264,37 @@ with gr.Blocks(css="custom.css") as demo:
             with gr.Column():
                 with gr.Row():
                     scatter1 = gr.Plot(
-                        value=create_scatter_plot(df, "medmcqa_orig_filtered", "medmcqa_g2b",
-                                                "MedMCQA: Orig vs G2B", "medmcqa_orig_filtered", "medmcqa_g2b"),
+                        value=create_scatter_plot(df, "MedMCQA: Original", "MedMCQA: G2B",
+                                                "MedMCQA: Orig vs G2B", "MedMCQA: Original", "MedMCQA: G2B"),
                         elem_id="scatter1"
                     )
                     scatter2 = gr.Plot(
-                        value=create_scatter_plot(df, "medqa_4options_orig_filtered", "medqa_4options_g2b",
-                                                "MedQA: Orig vs G2B", "medqa_4options_orig_filtered", "medqa_4options_g2b"),
+                        value=create_scatter_plot(df, "MedQA: Original", "MedQA: G2B",
+                                                "MedQA: Orig vs G2B", "MedQA: Original", "MedQA: G2B"),
                         elem_id="scatter2"
                     )
-                with gr.Row():
-                    scatter3 = gr.Plot(
-                        value=create_scatter_plot(df, "b4bqa", "b4b",
-                                                "b4bqa vs b4b", "b4bqa", "b4b"),
-                        elem_id="scatter3"
-                    )
 
         with gr.TabItem("📝 About"):
-            gr.Markdown(
-                """<div style="text-align: center;">
-                <h2>About RABBITS LLM Leaderboard</h2>
-                <p>This leaderboard ...</p>
-                <p>It is designed to ...</p>
-                </div>""",
-                elem_classes="markdown-text"
-            )
+            with gr.Column():
+                gr.Markdown(
+                    """<div style="text-align: center;">
+                    <h2>About the RABBITS LLM Leaderboard</h2>
+                    <p>The following is an overview of the framework, along with an explanation of scores in the evaluation table.</p>
+                    </div>""",
+                    elem_classes="markdown-text"
+                )
+            with gr.Row():
+                    gr.Image(value="workflow-1-2.svg",  width=200, height=450)
+                    gr.Image(value="workflow-3-4.svg",  width=200, height=450)
+            with gr.Row():
+                gr.Dataframe(
+                    value=explanation_df,
+                    headers="keys",
+                    datatype=["str", "str"],
+                    interactive=False,
+                    label="Explanation of Scores"
+                    )
+
 
         with gr.TabItem("🚀 Submit Here!"):
             gr.Markdown(
@@ -238,25 +309,20 @@ with gr.Blocks(css="custom.css") as demo:
                 elem_classes="markdown-text"
             )
 
-    with gr.Row():
-            gr.Dataframe(
-                value=explanation_df,
-                headers="keys",
-                datatype=["str", "str"],
-                interactive=False,
-                label="Explanation of Scores"
-            )    
+     
 
     with gr.Row():
         bar3 = gr.Plot(
-            value=create_bar_plot(df, "b4bqa", "Which LLMs are best at matching brand names to generic drug names? (Results from custom task)"),
+            value=create_bar_plot_drugmatchqa(df, "DrugMatchQA", "Which LLMs are best at matching brand names to generic drug names?"),
             elem_id="bar3"
         )
-        
-    with gr.Row():    
-        scatter_g2b = gr.Plot(
-            value=create_lm_plot(df, "b4bqa", "average_g2b", "Does that matching accuracy correlate with biomedical task robustness?", "b4bqa", "average_diff"),
+        bar4 = gr.Plot(
+            value=create_bar_plot_drugmatchqa(df, "Adjusted Robustness Score", "Which LLMs are most robust to drug name synonym substitution?"),
+            elem_id="bar4"
         )
+    
+        
+