updated with full category results

app.py

@@ -12,7 +12,6 @@ import pandas as pd
 # notebook_url = "https://colab.research.google.com/drive/1RAWb22-PFNI-X1gPVzc927SGUdfr6nsR?usp=sharing"
 notebook_url = "https://colab.research.google.com/drive/1KdwokPjirkTmpO_P1WByFNFiqxWQquwH#scrollTo=o_CpbkGEbhrK"
 
-
 basic_component_values = [None] * 6
 leader_component_values = [None] * 5
 
@@ -31,20 +30,25 @@ We've collected over **500,000** human preference votes to rank LLMs with the El
     return leaderboard_md
 
 
-def make_arena_leaderboard_md(arena_df, arena_subset_df=None, name="Overall"):
+def make_arena_leaderboard_md(arena_df):
     total_votes = sum(arena_df["num_battles"]) // 2
     total_models = len(arena_df)
     space = "   "
-    if arena_subset_df is not None:
-        total_subset_votes = sum(arena_subset_df["num_battles"]) // 2
-        total_subset_models = len(arena_subset_df)
-        vote_str = f"{space} {name} #models: **{total_subset_models}**.{space} {name} #votes: **{'{:,}'.format(total_subset_votes)}**."
-    else:
-        vote_str = ""
     leaderboard_md = f"""
-Total #models: **{total_models}**.{space} Total #votes: **{"{:,}".format(total_votes)}**.{vote_str}{space} Last updated: March 29, 2024.
+Total #models: **{total_models}**.{space} Total #votes: **{"{:,}".format(total_votes)}**.{space} Last updated: March 29, 2024.
+
+**NEW!** View ELO leaderboard and stats for different input categories.
+"""
+    return leaderboard_md
 
-**NEW!** Click the buttons below to view the ELO leaderboard and stats for different input categories. You are currently viewing **{name}** inputs.
+def make_category_arena_leaderboard_md(arena_df, arena_subset_df, name="Overall"):
+    total_votes = sum(arena_df["num_battles"]) // 2
+    total_models = len(arena_df)
+    space = "   "
+    total_subset_votes = sum(arena_subset_df["num_battles"]) // 2
+    total_subset_models = len(arena_subset_df)
+    leaderboard_md = f"""### {name} Question Coverage
+#models: **{total_subset_models} ({round(total_subset_models/total_models *100)}%)**.{space} #votes: **{"{:,}".format(total_subset_votes)} ({round(total_subset_votes/total_votes * 100)}%)**.{space}
 """
     return leaderboard_md
 
@@ -279,19 +283,11 @@ def get_arena_table(arena_df, model_table_df, arena_subset_df=None):
             print(f"{model_key} - {e}")
     return values
 
-def update_leaderboard_and_plots(button, arena_df, model_table_df, arena_subset_df, elo_subset_results):
-    arena_values = get_arena_table(arena_df, model_table_df, arena_subset_df)
-    p1 = elo_subset_results["win_fraction_heatmap"]
-    p2 = elo_subset_results["battle_count_heatmap"]
-    p3 = elo_subset_results["bootstrap_elo_rating"]
-    p4 = elo_subset_results["average_win_rate_bar"]
-    more_stats_md = f"""## More Statistics for Chatbot Arena ({button})
-    """
-    leaderboard_md = make_arena_leaderboard_md(arena_df, arena_subset_df, name=button)
-    return arena_values, p1, p2, p3, p4, more_stats_md, leaderboard_md
-
+key_to_category_name = {"full": "Total", "coding": "Coding", "long": "Long Conversation", "english": "English", "chinese": "Chinese"}
 
 def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=False):
+    arena_dfs = {}
+    category_elo_results = {}
     if elo_results_file is None:  # Do live update
         default_md = "Loading ..."
         p1 = p2 = p3 = p4 = None
@@ -299,25 +295,20 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
         with open(elo_results_file, "rb") as fin:
             elo_results = pickle.load(fin)
             if "full" in elo_results:
-                elo_chinese_results = elo_results["chinese"]
-                elo_long_results = elo_results["long"]
-                elo_english_results = elo_results["english"]
-                elo_coding_results = elo_results["coding"]
-                elo_results = elo_results["full"]
-
-        p1 = elo_results["win_fraction_heatmap"]
-        p2 = elo_results["battle_count_heatmap"]
-        p3 = elo_results["bootstrap_elo_rating"]
-        p4 = elo_results["average_win_rate_bar"]
-        arena_df = elo_results["leaderboard_table_df"]
-        arena_chinese_df = elo_chinese_results["leaderboard_table_df"]
-        arena_long_df = elo_long_results["leaderboard_table_df"]
-        arena_english_df = elo_english_results["leaderboard_table_df"]
-        arena_coding_df = elo_coding_results["leaderboard_table_df"]
-        default_md = make_default_md(arena_df, elo_results)
+                print("KEYS ", elo_results.keys())
+                for k in elo_results.keys():
+                    for k in key_to_category_name:
+                        arena_dfs[key_to_category_name[k]] = elo_results[k]["leaderboard_table_df"]
+                        category_elo_results[key_to_category_name[k]] = elo_results[k]
+
+        p1 = category_elo_results["Total"]["win_fraction_heatmap"]
+        p2 = category_elo_results["Total"]["battle_count_heatmap"]
+        p3 = category_elo_results["Total"]["bootstrap_elo_rating"]
+        p4 = category_elo_results["Total"]["average_win_rate_bar"]
+        arena_df = arena_dfs["Total"]
+        default_md = make_default_md(arena_df, category_elo_results["Total"])
 
     md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown")
-    # md = make_arena_leaderboard_md(arena_df, arena_chinese_df, arena_long_df, arena_english_df)
     if leaderboard_table_file:
         data = load_leaderboard_table_csv(leaderboard_table_file)
         model_table_df = pd.DataFrame(data)
@@ -329,20 +320,11 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
                 md = make_arena_leaderboard_md(arena_df)
                 leaderboard_markdown = gr.Markdown(md, elem_id="leaderboard_markdown")
                 with gr.Row():
-                    overall_rating = gr.Button("Overall")
-                    # update_overall_rating_df = lambda _: get_arena_table(arena_df, model_table_df)
-                    update_overall_rating_df = lambda x: update_leaderboard_and_plots(x, arena_df, model_table_df, None, elo_results)
-                    coding_rating = gr.Button("Coding")
-                    update_coding_rating_df = lambda x: update_leaderboard_and_plots(x, arena_df, model_table_df, arena_coding_df, elo_coding_results)
-                    long_context_rating = gr.Button("Long Conversation")
-                    update_long_context_rating_df = lambda x: update_leaderboard_and_plots(x, arena_df, model_table_df, arena_long_df, elo_long_results)
-                    # update_long_context_rating_df = lambda _: get_arena_table(arena_df, model_table_df, arena_long_df)
-                    english_rating = gr.Button("English")
-                    update_english_rating_df = lambda x: update_leaderboard_and_plots(x, arena_df, model_table_df, arena_english_df, elo_english_results)
-                    # update_english_rating_df = lambda _: get_arena_table(arena_df, model_table_df, arena_english_df)
-                    chinese_rating = gr.Button("Chinese")
-                    update_chinese_rating_df = lambda x: update_leaderboard_and_plots(x, arena_df, model_table_df, arena_chinese_df, elo_chinese_results)
-                    # update_chinese_rating_df = lambda _: get_arena_table(arena_df, model_table_df, arena_chinese_df)
+                    category_dropdown = gr.Dropdown(choices=list(arena_dfs.keys()), label="Category", value="Total")
+                    default_category_details = make_category_arena_leaderboard_md(arena_df, arena_df, name="Toal")
+                    with gr.Column(variant="panel"):
+                        category_deets = gr.Markdown(default_category_details, elem_id="category_deets")
+
                 elo_display_df = gr.Dataframe(
                     headers=[
                         "Rank",
@@ -371,6 +353,44 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
                     wrap=True,
                 )
 
+                gr.Markdown(
+                    f"""Note: we take the 95% confidence interval into account when determining a model's ranking.
+            A model is ranked higher only if its lower bound of model score is higher than the upper bound of the other model's score.
+            See Figure 3 below for visualization of the confidence intervals. Code to recreate these tables and plots in this [notebook]({notebook_url}) and more discussions in this blog [post](https://lmsys.org/blog/2023-12-07-leaderboard/).
+            """,
+                    elem_id="leaderboard_markdown"
+                )
+
+                leader_component_values[:] = [default_md, p1, p2, p3, p4]
+
+                if show_plot:
+                    more_stats_md = gr.Markdown(
+                        f"""## More Statistics for Chatbot Arena (Overall)""",
+                        elem_id="leaderboard_header_markdown"
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            gr.Markdown(
+                                "#### Figure 1: Fraction of Model A Wins for All Non-tied A vs. B Battles", elem_id="plot-title"
+                            )
+                            plot_1 = gr.Plot(p1, show_label=False, elem_id="plot-container")
+                        with gr.Column():
+                            gr.Markdown(
+                                "#### Figure 2: Battle Count for Each Combination of Models (without Ties)", elem_id="plot-title"
+                            )
+                            plot_2 = gr.Plot(p2, show_label=False)
+                    with gr.Row():
+                        with gr.Column():
+                            gr.Markdown(
+                                "#### Figure 3: Confidence Intervals on Model Strength (via Bootstrapping)", elem_id="plot-title"
+                            )
+                            plot_3 = gr.Plot(p3, show_label=False)
+                        with gr.Column():
+                            gr.Markdown(
+                                "#### Figure 4: Average Win Rate Against All Other Models (Assuming Uniform Sampling and No Ties)", elem_id="plot-title"
+                            )
+                            plot_4 = gr.Plot(p4, show_label=False)
+
             with gr.Tab("Full Leaderboard", id=1):
                 md = make_full_leaderboard_md(elo_results)
                 gr.Markdown(md, elem_id="leaderboard_markdown")
@@ -401,49 +421,21 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
     else:
         pass
 
-    gr.Markdown(
-        f"""Note: we take the 95% confidence interval into account when determining a model's ranking.
-A model is ranked higher only if its lower bound of model score is higher than the upper bound of the other model's score.
-See Figure 3 below for visualization of the confidence intervals. Code to recreate these tables and plots in this [notebook]({notebook_url}) and more discussions in this blog [post](https://lmsys.org/blog/2023-12-07-leaderboard/).
-""",
-        elem_id="leaderboard_markdown"
-    )
-
-    leader_component_values[:] = [default_md, p1, p2, p3, p4]
-
-    if show_plot:
-        more_stats_md = gr.Markdown(
-            f"""## More Statistics for Chatbot Arena (Overall)""",
-            elem_id="leaderboard_header_markdown"
-        )
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown(
-                    "#### Figure 1: Fraction of Model A Wins for All Non-tied A vs. B Battles", elem_id="plot-title"
-                )
-                plot_1 = gr.Plot(p1, show_label=False, elem_id="plot-container")
-            with gr.Column():
-                gr.Markdown(
-                    "#### Figure 2: Battle Count for Each Combination of Models (without Ties)", elem_id="plot-title"
-                )
-                plot_2 = gr.Plot(p2, show_label=False)
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown(
-                    "#### Figure 3: Confidence Intervals on Model Strength (via Bootstrapping)", elem_id="plot-title"
-                )
-                plot_3 = gr.Plot(p3, show_label=False)
-            with gr.Column():
-                gr.Markdown(
-                    "#### Figure 4: Average Win Rate Against All Other Models (Assuming Uniform Sampling and No Ties)", elem_id="plot-title"
-                )
-                plot_4 = gr.Plot(p4, show_label=False)
+    def update_leaderboard_and_plots(category):
+        arena_subset_df = arena_dfs[category]
+        elo_subset_results = category_elo_results[category]
+        arena_df = arena_dfs["Total"]
+        arena_values = get_arena_table(arena_df, model_table_df, arena_subset_df)
+        p1 = elo_subset_results["win_fraction_heatmap"]
+        p2 = elo_subset_results["battle_count_heatmap"]
+        p3 = elo_subset_results["bootstrap_elo_rating"]
+        p4 = elo_subset_results["average_win_rate_bar"]
+        more_stats_md = f"""## More Statistics for Chatbot Arena - {category}
+        """
+        leaderboard_md = make_category_arena_leaderboard_md(arena_df, arena_subset_df, name=category)
+        return arena_values, p1, p2, p3, p4, more_stats_md, leaderboard_md
                 
-    overall_rating.click(fn=update_overall_rating_df, inputs=overall_rating, outputs=[elo_display_df, plot_1, plot_2, plot_3, plot_4, more_stats_md, leaderboard_markdown])
-    coding_rating.click(fn=update_coding_rating_df, inputs=coding_rating, outputs=[elo_display_df, plot_1, plot_2, plot_3, plot_4, more_stats_md, leaderboard_markdown])
-    long_context_rating.click(fn=update_long_context_rating_df, inputs=long_context_rating, outputs=[elo_display_df, plot_1, plot_2, plot_3, plot_4, more_stats_md, leaderboard_markdown])
-    english_rating.click(fn=update_english_rating_df, inputs=english_rating, outputs=[elo_display_df, plot_1, plot_2, plot_3, plot_4, more_stats_md, leaderboard_markdown])
-    chinese_rating.click(fn=update_chinese_rating_df, inputs=chinese_rating ,outputs=[elo_display_df, plot_1, plot_2, plot_3, plot_4, more_stats_md, leaderboard_markdown])
+    category_dropdown.change(update_leaderboard_and_plots, inputs=[category_dropdown], outputs=[elo_display_df, plot_1, plot_2, plot_3, plot_4, more_stats_md, category_deets])
 
     with gr.Accordion(
         "📝 Citation",
@@ -482,6 +474,11 @@ block_css = """
     padding-bottom: 6px;
 }
 
+#category_deets {
+    text-align: center;
+    padding: 0px;
+}
+
 #leaderboard_markdown {
     font-size: 104%
 }

elo_results_20240403.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce8cebf41da8c06eee0f37156e01be83cc43182e0f00444311b4ad97a83154be
+size 690286