First commit

app.py

@@ -1,7 +1,356 @@
+import os
+import pickle
+
+import pandas as pd
+import numpy as np
 import gradio as gr
+from datetime import datetime
+from huggingface_hub import HfApi
+from apscheduler.schedulers.background import BackgroundScheduler
+import plotly.graph_objects as go
+
+from utils import (
+    KEY_TO_CATEGORY_NAME,
+    CAT_NAME_TO_EXPLANATION,
+    download_latest_data_from_space,
+    get_constants,
+    update_release_date_mapping,
+    format_data,
+    get_trendlines,
+    find_crossover_point,
+    sigmoid_transition
+)
+
+###################
+### Initialize scheduler
+###################
+
+
+def restart_space():
+    HfApi(token=os.getenv("HF_TOKEN", None)).restart_space(
+        repo_id="andrewrreed/closed-vs-open-arena-elo"
+    )
+    print(f"Space restarted on {datetime.now()}")
+
+
+# restart the space every day at 9am
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "cron", day_of_week="mon-sun", hour=7, minute=0)
+scheduler.start()
+
+###################
+### Load Data
+###################
+
+# gather ELO data
+latest_elo_file_local = download_latest_data_from_space(
+    repo_id="lmsys/chatbot-arena-leaderboard", file_type="pkl"
+)
+
+with open(latest_elo_file_local, "rb") as fin:
+    elo_results = pickle.load(fin)
+
+# TO-DO: need to also include vision
+elo_results = elo_results["text"]
+
+arena_dfs = {}
+for k in KEY_TO_CATEGORY_NAME.keys():
+    if k not in elo_results:
+        continue
+    arena_dfs[KEY_TO_CATEGORY_NAME[k]] = elo_results[k]["leaderboard_table_df"]
+
+# gather open llm leaderboard data
+latest_leaderboard_file_local = download_latest_data_from_space(
+    repo_id="lmsys/chatbot-arena-leaderboard", file_type="csv"
+)
+leaderboard_df = pd.read_csv(latest_leaderboard_file_local)
+
+# load release date mapping data
+release_date_mapping = pd.read_json("release_date_mapping.json", orient="records")
+
+###################
+### Prepare Data
+###################
+
+# update release date mapping with new models
+# check for new models in ELO data
+new_model_keys_to_add = [
+    model
+    for model in arena_dfs["Overall"].index.to_list()
+    if model not in release_date_mapping["key"].to_list()
+]
+if new_model_keys_to_add:
+    release_date_mapping = update_release_date_mapping(
+        new_model_keys_to_add, leaderboard_df, release_date_mapping
+    )
+
+# merge leaderboard data with ELO data
+merged_dfs = {}
+for k, v in arena_dfs.items():
+    merged_dfs[k] = (
+        pd.merge(arena_dfs[k], leaderboard_df, left_index=True, right_on="key")
+        .sort_values("rating", ascending=False)
+        .reset_index(drop=True)
+    )
+
+# add release dates into the merged data
+for k, v in merged_dfs.items():
+    merged_dfs[k] = pd.merge(
+        merged_dfs[k], release_date_mapping[["key", "Release Date"]], on="key"
+    )
+
+# format dataframes
+merged_dfs = {k: format_data(v) for k, v in merged_dfs.items()}
+
+# get constants
+min_elo_score, max_elo_score, _ = get_constants(merged_dfs)
+date_updated = elo_results["full"]["last_updated_datetime"].split(" ")[0]
+orgs = merged_dfs["Overall"].Organization.unique().tolist()
+
+###################
+### Build and Plot Data
+###################
+
+
+df = merged_dfs["Overall"]
+top_orgs = df.groupby("Organization")["rating"].max().nlargest(11).index.tolist()
+
+df = df.loc[(df["Organization"].isin(top_orgs)) & (df["rating"] > 1000)]
+print(df)
+
+df = df.loc[~df["Release Date"].isna()]
+
+def get_data_split(dfs, set_name):
+    df = dfs[set_name].copy(deep=True)
+    return df.reset_index(drop=True)
+
+
+def clean_df_for_display(df):
+    df = df.loc[
+        :,
+        [
+            "Model",
+            "rating",
+            "MMLU",
+            "MT-bench (score)",
+            "Release Date",
+            "Organization",
+            "License",
+            "Link",
+        ],
+    ].rename(columns={"rating": "ELO Score", "MT-bench (score)": "MT-Bench"})
+
+    df["Release Date"] = df["Release Date"].astype(str)
+    df.sort_values("ELO Score", ascending=False, inplace=True)
+    df.reset_index(drop=True, inplace=True)
+    return df
+
+def format_data(df):
+    """
+    Formats the given DataFrame by performing the following operations:
+    - Converts the 'License' column values to 'Proprietary LLM' if they are in PROPRIETARY_LICENSES, otherwise 'Open LLM'.
+    - Converts the 'Release Date' column to datetime format.
+    - Adds a new 'Month-Year' column by extracting the month and year from the 'Release Date' column.
+    - Rounds the 'rating' column to the nearest integer.
+    - Resets the index of the DataFrame.
+    Args:
+        df (pandas.DataFrame): The DataFrame to be formatted.
+    Returns:
+        pandas.DataFrame: The formatted DataFrame.
+    """
+
+    PROPRIETARY_LICENSES = ["Proprietary", "Proprietory"]
+
+    df["License"] = df["License"].apply(
+        lambda x: "Proprietary LLM" if x in PROPRIETARY_LICENSES else "Open LLM"
+    )
+    df["Release Date"] = pd.to_datetime(df["Release Date"])
+    df["Month-Year"] = df["Release Date"].dt.to_period("M")
+    df["rating"] = df["rating"].round()
+    return df.reset_index(drop=True)
+
+
+# Define organization to country mapping and colors
+org_info = {
+    "OpenAI": ("#00A67E", "🇺🇸"),  # Teal
+    "Google": ("#4285F4", "🇺🇸"),  # Google Blue
+    "xAI": ("black", "🇺🇸"),  # Bright Orange
+    "Anthropic": ("#cc785c", "🇺🇸"),  # Brown (as requested)
+    "Meta": ("#0064E0", "🇺🇸"),  # Facebook Blue
+    "Alibaba": ("#6958cf", "🇨🇳"),
+    "DeepSeek": ("#C70039", "🇨🇳"),
+    "01 AI": ("#11871e", "🇨🇳"),  # Bright Green
+    "DeepSeek AI": ("#9900CC", "🇨🇳"),  # Purple
+    "Mistral": ("#ff7000", "🇫🇷"),  # Mistral Orange (as requested)
+    "AI21 Labs": ("#1E90FF", "🇮🇱"),  # Dodger Blue,
+    "Reka AI": ("#FFC300", "🇺🇸"),
+    "Zhipu AI": ("#FFC300", "🇨🇳"),
+}
+
+def make_figure(df):
+    fig = go.Figure()
+
+    for i, org in enumerate(
+        df.groupby("Organization")["rating"]
+        .max()
+        .sort_values(ascending=False)
+        .index.tolist()
+    ):
+        org_data = df[df["Organization"] == org]
+
+        if len(org_data) > 0:
+            x_values = []
+            y_values = []
+            current_best = -np.inf
+            best_models = []
+
+            # Group by date and get the best model for each date
+            daily_best = org_data.groupby("Release Date").first().reset_index()
+
+            for _, row in daily_best.iterrows():
+                if row["rating"] > current_best:
+                    if len(x_values) > 0:
+                        # Create smooth transition
+                        transition_days = (row["Release Date"] - x_values[-1]).days
+                        transition_points = pd.date_range(
+                            x_values[-1],
+                            row["Release Date"],
+                            periods=max(100, transition_days),
+                        )
+                        x_values.extend(transition_points)
+
+                        transition_y = current_best + (
+                            row["rating"] - current_best
+                        ) * sigmoid_transition(
+                            np.linspace(-6, 6, len(transition_points)), 0, k=1
+                        )
+                        y_values.extend(transition_y)
+
+                    x_values.append(row["Release Date"])
+                    y_values.append(row["rating"])
+                    current_best = row["rating"]
+                    best_models.append(row)
+
+            # Extend the line to the current date
+            if x_values[-1] < current_date:
+                x_values.append(current_date)
+                y_values.append(current_best)
+
+            # Get org color and flag
+            color, flag = org_info.get(org, ("#808080", ""))
+
+            # Add line plot
+            fig.add_trace(
+                go.Scatter(
+                    x=x_values,
+                    y=y_values,
+                    mode="lines",
+                    name=f"{i+1}. {org} {flag}",
+                    line=dict(color=color, width=2),
+                    hoverinfo="skip",
+                )
+            )
+
+            # Add scatter plot for best model points
+            best_models_df = pd.DataFrame(best_models)
+            fig.add_trace(
+                go.Scatter(
+                    x=best_models_df["Release Date"],
+                    y=best_models_df["rating"],
+                    mode="markers",
+                    name=org,
+                    showlegend=False,
+                    marker=dict(color=color, size=8, symbol="circle"),
+                    text=best_models_df["Model"],
+                    hovertemplate="<b>%{text}</b><br>Date: %{x}<br>ELO Score: %{y:.2f}<extra></extra>",
+                )
+            )
+
+
+    # Update layout
+    fig.update_layout(
+        xaxis_title="Date",
+        title="La course au classement",
+        yaxis_title="Score ELO",
+        legend_title="Classement en Novembre 2024",
+        xaxis_range=[pd.Timestamp("2024-01-01"), current_date],  # Extend x-axis for labels
+        yaxis_range=[1103, 1350],
+    )
+    # apply_template(fig)
+
+    fig.update_xaxes(
+        tickformat="%m-%Y",
+    )
+
+    return fig, df
+
+def filter_df():
+    return df
+
+
+set_dark_mode = """
+function refresh() {
+    const url = new URL(window.location);
+
+    if (url.searchParams.get('__theme') !== 'dark') {
+        url.searchParams.set('__theme', 'dark');
+        window.location.href = url.href;
+    }
+}
+"""
+
+with gr.Blocks(
+    theme=gr.themes.Soft(
+        primary_hue=gr.themes.colors.sky,
+        secondary_hue=gr.themes.colors.green,
+        # spacing_size=gr.themes.sizes.spacing_sm,
+        text_size=gr.themes.sizes.text_sm,
+        font=[
+            gr.themes.GoogleFont("Open Sans"),
+            "ui-sans-serif",
+            "system-ui",
+            "sans-serif",
+        ],
+    ),
+    js=set_dark_mode,
+) as demo:
+    gr.Markdown(
+        """
+        <div style="text-align: center; max-width: 650px; margin: auto;">
+            <h1 style="font-weight: 900; margin-top: 5px;">🚀 The race for the best LLM 🚀</h1>
+            <p style="text-align: left; margin-top: 30px; margin-bottom: 30px; line-height: 20px;">
+            This app visualizes the progress of LLMs over time as scored by the <a href="https://leaderboard.lmsys.org/">LMSYS Chatbot Arena</a>.
+            The app is adapted from <a href="https://huggingface.co/spaces/andrewrreed/closed-vs-open-arena-elo"> this app</a> by Andew Reed,
+            and is intended to stay up-to-date as new models are released and evaluated.
+            <div style="text-align: left;">
+            <strong>Plot info:</strong>
+            <br>
+            <ul style="padding-left: 20px;">
+                <li> The ELO score (y-axis) is a measure of the relative strength of a model based on its performance against other models in the arena. </li>
+                <li> The Release Date (x-axis) corresponds to when the model was first publicly released or when its ELO results were first reported (for ease of automated updates). </li>
+                <li> Trend lines are based on Ordinary Least Squares (OLS) regression and adjust based on the filter criteria. </li>
+            <ul>
+            </div>
+            </p>
+        </div>
+        """
+    )
+    filtered_df = gr.State()
+    with gr.Group():
+        with gr.Tab("Plot"):
+            plot = gr.Plot(show_label=False)
+        with gr.Tab("Raw Data"):
+            display_df = gr.DataFrame()
+
 
-def greet(name):
-    return "Hello " + name + "!!"
+    demo.load(
+        fn=filter_df,
+        inputs=[],
+        outputs=filtered_df,
+    ).then(
+        fn=make_figure,
+        inputs=[filtered_df],
+        outputs=[plot, display_df],
+    )
 
-demo = gr.Interface(fn=greet, inputs="text", outputs="text")
 demo.launch()

release_date_mapping.json

@@ -0,0 +1,857 @@
+[
+    {
+        "key": "gpt-4-turbo-2024-04-09",
+        "Model": "GPT-4-Turbo-2024-04-09",
+        "Release Date": "2024-04-09"
+    },
+    {
+        "key": "gpt-4-1106-preview",
+        "Model": "GPT-4-1106-preview",
+        "Release Date": "2023-11-06"
+    },
+    {
+        "key": "claude-3-opus-20240229",
+        "Model": "Claude 3 Opus",
+        "Release Date": "2024-02-29"
+    },
+    {
+        "key": "gemini-1.5-pro-api-0409-preview",
+        "Model": "Gemini 1.5 Pro API-0409-Preview",
+        "Release Date": "2024-04-09"
+    },
+    {
+        "key": "gpt-4-0125-preview",
+        "Model": "GPT-4-0125-preview",
+        "Release Date": "2024-01-25"
+    },
+    {
+        "key": "bard-jan-24-gemini-pro",
+        "Model": "Bard (Gemini Pro)",
+        "Release Date": "2024-02-01"
+    },
+    {
+        "key": "llama-3-70b-instruct",
+        "Model": "Llama-3-70b-Instruct",
+        "Release Date": "2024-04-18"
+    },
+    {
+        "key": "claude-3-sonnet-20240229",
+        "Model": "Claude 3 Sonnet",
+        "Release Date": "2024-02-29"
+    },
+    {
+        "key": "command-r-plus",
+        "Model": "Command R+",
+        "Release Date": "2024-04-04"
+    },
+    {
+        "key": "gpt-4-0314",
+        "Model": "GPT-4-0314",
+        "Release Date": "2023-03-14"
+    },
+    {
+        "key": "claude-3-haiku-20240307",
+        "Model": "Claude 3 Haiku",
+        "Release Date": "2024-03-07"
+    },
+    {
+        "key": "gpt-4-0613",
+        "Model": "GPT-4-0613",
+        "Release Date": "2023-06-13"
+    },
+    {
+        "key": "mistral-large-2402",
+        "Model": "Mistral-Large-2402",
+        "Release Date": "2024-02-24"
+    },
+    {
+        "key": "qwen1.5-72b-chat",
+        "Model": "Qwen1.5-72B-Chat",
+        "Release Date": "2024-02-04"
+    },
+    {
+        "key": "reka-flash-21b-20240226-online",
+        "Model": "Reka-Flash-21B-online",
+        "Release Date": "2024-02-26"
+    },
+    {
+        "key": "claude-1",
+        "Model": "Claude-1",
+        "Release Date": "2023-03-14"
+    },
+    {
+        "key": "reka-flash-21b-20240226",
+        "Model": "Reka-Flash-21B",
+        "Release Date": "2024-02-26"
+    },
+    {
+        "key": "command-r",
+        "Model": "Command R",
+        "Release Date": "2024-03-11"
+    },
+    {
+        "key": "mistral-medium",
+        "Model": "Mistral Medium",
+        "Release Date": "2023-12-11"
+    },
+    {
+        "key": "mixtral-8x22b-instruct-v0.1",
+        "Model": "Mixtral-8x22b-Instruct-v0.1",
+        "Release Date": "2024-04-17"
+    },
+    {
+        "key": "llama-3-8b-instruct",
+        "Model": "Llama-3-8b-Instruct",
+        "Release Date": "2024-04-18"
+    },
+    {
+        "key": "gemini-pro-dev-api",
+        "Model": "Gemini Pro (Dev API)",
+        "Release Date": "2023-12-13"
+    },
+    {
+        "key": "qwen1.5-32b-chat",
+        "Model": "Qwen1.5-32B-Chat",
+        "Release Date": "2024-02-04"
+    },
+    {
+        "key": "claude-2.0",
+        "Model": "Claude-2.0",
+        "Release Date": "2023-07-11"
+    },
+    {
+        "key": "mistral-next",
+        "Model": "Mistral-Next",
+        "Release Date": "2024-02-17"
+    },
+    {
+        "key": "zephyr-orpo-141b-A35b-v0.1",
+        "Model": "Zephyr-ORPO-141b-A35b-v0.1",
+        "Release Date": "2024-04-12"
+    },
+    {
+        "key": "gpt-3.5-turbo-0613",
+        "Model": "GPT-3.5-Turbo-0613",
+        "Release Date": "2023-06-13"
+    },
+    {
+        "key": "claude-2.1",
+        "Model": "Claude-2.1",
+        "Release Date": "2023-11-21"
+    },
+    {
+        "key": "qwen1.5-14b-chat",
+        "Model": "Qwen1.5-14B-Chat",
+        "Release Date": "2024-02-04"
+    },
+    {
+        "key": "starling-lm-7b-beta",
+        "Model": "Starling-LM-7B-beta",
+        "Release Date": "2024-03-20"
+    },
+    {
+        "key": "gemini-pro",
+        "Model": "Gemini Pro",
+        "Release Date": "2023-12-13"
+    },
+    {
+        "key": "mixtral-8x7b-instruct-v0.1",
+        "Model": "Mixtral-8x7b-Instruct-v0.1",
+        "Release Date": "2023-12-11"
+    },
+    {
+        "key": "claude-instant-1",
+        "Model": "Claude-Instant-1",
+        "Release Date": "2023-03-14"
+    },
+    {
+        "key": "yi-34b-chat",
+        "Model": "Yi-34B-Chat",
+        "Release Date": "2024-01-23"
+    },
+    {
+        "key": "gpt-3.5-turbo-0314",
+        "Model": "GPT-3.5-Turbo-0314",
+        "Release Date": "2023-03-14"
+    },
+    {
+        "key": "wizardlm-70b",
+        "Model": "WizardLM-70B-v1.0",
+        "Release Date": "2023-08-09"
+    },
+    {
+        "key": "gpt-3.5-turbo-0125",
+        "Model": "GPT-3.5-Turbo-0125",
+        "Release Date": "2024-01-25"
+    },
+    {
+        "key": "tulu-2-dpo-70b",
+        "Model": "Tulu-2-DPO-70B",
+        "Release Date": "2023-11-12"
+    },
+    {
+        "key": "dbrx-instruct-preview",
+        "Model": "DBRX-Instruct-Preview",
+        "Release Date": "2024-03-27"
+    },
+    {
+        "key": "openchat-3.5-0106",
+        "Model": "OpenChat-3.5-0106",
+        "Release Date": "2024-01-06"
+    },
+    {
+        "key": "vicuna-33b",
+        "Model": "Vicuna-33B",
+        "Release Date": "2023-06-21"
+    },
+    {
+        "key": "starling-lm-7b-alpha",
+        "Model": "Starling-LM-7B-alpha",
+        "Release Date": "2023-11-25"
+    },
+    {
+        "key": "llama-2-70b-chat",
+        "Model": "Llama-2-70b-chat",
+        "Release Date": "2023-07-18"
+    },
+    {
+        "key": "nous-hermes-2-mixtral-8x7b-dpo",
+        "Model": "Nous-Hermes-2-Mixtral-8x7B-DPO",
+        "Release Date": "2024-01-13"
+    },
+    {
+        "key": "gemma-1.1-7b-it",
+        "Model": "Gemma-1.1-7B-it",
+        "Release Date": "2024-04-09"
+    },
+    {
+        "key": "llama2-70b-steerlm-chat",
+        "Model": "NV-Llama2-70B-SteerLM-Chat",
+        "Release Date": "2023-11-24"
+    },
+    {
+        "key": "deepseek-llm-67b-chat",
+        "Model": "DeepSeek-LLM-67B-Chat",
+        "Release Date": "2023-11-29"
+    },
+    {
+        "key": "openhermes-2.5-mistral-7b",
+        "Model": "OpenHermes-2.5-Mistral-7b",
+        "Release Date": "2023-10-29"
+    },
+    {
+        "key": "openchat-3.5",
+        "Model": "OpenChat-3.5",
+        "Release Date": "2023-11-16"
+    },
+    {
+        "key": "pplx-70b-online",
+        "Model": "pplx-70b-online",
+        "Release Date": "2023-11-29"
+    },
+    {
+        "key": "mistral-7b-instruct-v0.2",
+        "Model": "Mistral-7B-Instruct-v0.2",
+        "Release Date": "2023-12-11"
+    },
+    {
+        "key": "qwen1.5-7b-chat",
+        "Model": "Qwen1.5-7B-Chat",
+        "Release Date": "2024-02-04"
+    },
+    {
+        "key": "gpt-3.5-turbo-1106",
+        "Model": "GPT-3.5-Turbo-1106",
+        "Release Date": "2023-11-06"
+    },
+    {
+        "key": "dolphin-2.2.1-mistral-7b",
+        "Model": "Dolphin-2.2.1-Mistral-7B",
+        "Release Date": "2023-10-30"
+    },
+    {
+        "key": "solar-10.7b-instruct-v1.0",
+        "Model": "SOLAR-10.7B-Instruct-v1.0",
+        "Release Date": "2023-12-13"
+    },
+    {
+        "key": "phi-3-mini-128k-instruct",
+        "Model": "Phi-3-Mini-128k-Instruct",
+        "Release Date": "2024-04-23"
+    },
+    {
+        "key": "wizardlm-13b",
+        "Model": "WizardLM-13b-v1.2",
+        "Release Date": "2023-07-25"
+    },
+    {
+        "key": "llama-2-13b-chat",
+        "Model": "Llama-2-13b-chat",
+        "Release Date": "2023-07-18"
+    },
+    {
+        "key": "zephyr-7b-beta",
+        "Model": "Zephyr-7b-beta",
+        "Release Date": "2023-10-26"
+    },
+    {
+        "key": "codellama-70b-instruct",
+        "Model": "CodeLlama-70B-instruct",
+        "Release Date": "2024-01-29"
+    },
+    {
+        "key": "mpt-30b-chat",
+        "Model": "MPT-30B-chat",
+        "Release Date": "2023-06-09"
+    },
+    {
+        "key": "vicuna-13b",
+        "Model": "Vicuna-13B",
+        "Release Date": "2023-07-23"
+    },
+    {
+        "key": "codellama-34b-instruct",
+        "Model": "CodeLlama-34B-instruct",
+        "Release Date": "2023-08-24"
+    },
+    {
+        "key": "gemma-7b-it",
+        "Model": "Gemma-7B-it",
+        "Release Date": "2024-02-21"
+    },
+    {
+        "key": "pplx-7b-online",
+        "Model": "pplx-7b-online",
+        "Release Date": "2023-11-29"
+    },
+    {
+        "key": "zephyr-7b-alpha",
+        "Model": "Zephyr-7b-alpha",
+        "Release Date": "2023-10-09"
+    },
+    {
+        "key": "llama-2-7b-chat",
+        "Model": "Llama-2-7b-chat",
+        "Release Date": "2023-07-18"
+    },
+    {
+        "key": "qwen-14b-chat",
+        "Model": "Qwen-14B-Chat",
+        "Release Date": "2023-09-24"
+    },
+    {
+        "key": "falcon-180b-chat",
+        "Model": "falcon-180b-chat",
+        "Release Date": "2023-09-05"
+    },
+    {
+        "key": "guanaco-33b",
+        "Model": "Guanaco-33B",
+        "Release Date": "2023-05-22"
+    },
+    {
+        "key": "stripedhyena-nous-7b",
+        "Model": "StripedHyena-Nous-7B",
+        "Release Date": "2023-12-07"
+    },
+    {
+        "key": "olmo-7b-instruct",
+        "Model": "OLMo-7B-instruct",
+        "Release Date": "2024-02-23"
+    },
+    {
+        "key": "gemma-1.1-2b-it",
+        "Model": "Gemma-1.1-2B-it",
+        "Release Date": "2024-04-09"
+    },
+    {
+        "key": "mistral-7b-instruct",
+        "Model": "Mistral-7B-Instruct-v0.1",
+        "Release Date": "2023-09-27"
+    },
+    {
+        "key": "palm-2",
+        "Model": "PaLM-Chat-Bison-001",
+        "Release Date": "2023-07-10"
+    },
+    {
+        "key": "vicuna-7b",
+        "Model": "Vicuna-7B",
+        "Release Date": "2023-07-29"
+    },
+    {
+        "key": "qwen1.5-4b-chat",
+        "Model": "Qwen1.5-4B-Chat",
+        "Release Date": "2024-02-04"
+    },
+    {
+        "key": "gemma-2b-it",
+        "Model": "Gemma-2B-it",
+        "Release Date": "2024-02-21"
+    },
+    {
+        "key": "koala-13b",
+        "Model": "Koala-13B",
+        "Release Date": "2023-04-03"
+    },
+    {
+        "key": "chatglm3-6b",
+        "Model": "ChatGLM3-6B",
+        "Release Date": "2023-10-25"
+    },
+    {
+        "key": "gpt4all-13b-snoozy",
+        "Model": "GPT4All-13B-Snoozy",
+        "Release Date": "2023-04-24"
+    },
+    {
+        "key": "chatglm2-6b",
+        "Model": "ChatGLM2-6B",
+        "Release Date": "2023-06-25"
+    },
+    {
+        "key": "mpt-7b-chat",
+        "Model": "MPT-7B-Chat",
+        "Release Date": "2023-05-04"
+    },
+    {
+        "key": "RWKV-4-Raven-14B",
+        "Model": "RWKV-4-Raven-14B",
+        "Release Date": "2023-05-22"
+    },
+    {
+        "key": "alpaca-13b",
+        "Model": "Alpaca-13B",
+        "Release Date": "2023-03-13"
+    },
+    {
+        "key": "oasst-pythia-12b",
+        "Model": "OpenAssistant-Pythia-12B",
+        "Release Date": "2023-04-03"
+    },
+    {
+        "key": "chatglm-6b",
+        "Model": "ChatGLM-6B",
+        "Release Date": "2023-03-13"
+    },
+    {
+        "key": "fastchat-t5-3b",
+        "Model": "FastChat-T5-3B",
+        "Release Date": "2023-04-27"
+    },
+    {
+        "key": "stablelm-tuned-alpha-7b",
+        "Model": "StableLM-Tuned-Alpha-7B",
+        "Release Date": "2023-04-19"
+    },
+    {
+        "key": "dolly-v2-12b",
+        "Model": "Dolly-V2-12B",
+        "Release Date": "2023-04-12"
+    },
+    {
+        "key": "llama-13b",
+        "Model": "LLaMA-13B",
+        "Release Date": "2023-02-27"
+    },
+    {
+        "key": "snowflake-arctic-instruct",
+        "Model": "Snowflake Arctic Instruct",
+        "Release Date": "2024-04-24"
+    },
+    {
+        "key": "gpt-4o-2024-05-13",
+        "Model": "GPT-4o-2024-05-13",
+        "Release Date": "2024-05-16"
+    },
+    {
+        "key": "qwen-max-0428",
+        "Model": "Qwen-Max-0428",
+        "Release Date": "2024-05-16"
+    },
+    {
+        "key": "qwen1.5-110b-chat",
+        "Model": "Qwen1.5-110B-Chat",
+        "Release Date": "2024-05-16"
+    },
+    {
+        "key": "reka-core-20240501",
+        "Model": "Reka-Core-20240501",
+        "Release Date": "2024-05-16"
+    },
+    {
+        "key": "glm-4-0116",
+        "Model": "GLM-4-0116",
+        "Release Date": "2024-05-23"
+    },
+    {
+        "key": "phi-3-mini-4k-instruct",
+        "Model": "Phi-3-Mini-4k-Instruct",
+        "Release Date": "2024-05-23"
+    },
+    {
+        "key": "yi-large-preview",
+        "Model": "Yi-Large-preview",
+        "Release Date": "2024-05-23"
+    },
+    {
+        "key": "claude-3-5-sonnet-20240620",
+        "Model": "Claude 3.5 Sonnet",
+        "Release Date": "2024-06-20"
+    },
+    {
+        "key": "deepseek-coder-v2",
+        "Model": "DeepSeek-Coder-V2-Instruct",
+        "Release Date": "2024-06-17"
+    },
+    {
+        "key": "gemini-1.5-flash-api-0514",
+        "Model": "Gemini-1.5-Flash-API-0514",
+        "Release Date": "2024-05-24"
+    },
+    {
+        "key": "gemini-1.5-pro-api-0514",
+        "Model": "Gemini-1.5-Pro-API-0514",
+        "Release Date": "2024-05-24"
+    },
+    {
+        "key": "gemini-advanced-0514",
+        "Model": "Gemini-Advanced-0514",
+        "Release Date": "2024-05-24"
+    },
+    {
+        "key": "gemma-2-27b-it",
+        "Model": "Gemma-2-27B-it",
+        "Release Date": "2024-07-01"
+    },
+    {
+        "key": "gemma-2-9b-it",
+        "Model": "Gemma-2-9B-it",
+        "Release Date": "2024-06-27"
+    },
+    {
+        "key": "glm-4-0520",
+        "Model": "GLM-4-0520",
+        "Release Date": "2024-05-20"
+    },
+    {
+        "key": "nemotron-4-340b-instruct",
+        "Model": "Nemotron-4-340B-Instruct",
+        "Release Date": "2024-06-14"
+    },
+    {
+        "key": "phi-3-medium-4k-instruct",
+        "Model": "Phi-3-Medium-4k-Instruct",
+        "Release Date": "2024-05-21"
+    },
+    {
+        "key": "phi-3-small-8k-instruct",
+        "Model": "Phi-3-Small-8k-Instruct",
+        "Release Date": "2024-05-21"
+    },
+    {
+        "key": "qwen2-72b-instruct",
+        "Model": "Qwen2-72B-Instruct",
+        "Release Date": "2024-06-06"
+    },
+    {
+        "key": "reka-flash-preview-20240611",
+        "Model": "Reka-Flash-Preview-20240611",
+        "Release Date": "2024-06-11"
+    },
+    {
+        "key": "yi-1.5-34b-chat",
+        "Model": "Yi-1.5-34B-Chat",
+        "Release Date": "2024-05-13"
+    },
+    {
+        "key": "yi-large",
+        "Model": "Yi-Large",
+        "Release Date": "2024-05-13"
+    },
+    {
+        "key": "phi-3-mini-4k-instruct-june-2024",
+        "Model": "Phi-3-Mini-4k-Instruct-June-24",
+        "Release Date": "2024-06-24"
+    },
+    {
+        "key": "athene-70b-0725",
+        "Model": "athene-70b-0725",
+        "Release Date": "2024-07-25"
+    },
+    {
+        "key": "athene-70b-0725",
+        "Model": "athene-70b-0725",
+        "Release Date": "2024-07-25"
+    },
+    {
+        "key": "deepseek-coder-v2-0724",
+        "Model": "Deepseek-Coder-v2-0724",
+        "Release Date": "2024-07-24"
+    },
+    {
+        "key": "deepseek-v2-api-0628",
+        "Model": "Deepseek-v2-API-0628",
+        "Release Date": "2024-06-28"
+    },
+    {
+        "key": "gemini-1.5-pro-exp-0801",
+        "Model": "Gemini-1.5-Pro-Exp-0801",
+        "Release Date": "2024-08-01"
+    },
+    {
+        "key": "gemma-2-2b-it",
+        "Model": "Gemma-2-2b-it",
+        "Release Date": "2024-07-31"
+    },
+    {
+        "key": "gpt-4o-mini-2024-07-18",
+        "Model": "GPT-4o-mini-2024-07-18",
+        "Release Date": "2024-07-18"
+    },
+    {
+        "key": "llama-3.1-405b-instruct",
+        "Model": "Meta-Llama-3.1-405b-Instruct",
+        "Release Date": "2024-07-23"
+    },
+    {
+        "key": "llama-3.1-70b-instruct",
+        "Model": "Meta-Llama-3.1-70b-Instruct",
+        "Release Date": "2024-07-23"
+    },
+    {
+        "key": "llama-3.1-8b-instruct",
+        "Model": "Meta-Llama-3.1-8b-Instruct",
+        "Release Date": "2024-07-23"
+    },
+    {
+        "key": "mistral-large-2407",
+        "Model": "Mistral-Large-2407",
+        "Release Date": "2024-07-24"
+    },
+    {
+        "key": "reka-core-20240722",
+        "Model": "Reka-Core-20240722",
+        "Release Date": "2024-07-22"
+    },
+    {
+        "key": "reka-flash-20240722",
+        "Model": "Reka-Flash-20240722",
+        "Release Date": "2024-07-22"
+    },
+    {
+        "key": "chatgpt-4o-latest-20240808",
+        "Model": "ChatGPT-4o-latest (2024-08-08)",
+        "Release Date": "2024-08-08"
+    },
+    {
+        "key": "chatgpt-4o-latest-20240903",
+        "Model": "ChatGPT-4o-latest (2024-09-03)",
+        "Release Date": "2024-09-03"
+    },
+    {
+        "key": "command-r-08-2024",
+        "Model": "Command R (08-2024)",
+        "Release Date": "2024-08-08"
+    },
+    {
+        "key": "command-r-plus-08-2024",
+        "Model": "Command R+ (08-2024)",
+        "Release Date": "2024-08-08"
+    },
+    {
+        "key": "deepseek-v2.5",
+        "Model": "Deepseek-v2.5",
+        "Release Date": "2024-09-05"
+    },
+    {
+        "key": "gemini-1.5-flash-8b-exp-0827",
+        "Model": "Gemini-1.5-Flash-8b-Exp-0827",
+        "Release Date": "2024-08-27"
+    },
+    {
+        "key": "gemini-1.5-flash-exp-0827",
+        "Model": "Gemini-1.5-Flash-Exp-0827",
+        "Release Date": "2024-08-27"
+    },
+    {
+        "key": "gemini-1.5-pro-exp-0827",
+        "Model": "Gemini-1.5-Pro-Exp-0827",
+        "Release Date": "2024-08-27"
+    },
+    {
+        "key": "gemma-2-9b-it-simpo",
+        "Model": "Gemma-2-9b-it-SimPO",
+        "Release Date": "2024-09-07"
+    },
+    {
+        "key": "gpt-4o-2024-08-06",
+        "Model": "GPT-4o-2024-08-06",
+        "Release Date": "2024-08-06"
+    },
+    {
+        "key": "grok-2-2024-08-13",
+        "Model": "Grok-2-08-13",
+        "Release Date": "2024-08-13"
+    },
+    {
+        "key": "grok-2-mini-2024-08-13",
+        "Model": "Grok-2-Mini-08-13",
+        "Release Date": "2024-08-13"
+    },
+    {
+        "key": "internlm2_5-20b-chat",
+        "Model": "InternLM2.5-20b-chat",
+        "Release Date": "2024-07-30"
+    },
+    {
+        "key": "jamba-1.5-large",
+        "Model": "Jamba-1.5-Large",
+        "Release Date": "2024-08-22"
+    },
+    {
+        "key": "jamba-1.5-mini",
+        "Model": "Jamba-1.5-Mini",
+        "Release Date": "2024-08-22"
+    },
+    {
+        "key": "llama-3.1-405b-instruct-bf16",
+        "Model": "Meta-Llama-3.1-405b-Instruct-bf16",
+        "Release Date": "2024-07-23"
+    },
+    {
+        "key": "llama-3.1-405b-instruct-fp8",
+        "Model": "Meta-Llama-3.1-405b-Instruct-fp8",
+        "Release Date": "2024-07-23"
+    },
+    {
+        "key": "llama-3.2-1b-instruct",
+        "Model": "Meta-Llama-3.2-1b-Instruct",
+        "Release Date": "2024-09-25"
+    },
+    {
+        "key": "llama-3.2-3b-instruct",
+        "Model": "Meta-Llama-3.2-3b-Instruct",
+        "Release Date": "2024-09-25"
+    },
+    {
+        "key": "o1-mini",
+        "Model": "o1-mini",
+        "Release Date": "2024-09-12"
+    },
+    {
+        "key": "o1-preview",
+        "Model": "o1-preview",
+        "Release Date": "2024-09-12"
+    },
+    {
+        "key": "qwen-plus-0828",
+        "Model": "Qwen-Plus-0828",
+        "Release Date": "2024-08-28"
+    },
+    {
+        "key": "qwen2.5-72b-instruct",
+        "Model": "Qwen2.5-72b-Instruct",
+        "Release Date": "2024-09-19"
+    },
+    {
+        "key": "chatgpt-4o-latest-20241120",
+        "Model": "ChatGPT-4o-latest (2024-11-20)",
+        "Release Date": "2024-11-20"
+    },
+    {
+        "key": "claude-3-5-sonnet-20241022",
+        "Model": "Claude 3.5 Sonnet (20241022)",
+        "Release Date": "2024-10-22"
+    },
+    {
+        "key": "gemini-1.5-flash-001",
+        "Model": "Gemini-1.5-Flash-001",
+        "Release Date": "2024-05-24"
+    },
+    {
+        "key": "gemini-1.5-flash-002",
+        "Model": "Gemini-1.5-Flash-002",
+        "Release Date": "2024-09-24"
+    },
+    {
+        "key": "gemini-1.5-flash-8b-001",
+        "Model": "Gemini-1.5-Flash-8B-001",
+        "Release Date": "2024-10-03"
+    },
+    {
+        "key": "gemini-1.5-pro-001",
+        "Model": "Gemini-1.5-Pro-001",
+        "Release Date": "2024-05-24"
+    },
+    {
+        "key": "gemini-1.5-pro-002",
+        "Model": "Gemini-1.5-Pro-002",
+        "Release Date": "2024-09-24"
+    },
+    {
+        "key": "gemini-exp-1114",
+        "Model": "Gemini-Exp-1114",
+        "Release Date": "2024-11-14"
+    },
+    {
+        "key": "gemini-exp-1121",
+        "Model": "Gemini-Exp-1121",
+        "Release Date": "2024-11-21"
+    },
+    {
+        "key": "glm-4-plus",
+        "Model": "GLM-4-Plus",
+        "Release Date": "2024-08-30"
+    },
+    {
+        "key": "granite-3.0-2b-instruct",
+        "Model": "Granite-3.0-2B-Instruct",
+        "Release Date": "2024-10-21"
+    },
+    {
+        "key": "granite-3.0-8b-instruct",
+        "Model": "Granite-3.0-8B-Instruct",
+        "Release Date": "2024-10-21"
+    },
+    {
+        "key": "hunyuan-standard-256k",
+        "Model": "Hunyuan-Standard-256K",
+        "Release Date": "2024-05-28"
+    },
+    {
+        "key": "llama-3.1-nemotron-51b-instruct",
+        "Model": "Llama-3.1-Nemotron-51B-Instruct",
+        "Release Date": "2024-09-23"
+    },
+    {
+        "key": "llama-3.1-nemotron-70b-instruct",
+        "Model": "Llama-3.1-Nemotron-70B-Instruct",
+        "Release Date": "2024-10-11"
+    },
+    {
+        "key": "ministral-8b-2410",
+        "Model": "Ministral-8B-2410",
+        "Release Date": "2024-10-24"
+    },
+    {
+        "key": "qwen-max-0919",
+        "Model": "Qwen-Max-0919",
+        "Release Date": "2024-09-19"
+    },
+    {
+        "key": "qwen2.5-coder-32b-instruct",
+        "Model": "Qwen2.5-Coder-32B-Instruct",
+        "Release Date": "2024-11-05"
+    },
+    {
+        "key": "yi-lightning",
+        "Model": "Yi-Lightning",
+        "Release Date": "2024-10-16"
+    },
+    {
+        "key": "yi-lightning-lite",
+        "Model": "Yi-Lightning-lite",
+        "Release Date": "2024-10-16"
+    }
+]

requirements.txt

@@ -0,0 +1,7 @@
+huggingface_hub
+pandas
+numpy
+plotly
+gradio
+statsmodels
+apscheduler

utils.py

@@ -0,0 +1,234 @@
+import json
+from datetime import datetime
+
+from typing import Literal, List
+
+import pandas as pd
+import plotly.express as px
+from huggingface_hub import HfFileSystem, hf_hub_download
+
+# from: https://github.com/lm-sys/FastChat/blob/main/fastchat/serve/monitor/monitor.py#L389
+KEY_TO_CATEGORY_NAME = {
+    "full": "Overall",
+    "dedup": "De-duplicate Top Redundant Queries (soon to be default)",
+    "math": "Math",
+    "if": "Instruction Following",
+    "multiturn": "Multi-Turn",
+    "coding": "Coding",
+    "hard_6": "Hard Prompts (Overall)",
+    "hard_english_6": "Hard Prompts (English)",
+    "long_user": "Longer Query",
+    "english": "English",
+    "chinese": "Chinese",
+    "french": "French",
+    "german": "German",
+    "spanish": "Spanish",
+    "russian": "Russian",
+    "japanese": "Japanese",
+    "korean": "Korean",
+    "no_tie": "Exclude Ties",
+    "no_short": "Exclude Short Query (< 5 tokens)",
+    "no_refusal": "Exclude Refusal",
+    "overall_limit_5_user_vote": "overall_limit_5_user_vote",
+    "full_old": "Overall (Deprecated)",
+}
+
+CAT_NAME_TO_EXPLANATION = {
+    "Overall": "Overall Questions",
+    "De-duplicate Top Redundant Queries (soon to be default)": "De-duplicate top redundant queries (top 0.1%). See details in [blog post](https://lmsys.org/blog/2024-05-17-category-hard/#note-enhancing-quality-through-de-duplication).",
+    "Math": "Math",
+    "Instruction Following": "Instruction Following",
+    "Multi-Turn": "Multi-Turn Conversation (>= 2 turns)",
+    "Coding": "Coding: whether conversation contains code snippets",
+    "Hard Prompts (Overall)": "Hard Prompts (Overall): details in [blog post](https://lmsys.org/blog/2024-05-17-category-hard/)",
+    "Hard Prompts (English)": "Hard Prompts (English), note: the delta is to English Category. details in [blog post](https://lmsys.org/blog/2024-05-17-category-hard/)",
+    "Longer Query": "Longer Query (>= 500 tokens)",
+    "English": "English Prompts",
+    "Chinese": "Chinese Prompts",
+    "French": "French Prompts",
+    "German": "German Prompts",
+    "Spanish": "Spanish Prompts",
+    "Russian": "Russian Prompts",
+    "Japanese": "Japanese Prompts",
+    "Korean": "Korean Prompts",
+    "Exclude Ties": "Exclude Ties and Bothbad",
+    "Exclude Short Query (< 5 tokens)": "Exclude Short User Query (< 5 tokens)",
+    "Exclude Refusal": 'Exclude model responses with refusal (e.g., "I cannot answer")',
+    "overall_limit_5_user_vote": "overall_limit_5_user_vote",
+    "Overall (Deprecated)": "Overall without De-duplicating Top Redundant Queries (top 0.1%). See details in [blog post](https://lmsys.org/blog/2024-05-17-category-hard/#note-enhancing-quality-through-de-duplication).",
+}
+
+PROPRIETARY_LICENSES = ["Proprietary", "Proprietory"]
+
+
+def download_latest_data_from_space(
+    repo_id: str, file_type: Literal["pkl", "csv"]
+) -> str:
+    """
+    Downloads the latest data file of the specified file type from the given repository space.
+
+    Args:
+        repo_id (str): The ID of the repository space.
+        file_type (Literal["pkl", "csv"]): The type of the data file to download. Must be either "pkl" or "csv".
+
+    Returns:
+        str: The local file path of the downloaded data file.
+    """
+
+    def extract_date(filename):
+        return filename.split("/")[-1].split(".")[0].split("_")[-1]
+
+    fs = HfFileSystem()
+    data_file_path = f"spaces/{repo_id}/*.{file_type}"
+    files = fs.glob(data_file_path)
+    files = [
+        file for file in files if "leaderboard_table" in file or "elo_results" in file
+    ]
+    latest_file = sorted(files, key=extract_date, reverse=True)[0]
+
+    latest_filepath_local = hf_hub_download(
+        repo_id=repo_id,
+        filename=latest_file.split("/")[-1],
+        repo_type="space",
+    )
+    print(latest_file.split("/")[-1])
+    return latest_filepath_local
+
+
+def get_constants(dfs):
+    """
+    Calculate and return the minimum and maximum Elo scores, as well as the maximum number of models per month.
+
+    Parameters:
+    - dfs (dict): A dictionary containing DataFrames for different categories.
+
+    Returns:
+    - min_elo_score (float): The minimum Elo score across all DataFrames.
+    - max_elo_score (float): The maximum Elo score across all DataFrames.
+    - upper_models_per_month (int): The maximum number of models per month per license across all DataFrames.
+    """
+    filter_ranges = {}
+    for k, df in dfs.items():
+        filter_ranges[k] = {
+            "min_elo_score": df["rating"].min().round(),
+            "max_elo_score": df["rating"].max().round(),
+            "upper_models_per_month": int(
+                df.groupby(["Month-Year", "License"])["rating"]
+                .apply(lambda x: x.count())
+                .max()
+            ),
+        }
+
+    min_elo_score = float("inf")
+    max_elo_score = float("-inf")
+    upper_models_per_month = 0
+
+    for _, value in filter_ranges.items():
+        min_elo_score = min(min_elo_score, value["min_elo_score"])
+        max_elo_score = max(max_elo_score, value["max_elo_score"])
+        upper_models_per_month = max(
+            upper_models_per_month, value["upper_models_per_month"]
+        )
+    return min_elo_score, max_elo_score, upper_models_per_month
+
+
+def update_release_date_mapping(
+    new_model_keys_to_add: List[str],
+    leaderboard_df: pd.DataFrame,
+    release_date_mapping: pd.DataFrame,
+) -> pd.DataFrame:
+    """
+    Update the release date mapping with new model keys.
+
+    Args:
+        new_model_keys_to_add (List[str]): A list of new model keys to add to the release date mapping.
+        leaderboard_df (pd.DataFrame): The leaderboard DataFrame containing the model information.
+        release_date_mapping (pd.DataFrame): The current release date mapping DataFrame.
+
+    Returns:
+        pd.DataFrame: The updated release date mapping DataFrame.
+    """
+    # if any, add those to the release date mapping
+    if new_model_keys_to_add:
+        for key in new_model_keys_to_add:
+            new_entry = {
+                "key": key,
+                "Model": leaderboard_df[leaderboard_df["key"] == key]["Model"].values[
+                    0
+                ],
+                "Release Date": datetime.today().strftime("%Y-%m-%d"),
+            }
+
+            with open("release_date_mapping.json", "r") as file:
+                data = json.load(file)
+
+            data.append(new_entry)
+
+            with open("release_date_mapping.json", "w") as file:
+                json.dump(data, file, indent=4)
+
+            print(f"Added {key} to release_date_mapping.json")
+
+        # reload the release date mapping
+        release_date_mapping = pd.read_json(
+            "release_date_mapping.json", orient="records"
+        )
+    return release_date_mapping
+
+
+def format_data(df):
+    """
+    Formats the given DataFrame by performing the following operations:
+    - Converts the 'License' column values to 'Proprietary LLM' if they are in PROPRIETARY_LICENSES, otherwise 'Open LLM'.
+    - Converts the 'Release Date' column to datetime format.
+    - Adds a new 'Month-Year' column by extracting the month and year from the 'Release Date' column.
+    - Rounds the 'rating' column to the nearest integer.
+    - Resets the index of the DataFrame.
+
+    Args:
+        df (pandas.DataFrame): The DataFrame to be formatted.
+
+    Returns:
+        pandas.DataFrame: The formatted DataFrame.
+    """
+    df["License"] = df["License"].apply(
+        lambda x: "Proprietary LLM" if x in PROPRIETARY_LICENSES else "Open LLM"
+    )
+    df["Release Date"] = pd.to_datetime(df["Release Date"])
+    df["Month-Year"] = df["Release Date"].dt.to_period("M")
+    df["rating"] = df["rating"].round()
+    return df.reset_index(drop=True)
+
+
+def get_trendlines(fig):
+
+    trend_lines = px.get_trendline_results(fig)
+
+    return [
+        trend_lines.iloc[i]["px_fit_results"].params.tolist()
+        for i in range(len(trend_lines))
+    ]
+
+
+def find_crossover_point(b1, m1, b2, m2):
+    """
+    Determine the X value at which two trendlines will cross over.
+
+    Parameters:
+    m1 (float): Slope of the first trendline.
+    b1 (float): Intercept of the first trendline.
+    m2 (float): Slope of the second trendline.
+    b2 (float): Intercept of the second trendline.
+
+    Returns:
+    float: The X value where the two trendlines cross.
+    """
+    if m1 == m2:
+        raise ValueError("The trendlines are parallel and do not cross.")
+
+    x_crossover = (b2 - b1) / (m1 - m2)
+    return x_crossover
+
+# Function to create sigmoid transition
+def sigmoid_transition(x, x0, k=0.1):
+    return expit(k * (x - x0))