update

app.py

@@ -39,7 +39,7 @@ def make_arena_leaderboard_md(arena_df):
     leaderboard_md = f"""
 Total #models: **{total_models}**.{space} Total #votes: **{"{:,}".format(total_votes)}**.{space} Last updated: March 29, 2024.
 
-**NEW!** View ELO leaderboard and stats for different input categories.
+📣 **NEW!** View leaderboard for different categories (e.g., coding, long user query)!
 """
     return leaderboard_md
 
@@ -49,8 +49,8 @@ def make_category_arena_leaderboard_md(arena_df, arena_subset_df, name="Overall"
     space = "   "
     total_subset_votes = sum(arena_subset_df["num_battles"]) // 2
     total_subset_models = len(arena_subset_df)
-    leaderboard_md = f"""### {name} Question Coverage
-#models: **{total_subset_models} ({round(total_subset_models/total_models *100)}%)**.{space} #votes: **{"{:,}".format(total_subset_votes)} ({round(total_subset_votes/total_votes * 100)}%)**.{space}
+    leaderboard_md = f"""### {cat_name_to_explanation[name]}
+#### [Coverage] {space} #models: **{total_subset_models} ({round(total_subset_models/total_models *100)}%)** {space} #votes: **{"{:,}".format(total_subset_votes)} ({round(total_subset_votes/total_votes * 100)}%)**{space}
 """
     return leaderboard_md
 
@@ -259,10 +259,14 @@ def get_arena_table(arena_df, model_table_df, arena_subset_df=None):
         arena_subset_df["final_ranking_no_tie"] = range(1, len(arena_subset_df) + 1)
         arena_df["final_ranking_no_tie"] = range(1, len(arena_df) + 1)
         # join arena_df and arena_subset_df on index
-        arena_df = arena_subset_df.join(arena_df["final_ranking_no_tie"], rsuffix="_global", how="inner")
-        # arena_df = arena_subset_df.join(arena_df["rating"], rsuffix="_global", how="inner")
-        arena_df["ranking_difference"] =  arena_df["final_ranking_no_tie_global"] - arena_df["final_ranking_no_tie"]
-        # arena_df['ranking_difference'] =  arena_df['rating_global'] - arena_df['rating']
+        arena_df = arena_subset_df.join(arena_df["final_ranking"], rsuffix="_global", how="inner")
+        arena_df["ranking_difference"] =  arena_df["final_ranking_global"] - arena_df["final_ranking"]
+
+        # no tie version
+        # arena_df = arena_subset_df.join(arena_df["final_ranking_no_tie"], rsuffix="_global", how="inner")
+        # arena_df["ranking_difference"] =  arena_df["final_ranking_no_tie_global"] - arena_df["final_ranking_no_tie"]
+
+        arena_df = arena_df.sort_values(by=["rating"], ascending=False)
         arena_df = arena_df.sort_values(by=["final_ranking"], ascending=True)
         arena_df["final_ranking"] = arena_df.apply(lambda x: create_ranking_str(x["final_ranking"], x["ranking_difference"]), axis=1)
 
@@ -310,7 +314,25 @@ def get_arena_table(arena_df, model_table_df, arena_subset_df=None):
             print(f"{model_key} - {e}")
     return values
 
-key_to_category_name = {"full": "Total", "coding": "Coding", "long": "Long Conversation", "english": "English", "chinese": "Chinese"}
+key_to_category_name = {
+    "full": "Overall",
+    "coding": "Coding",
+    "long_user": "Longer query",
+    "english": "English",
+    "chinese": "Chinese",
+    "no_tie": "Exclude Ties",
+    "no_short": "Exclude Short",
+}
+cat_name_to_explanation = {
+    "Overall": "Overall Questions",
+    "Coding": "Coding: whether conversation contains code snippets",
+    "Longer query": "Longer user query (>= 500 tokens)",
+    "English": "English Prompts",
+    "Chinese": "Chinese Prompts",
+    "Exclude Ties": "Exclude Ties and Bothbad",
+    "Exclude Short": "User Query >= 5 tokens",
+}
+
 
 def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=False):
     arena_dfs = {}
@@ -328,12 +350,12 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
                         arena_dfs[key_to_category_name[k]] = elo_results[k]["leaderboard_table_df"]
                         category_elo_results[key_to_category_name[k]] = elo_results[k]
 
-        p1 = category_elo_results["Total"]["win_fraction_heatmap"]
-        p2 = category_elo_results["Total"]["battle_count_heatmap"]
-        p3 = category_elo_results["Total"]["bootstrap_elo_rating"]
-        p4 = category_elo_results["Total"]["average_win_rate_bar"]
-        arena_df = arena_dfs["Total"]
-        default_md = make_default_md(arena_df, category_elo_results["Total"])
+        p1 = category_elo_results["Overall"]["win_fraction_heatmap"]
+        p2 = category_elo_results["Overall"]["battle_count_heatmap"]
+        p3 = category_elo_results["Overall"]["bootstrap_elo_rating"]
+        p4 = category_elo_results["Overall"]["average_win_rate_bar"]
+        arena_df = arena_dfs["Overall"]
+        default_md = make_default_md(arena_df, category_elo_results["Overall"])
 
     md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown")
     if leaderboard_table_file:
@@ -347,9 +369,10 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
                 md = make_arena_leaderboard_md(arena_df)
                 leaderboard_markdown = gr.Markdown(md, elem_id="leaderboard_markdown")
                 with gr.Row():
-                    category_dropdown = gr.Dropdown(choices=list(arena_dfs.keys()), label="Category", value="Total")
-                    default_category_details = make_category_arena_leaderboard_md(arena_df, arena_df, name="Toal")
-                    with gr.Column(variant="panel"):
+                    with gr.Column(scale=2):
+                        category_dropdown = gr.Dropdown(choices=list(arena_dfs.keys()), label="Category", value="Overall")
+                    default_category_details = make_category_arena_leaderboard_md(arena_df, arena_df, name="Overall")
+                    with gr.Column(scale=4, variant="panel"):
                         category_deets = gr.Markdown(default_category_details, elem_id="category_deets")
 
                 elo_display_df = gr.Dataframe(
@@ -364,7 +387,7 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
                         "Knowledge Cutoff",
                     ],
                     datatype=[
-                        "str",
+                        "number",
                         "markdown",
                         "number",
                         "str",
@@ -449,7 +472,7 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
         pass
 
     def update_leaderboard_df(arena_table_vals):
-        elo_datarame = pd.DataFrame(arena_table_vals, columns=[ "Rank", "Δ", "🤖 Model", "⭐ Arena Elo", "📊 95% CI", "🗳️ Votes", "Organization", "License", "Knowledge Cutoff"])
+        elo_datarame = pd.DataFrame(arena_table_vals, columns=[ "Rank", "Delta", "🤖 Model", "⭐ Arena Elo", "📊 95% CI", "🗳️ Votes", "Organization", "License", "Knowledge Cutoff"])
 
         # goal: color the rows based on the rank with styler
         def highlight_max(s):
@@ -459,20 +482,20 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
         def highlight_rank_max(s):
             return ["color: green; font-weight: bold" if v > 0 else "color: red; font-weight: bold" if v < 0 else "" for v in s]
         
-        return elo_datarame.style.apply(highlight_max, subset=["Rank"]).apply(highlight_rank_max, subset=["Δ"])
+        return elo_datarame.style.apply(highlight_max, subset=["Rank"]).apply(highlight_rank_max, subset=["Delta"])
 
     def update_leaderboard_and_plots(category):
         arena_subset_df = arena_dfs[category]
         arena_subset_df = arena_subset_df[arena_subset_df["num_battles"] > 500]
         elo_subset_results = category_elo_results[category]
-        arena_df = arena_dfs["Total"]
-        arena_values = get_arena_table(arena_df, model_table_df, arena_subset_df = arena_subset_df if category != "Total" else None)
-        if category != "Total":
+        arena_df = arena_dfs["Overall"]
+        arena_values = get_arena_table(arena_df, model_table_df, arena_subset_df = arena_subset_df if category != "Overall" else None)
+        if category != "Overall":
             arena_values = update_leaderboard_df(arena_values)
             arena_values = gr.Dataframe(
                     headers=[
                         "Rank",
-                        "Δ",
+                        "Delta",
                         "🤖 Model",
                         "⭐ Arena Elo",
                         "📊 95% CI",
@@ -482,7 +505,7 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
                         "Knowledge Cutoff",
                     ],
                     datatype=[
-                        "str",
+                        "number",
                         "number",
                         "markdown",
                         "number",
@@ -495,7 +518,7 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
                     value=arena_values,
                     elem_id="arena_leaderboard_dataframe",
                     height=700,
-                    column_widths=[50, 50, 190, 110, 100, 90, 160, 150, 140],
+                    column_widths=[60, 70, 190, 110, 100, 90, 160, 150, 140],
                     wrap=True,
                 )
         else:
@@ -511,7 +534,7 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
                         "Knowledge Cutoff",
                     ],
                     datatype=[
-                        "str",
+                        "number",
                         "markdown",
                         "number",
                         "str",