"""A gradio app that renders a static leaderboard. This is used for Hugging Face Space."""
import ast
import argparse
import glob
import pickle
import gradio as gr
import numpy as np
import pandas as pd
leader_component_values = [None]
space = " "
def make_default_md():
leaderboard_md = f"""
# NeurIPS LLM Merging Competition Leaderboard
[Website](https://llm-merging.github.io/index) | [Starter Kit (Github)](https://github.com/llm-merging/LLM-Merging) | [Discord](https://discord.com/invite/dPBHEVnV)
"""
return leaderboard_md
def make_arena_leaderboard_md(model_table_df):
total_models = len(model_table_df)
leaderboard_md = f"""
Validation Benchmark Performance is averaged.
Final performance will be assessed at the end of the competition on a hidden test set, which may or may not be correlated with Validation performance.
If you've made a submission, but don't see your model below, or there is no score for your model, please be patient -- our current setup requires us to manually evaluate all submissions.
Higher values are better.
Total #models: **{total_models}**.{space}
"""
return leaderboard_md
def load_leaderboard_table_csv(filename, add_hyperlink=False):
lines = open(filename).readlines()
heads = [v.strip() for v in lines[0].split(",")]
rows = []
for i in range(1, len(lines)):
row = [v.strip() for v in lines[i].split(",")]
for j in range(len(heads)):
item = {}
for h, v in zip(heads, row):
if h == "Validation Score":
if v != "-":
v = ast.literal_eval(v)
else:
v = np.nan
item[h] = v
if add_hyperlink:
item["Model"] = f'{item["Model"]}'
rows.append(item)
rows.sort(key=lambda m: m.get("Validation Score", 0), reverse=True)
return rows
def get_full_table(model_table_df):
values = []
for i in range(len(model_table_df)):
row = []
ranking = i+1
row.append(ranking)
model_name = model_table_df.iloc[i]["Model"]
score = model_table_df.iloc[i]["Validation Score"]
# model display name
row.append(model_name)
row.append(score)
values.append(row)
# values.sort(key=lambda x: -x[1] if not np.isnan(x[1]) else 1e9)
return values
key_to_category_name = {
"full": "Overall",
}
cat_name_to_explanation = {
"Overall": "Overall Questions",
}
def build_leaderboard_tab(leaderboard_table_file, show_plot=False):
arena_dfs = {}
category_elo_results = {}
if leaderboard_table_file is None: # Do live update
default_md = "Loading ..."
else:
default_md = make_default_md()
md_1 = gr.Markdown(default_md, elem_id="leaderboard_markdown")
if leaderboard_table_file:
data = load_leaderboard_table_csv(leaderboard_table_file)
model_table_df = pd.DataFrame(data)
with gr.Tabs() as tabs:
arena_table_vals = get_full_table(model_table_df)
with gr.Tab("Full leaderboard", id=0):
md = make_arena_leaderboard_md(model_table_df)
leaderboard_markdown = gr.Markdown(md, elem_id="leaderboard_markdown")
display_df = gr.Dataframe(
headers=[
"Rank",
"🤖 Model / Submission Name",
"⭐ Validation Performance",
],
datatype=[
"number",
"markdown",
"number",
],
value=arena_table_vals,
elem_id="arena_leaderboard_dataframe",
height=700,
column_widths=[70, 190, 110],
wrap=True,
)
gr.Markdown(
f"""Note: .
""",
elem_id="leaderboard_markdown"
)
leader_component_values[:] = [default_md]
if not show_plot:
gr.Markdown(
""" ## Submit your model [here]().
""",
elem_id="leaderboard_markdown",
)
else:
pass
with gr.Accordion(
"📝 Citation",
open=True,
):
citation_md = """
### Citation
Please cite the following paper
"""
gr.Markdown(citation_md, elem_id="leaderboard_markdown")
gr.Markdown(acknowledgment_md)
return [md_1]
block_css = """
#notice_markdown {
font-size: 104%
}
#notice_markdown th {
display: none;
}
#notice_markdown td {
padding-top: 6px;
padding-bottom: 6px;
}
#category_deets {
text-align: center;
padding: 0px;
padding-left: 5px;
}
#leaderboard_markdown {
font-size: 104%
}
#leaderboard_markdown td {
padding-top: 6px;
padding-bottom: 6px;
}
#leaderboard_header_markdown {
font-size: 104%;
text-align: center;
display:block;
}
#leaderboard_dataframe td {
line-height: 0.1em;
}
#plot-title {
text-align: center;
display:block;
}
#non-interactive-button {
display: inline-block;
padding: 10px 10px;
background-color: #f7f7f7; /* Super light grey background */
text-align: center;
font-size: 26px; /* Larger text */
border-radius: 0; /* Straight edges, no border radius */
border: 0px solid #dcdcdc; /* A light grey border to match the background */
user-select: none; /* The text inside the button is not selectable */
pointer-events: none; /* The button is non-interactive */
}
footer {
display:none !important
}
.sponsor-image-about img {
margin: 0 20px;
margin-top: 20px;
height: 40px;
max-height: 100%;
width: auto;
float: left;
}
"""
acknowledgment_md = """
### Acknowledgment
We thank Hugging Face, sakana.ai, and arcee.ai for their generous [sponsorship](https://llm-merging.github.io/sponsors).
"""
def build_demo(leaderboard_table_file):
text_size = gr.themes.sizes.text_lg
theme = gr.themes.Base(text_size=text_size)
theme.set(button_secondary_background_fill_hover="*primary_300",
button_secondary_background_fill_hover_dark="*primary_700")
with gr.Blocks(
title="LLM Merging Leaderboard",
theme=theme,
# theme = gr.themes.Base.load("theme.json"), # uncomment to use new cool theme
css=block_css,
) as demo:
leader_components = build_leaderboard_tab(
leaderboard_table_file, show_plot=True
)
return demo
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--share", action="store_true")
parser.add_argument("--host", default="0.0.0.0")
parser.add_argument("--port", type=int, default=7860)
args = parser.parse_args()
leaderboard_table_files = glob.glob("leaderboard_table_*.csv")
leaderboard_table_files.sort(key=lambda x: int(x[18:-4]))
leaderboard_table_file = leaderboard_table_files[-1]
demo = build_demo(leaderboard_table_file)
demo.launch(share=args.share, server_name=args.host, server_port=args.port)