init

README.md

@@ -1,12 +1,13 @@
 ---
-title: BalmLeaderboard
-emoji: 🔥
-colorFrom: purple
-colorTo: green
+title: BALM Leaderboard
+emoji: 🦾
+colorFrom: orange
+colorTo: indigo
 sdk: gradio
-sdk_version: 3.32.0
+sdk_version: 3.27.0
 app_file: app.py
-pinned: false
+pinned: true
+license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,179 @@
+import os
+from email.utils import parseaddr
+
+import gradio as gr
+import pandas as pd
+
+from datasets import load_dataset
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import HfApi
+
+# InfoStrings
+from content import *
+
+BALM_TOKEN = os.environ.get("BALM_TOKEN", None)
+owner="clefourrier" # change to balm once possible
+
+api = HfApi()
+
+eval_results = {}
+eval_dataframe = {}
+for level in range(1, 4):
+    eval_results[level] = load_dataset(f"{owner}/BALM_ResultsDev{level}", token=BALM_TOKEN, split="dev")
+    eval_dataframe[level] = pd.DataFrame(eval_results[level].remove_column("mail"))
+
+def restart_space():
+    api.restart_space(repo_id=f"{owner}/BALM_Leaderboard", token=BALM_TOKEN)
+
+
+COLS = ["Model", "Organisation", "Reported accuracy ⬆️"]
+TYPES = ["str", "str", "number",]
+
+def add_new_eval(
+    level_of_dev: str,
+    model: str,
+    score: float,
+    organisation: str,
+    mail: str,
+):
+    level = int(level_of_dev.split(" ")[-1])
+
+    # Very basic email parsing
+    _, parsed_mail = parseaddr(mail)
+    if not "@" in parsed_mail:
+        valid_mail = "Please provide a valid email adress."
+        return f"<p style='color: orange; font-size: 20px; text-align: center;'>{valid_mail}</p>"
+
+    print("Adding new eval")
+
+    # Check if the combination model/org already exists and prints a warning message if yes
+    if model.lower() in set(eval_results[level]["model"]) and organisation.lower() in set(eval_results[level]["organisation"]):
+        duplicate_request_message = "This model has been already submitted."
+        return f"<p style='color: orange; font-size: 20px; text-align: center;'>{duplicate_request_message}</p>"
+
+    # Actual submission
+    eval_entry = {
+        "model": model,
+        "score": score,
+        "organisation": organisation,
+        "mail": mail,
+    }
+    eval_results[level].add_item(eval_entry)
+
+    success_message = f"Model {model} submitted by {organisation}."
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{success_message}</p>"
+
+
+def refresh():
+    eval_results = {}
+    eval_dataframe = {}
+    for level in range(1, 4):
+        eval_results[level] = load_dataset(f"{owner}/BALM_ResultsDev{level}", token=BALM_TOKEN, split="dev")
+        eval_dataframe[level] = pd.DataFrame(eval_results[level].remove_column("mail"))
+    return eval_dataframe[1],  eval_dataframe[2],  eval_dataframe[3]
+
+
+custom_css = """
+#changelog-text {
+    font-size: 16px !important;
+}
+
+#changelog-text h2 {
+    font-size: 18px !important;
+}
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+"""
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Row():
+        with gr.Column():
+            with gr.Accordion("📙 Citation", open=False):
+                citation_button = gr.Textbox(
+                    value=CITATION_BUTTON_TEXT,
+                    label=CITATION_BUTTON_LABEL,
+                    elem_id="citation-button",
+                ).style(show_copy_button=True)
+        with gr.Column():
+            with gr.Accordion("✨ CHANGELOG", open=False):
+                changelog = gr.Markdown(CHANGELOG_TEXT, elem_id="changelog-text")
+
+    with gr.Tab("Results: Level 1"):
+        with gr.Tab("Results on Dev Set"):
+            leaderboard_table_1 = gr.components.Dataframe(
+                value=eval_dataframe[1], headers=COLS, datatype=TYPES, max_rows=20
+            )
+        with gr.Tab("Results on Test Set"):
+            gr.Textbox(value="The test set is currently private! Come back when performances on the dev set increased!")
+    with gr.Tab("Results: Level 2"):
+        with gr.Tab("Results on Dev Set"):
+            leaderboard_table_2 = gr.components.Dataframe(
+                value=eval_dataframe[2], headers=COLS, datatype=TYPES, max_rows=20
+            )
+        with gr.Tab("Results on Test Set"):
+            gr.Textbox(value="The test set is currently private! Come back when performances on the dev set increased!")
+    with gr.Tab("Results: Level 3"):
+        with gr.Tab("Results on Dev Set"):
+            leaderboard_table_3 = gr.components.Dataframe(
+                value=eval_dataframe[3], headers=COLS, datatype=TYPES, max_rows=20
+            )
+        with gr.Tab("Results on Test Set"):
+            gr.Textbox(value="The test set is currently private! Come back when performances on the dev set increased!")
+
+    refresh_button = gr.Button("Refresh")
+    refresh_button.click(
+        refresh,
+        inputs=[],
+        outputs=[
+            eval_dataframe[1],
+            eval_dataframe[2],
+            eval_dataframe[3],
+        ],
+    )
+
+    with gr.Accordion("Submit a new model for evaluation"):
+        #with gr.Row():
+        with gr.Column():
+            level_of_dev = gr.Radio(["Level 1", "Level 2", "Level 3"], value="Level 1", label="Dev set")
+            model_name_textbox = gr.Textbox(label="Model name")
+            score = gr.Textbox(label="Score")
+            organisation = gr.Textbox(label="Organisation")
+            mail = gr.Textbox(label="Contact email")
+
+        submit_button = gr.Button("Submit Eval")
+        submission_result = gr.Markdown()
+        submit_button.click(
+            add_new_eval,
+            [
+                level_of_dev,
+                model_name_textbox,
+                score,
+                organisation,
+                mail
+            ],
+            submission_result,
+        )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=3600)
+scheduler.start()
+demo.launch()

content.py

@@ -0,0 +1,32 @@
+CHANGELOG_TEXT = f"""
+## [2023-06-02] 
+- Beta internal version of the leaderboard
+"""
+
+TITLE = """<h1 align="center" id="space-title">BALM Leaderboard</h1>"""
+
+CANARY_STRING = "" # TODO
+
+INTRODUCTION_TEXT = f"""
+Large language models have seen their potential capabilities increased by several orders of magnitude with the introduction of augmentations, from simple prompting adjustement to actual external tooling (calculators, vision models, ...) or online web retrieval.
+
+To evaluate the next generation of LLMs, we argue for a new kind of benchmark, simple and yet effective to measure actual progress on augmented capabilities,
+We therefore present BALM.
+
+BALM is made of 3 evaluation levels, depending on the added level of tooling and autonomy the model needs.
+We expect the level 1 to be breakable by very good LLMs, and the level 3 to indicate a strong jump in model capabilities.
+
+Each of these levels is divided into two sets: a public dev set, on which people can self report their results, and a private test set, which will be unlocked once public performance passes a threshold on the dev set.
+
+Please do not repost the public dev set, nor use it in training data for your models. Its canary string is """ + CANARY_STRING + """ and files containing this string should be removed from training data.
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@misc{balm, # TODO
+  author = {tbd},
+  title = {Benchmark for Augmented Language Models},
+  year = {2023},
+  #publisher = {Hugging Face},
+  #howpublished = "\url{https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard}"
+}"""
+

requirements.txt

@@ -0,0 +1,8 @@
+datasets
+gradio==3.27.0
+gradio_client==0.1.3
+huggingface-hub==0.13.4
+pandas==2.0.0
+tqdm==4.65.0
+transformers==4.28.1
+typing_extensions==4.5.0