submit-system-update (#838)

- add login at submit and user info (0b31d4e1ffcd10bcdf9603d588d3b1d28c77fdc2)
- use gradio-leaderboard 0.0.11 (c8c479205c76ee4c961253c5909ffaa1eb6b03ed)
- update and revised submission info (8013f916ae1e9df83a51c72a1b736f51f10c8bc5)

app.py

@@ -235,6 +235,7 @@ with main_block:
 
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+                login_button = gr.LoginButton(elem_id="oauth-button")
 
             with gr.Row():
                 with gr.Column():

pyproject.toml

@@ -45,7 +45,7 @@ tokenizers = ">=0.15.0"
 gradio-space-ci = {git = "https://huggingface.co/spaces/Wauplin/gradio-space-ci", rev = "0.2.3"}
 isort = "^5.13.2"
 ruff = "^0.3.5"
-gradio-leaderboard = "0.0.11rc0"
+gradio-leaderboard = "0.0.11"
 gradio = {extras = ["oauth"], version = "^4.36.1"}
 requests = "^2.31.0"
 requests-oauthlib = "^1.3.1"

requirements.txt

@@ -17,7 +17,7 @@ isort
 ruff
 gradio==4.31.0
 gradio[oauth]
-gradio_leaderboard==0.0.11rc0
+gradio_leaderboard==0.0.11
 requests==2.31.0
 requests-oauthlib== 1.3.1
 schedule == 1.2.2

src/display/about.py

@@ -179,35 +179,47 @@ EVALUATION_QUEUE_TEXT = f"""
 
 Models added here will be automatically evaluated on the 🤗 cluster.
 
-## Don't forget to read the FAQ and the About tabs for more information!
+> **Important:** Don't forget to read the [FAQ](https://huggingface.co/docs/leaderboards/open_llm_leaderboard/faq) and [documentation](https://huggingface.co/docs/leaderboards/open_llm_leaderboard/about) for more information! 📄
 
-## First steps before submitting a model
+## Submission Disclaimer
+**By submitting a model, you acknowledge that:**
+- We store information about who submitted each model in [Requests dataset](https://huggingface.co/datasets/open-llm-leaderboard/requests).
+- This practice helps maintain the integrity of our leaderboard, prevent spam, and ensure responsible submissions.
+- Your submission will be visible to the community and you may be contacted regarding your model.
+- Please submit carefully and responsibly 💛
+
+## First Steps Before Submitting a Model
+
+### 1. Ensure Your Model Loads with AutoClasses
+Verify that you can load your model and tokenizer using AutoClasses:
 
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
 ```python
 from transformers import AutoConfig, AutoModel, AutoTokenizer
 config = AutoConfig.from_pretrained("your model name", revision=revision)
 model = AutoModel.from_pretrained("your model name", revision=revision)
 tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
 ```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+Note:
+- If this step fails, debug your model before submitting.
+- Ensure your model is public.
+- We are working on adding support for models requiring `use_remote_code=True`.
 
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+### 2. Convert Weights to Safetensors
+[Safetensors](https://huggingface.co/docs/safetensors/index) is a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
 
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
+### 3. Verify Your Model Open License
 This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
 
-### 4) Fill up your model card
+### 4. Complete Your Model Card
 When we add extra information about models to the leaderboard, it will be automatically taken from the model card
 
-### 5) Select the correct precision
-Not all models are converted properly from `float16` to `bfloat16`, and selecting the wrong precision can sometimes cause evaluation error (as loading a `bf16` model in `fp16` can sometimes generate NaNs, depending on the weight range).
+### 5. Select Correct Precision
+Choose the right precision to avoid evaluation errors:
+- Not all models convert properly from float16 to bfloat16.
+- Incorrect precision can cause issues (e.g., loading a bf16 model in fp16 may generate NaNs).
+
+> **Important:** When submitting, git branches and tags will be strictly tied to the specific commit present at the time of submission to ensure revision consistency.
 
-<b>Note:</b> Please be advised that when submitting, git <b>branches</b> and <b>tags</b> will be strictly tied to the <b>specific commit</b> present at the time of submission. This ensures revision consistency.
 ## Model types
 {icons}
 """

src/submission/submit.py

@@ -1,5 +1,6 @@
 import json
 import os
+import gradio as gr
 from datetime import datetime, timezone
 
 from dataclasses import dataclass
@@ -59,16 +60,25 @@ def add_new_eval(
     weight_type: str,
     model_type: str,
     use_chat_template: bool,
-):
+    profile: gr.OAuthProfile | None
+):  
+    # Login require
+    if profile is None:
+        return styled_error("Hub Login Required")
+
+    # Name of the actual user who sent the request
+    username = profile.username
+
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
-    user_name = ""
+
+    org_or_user = ""
     model_path = model
     if "/" in model:
-        user_name = model.split("/")[0]
+        org_or_user = model.split("/")[0]
         model_path = model.split("/")[1]
 
     precision = precision.split(" ")[0]
@@ -77,10 +87,14 @@ def add_new_eval(
     if model_type is None or model_type == "":
         return styled_error("Please select a model type.")
 
+    # Is user submitting own model?
+    # Check that username in the org. 
+    # if org_or_user != profile.username:
+
     # Is the user rate limited?
-    if user_name != "":
+    if org_or_user != "":
         user_can_submit, error_msg = user_submission_permission(
-            user_name, USERS_TO_SUBMISSION_DATES, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+            org_or_user, USERS_TO_SUBMISSION_DATES, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
         )
         if not user_can_submit:
             return styled_error(error_msg)
@@ -144,7 +158,6 @@ def add_new_eval(
 
     # Seems good, creating the eval
     print("Adding new eval")
-
     eval_entry = {
         "model": model,
         "base_model": base_model,
@@ -159,10 +172,11 @@ def add_new_eval(
         "job_id": -1,
         "job_start_time": None,
         "use_chat_template": use_chat_template,
+        "sender": username
     }
 
     print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{org_or_user}"
     os.makedirs(OUT_DIR, exist_ok=True)
     out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
 
@@ -170,6 +184,7 @@ def add_new_eval(
         f.write(json.dumps(eval_entry))
 
     print("Uploading eval file")
+    print(eval_entry)
     API.upload_file(
         path_or_fileobj=out_path,
         path_in_repo=out_path.split("eval-queue/")[1],