Added admin exception for counters

main_backend_lighteval.py

@@ -81,44 +81,47 @@ def run_auto_eval():
         pp.pprint(dict(message="Number of params too big, can't run this model", params=eval_request.params))
         return
     
-    counter_key = f'count_{instance_size}_{instance_type}'
-    if not counter_key in requests_seen:
-        requests_seen[counter_key] = 0
-    if requests_seen[counter_key] >= cap:
-        set_eval_request(
-            api=API,
-            eval_request=eval_request,
-            set_to_status=REJECTED_STATUS,
-            hf_repo=QUEUE_REPO,
-            local_dir=EVAL_REQUESTS_PATH_BACKEND,
-        )
-        pp.pprint(dict(message="Reached maximum cap for requests of this instance type this month", counter=counter_key, instance_type=instance_type, cap=cap))
-        return
-
-    # next, check to see who made the last commit to this repo - keep track of that. One person shouldn't commit more 
-    # than 4 models in one month. 
-    user = eval_request.user_info['name']
-    if user in requests_seen and len(requests_seen[user]) >= 4:
-        set_eval_request(
+    # ignore counters, from admins
+    is_admin = any(org['name'] == 'hebrew-llm-leaderboard' for org in eval_request.user_info.get('orgs', []))
+    if not is_admin:
+        counter_key = f'count_{instance_size}_{instance_type}'
+        if not counter_key in requests_seen:
+            requests_seen[counter_key] = 0
+        if requests_seen[counter_key] >= cap:
+            set_eval_request(
+                api=API,
+                eval_request=eval_request,
+                set_to_status=REJECTED_STATUS,
+                hf_repo=QUEUE_REPO,
+                local_dir=EVAL_REQUESTS_PATH_BACKEND,
+            )
+            pp.pprint(dict(message="Reached maximum cap for requests of this instance type this month", counter=counter_key, instance_type=instance_type, cap=cap))
+            return
+
+        # next, check the user that submitted the request - allow up to 4 per user
+        user = eval_request.user_info['name']
+        if user in requests_seen and len(requests_seen[user]) >= 4:
+            set_eval_request(
+                api=API,
+                eval_request=eval_request,
+                set_to_status=REJECTED_STATUS,
+                hf_repo=QUEUE_REPO,
+                local_dir=EVAL_REQUESTS_PATH_BACKEND,
+            )
+            pp.pprint(dict(message="Reached maximum cap for requests for this user this month", counter=counter_key, user=user))
+            return
+        if not user in requests_seen:
+            requests_seen[user] = []
+        
+        requests_seen[user].append(dict(model_id=eval_request.model, revision=eval_request.revision))
+        requests_seen[counter_key] += 1
+        set_requests_seen(
             api=API,
-            eval_request=eval_request,
-            set_to_status=REJECTED_STATUS,
+            requests_seen=requests_seen,
             hf_repo=QUEUE_REPO,
-            local_dir=EVAL_REQUESTS_PATH_BACKEND,
+            local_dir=EVAL_REQUESTS_PATH_BACKEND
         )
-        pp.pprint(dict(message="Reached maximum cap for requests for this user this month", counter=counter_key, user=user))
-        return
-    if not user in requests_seen:
-        requests_seen[user] = []
-        
-    requests_seen[user].append(dict(model_id=eval_request.model, revision=eval_request.revision))
-    requests_seen[counter_key] += 1
-    set_requests_seen(
-        api=API,
-        requests_seen=requests_seen,
-        hf_repo=QUEUE_REPO,
-        local_dir=EVAL_REQUESTS_PATH_BACKEND
-    )
+        # end of counters check for non-admins
 
     set_eval_request(
         api=API,
@@ -127,7 +130,6 @@ def run_auto_eval():
         hf_repo=QUEUE_REPO,
         local_dir=EVAL_REQUESTS_PATH_BACKEND,
     )
-
     
 
     run_evaluation(