Updated backend to use newer version of lighteval

requirements.txt

@@ -14,7 +14,7 @@ tqdm==4.65.0
 transformers
 tokenizers>=0.15.0
 # git+https://github.com/huggingface/lighteval.git#egg=lighteval
-git+https://github.com/shaltielshmid/lighteval.git@increase-tgi-container#egg=lighteval
+git+https://github.com/shaltielshmid/lighteval.git@increase-tgi-container-1.5#egg=lighteval
 accelerate==0.24.1
 sentencepiece
 Levenshtein

src/backend/run_eval_suite_lighteval.py

@@ -21,14 +21,29 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
         print("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
 
     args = DefaultNamespace(**{
-            "endpoint_model_name": eval_request.model,
-            "model_dtype": eval_request.precision,
-            "revision": eval_request.revision,
-            "accelerator": accelerator,
-            "vendor": vendor,
-            "region": region,
-            "instance_size": instance_size,
-            "instance_type": instance_type,
+            "model_config": dict(model=dict(
+                type="endpoint",
+                base_params=dict(
+                    endpoint_name=f'{eval_request.model.split("/")[1].replace(".", "-").lower()}-lighteval'[-32:],
+                    model=eval_request.model,
+                    revision=eval_request.revision,
+                    dtype=eval_request.precision,
+                    reuse_existing=False
+                ),
+                instance=dict(
+                    accelerator=accelerator,
+                    region=region,
+                    vendor=vendor,
+                    instance_size=instance_size,
+                    instance_type=instance_type,
+                    framework='pytorch',
+                    endpoint_type='protected',
+                    namespace=OWNER
+                ),
+                generation=dict(
+                    add_special_tokens=True
+                )
+            )),
             "max_samples": limit,
             "job_id": str(datetime.now()),
             "push_results_to_hub": True,
@@ -42,9 +57,7 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
             "custom_tasks": "custom_tasks.py",
             "tasks": task_names,
             "dataset_loading_processes": 24,
-            "num_fewshot_seeds": 0,
-            "reuse_existing": False,
-            "namespace": OWNER
+            "num_fewshot_seeds": 0
     })
 
     try: