Muennighoff commited on
Commit
70c518e
1 Parent(s): 536679f
Files changed (2) hide show
  1. EXTERNAL_MODEL_RESULTS.json +0 -0
  2. app.py +7 -1
EXTERNAL_MODEL_RESULTS.json CHANGED
The diff for this file is too large to render. See raw diff
 
app.py CHANGED
@@ -242,7 +242,13 @@ def get_mteb_data(tasks=["Clustering"], langs=[], datasets=[], fillna=True, add_
242
  task_results = [sub_res for sub_res in meta["model-index"][0]["results"] if (sub_res.get("task", {}).get("type", "") in tasks) and (sub_res.get("dataset", {}).get("config", "default") in ("default", *langs))]
243
  else:
244
  task_results = [sub_res for sub_res in meta["model-index"][0]["results"] if (sub_res.get("task", {}).get("type", "") in tasks)]
245
- out = [{res["dataset"]["name"].replace("MTEB ", ""): [round(score["value"], 2) for score in res["metrics"] if score["type"] == task_to_metric.get(res["task"]["type"])][0]} for res in task_results]
 
 
 
 
 
 
246
  out = {k: v for d in out for k, v in d.items()}
247
  out["Model"] = make_clickable_model(model.modelId)
248
  # Model & at least one result
 
242
  task_results = [sub_res for sub_res in meta["model-index"][0]["results"] if (sub_res.get("task", {}).get("type", "") in tasks) and (sub_res.get("dataset", {}).get("config", "default") in ("default", *langs))]
243
  else:
244
  task_results = [sub_res for sub_res in meta["model-index"][0]["results"] if (sub_res.get("task", {}).get("type", "") in tasks)]
245
+ # if model.modelId == "w601sxs/b1ade-embed-kd_3":
246
+ # import pdb; pdb.set_trace()
247
+ try:
248
+ out = [{res["dataset"]["name"].replace("MTEB ", ""): [round(score["value"], 2) for score in res["metrics"] if score["type"] == task_to_metric.get(res["task"]["type"])][0]} for res in task_results]
249
+ except:
250
+ print("ERROR", model.modelId)
251
+ continue
252
  out = {k: v for d in out for k, v in d.items()}
253
  out["Model"] = make_clickable_model(model.modelId)
254
  # Model & at least one result