update

gen_table.py

@@ -7,7 +7,7 @@ import gradio as gr
 import numpy as np
 import pandas as pd
 
-from meta_data import META_FIELDS, URL
+from meta_data import DEFAULT_BENCH, META_FIELDS, URL
 
 
 def listinstr(lst, s):
@@ -62,16 +62,15 @@ def model_type_flag(line, FIELDS):
 def BUILD_L1_DF(results, fields):
     check_box = {}
     check_box['essential'] = ['Method', 'Parameters (B)', 'Language Model', 'Vision Model']
-    # revise there to set defualt dataset
-    defualt_dataset = ['MMBench_TEST_EN', 'MMBench_TEST_CN', 'MMStar', 'MME', 'MMMU_VAL', 'MathVista', 'OCRBench', 'MMVet', 'AI2D', 'HallusionBench', 'LLaVABench', 'SEEDBench_IMG']
-    check_box['required'] = ['Avg Score', 'Avg Rank'] + defualt_dataset
+    # revise there to set default dataset
+    check_box['required'] = ['Avg Score', 'Avg Rank'] + DEFAULT_BENCH
     check_box['avg'] = ['Avg Score', 'Avg Rank']
     check_box['all'] = check_box['avg'] + fields
     type_map = defaultdict(lambda: 'number')
     type_map['Method'] = 'html'
     type_map['Language Model'] = type_map['Vision Model'] = type_map['OpenSource'] = type_map['Verified'] = 'str'
     check_box['type_map'] = type_map
-    
+
     res = generate_table(results, fields)
     df = pd.DataFrame(res)
     df = df.sort_values('Avg Score')
@@ -169,4 +168,4 @@ def generate_table(results, fields, df=None):
         df.update(res)
         df = df.sort_values('Avg Score')
         df = df.iloc[::-1]
-    return df
+    return df

meta_data.py

@@ -24,6 +24,11 @@ MAIN_FIELDS = [
     'MMMU_VAL', 'MathVista', 'OCRBench', 'AI2D', 
     'HallusionBench', 'SEEDBench_IMG', 'MMVet', 'LLaVABench', 'CCBench', 'RealWorldQA'
 ]
+DEFAULT_BENCH = [
+    'MMBench_TEST_EN', 'MMBench_TEST_CN', 'MMStar', 'MME',
+    'MMMU_VAL', 'MathVista', 'OCRBench', 'AI2D', 
+    'HallusionBench', 'SEEDBench_IMG', 'MMVet', 'LLaVABench'
+]
 MMBENCH_FIELDS = ['MMBench_TEST_EN', 'MMBench_DEV_EN', 'MMBench_TEST_CN', 'MMBench_DEV_CN', 'CCBench']
 MODEL_SIZE = ['<10B', '10B-20B', '20B-40B', '>40B', 'Unknown']
 MODEL_TYPE = ['API', 'OpenSource', 'Proprietary']
@@ -37,8 +42,9 @@ LEADERBOARD_MD['MAIN'] = f"""
 - Metrics:
   - Avg Score: The average score on all VLM Benchmarks (normalized to 0 - 100, the higher the better). 
   - Avg Rank: The average rank on all VLM Benchmarks (the lower the better). 
-- The overall evaluation results on {len(MAIN_FIELDS)} VLM benchmarks, sorted by the descending order of Avg Score. 
-  - The following datasets are included in the main results: {', '.join(MAIN_FIELDS)}. 
+  - Avg Score & Rank are calculated based on selected benchmark. 
+- By default, we present the overall evaluation results based on {len(DEFAULT_BENCH)} VLM benchmarks, sorted by the descending order of Avg Score. 
+  - The following datasets are included in the main results: {', '.join(DEFAULT_BENCH)}. 
   - Detailed evaluation results for each dataset (included or not included in main) are provided in the consequent tabs. 
 """