diff --git a/app.py b/app.py index 2821630a98f6a1b3e014cd08aef49f5720a16d5c..a5555bfdf2e7667ac73cc190432ea53a96376709 100644 --- a/app.py +++ b/app.py @@ -1,3 +1,4 @@ +from datasets import load_dataset import gradio as gr import pandas as pd from huggingface_hub import HfApi, hf_hub_download @@ -29,6 +30,8 @@ TASK_LIST_CLASSIFICATION = [ "TweetSentimentExtractionClassification", ] +TASK_LIST_CLASSIFICATION_NORM = [x.replace(" (en)", "") for x in TASK_LIST_CLASSIFICATION] + TASK_LIST_CLUSTERING = [ "ArxivClusteringP2P", "ArxivClusteringS2S", @@ -74,6 +77,20 @@ TASK_LIST_RETRIEVAL = [ "TRECCOVID", ] +TASK_LIST_RETRIEVAL_NORM = TASK_LIST_RETRIEVAL + ["CQADupstackAndroidRetrieval", + "CQADupstackEnglishRetrieval", + "CQADupstackGamingRetrieval", + "CQADupstackGisRetrieval", + "CQADupstackMathematicaRetrieval", + "CQADupstackPhysicsRetrieval", + "CQADupstackProgrammersRetrieval", + "CQADupstackStatsRetrieval", + "CQADupstackTexRetrieval", + "CQADupstackUnixRetrieval", + "CQADupstackWebmastersRetrieval", + "CQADupstackWordpressRetrieval" +] + TASK_LIST_STS = [ "BIOSSES", "SICK-R", @@ -87,6 +104,7 @@ TASK_LIST_STS = [ "STSBenchmark", ] +TASK_LIST_STS_NORM = [x.replace(" (en)", "").replace(" (en-en)", "") for x in TASK_LIST_STS] TASK_LIST_SUMMARIZATION = [ "SummEval", @@ -105,19 +123,107 @@ TASK_TO_METRIC = { "Summarization": "cos_sim_spearman", } -def make_clickable_model(model_name): +def make_clickable_model(model_name, link=None): # Remove user from model name - model_name_show = " ".join(model_name.split("/")[1:]) - link = "https://huggingface.co/" + model_name + model_name = model_name.split("/")[-1] + if link is None: + link = "https://huggingface.co/" + model_name return ( - f'{model_name_show}' + f'{model_name}' ) +# Models without metadata, thus we cannot fetch their results naturally +EXTERNAL_MODELS = [ + "LASER2", + "LaBSE", + "all-MiniLM-L12-v2", + "all-MiniLM-L6-v2", + "all-mpnet-base-v2", + "allenai-specter", + "bert-base-uncased", + "contriever-base-msmarco", + "glove.6B.300d", + "gtr-t5-base", + "gtr-t5-large", + "gtr-t5-xl", + "gtr-t5-xxl", + "komninos", + "msmarco-bert-co-condensor", + "paraphrase-multilingual-MiniLM-L12-v2", + "paraphrase-multilingual-mpnet-base-v2", + "sentence-t5-base", + "sentence-t5-large", + "sentence-t5-xl", + "sentence-t5-xxl", + "sgpt-bloom-1b3-nli", + "sgpt-bloom-7b1-msmarco", + "sgpt-nli-bloom-1b3", + "sup-simcse-bert-base-uncased", +# "text-similarity-ada-001", + "unsup-simcse-bert-base-uncased", +] +EXTERNAL_MODEL_TO_LINK = { + "LASER2": "https://github.com/facebookresearch/LASER", + "text-similarity-ada-001": "https://beta.openai.com/docs/guides/embeddings/types-of-embedding-models", +} +EXTERNAL_MODEL_RESULTS = {model: {k: {v: []} for k, v in TASK_TO_METRIC.items()} for model in EXTERNAL_MODELS} + +def add_lang(examples): + if not(examples["eval_language"]): + examples["mteb_dataset_name_with_lang"] = examples["mteb_dataset_name"] + else: + examples["mteb_dataset_name_with_lang"] = examples["mteb_dataset_name"] + f' ({examples["eval_language"]})' + return examples + +def add_task(examples): + # Could be added to the dataset loading script instead + if examples["mteb_dataset_name"] in TASK_LIST_CLASSIFICATION_NORM: + examples["mteb_task"] = "Classification" + elif examples["mteb_dataset_name"] in TASK_LIST_CLUSTERING: + examples["mteb_task"] = "Clustering" + elif examples["mteb_dataset_name"] in TASK_LIST_PAIR_CLASSIFICATION: + examples["mteb_task"] = "PairClassification" + elif examples["mteb_dataset_name"] in TASK_LIST_RERANKING: + examples["mteb_task"] = "Reranking" + elif examples["mteb_dataset_name"] in TASK_LIST_RETRIEVAL_NORM: + examples["mteb_task"] = "Retrieval" + elif examples["mteb_dataset_name"] in TASK_LIST_STS_NORM: + examples["mteb_task"] = "STS" + elif examples["mteb_dataset_name"] in TASK_LIST_SUMMARIZATION: + examples["mteb_task"] = "Summarization" + else: + examples["mteb_task"] = "BitextMining" + return examples + +for model in EXTERNAL_MODELS: + ds = load_dataset("mteb/results", model) + ds = ds.map(add_lang) + ds = ds.map(add_task) + base_dict = {"Model": make_clickable_model(model, link=EXTERNAL_MODEL_TO_LINK.get(model, "https://huggingface.co/mteb/leaderboard"))} + # For now only one metric per task - Could add more metrics lateron + for task, metric in TASK_TO_METRIC.items(): + ds_dict = ds.filter(lambda x: (x["mteb_task"] == task) and (x["metric"] == metric))["test"].to_dict() + ds_dict = {k: round(v, 2) for k, v in zip(ds_dict["mteb_dataset_name_with_lang"], ds_dict["score"])} + EXTERNAL_MODEL_RESULTS[model][task][metric].append({**base_dict, **ds_dict}) + def get_mteb_data(tasks=["Clustering"], langs=[], cast_to_str=True, task_to_metric=TASK_TO_METRIC): api = HfApi() models = api.list_models(filter="mteb") + # Initialize list to models that we cannot fetch metadata from df_list = [] + for model in EXTERNAL_MODEL_RESULTS: + results_list = [res for task in tasks for res in EXTERNAL_MODEL_RESULTS[model][task][task_to_metric[task]]] + if langs: + # Would be cleaner to rely on an extra language column instead + langs_format = [f"({lang})" for lang in langs] + res = {k: v for d in results_list for k, v in d.items() if any([k.split(" ")[-1] in (k, x) for x in langs_format])} + else: + res = {k: v for d in results_list for k, v in d.items()} + # Model & at least one result + if len(res) > 1: + df_list.append(res) + for model in models: readme_path = hf_hub_download(model.modelId, filename="README.md") meta = metadata_load(readme_path) @@ -154,8 +260,8 @@ def get_mteb_data(tasks=["Clustering"], langs=[], cast_to_str=True, task_to_metr return df.astype(str) # Cast to str as Gradio does not accept floats return df -def get_mteb_average(get_all_avgs=False): - global DATA_OVERALL, DATA_CLASSIFICATION_EN, DATA_CLUSTERING, DATA_PAIR_CLASSIFICATION, DATA_RERANKING, DATA_RETRIEVAL, DATA_STS_EN, DATA_SUMMARIZATION +def get_mteb_average(): + global DATA_OVERALL, DATA_CLASSIFICATION_EN, DATA_CLUSTERING, DATA_PAIR_CLASSIFICATION, DATA_RERANKING, DATA_RETRIEVAL, DATA_STS_EN, DATA_SUMMARIZATION, NUM_SCORES DATA_OVERALL = get_mteb_data( tasks=[ "Classification", @@ -169,6 +275,11 @@ def get_mteb_average(get_all_avgs=False): langs=["en", "en-en"], cast_to_str=False ) + # Approximation (Missing Bitext Mining & including some nans) + NUM_SCORES = DATA_OVERALL.shape[0] * DATA_OVERALL.shape[1] + + print("OVERALLDATA", DATA_OVERALL) + DATA_OVERALL.to_csv("overall.csv") DATA_OVERALL.insert(1, f"Average ({len(TASK_LIST_EN)} datasets)", DATA_OVERALL[TASK_LIST_EN].mean(axis=1, skipna=False)) DATA_OVERALL.insert(2, f"Classification Average ({len(TASK_LIST_CLASSIFICATION)} datasets)", DATA_OVERALL[TASK_LIST_CLASSIFICATION].mean(axis=1, skipna=False)) @@ -204,7 +315,7 @@ with block: gr.Markdown(f""" Massive Text Embedding Benchmark (MTEB) Leaderboard. To submit, refer to the MTEB GitHub repository 🤗 - - **Total Scores**: TODO + - **Total Scores**: >{NUM_SCORES} - **Total Models**: {len(DATA_OVERALL)} - **Total Users**: TODO """) @@ -232,7 +343,7 @@ with block: gr.Markdown(""" **Bitext Mining Leaderboard 🎌** - - **Metric:** Accuracy (accuracy) + - **Metric:** F1 (f1) - **Languages:** 117 """) with gr.Row(): @@ -242,7 +353,7 @@ with block: ) with gr.Row(): data_run = gr.Button("Refresh") - task_bitext_mining = gr.Variable(value="BitextMining") + task_bitext_mining = gr.Variable(value=["BitextMining"]) data_run.click( get_mteb_data, inputs=[task_bitext_mining], @@ -265,7 +376,7 @@ with block: ) with gr.Row(): data_run_classification_en = gr.Button("Refresh") - task_classification_en = gr.Variable(value="Classification") + task_classification_en = gr.Variable(value=["Classification"]) lang_classification_en = gr.Variable(value=["en"]) data_run_classification_en.click( get_mteb_data, @@ -285,12 +396,12 @@ with block: """) with gr.Row(): data_classification = gr.components.Dataframe( - datatype=["markdown"] * 500, # hack when we don't know how many columns + datatype=["markdown"] * 200, # hack when we don't know how many columns type="pandas", ) with gr.Row(): data_run = gr.Button("Refresh") - task_classification = gr.Variable(value="Classification") + task_classification = gr.Variable(value=["Classification"]) data_run.click( get_mteb_data, inputs=[task_classification], @@ -307,13 +418,12 @@ with block: with gr.Row(): data_clustering = gr.components.Dataframe( DATA_CLUSTERING, - datatype="markdown", + datatype=["markdown"] * len(DATA_CLUSTERING.columns) * 2, type="pandas", - col_count=(len(DATA_CLUSTERING.columns), "fixed"), ) with gr.Row(): data_run = gr.Button("Refresh") - task_clustering = gr.Variable(value="Clustering") + task_clustering = gr.Variable(value=["Clustering"]) data_run.click( get_mteb_data, inputs=[task_clustering], @@ -330,13 +440,12 @@ with block: with gr.Row(): data_pair_classification = gr.components.Dataframe( DATA_PAIR_CLASSIFICATION, - datatype="markdown", + datatype=["markdown"] * len(DATA_PAIR_CLASSIFICATION.columns) * 2, type="pandas", - col_count=(len(DATA_PAIR_CLASSIFICATION.columns), "fixed"), ) with gr.Row(): data_run = gr.Button("Refresh") - task_pair_classification = gr.Variable(value="PairClassification") + task_pair_classification = gr.Variable(value=["PairClassification"]) data_run.click( get_mteb_data, inputs=[task_pair_classification], @@ -358,7 +467,7 @@ with block: ) with gr.Row(): data_run = gr.Button("Refresh") - task_retrieval = gr.Variable(value="Retrieval") + task_retrieval = gr.Variable(value=["Retrieval"]) data_run.click( get_mteb_data, inputs=[task_retrieval], outputs=data_retrieval ) @@ -373,13 +482,12 @@ with block: with gr.Row(): data_reranking = gr.components.Dataframe( DATA_RERANKING, - datatype="markdown", + datatype=["markdown"] * len(DATA_RERANKING.columns) * 2, type="pandas", - col_count=(len(DATA_RERANKING.columns), "fixed"), ) with gr.Row(): data_run = gr.Button("Refresh") - task_reranking = gr.Variable(value="Reranking") + task_reranking = gr.Variable(value=["Reranking"]) metric_reranking = gr.Variable(value="map") data_run.click( get_mteb_data, inputs=[task_reranking], outputs=data_reranking @@ -396,15 +504,14 @@ with block: with gr.Row(): data_sts_en = gr.components.Dataframe( DATA_STS_EN, - datatype="markdown", + datatype=["markdown"] * len(DATA_STS_EN.columns) * 2, type="pandas", - col_count=(len(DATA_STS_EN.columns), "fixed"), ) with gr.Row(): - data_run_en = gr.Button("Refresh") - task_sts_en = gr.Variable(value="STS") + data_run_sts_en = gr.Button("Refresh") + task_sts_en = gr.Variable(value=["STS"]) lang_sts_en = gr.Variable(value=["en", "en-en"]) - data_run.click( + data_run_sts_en.click( get_mteb_data, inputs=[task_sts_en, lang_sts_en], outputs=data_sts_en, @@ -424,7 +531,7 @@ with block: ) with gr.Row(): data_run = gr.Button("Refresh") - task_sts = gr.Variable(value="STS") + task_sts = gr.Variable(value=["STS"]) data_run.click(get_mteb_data, inputs=[task_sts], outputs=data_sts) with gr.TabItem("Summarization"): with gr.Row(): @@ -436,14 +543,13 @@ with block: """) with gr.Row(): data_summarization = gr.components.Dataframe( - DATA_SUMMARIZATION, + DATA_SUMMARIZATION * len(DATA_SUMMARIZATION.columns) * 2, datatype="markdown", type="pandas", - col_count=(len(DATA_SUMMARIZATION.columns), "fixed"), ) with gr.Row(): data_run = gr.Button("Refresh") - task_summarization = gr.Variable(value="Summarization") + task_summarization = gr.Variable(value=["Summarization"]) data_run.click( get_mteb_data, inputs=[task_summarization], diff --git a/results/.DS_Store b/results/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ab8e04fee59d1d96e58bad83151806569670083e Binary files /dev/null and b/results/.DS_Store differ diff --git a/results/LASER2/AmazonCounterfactualClassification.json b/results/LASER2/AmazonCounterfactualClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..9c08bc941e8619a754cb1b072f7e60fb18d66385 --- /dev/null +++ b/results/LASER2/AmazonCounterfactualClassification.json @@ -0,0 +1,43 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "de": { + "accuracy": 0.6781584582441113, + "accuracy_stderr": 0.060279798858073545, + "ap": 0.8036240553535807, + "ap_stderr": 0.03476499899077643, + "f1": 0.6628493463277175, + "f1_stderr": 0.05804533112556245, + "main_score": 0.6781584582441113 + }, + "en": { + "accuracy": 0.7683582089552239, + "accuracy_stderr": 0.03737785483516161, + "ap": 0.40076479274021654, + "ap_stderr": 0.05081532982471566, + "f1": 0.70787800776529, + "f1_stderr": 0.03884967003850526, + "main_score": 0.7683582089552239 + }, + "en-ext": { + "accuracy": 0.7616941529235383, + "accuracy_stderr": 0.05609726317155699, + "ap": 0.23620239901382217, + "ap_stderr": 0.055900376704944924, + "f1": 0.6259005944326002, + "f1_stderr": 0.057023255773266515, + "main_score": 0.7616941529235383 + }, + "evaluation_time": 162.63, + "ja": { + "accuracy": 0.6875802997858672, + "accuracy_stderr": 0.057291619728276316, + "ap": 0.18157282477398815, + "ap_stderr": 0.0359805625991896, + "f1": 0.5601658468471795, + "f1_stderr": 0.047780178480722454, + "main_score": 0.6875802997858672 + } + } +} \ No newline at end of file diff --git a/results/LASER2/AmazonPolarityClassification.json b/results/LASER2/AmazonPolarityClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..0da931bc31e009d1ceb903ea06b8fbe8ae6aa514 --- /dev/null +++ b/results/LASER2/AmazonPolarityClassification.json @@ -0,0 +1,14 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "accuracy": 0.6100945, + "accuracy_stderr": 0.032754584770685165, + "ap": 0.5701234744666878, + "ap_stderr": 0.023665879113998516, + "evaluation_time": 169.16, + "f1": 0.6049258458477238, + "f1_stderr": 0.03882387073720782, + "main_score": 0.6100945 + } +} \ No newline at end of file diff --git a/results/LASER2/AmazonReviewsClassification.json b/results/LASER2/AmazonReviewsClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..26d9074b3f7edf80123bfc2f3cf32f05b7340f2a --- /dev/null +++ b/results/LASER2/AmazonReviewsClassification.json @@ -0,0 +1,49 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "de": { + "accuracy": 0.31068, + "accuracy_stderr": 0.030799766232879097, + "f1": 0.2938071341251565, + "f1_stderr": 0.03301311313116498, + "main_score": 0.31068 + }, + "en": { + "accuracy": 0.2871, + "accuracy_stderr": 0.031948740194254914, + "f1": 0.2763831660571802, + "f1_stderr": 0.03216710195455939, + "main_score": 0.2871 + }, + "es": { + "accuracy": 0.32724000000000003, + "accuracy_stderr": 0.015933311018115466, + "f1": 0.310782596824498, + "f1_stderr": 0.022188814588153163, + "main_score": 0.32724000000000003 + }, + "evaluation_time": 295.02, + "fr": { + "accuracy": 0.31116, + "accuracy_stderr": 0.030352304690089024, + "f1": 0.2995469284574527, + "f1_stderr": 0.03141580285250744, + "main_score": 0.31116 + }, + "ja": { + "accuracy": 0.28935999999999995, + "accuracy_stderr": 0.024075680675735834, + "f1": 0.2818735717046802, + "f1_stderr": 0.023753772760779744, + "main_score": 0.28935999999999995 + }, + "zh": { + "accuracy": 0.30892000000000003, + "accuracy_stderr": 0.02032696730946355, + "f1": 0.2990186813313857, + "f1_stderr": 0.021581437215568936, + "main_score": 0.30892000000000003 + } + } +} \ No newline at end of file diff --git a/results/LASER2/ArguAna.json b/results/LASER2/ArguAna.json new file mode 100644 index 0000000000000000000000000000000000000000..76fc68960b8107b8884c07236cd348c1bc8fa32a --- /dev/null +++ b/results/LASER2/ArguAna.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 62.18, + "map_at_1": 0.06117, + "map_at_10": 0.10455, + "map_at_100": 0.11002, + "map_at_1000": 0.11084, + "map_at_3": 0.09187, + "map_at_5": 0.09834, + "ndcg_at_1": 0.06117, + "ndcg_at_10": 0.12856, + "ndcg_at_100": 0.1601, + "ndcg_at_1000": 0.18712, + "ndcg_at_3": 0.10188, + "ndcg_at_5": 0.11358, + "precision_at_1": 0.06117, + "precision_at_10": 0.02055, + "precision_at_100": 0.00365, + "precision_at_1000": 0.00059, + "precision_at_3": 0.04362, + "precision_at_5": 0.03186, + "recall_at_1": 0.06117, + "recall_at_10": 0.20555, + "recall_at_100": 0.36486, + "recall_at_1000": 0.58962, + "recall_at_3": 0.13087, + "recall_at_5": 0.15932 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/ArxivClusteringP2P.json b/results/LASER2/ArxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..f6d63ab6d809a31d12a0dd7ebe891ea064c6bf71 --- /dev/null +++ b/results/LASER2/ArxivClusteringP2P.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 3687.79, + "v_measure": 0.1776823856238192, + "v_measure_std": 0.15680242731305624 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/ArxivClusteringS2S.json b/results/LASER2/ArxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..3f0533ae421a3b137157f934f3e599e4bb7ebd64 --- /dev/null +++ b/results/LASER2/ArxivClusteringS2S.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 398.92, + "v_measure": 0.1239260518556585, + "v_measure_std": 0.16362867463758127 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/AskUbuntuDupQuestions.json b/results/LASER2/AskUbuntuDupQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..c6541b06bd1b7ac1429d622090b16d660568bf8b --- /dev/null +++ b/results/LASER2/AskUbuntuDupQuestions.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 12.12, + "map": 0.4898595820868522, + "mrr": 0.6257276964340676 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/BIOSSES.json b/results/LASER2/BIOSSES.json new file mode 100644 index 0000000000000000000000000000000000000000..ea06658b5d6a975c264ead78542b2f16ea6e7142 --- /dev/null +++ b/results/LASER2/BIOSSES.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.6212936580597783, + "spearman": 0.6200864182463187 + }, + "euclidean": { + "pearson": 0.6471589936856987, + "spearman": 0.68256374544906 + }, + "evaluation_time": 10.33, + "manhattan": { + "pearson": 0.66773509051844, + "spearman": 0.7192212072202181 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/BUCC.json b/results/LASER2/BUCC.json new file mode 100644 index 0000000000000000000000000000000000000000..3bfc2c5786c28c9779135a8b3b74f9dbf75ad6e6 --- /dev/null +++ b/results/LASER2/BUCC.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "de-en": { + "accuracy": 0.9926931106471816, + "f1": 0.9921016005567153, + "precision": 0.9918058455114822, + "recall": 0.9926931106471816 + }, + "evaluation_time": 642.77, + "fr-en": { + "accuracy": 0.985362095531587, + "f1": 0.9838946364370093, + "precision": 0.9831609068897205, + "recall": 0.985362095531587 + }, + "ru-en": { + "accuracy": 0.9774160027710426, + "f1": 0.9762152176423046, + "precision": 0.9756148250779356, + "recall": 0.9774160027710426 + }, + "zh-en": { + "accuracy": 0.9778830963665087, + "f1": 0.9770054414604177, + "precision": 0.9765666140073723, + "recall": 0.9778830963665087 + } + } +} \ No newline at end of file diff --git a/results/LASER2/Banking77Classification.json b/results/LASER2/Banking77Classification.json new file mode 100644 index 0000000000000000000000000000000000000000..df411c02cb331dfd92a78845a8ae9107145447e1 --- /dev/null +++ b/results/LASER2/Banking77Classification.json @@ -0,0 +1,12 @@ +{ + "test": { + "accuracy": 0.5775974025974026, + "accuracy_stderr": 0.01037438007056524, + "evaluation_time": 121.97, + "f1": 0.5693279554001911, + "f1_stderr": 0.010505608700462006, + "main_score": 0.5775974025974026 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/BiorxivClusteringP2P.json b/results/LASER2/BiorxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..9fc529b21b4ee235d5956914faa241d570e6dd7e --- /dev/null +++ b/results/LASER2/BiorxivClusteringP2P.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 768.68, + "v_measure": 0.12399936477309108, + "v_measure_std": 0.005255888066724005 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/BiorxivClusteringS2S.json b/results/LASER2/BiorxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..c7a5486b30f5498af28ac3bb88b7815df2a57242 --- /dev/null +++ b/results/LASER2/BiorxivClusteringS2S.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 85.25, + "v_measure": 0.08827421024926384, + "v_measure_std": 0.005326348835553592 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackAndroidRetrieval.json b/results/LASER2/CQADupstackAndroidRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..ffc1545851adbb064fbdd3f72a6a05f9db747d5c --- /dev/null +++ b/results/LASER2/CQADupstackAndroidRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 91.48, + "map_at_1": 0.0, + "map_at_10": 0.0, + "map_at_100": 0.0, + "map_at_1000": 0.0, + "map_at_3": 0.0, + "map_at_5": 0.0, + "ndcg_at_1": 0.0, + "ndcg_at_10": 0.0, + "ndcg_at_100": 0.0, + "ndcg_at_1000": 0.0, + "ndcg_at_3": 0.0, + "ndcg_at_5": 0.0, + "precision_at_1": 0.0, + "precision_at_10": 0.0, + "precision_at_100": 0.0, + "precision_at_1000": 0.0, + "precision_at_3": 0.0, + "precision_at_5": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_100": 0.0, + "recall_at_1000": 0.0, + "recall_at_3": 0.0, + "recall_at_5": 0.0 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackEnglishRetrieval.json b/results/LASER2/CQADupstackEnglishRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..74badd397d1fdfe31e507149ede09b3cd8f240c2 --- /dev/null +++ b/results/LASER2/CQADupstackEnglishRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 101.88, + "map_at_1": 0.0418, + "map_at_10": 0.06151, + "map_at_100": 0.06587, + "map_at_1000": 0.06677, + "map_at_3": 0.05546, + "map_at_5": 0.05904, + "ndcg_at_1": 0.0535, + "ndcg_at_10": 0.07521, + "ndcg_at_100": 0.09953, + "ndcg_at_1000": 0.12755, + "ndcg_at_3": 0.06367, + "ndcg_at_5": 0.06975, + "precision_at_1": 0.0535, + "precision_at_10": 0.01459, + "precision_at_100": 0.00331, + "precision_at_1000": 0.00076, + "precision_at_3": 0.031, + "precision_at_5": 0.02318, + "recall_at_1": 0.0418, + "recall_at_10": 0.10225, + "recall_at_100": 0.21357, + "recall_at_1000": 0.41667, + "recall_at_3": 0.06958, + "recall_at_5": 0.08551 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackGamingRetrieval.json b/results/LASER2/CQADupstackGamingRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..35c1214c8c771087e8907d3dbbb1f43c61d40dce --- /dev/null +++ b/results/LASER2/CQADupstackGamingRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 147.83, + "map_at_1": 0.05906, + "map_at_10": 0.08391, + "map_at_100": 0.08922, + "map_at_1000": 0.09014, + "map_at_3": 0.07512, + "map_at_5": 0.07958, + "ndcg_at_1": 0.06897, + "ndcg_at_10": 0.1013, + "ndcg_at_100": 0.13153, + "ndcg_at_1000": 0.16093, + "ndcg_at_3": 0.08346, + "ndcg_at_5": 0.09095, + "precision_at_1": 0.06897, + "precision_at_10": 0.01799, + "precision_at_100": 0.00366, + "precision_at_1000": 0.00068, + "precision_at_3": 0.03845, + "precision_at_5": 0.02796, + "recall_at_1": 0.05906, + "recall_at_10": 0.1418, + "recall_at_100": 0.28657, + "recall_at_1000": 0.51596, + "recall_at_3": 0.09328, + "recall_at_5": 0.11166 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackGisRetrieval.json b/results/LASER2/CQADupstackGisRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..a82409925bf65094b038c5b3c80e99c235939d2c --- /dev/null +++ b/results/LASER2/CQADupstackGisRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 324.7, + "map_at_1": 0.01808, + "map_at_10": 0.02773, + "map_at_100": 0.03035, + "map_at_1000": 0.03104, + "map_at_3": 0.02467, + "map_at_5": 0.02658, + "ndcg_at_1": 0.02034, + "ndcg_at_10": 0.03397, + "ndcg_at_100": 0.05043, + "ndcg_at_1000": 0.07373, + "ndcg_at_3": 0.02754, + "ndcg_at_5": 0.03101, + "precision_at_1": 0.02034, + "precision_at_10": 0.00565, + "precision_at_100": 0.0015, + "precision_at_1000": 0.00038, + "precision_at_3": 0.01205, + "precision_at_5": 0.00904, + "recall_at_1": 0.01808, + "recall_at_10": 0.05104, + "recall_at_100": 0.13422, + "recall_at_1000": 0.31965, + "recall_at_3": 0.0339, + "recall_at_5": 0.042 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackMathematicaRetrieval.json b/results/LASER2/CQADupstackMathematicaRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..686337bc84a4e9b16c7b83804cedf9dcca0b46b3 --- /dev/null +++ b/results/LASER2/CQADupstackMathematicaRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 217.6, + "map_at_1": 0.01116, + "map_at_10": 0.01733, + "map_at_100": 0.02, + "map_at_1000": 0.02081, + "map_at_3": 0.01452, + "map_at_5": 0.0161, + "ndcg_at_1": 0.01493, + "ndcg_at_10": 0.02331, + "ndcg_at_100": 0.03985, + "ndcg_at_1000": 0.06932, + "ndcg_at_3": 0.01734, + "ndcg_at_5": 0.02033, + "precision_at_1": 0.01493, + "precision_at_10": 0.00485, + "precision_at_100": 0.00162, + "precision_at_1000": 0.0005, + "precision_at_3": 0.00912, + "precision_at_5": 0.00746, + "recall_at_1": 0.01116, + "recall_at_10": 0.03629, + "recall_at_100": 0.11457, + "recall_at_1000": 0.34389, + "recall_at_3": 0.02008, + "recall_at_5": 0.02748 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackPhysicsRetrieval.json b/results/LASER2/CQADupstackPhysicsRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..64d50ebf4962f8cab2d04f2c9121f4b0ce858e8d --- /dev/null +++ b/results/LASER2/CQADupstackPhysicsRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 209.46, + "map_at_1": 0.03631, + "map_at_10": 0.05496, + "map_at_100": 0.05935, + "map_at_1000": 0.0603, + "map_at_3": 0.05063, + "map_at_5": 0.05293, + "ndcg_at_1": 0.04812, + "ndcg_at_10": 0.06844, + "ndcg_at_100": 0.09373, + "ndcg_at_1000": 0.12299, + "ndcg_at_3": 0.06059, + "ndcg_at_5": 0.06386, + "precision_at_1": 0.04812, + "precision_at_10": 0.01347, + "precision_at_100": 0.00319, + "precision_at_1000": 0.0007, + "precision_at_3": 0.03144, + "precision_at_5": 0.02214, + "recall_at_1": 0.03631, + "recall_at_10": 0.09384, + "recall_at_100": 0.20952, + "recall_at_1000": 0.42465, + "recall_at_3": 0.06884, + "recall_at_5": 0.07864 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackProgrammersRetrieval.json b/results/LASER2/CQADupstackProgrammersRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..1854b7f7c3725beebc218519ee07b0c500807b68 --- /dev/null +++ b/results/LASER2/CQADupstackProgrammersRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 222.46, + "map_at_1": 0.03015, + "map_at_10": 0.04033, + "map_at_100": 0.04335, + "map_at_1000": 0.04404, + "map_at_3": 0.03578, + "map_at_5": 0.03706, + "ndcg_at_1": 0.04224, + "ndcg_at_10": 0.05053, + "ndcg_at_100": 0.06996, + "ndcg_at_1000": 0.09367, + "ndcg_at_3": 0.04125, + "ndcg_at_5": 0.04305, + "precision_at_1": 0.04224, + "precision_at_10": 0.00959, + "precision_at_100": 0.00232, + "precision_at_1000": 0.00052, + "precision_at_3": 0.01941, + "precision_at_5": 0.01324, + "recall_at_1": 0.03015, + "recall_at_10": 0.07077, + "recall_at_100": 0.16386, + "recall_at_1000": 0.34298, + "recall_at_3": 0.04318, + "recall_at_5": 0.0486 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackRetrieval.json b/results/LASER2/CQADupstackRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..8cb9845c826b24a1ec2943d0b2c00bf641f31b9c --- /dev/null +++ b/results/LASER2/CQADupstackRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 485.55, + "map_at_1": 0.02353666666666667, + "map_at_10": 0.033482500000000005, + "map_at_100": 0.0360425, + "map_at_1000": 0.03663416666666667, + "map_at_3": 0.029990833333333335, + "map_at_5": 0.031795833333333336, + "ndcg_at_1": 0.02917833333333333, + "ndcg_at_10": 0.04119, + "ndcg_at_100": 0.056311666666666677, + "ndcg_at_1000": 0.07594583333333334, + "ndcg_at_3": 0.034133333333333335, + "ndcg_at_5": 0.03712166666666667, + "precision_at_1": 0.02917833333333333, + "precision_at_10": 0.007701666666666667, + "precision_at_100": 0.0017991666666666666, + "precision_at_1000": 0.00041666666666666664, + "precision_at_3": 0.016209166666666667, + "precision_at_5": 0.011824999999999999, + "recall_at_1": 0.02353666666666667, + "recall_at_10": 0.0576375, + "recall_at_100": 0.1290925, + "recall_at_1000": 0.279035, + "recall_at_3": 0.03772166666666667, + "recall_at_5": 0.04548166666666667 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackStatsRetrieval.json b/results/LASER2/CQADupstackStatsRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..28144c7428031b57ce18b6b03a0ef53240674778 --- /dev/null +++ b/results/LASER2/CQADupstackStatsRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 344.51, + "map_at_1": 0.03055, + "map_at_10": 0.04272, + "map_at_100": 0.0457, + "map_at_1000": 0.04643, + "map_at_3": 0.03867, + "map_at_5": 0.04104, + "ndcg_at_1": 0.03681, + "ndcg_at_10": 0.05179, + "ndcg_at_100": 0.06878, + "ndcg_at_1000": 0.0944, + "ndcg_at_3": 0.04365, + "ndcg_at_5": 0.04723, + "precision_at_1": 0.03681, + "precision_at_10": 0.00982, + "precision_at_100": 0.00204, + "precision_at_1000": 0.00048, + "precision_at_3": 0.02096, + "precision_at_5": 0.01472, + "recall_at_1": 0.03055, + "recall_at_10": 0.0699, + "recall_at_100": 0.15083, + "recall_at_1000": 0.35566, + "recall_at_3": 0.04728, + "recall_at_5": 0.05725 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackTexRetrieval.json b/results/LASER2/CQADupstackTexRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..fc2a83de96fee985740dae9ac978876678339166 --- /dev/null +++ b/results/LASER2/CQADupstackTexRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 1027.41, + "map_at_1": 0.02132, + "map_at_10": 0.02908, + "map_at_100": 0.03136, + "map_at_1000": 0.03195, + "map_at_3": 0.02581, + "map_at_5": 0.02767, + "ndcg_at_1": 0.02512, + "ndcg_at_10": 0.03569, + "ndcg_at_100": 0.04952, + "ndcg_at_1000": 0.06975, + "ndcg_at_3": 0.02911, + "ndcg_at_5": 0.03214, + "precision_at_1": 0.02512, + "precision_at_10": 0.00664, + "precision_at_100": 0.0017, + "precision_at_1000": 0.00042, + "precision_at_3": 0.01342, + "precision_at_5": 0.00998, + "recall_at_1": 0.02132, + "recall_at_10": 0.05016, + "recall_at_100": 0.11483, + "recall_at_1000": 0.26865, + "recall_at_3": 0.03175, + "recall_at_5": 0.0398 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackUnixRetrieval.json b/results/LASER2/CQADupstackUnixRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..d60e3d0ce94fbf76bf0a55941a5c4575a1ce7f05 --- /dev/null +++ b/results/LASER2/CQADupstackUnixRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 528.37, + "map_at_1": 0.03401, + "map_at_10": 0.04402, + "map_at_100": 0.04709, + "map_at_1000": 0.04787, + "map_at_3": 0.03923, + "map_at_5": 0.04155, + "ndcg_at_1": 0.04011, + "ndcg_at_10": 0.05332, + "ndcg_at_100": 0.07162, + "ndcg_at_1000": 0.09671, + "ndcg_at_3": 0.04299, + "ndcg_at_5": 0.04714, + "precision_at_1": 0.04011, + "precision_at_10": 0.00942, + "precision_at_100": 0.00215, + "precision_at_1000": 0.0005, + "precision_at_3": 0.01866, + "precision_at_5": 0.01418, + "recall_at_1": 0.03401, + "recall_at_10": 0.07361, + "recall_at_100": 0.15881, + "recall_at_1000": 0.34893, + "recall_at_3": 0.04477, + "recall_at_5": 0.05484 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackWebmastersRetrieval.json b/results/LASER2/CQADupstackWebmastersRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..23d4fab67d0ed76fe8b477531c22ef6efb80b566 --- /dev/null +++ b/results/LASER2/CQADupstackWebmastersRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 16.59, + "map_at_1": 0.0, + "map_at_10": 0.0002, + "map_at_100": 0.00022, + "map_at_1000": 0.00026, + "map_at_3": 0.0, + "map_at_5": 0.0, + "ndcg_at_1": 0.0, + "ndcg_at_10": 0.00072, + "ndcg_at_100": 0.00079, + "ndcg_at_1000": 0.0023, + "ndcg_at_3": 0.0, + "ndcg_at_5": 0.0, + "precision_at_1": 0.0, + "precision_at_10": 0.0004, + "precision_at_100": 0.0001, + "precision_at_1000": 6e-05, + "precision_at_3": 0.0, + "precision_at_5": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.00199, + "recall_at_100": 0.00233, + "recall_at_1000": 0.01138, + "recall_at_3": 0.0, + "recall_at_5": 0.0 + } +} \ No newline at end of file diff --git a/results/LASER2/CQADupstackWordpressRetrieval.json b/results/LASER2/CQADupstackWordpressRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..ba61f0e5a4fe03df28774b1604ce40d60770b6e7 --- /dev/null +++ b/results/LASER2/CQADupstackWordpressRetrieval.json @@ -0,0 +1,31 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 485.55, + "map_at_1": 0.0, + "map_at_10": 0.0, + "map_at_100": 0.0, + "map_at_1000": 0.0, + "map_at_3": 0.0, + "map_at_5": 0.0, + "ndcg_at_1": 0.0, + "ndcg_at_10": 0.0, + "ndcg_at_100": 0.0, + "ndcg_at_1000": 0.0, + "ndcg_at_3": 0.0, + "ndcg_at_5": 0.0, + "precision_at_1": 0.0, + "precision_at_10": 0.0, + "precision_at_100": 0.0, + "precision_at_1000": 0.0, + "precision_at_3": 0.0, + "precision_at_5": 0.0, + "recall_at_1": 0.0, + "recall_at_10": 0.0, + "recall_at_100": 0.0, + "recall_at_1000": 0.0, + "recall_at_3": 0.0, + "recall_at_5": 0.0 + } +} \ No newline at end of file diff --git a/results/LASER2/ClimateFEVER.json b/results/LASER2/ClimateFEVER.json new file mode 100644 index 0000000000000000000000000000000000000000..05b792c0c6f1044afa3ae6f19f14ffe1d4f4d095 --- /dev/null +++ b/results/LASER2/ClimateFEVER.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 14616.31, + "map_at_1": 0.0017, + "map_at_10": 0.00242, + "map_at_100": 0.00249, + "map_at_1000": 0.00253, + "map_at_3": 0.00216, + "map_at_5": 0.00236, + "ndcg_at_1": 0.00391, + "ndcg_at_10": 0.0036, + "ndcg_at_100": 0.00436, + "ndcg_at_1000": 0.00594, + "ndcg_at_3": 0.00321, + "ndcg_at_5": 0.0034, + "precision_at_1": 0.00391, + "precision_at_10": 0.00098, + "precision_at_100": 0.00019, + "precision_at_1000": 5e-05, + "precision_at_3": 0.00239, + "precision_at_5": 0.00169, + "recall_at_1": 0.0017, + "recall_at_10": 0.00409, + "recall_at_100": 0.00711, + "recall_at_1000": 0.0171, + "recall_at_3": 0.00279, + "recall_at_5": 0.00366 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/DBPedia.json b/results/LASER2/DBPedia.json new file mode 100644 index 0000000000000000000000000000000000000000..b2be1afddc841565982c558544ba3cd1a4077db2 --- /dev/null +++ b/results/LASER2/DBPedia.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 5917.84, + "map_at_1": 0.00141, + "map_at_10": 0.00304, + "map_at_100": 0.0043, + "map_at_1000": 0.0048, + "map_at_3": 0.00219, + "map_at_5": 0.00252, + "ndcg_at_1": 0.0275, + "ndcg_at_10": 0.01526, + "ndcg_at_100": 0.0154, + "ndcg_at_1000": 0.02439, + "ndcg_at_3": 0.0202, + "ndcg_at_5": 0.01735, + "precision_at_1": 0.04, + "precision_at_10": 0.016, + "precision_at_100": 0.00582, + "precision_at_1000": 0.00199, + "precision_at_3": 0.0275, + "precision_at_5": 0.021, + "recall_at_1": 0.00141, + "recall_at_10": 0.00557, + "recall_at_100": 0.01832, + "recall_at_1000": 0.05513, + "recall_at_3": 0.00273, + "recall_at_5": 0.00369 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/EmotionClassification.json b/results/LASER2/EmotionClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..6c3638c893b1bdd8041f6a30495e64e5472f59af --- /dev/null +++ b/results/LASER2/EmotionClassification.json @@ -0,0 +1,12 @@ +{ + "test": { + "accuracy": 0.24830000000000002, + "accuracy_stderr": 0.03350611884417532, + "evaluation_time": 114.99, + "f1": 0.22363001081759562, + "f1_stderr": 0.028095131986252592, + "main_score": 0.24830000000000002 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/FEVER.json b/results/LASER2/FEVER.json new file mode 100644 index 0000000000000000000000000000000000000000..a9eced58fd3c6b1887da8715556efe81d06a9226 --- /dev/null +++ b/results/LASER2/FEVER.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 16228.4, + "map_at_1": 0.00572, + "map_at_10": 0.00687, + "map_at_100": 0.00718, + "map_at_1000": 0.00724, + "map_at_3": 0.0064, + "map_at_5": 0.00666, + "ndcg_at_1": 0.006, + "ndcg_at_10": 0.00767, + "ndcg_at_100": 0.0093, + "ndcg_at_1000": 0.01125, + "ndcg_at_3": 0.00672, + "ndcg_at_5": 0.00718, + "precision_at_1": 0.006, + "precision_at_10": 0.00104, + "precision_at_100": 0.00019, + "precision_at_1000": 4e-05, + "precision_at_3": 0.00255, + "precision_at_5": 0.00177, + "recall_at_1": 0.00572, + "recall_at_10": 0.01, + "recall_at_100": 0.0178, + "recall_at_1000": 0.03382, + "recall_at_3": 0.0073, + "recall_at_5": 0.0085 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/FiQA2018.json b/results/LASER2/FiQA2018.json new file mode 100644 index 0000000000000000000000000000000000000000..8b921e4cc8ba6f88b309ef70ab2d91e286917905 --- /dev/null +++ b/results/LASER2/FiQA2018.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 233.94, + "map_at_1": 0.00421, + "map_at_10": 0.01026, + "map_at_100": 0.01161, + "map_at_1000": 0.01212, + "map_at_3": 0.00814, + "map_at_5": 0.00903, + "ndcg_at_1": 0.00926, + "ndcg_at_10": 0.01725, + "ndcg_at_100": 0.02736, + "ndcg_at_1000": 0.04678, + "ndcg_at_3": 0.0134, + "ndcg_at_5": 0.01445, + "precision_at_1": 0.00926, + "precision_at_10": 0.00602, + "precision_at_100": 0.00182, + "precision_at_1000": 0.00052, + "precision_at_3": 0.0108, + "precision_at_5": 0.00864, + "recall_at_1": 0.00421, + "recall_at_10": 0.02583, + "recall_at_100": 0.06514, + "recall_at_1000": 0.19262, + "recall_at_3": 0.01344, + "recall_at_5": 0.01738 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/HotpotQA.json b/results/LASER2/HotpotQA.json new file mode 100644 index 0000000000000000000000000000000000000000..c6d0d937d3a30ab500628531b86e879bf00787fd --- /dev/null +++ b/results/LASER2/HotpotQA.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 7848.6, + "map_at_1": 0.03167, + "map_at_10": 0.04056, + "map_at_100": 0.04202, + "map_at_1000": 0.04232, + "map_at_3": 0.03754, + "map_at_5": 0.03914, + "ndcg_at_1": 0.06334, + "ndcg_at_10": 0.05504, + "ndcg_at_100": 0.06469, + "ndcg_at_1000": 0.0756, + "ndcg_at_3": 0.04785, + "ndcg_at_5": 0.0512, + "precision_at_1": 0.06334, + "precision_at_10": 0.01214, + "precision_at_100": 0.00201, + "precision_at_1000": 0.00035, + "precision_at_3": 0.02957, + "precision_at_5": 0.02039, + "recall_at_1": 0.03167, + "recall_at_10": 0.0607, + "recall_at_100": 0.10061, + "recall_at_1000": 0.17508, + "recall_at_3": 0.04436, + "recall_at_5": 0.05098 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/ImdbClassification.json b/results/LASER2/ImdbClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..742e6585142b63d04d8a43d5028623121a03355d --- /dev/null +++ b/results/LASER2/ImdbClassification.json @@ -0,0 +1,14 @@ +{ + "test": { + "accuracy": 0.57584, + "accuracy_stderr": 0.027472639480035414, + "ap": 0.5455790619149001, + "ap_stderr": 0.0190473646119646, + "evaluation_time": 2045.42, + "f1": 0.5721092756521572, + "f1_stderr": 0.02840547286714844, + "main_score": 0.57584 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/MSMARCO.json b/results/LASER2/MSMARCO.json new file mode 100644 index 0000000000000000000000000000000000000000..06a8ca4743d3e2ae3a9fab57a7b3201082a86915 --- /dev/null +++ b/results/LASER2/MSMARCO.json @@ -0,0 +1,58 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "dev": { + "evaluation_time": 14000.54, + "map_at_1": 0.0053, + "map_at_10": 0.00883, + "map_at_100": 0.01009, + "map_at_1000": 0.01042, + "map_at_3": 0.00775, + "map_at_5": 0.00829, + "ndcg_at_1": 0.00544, + "ndcg_at_10": 0.01087, + "ndcg_at_100": 0.01866, + "ndcg_at_1000": 0.0308, + "ndcg_at_3": 0.00854, + "ndcg_at_5": 0.00953, + "precision_at_1": 0.00544, + "precision_at_10": 0.00176, + "precision_at_100": 0.00059, + "precision_at_1000": 0.00017, + "precision_at_3": 0.00368, + "precision_at_5": 0.00269, + "recall_at_1": 0.0053, + "recall_at_10": 0.01734, + "recall_at_100": 0.05722, + "recall_at_1000": 0.15848, + "recall_at_3": 0.01074, + "recall_at_5": 0.01318 + }, + "test": { + "evaluation_time": 11497.8, + "map_at_1": 0.00015, + "map_at_10": 0.00195, + "map_at_100": 0.00444, + "map_at_1000": 0.00628, + "map_at_3": 0.00091, + "map_at_5": 0.00131, + "ndcg_at_1": 0.03101, + "ndcg_at_10": 0.03593, + "ndcg_at_100": 0.02811, + "ndcg_at_1000": 0.04373, + "ndcg_at_3": 0.03705, + "ndcg_at_5": 0.0394, + "precision_at_1": 0.04651, + "precision_at_10": 0.05116, + "precision_at_100": 0.02256, + "precision_at_1000": 0.00751, + "precision_at_3": 0.06202, + "precision_at_5": 0.06047, + "recall_at_1": 0.00015, + "recall_at_10": 0.00374, + "recall_at_100": 0.01999, + "recall_at_1000": 0.0669, + "recall_at_3": 0.00186, + "recall_at_5": 0.00244 + } +} \ No newline at end of file diff --git a/results/LASER2/MTOPDomainClassification.json b/results/LASER2/MTOPDomainClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..e8c8970042b55888c50729715e9c86a73a954b96 --- /dev/null +++ b/results/LASER2/MTOPDomainClassification.json @@ -0,0 +1,49 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "de": { + "accuracy": 0.7407720484643561, + "accuracy_stderr": 0.02097470751351384, + "f1": 0.7246747959017267, + "f1_stderr": 0.021409255315184686, + "main_score": 0.7407720484643561 + }, + "en": { + "accuracy": 0.7535795713634291, + "accuracy_stderr": 0.01932929756272018, + "f1": 0.7471023646717295, + "f1_stderr": 0.020603379350450358, + "main_score": 0.7535795713634291 + }, + "es": { + "accuracy": 0.7346897931954637, + "accuracy_stderr": 0.029275171505061988, + "f1": 0.7201695057514836, + "f1_stderr": 0.02619166045098598, + "main_score": 0.7346897931954637 + }, + "evaluation_time": 238.88, + "fr": { + "accuracy": 0.7226119636705294, + "accuracy_stderr": 0.017579729530633047, + "f1": 0.7138676065809201, + "f1_stderr": 0.016450702159748233, + "main_score": 0.7226119636705294 + }, + "hi": { + "accuracy": 0.7295087845105772, + "accuracy_stderr": 0.022688800454798208, + "f1": 0.7163844311220117, + "f1_stderr": 0.019843811193564726, + "main_score": 0.7295087845105772 + }, + "th": { + "accuracy": 0.7267631103074141, + "accuracy_stderr": 0.014703529100373161, + "f1": 0.7222556760062775, + "f1_stderr": 0.012793360924583051, + "main_score": 0.7267631103074141 + } + } +} \ No newline at end of file diff --git a/results/LASER2/MTOPIntentClassification.json b/results/LASER2/MTOPIntentClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..e59a5b8f6ad3d0e72b50a6b141dae16717b77039 --- /dev/null +++ b/results/LASER2/MTOPIntentClassification.json @@ -0,0 +1,49 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "de": { + "accuracy": 0.5162299239222317, + "accuracy_stderr": 0.01832258118122447, + "f1": 0.3256925158951781, + "f1_stderr": 0.006031602034364266, + "main_score": 0.5162299239222317 + }, + "en": { + "accuracy": 0.49473324213406294, + "accuracy_stderr": 0.026360858836492213, + "f1": 0.3300910657296617, + "f1_stderr": 0.012151253776376927, + "main_score": 0.49473324213406294 + }, + "es": { + "accuracy": 0.5274516344229486, + "accuracy_stderr": 0.01871901414866834, + "f1": 0.33394567108321266, + "f1_stderr": 0.012228976278261429, + "main_score": 0.5274516344229486 + }, + "evaluation_time": 288.21, + "fr": { + "accuracy": 0.5011901033510806, + "accuracy_stderr": 0.02349600059161113, + "f1": 0.33814182942295407, + "f1_stderr": 0.008555560889430382, + "main_score": 0.5011901033510806 + }, + "hi": { + "accuracy": 0.45546790964503403, + "accuracy_stderr": 0.019083595788646334, + "f1": 0.27716607594942344, + "f1_stderr": 0.008465873041149053, + "main_score": 0.45546790964503403 + }, + "th": { + "accuracy": 0.5006871609403254, + "accuracy_stderr": 0.032474576694908, + "f1": 0.3475254801351875, + "f1_stderr": 0.01064830577932404, + "main_score": 0.5006871609403254 + } + } +} \ No newline at end of file diff --git a/results/LASER2/MassiveIntentClassification.json b/results/LASER2/MassiveIntentClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..35e862232a939c9454407b2c876fe0b70bb3d6f9 --- /dev/null +++ b/results/LASER2/MassiveIntentClassification.json @@ -0,0 +1,364 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "af": { + "accuracy": 0.38012777404169473, + "accuracy_stderr": 0.018054485755774126, + "f1": 0.36372620791961896, + "f1_stderr": 0.011605337850752963, + "main_score": 0.38012777404169473 + }, + "am": { + "accuracy": 0.1269670477471419, + "accuracy_stderr": 0.024740686380275122, + "f1": 0.10125430985923822, + "f1_stderr": 0.014610275424102794, + "main_score": 0.1269670477471419 + }, + "ar": { + "accuracy": 0.3716207128446537, + "accuracy_stderr": 0.015242658586425109, + "f1": 0.3513819183071391, + "f1_stderr": 0.00952968827923218, + "main_score": 0.3716207128446537 + }, + "az": { + "accuracy": 0.19979825151311364, + "accuracy_stderr": 0.023512716506208713, + "f1": 0.20755182437536165, + "f1_stderr": 0.020871688941913907, + "main_score": 0.19979825151311364 + }, + "bn": { + "accuracy": 0.4251176866173504, + "accuracy_stderr": 0.015417634386490787, + "f1": 0.41808038563926464, + "f1_stderr": 0.015199386985110714, + "main_score": 0.4251176866173504 + }, + "cy": { + "accuracy": 0.17326832548755883, + "accuracy_stderr": 0.012898104386975457, + "f1": 0.18929774881076256, + "f1_stderr": 0.011323358864015197, + "main_score": 0.17326832548755883 + }, + "da": { + "accuracy": 0.4560860793544048, + "accuracy_stderr": 0.01951406090649724, + "f1": 0.4499828344382091, + "f1_stderr": 0.012363163669498014, + "main_score": 0.4560860793544048 + }, + "de": { + "accuracy": 0.44788164088769333, + "accuracy_stderr": 0.014665941706559757, + "f1": 0.4331277340788812, + "f1_stderr": 0.01522229039527465, + "main_score": 0.44788164088769333 + }, + "el": { + "accuracy": 0.4670813718897109, + "accuracy_stderr": 0.016337233490102906, + "f1": 0.4610946954649364, + "f1_stderr": 0.0176594232674652, + "main_score": 0.4670813718897109 + }, + "en": { + "accuracy": 0.4790854068594485, + "accuracy_stderr": 0.01717352517420073, + "f1": 0.464939146287881, + "f1_stderr": 0.012710556685691056, + "main_score": 0.4790854068594485 + }, + "es": { + "accuracy": 0.45437121721587087, + "accuracy_stderr": 0.014659040367775445, + "f1": 0.46286614708385815, + "f1_stderr": 0.01400822366938995, + "main_score": 0.45437121721587087 + }, + "evaluation_time": 2187.96, + "fa": { + "accuracy": 0.45013449899125757, + "accuracy_stderr": 0.018101267052589458, + "f1": 0.4440276709633043, + "f1_stderr": 0.02139785887315828, + "main_score": 0.45013449899125757 + }, + "fi": { + "accuracy": 0.45938130464021515, + "accuracy_stderr": 0.019983852698166247, + "f1": 0.4467651558171412, + "f1_stderr": 0.013228992239689133, + "main_score": 0.45938130464021515 + }, + "fr": { + "accuracy": 0.4613315400134499, + "accuracy_stderr": 0.019672058058454584, + "f1": 0.46149191584616656, + "f1_stderr": 0.01462338783534165, + "main_score": 0.4613315400134499 + }, + "he": { + "accuracy": 0.42545393409549426, + "accuracy_stderr": 0.015641146997013137, + "f1": 0.406277912154974, + "f1_stderr": 0.015808885221407522, + "main_score": 0.42545393409549426 + }, + "hi": { + "accuracy": 0.40201748486886346, + "accuracy_stderr": 0.014870504329345259, + "f1": 0.39955511381980663, + "f1_stderr": 0.012540162228863491, + "main_score": 0.40201748486886346 + }, + "hu": { + "accuracy": 0.4277404169468729, + "accuracy_stderr": 0.012916849604535326, + "f1": 0.41428552778550787, + "f1_stderr": 0.01796504857153477, + "main_score": 0.4277404169468729 + }, + "hy": { + "accuracy": 0.28073301950235374, + "accuracy_stderr": 0.022799686666722056, + "f1": 0.27060617914010276, + "f1_stderr": 0.011002544912943118, + "main_score": 0.28073301950235374 + }, + "id": { + "accuracy": 0.4581035642232683, + "accuracy_stderr": 0.01719550009501075, + "f1": 0.44698433663656323, + "f1_stderr": 0.016362307533275874, + "main_score": 0.4581035642232683 + }, + "is": { + "accuracy": 0.39862138533960995, + "accuracy_stderr": 0.016819937773966665, + "f1": 0.3920844117598715, + "f1_stderr": 0.015188173368411415, + "main_score": 0.39862138533960995 + }, + "it": { + "accuracy": 0.4825487558843308, + "accuracy_stderr": 0.017093447378236502, + "f1": 0.4800020656000982, + "f1_stderr": 0.014433139843477095, + "main_score": 0.4825487558843308 + }, + "ja": { + "accuracy": 0.45299260255548085, + "accuracy_stderr": 0.020810866845063326, + "f1": 0.45499154063652886, + "f1_stderr": 0.015152638500548954, + "main_score": 0.45299260255548085 + }, + "jv": { + "accuracy": 0.24303967720242098, + "accuracy_stderr": 0.018405242193104722, + "f1": 0.24206441275862223, + "f1_stderr": 0.01230622384839183, + "main_score": 0.24303967720242098 + }, + "ka": { + "accuracy": 0.22700067249495626, + "accuracy_stderr": 0.011315564480105716, + "f1": 0.2262620454603394, + "f1_stderr": 0.013641428742103908, + "main_score": 0.22700067249495626 + }, + "km": { + "accuracy": 0.22481506388702083, + "accuracy_stderr": 0.017626681644496683, + "f1": 0.20273821769692973, + "f1_stderr": 0.010423853100542939, + "main_score": 0.22481506388702083 + }, + "kn": { + "accuracy": 0.043207800941492944, + "accuracy_stderr": 0.011439581992919204, + "f1": 0.019591698863663035, + "f1_stderr": 0.004395348203545399, + "main_score": 0.043207800941492944 + }, + "ko": { + "accuracy": 0.4426025554808339, + "accuracy_stderr": 0.024004890918808426, + "f1": 0.44147618521379844, + "f1_stderr": 0.01371530289091307, + "main_score": 0.4426025554808339 + }, + "lv": { + "accuracy": 0.3975453934095495, + "accuracy_stderr": 0.02350514180018239, + "f1": 0.4061787392665471, + "f1_stderr": 0.02079966697693215, + "main_score": 0.3975453934095495 + }, + "ml": { + "accuracy": 0.4133490248823134, + "accuracy_stderr": 0.022600758621339218, + "f1": 0.40286261835633236, + "f1_stderr": 0.019466174082752453, + "main_score": 0.4133490248823134 + }, + "mn": { + "accuracy": 0.16200403496973773, + "accuracy_stderr": 0.0100095627651343, + "f1": 0.16450176344768086, + "f1_stderr": 0.01474426560593057, + "main_score": 0.16200403496973773 + }, + "ms": { + "accuracy": 0.4322797579018157, + "accuracy_stderr": 0.01759072503596667, + "f1": 0.4248684950682716, + "f1_stderr": 0.016635102291421434, + "main_score": 0.4322797579018157 + }, + "my": { + "accuracy": 0.25373234700739744, + "accuracy_stderr": 0.014838536825778608, + "f1": 0.2222952356057096, + "f1_stderr": 0.013501549124465934, + "main_score": 0.25373234700739744 + }, + "nb": { + "accuracy": 0.3773705447209146, + "accuracy_stderr": 0.021859215543374035, + "f1": 0.3822420276063557, + "f1_stderr": 0.01781935165480971, + "main_score": 0.3773705447209146 + }, + "nl": { + "accuracy": 0.45000000000000007, + "accuracy_stderr": 0.016798952275448243, + "f1": 0.4390949593879808, + "f1_stderr": 0.013457897429685057, + "main_score": 0.45000000000000007 + }, + "pl": { + "accuracy": 0.4499327505043712, + "accuracy_stderr": 0.018087145352772453, + "f1": 0.42978932594492064, + "f1_stderr": 0.020420111856607633, + "main_score": 0.4499327505043712 + }, + "pt": { + "accuracy": 0.4855413584398117, + "accuracy_stderr": 0.01347928867036143, + "f1": 0.48571088423157766, + "f1_stderr": 0.013222183663454462, + "main_score": 0.4855413584398117 + }, + "ro": { + "accuracy": 0.44300605245460656, + "accuracy_stderr": 0.021637858328300265, + "f1": 0.44740069972880264, + "f1_stderr": 0.01577190962835917, + "main_score": 0.44300605245460656 + }, + "ru": { + "accuracy": 0.44290517821116343, + "accuracy_stderr": 0.021596827649085326, + "f1": 0.43747519563352766, + "f1_stderr": 0.019105088547632505, + "main_score": 0.44290517821116343 + }, + "sl": { + "accuracy": 0.44717552118359116, + "accuracy_stderr": 0.01726980272383147, + "f1": 0.4583434006703727, + "f1_stderr": 0.016199198460333516, + "main_score": 0.44717552118359116 + }, + "sq": { + "accuracy": 0.4612306657700067, + "accuracy_stderr": 0.01641084055632617, + "f1": 0.4549674955169706, + "f1_stderr": 0.013851645495939875, + "main_score": 0.4612306657700067 + }, + "sv": { + "accuracy": 0.4594821788836584, + "accuracy_stderr": 0.020832831141935834, + "f1": 0.44306077100248525, + "f1_stderr": 0.019118675204039762, + "main_score": 0.4594821788836584 + }, + "sw": { + "accuracy": 0.31893073301950237, + "accuracy_stderr": 0.011235144820777232, + "f1": 0.31391383206208073, + "f1_stderr": 0.014373830497544215, + "main_score": 0.31893073301950237 + }, + "ta": { + "accuracy": 0.2963349024882313, + "accuracy_stderr": 0.012793365954792035, + "f1": 0.2888382113446147, + "f1_stderr": 0.01179745104077985, + "main_score": 0.2963349024882313 + }, + "te": { + "accuracy": 0.36028917283120376, + "accuracy_stderr": 0.018898933014207392, + "f1": 0.3438728561319695, + "f1_stderr": 0.015037693640197282, + "main_score": 0.36028917283120376 + }, + "th": { + "accuracy": 0.4338601210490921, + "accuracy_stderr": 0.01838326805184409, + "f1": 0.44123009965336957, + "f1_stderr": 0.014967999591595584, + "main_score": 0.4338601210490921 + }, + "tl": { + "accuracy": 0.2973436449226631, + "accuracy_stderr": 0.01833399973821919, + "f1": 0.28090429460991373, + "f1_stderr": 0.012358692677216883, + "main_score": 0.2973436449226631 + }, + "tr": { + "accuracy": 0.43930733019502355, + "accuracy_stderr": 0.02477039571658176, + "f1": 0.4265561991600836, + "f1_stderr": 0.022060686946343926, + "main_score": 0.43930733019502355 + }, + "ur": { + "accuracy": 0.26109616677874925, + "accuracy_stderr": 0.011974612577756474, + "f1": 0.27607059745773804, + "f1_stderr": 0.011634207629783868, + "main_score": 0.26109616677874925 + }, + "vi": { + "accuracy": 0.44327505043712173, + "accuracy_stderr": 0.022061037910983305, + "f1": 0.43953676685827936, + "f1_stderr": 0.01965182342188036, + "main_score": 0.44327505043712173 + }, + "zh-CN": { + "accuracy": 0.40618695359784807, + "accuracy_stderr": 0.01878416975952488, + "f1": 0.40304322644486057, + "f1_stderr": 0.016358332856736345, + "main_score": 0.40618695359784807 + }, + "zh-TW": { + "accuracy": 0.3292871553463349, + "accuracy_stderr": 0.012964553395663082, + "f1": 0.3310118176304551, + "f1_stderr": 0.00939726932836698, + "main_score": 0.3292871553463349 + } + } +} \ No newline at end of file diff --git a/results/LASER2/MassiveScenarioClassification.json b/results/LASER2/MassiveScenarioClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..2bcebfee52c14eedc9fe14c2d063bc5feb48e1fe --- /dev/null +++ b/results/LASER2/MassiveScenarioClassification.json @@ -0,0 +1,364 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "af": { + "accuracy": 0.4710154673839947, + "accuracy_stderr": 0.018564271686437515, + "f1": 0.4451973179524046, + "f1_stderr": 0.016582236744477792, + "main_score": 0.4710154673839947 + }, + "am": { + "accuracy": 0.17696704774714192, + "accuracy_stderr": 0.03488947970284516, + "f1": 0.15821694371719605, + "f1_stderr": 0.018041297388910648, + "main_score": 0.17696704774714192 + }, + "ar": { + "accuracy": 0.4520847343644922, + "accuracy_stderr": 0.02052556073259087, + "f1": 0.4493556808838758, + "f1_stderr": 0.02219131959628148, + "main_score": 0.4520847343644922 + }, + "az": { + "accuracy": 0.2821116341627438, + "accuracy_stderr": 0.03302593905078847, + "f1": 0.25909697848350605, + "f1_stderr": 0.025892139767993743, + "main_score": 0.2821116341627438 + }, + "bn": { + "accuracy": 0.5052454606590451, + "accuracy_stderr": 0.025871484930688898, + "f1": 0.4856524051409231, + "f1_stderr": 0.02999372933773852, + "main_score": 0.5052454606590451 + }, + "cy": { + "accuracy": 0.2257565568258238, + "accuracy_stderr": 0.029955555426665198, + "f1": 0.22253995596411508, + "f1_stderr": 0.024577736879299292, + "main_score": 0.2257565568258238 + }, + "da": { + "accuracy": 0.5486550100874243, + "accuracy_stderr": 0.024402881290646793, + "f1": 0.5266171141921359, + "f1_stderr": 0.022103636920251905, + "main_score": 0.5486550100874243 + }, + "de": { + "accuracy": 0.543375924680565, + "accuracy_stderr": 0.021944881957252572, + "f1": 0.5252613318654367, + "f1_stderr": 0.026364483689257163, + "main_score": 0.543375924680565 + }, + "el": { + "accuracy": 0.5547074646940148, + "accuracy_stderr": 0.025938196719597618, + "f1": 0.5349766031105144, + "f1_stderr": 0.029002837280894762, + "main_score": 0.5547074646940148 + }, + "en": { + "accuracy": 0.5592131809011432, + "accuracy_stderr": 0.02460975514543639, + "f1": 0.5360164729184802, + "f1_stderr": 0.023764425252363358, + "main_score": 0.5592131809011432 + }, + "es": { + "accuracy": 0.5277404169468729, + "accuracy_stderr": 0.01930086885231061, + "f1": 0.5147488045351095, + "f1_stderr": 0.02102482512919075, + "main_score": 0.5277404169468729 + }, + "evaluation_time": 2170.8, + "fa": { + "accuracy": 0.5250168123739072, + "accuracy_stderr": 0.02656193333119369, + "f1": 0.5107120593131073, + "f1_stderr": 0.027581382894975626, + "main_score": 0.5250168123739072 + }, + "fi": { + "accuracy": 0.5263281775386685, + "accuracy_stderr": 0.028049993574005946, + "f1": 0.5028105658943992, + "f1_stderr": 0.029331037664081416, + "main_score": 0.5263281775386685 + }, + "fr": { + "accuracy": 0.5431741761936786, + "accuracy_stderr": 0.02330862085752492, + "f1": 0.5225553974023132, + "f1_stderr": 0.025371329635383216, + "main_score": 0.5431741761936786 + }, + "he": { + "accuracy": 0.5241425689307331, + "accuracy_stderr": 0.023080787167122134, + "f1": 0.5053900298960137, + "f1_stderr": 0.023963342674475203, + "main_score": 0.5241425689307331 + }, + "hi": { + "accuracy": 0.47370544720914587, + "accuracy_stderr": 0.02723430216840629, + "f1": 0.4497838830218261, + "f1_stderr": 0.029160940440385055, + "main_score": 0.47370544720914587 + }, + "hu": { + "accuracy": 0.5343308675184936, + "accuracy_stderr": 0.02571158946620075, + "f1": 0.5127958383061038, + "f1_stderr": 0.02800743447651377, + "main_score": 0.5343308675184936 + }, + "hy": { + "accuracy": 0.33574310692669806, + "accuracy_stderr": 0.0279984332671053, + "f1": 0.31782884203375084, + "f1_stderr": 0.024137129936083107, + "main_score": 0.33574310692669806 + }, + "id": { + "accuracy": 0.543813046402152, + "accuracy_stderr": 0.02173056397773423, + "f1": 0.5300232571973252, + "f1_stderr": 0.02368021779116069, + "main_score": 0.543813046402152 + }, + "is": { + "accuracy": 0.49778076664425014, + "accuracy_stderr": 0.02425237237474775, + "f1": 0.4706894365473266, + "f1_stderr": 0.026960565416528054, + "main_score": 0.49778076664425014 + }, + "it": { + "accuracy": 0.5483523873570949, + "accuracy_stderr": 0.0236096496608969, + "f1": 0.5335311466130319, + "f1_stderr": 0.02545386322306336, + "main_score": 0.5483523873570949 + }, + "ja": { + "accuracy": 0.541223940820444, + "accuracy_stderr": 0.02132943677295098, + "f1": 0.5297398634789287, + "f1_stderr": 0.02456627872645516, + "main_score": 0.541223940820444 + }, + "jv": { + "accuracy": 0.32713517148621385, + "accuracy_stderr": 0.035614449716496376, + "f1": 0.31631803435296824, + "f1_stderr": 0.02907835125753927, + "main_score": 0.32713517148621385 + }, + "ka": { + "accuracy": 0.2691997310020175, + "accuracy_stderr": 0.014025349472741056, + "f1": 0.2553767051487454, + "f1_stderr": 0.01162136511437043, + "main_score": 0.2691997310020175 + }, + "km": { + "accuracy": 0.2723268325487559, + "accuracy_stderr": 0.02335380952684497, + "f1": 0.2515485444512877, + "f1_stderr": 0.017257971891773076, + "main_score": 0.2723268325487559 + }, + "kn": { + "accuracy": 0.10063887020847344, + "accuracy_stderr": 0.024999293350221934, + "f1": 0.064525855332009, + "f1_stderr": 0.013368181011921797, + "main_score": 0.10063887020847344 + }, + "ko": { + "accuracy": 0.5200739744451917, + "accuracy_stderr": 0.028003601644626674, + "f1": 0.508622045242862, + "f1_stderr": 0.02866048570926957, + "main_score": 0.5200739744451917 + }, + "lv": { + "accuracy": 0.44821788836583726, + "accuracy_stderr": 0.03405873263075454, + "f1": 0.44404598642640175, + "f1_stderr": 0.034820002597344316, + "main_score": 0.44821788836583726 + }, + "ml": { + "accuracy": 0.49098856758574316, + "accuracy_stderr": 0.036953827315555375, + "f1": 0.477330044105713, + "f1_stderr": 0.04031567283455406, + "main_score": 0.49098856758574316 + }, + "mn": { + "accuracy": 0.2150975117686617, + "accuracy_stderr": 0.031789545064276226, + "f1": 0.19956650806636905, + "f1_stderr": 0.02753725989804031, + "main_score": 0.2150975117686617 + }, + "ms": { + "accuracy": 0.5360121049092131, + "accuracy_stderr": 0.024483088411342932, + "f1": 0.5186987112385978, + "f1_stderr": 0.02535803268828841, + "main_score": 0.5360121049092131 + }, + "my": { + "accuracy": 0.29717552118359114, + "accuracy_stderr": 0.028042233316740755, + "f1": 0.2811205919315657, + "f1_stderr": 0.027129866000958187, + "main_score": 0.29717552118359114 + }, + "nb": { + "accuracy": 0.439004707464694, + "accuracy_stderr": 0.027998816890400836, + "f1": 0.4292074186822793, + "f1_stderr": 0.026243622116762927, + "main_score": 0.439004707464694 + }, + "nl": { + "accuracy": 0.5333221250840617, + "accuracy_stderr": 0.025110033232975306, + "f1": 0.5077836780602125, + "f1_stderr": 0.027053518325785973, + "main_score": 0.5333221250840617 + }, + "pl": { + "accuracy": 0.5291526563550774, + "accuracy_stderr": 0.02756493168576702, + "f1": 0.5199897798231456, + "f1_stderr": 0.029788904479210038, + "main_score": 0.5291526563550774 + }, + "pt": { + "accuracy": 0.5341291190316073, + "accuracy_stderr": 0.0218184197703392, + "f1": 0.5199879514297052, + "f1_stderr": 0.02121614576868715, + "main_score": 0.5341291190316073 + }, + "ro": { + "accuracy": 0.5047747141896436, + "accuracy_stderr": 0.02555648952102798, + "f1": 0.4877286315335815, + "f1_stderr": 0.02639141604347184, + "main_score": 0.5047747141896436 + }, + "ru": { + "accuracy": 0.5184263618022864, + "accuracy_stderr": 0.030568502165383708, + "f1": 0.5141368540903201, + "f1_stderr": 0.033485640371287616, + "main_score": 0.5184263618022864 + }, + "sl": { + "accuracy": 0.5128782784129118, + "accuracy_stderr": 0.02677596327757706, + "f1": 0.5069111807663103, + "f1_stderr": 0.028984297990527246, + "main_score": 0.5128782784129118 + }, + "sq": { + "accuracy": 0.556523201075992, + "accuracy_stderr": 0.022907536560761797, + "f1": 0.5280391253265858, + "f1_stderr": 0.02766946734240689, + "main_score": 0.556523201075992 + }, + "sv": { + "accuracy": 0.5464357767316745, + "accuracy_stderr": 0.02703916904648807, + "f1": 0.5289646853571585, + "f1_stderr": 0.027215870565783958, + "main_score": 0.5464357767316745 + }, + "sw": { + "accuracy": 0.4204438466711499, + "accuracy_stderr": 0.02543427134780407, + "f1": 0.39698226248309787, + "f1_stderr": 0.02495774581205971, + "main_score": 0.4204438466711499 + }, + "ta": { + "accuracy": 0.3672158708809684, + "accuracy_stderr": 0.024620778778865062, + "f1": 0.35611628288260666, + "f1_stderr": 0.02508451846783393, + "main_score": 0.3672158708809684 + }, + "te": { + "accuracy": 0.4208137188971082, + "accuracy_stderr": 0.02723531925936748, + "f1": 0.39746768684058836, + "f1_stderr": 0.029476453506871073, + "main_score": 0.4208137188971082 + }, + "th": { + "accuracy": 0.521486213853396, + "accuracy_stderr": 0.02513073701134646, + "f1": 0.5038882871449852, + "f1_stderr": 0.025901347057158762, + "main_score": 0.521486213853396 + }, + "tl": { + "accuracy": 0.3734028244788164, + "accuracy_stderr": 0.036431060397290715, + "f1": 0.3428567103143635, + "f1_stderr": 0.03562323164275494, + "main_score": 0.3734028244788164 + }, + "tr": { + "accuracy": 0.5255548083389374, + "accuracy_stderr": 0.030376754244448495, + "f1": 0.5139704021609626, + "f1_stderr": 0.02797264681517368, + "main_score": 0.5255548083389374 + }, + "ur": { + "accuracy": 0.32599193006052457, + "accuracy_stderr": 0.03133503581066815, + "f1": 0.32576383434482714, + "f1_stderr": 0.02553427963140292, + "main_score": 0.32599193006052457 + }, + "vi": { + "accuracy": 0.5097175521183592, + "accuracy_stderr": 0.01667763764284436, + "f1": 0.49826499437648436, + "f1_stderr": 0.019814421278178205, + "main_score": 0.5097175521183592 + }, + "zh-CN": { + "accuracy": 0.5021856086079354, + "accuracy_stderr": 0.01872946746711234, + "f1": 0.49069359630568155, + "f1_stderr": 0.022400099688479058, + "main_score": 0.5021856086079354 + }, + "zh-TW": { + "accuracy": 0.4232347007397445, + "accuracy_stderr": 0.01456541641125866, + "f1": 0.41073538173900237, + "f1_stderr": 0.015525857226939374, + "main_score": 0.4232347007397445 + } + } +} \ No newline at end of file diff --git a/results/LASER2/MedrxivClusteringP2P.json b/results/LASER2/MedrxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..1c42067ca3e16df014e74967da894e17497a9205 --- /dev/null +++ b/results/LASER2/MedrxivClusteringP2P.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 549.14, + "v_measure": 0.17908142247465778, + "v_measure_std": 0.01580960913642727 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/MedrxivClusteringS2S.json b/results/LASER2/MedrxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..9e14569a41bb3293c6f0de3061b43c4b8029ee7a --- /dev/null +++ b/results/LASER2/MedrxivClusteringS2S.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 73.93, + "v_measure": 0.1662952889451872, + "v_measure_std": 0.018792494328507908 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/MindSmallReranking.json b/results/LASER2/MindSmallReranking.json new file mode 100644 index 0000000000000000000000000000000000000000..a0423d7d84f328eb9d873ddbfa6900a36e0e25b3 --- /dev/null +++ b/results/LASER2/MindSmallReranking.json @@ -0,0 +1,9 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 103.4, + "map": 0.24788913161151171, + "mrr": 0.25454137908041313 + } +} \ No newline at end of file diff --git a/results/LASER2/NFCorpus.json b/results/LASER2/NFCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..5cca7e0631f5883db9a1b95916c1e08a8d666ea0 --- /dev/null +++ b/results/LASER2/NFCorpus.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 44.54, + "map_at_1": 0.00051, + "map_at_10": 0.00303, + "map_at_100": 0.00588, + "map_at_1000": 0.01164, + "map_at_3": 0.00166, + "map_at_5": 0.00244, + "ndcg_at_1": 0.01548, + "ndcg_at_10": 0.02439, + "ndcg_at_100": 0.03883, + "ndcg_at_1000": 0.13695, + "ndcg_at_3": 0.0188, + "ndcg_at_5": 0.02153, + "precision_at_1": 0.01858, + "precision_at_10": 0.02508, + "precision_at_100": 0.01851, + "precision_at_1000": 0.01354, + "precision_at_3": 0.02167, + "precision_at_5": 0.02415, + "recall_at_1": 0.00051, + "recall_at_10": 0.01005, + "recall_at_100": 0.06311, + "recall_at_1000": 0.40172, + "recall_at_3": 0.00342, + "recall_at_5": 0.00638 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/NQ.json b/results/LASER2/NQ.json new file mode 100644 index 0000000000000000000000000000000000000000..1e4874df076f6ef63a6cc8c47997218473f86b90 --- /dev/null +++ b/results/LASER2/NQ.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 6367.75, + "map_at_1": 0.00261, + "map_at_10": 0.00499, + "map_at_100": 0.00563, + "map_at_1000": 0.0058, + "map_at_3": 0.0044, + "map_at_5": 0.00473, + "ndcg_at_1": 0.00319, + "ndcg_at_10": 0.00642, + "ndcg_at_100": 0.01013, + "ndcg_at_1000": 0.01617, + "ndcg_at_3": 0.00514, + "ndcg_at_5": 0.00576, + "precision_at_1": 0.00319, + "precision_at_10": 0.00116, + "precision_at_100": 0.00033, + "precision_at_1000": 9e-05, + "precision_at_3": 0.00251, + "precision_at_5": 0.00185, + "recall_at_1": 0.00261, + "recall_at_10": 0.01004, + "recall_at_100": 0.02728, + "recall_at_1000": 0.07648, + "recall_at_3": 0.00661, + "recall_at_5": 0.00806 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/QuoraRetrieval.json b/results/LASER2/QuoraRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..ef494bdc8c0df4bad4581abc0241f41ede3c0094 --- /dev/null +++ b/results/LASER2/QuoraRetrieval.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 393.84, + "map_at_1": 0.54965, + "map_at_10": 0.66201, + "map_at_100": 0.6704, + "map_at_1000": 0.6709, + "map_at_3": 0.63478, + "map_at_5": 0.65101, + "ndcg_at_1": 0.6354, + "ndcg_at_10": 0.71145, + "ndcg_at_100": 0.74023, + "ndcg_at_1000": 0.7486, + "ndcg_at_3": 0.67488, + "ndcg_at_5": 0.69306, + "precision_at_1": 0.6354, + "precision_at_10": 0.10798, + "precision_at_100": 0.01348, + "precision_at_1000": 0.00149, + "precision_at_3": 0.29257, + "precision_at_5": 0.19412, + "recall_at_1": 0.54965, + "recall_at_10": 0.80505, + "recall_at_100": 0.92322, + "recall_at_1000": 0.97766, + "recall_at_3": 0.69938, + "recall_at_5": 0.7496 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/RedditClustering.json b/results/LASER2/RedditClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..97ee45cf3e1fefcabb6a472fd9aa58ce9f134a0b --- /dev/null +++ b/results/LASER2/RedditClustering.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 251.71, + "v_measure": 0.0996202584007419, + "v_measure_std": 0.013494785830352516 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/RedditClusteringP2P.json b/results/LASER2/RedditClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..8562d77701700fee9be4ec7b98576dc75d94f79d --- /dev/null +++ b/results/LASER2/RedditClusteringP2P.json @@ -0,0 +1,9 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 1784.6, + "v_measure": 0.264185580282609, + "v_measure_std": 0.08501487346178989 + } +} \ No newline at end of file diff --git a/results/LASER2/SCIDOCS.json b/results/LASER2/SCIDOCS.json new file mode 100644 index 0000000000000000000000000000000000000000..6976adc7f93c7920a2462a265b43eddd5d730fa8 --- /dev/null +++ b/results/LASER2/SCIDOCS.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 164.89, + "map_at_1": 0.003, + "map_at_10": 0.00387, + "map_at_100": 0.00459, + "map_at_1000": 0.00532, + "map_at_3": 0.00333, + "map_at_5": 0.00356, + "ndcg_at_1": 0.015, + "ndcg_at_10": 0.00777, + "ndcg_at_100": 0.01565, + "ndcg_at_1000": 0.04821, + "ndcg_at_3": 0.00787, + "ndcg_at_5": 0.00647, + "precision_at_1": 0.015, + "precision_at_10": 0.0035, + "precision_at_100": 0.00162, + "precision_at_1000": 0.00098, + "precision_at_3": 0.006, + "precision_at_5": 0.0046, + "recall_at_1": 0.003, + "recall_at_10": 0.00718, + "recall_at_100": 0.03288, + "recall_at_1000": 0.19957, + "recall_at_3": 0.00373, + "recall_at_5": 0.00473 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/SICK-R.json b/results/LASER2/SICK-R.json new file mode 100644 index 0000000000000000000000000000000000000000..5e33aaf9392f81113484234405c35859d643f6e2 --- /dev/null +++ b/results/LASER2/SICK-R.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.6710717395109536, + "spearman": 0.6285761430269268 + }, + "euclidean": { + "pearson": 0.5991815699401573, + "spearman": 0.5964342697857706 + }, + "evaluation_time": 14.19, + "manhattan": { + "pearson": 0.5607429216703508, + "spearman": 0.5531149921833667 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/STS12.json b/results/LASER2/STS12.json new file mode 100644 index 0000000000000000000000000000000000000000..0ef6fa86f181525fb9d60e08720f668e9b98d71b --- /dev/null +++ b/results/LASER2/STS12.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.5997074409143551, + "spearman": 0.6260375600516247 + }, + "euclidean": { + "pearson": 0.49354765627824515, + "spearman": 0.5909106889528086 + }, + "evaluation_time": 11.98, + "manhattan": { + "pearson": 0.4558810047629682, + "spearman": 0.5570403794622609 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/STS13.json b/results/LASER2/STS13.json new file mode 100644 index 0000000000000000000000000000000000000000..772d95c093daaf939c56ade16d23f2f40a31493b --- /dev/null +++ b/results/LASER2/STS13.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.5482084934083251, + "spearman": 0.5961753521490076 + }, + "euclidean": { + "pearson": 0.48749678842541166, + "spearman": 0.5134313640207328 + }, + "evaluation_time": 11.09, + "manhattan": { + "pearson": 0.4650342089534767, + "spearman": 0.5049636504256733 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/STS14.json b/results/LASER2/STS14.json new file mode 100644 index 0000000000000000000000000000000000000000..d9b01f4d1d9713404dd85472e6fb949667694165 --- /dev/null +++ b/results/LASER2/STS14.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.5818703847325283, + "spearman": 0.5702962555736466 + }, + "euclidean": { + "pearson": 0.45125157337180055, + "spearman": 0.4602986618017823 + }, + "evaluation_time": 12.02, + "manhattan": { + "pearson": 0.43130231200967645, + "spearman": 0.4424979462930574 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/STS15.json b/results/LASER2/STS15.json new file mode 100644 index 0000000000000000000000000000000000000000..d2a6f43b32d84f33eaf9d97abbff0388d1e40eec --- /dev/null +++ b/results/LASER2/STS15.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.7149026229332927, + "spearman": 0.7156560343816677 + }, + "euclidean": { + "pearson": 0.6468455575157394, + "spearman": 0.638486354901432 + }, + "evaluation_time": 11.65, + "manhattan": { + "pearson": 0.6245348235089776, + "spearman": 0.6191555783086348 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/STS16.json b/results/LASER2/STS16.json new file mode 100644 index 0000000000000000000000000000000000000000..584fbf1af8bcd21de6a8eed0b5174b7ce9972349 --- /dev/null +++ b/results/LASER2/STS16.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.7054325243064823, + "spearman": 0.7075338261767046 + }, + "euclidean": { + "pearson": 0.6696113504818519, + "spearman": 0.6636321693884267 + }, + "evaluation_time": 10.97, + "manhattan": { + "pearson": 0.6383689807645614, + "spearman": 0.6311489646748284 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/STS17.json b/results/LASER2/STS17.json new file mode 100644 index 0000000000000000000000000000000000000000..427b60677d721eaa3051308d168eedf83cedc27f --- /dev/null +++ b/results/LASER2/STS17.json @@ -0,0 +1,161 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "ar-ar": { + "cos_sim": { + "pearson": 0.6801436364905892, + "spearman": 0.67471290685683 + }, + "euclidean": { + "pearson": 0.688464714661799, + "spearman": 0.6791576163557862 + }, + "manhattan": { + "pearson": 0.6775433103804529, + "spearman": 0.6740870980081657 + } + }, + "en-ar": { + "cos_sim": { + "pearson": 0.6331345024050433, + "spearman": 0.6504533249570149 + }, + "euclidean": { + "pearson": 0.6272572617459065, + "spearman": 0.6297211537313641 + }, + "manhattan": { + "pearson": 0.5915760543288405, + "spearman": 0.5878981670898759 + } + }, + "en-de": { + "cos_sim": { + "pearson": 0.6759917647701135, + "spearman": 0.6666485598206134 + }, + "euclidean": { + "pearson": 0.6751135165174461, + "spearman": 0.6684540704210077 + }, + "manhattan": { + "pearson": 0.6461132109329352, + "spearman": 0.640981633312306 + } + }, + "en-en": { + "cos_sim": { + "pearson": 0.7613689489191132, + "spearman": 0.7672812705124583 + }, + "euclidean": { + "pearson": 0.7516161468369351, + "spearman": 0.7509521050228384 + }, + "manhattan": { + "pearson": 0.7372390021348041, + "spearman": 0.7327051851399107 + } + }, + "en-tr": { + "cos_sim": { + "pearson": 0.7135050517199619, + "spearman": 0.7005300627762052 + }, + "euclidean": { + "pearson": 0.7371308386166394, + "spearman": 0.7396080617864679 + }, + "manhattan": { + "pearson": 0.6992109263147055, + "spearman": 0.7001513590146807 + } + }, + "es-en": { + "cos_sim": { + "pearson": 0.573170007653706, + "spearman": 0.5530160205961064 + }, + "euclidean": { + "pearson": 0.5774766298250574, + "spearman": 0.5571644221078261 + }, + "manhattan": { + "pearson": 0.5696185343814256, + "spearman": 0.5562636988438135 + } + }, + "es-es": { + "cos_sim": { + "pearson": 0.7970998417083257, + "spearman": 0.7967371935133953 + }, + "euclidean": { + "pearson": 0.7773043111963374, + "spearman": 0.7718331961142014 + }, + "manhattan": { + "pearson": 0.7523140795832627, + "spearman": 0.7412617185916907 + } + }, + "evaluation_time": 76.09, + "fr-en": { + "cos_sim": { + "pearson": 0.705285477974541, + "spearman": 0.7081553150090238 + }, + "euclidean": { + "pearson": 0.7139001856760165, + "spearman": 0.7162749233573582 + }, + "manhattan": { + "pearson": 0.6883374546133774, + "spearman": 0.6880160109051592 + } + }, + "it-en": { + "cos_sim": { + "pearson": 0.7126560928107515, + "spearman": 0.7097609394028597 + }, + "euclidean": { + "pearson": 0.7209539407598131, + "spearman": 0.7189733871458971 + }, + "manhattan": { + "pearson": 0.6928825782754588, + "spearman": 0.6894959376267937 + } + }, + "ko-ko": { + "cos_sim": { + "pearson": 0.7014595759675321, + "spearman": 0.7052134871940469 + }, + "euclidean": { + "pearson": 0.5415733866419151, + "spearman": 0.5395714675389758 + }, + "manhattan": { + "pearson": 0.5011233908837701, + "spearman": 0.49705478532188346 + } + }, + "nl-en": { + "cos_sim": { + "pearson": 0.6880208673096178, + "spearman": 0.6812483252269519 + }, + "euclidean": { + "pearson": 0.6939602449143909, + "spearman": 0.6984331574392624 + }, + "manhattan": { + "pearson": 0.6678300023562561, + "spearman": 0.6658040405979041 + } + } + } +} \ No newline at end of file diff --git a/results/LASER2/STS22.json b/results/LASER2/STS22.json new file mode 100644 index 0000000000000000000000000000000000000000..eb37cdcd36f070b0fe695bc3cd6f1519700c0535 --- /dev/null +++ b/results/LASER2/STS22.json @@ -0,0 +1,259 @@ +{ + "dataset_version": "6d1ba47164174a496b7fa5d3569dae26a6813b80", + "mteb_version": "0.0.2", + "test": { + "ar": { + "cos_sim": { + "pearson": 0.22399825612780572, + "spearman": 0.4257422635873906 + }, + "euclidean": { + "pearson": 0.3030090651926248, + "spearman": 0.4077978487387852 + }, + "manhattan": { + "pearson": 0.34953847729812365, + "spearman": 0.4260211243963624 + } + }, + "de": { + "cos_sim": { + "pearson": 0.09239218701232288, + "spearman": 0.25689644701176306 + }, + "euclidean": { + "pearson": 0.053531273690696504, + "spearman": 0.18510543664936038 + }, + "manhattan": { + "pearson": 0.08670081522167335, + "spearman": 0.17996657229567936 + } + }, + "de-en": { + "cos_sim": { + "pearson": 0.0013479727118452453, + "spearman": 0.3235250562269627 + }, + "euclidean": { + "pearson": 0.15012095640126139, + "spearman": 0.31471527281481654 + }, + "manhattan": { + "pearson": 0.2109357139582619, + "spearman": 0.3401947489977194 + } + }, + "de-fr": { + "cos_sim": { + "pearson": 0.22705709977476957, + "spearman": 0.37405107705079765 + }, + "euclidean": { + "pearson": 0.30561393923670876, + "spearman": 0.4155220554329526 + }, + "manhattan": { + "pearson": 0.319779169746448, + "spearman": 0.44295878541809985 + } + }, + "de-pl": { + "cos_sim": { + "pearson": 0.0662198588552205, + "spearman": 0.1567419868131044 + }, + "euclidean": { + "pearson": 0.057382915823515984, + "spearman": 0.146271991623231 + }, + "manhattan": { + "pearson": 0.0933763453225985, + "spearman": 0.17521844891288094 + } + }, + "en": { + "cos_sim": { + "pearson": 0.2101016303872804, + "spearman": 0.39754999965029375 + }, + "euclidean": { + "pearson": 0.31478530690667683, + "spearman": 0.4042210376225253 + }, + "manhattan": { + "pearson": 0.34924497920234676, + "spearman": 0.41953223735822426 + } + }, + "es": { + "cos_sim": { + "pearson": 0.1287207977520764, + "spearman": 0.5492031963435001 + }, + "euclidean": { + "pearson": 0.38635728110362055, + "spearman": 0.5775391865115991 + }, + "manhattan": { + "pearson": 0.43765768532833255, + "spearman": 0.5877351793460481 + } + }, + "es-en": { + "cos_sim": { + "pearson": 0.22351479991828302, + "spearman": 0.5433790834604113 + }, + "euclidean": { + "pearson": 0.4178305921193284, + "spearman": 0.59311251968952 + }, + "manhattan": { + "pearson": 0.46823770131717773, + "spearman": 0.6141448683663221 + } + }, + "es-it": { + "cos_sim": { + "pearson": 0.22733678512149227, + "spearman": 0.42205155600580757 + }, + "euclidean": { + "pearson": 0.41160388814187443, + "spearman": 0.4965854702700807 + }, + "manhattan": { + "pearson": 0.44501006110372354, + "spearman": 0.5303101065840476 + } + }, + "evaluation_time": 220.87, + "fr": { + "cos_sim": { + "pearson": 0.28774388241277926, + "spearman": 0.5860942035161938 + }, + "euclidean": { + "pearson": 0.4088544804074059, + "spearman": 0.5748634171457377 + }, + "manhattan": { + "pearson": 0.49020948352681765, + "spearman": 0.5965275375999903 + } + }, + "fr-pl": { + "cos_sim": { + "pearson": 0.5270303049281483, + "spearman": 0.3944053188733078 + }, + "euclidean": { + "pearson": 0.598217794405991, + "spearman": 0.50709255283711 + }, + "manhattan": { + "pearson": 0.5992862452592251, + "spearman": 0.50709255283711 + } + }, + "it": { + "cos_sim": { + "pearson": 0.453952624530108, + "spearman": 0.6031219163772915 + }, + "euclidean": { + "pearson": 0.5689518423872097, + "spearman": 0.6147043724832666 + }, + "manhattan": { + "pearson": 0.6083317528035341, + "spearman": 0.6498922552131471 + } + }, + "pl": { + "cos_sim": { + "pearson": 0.018370756227069088, + "spearman": 0.18342486158185198 + }, + "euclidean": { + "pearson": 0.025951836804318947, + "spearman": 0.19016091804034613 + }, + "manhattan": { + "pearson": 0.003332464637298374, + "spearman": 0.200323932353402 + } + }, + "pl-en": { + "cos_sim": { + "pearson": 0.3339894418147491, + "spearman": 0.536300053844228 + }, + "euclidean": { + "pearson": 0.5123403560143394, + "spearman": 0.5872472566280702 + }, + "manhattan": { + "pearson": 0.5505556232937919, + "spearman": 0.6170445981335483 + } + }, + "ru": { + "cos_sim": { + "pearson": 0.09664268900587175, + "spearman": 0.39238028677590087 + }, + "euclidean": { + "pearson": 0.22732815251989258, + "spearman": 0.3885765342482269 + }, + "manhattan": { + "pearson": 0.24867681226319566, + "spearman": 0.40376765499506 + } + }, + "tr": { + "cos_sim": { + "pearson": 0.17438905289475365, + "spearman": 0.3697080711967222 + }, + "euclidean": { + "pearson": 0.261814843856061, + "spearman": 0.3318478857563501 + }, + "manhattan": { + "pearson": 0.2871069547438314, + "spearman": 0.32986592547433413 + } + }, + "zh": { + "cos_sim": { + "pearson": 0.11237535228282351, + "spearman": 0.49409672527960347 + }, + "euclidean": { + "pearson": 0.27025960507413893, + "spearman": 0.4798215013916385 + }, + "manhattan": { + "pearson": 0.3392669328649448, + "spearman": 0.5093712161569486 + } + }, + "zh-en": { + "cos_sim": { + "pearson": 0.25668488486166574, + "spearman": 0.46190801321486336 + }, + "euclidean": { + "pearson": 0.40875508271480476, + "spearman": 0.5086008839147637 + }, + "manhattan": { + "pearson": 0.43147436155156527, + "spearman": 0.510828738311338 + } + } + } +} \ No newline at end of file diff --git a/results/LASER2/STSBenchmark.json b/results/LASER2/STSBenchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..aa0cbeda00481cc00e46476366537e8b7d6fb22c --- /dev/null +++ b/results/LASER2/STSBenchmark.json @@ -0,0 +1,19 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "cos_sim": { + "pearson": 0.7021304131403056, + "spearman": 0.6977241730309153 + }, + "euclidean": { + "pearson": 0.5262694226761778, + "spearman": 0.5220118701934681 + }, + "evaluation_time": 11.79, + "manhattan": { + "pearson": 0.46402849152606146, + "spearman": 0.4656295330158784 + } + } +} \ No newline at end of file diff --git a/results/LASER2/SciDocsRR.json b/results/LASER2/SciDocsRR.json new file mode 100644 index 0000000000000000000000000000000000000000..9f0a88ec571db507e9bb5b279851578c525c7b09 --- /dev/null +++ b/results/LASER2/SciDocsRR.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 42.51, + "map": 0.5498656851897086, + "mrr": 0.7942668805413904 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/SciFact.json b/results/LASER2/SciFact.json new file mode 100644 index 0000000000000000000000000000000000000000..d1795914e4beb5847ddc2428c72a711d60f71b30 --- /dev/null +++ b/results/LASER2/SciFact.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 55.87, + "map_at_1": 0.02, + "map_at_10": 0.03188, + "map_at_100": 0.0359, + "map_at_1000": 0.03746, + "map_at_3": 0.02722, + "map_at_5": 0.03031, + "ndcg_at_1": 0.02, + "ndcg_at_10": 0.04038, + "ndcg_at_100": 0.06922, + "ndcg_at_1000": 0.11648, + "ndcg_at_3": 0.03044, + "ndcg_at_5": 0.03627, + "precision_at_1": 0.02, + "precision_at_10": 0.007, + "precision_at_100": 0.0024, + "precision_at_1000": 0.00067, + "precision_at_3": 0.01333, + "precision_at_5": 0.01133, + "recall_at_1": 0.02, + "recall_at_10": 0.06667, + "recall_at_100": 0.21844, + "recall_at_1000": 0.594, + "recall_at_3": 0.04, + "recall_at_5": 0.05333 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/SprintDuplicateQuestions.json b/results/LASER2/SprintDuplicateQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..3c22f1b8a06551aea33c7f73b67882312e69f927 --- /dev/null +++ b/results/LASER2/SprintDuplicateQuestions.json @@ -0,0 +1,48 @@ +{ + "test": { + "cos_sim": { + "accuracy": 0.9935643564356436, + "accuracy_threshold": 0.8075398206710815, + "ap": 0.6554452704075004, + "f1": 0.6178686759956943, + "f1_threshold": 0.7865520715713501, + "precision": 0.668997668997669, + "recall": 0.574 + }, + "dot": { + "accuracy": 0.9907227722772277, + "accuracy_threshold": 0.38118740916252136, + "ap": 0.24298813067700603, + "f1": 0.30387409200968524, + "f1_threshold": 0.3419535160064697, + "precision": 0.38496932515337423, + "recall": 0.251 + }, + "euclidean": { + "accuracy": 0.9937425742574257, + "accuracy_threshold": 0.38275259733200073, + "ap": 0.6555481590642165, + "f1": 0.6334246575342466, + "f1_threshold": 0.40048980712890625, + "precision": 0.7006060606060606, + "recall": 0.578 + }, + "evaluation_time": 13.32, + "manhattan": { + "accuracy": 0.9938811881188119, + "accuracy_threshold": 7.654071807861328, + "ap": 0.6848476246224511, + "f1": 0.654925701706109, + "f1_threshold": 7.864644527435303, + "precision": 0.7282741738066095, + "recall": 0.595 + }, + "max": { + "accuracy": 0.9938811881188119, + "ap": 0.6848476246224511, + "f1": 0.654925701706109 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/StackExchangeClustering.json b/results/LASER2/StackExchangeClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..0af655eda747653ceb5755a3ca96e059f866da1d --- /dev/null +++ b/results/LASER2/StackExchangeClustering.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 225.82, + "v_measure": 0.15794084761797694, + "v_measure_std": 0.027003105512856673 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/StackExchangeClusteringP2P.json b/results/LASER2/StackExchangeClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..1bf6838c63dea7d4597a25fc05fd38d7c2ec74e4 --- /dev/null +++ b/results/LASER2/StackExchangeClusteringP2P.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 591.96, + "v_measure": 0.18627980064437472, + "v_measure_std": 0.01991546180991416 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/StackOverflowDupQuestions.json b/results/LASER2/StackOverflowDupQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..0149418f170429514e30c8e2f35518e1a64a92c0 --- /dev/null +++ b/results/LASER2/StackOverflowDupQuestions.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 30.27, + "map": 0.36983704984940896, + "mrr": 0.3692667218402512 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/SummEval.json b/results/LASER2/SummEval.json new file mode 100644 index 0000000000000000000000000000000000000000..f903086ca691bc66300778b1f1f7a30f13439fec --- /dev/null +++ b/results/LASER2/SummEval.json @@ -0,0 +1,15 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.2546339001703312, + "spearman": 0.268033428993562 + }, + "dot": { + "pearson": 0.2607652569492036, + "spearman": 0.27691547366916064 + }, + "evaluation_time": 8729.84 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/TRECCOVID.json b/results/LASER2/TRECCOVID.json new file mode 100644 index 0000000000000000000000000000000000000000..17cff63afe1c33755f459a4677601dabac757a99 --- /dev/null +++ b/results/LASER2/TRECCOVID.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 1188.13, + "map_at_1": 0.00031, + "map_at_10": 0.00132, + "map_at_100": 0.00253, + "map_at_1000": 0.00342, + "map_at_3": 0.00084, + "map_at_5": 0.00102, + "ndcg_at_1": 0.14, + "ndcg_at_10": 0.10972, + "ndcg_at_100": 0.05153, + "ndcg_at_1000": 0.03504, + "ndcg_at_3": 0.14235, + "ndcg_at_5": 0.13033, + "precision_at_1": 0.14, + "precision_at_10": 0.108, + "precision_at_100": 0.0458, + "precision_at_1000": 0.01454, + "precision_at_3": 0.16, + "precision_at_5": 0.136, + "recall_at_1": 0.00031, + "recall_at_10": 0.00242, + "recall_at_100": 0.0094, + "recall_at_1000": 0.0301, + "recall_at_3": 0.00106, + "recall_at_5": 0.00144 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/Tatoeba.json b/results/LASER2/Tatoeba.json new file mode 100644 index 0000000000000000000000000000000000000000..bc57f751663c8b53faa007b3c8bd27e819b2d418 --- /dev/null +++ b/results/LASER2/Tatoeba.json @@ -0,0 +1,679 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "afr-eng": { + "accuracy": 0.943, + "f1": 0.9259, + "precision": 0.91775, + "recall": 0.943 + }, + "amh-eng": { + "accuracy": 0.8511904761904762, + "f1": 0.8082482993197279, + "precision": 0.7886904761904762, + "recall": 0.8511904761904762 + }, + "ang-eng": { + "accuracy": 0.3208955223880597, + "f1": 0.25217262617394304, + "precision": 0.23799630096271887, + "recall": 0.3208955223880597 + }, + "ara-eng": { + "accuracy": 0.923, + "f1": 0.9014, + "precision": 0.8909166666666666, + "recall": 0.923 + }, + "arq-eng": { + "accuracy": 0.31284302963776073, + "f1": 0.2663221313915725, + "precision": 0.2526664491150314, + "recall": 0.31284302963776073 + }, + "arz-eng": { + "accuracy": 0.7085953878406709, + "f1": 0.6616052710392333, + "precision": 0.6405310971348708, + "recall": 0.7085953878406709 + }, + "ast-eng": { + "accuracy": 0.8110236220472441, + "f1": 0.763517060367454, + "precision": 0.7454068241469816, + "recall": 0.8110236220472441 + }, + "awa-eng": { + "accuracy": 0.38961038961038963, + "f1": 0.3373766445195016, + "precision": 0.3219748505462791, + "recall": 0.38961038961038963 + }, + "aze-eng": { + "accuracy": 0.856, + "f1": 0.8241380952380952, + "precision": 0.8107277777777776, + "recall": 0.856 + }, + "bel-eng": { + "accuracy": 0.833, + "f1": 0.7953912698412698, + "precision": 0.7797892857142857, + "recall": 0.833 + }, + "ben-eng": { + "accuracy": 0.917, + "f1": 0.8943333333333334, + "precision": 0.8836666666666667, + "recall": 0.917 + }, + "ber-eng": { + "accuracy": 0.82, + "f1": 0.7763, + "precision": 0.7568111111111111, + "recall": 0.82 + }, + "bos-eng": { + "accuracy": 0.9689265536723164, + "f1": 0.9585687382297553, + "precision": 0.9533898305084746, + "recall": 0.9689265536723164 + }, + "bre-eng": { + "accuracy": 0.369, + "f1": 0.312032490590085, + "precision": 0.29087169913419914, + "recall": 0.369 + }, + "bul-eng": { + "accuracy": 0.949, + "f1": 0.9356666666666666, + "precision": 0.9291666666666667, + "recall": 0.949 + }, + "cat-eng": { + "accuracy": 0.968, + "f1": 0.958, + "precision": 0.9533333333333333, + "recall": 0.968 + }, + "cbk-eng": { + "accuracy": 0.811, + "f1": 0.7716785714285713, + "precision": 0.7560261904761906, + "recall": 0.811 + }, + "ceb-eng": { + "accuracy": 0.14166666666666666, + "f1": 0.09933432192620326, + "precision": 0.08962492368742368, + "recall": 0.14166666666666666 + }, + "ces-eng": { + "accuracy": 0.966, + "f1": 0.9551666666666666, + "precision": 0.9498333333333333, + "recall": 0.966 + }, + "cha-eng": { + "accuracy": 0.1897810218978102, + "f1": 0.14863138686131386, + "precision": 0.14101868358991032, + "recall": 0.1897810218978102 + }, + "cmn-eng": { + "accuracy": 0.886, + "f1": 0.8562333333333332, + "precision": 0.8424499999999999, + "recall": 0.886 + }, + "cor-eng": { + "accuracy": 0.062, + "f1": 0.0444599012766252, + "precision": 0.04032906097625029, + "recall": 0.062 + }, + "csb-eng": { + "accuracy": 0.3359683794466403, + "f1": 0.2702575348820408, + "precision": 0.2521517267317663, + "recall": 0.3359683794466403 + }, + "cym-eng": { + "accuracy": 0.0817391304347826, + "f1": 0.05854842365274456, + "precision": 0.05352279967531011, + "recall": 0.0817391304347826 + }, + "dan-eng": { + "accuracy": 0.962, + "f1": 0.9522333333333333, + "precision": 0.9475833333333333, + "recall": 0.962 + }, + "deu-eng": { + "accuracy": 0.993, + "f1": 0.9906666666666667, + "precision": 0.9895, + "recall": 0.993 + }, + "dsb-eng": { + "accuracy": 0.4801670146137787, + "f1": 0.42336382012791196, + "precision": 0.40368641449017234, + "recall": 0.4801670146137787 + }, + "dtp-eng": { + "accuracy": 0.098, + "f1": 0.07389854447432771, + "precision": 0.07018063241444196, + "recall": 0.098 + }, + "ell-eng": { + "accuracy": 0.971, + "f1": 0.962, + "precision": 0.9575, + "recall": 0.971 + }, + "epo-eng": { + "accuracy": 0.974, + "f1": 0.9660666666666666, + "precision": 0.96225, + "recall": 0.974 + }, + "est-eng": { + "accuracy": 0.973, + "f1": 0.9643333333333333, + "precision": 0.96, + "recall": 0.973 + }, + "eus-eng": { + "accuracy": 0.948, + "f1": 0.9332333333333334, + "precision": 0.92625, + "recall": 0.948 + }, + "evaluation_time": 746.28, + "fao-eng": { + "accuracy": 0.6335877862595419, + "f1": 0.5703595386801493, + "precision": 0.5466375863322428, + "recall": 0.6335877862595419 + }, + "fin-eng": { + "accuracy": 0.977, + "f1": 0.9698333333333333, + "precision": 0.9663333333333334, + "recall": 0.977 + }, + "fra-eng": { + "accuracy": 0.955, + "f1": 0.9428333333333334, + "precision": 0.937, + "recall": 0.955 + }, + "fry-eng": { + "accuracy": 0.4797687861271676, + "f1": 0.42067162124965585, + "precision": 0.4023188757292803, + "recall": 0.4797687861271676 + }, + "gla-eng": { + "accuracy": 0.025331724969843185, + "f1": 0.01518224439521446, + "precision": 0.0140455035383427, + "recall": 0.025331724969843185 + }, + "gle-eng": { + "accuracy": 0.062, + "f1": 0.04204012349091683, + "precision": 0.03848639740560742, + "recall": 0.062 + }, + "glg-eng": { + "accuracy": 0.97, + "f1": 0.9614000000000001, + "precision": 0.9574166666666667, + "recall": 0.97 + }, + "gsw-eng": { + "accuracy": 0.3333333333333333, + "f1": 0.2752179594284857, + "precision": 0.25588793922127256, + "recall": 0.3333333333333333 + }, + "heb-eng": { + "accuracy": 0.0, + "f1": 0.0, + "precision": 0.0, + "recall": 0.0 + }, + "hin-eng": { + "accuracy": 0.964, + "f1": 0.9531666666666666, + "precision": 0.9478333333333333, + "recall": 0.964 + }, + "hrv-eng": { + "accuracy": 0.975, + "f1": 0.9671666666666666, + "precision": 0.9633333333333333, + "recall": 0.975 + }, + "hsb-eng": { + "accuracy": 0.5072463768115942, + "f1": 0.4574828409475455, + "precision": 0.4401859924814286, + "recall": 0.5072463768115942 + }, + "hun-eng": { + "accuracy": 0.963, + "f1": 0.952, + "precision": 0.9466666666666668, + "recall": 0.963 + }, + "hye-eng": { + "accuracy": 0.9123989218328841, + "f1": 0.8872416891284817, + "precision": 0.8751123090745733, + "recall": 0.9123989218328841 + }, + "ido-eng": { + "accuracy": 0.843, + "f1": 0.8086285714285714, + "precision": 0.7947295546558705, + "recall": 0.843 + }, + "ile-eng": { + "accuracy": 0.904, + "f1": 0.8788341991341991, + "precision": 0.8676833333333333, + "recall": 0.904 + }, + "ina-eng": { + "accuracy": 0.953, + "f1": 0.9393333333333334, + "precision": 0.9326666666666668, + "recall": 0.953 + }, + "ind-eng": { + "accuracy": 0.945, + "f1": 0.9298333333333333, + "precision": 0.9227000000000001, + "recall": 0.945 + }, + "isl-eng": { + "accuracy": 0.955, + "f1": 0.9431666666666666, + "precision": 0.9373333333333332, + "recall": 0.955 + }, + "ita-eng": { + "accuracy": 0.954, + "f1": 0.9431666666666666, + "precision": 0.9380333333333333, + "recall": 0.954 + }, + "jav-eng": { + "accuracy": 0.14634146341463414, + "f1": 0.09948930784722623, + "precision": 0.08895197620807377, + "recall": 0.14634146341463414 + }, + "jpn-eng": { + "accuracy": 0.952, + "f1": 0.9378333333333333, + "precision": 0.9308333333333333, + "recall": 0.952 + }, + "kab-eng": { + "accuracy": 0.709, + "f1": 0.6588357142857143, + "precision": 0.638541341991342, + "recall": 0.709 + }, + "kat-eng": { + "accuracy": 0.8431635388739946, + "f1": 0.8116207072641389, + "precision": 0.7982254564024, + "recall": 0.8431635388739946 + }, + "kaz-eng": { + "accuracy": 0.591304347826087, + "f1": 0.5330062111801241, + "precision": 0.5096153427045875, + "recall": 0.591304347826087 + }, + "khm-eng": { + "accuracy": 0.7867036011080333, + "f1": 0.7418744228993537, + "precision": 0.7214219759926132, + "recall": 0.7867036011080333 + }, + "kor-eng": { + "accuracy": 0.906, + "f1": 0.8797333333333333, + "precision": 0.8675833333333334, + "recall": 0.906 + }, + "kur-eng": { + "accuracy": 0.23414634146341465, + "f1": 0.19090883135952066, + "precision": 0.17855863409665418, + "recall": 0.23414634146341465 + }, + "kzj-eng": { + "accuracy": 0.066, + "f1": 0.04459115775646192, + "precision": 0.04100426498575014, + "recall": 0.066 + }, + "lat-eng": { + "accuracy": 0.695, + "f1": 0.6481185897435897, + "precision": 0.6287223877068557, + "recall": 0.695 + }, + "lfn-eng": { + "accuracy": 0.68, + "f1": 0.6338830128205128, + "precision": 0.6169762035825996, + "recall": 0.68 + }, + "lit-eng": { + "accuracy": 0.97, + "f1": 0.962, + "precision": 0.958, + "recall": 0.97 + }, + "lvs-eng": { + "accuracy": 0.964, + "f1": 0.9533333333333333, + "precision": 0.9481666666666667, + "recall": 0.964 + }, + "mal-eng": { + "accuracy": 0.9854439592430859, + "f1": 0.9815623483745755, + "precision": 0.9796215429403202, + "recall": 0.9854439592430859 + }, + "mar-eng": { + "accuracy": 0.946, + "f1": 0.9293333333333332, + "precision": 0.921, + "recall": 0.946 + }, + "max-eng": { + "accuracy": 0.44014084507042256, + "f1": 0.36957126531116524, + "precision": 0.3485877631379676, + "recall": 0.44014084507042256 + }, + "mhr-eng": { + "accuracy": 0.094, + "f1": 0.06856864012159629, + "precision": 0.06307192561559993, + "recall": 0.094 + }, + "mkd-eng": { + "accuracy": 0.95, + "f1": 0.9363333333333334, + "precision": 0.9298333333333334, + "recall": 0.95 + }, + "mon-eng": { + "accuracy": 0.04772727272727273, + "f1": 0.03420342798332415, + "precision": 0.03211418279432985, + "recall": 0.04772727272727273 + }, + "nds-eng": { + "accuracy": 0.812, + "f1": 0.7712746031746032, + "precision": 0.754665909090909, + "recall": 0.812 + }, + "nld-eng": { + "accuracy": 0.964, + "f1": 0.9535, + "precision": 0.9483333333333333, + "recall": 0.964 + }, + "nno-eng": { + "accuracy": 0.773, + "f1": 0.7275008658008658, + "precision": 0.7089904761904761, + "recall": 0.773 + }, + "nob-eng": { + "accuracy": 0.968, + "f1": 0.9576666666666667, + "precision": 0.9526666666666667, + "recall": 0.968 + }, + "nov-eng": { + "accuracy": 0.6653696498054474, + "f1": 0.6001729355814959, + "precision": 0.57519645990692, + "recall": 0.6653696498054474 + }, + "oci-eng": { + "accuracy": 0.635, + "f1": 0.5812726735253052, + "precision": 0.5620229630448671, + "recall": 0.635 + }, + "orv-eng": { + "accuracy": 0.2838323353293413, + "f1": 0.2323980573651232, + "precision": 0.2162651058055749, + "recall": 0.2838323353293413 + }, + "pam-eng": { + "accuracy": 0.048, + "f1": 0.03241787254474141, + "precision": 0.02900245857293385, + "recall": 0.048 + }, + "pes-eng": { + "accuracy": 0.945, + "f1": 0.9313333333333332, + "precision": 0.9248333333333333, + "recall": 0.945 + }, + "pms-eng": { + "accuracy": 0.4342857142857143, + "f1": 0.3623231493063426, + "precision": 0.34115733645236757, + "recall": 0.4342857142857143 + }, + "pol-eng": { + "accuracy": 0.979, + "f1": 0.9731666666666666, + "precision": 0.9705, + "recall": 0.979 + }, + "por-eng": { + "accuracy": 0.956, + "f1": 0.9453523809523809, + "precision": 0.9404166666666667, + "recall": 0.956 + }, + "ron-eng": { + "accuracy": 0.973, + "f1": 0.9651666666666666, + "precision": 0.9613333333333334, + "recall": 0.973 + }, + "rus-eng": { + "accuracy": 0.942, + "f1": 0.9258333333333333, + "precision": 0.9178333333333333, + "recall": 0.942 + }, + "slk-eng": { + "accuracy": 0.967, + "f1": 0.9581666666666666, + "precision": 0.9538333333333334, + "recall": 0.967 + }, + "slv-eng": { + "accuracy": 0.9647630619684082, + "f1": 0.9540299716484406, + "precision": 0.9487646820575131, + "recall": 0.9647630619684082 + }, + "spa-eng": { + "accuracy": 0.98, + "f1": 0.9733333333333333, + "precision": 0.97, + "recall": 0.98 + }, + "sqi-eng": { + "accuracy": 0.978, + "f1": 0.9721666666666666, + "precision": 0.9693333333333334, + "recall": 0.978 + }, + "srp-eng": { + "accuracy": 0.951, + "f1": 0.9364, + "precision": 0.9294166666666667, + "recall": 0.951 + }, + "swe-eng": { + "accuracy": 0.962, + "f1": 0.9530666666666666, + "precision": 0.94875, + "recall": 0.962 + }, + "swg-eng": { + "accuracy": 0.4017857142857143, + "f1": 0.33101851851851855, + "precision": 0.3076350732600733, + "recall": 0.4017857142857143 + }, + "swh-eng": { + "accuracy": 0.6102564102564103, + "f1": 0.5566222666222667, + "precision": 0.5378217697335345, + "recall": 0.6102564102564103 + }, + "tam-eng": { + "accuracy": 0.8925081433224755, + "f1": 0.8731813246471225, + "precision": 0.8640065146579805, + "recall": 0.8925081433224755 + }, + "tat-eng": { + "accuracy": 0.411, + "f1": 0.3474027601053917, + "precision": 0.324908567007705, + "recall": 0.411 + }, + "tel-eng": { + "accuracy": 0.9743589743589743, + "f1": 0.9672364672364672, + "precision": 0.9636752136752137, + "recall": 0.9743589743589743 + }, + "tgl-eng": { + "accuracy": 0.682, + "f1": 0.6318801587301587, + "precision": 0.612822619047619, + "recall": 0.682 + }, + "tha-eng": { + "accuracy": 0.9726277372262774, + "f1": 0.9638077858880778, + "precision": 0.9595498783454987, + "recall": 0.9726277372262774 + }, + "tuk-eng": { + "accuracy": 0.2019704433497537, + "f1": 0.16345815233094282, + "precision": 0.15540068169378515, + "recall": 0.2019704433497537 + }, + "tur-eng": { + "accuracy": 0.985, + "f1": 0.9803333333333333, + "precision": 0.978, + "recall": 0.985 + }, + "tzl-eng": { + "accuracy": 0.4230769230769231, + "f1": 0.3655837218337218, + "precision": 0.3511790293040293, + "recall": 0.4230769230769231 + }, + "uig-eng": { + "accuracy": 0.626, + "f1": 0.564875, + "precision": 0.5398095238095237, + "recall": 0.626 + }, + "ukr-eng": { + "accuracy": 0.95, + "f1": 0.9351666666666667, + "precision": 0.9278333333333333, + "recall": 0.95 + }, + "urd-eng": { + "accuracy": 0.874, + "f1": 0.8422999999999999, + "precision": 0.8280333333333333, + "recall": 0.874 + }, + "uzb-eng": { + "accuracy": 0.2757009345794392, + "f1": 0.2320341813596971, + "precision": 0.22032813906036172, + "recall": 0.2757009345794392 + }, + "vie-eng": { + "accuracy": 0.975, + "f1": 0.9673333333333334, + "precision": 0.9635, + "recall": 0.975 + }, + "war-eng": { + "accuracy": 0.107, + "f1": 0.08249952522272298, + "precision": 0.07706462772350414, + "recall": 0.107 + }, + "wuu-eng": { + "accuracy": 0.796, + "f1": 0.7509094017094017, + "precision": 0.7317416666666665, + "recall": 0.796 + }, + "xho-eng": { + "accuracy": 0.06338028169014084, + "f1": 0.046788305591122493, + "precision": 0.043479627766599596, + "recall": 0.06338028169014084 + }, + "yid-eng": { + "accuracy": 0.03773584905660377, + "f1": 0.024940123332144272, + "precision": 0.022748795893729884, + "recall": 0.03773584905660377 + }, + "yue-eng": { + "accuracy": 0.904, + "f1": 0.8775, + "precision": 0.8650333333333334, + "recall": 0.904 + }, + "zsm-eng": { + "accuracy": 0.965, + "f1": 0.9540666666666666, + "precision": 0.94875, + "recall": 0.965 + } + } +} \ No newline at end of file diff --git a/results/LASER2/Touche2020.json b/results/LASER2/Touche2020.json new file mode 100644 index 0000000000000000000000000000000000000000..d587370d78528b2b5a9727397080c30ba61f1317 --- /dev/null +++ b/results/LASER2/Touche2020.json @@ -0,0 +1,31 @@ +{ + "test": { + "evaluation_time": 2868.01, + "map_at_1": 0.00146, + "map_at_10": 0.0036, + "map_at_100": 0.00428, + "map_at_1000": 0.00468, + "map_at_3": 0.00292, + "map_at_5": 0.00316, + "ndcg_at_1": 0.02041, + "ndcg_at_10": 0.0106, + "ndcg_at_100": 0.01767, + "ndcg_at_1000": 0.0329, + "ndcg_at_3": 0.0126, + "ndcg_at_5": 0.01209, + "precision_at_1": 0.02041, + "precision_at_10": 0.0102, + "precision_at_100": 0.00469, + "precision_at_1000": 0.00161, + "precision_at_3": 0.01361, + "precision_at_5": 0.01224, + "recall_at_1": 0.00146, + "recall_at_10": 0.00788, + "recall_at_100": 0.02675, + "recall_at_1000": 0.08458, + "recall_at_3": 0.00292, + "recall_at_5": 0.00389 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/ToxicConversationsClassification.json b/results/LASER2/ToxicConversationsClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..e32bf42d6b52a0ebf511ea815d60f07a62fffdb4 --- /dev/null +++ b/results/LASER2/ToxicConversationsClassification.json @@ -0,0 +1,14 @@ +{ + "test": { + "accuracy": 0.5404779999999999, + "accuracy_stderr": 0.058204167170401125, + "ap": 0.09680562314649051, + "ap_stderr": 0.0130349361795604, + "evaluation_time": 669.03, + "f1": 0.42952438913599666, + "f1_stderr": 0.037157125631979135, + "main_score": 0.09680562314649051 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/TweetSentimentExtractionClassification.json b/results/LASER2/TweetSentimentExtractionClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..7e85712e788c20c94bc48a2d6a56e78c567a4187 --- /dev/null +++ b/results/LASER2/TweetSentimentExtractionClassification.json @@ -0,0 +1,12 @@ +{ + "test": { + "accuracy": 0.4872665534804754, + "accuracy_stderr": 0.03130421904163542, + "evaluation_time": 118.48, + "f1": 0.48801591534886074, + "f1_stderr": 0.031625293531795155, + "main_score": 0.4872665534804754 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/TwentyNewsgroupsClustering.json b/results/LASER2/TwentyNewsgroupsClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..968ac735f8da8715c5bd34211552da3e3b68e630 --- /dev/null +++ b/results/LASER2/TwentyNewsgroupsClustering.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 64.96, + "v_measure": 0.11378130183913912, + "v_measure_std": 0.02245543556240757 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/TwitterSemEval2015.json b/results/LASER2/TwitterSemEval2015.json new file mode 100644 index 0000000000000000000000000000000000000000..0a3efa0827a945e513ce5dbeda35a90186878942 --- /dev/null +++ b/results/LASER2/TwitterSemEval2015.json @@ -0,0 +1,48 @@ +{ + "test": { + "cos_sim": { + "accuracy": 0.8207665256005245, + "accuracy_threshold": 0.7753598093986511, + "ap": 0.5956789548901481, + "f1": 0.5697118332311466, + "f1_threshold": 0.7282990217208862, + "precision": 0.5321878579610538, + "recall": 0.612928759894459 + }, + "dot": { + "accuracy": 0.7919771115217261, + "accuracy_threshold": 0.29487645626068115, + "ap": 0.46260703584680707, + "f1": 0.4874578809434669, + "f1_threshold": 0.24640625715255737, + "precision": 0.46246744020838265, + "recall": 0.5153034300791557 + }, + "euclidean": { + "accuracy": 0.8066400429158967, + "accuracy_threshold": 0.3704949915409088, + "ap": 0.5403708142853088, + "f1": 0.5144664402014759, + "f1_threshold": 0.41998252272605896, + "precision": 0.46260796292395195, + "recall": 0.5794195250659631 + }, + "evaluation_time": 9.27, + "manhattan": { + "accuracy": 0.7983548906240686, + "accuracy_threshold": 6.359255790710449, + "ap": 0.497610338575302, + "f1": 0.48250116658889414, + "f1_threshold": 8.010211944580078, + "precision": 0.43245503973232957, + "recall": 0.545646437994723 + }, + "max": { + "accuracy": 0.8207665256005245, + "ap": 0.5956789548901481, + "f1": 0.5697118332311466 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/results/LASER2/TwitterURLCorpus.json b/results/LASER2/TwitterURLCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..5c8da443169d5cd2d014e85d9456e4c48f90cfdc --- /dev/null +++ b/results/LASER2/TwitterURLCorpus.json @@ -0,0 +1,48 @@ +{ + "test": { + "cos_sim": { + "accuracy": 0.8721426630962084, + "accuracy_threshold": 0.7175396680831909, + "ap": 0.8147393118163185, + "f1": 0.7399120145631068, + "f1_threshold": 0.6966373324394226, + "precision": 0.7290732436472347, + "recall": 0.7510779180782261 + }, + "dot": { + "accuracy": 0.8032366981022238, + "accuracy_threshold": 0.3221713900566101, + "ap": 0.6165376099028658, + "f1": 0.5896786618638326, + "f1_threshold": 0.2890387177467346, + "precision": 0.5492658295129892, + "recall": 0.6365106251924854 + }, + "euclidean": { + "accuracy": 0.8676019715139519, + "accuracy_threshold": 0.47812920808792114, + "ap": 0.8026557073495436, + "f1": 0.7276556438204322, + "f1_threshold": 0.5027191638946533, + "precision": 0.6995098387440506, + "recall": 0.7581613797351401 + }, + "evaluation_time": 22.67, + "manhattan": { + "accuracy": 0.868979702720534, + "accuracy_threshold": 9.598307609558105, + "ap": 0.8097691582914013, + "f1": 0.7321912972390474, + "f1_threshold": 10.176679611206055, + "precision": 0.7076061193708253, + "recall": 0.7585463504773637 + }, + "max": { + "accuracy": 0.8721426630962084, + "ap": 0.8147393118163185, + "f1": 0.7399120145631068 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file