Upload folder using huggingface_hub

.__Models__SciPhi-Mistral-7B-32k/results_2024-07-02T19-17-44.679074.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.5982871937860984,
+      "acc_stderr,none": 0.004892425356375702,
+      "acc_norm,none": 0.7794264090818562,
+      "acc_norm_stderr,none": 0.004137860370785822,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 27.39318854829527,
+      "eqbench_stderr,none": 3.215602271541554,
+      "percent_parseable,none": 88.88888888888889,
+      "percent_parseable_stderr,none": 2.41033842020729,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=./Models/SciPhi-Mistral-7B-32k",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719946299.4443362,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.42.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "./Models/SciPhi-Mistral-7B-32k",
+  "model_name_sanitized": ".__Models__SciPhi-Mistral-7B-32k",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 157594.499625904,
+  "end_time": 159166.6084983,
+  "total_evaluation_time_seconds": "1572.1088723959983"
+}

.__Models__cognitivecomputations__samantha-mistral-instruct-7b/results_2024-07-02T19-23-24.931098.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.5549691296554471,
+      "acc_stderr,none": 0.004959535443170614,
+      "acc_norm,none": 0.7365066719776937,
+      "acc_norm_stderr,none": 0.004396273173717414,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 18.697258501455963,
+      "eqbench_stderr,none": 3.7651200808947474,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=./Models/cognitivecomputations/samantha-mistral-instruct-7b",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719946631.0748322,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.42.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "./Models/cognitivecomputations/samantha-mistral-instruct-7b",
+  "model_name_sanitized": ".__Models__cognitivecomputations__samantha-mistral-instruct-7b",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 157926.171562761,
+  "end_time": 159506.860535868,
+  "total_evaluation_time_seconds": "1580.6889731069969"
+}

Crimvael__Raphael-7B/results_2024-07-02T03-45-26.455365.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6527584146584345,
+      "acc_stderr,none": 0.004751203378888043,
+      "acc_norm,none": 0.8346942840071699,
+      "acc_norm_stderr,none": 0.0037069708564110657,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 68.72823492962466,
+      "eqbench_stderr,none": 2.1836213516902125,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Crimvael/Raphael-7B,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "c5a262bacfaf9f4a822954223462f428f7cb6290",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719890419.8683555,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Crimvael/Raphael-7B",
+  "model_name_sanitized": "Crimvael__Raphael-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 101714.859551874,
+  "end_time": 103228.384796742,
+  "total_evaluation_time_seconds": "1513.5252448679967"
+}

Delcos__Mistral-Pygmalion-7b/results_2024-07-02T07-11-52.058605.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.571400119498108,
+      "acc_stderr,none": 0.004938643787869521,
+      "acc_norm,none": 0.7660824536944831,
+      "acc_norm_stderr,none": 0.004224552134436904,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 17.344080969420517,
+      "eqbench_stderr,none": 3.6133795461572396,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Delcos/Mistral-Pygmalion-7b,trust_remote_code=True",
+    "model_num_parameters": 6738415616,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "2cf8706d62541ba6d647562055cdc08bc70500a1",
+    "batch_size": "auto",
+    "batch_sizes": [
+      32
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719902850.114841,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Delcos/Mistral-Pygmalion-7b",
+  "model_name_sanitized": "Delcos__Mistral-Pygmalion-7b",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 114145.16115359,
+  "end_time": 115613.988031289,
+  "total_evaluation_time_seconds": "1468.8268776990008"
+}

HuggingFaceH4__zephyr-7b-beta/results_2024-07-02T05-33-39.653334.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6398127862975503,
+      "acc_stderr,none": 0.0047907346837046125,
+      "acc_norm,none": 0.8201553475403306,
+      "acc_norm_stderr,none": 0.0038327310175919993,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 58.1357621766725,
+      "eqbench_stderr,none": 2.8035833778342916,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=HuggingFaceH4/zephyr-7b-beta,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "b70e0c9a2d9e14bd1e812d3c398e5f313e93b473",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719896897.7463036,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "HuggingFaceH4/zephyr-7b-beta",
+  "model_name_sanitized": "HuggingFaceH4__zephyr-7b-beta",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 108192.842263836,
+  "end_time": 109721.582768754,
+  "total_evaluation_time_seconds": "1528.7405049179943"
+}

KatyTheCutie__LemonadeRP-4.5.3/results_2024-07-02T08-08-46.956689.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6444931288587931,
+      "acc_stderr,none": 0.004776883632722606,
+      "acc_norm,none": 0.8265285799641505,
+      "acc_norm_stderr,none": 0.0037788044746058284,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 63.22759969511479,
+      "eqbench_stderr,none": 2.4086520534332245,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=KatyTheCutie/LemonadeRP-4.5.3,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "3f2309618a48035253889f01d4df2d7f1e81b730",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719906146.1677766,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "KatyTheCutie/LemonadeRP-4.5.3",
+  "model_name_sanitized": "KatyTheCutie__LemonadeRP-4.5.3",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 117441.278859038,
+  "end_time": 119028.886089565,
+  "total_evaluation_time_seconds": "1587.6072305270063"
+}

Norquinal__Mistral-7B-claude-chat/results_2024-07-02T07-25-06.524375.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6319458275243975,
+      "acc_stderr,none": 0.004812905279066437,
+      "acc_norm,none": 0.8306114319856602,
+      "acc_norm_stderr,none": 0.003743281749373698,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 16.33570389924275,
+      "eqbench_stderr,none": 2.9383702981155455,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.5847953216374279,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Norquinal/Mistral-7B-claude-chat,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "781aa96659a06a1c70fb1f27f68c7db3f5ab760e",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719903556.2722895,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Norquinal/Mistral-7B-claude-chat",
+  "model_name_sanitized": "Norquinal__Mistral-7B-claude-chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 114851.308841458,
+  "end_time": 116408.453788708,
+  "total_evaluation_time_seconds": "1557.1449472499953"
+}

NousResearch__Nous-Capybara-7B-V1.9/results_2024-07-02T07-40-59.772360.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6075482971519618,
+      "acc_stderr,none": 0.004872984492967986,
+      "acc_norm,none": 0.7870942043417646,
+      "acc_norm_stderr,none": 0.004085249783499773,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 19.249537416299493,
+      "eqbench_stderr,none": 3.5871216396534114,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=NousResearch/Nous-Capybara-7B-V1.9,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "ea08e10fb568f676e19e810d11d4a5ee6b3f02b3",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719904544.6768596,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "NousResearch/Nous-Capybara-7B-V1.9",
+  "model_name_sanitized": "NousResearch__Nous-Capybara-7B-V1.9",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 115839.811088593,
+  "end_time": 117361.70177151,
+  "total_evaluation_time_seconds": "1521.890682917001"
+}

SanjiWatsuki__Kunoichi-7B/results_2024-07-02T06-27-22.305698.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6803425612427804,
+      "acc_stderr,none": 0.004653907471785688,
+      "acc_norm,none": 0.8525194184425413,
+      "acc_norm_stderr,none": 0.003538596773704852,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 72.35673723130577,
+      "eqbench_stderr,none": 1.842888264461036,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=SanjiWatsuki/Kunoichi-7B,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "b2c23b9d0036e6e74e5f61de74776e9091956c83",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719900175.0852504,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "SanjiWatsuki/Kunoichi-7B",
+  "model_name_sanitized": "SanjiWatsuki__Kunoichi-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 111470.107385929,
+  "end_time": 112944.23512605,
+  "total_evaluation_time_seconds": "1474.1277401209954"
+}

SanjiWatsuki__Loyal-Macaroni-Maid-7B/results_2024-07-02T06-14-04.529485.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6699860585540729,
+      "acc_stderr,none": 0.004692567655961757,
+      "acc_norm,none": 0.8453495319657439,
+      "acc_norm_stderr,none": 0.0036083220651419597,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 73.66931196891234,
+      "eqbench_stderr,none": 1.6676417973789068,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=SanjiWatsuki/Loyal-Macaroni-Maid-7B,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "831837e474f6c474f68f3c31a62ef7eb01b9f5b7",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719899408.9462144,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "SanjiWatsuki/Loyal-Macaroni-Maid-7B",
+  "model_name_sanitized": "SanjiWatsuki__Loyal-Macaroni-Maid-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 110704.019752883,
+  "end_time": 112146.458918638,
+  "total_evaluation_time_seconds": "1442.4391657550004"
+}

SanjiWatsuki__Silicon-Maid-7B/results_2024-07-02T06-55-56.426785.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6676956781517626,
+      "acc_stderr,none": 0.0047007677417355885,
+      "acc_norm,none": 0.8455486954789883,
+      "acc_norm_stderr,none": 0.0036064226236399086,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 68.74376746724828,
+      "eqbench_stderr,none": 1.9296099363146424,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=SanjiWatsuki/Silicon-Maid-7B,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "4e43d81f3fff1091df7cb2d85e9e306d25235701",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719901923.6482406,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "SanjiWatsuki/Silicon-Maid-7B",
+  "model_name_sanitized": "SanjiWatsuki__Silicon-Maid-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 113218.750420381,
+  "end_time": 114658.35620432,
+  "total_evaluation_time_seconds": "1439.6057839390123"
+}

TeeZee__DarkSapling-7B-v2.0/results_2024-07-02T03-18-06.078821.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.637024497112129,
+      "acc_stderr,none": 0.0047987512815608575,
+      "acc_norm,none": 0.8256323441545509,
+      "acc_norm_stderr,none": 0.0037864988567691974,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 62.191078817329824,
+      "eqbench_stderr,none": 2.466355668906657,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=TeeZee/DarkSapling-7B-v2.0,trust_remote_code=True",
+    "model_num_parameters": 7241748480,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "0290b688e6aa6620f88ead6903487078ab24f96c",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719888689.565892,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "TeeZee/DarkSapling-7B-v2.0",
+  "model_name_sanitized": "TeeZee__DarkSapling-7B-v2.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 99984.687478048,
+  "end_time": 101588.008165402,
+  "total_evaluation_time_seconds": "1603.320687354004"
+}

argilla__CapybaraHermes-2.5-Mistral-7B/results_2024-07-02T04-12-24.235824.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.648177653853814,
+      "acc_stderr,none": 0.004765629263643518,
+      "acc_norm,none": 0.8303126867157936,
+      "acc_norm_stderr,none": 0.0037459074237766016,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 66.52043170781761,
+      "eqbench_stderr,none": 2.1880719529261583,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=argilla/CapybaraHermes-2.5-Mistral-7B,trust_remote_code=True",
+    "model_num_parameters": 7241748480,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "d06c86726aadd8dadb92c5b9b9e3ce8ef246c471",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719891965.445841,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    32000
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 32000,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "argilla/CapybaraHermes-2.5-Mistral-7B",
+  "model_name_sanitized": "argilla__CapybaraHermes-2.5-Mistral-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 103260.591614457,
+  "end_time": 104846.165257457,
+  "total_evaluation_time_seconds": "1585.5736430000106"
+}

berkeley-nest__Starling-LM-7B-alpha/results_2024-07-02T05-17-10.530751.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.642899820752838,
+      "acc_stderr,none": 0.004781654610857148,
+      "acc_norm,none": 0.8217486556462856,
+      "acc_norm_stderr,none": 0.0038194200585540956,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 69.78303111392624,
+      "eqbench_stderr,none": 2.1554319334518173,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=berkeley-nest/Starling-LM-7B-alpha,trust_remote_code=True",
+    "model_num_parameters": 7241748480,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "1dddf3b95bc1391f6307299eb1c162c194bde9bd",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719895983.4735777,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|end_of_turn|>",
+    32000
+  ],
+  "tokenizer_eos_token": [
+    "<|end_of_turn|>",
+    32000
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 32000,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "berkeley-nest/Starling-LM-7B-alpha",
+  "model_name_sanitized": "berkeley-nest__Starling-LM-7B-alpha",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 107278.552702596,
+  "end_time": 108732.460166266,
+  "total_evaluation_time_seconds": "1453.9074636699952"
+}

cgato__Thespis-Mistral-7b-v0.6/results_2024-07-02T03-55-19.886617.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6221868153754232,
+      "acc_stderr,none": 0.004838496966823936,
+      "acc_norm,none": 0.818263294164509,
+      "acc_norm_stderr,none": 0.003848392656939309,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 29.12698576180375,
+      "eqbench_stderr,none": 3.2541125218508933,
+      "percent_parseable,none": 79.53216374269006,
+      "percent_parseable_stderr,none": 3.0944459778533218,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=cgato/Thespis-Mistral-7b-v0.6,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "8f02867239bb861de358813db56bd68440534553",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719891027.5687327,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "cgato/Thespis-Mistral-7b-v0.6",
+  "model_name_sanitized": "cgato__Thespis-Mistral-7b-v0.6",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 102322.69548256,
+  "end_time": 103821.816040654,
+  "total_evaluation_time_seconds": "1499.1205580940004"
+}

chargoddard__loyal-piano-m7/results_2024-07-02T04-51-42.336742.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6439952200756821,
+      "acc_stderr,none": 0.004778380758851119,
+      "acc_norm,none": 0.8328022306313483,
+      "acc_norm_stderr,none": 0.003723897305645462,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 72.14962256589271,
+      "eqbench_stderr,none": 2.006126552035965,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=chargoddard/loyal-piano-m7,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "9ad4f1ce2cf75c3ab54af6f9872356cb199b330e",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719894335.7498412,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "chargoddard/loyal-piano-m7",
+  "model_name_sanitized": "chargoddard__loyal-piano-m7",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 105630.829033957,
+  "end_time": 107204.26616981,
+  "total_evaluation_time_seconds": "1573.4371358530043"
+}

cognitivecomputations__dolphin-2.2.1-mistral-7b/results_2024-07-02T06-02-40.816103.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6285600477992431,
+      "acc_stderr,none": 0.004822022254886004,
+      "acc_norm,none": 0.8146783509261103,
+      "acc_norm_stderr,none": 0.003877641746375665,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 67.32561567936949,
+      "eqbench_stderr,none": 2.414051136188407,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=cognitivecomputations/dolphin-2.2.1-mistral-7b,trust_remote_code=True",
+    "model_num_parameters": 7241748480,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "20f78ab87598cab137c8ce00855464cae403a3fd",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719898636.197451,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    32000
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 32000,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "cognitivecomputations/dolphin-2.2.1-mistral-7b",
+  "model_name_sanitized": "cognitivecomputations__dolphin-2.2.1-mistral-7b",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 109931.241971652,
+  "end_time": 111462.745517669,
+  "total_evaluation_time_seconds": "1531.5035460170038"
+}

cognitivecomputations__dolphin-2.6-mistral-7b-dpo-laser/results_2024-07-02T04-21-44.877903.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6483768173670583,
+      "acc_stderr,none": 0.004765012078929417,
+      "acc_norm,none": 0.8360884285998805,
+      "acc_norm_stderr,none": 0.003694387361177522,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 71.57020405475217,
+      "eqbench_stderr,none": 2.0257691517275624,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.5847953216374273,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=cognitivecomputations/dolphin-2.6-mistral-7b-dpo-laser,trust_remote_code=True",
+    "model_num_parameters": 7241740288,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "95999a69464ffcf4e25854208271000dfc255696",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719892564.0480487,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|im_end|>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "cognitivecomputations/dolphin-2.6-mistral-7b-dpo-laser",
+  "model_name_sanitized": "cognitivecomputations__dolphin-2.6-mistral-7b-dpo-laser",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 103859.073714916,
+  "end_time": 105406.807341044,
+  "total_evaluation_time_seconds": "1547.7336261279997"
+}

head-empty-ai__Mytho-Lemon-11B/results_2024-07-02T03-27-57.446245.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6298546106353317,
+      "acc_stderr,none": 0.004818566366066908,
+      "acc_norm,none": 0.7977494523003386,
+      "acc_norm_stderr,none": 0.0040085714314836915,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 60.025207476634726,
+      "eqbench_stderr,none": 2.7274520011253087,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=head-empty-ai/Mytho-Lemon-11B,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "793a2d1bf7fe26da4824e13b7992465f2f4765d6",
+    "batch_size": "auto",
+    "batch_sizes": [
+      8
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719888600.6598475,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "head-empty-ai/Mytho-Lemon-11B",
+  "model_name_sanitized": "head-empty-ai__Mytho-Lemon-11B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 99895.742790118,
+  "end_time": 102179.375651272,
+  "total_evaluation_time_seconds": "2283.632861153994"
+}

maywell__Synatra-7B-v0.3-RP/results_2024-07-02T06-46-15.142587.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6164110734913364,
+      "acc_stderr,none": 0.004852658876775384,
+      "acc_norm,none": 0.8046205935072694,
+      "acc_norm_stderr,none": 0.0039568217050184535,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 54.93035121530972,
+      "eqbench_stderr,none": 2.672374443919001,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=maywell/Synatra-7B-v0.3-RP,trust_remote_code=True",
+    "model_num_parameters": 7241748480,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "a994747e68972f9018cd454730174211f9e46736",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719901236.0567749,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    32000
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 32000,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "maywell/Synatra-7B-v0.3-RP",
+  "model_name_sanitized": "maywell__Synatra-7B-v0.3-RP",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 112531.153671088,
+  "end_time": 114077.071988352,
+  "total_evaluation_time_seconds": "1545.9183172640041"
+}

mistralai__Mistral-7B-Instruct-v0.1/results_2024-07-02T04-41-17.557455.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.5630352519418442,
+      "acc_stderr,none": 0.004949969363017642,
+      "acc_norm,none": 0.7466640111531567,
+      "acc_norm_stderr,none": 0.004340328204135102,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 46.82017378717466,
+      "eqbench_stderr,none": 3.211082755225061,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=mistralai/Mistral-7B-Instruct-v0.1,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "86370fc1f5e0aa51b50dcdf6eada80697b570099",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719893767.2215395,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "mistralai/Mistral-7B-Instruct-v0.1",
+  "model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.1",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 105062.200308793,
+  "end_time": 106579.486870813,
+  "total_evaluation_time_seconds": "1517.2865620199882"
+}

mistralai__Mistral-7B-Instruct-v0.2/results_2024-07-02T05-07-32.922766.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6609241187014538,
+      "acc_stderr,none": 0.004724281487819372,
+      "acc_norm,none": 0.8365863373829915,
+      "acc_norm_stderr,none": 0.0036898701424130766,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 65.49565100216773,
+      "eqbench_stderr,none": 2.53483923149953,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.5847953216374286,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=mistralai/Mistral-7B-Instruct-v0.2,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "99259002b41e116d28ccb2d04a9fbe22baed0c7f",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719895327.2738423,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "mistralai/Mistral-7B-Instruct-v0.2",
+  "model_name_sanitized": "mistralai__Mistral-7B-Instruct-v0.2",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 106622.397945177,
+  "end_time": 108154.852197086,
+  "total_evaluation_time_seconds": "1532.4542519090028"
+}

teknium__Hermes-Trismegistus-Mistral-7B/results_2024-07-02T05-46-44.024042.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.612427803226449,
+      "acc_stderr,none": 0.004862003566798519,
+      "acc_norm,none": 0.7921728739294961,
+      "acc_norm_stderr,none": 0.00404923158643312,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 64.86049373791583,
+      "eqbench_stderr,none": 2.1996109709512552,
+      "percent_parseable,none": 98.83040935672514,
+      "percent_parseable_stderr,none": 0.8245894595446,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=teknium/Hermes-Trismegistus-Mistral-7B,trust_remote_code=True",
+    "model_num_parameters": 7241748480,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "d5757feda9c0c62c7a56fdc2eff7d5041c970a83",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719897658.008983,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    32000
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 32000,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "teknium/Hermes-Trismegistus-Mistral-7B",
+  "model_name_sanitized": "teknium__Hermes-Trismegistus-Mistral-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 108953.100902165,
+  "end_time": 110505.95347585,
+  "total_evaluation_time_seconds": "1552.8525736849988"
+}