Upload folder using huggingface_hub

ABX-AI__Silver-Sun-v2-11B/results_2024-07-02T00-46-34.040470.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6798446524596694,
+      "acc_stderr,none": 0.0046558259808919715,
+      "acc_norm,none": 0.8639713204540929,
+      "acc_norm_stderr,none": 0.003421183909320265,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 69.92376298818061,
+      "eqbench_stderr,none": 2.302476504280005,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=ABX-AI/Silver-Sun-v2-11B,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "052f5514e25a2c0d3622f2aa84c9662ebca41eba",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719878990.231564,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "ABX-AI/Silver-Sun-v2-11B",
+  "model_name_sanitized": "ABX-AI__Silver-Sun-v2-11B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 90285.382371444,
+  "end_time": 92495.969871496,
+  "total_evaluation_time_seconds": "2210.5875000520027"
+}

BlueNipples__SnowLotus-v2-10.7B/results_2024-07-01T22-45-32.913168.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6488747261501693,
+      "acc_stderr,none": 0.004763465139038578,
+      "acc_norm,none": 0.834196375224059,
+      "acc_norm_stderr,none": 0.0037114419828661186,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 60.53953285457271,
+      "eqbench_stderr,none": 2.6096565956249744,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.5847953216374273,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=BlueNipples/SnowLotus-v2-10.7B,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "e655cc010d33dca9a7ac37a640ea45b82fef054a",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719871605.6418686,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "BlueNipples/SnowLotus-v2-10.7B",
+  "model_name_sanitized": "BlueNipples__SnowLotus-v2-10.7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 82900.70551701,
+  "end_time": 85234.842583591,
+  "total_evaluation_time_seconds": "2334.1370665809955"
+}

FallenMerick__Chewy-Lemon-Cookie-11B/.ipynb_checkpoints/results_2024-07-01T19-12-13.115090-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6618203545110536,
+      "acc_stderr,none": 0.004721231637092694,
+      "acc_norm,none": 0.843855805616411,
+      "acc_norm_stderr,none": 0.0036225013703318895,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 76.2370547740446,
+      "eqbench_stderr,none": 1.7148952450726893,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FallenMerick/Chewy-Lemon-Cookie-11B,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "0f5d0d6d218b3ef034f58eba32d6fe7ac4c237ae",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719858887.6024628,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "FallenMerick/Chewy-Lemon-Cookie-11B",
+  "model_name_sanitized": "FallenMerick__Chewy-Lemon-Cookie-11B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 70182.562797015,
+  "end_time": 72435.044500542,
+  "total_evaluation_time_seconds": "2252.48170352701"
+}

FallenMerick__Chewy-Lemon-Cookie-11B/results_2024-07-01T19-12-13.115090.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6618203545110536,
+      "acc_stderr,none": 0.004721231637092694,
+      "acc_norm,none": 0.843855805616411,
+      "acc_norm_stderr,none": 0.0036225013703318895,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 76.2370547740446,
+      "eqbench_stderr,none": 1.7148952450726893,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FallenMerick/Chewy-Lemon-Cookie-11B,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "0f5d0d6d218b3ef034f58eba32d6fe7ac4c237ae",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719858887.6024628,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "FallenMerick/Chewy-Lemon-Cookie-11B",
+  "model_name_sanitized": "FallenMerick__Chewy-Lemon-Cookie-11B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 70182.562797015,
+  "end_time": 72435.044500542,
+  "total_evaluation_time_seconds": "2252.48170352701"
+}

FallenMerick__Chunky-Lemon-Cookie-11B/.ipynb_checkpoints/results_2024-07-01T18-34-32.911166-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6622186815375424,
+      "acc_stderr,none": 0.004719870074967253,
+      "acc_norm,none": 0.8435570603465445,
+      "acc_norm_stderr,none": 0.0036253232211662535,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 76.2907953282312,
+      "eqbench_stderr,none": 1.7296444904041766,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FallenMerick/Chunky-Lemon-Cookie-11B,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "849afd37a9995d7a88503e4ef4847c5d9d239e2a",
+    "batch_size": "auto",
+    "batch_sizes": [
+      8
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719856428.7962418,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "FallenMerick/Chunky-Lemon-Cookie-11B",
+  "model_name_sanitized": "FallenMerick__Chunky-Lemon-Cookie-11B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 67723.856639362,
+  "end_time": 70174.840569089,
+  "total_evaluation_time_seconds": "2450.983929727008"
+}

FallenMerick__Chunky-Lemon-Cookie-11B/results_2024-07-01T18-34-32.911166.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6622186815375424,
+      "acc_stderr,none": 0.004719870074967253,
+      "acc_norm,none": 0.8435570603465445,
+      "acc_norm_stderr,none": 0.0036253232211662535,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 76.2907953282312,
+      "eqbench_stderr,none": 1.7296444904041766,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FallenMerick/Chunky-Lemon-Cookie-11B,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "849afd37a9995d7a88503e4ef4847c5d9d239e2a",
+    "batch_size": "auto",
+    "batch_sizes": [
+      8
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719856428.7962418,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "FallenMerick/Chunky-Lemon-Cookie-11B",
+  "model_name_sanitized": "FallenMerick__Chunky-Lemon-Cookie-11B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 67723.856639362,
+  "end_time": 70174.840569089,
+  "total_evaluation_time_seconds": "2450.983929727008"
+}

Himitsui__KuroMitsu-11B/results_2024-07-01T22-05-02.101817.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6841266679944235,
+      "acc_stderr,none": 0.004639126951051454,
+      "acc_norm,none": 0.8632742481577375,
+      "acc_norm_stderr,none": 0.0034285545959502227,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 70.49864838913086,
+      "eqbench_stderr,none": 2.176357434703893,
+      "percent_parseable,none": 98.83040935672514,
+      "percent_parseable_stderr,none": 0.824589459544595,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Himitsui/KuroMitsu-11B,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "7bd8487fc3a5c3bac022bfe8c34d2f630c123d40",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719869292.2336426,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Himitsui/KuroMitsu-11B",
+  "model_name_sanitized": "Himitsui__KuroMitsu-11B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 80587.309586819,
+  "end_time": 82804.031242981,
+  "total_evaluation_time_seconds": "2216.7216561620007"
+}

Intel__neural-chat-7b-v3-1/.ipynb_checkpoints/results_2024-06-27T21-55-55.507233-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6323441545508863,
+      "acc_stderr,none": 0.004811815959388812,
+      "acc_norm,none": 0.7975502887870942,
+      "acc_norm_stderr,none": 0.004010043978333027,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 62.2626522660805,
+      "eqbench_stderr,none": 2.2134366454600554,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Intel/neural-chat-7b-v3-1,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "c0d379a49c1c0579529d5e6f2e936ddb759552a8",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719523797.1185606,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Intel/neural-chat-7b-v3-1",
+  "model_name_sanitized": "Intel__neural-chat-7b-v3-1",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 76180.270750278,
+  "end_time": 77745.57190531,
+  "total_evaluation_time_seconds": "1565.301155032008"
+}

KoboldAI__Mistral-7B-Erebus-v3/.ipynb_checkpoints/results_2024-06-28T02-01-18.290687-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.5837482573192591,
+      "acc_stderr,none": 0.0049192891130275095,
+      "acc_norm,none": 0.7665803624775941,
+      "acc_norm_stderr,none": 0.004221424792919153,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 18.19761609584577,
+      "eqbench_stderr,none": 3.474273216617232,
+      "percent_parseable,none": 97.6608187134503,
+      "percent_parseable_stderr,none": 1.1592247905734945,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=KoboldAI/Mistral-7B-Erebus-v3,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "476c2eed031028f2fcfb9b8d0115b83363cec904",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719538494.5098195,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "KoboldAI/Mistral-7B-Erebus-v3",
+  "model_name_sanitized": "KoboldAI__Mistral-7B-Erebus-v3",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 90877.776869387,
+  "end_time": 92468.355496828,
+  "total_evaluation_time_seconds": "1590.578627440991"
+}

KoboldAI__Mistral-7B-Holodeck-1/.ipynb_checkpoints/results_2024-06-28T01-04-59.368025-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6026687910774746,
+      "acc_stderr,none": 0.004883455188908956,
+      "acc_norm,none": 0.7918741286596296,
+      "acc_norm_stderr,none": 0.0040513767194979506,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 2.099910527905425,
+      "eqbench_stderr,none": 2.491702523648299,
+      "percent_parseable,none": 98.24561403508773,
+      "percent_parseable_stderr,none": 1.0069193740062292,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=KoboldAI/Mistral-7B-Holodeck-1,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "76057cc5c1923921162133c81ae7ca0e92755810",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719535119.7065547,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "KoboldAI/Mistral-7B-Holodeck-1",
+  "model_name_sanitized": "KoboldAI__Mistral-7B-Holodeck-1",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 87502.926965946,
+  "end_time": 89089.432844292,
+  "total_evaluation_time_seconds": "1586.5058783459972"
+}

NeverSleep__Mistral-11B-SynthIAirOmniMix/results_2024-07-01T23-28-29.609057.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6276638119896435,
+      "acc_stderr,none": 0.00482439307682663,
+      "acc_norm,none": 0.8157737502489544,
+      "acc_norm_stderr,none": 0.0038687630867377757,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 55.19425772388985,
+      "eqbench_stderr,none": 2.8835673004687723,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=NeverSleep/Mistral-11B-SynthIAirOmniMix,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "639cca3fff101891a70eb2f12fc835598ed50eb0",
+    "batch_size": "auto",
+    "batch_sizes": [
+      8
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719874190.80644,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "NeverSleep/Mistral-11B-SynthIAirOmniMix",
+  "model_name_sanitized": "NeverSleep__Mistral-11B-SynthIAirOmniMix",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 85485.875698911,
+  "end_time": 87811.538471,
+  "total_evaluation_time_seconds": "2325.66277208901"
+}

NousResearch__Hermes-2-Pro-Mistral-7B/.ipynb_checkpoints/results_2024-06-28T00-36-44.931474-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6267675761800439,
+      "acc_stderr,none": 0.004826746160830164,
+      "acc_norm,none": 0.8055168293168692,
+      "acc_norm_stderr,none": 0.003949933997955457,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 65.92538496559615,
+      "eqbench_stderr,none": 2.278499695256187,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=NousResearch/Hermes-2-Pro-Mistral-7B,trust_remote_code=True",
+    "model_num_parameters": 7241994240,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "09317b1d8da639b5d9af77c06aa17cde0f0f91c0",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719533491.7175071,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    32000
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 32000,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "NousResearch/Hermes-2-Pro-Mistral-7B",
+  "model_name_sanitized": "NousResearch__Hermes-2-Pro-Mistral-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 85874.893829605,
+  "end_time": 87394.996271898,
+  "total_evaluation_time_seconds": "1520.1024422929913"
+}

NousResearch__Nous-Hermes-2-SOLAR-10.7B/results_2024-07-01T22-46-11.267534.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6467835092611034,
+      "acc_stderr,none": 0.004769924131304646,
+      "acc_norm,none": 0.8324039036048596,
+      "acc_norm_stderr,none": 0.0037274387865133193,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 63.524669890379585,
+      "eqbench_stderr,none": 2.5941253832883335,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=NousResearch/Nous-Hermes-2-SOLAR-10.7B,trust_remote_code=True",
+    "model_num_parameters": 10731540480,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "14c1fbe2f71acdcd58247b30d5439bd572d52386",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719871760.8771381,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    32000
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 32000,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "NousResearch/Nous-Hermes-2-SOLAR-10.7B",
+  "model_name_sanitized": "NousResearch__Nous-Hermes-2-SOLAR-10.7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 83055.962424888,
+  "end_time": 85273.196902306,
+  "total_evaluation_time_seconds": "2217.2344774179946"
+}

Open-Orca__Mistral-7B-OpenOrca/.ipynb_checkpoints/results_2024-06-27T21-00-54.306241-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6379207329217288,
+      "acc_stderr,none": 0.004796193584930065,
+      "acc_norm,none": 0.8166699860585541,
+      "acc_norm_stderr,none": 0.0038614605262315377,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 63.978950638437865,
+      "eqbench_stderr,none": 2.3824356593314344,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.5847953216374284,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Open-Orca/Mistral-7B-OpenOrca,trust_remote_code=True",
+    "model_num_parameters": 7241748480,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "4a37328cef00f524d3791b1c0cc559a3cc6af14d",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719520557.5287726,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    32000
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 32000,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Open-Orca/Mistral-7B-OpenOrca",
+  "model_name_sanitized": "Open-Orca__Mistral-7B-OpenOrca",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 72940.705278236,
+  "end_time": 74444.371073833,
+  "total_evaluation_time_seconds": "1503.6657955970004"
+}

SanjiWatsuki__Kunoichi-7B/.ipynb_checkpoints/results_2024-06-27T20-34-47.197919-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6803425612427804,
+      "acc_stderr,none": 0.004653907471785688,
+      "acc_norm,none": 0.8525194184425413,
+      "acc_norm_stderr,none": 0.003538596773704852,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 72.35673723130577,
+      "eqbench_stderr,none": 1.842888264461036,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=SanjiWatsuki/Kunoichi-7B,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "b2c23b9d0036e6e74e5f61de74776e9091956c83",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719519064.691441,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "SanjiWatsuki/Kunoichi-7B",
+  "model_name_sanitized": "SanjiWatsuki__Kunoichi-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 71447.838088771,
+  "end_time": 72877.26274353,
+  "total_evaluation_time_seconds": "1429.424654759001"
+}

Sao10K__Fimbulvetr-10.7B-v1/results_2024-07-01T21-25-41.128938.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6694881497709619,
+      "acc_stderr,none": 0.004694360968929443,
+      "acc_norm,none": 0.8580959968133838,
+      "acc_norm_stderr,none": 0.0034823849566329064,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 65.41948210475555,
+      "eqbench_stderr,none": 2.4500037057733617,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Sao10K/Fimbulvetr-10.7B-v1,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "30d93aaba30d8b9eba0ce46fb68a468ea242174a",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719866827.6673388,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Sao10K/Fimbulvetr-10.7B-v1",
+  "model_name_sanitized": "Sao10K__Fimbulvetr-10.7B-v1",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 78122.684526781,
+  "end_time": 80443.05835636,
+  "total_evaluation_time_seconds": "2320.3738295789954"
+}

Sao10K__Fimbulvetr-11B-v2/.ipynb_checkpoints/results_2024-06-28T04-32-22.127106-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.68123879705238,
+      "acc_stderr,none": 0.004650438781745276,
+      "acc_norm,none": 0.8660625373431587,
+      "acc_norm_stderr,none": 0.003398890525229556,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 70.00837363646892,
+      "eqbench_stderr,none": 2.230997557081673,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.5847953216374293,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Sao10K/Fimbulvetr-11B-v2",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "b2dcd534dc3a53ff84e60a53b87816185169be19",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719546844.0477293,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Sao10K/Fimbulvetr-11B-v2",
+  "model_name_sanitized": "Sao10K__Fimbulvetr-11B-v2",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 99227.279509843,
+  "end_time": 101532.191916139,
+  "total_evaluation_time_seconds": "2304.912406295989"
+}

Sao10K__Frostwind-10.7B-v1/results_2024-07-01T20-07-44.450930.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.644991037641904,
+      "acc_stderr,none": 0.004775380866948014,
+      "acc_norm,none": 0.8414658434574785,
+      "acc_norm_stderr,none": 0.0036449467300447647,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 55.726607928145825,
+      "eqbench_stderr,none": 2.887808692501527,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.5847953216374293,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Sao10K/Frostwind-10.7B-v1,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5b465f636e1d354718e393e85914865a64840903",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719862245.485066,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Sao10K/Frostwind-10.7B-v1",
+  "model_name_sanitized": "Sao10K__Frostwind-10.7B-v1",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 73540.501016482,
+  "end_time": 75766.380380746,
+  "total_evaluation_time_seconds": "2225.8793642640085"
+}

Sao10K__Solstice-11B-v1/results_2024-07-01T20-47-26.616675.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6811392152957578,
+      "acc_stderr,none": 0.004650825168905211,
+      "acc_norm,none": 0.8641704839673372,
+      "acc_norm_stderr,none": 0.003419072480735363,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 68.23650356231931,
+      "eqbench_stderr,none": 2.2992289426525265,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.584795321637429,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Sao10K/Solstice-11B-v1,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "6db53c612c43e7b34c2f76a662abbd8e19b0c5f6",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719864640.8746834,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Sao10K/Solstice-11B-v1",
+  "model_name_sanitized": "Sao10K__Solstice-11B-v1",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 75935.867073192,
+  "end_time": 78148.546093816,
+  "total_evaluation_time_seconds": "2212.6790206239966"
+}

TheDrummer__Moistral-11B-v3/results_2024-07-02T00-08-37.869624.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6810396335391357,
+      "acc_stderr,none": 0.004651211311633819,
+      "acc_norm,none": 0.8664608643696475,
+      "acc_norm_stderr,none": 0.003394613020442028,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 69.75227204221265,
+      "eqbench_stderr,none": 2.2281362165770795,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.5847953216374293,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=TheDrummer/Moistral-11B-v3,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "c061f49aad10acb47e40cf63e7840a906a3ad2c1",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719876604.3651178,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "TheDrummer/Moistral-11B-v3",
+  "model_name_sanitized": "TheDrummer__Moistral-11B-v3",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 87899.523081436,
+  "end_time": 90219.799023715,
+  "total_evaluation_time_seconds": "2320.275942278997"
+}

Undi95__Borealis-10.7B/results_2024-07-02T01-25-42.423826.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.5981876120294762,
+      "acc_stderr,none": 0.00489262449093723,
+      "acc_norm,none": 0.7957578171678948,
+      "acc_norm_stderr,none": 0.0040232284614061305,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 8.265798945931458,
+      "eqbench_stderr,none": 2.400702056761244,
+      "percent_parseable,none": 44.44444444444444,
+      "percent_parseable_stderr,none": 3.811079669833523,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Undi95/Borealis-10.7B,trust_remote_code=True",
+    "model_num_parameters": 10731556864,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "b2c1ed5e4b64dc8499e7c4a5bcc0c7fefed738cb",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719881280.6823523,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    32000
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 32000,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Undi95/Borealis-10.7B",
+  "model_name_sanitized": "Undi95__Borealis-10.7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 92575.714752079,
+  "end_time": 94844.35324392,
+  "total_evaluation_time_seconds": "2268.638491841004"
+}

Undi95__Toppy-M-7B/.ipynb_checkpoints/results_2024-06-28T02-28-16.478931-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6571400119498108,
+      "acc_stderr,none": 0.00473695081061781,
+      "acc_norm,none": 0.8351921927902808,
+      "acc_norm_stderr,none": 0.003702487662126953,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 66.56565114431275,
+      "eqbench_stderr,none": 2.1832557339862837,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=Undi95/Toppy-M-7B,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5d0c492effbb2e52ea04e5100c6ce02eba48a793",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719540198.0639265,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "Undi95/Toppy-M-7B",
+  "model_name_sanitized": "Undi95__Toppy-M-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 92581.210827571,
+  "end_time": 94086.543624108,
+  "total_evaluation_time_seconds": "1505.3327965369972"
+}

athirdpath__NSFW_DPO_vmgb-7b/.ipynb_checkpoints/results_2024-06-28T02-55-12.160237-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6730730930093607,
+      "acc_stderr,none": 0.0046813160644444095,
+      "acc_norm,none": 0.8536148177653854,
+      "acc_norm_stderr,none": 0.0035276951498235012,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 74.82935321697907,
+      "eqbench_stderr,none": 1.6591997145588517,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=athirdpath/NSFW_DPO_vmgb-7b,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "b667d7810267685cd4f32a8f82044e419c010abe",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719541817.2830012,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "athirdpath/NSFW_DPO_vmgb-7b",
+  "model_name_sanitized": "athirdpath__NSFW_DPO_vmgb-7b",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 94200.513127129,
+  "end_time": 95702.225070561,
+  "total_evaluation_time_seconds": "1501.7119434320048"
+}

backyardai__Fimbulvetr-Holodeck-Erebus-Westlake-10.7B/results_2024-07-02T00-45-44.704724.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6837283409679347,
+      "acc_stderr,none": 0.004640699483543287,
+      "acc_norm,none": 0.8599880501892053,
+      "acc_norm_stderr,none": 0.0034629026011361993,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 69.25278425131637,
+      "eqbench_stderr,none": 2.0742624474888856,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=backyardai/Fimbulvetr-Holodeck-Erebus-Westlake-10.7B,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "e47cd0e9dd63b41f649b8da31651689067ac217b",
+    "batch_size": "auto",
+    "batch_sizes": [
+      8
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719878717.9026196,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "backyardai/Fimbulvetr-Holodeck-Erebus-Westlake-10.7B",
+  "model_name_sanitized": "backyardai__Fimbulvetr-Holodeck-Erebus-Westlake-10.7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 90012.968120256,
+  "end_time": 92446.634152182,
+  "total_evaluation_time_seconds": "2433.6660319259972"
+}

froggeric__WestLake-10.7B-v2/results_2024-07-01T22-07-10.044094.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6982672774347739,
+      "acc_stderr,none": 0.004580718115992501,
+      "acc_norm,none": 0.8673571001792472,
+      "acc_norm_stderr,none": 0.0033849518032134326,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 73.35343055205183,
+      "eqbench_stderr,none": 2.018556635561309,
+      "percent_parseable,none": 95.32163742690058,
+      "percent_parseable_stderr,none": 1.619638995623559,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=froggeric/WestLake-10.7B-v2,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "de1f0f286ef6d5a6e10627ac05f8cfb9baaa36a5",
+    "batch_size": "auto",
+    "batch_sizes": [
+      8
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719869203.7996294,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "froggeric/WestLake-10.7B-v2",
+  "model_name_sanitized": "froggeric__WestLake-10.7B-v2",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 80498.777827703,
+  "end_time": 82931.973511543,
+  "total_evaluation_time_seconds": "2433.195683839993"
+}

jondurbin__airoboros-m-7b-3.1.2/.ipynb_checkpoints/results_2024-06-27T21-27-37.734965-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6262696673969329,
+      "acc_stderr,none": 0.0048280457747349,
+      "acc_norm,none": 0.8133837880900219,
+      "acc_norm_stderr,none": 0.0038880689432920544,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 38.51541437922841,
+      "eqbench_stderr,none": 3.511567923871361,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=jondurbin/airoboros-m-7b-3.1.2,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "e9a7f0271fa442d65bf6be87feeb3f4de2f5760e",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719522153.290284,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "jondurbin/airoboros-m-7b-3.1.2",
+  "model_name_sanitized": "jondurbin__airoboros-m-7b-3.1.2",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 74536.468290869,
+  "end_time": 76047.79978283,
+  "total_evaluation_time_seconds": "1511.3314919609984"
+}

jondurbin__cinematika-7b-v0.1/.ipynb_checkpoints/results_2024-06-27T23-16-51.732979-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6138219478191596,
+      "acc_stderr,none": 0.004858771963468838,
+      "acc_norm,none": 0.8031268671579367,
+      "acc_norm_stderr,none": 0.00396822985262125,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 44.84948931109151,
+      "eqbench_stderr,none": 3.1571076496385277,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=jondurbin/cinematika-7b-v0.1,trust_remote_code=True",
+    "model_num_parameters": 7241756672,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "6df1846af7de7ab8e2201ad87071ed661e3b0de2",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719528705.1154015,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "[PAD]",
+    32000
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "jondurbin/cinematika-7b-v0.1",
+  "model_name_sanitized": "jondurbin__cinematika-7b-v0.1",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 81088.287076101,
+  "end_time": 82601.797810444,
+  "total_evaluation_time_seconds": "1513.510734342999"
+}

kyujinpy__SOLAR-Platypus-10.7B-v2/results_2024-07-02T00-03-58.332402.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6271659032065325,
+      "acc_stderr,none": 0.004825702533920419,
+      "acc_norm,none": 0.8204540928101972,
+      "acc_norm_stderr,none": 0.0038302437643328983,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 25.109901450892778,
+      "eqbench_stderr,none": 2.902801491896888,
+      "percent_parseable,none": 45.6140350877193,
+      "percent_parseable_stderr,none": 3.820042586602962,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=kyujinpy/SOLAR-Platypus-10.7B-v2,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "2a08546624dd10d139dd030f0ce7cf2199dd7a6a",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719876463.3489583,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "kyujinpy/SOLAR-Platypus-10.7B-v2",
+  "model_name_sanitized": "kyujinpy__SOLAR-Platypus-10.7B-v2",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 87758.502834899,
+  "end_time": 89940.261818391,
+  "total_evaluation_time_seconds": "2181.7589834919927"
+}

kyujinpy__SOLAR-Platypus-10.7B-v2/results_2024-07-02T01-23-06.754731.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6271659032065325,
+      "acc_stderr,none": 0.004825702533920419,
+      "acc_norm,none": 0.8204540928101972,
+      "acc_norm_stderr,none": 0.0038302437643328983,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 25.109901450892778,
+      "eqbench_stderr,none": 2.902801491896888,
+      "percent_parseable,none": 45.6140350877193,
+      "percent_parseable_stderr,none": 3.820042586602962,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=kyujinpy/SOLAR-Platypus-10.7B-v2,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "2a08546624dd10d139dd030f0ce7cf2199dd7a6a",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719881209.3548667,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "kyujinpy/SOLAR-Platypus-10.7B-v2",
+  "model_name_sanitized": "kyujinpy__SOLAR-Platypus-10.7B-v2",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 92504.079796018,
+  "end_time": 94688.684219003,
+  "total_evaluation_time_seconds": "2184.604422984994"
+}

migtissera__Synthia-7B-v3.0/.ipynb_checkpoints/results_2024-06-27T22-50-03.654626-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6188010356502689,
+      "acc_stderr,none": 0.004846886929763468,
+      "acc_norm,none": 0.8173670583549094,
+      "acc_norm_stderr,none": 0.0038557568514416335,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 15.026068685500109,
+      "eqbench_stderr,none": 2.8920476087351132,
+      "percent_parseable,none": 94.73684210526316,
+      "percent_parseable_stderr,none": 1.7126088775157098,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=migtissera/Synthia-7B-v3.0,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "93c2e8b8055b42779f2b68059ebe38af6f2789c4",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719527019.219691,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "migtissera/Synthia-7B-v3.0",
+  "model_name_sanitized": "migtissera__Synthia-7B-v3.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 79402.540235241,
+  "end_time": 80993.719447117,
+  "total_evaluation_time_seconds": "1591.1792118759913"
+}

migtissera__Tess-10.7B-v1.5b/results_2024-07-01T21-27-23.093748.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6458872734515037,
+      "acc_stderr,none": 0.004772661659628842,
+      "acc_norm,none": 0.8381796454889464,
+      "acc_norm_stderr,none": 0.0036753325906809194,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 61.834945599309265,
+      "eqbench_stderr,none": 2.695505491187219,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.5847953216374285,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=migtissera/Tess-10.7B-v1.5b,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "67fd613302b31ff4a60e320685813584851a214f",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719867017.0855958,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "migtissera/Tess-10.7B-v1.5b",
+  "model_name_sanitized": "migtissera__Tess-10.7B-v1.5b",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 78312.095615139,
+  "end_time": 80545.023168157,
+  "total_evaluation_time_seconds": "2232.927553018002"
+}

mlabonne__NeuralBeagle14-7B/.ipynb_checkpoints/results_2024-06-28T00-10-47.687175-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.7003584943238399,
+      "acc_stderr,none": 0.004571647137441099,
+      "acc_norm,none": 0.8645688109938259,
+      "acc_norm_stderr,none": 0.003414842236516961,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 74.20803234078544,
+      "eqbench_stderr,none": 1.9057062958788094,
+      "percent_parseable,none": 99.41520467836257,
+      "percent_parseable_stderr,none": 0.5847953216374271,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=mlabonne/NeuralBeagle14-7B,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "1567ad618a0998139654cb355738bb9bc018ca64",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719531860.3285184,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "mlabonne/NeuralBeagle14-7B",
+  "model_name_sanitized": "mlabonne__NeuralBeagle14-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 84243.603381348,
+  "end_time": 85837.751968409,
+  "total_evaluation_time_seconds": "1594.1485870609904"
+}

rwitz__go-bruins/.ipynb_checkpoints/results_2024-06-27T22-21-09.060416-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6664011153156741,
+      "acc_stderr,none": 0.0047053471376996584,
+      "acc_norm,none": 0.8492332204740092,
+      "acc_norm_stderr,none": 0.0035709011883580865,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 73.61594324522352,
+      "eqbench_stderr,none": 1.8000447804156592,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=rwitz/go-bruins,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "27b510cc158d83cad4f4df4f5cee65353647e080",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719525369.5356786,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "rwitz/go-bruins",
+  "model_name_sanitized": "rwitz__go-bruins",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 77752.802678043,
+  "end_time": 79259.12526582,
+  "total_evaluation_time_seconds": "1506.322587777002"
+}

saishf__Fimbulvetr-Kuro-Lotus-10.7B/results_2024-07-01T23-24-05.421876.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.676458872734515,
+      "acc_stderr,none": 0.0046687106891924075,
+      "acc_norm,none": 0.86247759410476,
+      "acc_norm_stderr,none": 0.0034369416417827595,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 65.8510580469694,
+      "eqbench_stderr,none": 2.304299203691162,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=saishf/Fimbulvetr-Kuro-Lotus-10.7B,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "ec1288fd8c06ac408a2a7e503ea62ac300e474e1",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:1",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719874027.8025355,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "saishf/Fimbulvetr-Kuro-Lotus-10.7B",
+  "model_name_sanitized": "saishf__Fimbulvetr-Kuro-Lotus-10.7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 85322.658459155,
+  "end_time": 87547.351305784,
+  "total_evaluation_time_seconds": "2224.6928466289974"
+}

senseable__WestLake-7B-v2/.ipynb_checkpoints/results_2024-06-28T01-32-26.319492-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.7048396733718383,
+      "acc_stderr,none": 0.0045518262729780865,
+      "acc_norm,none": 0.874228241386178,
+      "acc_norm_stderr,none": 0.0033091427273509244,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 77.87295612615341,
+      "eqbench_stderr,none": 1.577200540645531,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=senseable/WestLake-7B-v2,trust_remote_code=True",
+    "model_num_parameters": 7241732096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "41625004c47628837678859753b94c50c82f3bec",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719536781.997623,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "senseable/WestLake-7B-v2",
+  "model_name_sanitized": "senseable__WestLake-7B-v2",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 89165.24699567,
+  "end_time": 90736.384333217,
+  "total_evaluation_time_seconds": "1571.1373375470139"
+}

teknium__OpenHermes-2.5-Mistral-7B/.ipynb_checkpoints/results_2024-06-27T23-43-07.467674-checkpoint.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6302529376618203,
+      "acc_stderr,none": 0.004817495546789561,
+      "acc_norm,none": 0.8167695678151763,
+      "acc_norm_stderr,none": 0.003860646998897285,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 65.75110483136034,
+      "eqbench_stderr,none": 2.270775919439369,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=teknium/OpenHermes-2.5-Mistral-7B,trust_remote_code=True",
+    "model_num_parameters": 7241748480,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "24c0bea14d53e6f67f1fbe2eca5bfe7cae389b33",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719530289.0024347,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             24\nOn-line CPU(s) list:                0-23\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 12\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          384 KiB (12 instances)\nL1i cache:                          384 KiB (12 instances)\nL2 cache:                           12 MiB (12 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "<|im_end|>",
+    32000
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 32000,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "teknium/OpenHermes-2.5-Mistral-7B",
+  "model_name_sanitized": "teknium__OpenHermes-2.5-Mistral-7B",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 82672.316515428,
+  "end_time": 84177.532482507,
+  "total_evaluation_time_seconds": "1505.2159670789988"
+}

upstage__SOLAR-10.7B-Instruct-v1.0/results_2024-07-01T20-44-42.759467.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6866162119099781,
+      "acc_stderr,none": 0.004629209184813552,
+      "acc_norm,none": 0.8634734116709819,
+      "acc_norm_stderr,none": 0.003426451744507963,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 68.65298935843904,
+      "eqbench_stderr,none": 2.3895445747636623,
+      "percent_parseable,none": 98.24561403508773,
+      "percent_parseable_stderr,none": 1.006919374006229,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=upstage/SOLAR-10.7B-Instruct-v1.0,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "c08c25ed66414a878fe0401a3596d536c083606c",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719864409.1868505,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "upstage/SOLAR-10.7B-Instruct-v1.0",
+  "model_name_sanitized": "upstage__SOLAR-10.7B-Instruct-v1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 75704.269434269,
+  "end_time": 77984.688135683,
+  "total_evaluation_time_seconds": "2280.4187014140043"
+}

upstage__SOLAR-10.7B-v1.0/results_2024-07-01T20-06-05.907692.json

@@ -0,0 +1,177 @@
+{
+  "results": {
+    "hellaswag": {
+      "acc,none": 0.6394144592710616,
+      "acc_stderr,none": 0.004791890625834175,
+      "acc_norm,none": 0.831009759012149,
+      "acc_norm_stderr,none": 0.0037397742854184536,
+      "alias": "hellaswag"
+    },
+    "eq_bench": {
+      "eqbench,none": 28.659786009514935,
+      "eqbench_stderr,none": 3.1809125071508557,
+      "percent_parseable,none": 100.0,
+      "percent_parseable_stderr,none": 0.0,
+      "alias": "eq_bench"
+    }
+  },
+  "group_subtasks": {
+    "eq_bench": [],
+    "hellaswag": []
+  },
+  "configs": {
+    "eq_bench": {
+      "task": "eq_bench",
+      "dataset_path": "pbevan11/EQ-Bench",
+      "validation_split": "validation",
+      "doc_to_text": "prompt",
+      "doc_to_target": "reference_answer_fullscale",
+      "process_results": "def calculate_score_fullscale(docs, results):\n    reference = eval(docs[\"reference_answer_fullscale\"])\n    user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n    # First check that the emotions specified in the answer match those in the reference\n    if len(user.items()) != 4:\n        # print('! Error: 4 emotions were not returned')\n        # print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n    emotions_dict = {}\n    for emotion, user_emotion_score in user.items():\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                emotions_dict[emotion] = True\n    if len(emotions_dict) != 4:\n        print(\"! Error: emotions did not match reference\")\n        print(user)\n        return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n    difference_tally = (\n        0  # Tally of differerence from reference answers for this question\n    )\n\n    # Iterate over each emotion in the user's answers.\n    for emotion, user_emotion_score in user.items():\n        # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n        for i in range(1, 5):\n            if emotion == reference[f\"emotion{i}\"]:\n                d = abs(\n                    float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n                )\n                # this will be a value between 0 and 10\n                if d == 0:\n                    scaled_difference = 0\n                elif d <= 5:\n                    # S-shaped scaling function\n                    # https://www.desmos.com/calculator\n                    # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n                    scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n                else:\n                    scaled_difference = d\n                difference_tally += scaled_difference\n\n    # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n    # The adjustment constant is chosen such that answering randomly produces a score of zero.\n    adjust_const = 0.7477\n    final_score = 10 - (difference_tally * adjust_const)\n    final_score_percent = final_score * 10\n\n    return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "eqbench",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "percent_parseable",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 80,
+        "until": [
+          "\n\n"
+        ]
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.1
+      }
+    },
+    "hellaswag": {
+      "task": "hellaswag",
+      "group": [
+        "multiple_choice"
+      ],
+      "dataset_path": "hellaswag",
+      "training_split": "train",
+      "validation_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_doc(doc):\n        ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n        out_doc = {\n            \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n            \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n            \"gold\": int(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_doc)\n",
+      "doc_to_text": "{{query}}",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": "choices",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "eq_bench": 2.1,
+    "hellaswag": 1.0
+  },
+  "n-shot": {
+    "eq_bench": 0,
+    "hellaswag": 0
+  },
+  "higher_is_better": {
+    "eq_bench": {
+      "eqbench": true,
+      "percent_parseable": true
+    },
+    "hellaswag": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "hellaswag": {
+      "original": 10042,
+      "effective": 10042
+    },
+    "eq_bench": {
+      "original": 171,
+      "effective": 171
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=upstage/SOLAR-10.7B-v1.0,trust_remote_code=True",
+    "model_num_parameters": 10731524096,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "a45090b8e56bdc2b8e32e46b3cd782fc0bea1fa5",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": "cuda:0",
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": null,
+  "date": 1719862032.513694,
+  "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      46 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          GenuineIntel\nModel name:                         Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family:                         6\nModel:                              85\nThread(s) per core:                 2\nCore(s) per socket:                 24\nSocket(s):                          1\nStepping:                           7\nBogoMIPS:                           4400.47\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor:                  KVM\nVirtualization type:                full\nL1d cache:                          768 KiB (24 instances)\nL1i cache:                          768 KiB (24 instances)\nL2 cache:                           24 MiB (24 instances)\nL3 cache:                           38.5 MiB (1 instance)\nNUMA node(s):                       1\nNUMA node0 CPU(s):                  0-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed:             Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect",
+  "transformers_version": "4.41.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    0
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    2
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    1
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "upstage/SOLAR-10.7B-v1.0",
+  "model_name_sanitized": "upstage__SOLAR-10.7B-v1.0",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 73327.637521213,
+  "end_time": 75667.837128227,
+  "total_evaluation_time_seconds": "2340.1996070140012"
+}