| { | |
| "results": { | |
| "hellaswag": { | |
| "acc,none": 0.5837482573192591, | |
| "acc_stderr,none": 0.0049192891130275095, | |
| "acc_norm,none": 0.7665803624775941, | |
| "acc_norm_stderr,none": 0.004221424792919153, | |
| "alias": "hellaswag" | |
| }, | |
| "eq_bench": { | |
| "eqbench,none": 18.19761609584577, | |
| "eqbench_stderr,none": 3.474273216617232, | |
| "percent_parseable,none": 97.6608187134503, | |
| "percent_parseable_stderr,none": 1.1592247905734945, | |
| "alias": "eq_bench" | |
| } | |
| }, | |
| "group_subtasks": { | |
| "eq_bench": [], | |
| "hellaswag": [] | |
| }, | |
| "configs": { | |
| "eq_bench": { | |
| "task": "eq_bench", | |
| "dataset_path": "pbevan11/EQ-Bench", | |
| "validation_split": "validation", | |
| "doc_to_text": "prompt", | |
| "doc_to_target": "reference_answer_fullscale", | |
| "process_results": "def calculate_score_fullscale(docs, results):\n reference = eval(docs[\"reference_answer_fullscale\"])\n user = dict(re.findall(r\"(\\w+):\\s+(\\d+)\", results[0]))\n # First check that the emotions specified in the answer match those in the reference\n if len(user.items()) != 4:\n # print('! Error: 4 emotions were not returned')\n # print(user)\n return {\"eqbench\": 0, \"percent_parseable\": 0}\n emotions_dict = {}\n for emotion, user_emotion_score in user.items():\n for i in range(1, 5):\n if emotion == reference[f\"emotion{i}\"]:\n emotions_dict[emotion] = True\n if len(emotions_dict) != 4:\n print(\"! Error: emotions did not match reference\")\n print(user)\n return {\"eqbench\": 0, \"percent_parseable\": 0}\n\n difference_tally = (\n 0 # Tally of differerence from reference answers for this question\n )\n\n # Iterate over each emotion in the user's answers.\n for emotion, user_emotion_score in user.items():\n # If this emotion is in the reference, calculate the difference between the user's score and the reference score.\n for i in range(1, 5):\n if emotion == reference[f\"emotion{i}\"]:\n d = abs(\n float(user_emotion_score) - float(reference[f\"emotion{i}_score\"])\n )\n # this will be a value between 0 and 10\n if d == 0:\n scaled_difference = 0\n elif d <= 5:\n # S-shaped scaling function\n # https://www.desmos.com/calculator\n # 6.5\\cdot\\ \\frac{1}{\\left(1\\ +\\ e^{\\left(-1.2\\cdot\\left(x-4\\right)\\right)}\\right)}\n scaled_difference = 6.5 * (1 / (1 + math.e ** (-1.2 * (d - 4))))\n\n else:\n scaled_difference = d\n difference_tally += scaled_difference\n\n # Inverting the difference tally so that the closer the answer is to reference, the higher the score.\n # The adjustment constant is chosen such that answering randomly produces a score of zero.\n adjust_const = 0.7477\n final_score = 10 - (difference_tally * adjust_const)\n final_score_percent = final_score * 10\n\n return {\"eqbench\": final_score_percent, \"percent_parseable\": 100}\n", | |
| "description": "", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "eqbench", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "percent_parseable", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "generate_until", | |
| "generation_kwargs": { | |
| "do_sample": false, | |
| "temperature": 0.0, | |
| "max_gen_toks": 80, | |
| "until": [ | |
| "\n\n" | |
| ] | |
| }, | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 2.1 | |
| } | |
| }, | |
| "hellaswag": { | |
| "task": "hellaswag", | |
| "group": [ | |
| "multiple_choice" | |
| ], | |
| "dataset_path": "hellaswag", | |
| "training_split": "train", | |
| "validation_split": "validation", | |
| "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", | |
| "doc_to_text": "{{query}}", | |
| "doc_to_target": "{{label}}", | |
| "doc_to_choice": "choices", | |
| "description": "", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc_norm", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": false, | |
| "metadata": { | |
| "version": 1.0 | |
| } | |
| } | |
| }, | |
| "versions": { | |
| "eq_bench": 2.1, | |
| "hellaswag": 1.0 | |
| }, | |
| "n-shot": { | |
| "eq_bench": 0, | |
| "hellaswag": 0 | |
| }, | |
| "higher_is_better": { | |
| "eq_bench": { | |
| "eqbench": true, | |
| "percent_parseable": true | |
| }, | |
| "hellaswag": { | |
| "acc": true, | |
| "acc_norm": true | |
| } | |
| }, | |
| "n-samples": { | |
| "hellaswag": { | |
| "original": 10042, | |
| "effective": 10042 | |
| }, | |
| "eq_bench": { | |
| "original": 171, | |
| "effective": 171 | |
| } | |
| }, | |
| "config": { | |
| "model": "hf", | |
| "model_args": "pretrained=KoboldAI/Mistral-7B-Erebus-v3,trust_remote_code=True", | |
| "model_num_parameters": 7241732096, | |
| "model_dtype": "torch.float16", | |
| "model_revision": "main", | |
| "model_sha": "476c2eed031028f2fcfb9b8d0115b83363cec904", | |
| "batch_size": "auto", | |
| "batch_sizes": [ | |
| 64 | |
| ], | |
| "device": null, | |
| "use_cache": null, | |
| "limit": null, | |
| "bootstrap_iters": 100000, | |
| "gen_kwargs": null, | |
| "random_seed": 0, | |
| "numpy_seed": 1234, | |
| "torch_seed": 1234, | |
| "fewshot_seed": 1234 | |
| }, | |
| "git_hash": null, | |
| "date": 1719538494.5098195, | |
| "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.4 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: Could not collect\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-6.5.0-1022-gcp-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.1.105\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA L4\nGPU 1: NVIDIA L4\n\nNvidia driver version: 555.42.02\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 24\nOn-line CPU(s) list: 0-23\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) CPU @ 2.20GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 12\nSocket(s): 1\nStepping: 7\nBogoMIPS: 4400.47\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat avx512_vnni md_clear arch_capabilities\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 384 KiB (12 instances)\nL1i cache: 384 KiB (12 instances)\nL2 cache: 12 MiB (12 instances)\nL3 cache: 38.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-23\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Mitigation; Clear CPU buffers; SMT Host state unknown\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI Syscall hardening, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Mitigation; Clear CPU buffers; SMT Host state unknown\n\nVersions of relevant libraries:\n[pip3] numpy==2.0.0\n[pip3] torch==2.3.1\n[pip3] triton==2.3.1\n[conda] Could not collect", | |
| "transformers_version": "4.41.2", | |
| "upper_git_hash": null, | |
| "tokenizer_pad_token": [ | |
| "<unk>", | |
| 0 | |
| ], | |
| "tokenizer_eos_token": [ | |
| "</s>", | |
| 2 | |
| ], | |
| "tokenizer_bos_token": [ | |
| "<s>", | |
| 1 | |
| ], | |
| "eot_token_id": 2, | |
| "max_length": 32768, | |
| "task_hashes": {}, | |
| "model_source": "hf", | |
| "model_name": "KoboldAI/Mistral-7B-Erebus-v3", | |
| "model_name_sanitized": "KoboldAI__Mistral-7B-Erebus-v3", | |
| "system_instruction": null, | |
| "system_instruction_sha": null, | |
| "fewshot_as_multiturn": false, | |
| "chat_template": null, | |
| "chat_template_sha": null, | |
| "start_time": 90877.776869387, | |
| "end_time": 92468.355496828, | |
| "total_evaluation_time_seconds": "1590.578627440991" | |
| } |