| import os | |
| import argparse | |
| import json | |
| import re | |
| from llava.eval.m4c_evaluator import TextVQAAccuracyEvaluator | |
| def get_args(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--annotation-file', type=str) | |
| parser.add_argument('--result-file', type=str) | |
| parser.add_argument('--result-dir', type=str) | |
| return parser.parse_args() | |
| def prompt_processor(prompt): | |
| if prompt.startswith('OCR tokens: '): | |
| pattern = r"Question: (.*?) Short answer:" | |
| match = re.search(pattern, prompt, re.DOTALL) | |
| question = match.group(1) | |
| elif 'Reference OCR token: ' in prompt and len(prompt.split('\n')) == 3: | |
| if prompt.startswith('Reference OCR token:'): | |
| question = prompt.split('\n')[1] | |
| else: | |
| question = prompt.split('\n')[0] | |
| elif len(prompt.split('\n')) == 2: | |
| question = prompt.split('\n')[0] | |
| else: | |
| assert False | |
| return question.lower() | |
| def eval_single(annotation_file, result_file): | |
| experiment_name = os.path.splitext(os.path.basename(result_file))[0] | |
| print(experiment_name) | |
| annotations = json.load(open(annotation_file))['data'] | |
| annotations = {(annotation['image_id'], annotation['question'].lower()): annotation for annotation in annotations} | |
| results = [json.loads(line) for line in open(result_file)] | |
| pred_list = [] | |
| for result in results: | |
| annotation = annotations[(result['question_id'], prompt_processor(result['prompt']))] | |
| pred_list.append({ | |
| "pred_answer": result['text'], | |
| "gt_answers": annotation['answers'], | |
| }) | |
| evaluator = TextVQAAccuracyEvaluator() | |
| print('Samples: {}\nAccuracy: {:.2f}%\n'.format(len(pred_list), 100. * evaluator.eval_pred_list(pred_list))) | |
| if __name__ == "__main__": | |
| args = get_args() | |
| if args.result_file is not None: | |
| eval_single(args.annotation_file, args.result_file) | |
| if args.result_dir is not None: | |
| for result_file in sorted(os.listdir(args.result_dir)): | |
| if not result_file.endswith('.jsonl'): | |
| print(f'Skipping {result_file}') | |
| continue | |
| eval_single(args.annotation_file, os.path.join(args.result_dir, result_file)) | |