|
import pandas as pd |
|
from fuzzywuzzy import fuzz |
|
from collections import Counter |
|
from nltk.stem import PorterStemmer |
|
from ast import literal_eval |
|
from typing import Union, List |
|
|
|
class KBVQAEvaluator: |
|
def __init__(self): |
|
""" |
|
Initialize the VQA Processor with the dataset and configuration settings. |
|
""" |
|
self.use_fuzzy = False |
|
self.stemmer = PorterStemmer() |
|
self.df = pd.read_excel('evaluation_results_final.xlsx') |
|
self.scores_df = pd.read_excel(data_path, sheet_name="Scores") |
|
self.df = pd.read_excel(data_path, sheet_name="Main Data") |
|
self.vqa_scores = {} |
|
self.exact_match_scores = {} |
|
|
|
def stem_answers(self, answers: Union[str, List[str]]) -> Union[str, List[str]]: |
|
""" |
|
Apply Porter Stemmer to either a single string or a list of strings. |
|
""" |
|
if isinstance(answers, list): |
|
return [" ".join(self.stemmer.stem(word.strip()) for word in answer.split()) for answer in answers] |
|
else: |
|
words = answers.split() |
|
return " ".join(self.stemmer.stem(word.strip()) for word in words) |
|
|
|
def calculate_vqa_score(self, ground_truths, model_answer): |
|
""" |
|
Calculate VQA score based on the number of matching answers, with optional fuzzy matching. |
|
""" |
|
if self.use_fuzzy: |
|
fuzzy_matches = sum(fuzz.partial_ratio(model_answer, gt) >= 80 for gt in ground_truths) |
|
return min(fuzzy_matches / 3, 1) |
|
else: |
|
count = Counter(ground_truths) |
|
return min(count.get(model_answer, 0) / 3, 1) |
|
|
|
def calculate_exact_match_score(self, ground_truths, model_answer): |
|
""" |
|
Calculate Exact Match score, with optional fuzzy matching. |
|
""" |
|
if self.use_fuzzy: |
|
return int(any(fuzz.partial_ratio(model_answer, gt) >= 80 for gt in ground_truths)) |
|
else: |
|
return int(model_answer in ground_truths) |
|
|
|
def evaluate(self): |
|
""" |
|
Process the DataFrame: stem answers, calculate scores, and store results. |
|
""" |
|
self.df['raw_answers_stemmed'] = self.df['raw_answers'].apply(literal_eval).apply(self.stem_answers) |
|
model_configurations = ['caption+detic', 'caption+yolov5', 'only_caption', 'only_detic', 'only_yolov5'] |
|
model_names = ['13b', '7b'] |
|
|
|
for name in model_names: |
|
for config in model_configurations: |
|
full_config = f'{name}_{config}' |
|
self.df[f'{full_config}_stemmed'] = self.df[full_config].apply(self.stem_answers) |
|
|
|
self.df[f'vqa_score_{full_config}'] = self.df.apply(lambda x: self.calculate_vqa_score(x['raw_answers_stemmed'], x[f'{full_config}_stemmed']), axis=1) |
|
self.df[f'exact_match_score_{full_config}'] = self.df.apply(lambda x: self.calculate_exact_match_score(x['raw_answers_stemmed'], x[f'{full_config}_stemmed']), axis=1) |
|
|
|
self.vqa_scores[full_config] = round(self.df[f'vqa_score_{full_config}'].mean()*100, 2) |
|
self.exact_match_scores[full_config] = round(self.df[f'exact_match_score_{full_config}'].mean()*100, 2) |
|
|
|
def save_results(self): |
|
|
|
scores_data = { |
|
'Model Configuration': list(self.vqa_scores.keys()), |
|
'VQA Score': list(self.vqa_scores.values()), |
|
'Exact Match Score': list(self.exact_match_scores.values()) |
|
} |
|
scores_df = pd.DataFrame(scores_data) |
|
|
|
|
|
with pd.ExcelWriter('evaluation_results_final.xlsx', engine='openpyxl', mode='w') as writer: |
|
scores_df.to_excel(writer, sheet_name='Scores', index=False) |
|
|
|
def run_evaluator(self): |
|
|
|
st.table(self.scores_df) |
|
st.write(self.scores_df) |
|
|
|
|
|
|
