LanguageBind/Video-LLaVA

1import argparse2 3import openai4import json5import os6from tqdm import tqdm7import pandas as pd8import numpy as np9from collections import Counter10import time11 12 13 14parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')15parser.add_argument('--mmvet_path')16parser.add_argument('--ckpt_name')17parser.add_argument('--result_path')18args = parser.parse_args()19 20 21openai.api_base = "https://api.aiguoguo199.com/v1"22openai.api_key = 'sk-eionFWpNThMNy4eeFdC25789F60a4cC2A66b2c94D3948bA6'23 24gpt_model = "gpt-3.5-turbo"25 26 27prompt = """Compare the ground truth and prediction from AI models, to give a correctness score for the prediction. <AND> in the ground truth means it is totally right only when all elements in the ground truth are present in the prediction, and <OR> means it is totally right when any one element in the ground truth is present in the prediction. The correctness score is 0.0 (totally wrong), 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, or 1.0 (totally right). Just complete the last space of the correctness score.28 29Question | Ground truth | Prediction | Correctness30--- | --- | --- | ---31What is x in the equation? | -1 <AND> -5 | x = 3 | 0.032What is x in the equation? | -1 <AND> -5 | x = -1 | 0.533What is x in the equation? | -1 <AND> -5 | x = -5 | 0.534What is x in the equation? | -1 <AND> -5 | x = -5 or 5 | 0.535What is x in the equation? | -1 <AND> -5 | x = -1 or x = -5 | 1.036Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme talks about Iceland and Greenland. It's pointing out that despite their names, Iceland is not very icy and Greenland isn't very green. | 0.437Can you explain this meme? | This meme is poking fun at the fact that the names of the countries Iceland and Greenland are misleading. Despite its name, Iceland is known for its beautiful green landscapes, while Greenland is mostly covered in ice and snow. The meme is saying that the person has trust issues because the names of these countries do not accurately represent their landscapes. | The meme is using humor to point out the misleading nature of Iceland's and Greenland's names. Iceland, despite its name, has lush green landscapes while Greenland is mostly covered in ice and snow. The text 'This is why I have trust issues' is a playful way to suggest that these contradictions can lead to distrust or confusion. The humor in this meme is derived from the unexpected contrast between the names of the countries and their actual physical characteristics. | 1.038"""39 40# load metadata41# Download mm-vet.zip and `unzip mm-vet.zip` and change the path below42mmvet_path = args.mmvet_path43use_sub_set = False44decimal_places = 1  # number of decimal places to round to45 46if use_sub_set:47    bard_set_file = os.path.join(mmvet_path, "bard_set.json")48    with open(bard_set_file, 'r') as f:49        sub_set = json.load(f)50    sub_set_name = 'bardset'51    sub_set_name = sub_set_name + '_'52else:53    sub_set = None54    sub_set_name = ''55 56mmvet_metadata = os.path.join(mmvet_path, "mm-vet.json")57with open(mmvet_metadata, 'r') as f:58    data = json.load(f)59 60counter = Counter()61cap_set_list = []62cap_set_counter = []63len_data = 064for id, value in data.items():65    if sub_set is not None and id not in sub_set:66        continue67    question = value["question"]68    answer = value["answer"]69    cap = value["capability"]70    cap = set(cap)71    counter.update(cap)72    if cap not in cap_set_list:73        cap_set_list.append(cap)74        cap_set_counter.append(1)75    else:76        cap_set_counter[cap_set_list.index(cap)] += 177 78    len_data += 179 80sorted_list = counter.most_common()81columns = [k for k, v in sorted_list]82columns.append("total")83columns.append("std")84columns.append('runs')85df = pd.DataFrame(columns=columns)86 87cap_set_sorted_indices = np.argsort(-np.array(cap_set_counter))88new_cap_set_list = []89new_cap_set_counter = []90for index in cap_set_sorted_indices:91    new_cap_set_list.append(cap_set_list[index])92    new_cap_set_counter.append(cap_set_counter[index])93 94cap_set_list = new_cap_set_list95cap_set_counter = new_cap_set_counter96cap_set_names = ["_".join(list(cap_set)) for cap_set in cap_set_list]97 98columns2 = cap_set_names99columns2.append("total")100columns2.append("std")101columns2.append('runs')102df2 = pd.DataFrame(columns=columns2)103 104 105 106 107 108 109 110 111###### change your model name ######112model = args.ckpt_name113result_path = args.result_path114num_run = 1 # we set it as 5 in the paper115model_results_file = os.path.join(result_path, f"{model}.json")116 117# grade results for each sample to svae118grade_file = f'{model}_{gpt_model}-grade-{num_run}runs.json'119grade_file = os.path.join(result_path, grade_file)120 121# score results regarding capabilities/capability integration to save122cap_score_file = f'{model}_{sub_set_name}{gpt_model}-cap-score-{num_run}runs.csv'123cap_score_file = os.path.join(result_path, cap_score_file)124cap_int_score_file = f'{model}_{sub_set_name}{gpt_model}-cap-int-score-{num_run}runs.csv'125cap_int_score_file = os.path.join(result_path, cap_int_score_file)126 127with open(model_results_file) as f:128    results = json.load(f)129if os.path.exists(grade_file):130    with open(grade_file, 'r') as f:131        grade_results = json.load(f)132else:133    grade_results = {}134 135 136def need_more_runs():137    need_more_runs = False138    if len(grade_results) > 0:139        for k, v in grade_results.items():140            if len(v['score']) < num_run:141                need_more_runs = True142                break143    return need_more_runs or len(grade_results) < len_data144 145 146while need_more_runs():147    for j in range(num_run):148        print(f'eval run {j}')149        for id, line in tqdm(data.items()):150            if sub_set is not None and id not in sub_set:151                continue152            if id in grade_results and len(grade_results[id]['score']) >= (j + 1):153                continue154 155            model_pred = results[id]156 157            question = prompt + '\n' + ' | '.join(158                [line['question'], line['answer'].replace("<AND>", " <AND> ").replace("<OR>", " <OR> "), model_pred,159                 ""])160            messages = [161                {"role": "user", "content": question},162            ]163 164            if id not in grade_results:165                sample_grade = {'model': [], 'content': [], 'score': []}166            else:167                sample_grade = grade_results[id]168 169            grade_sample_run_complete = False170            temperature = 0.0171 172            while not grade_sample_run_complete:173                try:174                    response = openai.ChatCompletion.create(175                        model=gpt_model,176                        max_tokens=3,177                        temperature=temperature,178                        messages=messages)179                    # print(response['model'])180                    content = response['choices'][0]['message']['content']181                    flag = True182                    try_time = 1183                    while flag:184                        try:185                            content = content.split(' ')[0].strip()186                            score = float(content)187                            if score > 1.0 or score < 0.0:188                                assert False189                            flag = False190                        except:191                            question = prompt + '\n' + ' | '.join(192                                [line['question'], line['answer'].replace("<AND>", " <AND> ").replace("<OR>", " <OR> "),193                                 model_pred, ""]) + "\nPredict the correctness of the answer (digit): "194                            messages = [195                                {"role": "user", "content": question},196                            ]197                            response = openai.ChatCompletion.create(198                                model=gpt_model,199                                max_tokens=3,200                                temperature=temperature,201                                messages=messages)202                            # print(response)203                            content = response['choices'][0]['message']['content']204                            try_time += 1205                            temperature += 0.5206                            print(f"{id} try {try_time} times")207                            print(content)208                            if try_time > 5:209                                score = 0.0210                                flag = False211                    grade_sample_run_complete = True212                except:213                    # gpt4 may have token rate limit214                    print("sleep 1s")215                    time.sleep(1)216 217            if len(sample_grade['model']) >= j + 1:218                # sample_grade['model'][j] = response['model']219                sample_grade['content'][j] = content220                sample_grade['score'][j] = score221            else:222                # sample_grade['model'].append(response['model'])223                sample_grade['content'].append(content)224                sample_grade['score'].append(score)225            grade_results[id] = sample_grade226 227            with open(grade_file, 'w') as f:228                json.dump(grade_results, f, indent=4)229 230assert not need_more_runs()231cap_socres = {k: [0.0] * num_run for k in columns[:-2]}232counter['total'] = len_data233 234cap_socres2 = {k: [0.0] * num_run for k in columns2[:-2]}235counter2 = {columns2[i]: cap_set_counter[i] for i in range(len(cap_set_counter))}236counter2['total'] = len_data237 238for k, v in grade_results.items():239    if sub_set is not None and k not in sub_set:240        continue241    for i in range(num_run):242        score = v['score'][i]243        caps = set(data[k]['capability'])244        for c in caps:245            cap_socres[c][i] += score246 247        cap_socres['total'][i] += score248 249        index = cap_set_list.index(caps)250        cap_socres2[cap_set_names[index]][i] += score251        cap_socres2['total'][i] += score252 253for k, v in cap_socres.items():254    cap_socres[k] = np.array(v) / counter[k] * 100255 256std = round(cap_socres['total'].std(), decimal_places)257total_copy = cap_socres['total'].copy()258runs = str(list(np.round(total_copy, decimal_places)))259 260for k, v in cap_socres.items():261    cap_socres[k] = round(v.mean(), decimal_places)262 263cap_socres['std'] = std264cap_socres['runs'] = runs265df.loc[model] = cap_socres266 267for k, v in cap_socres2.items():268    cap_socres2[k] = round(np.mean(np.array(v) / counter2[k] * 100), decimal_places)269cap_socres2['std'] = std270cap_socres2['runs'] = runs271df2.loc[model] = cap_socres2272 273df.to_csv(cap_score_file)274df2.to_csv(cap_int_score_file)275print(df)276print(df2)