nvidia/NVLM-D-72B

1# coding=utf-82 3__author__ = "aagrawal"4 5# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link:6# (https://github.com/tylin/coco-caption/blob/master/pycocoevalcap/eval.py).7import sys8import re9 10 11class VQAEval:12    def __init__(self, vqa=None, vqaRes=None, n=2):13        self.n = n14        self.accuracy = {}15        self.evalQA = {}16        self.evalQuesType = {}17        self.evalAnsType = {}18        self.vqa = vqa19        self.vqaRes = vqaRes20        if vqa is not None:21            self.params = {"question_id": vqa.getQuesIds()}22        self.contractions = {23            "aint": "ain't",24            "arent": "aren't",25            "cant": "can't",26            "couldve": "could've",27            "couldnt": "couldn't",28            "couldn'tve": "couldn't've",29            "couldnt've": "couldn't've",30            "didnt": "didn't",31            "doesnt": "doesn't",32            "dont": "don't",33            "hadnt": "hadn't",34            "hadnt've": "hadn't've",35            "hadn'tve": "hadn't've",36            "hasnt": "hasn't",37            "havent": "haven't",38            "hed": "he'd",39            "hed've": "he'd've",40            "he'dve": "he'd've",41            "hes": "he's",42            "howd": "how'd",43            "howll": "how'll",44            "hows": "how's",45            "Id've": "I'd've",46            "I'dve": "I'd've",47            "Im": "I'm",48            "Ive": "I've",49            "isnt": "isn't",50            "itd": "it'd",51            "itd've": "it'd've",52            "it'dve": "it'd've",53            "itll": "it'll",54            "let's": "let's",55            "maam": "ma'am",56            "mightnt": "mightn't",57            "mightnt've": "mightn't've",58            "mightn'tve": "mightn't've",59            "mightve": "might've",60            "mustnt": "mustn't",61            "mustve": "must've",62            "neednt": "needn't",63            "notve": "not've",64            "oclock": "o'clock",65            "oughtnt": "oughtn't",66            "ow's'at": "'ow's'at",67            "'ows'at": "'ow's'at",68            "'ow'sat": "'ow's'at",69            "shant": "shan't",70            "shed've": "she'd've",71            "she'dve": "she'd've",72            "she's": "she's",73            "shouldve": "should've",74            "shouldnt": "shouldn't",75            "shouldnt've": "shouldn't've",76            "shouldn'tve": "shouldn't've",77            "somebody'd": "somebodyd",78            "somebodyd've": "somebody'd've",79            "somebody'dve": "somebody'd've",80            "somebodyll": "somebody'll",81            "somebodys": "somebody's",82            "someoned": "someone'd",83            "someoned've": "someone'd've",84            "someone'dve": "someone'd've",85            "someonell": "someone'll",86            "someones": "someone's",87            "somethingd": "something'd",88            "somethingd've": "something'd've",89            "something'dve": "something'd've",90            "somethingll": "something'll",91            "thats": "that's",92            "thered": "there'd",93            "thered've": "there'd've",94            "there'dve": "there'd've",95            "therere": "there're",96            "theres": "there's",97            "theyd": "they'd",98            "theyd've": "they'd've",99            "they'dve": "they'd've",100            "theyll": "they'll",101            "theyre": "they're",102            "theyve": "they've",103            "twas": "'twas",104            "wasnt": "wasn't",105            "wed've": "we'd've",106            "we'dve": "we'd've",107            "weve": "we've",108            "werent": "weren't",109            "whatll": "what'll",110            "whatre": "what're",111            "whats": "what's",112            "whatve": "what've",113            "whens": "when's",114            "whered": "where'd",115            "wheres": "where's",116            "whereve": "where've",117            "whod": "who'd",118            "whod've": "who'd've",119            "who'dve": "who'd've",120            "wholl": "who'll",121            "whos": "who's",122            "whove": "who've",123            "whyll": "why'll",124            "whyre": "why're",125            "whys": "why's",126            "wont": "won't",127            "wouldve": "would've",128            "wouldnt": "wouldn't",129            "wouldnt've": "wouldn't've",130            "wouldn'tve": "wouldn't've",131            "yall": "y'all",132            "yall'll": "y'all'll",133            "y'allll": "y'all'll",134            "yall'd've": "y'all'd've",135            "y'alld've": "y'all'd've",136            "y'all'dve": "y'all'd've",137            "youd": "you'd",138            "youd've": "you'd've",139            "you'dve": "you'd've",140            "youll": "you'll",141            "youre": "you're",142            "youve": "you've",143        }144        self.manualMap = {145            "none": "0",146            "zero": "0",147            "one": "1",148            "two": "2",149            "three": "3",150            "four": "4",151            "five": "5",152            "six": "6",153            "seven": "7",154            "eight": "8",155            "nine": "9",156            "ten": "10",157        }158        self.articles = ["a", "an", "the"]159 160        self.periodStrip = re.compile("(?!<=\d)(\.)(?!\d)")161        self.commaStrip = re.compile("(\d)(,)(\d)")162        self.punct = [163            ";",164            r"/",165            "[",166            "]",167            '"',168            "{",169            "}",170            "(",171            ")",172            "=",173            "+",174            "\\",175            "_",176            "-",177            ">",178            "<",179            "@",180            "`",181            ",",182            "?",183            "!",184        ]185 186    def evaluate(self, quesIds=None):187        if quesIds == None:188            quesIds = [quesId for quesId in self.params["question_id"]]189        gts = {}190        res = {}191        for quesId in quesIds:192            gts[quesId] = self.vqa.qa[quesId]193            res[quesId] = self.vqaRes.qa[quesId]194 195        # =================================================196        # Compute accuracy197        # =================================================198        accQA = []199        accQuesType = {}200        accAnsType = {}201        print("computing accuracy")202        step = 0203        for quesId in quesIds:204            resAns = res[quesId]["answer"]205            resAns = resAns.replace("\n", " ")206            resAns = resAns.replace("\t", " ")207            resAns = resAns.strip()208            resAns = self.processPunctuation(resAns)209            resAns = self.processDigitArticle(resAns)210            gtAcc = []211            gtAnswers = [ans["answer"] for ans in gts[quesId]["answers"]]212            if len(set(gtAnswers)) > 1:213                for ansDic in gts[quesId]["answers"]:214                    ansDic["answer"] = self.processPunctuation(ansDic["answer"])215            for gtAnsDatum in gts[quesId]["answers"]:216                otherGTAns = [217                    item for item in gts[quesId]["answers"] if item != gtAnsDatum218                ]219                matchingAns = [item for item in otherGTAns if item["answer"] == resAns]220                acc = min(1, float(len(matchingAns)) / 3)221                gtAcc.append(acc)222            quesType = gts[quesId]["question_type"]223            ansType = gts[quesId]["answer_type"]224            avgGTAcc = float(sum(gtAcc)) / len(gtAcc)225            accQA.append(avgGTAcc)226            if quesType not in accQuesType:227                accQuesType[quesType] = []228            accQuesType[quesType].append(avgGTAcc)229            if ansType not in accAnsType:230                accAnsType[ansType] = []231            accAnsType[ansType].append(avgGTAcc)232            self.setEvalQA(quesId, avgGTAcc)233            self.setEvalQuesType(quesId, quesType, avgGTAcc)234            self.setEvalAnsType(quesId, ansType, avgGTAcc)235            if step % 100 == 0:236                self.updateProgress(step / float(len(quesIds)))237            step = step + 1238 239        self.setAccuracy(accQA, accQuesType, accAnsType)240        print("Done computing accuracy")241 242    def processPunctuation(self, inText):243        outText = inText244        for p in self.punct:245            if (p + " " in inText or " " + p in inText) or (246                re.search(self.commaStrip, inText) != None247            ):248                outText = outText.replace(p, "")249            else:250                outText = outText.replace(p, " ")251        outText = self.periodStrip.sub("", outText, re.UNICODE)252        return outText253 254    def processDigitArticle(self, inText):255        outText = []256        tempText = inText.lower().split()257        for word in tempText:258            word = self.manualMap.setdefault(word, word)259            if word not in self.articles:260                outText.append(word)261            else:262                pass263        for wordId, word in enumerate(outText):264            if word in self.contractions:265                outText[wordId] = self.contractions[word]266        outText = " ".join(outText)267        return outText268 269    def setAccuracy(self, accQA, accQuesType, accAnsType):270        self.accuracy["overall"] = round(100 * float(sum(accQA)) / len(accQA), self.n)271        self.accuracy["perQuestionType"] = {272            quesType: round(273                100 * float(sum(accQuesType[quesType])) / len(accQuesType[quesType]),274                self.n,275            )276            for quesType in accQuesType277        }278        self.accuracy["perAnswerType"] = {279            ansType: round(280                100 * float(sum(accAnsType[ansType])) / len(accAnsType[ansType]), self.n281            )282            for ansType in accAnsType283        }284 285    def setEvalQA(self, quesId, acc):286        self.evalQA[quesId] = round(100 * acc, self.n)287 288    def setEvalQuesType(self, quesId, quesType, acc):289        if quesType not in self.evalQuesType:290            self.evalQuesType[quesType] = {}291        self.evalQuesType[quesType][quesId] = round(100 * acc, self.n)292 293    def setEvalAnsType(self, quesId, ansType, acc):294        if ansType not in self.evalAnsType:295            self.evalAnsType[ansType] = {}296        self.evalAnsType[ansType][quesId] = round(100 * acc, self.n)297 298    def updateProgress(self, progress):299        barLength = 20300        status = ""301        if isinstance(progress, int):302            progress = float(progress)303        if not isinstance(progress, float):304            progress = 0305            status = "error: progress var must be float\r\n"306        if progress < 0:307            progress = 0308            status = "Halt...\r\n"309        if progress >= 1:310            progress = 1311            status = "Done...\r\n"312        block = int(round(barLength * progress))313        text = "\rFinshed Percent: [{0}] {1}% {2}".format(314            "#" * block + "-" * (barLength - block), int(progress * 100), status315        )316        sys.stdout.write(text)317        sys.stdout.flush()