SubashSK777/Visual-Question-Answering

1#!/usr/bin/env python3 -u2# Copyright (c) Facebook, Inc. and its affiliates.3#4# This source code is licensed under the MIT license found in the5# LICENSE file in the root directory of this source tree.6 7import logging8import os9import sys10import json11from itertools import chain12 13import numpy as np14import torch15import torch.distributed as dist16from fairseq import distributed_utils, options, tasks, utils17from fairseq.dataclass.utils import convert_namespace_to_omegaconf18from fairseq.logging import progress_bar19from fairseq.utils import reset_logging20from omegaconf import DictConfig21 22from utils import checkpoint_utils23from utils.eval_utils import eval_step24 25logging.basicConfig(26    format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",27    datefmt="%Y-%m-%d %H:%M:%S",28    level=os.environ.get("LOGLEVEL", "INFO").upper(),29    stream=sys.stdout,30)31logger = logging.getLogger("ofa.evaluate")32 33 34def apply_half(t):35    if t.dtype is torch.float32:36        return t.to(dtype=torch.half)37    return t38 39 40def main(cfg: DictConfig, **kwargs):41    utils.import_user_module(cfg.common)42 43    reset_logging()44    logger.info(cfg)45 46    assert (47        cfg.dataset.max_tokens is not None or cfg.dataset.batch_size is not None48    ), "Must specify batch size either with --max-tokens or --batch-size"49 50    # Fix seed for stochastic decoding51    if cfg.common.seed is not None and not cfg.generation.no_seed_provided:52        np.random.seed(cfg.common.seed)53        utils.set_torch_seed(cfg.common.seed)54 55    use_fp16 = cfg.common.fp1656    use_cuda = torch.cuda.is_available() and not cfg.common.cpu57 58    if use_cuda:59        torch.cuda.set_device(cfg.distributed_training.device_id)60 61    # Load ensemble62    overrides = eval(cfg.common_eval.model_overrides)63    logger.info("loading model(s) from {}".format(cfg.common_eval.path))64    models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(65        utils.split_paths(cfg.common_eval.path),66        arg_overrides=overrides,67        suffix=cfg.checkpoint.checkpoint_suffix,68        strict=(cfg.checkpoint.checkpoint_shard_count == 1),69        num_shards=cfg.checkpoint.checkpoint_shard_count,70    )71 72    # loading the dataset should happen after the checkpoint has been loaded so we can give it the saved task config73    task.load_dataset(cfg.dataset.gen_subset, task_cfg=saved_cfg.task)74 75    # Move models to GPU76    for model, ckpt_path in zip(models, utils.split_paths(cfg.common_eval.path)):77        if kwargs['ema_eval']:78            logger.info("loading EMA weights from {}".format(ckpt_path))79            model.load_state_dict(checkpoint_utils.load_ema_from_checkpoint(ckpt_path)['model'])80        model.eval()81        if use_fp16:82            model.half()83        if use_cuda and not cfg.distributed_training.pipeline_model_parallel:84            model.cuda()85        model.prepare_for_inference_(cfg)86 87    # Load dataset (possibly sharded)88    itr = task.get_batch_iterator(89        dataset=task.dataset(cfg.dataset.gen_subset),90        max_tokens=cfg.dataset.max_tokens,91        max_sentences=cfg.dataset.batch_size,92        max_positions=utils.resolve_max_positions(93            task.max_positions(), *[m.max_positions() for m in models]94        ),95        ignore_invalid_inputs=cfg.dataset.skip_invalid_size_inputs_valid_test,96        required_batch_size_multiple=cfg.dataset.required_batch_size_multiple,97        seed=cfg.common.seed,98        num_shards=cfg.distributed_training.distributed_world_size,99        shard_id=cfg.distributed_training.distributed_rank,100        num_workers=cfg.dataset.num_workers,101        data_buffer_size=cfg.dataset.data_buffer_size,102    ).next_epoch_itr(shuffle=False)103    progress = progress_bar.progress_bar(104        itr,105        log_format=cfg.common.log_format,106        log_interval=cfg.common.log_interval,107        default_log_format=("tqdm" if not cfg.common.no_progress_bar else "simple"),108    )109 110    # Initialize generator111    generator = task.build_generator(models, cfg.generation)112 113    results = []114    score_sum = torch.FloatTensor([0]).cuda()115    score_cnt = torch.FloatTensor([0]).cuda()116    for sample in progress:117        if "net_input" not in sample:118            continue119        sample = utils.move_to_cuda(sample) if use_cuda else sample120        sample = utils.apply_to_sample(apply_half, sample) if cfg.common.fp16 else sample121        with torch.no_grad():122            result, scores = eval_step(task, generator, models, sample)123        results += result124        score_sum += sum(scores) if scores is not None else 0125        score_cnt += len(scores) if scores is not None else 0126        progress.log({"sentences": sample["nsentences"]})127 128    gather_results = None129    if cfg.distributed_training.distributed_world_size > 1:130        gather_results = [None for _ in range(dist.get_world_size())]131        dist.all_gather_object(gather_results, results)132        dist.all_reduce(score_sum.data)133        dist.all_reduce(score_cnt.data)134    if score_cnt.item() > 0:135        logger.info("score_sum: {}, score_cnt: {}, score: {}".format(136            score_sum, score_cnt, round(score_sum.item() / score_cnt.item(), 4)137        ))138 139    if cfg.distributed_training.distributed_world_size == 1 or dist.get_rank() == 0:140        os.makedirs(cfg.common_eval.results_path, exist_ok=True)141        output_path = os.path.join(cfg.common_eval.results_path, "{}_predict.json".format(cfg.dataset.gen_subset))142        gather_results = list(chain(*gather_results)) if gather_results is not None else results143        with open(output_path, 'w') as fw:144            json.dump(gather_results, fw)145 146 147def cli_main():148    parser = options.get_generation_parser()149    parser.add_argument("--ema-eval", action='store_true', help="Use EMA weights to make evaluation.")150    args = options.parse_args_and_arch(parser)151    cfg = convert_namespace_to_omegaconf(args)152    distributed_utils.call_main(cfg, main, ema_eval=args.ema_eval)153 154 155if __name__ == "__main__":156    cli_main()