coreml-community/ControlNet-v1-1-Annotators-cpu

1# ------------------------------------------------------------------------------2# Reference: https://github.com/facebookresearch/detectron2/blob/main/detectron2/evaluation/evaluator.py3# Modified by Jitesh Jain (https://github.com/praeclarumjj3)4# ------------------------------------------------------------------------------5 6import datetime7import logging8import time9from collections import OrderedDict, abc10from contextlib import ExitStack, contextmanager11from typing import List, Union12import torch13from torch import nn14 15from annotator.oneformer.detectron2.utils.comm import get_world_size, is_main_process16from annotator.oneformer.detectron2.utils.logger import log_every_n_seconds17 18 19class DatasetEvaluator:20    """21    Base class for a dataset evaluator.22 23    The function :func:`inference_on_dataset` runs the model over24    all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs.25 26    This class will accumulate information of the inputs/outputs (by :meth:`process`),27    and produce evaluation results in the end (by :meth:`evaluate`).28    """29 30    def reset(self):31        """32        Preparation for a new round of evaluation.33        Should be called before starting a round of evaluation.34        """35        pass36 37    def process(self, inputs, outputs):38        """39        Process the pair of inputs and outputs.40        If they contain batches, the pairs can be consumed one-by-one using `zip`:41 42        .. code-block:: python43 44            for input_, output in zip(inputs, outputs):45                # do evaluation on single input/output pair46                ...47 48        Args:49            inputs (list): the inputs that's used to call the model.50            outputs (list): the return value of `model(inputs)`51        """52        pass53 54    def evaluate(self):55        """56        Evaluate/summarize the performance, after processing all input/output pairs.57 58        Returns:59            dict:60                A new evaluator class can return a dict of arbitrary format61                as long as the user can process the results.62                In our train_net.py, we expect the following format:63 64                * key: the name of the task (e.g., bbox)65                * value: a dict of {metric name: score}, e.g.: {"AP50": 80}66        """67        pass68 69 70class DatasetEvaluators(DatasetEvaluator):71    """72    Wrapper class to combine multiple :class:`DatasetEvaluator` instances.73 74    This class dispatches every evaluation call to75    all of its :class:`DatasetEvaluator`.76    """77 78    def __init__(self, evaluators):79        """80        Args:81            evaluators (list): the evaluators to combine.82        """83        super().__init__()84        self._evaluators = evaluators85 86    def reset(self):87        for evaluator in self._evaluators:88            evaluator.reset()89 90    def process(self, inputs, outputs):91        for evaluator in self._evaluators:92            evaluator.process(inputs, outputs)93 94    def evaluate(self):95        results = OrderedDict()96        for evaluator in self._evaluators:97            result = evaluator.evaluate()98            if is_main_process() and result is not None:99                for k, v in result.items():100                    assert (101                        k not in results102                    ), "Different evaluators produce results with the same key {}".format(k)103                    results[k] = v104        return results105 106 107def inference_on_dataset(108    model, data_loader, evaluator: Union[DatasetEvaluator, List[DatasetEvaluator], None]109):110    """111    Run model on the data_loader and evaluate the metrics with evaluator.112    Also benchmark the inference speed of `model.__call__` accurately.113    The model will be used in eval mode.114 115    Args:116        model (callable): a callable which takes an object from117            `data_loader` and returns some outputs.118 119            If it's an nn.Module, it will be temporarily set to `eval` mode.120            If you wish to evaluate a model in `training` mode instead, you can121            wrap the given model and override its behavior of `.eval()` and `.train()`.122        data_loader: an iterable object with a length.123            The elements it generates will be the inputs to the model.124        evaluator: the evaluator(s) to run. Use `None` if you only want to benchmark,125            but don't want to do any evaluation.126 127    Returns:128        The return value of `evaluator.evaluate()`129    """130    num_devices = get_world_size()131    logger = logging.getLogger(__name__)132    logger.info("Start inference on {} batches".format(len(data_loader)))133 134    total = len(data_loader)  # inference data loader must have a fixed length135    if evaluator is None:136        # create a no-op evaluator137        evaluator = DatasetEvaluators([])138    if isinstance(evaluator, abc.MutableSequence):139        evaluator = DatasetEvaluators(evaluator)140    evaluator.reset()141 142    num_warmup = min(5, total - 1)143    start_time = time.perf_counter()144    total_data_time = 0145    total_compute_time = 0146    total_eval_time = 0147    with ExitStack() as stack:148        if isinstance(model, nn.Module):149            stack.enter_context(inference_context(model))150        stack.enter_context(torch.no_grad())151 152        start_data_time = time.perf_counter()153        for idx, inputs in enumerate(data_loader):154            total_data_time += time.perf_counter() - start_data_time155            if idx == num_warmup:156                start_time = time.perf_counter()157                total_data_time = 0158                total_compute_time = 0159                total_eval_time = 0160 161            start_compute_time = time.perf_counter()162            outputs = model(inputs)163            if torch.cuda.is_available():164                torch.cuda.synchronize()165            total_compute_time += time.perf_counter() - start_compute_time166 167            start_eval_time = time.perf_counter()168            evaluator.process(inputs, outputs)169            total_eval_time += time.perf_counter() - start_eval_time170 171            iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup)172            data_seconds_per_iter = total_data_time / iters_after_start173            compute_seconds_per_iter = total_compute_time / iters_after_start174            eval_seconds_per_iter = total_eval_time / iters_after_start175            total_seconds_per_iter = (time.perf_counter() - start_time) / iters_after_start176            if idx >= num_warmup * 2 or compute_seconds_per_iter > 5:177                eta = datetime.timedelta(seconds=int(total_seconds_per_iter * (total - idx - 1)))178                log_every_n_seconds(179                    logging.INFO,180                    (181                        f"Inference done {idx + 1}/{total}. "182                        f"Dataloading: {data_seconds_per_iter:.4f} s/iter. "183                        f"Inference: {compute_seconds_per_iter:.4f} s/iter. "184                        f"Eval: {eval_seconds_per_iter:.4f} s/iter. "185                        f"Total: {total_seconds_per_iter:.4f} s/iter. "186                        f"ETA={eta}"187                    ),188                    n=5,189                )190            start_data_time = time.perf_counter()191 192    # Measure the time only for this worker (before the synchronization barrier)193    total_time = time.perf_counter() - start_time194    total_time_str = str(datetime.timedelta(seconds=total_time))195    # NOTE this format is parsed by grep196    logger.info(197        "Total inference time: {} ({:.6f} s / iter per device, on {} devices)".format(198            total_time_str, total_time / (total - num_warmup), num_devices199        )200    )201    total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time)))202    logger.info(203        "Total inference pure compute time: {} ({:.6f} s / iter per device, on {} devices)".format(204            total_compute_time_str, total_compute_time / (total - num_warmup), num_devices205        )206    )207 208    results = evaluator.evaluate()209    # An evaluator may return None when not in main process.210    # Replace it by an empty dict instead to make it easier for downstream code to handle211    if results is None:212        results = {}213    return results214 215 216@contextmanager217def inference_context(model):218    """219    A context where the model is temporarily changed to eval mode,220    and restored to previous mode afterwards.221 222    Args:223        model: a torch Module224    """225    training_mode = model.training226    model.eval()227    yield228    model.train(training_mode)229