Aluode/PerceptionLabPortable

1# Copyright 2020 Optuna, Hugging Face2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""Logging utilities."""15 16import functools17import logging18import os19import sys20import threading21from logging import (22    CRITICAL,  # NOQA23    DEBUG,24    ERROR,25    FATAL,  # NOQA26    INFO,27    NOTSET,  # NOQA28    WARN,  # NOQA29    WARNING,30)31from logging import captureWarnings as _captureWarnings32from typing import Optional33 34import huggingface_hub.utils as hf_hub_utils35from tqdm import auto as tqdm_lib36 37 38_lock = threading.Lock()39_default_handler: Optional[logging.Handler] = None40 41log_levels = {42    "detail": logging.DEBUG,  # will also print filename and line number43    "debug": logging.DEBUG,44    "info": logging.INFO,45    "warning": logging.WARNING,46    "error": logging.ERROR,47    "critical": logging.CRITICAL,48}49 50_default_log_level = logging.WARNING51 52_tqdm_active = not hf_hub_utils.are_progress_bars_disabled()53 54 55def _get_default_logging_level():56    """57    If TRANSFORMERS_VERBOSITY env var is set to one of the valid choices return that as the new default level. If it is58    not - fall back to `_default_log_level`59    """60    env_level_str = os.getenv("TRANSFORMERS_VERBOSITY", None)61    if env_level_str:62        if env_level_str in log_levels:63            return log_levels[env_level_str]64        else:65            logging.getLogger().warning(66                f"Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, "67                f"has to be one of: {', '.join(log_levels.keys())}"68            )69    return _default_log_level70 71 72def _get_library_name() -> str:73    return __name__.split(".")[0]74 75 76def _get_library_root_logger() -> logging.Logger:77    return logging.getLogger(_get_library_name())78 79 80def _configure_library_root_logger() -> None:81    global _default_handler82 83    with _lock:84        if _default_handler:85            # This library has already configured the library root logger.86            return87        _default_handler = logging.StreamHandler()  # Set sys.stderr as stream.88        # set defaults based on https://github.com/pyinstaller/pyinstaller/issues/7334#issuecomment-135744717689        if sys.stderr is None:90            sys.stderr = open(os.devnull, "w")91 92        _default_handler.flush = sys.stderr.flush93 94        # Apply our default configuration to the library root logger.95        library_root_logger = _get_library_root_logger()96        library_root_logger.addHandler(_default_handler)97        library_root_logger.setLevel(_get_default_logging_level())98        # if logging level is debug, we add pathname and lineno to formatter for easy debugging99        if os.getenv("TRANSFORMERS_VERBOSITY", None) == "detail":100            formatter = logging.Formatter("[%(levelname)s|%(pathname)s:%(lineno)s] %(asctime)s >> %(message)s")101            _default_handler.setFormatter(formatter)102 103        is_ci = os.getenv("CI") is not None and os.getenv("CI").upper() in {"1", "ON", "YES", "TRUE"}104        library_root_logger.propagate = is_ci105 106 107def _reset_library_root_logger() -> None:108    global _default_handler109 110    with _lock:111        if not _default_handler:112            return113 114        library_root_logger = _get_library_root_logger()115        library_root_logger.removeHandler(_default_handler)116        library_root_logger.setLevel(logging.NOTSET)117        _default_handler = None118 119 120def get_log_levels_dict():121    return log_levels122 123 124def captureWarnings(capture):125    """126    Calls the `captureWarnings` method from the logging library to enable management of the warnings emitted by the127    `warnings` library.128 129    Read more about this method here:130    https://docs.python.org/3/library/logging.html#integration-with-the-warnings-module131 132    All warnings will be logged through the `py.warnings` logger.133 134    Careful: this method also adds a handler to this logger if it does not already have one, and updates the logging135    level of that logger to the library's root logger.136    """137    logger = get_logger("py.warnings")138 139    if not logger.handlers:140        logger.addHandler(_default_handler)141 142    logger.setLevel(_get_library_root_logger().level)143 144    _captureWarnings(capture)145 146 147def get_logger(name: Optional[str] = None) -> logging.Logger:148    """149    Return a logger with the specified name.150 151    This function is not supposed to be directly accessed unless you are writing a custom transformers module.152    """153 154    if name is None:155        name = _get_library_name()156 157    _configure_library_root_logger()158    return logging.getLogger(name)159 160 161def get_verbosity() -> int:162    """163    Return the current level for the 🤗 Transformers's root logger as an int.164 165    Returns:166        `int`: The logging level.167 168    <Tip>169 170    🤗 Transformers has following logging levels:171 172    - 50: `transformers.logging.CRITICAL` or `transformers.logging.FATAL`173    - 40: `transformers.logging.ERROR`174    - 30: `transformers.logging.WARNING` or `transformers.logging.WARN`175    - 20: `transformers.logging.INFO`176    - 10: `transformers.logging.DEBUG`177 178    </Tip>"""179 180    _configure_library_root_logger()181    return _get_library_root_logger().getEffectiveLevel()182 183 184def set_verbosity(verbosity: int) -> None:185    """186    Set the verbosity level for the 🤗 Transformers's root logger.187 188    Args:189        verbosity (`int`):190            Logging level, e.g., one of:191 192            - `transformers.logging.CRITICAL` or `transformers.logging.FATAL`193            - `transformers.logging.ERROR`194            - `transformers.logging.WARNING` or `transformers.logging.WARN`195            - `transformers.logging.INFO`196            - `transformers.logging.DEBUG`197    """198 199    _configure_library_root_logger()200    _get_library_root_logger().setLevel(verbosity)201 202 203def set_verbosity_info():204    """Set the verbosity to the `INFO` level."""205    return set_verbosity(INFO)206 207 208def set_verbosity_warning():209    """Set the verbosity to the `WARNING` level."""210    return set_verbosity(WARNING)211 212 213def set_verbosity_debug():214    """Set the verbosity to the `DEBUG` level."""215    return set_verbosity(DEBUG)216 217 218def set_verbosity_error():219    """Set the verbosity to the `ERROR` level."""220    return set_verbosity(ERROR)221 222 223def disable_default_handler() -> None:224    """Disable the default handler of the HuggingFace Transformers's root logger."""225 226    _configure_library_root_logger()227 228    assert _default_handler is not None229    _get_library_root_logger().removeHandler(_default_handler)230 231 232def enable_default_handler() -> None:233    """Enable the default handler of the HuggingFace Transformers's root logger."""234 235    _configure_library_root_logger()236 237    assert _default_handler is not None238    _get_library_root_logger().addHandler(_default_handler)239 240 241def add_handler(handler: logging.Handler) -> None:242    """adds a handler to the HuggingFace Transformers's root logger."""243 244    _configure_library_root_logger()245 246    assert handler is not None247    _get_library_root_logger().addHandler(handler)248 249 250def remove_handler(handler: logging.Handler) -> None:251    """removes given handler from the HuggingFace Transformers's root logger."""252 253    _configure_library_root_logger()254 255    assert handler is not None and handler not in _get_library_root_logger().handlers256    _get_library_root_logger().removeHandler(handler)257 258 259def disable_propagation() -> None:260    """261    Disable propagation of the library log outputs. Note that log propagation is disabled by default.262    """263 264    _configure_library_root_logger()265    _get_library_root_logger().propagate = False266 267 268def enable_propagation() -> None:269    """270    Enable propagation of the library log outputs. Please disable the HuggingFace Transformers's default handler to271    prevent double logging if the root logger has been configured.272    """273 274    _configure_library_root_logger()275    _get_library_root_logger().propagate = True276 277 278def enable_explicit_format() -> None:279    """280    Enable explicit formatting for every HuggingFace Transformers's logger. The explicit formatter is as follows:281    ```282        [LEVELNAME|FILENAME|LINE NUMBER] TIME >> MESSAGE283    ```284    All handlers currently bound to the root logger are affected by this method.285    """286    handlers = _get_library_root_logger().handlers287 288    for handler in handlers:289        formatter = logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s")290        handler.setFormatter(formatter)291 292 293def reset_format() -> None:294    """295    Resets the formatting for HuggingFace Transformers's loggers.296 297    All handlers currently bound to the root logger are affected by this method.298    """299    handlers = _get_library_root_logger().handlers300 301    for handler in handlers:302        handler.setFormatter(None)303 304 305def warning_advice(self, *args, **kwargs):306    """307    This method is identical to `logger.warning()`, but if env var TRANSFORMERS_NO_ADVISORY_WARNINGS=1 is set, this308    warning will not be printed309    """310    no_advisory_warnings = os.getenv("TRANSFORMERS_NO_ADVISORY_WARNINGS")311    if no_advisory_warnings:312        return313    self.warning(*args, **kwargs)314 315 316logging.Logger.warning_advice = warning_advice317 318 319@functools.lru_cache(None)320def warning_once(self, *args, **kwargs):321    """322    This method is identical to `logger.warning()`, but will emit the warning with the same message only once323 324    Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the cache.325    The assumption here is that all warning messages are unique across the code. If they aren't then need to switch to326    another type of cache that includes the caller frame information in the hashing function.327    """328    self.warning(*args, **kwargs)329 330 331logging.Logger.warning_once = warning_once332 333 334@functools.lru_cache(None)335def info_once(self, *args, **kwargs):336    """337    This method is identical to `logger.info()`, but will emit the info with the same message only once338 339    Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the cache.340    The assumption here is that all warning messages are unique across the code. If they aren't then need to switch to341    another type of cache that includes the caller frame information in the hashing function.342    """343    self.info(*args, **kwargs)344 345 346logging.Logger.info_once = info_once347 348 349class EmptyTqdm:350    """Dummy tqdm which doesn't do anything."""351 352    def __init__(self, *args, **kwargs):  # pylint: disable=unused-argument353        self._iterator = args[0] if args else None354 355    def __iter__(self):356        return iter(self._iterator)357 358    def __getattr__(self, _):359        """Return empty function."""360 361        def empty_fn(*args, **kwargs):  # pylint: disable=unused-argument362            return363 364        return empty_fn365 366    def __enter__(self):367        return self368 369    def __exit__(self, type_, value, traceback):370        return371 372 373class _tqdm_cls:374    def __call__(self, *args, **kwargs):375        if _tqdm_active:376            return tqdm_lib.tqdm(*args, **kwargs)377        else:378            return EmptyTqdm(*args, **kwargs)379 380    def set_lock(self, *args, **kwargs):381        self._lock = None382        if _tqdm_active:383            return tqdm_lib.tqdm.set_lock(*args, **kwargs)384 385    def get_lock(self):386        if _tqdm_active:387            return tqdm_lib.tqdm.get_lock()388 389 390tqdm = _tqdm_cls()391 392 393def is_progress_bar_enabled() -> bool:394    """Return a boolean indicating whether tqdm progress bars are enabled."""395    return bool(_tqdm_active)396 397 398def enable_progress_bar():399    """Enable tqdm progress bar."""400    global _tqdm_active401    _tqdm_active = True402    hf_hub_utils.enable_progress_bars()403 404 405def disable_progress_bar():406    """Disable tqdm progress bar."""407    global _tqdm_active408    _tqdm_active = False409    hf_hub_utils.disable_progress_bars()410