Aluode/PerceptionLabPortable

1# Copyright 2018 The HuggingFace Inc. team.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14"""Configuration base class and utilities."""15 16import copy17import json18import os19import warnings20from dataclasses import dataclass21from pathlib import Path22from typing import Any, Optional, Union23 24import requests25import yaml26from huggingface_hub import model_info27from huggingface_hub.errors import OfflineModeIsEnabled28from huggingface_hub.utils import HFValidationError29 30from . import __version__31from .models.auto.modeling_auto import (32    MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,33    MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,34    MODEL_FOR_CTC_MAPPING_NAMES,35    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,36    MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES,37    MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES,38    MODEL_FOR_MASKED_LM_MAPPING_NAMES,39    MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,40    MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,41    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,42    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,43    MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES,44    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES,45    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,46    MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES,47)48from .training_args import ParallelMode49from .utils import (50    MODEL_CARD_NAME,51    cached_file,52    is_datasets_available,53    is_offline_mode,54    is_tf_available,55    is_tokenizers_available,56    is_torch_available,57    logging,58)59 60 61TASK_MAPPING = {62    "text-generation": MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,63    "image-classification": MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,64    "image-segmentation": MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES,65    "fill-mask": MODEL_FOR_MASKED_LM_MAPPING_NAMES,66    "object-detection": MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,67    "question-answering": MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,68    "text2text-generation": MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,69    "text-classification": MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,70    "table-question-answering": MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES,71    "token-classification": MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,72    "audio-classification": MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,73    "automatic-speech-recognition": {**MODEL_FOR_CTC_MAPPING_NAMES, **MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES},74    "zero-shot-image-classification": MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES,75    "image-text-to-text": MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES,76}77 78logger = logging.get_logger(__name__)79 80 81class ModelCard:82    r"""83    Structured Model Card class. Store model card as well as methods for loading/downloading/saving model cards.84 85    Please read the following paper for details and explanation on the sections: "Model Cards for Model Reporting" by86    Margaret Mitchell, Simone Wu, Andrew Zaldivar, Parker Barnes, Lucy Vasserman, Ben Hutchinson, Elena Spitzer,87    Inioluwa Deborah Raji and Timnit Gebru for the proposal behind model cards. Link: https://huggingface.co/papers/1810.0399388 89    Note: A model card can be loaded and saved to disk.90    """91 92    def __init__(self, **kwargs):93        warnings.warn(94            "The class `ModelCard` is deprecated and will be removed in version 5 of Transformers", FutureWarning95        )96        # Recommended attributes from https://huggingface.co/papers/1810.03993 (see papers)97        self.model_details = kwargs.pop("model_details", {})98        self.intended_use = kwargs.pop("intended_use", {})99        self.factors = kwargs.pop("factors", {})100        self.metrics = kwargs.pop("metrics", {})101        self.evaluation_data = kwargs.pop("evaluation_data", {})102        self.training_data = kwargs.pop("training_data", {})103        self.quantitative_analyses = kwargs.pop("quantitative_analyses", {})104        self.ethical_considerations = kwargs.pop("ethical_considerations", {})105        self.caveats_and_recommendations = kwargs.pop("caveats_and_recommendations", {})106 107        # Open additional attributes108        for key, value in kwargs.items():109            try:110                setattr(self, key, value)111            except AttributeError as err:112                logger.error(f"Can't set {key} with value {value} for {self}")113                raise err114 115    def save_pretrained(self, save_directory_or_file):116        """Save a model card object to the directory or file `save_directory_or_file`."""117        if os.path.isdir(save_directory_or_file):118            # If we save using the predefined names, we can load using `from_pretrained`119            output_model_card_file = os.path.join(save_directory_or_file, MODEL_CARD_NAME)120        else:121            output_model_card_file = save_directory_or_file122 123        self.to_json_file(output_model_card_file)124        logger.info(f"Model card saved in {output_model_card_file}")125 126    @classmethod127    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):128        r"""129        Instantiate a [`ModelCard`] from a pre-trained model model card.130 131        Parameters:132            pretrained_model_name_or_path: either:133 134                - a string, the *model id* of a pretrained model card hosted inside a model repo on huggingface.co.135                - a path to a *directory* containing a model card file saved using the [`~ModelCard.save_pretrained`]136                  method, e.g.: `./my_model_directory/`.137                - a path or url to a saved model card JSON *file*, e.g.: `./my_model_directory/modelcard.json`.138 139            cache_dir: (*optional*) string:140                Path to a directory in which a downloaded pre-trained model card should be cached if the standard cache141                should not be used.142 143            kwargs: (*optional*) dict: key/value pairs with which to update the ModelCard object after loading.144 145                - The values in kwargs of any keys which are model card attributes will be used to override the loaded146                  values.147                - Behavior concerning key/value pairs whose keys are *not* model card attributes is controlled by the148                  *return_unused_kwargs* keyword parameter.149 150            proxies: (*optional*) dict, default None:151                A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128',152                'http://hostname': 'foo.bar:4012'}. The proxies are used on each request.153 154            return_unused_kwargs: (*optional*) bool:155 156                - If False, then this function returns just the final model card object.157                - If True, then this functions returns a tuple *(model card, unused_kwargs)* where *unused_kwargs* is a158                  dictionary consisting of the key/value pairs whose keys are not model card attributes: ie the part of159                  kwargs which has not been used to update *ModelCard* and is otherwise ignored.160 161        Examples:162 163        ```python164        # Download model card from huggingface.co and cache.165        modelcard = ModelCard.from_pretrained("google-bert/bert-base-uncased")166        # Model card was saved using *save_pretrained('./test/saved_model/')*167        modelcard = ModelCard.from_pretrained("./test/saved_model/")168        modelcard = ModelCard.from_pretrained("./test/saved_model/modelcard.json")169        modelcard = ModelCard.from_pretrained("google-bert/bert-base-uncased", output_attentions=True, foo=False)170        ```"""171        cache_dir = kwargs.pop("cache_dir", None)172        proxies = kwargs.pop("proxies", None)173        return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)174        from_pipeline = kwargs.pop("_from_pipeline", None)175 176        user_agent = {"file_type": "model_card"}177        if from_pipeline is not None:178            user_agent["using_pipeline"] = from_pipeline179 180        is_local = os.path.isdir(pretrained_model_name_or_path)181        if os.path.isfile(pretrained_model_name_or_path):182            resolved_model_card_file = pretrained_model_name_or_path183            is_local = True184        else:185            try:186                # Load from URL or cache if already cached187                resolved_model_card_file = cached_file(188                    pretrained_model_name_or_path,189                    filename=MODEL_CARD_NAME,190                    cache_dir=cache_dir,191                    proxies=proxies,192                    user_agent=user_agent,193                )194                if is_local:195                    logger.info(f"loading model card file {resolved_model_card_file}")196                else:197                    logger.info(f"loading model card file {MODEL_CARD_NAME} from cache at {resolved_model_card_file}")198                # Load model card199                modelcard = cls.from_json_file(resolved_model_card_file)200 201            except (OSError, json.JSONDecodeError):202                # We fall back on creating an empty model card203                modelcard = cls()204 205        # Update model card with kwargs if needed206        to_remove = []207        for key, value in kwargs.items():208            if hasattr(modelcard, key):209                setattr(modelcard, key, value)210                to_remove.append(key)211        for key in to_remove:212            kwargs.pop(key, None)213 214        logger.info(f"Model card: {modelcard}")215        if return_unused_kwargs:216            return modelcard, kwargs217        else:218            return modelcard219 220    @classmethod221    def from_dict(cls, json_object):222        """Constructs a `ModelCard` from a Python dictionary of parameters."""223        return cls(**json_object)224 225    @classmethod226    def from_json_file(cls, json_file):227        """Constructs a `ModelCard` from a json file of parameters."""228        with open(json_file, encoding="utf-8") as reader:229            text = reader.read()230        dict_obj = json.loads(text)231        return cls(**dict_obj)232 233    def __eq__(self, other):234        return self.__dict__ == other.__dict__235 236    def __repr__(self):237        return str(self.to_json_string())238 239    def to_dict(self):240        """Serializes this instance to a Python dictionary."""241        output = copy.deepcopy(self.__dict__)242        return output243 244    def to_json_string(self):245        """Serializes this instance to a JSON string."""246        return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n"247 248    def to_json_file(self, json_file_path):249        """Save this instance to a json file."""250        with open(json_file_path, "w", encoding="utf-8") as writer:251            writer.write(self.to_json_string())252 253 254AUTOGENERATED_TRAINER_COMMENT = """255<!-- This model card has been generated automatically according to the information the Trainer had access to. You256should probably proofread and complete it, then remove this comment. -->257"""258 259AUTOGENERATED_KERAS_COMMENT = """260<!-- This model card has been generated automatically according to the information Keras had access to. You should261probably proofread and complete it, then remove this comment. -->262"""263 264 265TASK_TAG_TO_NAME_MAPPING = {266    "fill-mask": "Masked Language Modeling",267    "image-classification": "Image Classification",268    "image-segmentation": "Image Segmentation",269    "multiple-choice": "Multiple Choice",270    "object-detection": "Object Detection",271    "question-answering": "Question Answering",272    "summarization": "Summarization",273    "table-question-answering": "Table Question Answering",274    "text-classification": "Text Classification",275    "text-generation": "Causal Language Modeling",276    "text2text-generation": "Sequence-to-sequence Language Modeling",277    "token-classification": "Token Classification",278    "translation": "Translation",279    "zero-shot-classification": "Zero Shot Classification",280    "automatic-speech-recognition": "Automatic Speech Recognition",281    "audio-classification": "Audio Classification",282}283 284 285METRIC_TAGS = [286    "accuracy",287    "bleu",288    "f1",289    "matthews_correlation",290    "pearsonr",291    "precision",292    "recall",293    "rouge",294    "sacrebleu",295    "spearmanr",296    "wer",297]298 299 300def _listify(obj):301    if obj is None:302        return []303    elif isinstance(obj, str):304        return [obj]305    else:306        return obj307 308 309def _insert_values_as_list(metadata, name, values):310    if values is None:311        return metadata312    if isinstance(values, str):313        values = [values]314    values = [v for v in values if v is not None]315    if len(values) == 0:316        return metadata317    metadata[name] = values318    return metadata319 320 321def infer_metric_tags_from_eval_results(eval_results):322    if eval_results is None:323        return {}324    result = {}325    for key in eval_results:326        if key.lower().replace(" ", "_") in METRIC_TAGS:327            result[key.lower().replace(" ", "_")] = key328        elif key.lower() == "rouge1":329            result["rouge"] = key330    return result331 332 333def _insert_value(metadata, name, value):334    if value is None:335        return metadata336    metadata[name] = value337    return metadata338 339 340def is_hf_dataset(dataset):341    if not is_datasets_available():342        return False343 344    from datasets import Dataset, IterableDataset345 346    return isinstance(dataset, (Dataset, IterableDataset))347 348 349def _get_mapping_values(mapping):350    result = []351    for v in mapping.values():352        if isinstance(v, (tuple, list)):353            result += list(v)354        else:355            result.append(v)356    return result357 358 359@dataclass360class TrainingSummary:361    model_name: str362    language: Optional[Union[str, list[str]]] = None363    license: Optional[str] = None364    tags: Optional[Union[str, list[str]]] = None365    finetuned_from: Optional[str] = None366    tasks: Optional[Union[str, list[str]]] = None367    dataset: Optional[Union[str, list[str]]] = None368    dataset_tags: Optional[Union[str, list[str]]] = None369    dataset_args: Optional[Union[str, list[str]]] = None370    dataset_metadata: Optional[dict[str, Any]] = None371    eval_results: Optional[dict[str, float]] = None372    eval_lines: Optional[list[str]] = None373    hyperparameters: Optional[dict[str, Any]] = None374    source: Optional[str] = "trainer"375 376    def __post_init__(self):377        # Infer default license from the checkpoint used, if possible.378        if (379            self.license is None380            and not is_offline_mode()381            and self.finetuned_from is not None382            and len(self.finetuned_from) > 0383        ):384            try:385                info = model_info(self.finetuned_from)386                for tag in info.tags:387                    if tag.startswith("license:"):388                        self.license = tag[8:]389            except (390                requests.exceptions.HTTPError,391                requests.exceptions.ConnectionError,392                HFValidationError,393                OfflineModeIsEnabled,394            ):395                pass396 397    def create_model_index(self, metric_mapping):398        model_index = {"name": self.model_name}399 400        # Dataset mapping tag -> name401        dataset_names = _listify(self.dataset)402        dataset_tags = _listify(self.dataset_tags)403        dataset_args = _listify(self.dataset_args)404        dataset_metadata = _listify(self.dataset_metadata)405        if len(dataset_args) < len(dataset_tags):406            dataset_args = dataset_args + [None] * (len(dataset_tags) - len(dataset_args))407        dataset_mapping = dict(zip(dataset_tags, dataset_names))408        dataset_arg_mapping = dict(zip(dataset_tags, dataset_args))409        dataset_metadata_mapping = dict(zip(dataset_tags, dataset_metadata))410 411        task_mapping = {412            task: TASK_TAG_TO_NAME_MAPPING[task] for task in _listify(self.tasks) if task in TASK_TAG_TO_NAME_MAPPING413        }414 415        model_index["results"] = []416 417        if len(task_mapping) == 0 and len(dataset_mapping) == 0:418            return [model_index]419        if len(task_mapping) == 0:420            task_mapping = {None: None}421        if len(dataset_mapping) == 0:422            dataset_mapping = {None: None}423 424        # One entry per dataset and per task425        all_possibilities = [(task_tag, ds_tag) for task_tag in task_mapping for ds_tag in dataset_mapping]426        for task_tag, ds_tag in all_possibilities:427            result = {}428            if task_tag is not None:429                result["task"] = {"name": task_mapping[task_tag], "type": task_tag}430 431            if ds_tag is not None:432                metadata = dataset_metadata_mapping.get(ds_tag, {})433                result["dataset"] = {434                    "name": dataset_mapping[ds_tag],435                    "type": ds_tag,436                    **metadata,437                }438                if dataset_arg_mapping[ds_tag] is not None:439                    result["dataset"]["args"] = dataset_arg_mapping[ds_tag]440 441            if len(metric_mapping) > 0:442                result["metrics"] = []443                for metric_tag, metric_name in metric_mapping.items():444                    result["metrics"].append(445                        {446                            "name": metric_name,447                            "type": metric_tag,448                            "value": self.eval_results[metric_name],449                        }450                    )451 452            # Remove partial results to avoid the model card being rejected.453            if "task" in result and "dataset" in result and "metrics" in result:454                model_index["results"].append(result)455            else:456                logger.info(f"Dropping the following result as it does not have all the necessary fields:\n{result}")457 458        return [model_index]459 460    def create_metadata(self):461        metric_mapping = infer_metric_tags_from_eval_results(self.eval_results)462 463        metadata = {}464        metadata = _insert_value(metadata, "library_name", "transformers")465        metadata = _insert_values_as_list(metadata, "language", self.language)466        metadata = _insert_value(metadata, "license", self.license)467        if self.finetuned_from is not None and isinstance(self.finetuned_from, str) and len(self.finetuned_from) > 0:468            metadata = _insert_value(metadata, "base_model", self.finetuned_from)469        metadata = _insert_values_as_list(metadata, "tags", self.tags)470        metadata = _insert_values_as_list(metadata, "datasets", self.dataset_tags)471        metadata = _insert_values_as_list(metadata, "metrics", list(metric_mapping.keys()))472        metadata["model-index"] = self.create_model_index(metric_mapping)473 474        return metadata475 476    def to_model_card(self):477        model_card = ""478 479        metadata = yaml.dump(self.create_metadata(), sort_keys=False)480        if len(metadata) > 0:481            model_card = f"---\n{metadata}---\n"482 483        # Now the model card for realsies.484        if self.source == "trainer":485            model_card += AUTOGENERATED_TRAINER_COMMENT486        else:487            model_card += AUTOGENERATED_KERAS_COMMENT488 489        model_card += f"\n# {self.model_name}\n\n"490 491        if self.finetuned_from is None:492            model_card += "This model was trained from scratch on "493        else:494            model_card += (495                "This model is a fine-tuned version of"496                f" [{self.finetuned_from}](https://huggingface.co/{self.finetuned_from}) on "497            )498 499        if self.dataset is None or (isinstance(self.dataset, list) and len(self.dataset) == 0):500            model_card += "an unknown dataset."501        else:502            if isinstance(self.dataset, str):503                model_card += f"the {self.dataset} dataset."504            elif isinstance(self.dataset, (tuple, list)) and len(self.dataset) == 1:505                model_card += f"the {self.dataset[0]} dataset."506            else:507                model_card += (508                    ", ".join([f"the {ds}" for ds in self.dataset[:-1]]) + f" and the {self.dataset[-1]} datasets."509                )510 511        if self.eval_results is not None:512            model_card += "\nIt achieves the following results on the evaluation set:\n"513            model_card += "\n".join([f"- {name}: {_maybe_round(value)}" for name, value in self.eval_results.items()])514        model_card += "\n"515 516        model_card += "\n## Model description\n\nMore information needed\n"517        model_card += "\n## Intended uses & limitations\n\nMore information needed\n"518        model_card += "\n## Training and evaluation data\n\nMore information needed\n"519 520        model_card += "\n## Training procedure\n"521        model_card += "\n### Training hyperparameters\n"522        if self.hyperparameters is not None:523            model_card += "\nThe following hyperparameters were used during training:\n"524            model_card += "\n".join([f"- {name}: {value}" for name, value in self.hyperparameters.items()])525            model_card += "\n"526        else:527            model_card += "\nMore information needed\n"528 529        if self.eval_lines is not None:530            model_card += "\n### Training results\n\n"531            model_card += make_markdown_table(self.eval_lines)532            model_card += "\n"533 534        model_card += "\n### Framework versions\n\n"535        model_card += f"- Transformers {__version__}\n"536 537        if self.source == "trainer" and is_torch_available():538            import torch539 540            model_card += f"- Pytorch {torch.__version__}\n"541        elif self.source == "keras" and is_tf_available():542            import tensorflow as tf543 544            model_card += f"- TensorFlow {tf.__version__}\n"545        if is_datasets_available():546            import datasets547 548            model_card += f"- Datasets {datasets.__version__}\n"549        if is_tokenizers_available():550            import tokenizers551 552            model_card += f"- Tokenizers {tokenizers.__version__}\n"553 554        return model_card555 556    @classmethod557    def from_trainer(558        cls,559        trainer,560        language=None,561        license=None,562        tags=None,563        model_name=None,564        finetuned_from=None,565        tasks=None,566        dataset_tags=None,567        dataset_metadata=None,568        dataset=None,569        dataset_args=None,570    ):571        # Infer default from dataset572        one_dataset = trainer.eval_dataset if trainer.eval_dataset is not None else trainer.train_dataset573        if is_hf_dataset(one_dataset) and (dataset_tags is None or dataset_args is None or dataset_metadata is None):574            default_tag = one_dataset.builder_name575            # Those are not real datasets from the Hub so we exclude them.576            if default_tag not in ["csv", "json", "pandas", "parquet", "text"]:577                if dataset_metadata is None:578                    dataset_metadata = [{"config": one_dataset.config_name, "split": str(one_dataset.split)}]579                if dataset_tags is None:580                    dataset_tags = [default_tag]581                if dataset_args is None:582                    dataset_args = [one_dataset.config_name]583 584        if dataset is None and dataset_tags is not None:585            dataset = dataset_tags586 587        # Infer default finetuned_from588        if (589            finetuned_from is None590            and hasattr(trainer.model.config, "_name_or_path")591            and not os.path.isdir(trainer.model.config._name_or_path)592        ):593            finetuned_from = trainer.model.config._name_or_path594 595        # Infer default task tag:596        if tasks is None:597            model_class_name = trainer.model.__class__.__name__598            for task, mapping in TASK_MAPPING.items():599                if model_class_name in _get_mapping_values(mapping):600                    tasks = task601 602        if model_name is None:603            model_name = Path(trainer.args.output_dir).name604        if len(model_name) == 0:605            model_name = finetuned_from606 607        # Add `generated_from_trainer` to the tags608        if tags is None:609            tags = ["generated_from_trainer"]610        elif isinstance(tags, str) and tags != "generated_from_trainer":611            tags = [tags, "generated_from_trainer"]612        elif "generated_from_trainer" not in tags:613            tags.append("generated_from_trainer")614 615        _, eval_lines, eval_results = parse_log_history(trainer.state.log_history)616        hyperparameters = extract_hyperparameters_from_trainer(trainer)617 618        return cls(619            language=language,620            license=license,621            tags=tags,622            model_name=model_name,623            finetuned_from=finetuned_from,624            tasks=tasks,625            dataset=dataset,626            dataset_tags=dataset_tags,627            dataset_args=dataset_args,628            dataset_metadata=dataset_metadata,629            eval_results=eval_results,630            eval_lines=eval_lines,631            hyperparameters=hyperparameters,632        )633 634    @classmethod635    def from_keras(636        cls,637        model,638        model_name,639        keras_history=None,640        language=None,641        license=None,642        tags=None,643        finetuned_from=None,644        tasks=None,645        dataset_tags=None,646        dataset=None,647        dataset_args=None,648    ):649        # Infer default from dataset650        if dataset is not None:651            if is_hf_dataset(dataset) and (dataset_tags is None or dataset_args is None):652                default_tag = dataset.builder_name653                # Those are not real datasets from the Hub so we exclude them.654                if default_tag not in ["csv", "json", "pandas", "parquet", "text"]:655                    if dataset_tags is None:656                        dataset_tags = [default_tag]657                    if dataset_args is None:658                        dataset_args = [dataset.config_name]659 660        if dataset is None and dataset_tags is not None:661            dataset = dataset_tags662 663        # Infer default finetuned_from664        if (665            finetuned_from is None666            and hasattr(model.config, "_name_or_path")667            and not os.path.isdir(model.config._name_or_path)668        ):669            finetuned_from = model.config._name_or_path670 671        # Infer default task tag:672        if tasks is None:673            model_class_name = model.__class__.__name__674            for task, mapping in TASK_MAPPING.items():675                if model_class_name in _get_mapping_values(mapping):676                    tasks = task677 678        # Add `generated_from_keras_callback` to the tags679        if tags is None:680            tags = ["generated_from_keras_callback"]681        elif isinstance(tags, str) and tags != "generated_from_keras_callback":682            tags = [tags, "generated_from_keras_callback"]683        elif "generated_from_keras_callback" not in tags:684            tags.append("generated_from_keras_callback")685 686        if keras_history is not None:687            _, eval_lines, eval_results = parse_keras_history(keras_history)688        else:689            eval_lines = []690            eval_results = {}691        hyperparameters = extract_hyperparameters_from_keras(model)692 693        return cls(694            language=language,695            license=license,696            tags=tags,697            model_name=model_name,698            finetuned_from=finetuned_from,699            tasks=tasks,700            dataset_tags=dataset_tags,701            dataset=dataset,702            dataset_args=dataset_args,703            eval_results=eval_results,704            eval_lines=eval_lines,705            hyperparameters=hyperparameters,706            source="keras",707        )708 709 710def parse_keras_history(logs):711    """712    Parse the `logs` of either a `keras.History` object returned by `model.fit()` or an accumulated logs `dict`713    passed to the `PushToHubCallback`. Returns lines and logs compatible with those returned by `parse_log_history`.714    """715    if hasattr(logs, "history"):716        # This looks like a `History` object717        if not hasattr(logs, "epoch"):718            # This history looks empty, return empty results719            return None, [], {}720        logs.history["epoch"] = logs.epoch721        logs = logs.history722    else:723        # Training logs is a list of dicts, let's invert it to a dict of lists to match a History object724        logs = {log_key: [single_dict[log_key] for single_dict in logs] for log_key in logs[0]}725 726    lines = []727    for i in range(len(logs["epoch"])):728        epoch_dict = {log_key: log_value_list[i] for log_key, log_value_list in logs.items()}729        values = {}730        for k, v in epoch_dict.items():731            if k.startswith("val_"):732                k = "validation_" + k[4:]733            elif k != "epoch":734                k = "train_" + k735            splits = k.split("_")736            name = " ".join([part.capitalize() for part in splits])737            values[name] = v738        lines.append(values)739 740    eval_results = lines[-1]741 742    return logs, lines, eval_results743 744 745def parse_log_history(log_history):746    """747    Parse the `log_history` of a Trainer to get the intermediate and final evaluation results.748    """749    idx = 0750    while idx < len(log_history) and "train_runtime" not in log_history[idx]:751        idx += 1752 753    # If there are no training logs754    if idx == len(log_history):755        idx -= 1756        while idx >= 0 and "eval_loss" not in log_history[idx]:757            idx -= 1758 759        if idx >= 0:760            return None, None, log_history[idx]761        else:762            return None, None, None763 764    # From now one we can assume we have training logs:765    train_log = log_history[idx]766    lines = []767    training_loss = "No log"768    for i in range(idx):769        if "loss" in log_history[i]:770            training_loss = log_history[i]["loss"]771        if "eval_loss" in log_history[i]:772            metrics = log_history[i].copy()773            _ = metrics.pop("total_flos", None)774            epoch = metrics.pop("epoch", None)775            step = metrics.pop("step", None)776            _ = metrics.pop("eval_runtime", None)777            _ = metrics.pop("eval_samples_per_second", None)778            _ = metrics.pop("eval_steps_per_second", None)779            _ = metrics.pop("eval_jit_compilation_time", None)780            values = {"Training Loss": training_loss, "Epoch": epoch, "Step": step}781            for k, v in metrics.items():782                if k == "eval_loss":783                    values["Validation Loss"] = v784                else:785                    splits = k.split("_")786                    name = " ".join([part.capitalize() for part in splits[1:]])787                    values[name] = v788            lines.append(values)789 790    idx = len(log_history) - 1791    while idx >= 0 and "eval_loss" not in log_history[idx]:792        idx -= 1793 794    if idx > 0:795        eval_results = {}796        for key, value in log_history[idx].items():797            key = key.removeprefix("eval_")798            if key not in ["runtime", "samples_per_second", "steps_per_second", "epoch", "step"]:799                camel_cased_key = " ".join([part.capitalize() for part in key.split("_")])800                eval_results[camel_cased_key] = value801        return train_log, lines, eval_results802    else:803        return train_log, lines, None804 805 806def extract_hyperparameters_from_keras(model):807    from .modeling_tf_utils import keras808 809    hyperparameters = {}810    if hasattr(model, "optimizer") and model.optimizer is not None:811        hyperparameters["optimizer"] = model.optimizer.get_config()812    else:813        hyperparameters["optimizer"] = None814    hyperparameters["training_precision"] = keras.mixed_precision.global_policy().name815 816    return hyperparameters817 818 819def _maybe_round(v, decimals=4):820    if isinstance(v, float) and len(str(v).split(".")) > 1 and len(str(v).split(".")[1]) > decimals:821        return f"{v:.{decimals}f}"822    return str(v)823 824 825def _regular_table_line(values, col_widths):826    values_with_space = [f"| {v}" + " " * (w - len(v) + 1) for v, w in zip(values, col_widths)]827    return "".join(values_with_space) + "|\n"828 829 830def _second_table_line(col_widths):831    values = ["|:" + "-" * w + ":" for w in col_widths]832    return "".join(values) + "|\n"833 834 835def make_markdown_table(lines):836    """837    Create a nice Markdown table from the results in `lines`.838    """839    if lines is None or len(lines) == 0:840        return ""841    col_widths = {key: len(str(key)) for key in lines[0]}842    for line in lines:843        for key, value in line.items():844            if col_widths[key] < len(_maybe_round(value)):845                col_widths[key] = len(_maybe_round(value))846 847    table = _regular_table_line(list(lines[0].keys()), list(col_widths.values()))848    table += _second_table_line(list(col_widths.values()))849    for line in lines:850        table += _regular_table_line([_maybe_round(v) for v in line.values()], list(col_widths.values()))851    return table852 853 854_TRAINING_ARGS_KEYS = [855    "learning_rate",856    "train_batch_size",857    "eval_batch_size",858    "seed",859]860 861 862def extract_hyperparameters_from_trainer(trainer):863    hyperparameters = {k: getattr(trainer.args, k) for k in _TRAINING_ARGS_KEYS}864 865    if trainer.args.parallel_mode not in [ParallelMode.NOT_PARALLEL, ParallelMode.NOT_DISTRIBUTED]:866        hyperparameters["distributed_type"] = (867            "multi-GPU" if trainer.args.parallel_mode == ParallelMode.DISTRIBUTED else trainer.args.parallel_mode.value868        )869    if trainer.args.world_size > 1:870        hyperparameters["num_devices"] = trainer.args.world_size871    if trainer.args.gradient_accumulation_steps > 1:872        hyperparameters["gradient_accumulation_steps"] = trainer.args.gradient_accumulation_steps873 874    total_train_batch_size = (875        trainer.args.train_batch_size * trainer.args.world_size * trainer.args.gradient_accumulation_steps876    )877    if total_train_batch_size != hyperparameters["train_batch_size"]:878        hyperparameters["total_train_batch_size"] = total_train_batch_size879    total_eval_batch_size = trainer.args.eval_batch_size * trainer.args.world_size880    if total_eval_batch_size != hyperparameters["eval_batch_size"]:881        hyperparameters["total_eval_batch_size"] = total_eval_batch_size882 883    if trainer.args.optim:884        optimizer_name = trainer.args.optim885        optimizer_args = trainer.args.optim_args if trainer.args.optim_args else "No additional optimizer arguments"886 887        if "adam" in optimizer_name.lower():888            hyperparameters["optimizer"] = (889                f"Use {optimizer_name} with betas=({trainer.args.adam_beta1},{trainer.args.adam_beta2}) and"890                f" epsilon={trainer.args.adam_epsilon} and optimizer_args={optimizer_args}"891            )892        else:893            hyperparameters["optimizer"] = f"Use {optimizer_name} and the args are:\n{optimizer_args}"894 895    hyperparameters["lr_scheduler_type"] = trainer.args.lr_scheduler_type.value896    if trainer.args.warmup_ratio != 0.0:897        hyperparameters["lr_scheduler_warmup_ratio"] = trainer.args.warmup_ratio898    if trainer.args.warmup_steps != 0.0:899        hyperparameters["lr_scheduler_warmup_steps"] = trainer.args.warmup_steps900    if trainer.args.max_steps != -1:901        hyperparameters["training_steps"] = trainer.args.max_steps902    else:903        hyperparameters["num_epochs"] = trainer.args.num_train_epochs904 905    if trainer.args.fp16:906        if trainer.use_apex:907            hyperparameters["mixed_precision_training"] = f"Apex, opt level {trainer.args.fp16_opt_level}"908        else:909            hyperparameters["mixed_precision_training"] = "Native AMP"910 911    if trainer.args.label_smoothing_factor != 0.0:912        hyperparameters["label_smoothing_factor"] = trainer.args.label_smoothing_factor913 914    return hyperparameters915