Aluode/PerceptionLabPortable
0
1# Copyright 2020 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import contextlib16import json17import math18import os19import warnings20from dataclasses import asdict, dataclass, field, fields21from datetime import timedelta22from enum import Enum23from functools import cached_property24from pathlib import Path25from typing import Any, Optional, Union26 27from huggingface_hub import get_full_repo_name28 29from .debug_utils import DebugOption30from .trainer_utils import (31 EvaluationStrategy,32 FSDPOption,33 HubStrategy,34 IntervalStrategy,35 SaveStrategy,36 SchedulerType,37)38from .utils import (39 ACCELERATE_MIN_VERSION,40 ExplicitEnum,41 is_accelerate_available,42 is_apex_available,43 is_ipex_available,44 is_sagemaker_dp_enabled,45 is_sagemaker_mp_enabled,46 is_torch_available,47 is_torch_bf16_gpu_available,48 is_torch_cuda_available,49 is_torch_hpu_available,50 is_torch_mlu_available,51 is_torch_mps_available,52 is_torch_musa_available,53 is_torch_neuroncore_available,54 is_torch_npu_available,55 is_torch_tf32_available,56 is_torch_xla_available,57 is_torch_xpu_available,58 logging,59 requires_backends,60)61from .utils.generic import strtobool62from .utils.import_utils import is_optimum_neuron_available63 64 65logger = logging.get_logger(__name__)66log_levels = logging.get_log_levels_dict().copy()67trainer_log_levels = dict(**log_levels, passive=-1)68 69if is_torch_available():70 import torch71 import torch.distributed as dist72 73if is_accelerate_available():74 from accelerate.state import AcceleratorState, PartialState75 from accelerate.utils import DistributedType76 77 from .trainer_pt_utils import AcceleratorConfig78 79if is_accelerate_available("1.10.1"):80 from accelerate.parallelism_config import ParallelismConfig81else:82 ParallelismConfig = Any83 84if is_torch_xla_available():85 import torch_xla.core.xla_model as xm86 87if is_torch_neuroncore_available(check_device=False):88 # torchrun support89 # https://github.com/pytorch/xla/pull/360990 if os.environ.get("TORCHELASTIC_RUN_ID"):91 if is_optimum_neuron_available():92 logger.info(93 "Make sure that you are performing the training with the NeuronTrainer from optimum[neuron], this "94 "will fail otherwise."95 )96 else:97 logger.warning(98 "Please use the NeuronTrainer from optimum[neuron] instead of the Transformers library to perform "99 "training on AWS Trainium instances. More information here: "100 "https://github.com/huggingface/optimum-neuron"101 )102 import torch_xla.distributed.xla_backend as xbn103 104 if not isinstance(dist.group.WORLD, xbn.ProcessGroupXla):105 dist.init_process_group(backend="xla")106 if not isinstance(dist.group.WORLD, xbn.ProcessGroupXla):107 raise AssertionError("Failed to initialize torch.distributed process group using XLA backend.")108 109 110if is_sagemaker_mp_enabled():111 import smdistributed.modelparallel.torch as smp112 113 smp.init()114 115 116def default_logdir() -> str:117 """118 Same default as PyTorch119 """120 import socket121 from datetime import datetime122 123 current_time = datetime.now().strftime("%b%d_%H-%M-%S")124 return os.path.join("runs", current_time + "_" + socket.gethostname())125 126 127def get_int_from_env(env_keys, default):128 """Returns the first positive env value found in the `env_keys` list or the default."""129 for e in env_keys:130 val = int(os.environ.get(e, "-1"))131 if val >= 0:132 return val133 return default134 135 136def get_xla_device_type(device: "torch.device") -> Optional[str]:137 """138 Returns the xla device type (CPU|GPU|TPU) or None if the device is a non-xla device.139 """140 if is_torch_xla_available():141 if device.type == "cpu":142 return "CPU"143 return xm.xla_real_devices([device])[0].split(":")[0]144 return None145 146 147class OptimizerNames(ExplicitEnum):148 """149 Stores the acceptable string identifiers for optimizers.150 """151 152 ADAMW_TORCH = "adamw_torch"153 ADAMW_TORCH_FUSED = "adamw_torch_fused"154 ADAMW_TORCH_XLA = "adamw_torch_xla"155 ADAMW_TORCH_NPU_FUSED = "adamw_torch_npu_fused"156 ADAMW_APEX_FUSED = "adamw_apex_fused"157 ADAFACTOR = "adafactor"158 ADAMW_ANYPRECISION = "adamw_anyprecision"159 ADAMW_TORCH_4BIT = "adamw_torch_4bit"160 ADAMW_TORCH_8BIT = "adamw_torch_8bit"161 ADEMAMIX = "ademamix"162 SGD = "sgd"163 ADAGRAD = "adagrad"164 ADAMW_BNB = "adamw_bnb_8bit"165 ADAMW_8BIT = "adamw_8bit" # just an alias for adamw_bnb_8bit166 ADEMAMIX_8BIT = "ademamix_8bit"167 LION_8BIT = "lion_8bit"168 LION = "lion_32bit"169 PAGED_ADAMW = "paged_adamw_32bit"170 PAGED_ADAMW_8BIT = "paged_adamw_8bit"171 PAGED_ADEMAMIX = "paged_ademamix_32bit"172 PAGED_ADEMAMIX_8BIT = "paged_ademamix_8bit"173 PAGED_LION = "paged_lion_32bit"174 PAGED_LION_8BIT = "paged_lion_8bit"175 RMSPROP = "rmsprop"176 RMSPROP_BNB = "rmsprop_bnb"177 RMSPROP_8BIT = "rmsprop_bnb_8bit"178 RMSPROP_32BIT = "rmsprop_bnb_32bit"179 GALORE_ADAMW = "galore_adamw"180 GALORE_ADAMW_8BIT = "galore_adamw_8bit"181 GALORE_ADAFACTOR = "galore_adafactor"182 GALORE_ADAMW_LAYERWISE = "galore_adamw_layerwise"183 GALORE_ADAMW_8BIT_LAYERWISE = "galore_adamw_8bit_layerwise"184 GALORE_ADAFACTOR_LAYERWISE = "galore_adafactor_layerwise"185 LOMO = "lomo"186 ADALOMO = "adalomo"187 GROKADAMW = "grokadamw"188 SCHEDULE_FREE_RADAM = "schedule_free_radam"189 SCHEDULE_FREE_ADAMW = "schedule_free_adamw"190 SCHEDULE_FREE_SGD = "schedule_free_sgd"191 APOLLO_ADAMW = "apollo_adamw"192 APOLLO_ADAMW_LAYERWISE = "apollo_adamw_layerwise"193 STABLE_ADAMW = "stable_adamw"194 195 196def _convert_str_dict(passed_value: dict):197 "Safely checks that a passed value is a dictionary and converts any string values to their appropriate types."198 for key, value in passed_value.items():199 if isinstance(value, dict):200 passed_value[key] = _convert_str_dict(value)201 elif isinstance(value, str):202 # First check for bool and convert203 if value.lower() in ("true", "false"):204 passed_value[key] = value.lower() == "true"205 # Check for digit206 elif value.isdigit():207 passed_value[key] = int(value)208 elif value.replace(".", "", 1).isdigit():209 passed_value[key] = float(value)210 211 return passed_value212 213 214# TODO: `TrainingArguments` users rely on it being fully mutable. In the future see if we can narrow this to a few keys: https://github.com/huggingface/transformers/pull/25903215@dataclass216class TrainingArguments:217 """218 TrainingArguments is the subset of the arguments we use in our example scripts **which relate to the training loop219 itself**.220 221 Using [`HfArgumentParser`] we can turn this class into222 [argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the223 command line.224 225 Parameters:226 output_dir (`str`, *optional*, defaults to `"trainer_output"`):227 The output directory where the model predictions and checkpoints will be written.228 overwrite_output_dir (`bool`, *optional*, defaults to `False`):229 If `True`, overwrite the content of the output directory. Use this to continue training if `output_dir`230 points to a checkpoint directory.231 do_train (`bool`, *optional*, defaults to `False`):232 Whether to run training or not. This argument is not directly used by [`Trainer`], it's intended to be used233 by your training/evaluation scripts instead. See the [example234 scripts](https://github.com/huggingface/transformers/tree/main/examples) for more details.235 do_eval (`bool`, *optional*):236 Whether to run evaluation on the validation set or not. Will be set to `True` if `eval_strategy` is237 different from `"no"`. This argument is not directly used by [`Trainer`], it's intended to be used by your238 training/evaluation scripts instead. See the [example239 scripts](https://github.com/huggingface/transformers/tree/main/examples) for more details.240 do_predict (`bool`, *optional*, defaults to `False`):241 Whether to run predictions on the test set or not. This argument is not directly used by [`Trainer`], it's242 intended to be used by your training/evaluation scripts instead. See the [example243 scripts](https://github.com/huggingface/transformers/tree/main/examples) for more details.244 eval_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`):245 The evaluation strategy to adopt during training. Possible values are:246 247 - `"no"`: No evaluation is done during training.248 - `"steps"`: Evaluation is done (and logged) every `eval_steps`.249 - `"epoch"`: Evaluation is done at the end of each epoch.250 251 prediction_loss_only (`bool`, *optional*, defaults to `False`):252 When performing evaluation and generating predictions, only returns the loss.253 per_device_train_batch_size (`int`, *optional*, defaults to 8):254 The batch size *per device*. The **global batch size** is computed as:255 `per_device_train_batch_size * number_of_devices` in multi-GPU or distributed setups.256 per_device_eval_batch_size (`int`, *optional*, defaults to 8):257 The batch size per device accelerator core/CPU for evaluation.258 gradient_accumulation_steps (`int`, *optional*, defaults to 1):259 Number of updates steps to accumulate the gradients for, before performing a backward/update pass.260 261 <Tip warning={true}>262 263 When using gradient accumulation, one step is counted as one step with backward pass. Therefore, logging,264 evaluation, save will be conducted every `gradient_accumulation_steps * xxx_step` training examples.265 266 </Tip>267 268 eval_accumulation_steps (`int`, *optional*):269 Number of predictions steps to accumulate the output tensors for, before moving the results to the CPU. If270 left unset, the whole predictions are accumulated on the device accelerator before being moved to the CPU (faster but271 requires more memory).272 eval_delay (`float`, *optional*):273 Number of epochs or steps to wait for before the first evaluation can be performed, depending on the274 eval_strategy.275 torch_empty_cache_steps (`int`, *optional*):276 Number of steps to wait before calling `torch.<device>.empty_cache()`. If left unset or set to None, cache will not be emptied.277 278 <Tip>279 280 This can help avoid CUDA out-of-memory errors by lowering peak VRAM usage at a cost of about [10% slower performance](https://github.com/huggingface/transformers/issues/31372).281 282 </Tip>283 284 learning_rate (`float`, *optional*, defaults to 5e-5):285 The initial learning rate for [`AdamW`] optimizer.286 weight_decay (`float`, *optional*, defaults to 0):287 The weight decay to apply (if not zero) to all layers except all bias and LayerNorm weights in [`AdamW`]288 optimizer.289 adam_beta1 (`float`, *optional*, defaults to 0.9):290 The beta1 hyperparameter for the [`AdamW`] optimizer.291 adam_beta2 (`float`, *optional*, defaults to 0.999):292 The beta2 hyperparameter for the [`AdamW`] optimizer.293 adam_epsilon (`float`, *optional*, defaults to 1e-8):294 The epsilon hyperparameter for the [`AdamW`] optimizer.295 max_grad_norm (`float`, *optional*, defaults to 1.0):296 Maximum gradient norm (for gradient clipping).297 num_train_epochs(`float`, *optional*, defaults to 3.0):298 Total number of training epochs to perform (if not an integer, will perform the decimal part percents of299 the last epoch before stopping training).300 max_steps (`int`, *optional*, defaults to -1):301 If set to a positive number, the total number of training steps to perform. Overrides `num_train_epochs`.302 For a finite dataset, training is reiterated through the dataset (if all data is exhausted) until303 `max_steps` is reached.304 lr_scheduler_type (`str` or [`SchedulerType`], *optional*, defaults to `"linear"`):305 The scheduler type to use. See the documentation of [`SchedulerType`] for all possible values.306 lr_scheduler_kwargs ('dict', *optional*, defaults to {}):307 The extra arguments for the lr_scheduler. See the documentation of each scheduler for possible values.308 warmup_ratio (`float`, *optional*, defaults to 0.0):309 Ratio of total training steps used for a linear warmup from 0 to `learning_rate`.310 warmup_steps (`int`, *optional*, defaults to 0):311 Number of steps used for a linear warmup from 0 to `learning_rate`. Overrides any effect of `warmup_ratio`.312 log_level (`str`, *optional*, defaults to `passive`):313 Logger log level to use on the main process. Possible choices are the log levels as strings: 'debug',314 'info', 'warning', 'error' and 'critical', plus a 'passive' level which doesn't set anything and keeps the315 current log level for the Transformers library (which will be `"warning"` by default).316 log_level_replica (`str`, *optional*, defaults to `"warning"`):317 Logger log level to use on replicas. Same choices as `log_level`"318 log_on_each_node (`bool`, *optional*, defaults to `True`):319 In multinode distributed training, whether to log using `log_level` once per node, or only on the main320 node.321 logging_dir (`str`, *optional*):322 [TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to323 *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***.324 logging_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"steps"`):325 The logging strategy to adopt during training. Possible values are:326 327 - `"no"`: No logging is done during training.328 - `"epoch"`: Logging is done at the end of each epoch.329 - `"steps"`: Logging is done every `logging_steps`.330 331 logging_first_step (`bool`, *optional*, defaults to `False`):332 Whether to log the first `global_step` or not.333 logging_steps (`int` or `float`, *optional*, defaults to 500):334 Number of update steps between two logs if `logging_strategy="steps"`. Should be an integer or a float in335 range `[0,1)`. If smaller than 1, will be interpreted as ratio of total training steps.336 logging_nan_inf_filter (`bool`, *optional*, defaults to `True`):337 Whether to filter `nan` and `inf` losses for logging. If set to `True` the loss of every step that is `nan`338 or `inf` is filtered and the average loss of the current logging window is taken instead.339 340 <Tip>341 342 `logging_nan_inf_filter` only influences the logging of loss values, it does not change the behavior the343 gradient is computed or applied to the model.344 345 </Tip>346 347 save_strategy (`str` or [`~trainer_utils.SaveStrategy`], *optional*, defaults to `"steps"`):348 The checkpoint save strategy to adopt during training. Possible values are:349 350 - `"no"`: No save is done during training.351 - `"epoch"`: Save is done at the end of each epoch.352 - `"steps"`: Save is done every `save_steps`.353 - `"best"`: Save is done whenever a new `best_metric` is achieved.354 355 If `"epoch"` or `"steps"` is chosen, saving will also be performed at the356 very end of training, always.357 save_steps (`int` or `float`, *optional*, defaults to 500):358 Number of updates steps before two checkpoint saves if `save_strategy="steps"`. Should be an integer or a359 float in range `[0,1)`. If smaller than 1, will be interpreted as ratio of total training steps.360 save_total_limit (`int`, *optional*):361 If a value is passed, will limit the total amount of checkpoints. Deletes the older checkpoints in362 `output_dir`. When `load_best_model_at_end` is enabled, the "best" checkpoint according to363 `metric_for_best_model` will always be retained in addition to the most recent ones. For example, for364 `save_total_limit=5` and `load_best_model_at_end`, the four last checkpoints will always be retained365 alongside the best model. When `save_total_limit=1` and `load_best_model_at_end`, it is possible that two366 checkpoints are saved: the last one and the best one (if they are different).367 save_safetensors (`bool`, *optional*, defaults to `True`):368 Use [safetensors](https://huggingface.co/docs/safetensors) saving and loading for state dicts instead of369 default `torch.load` and `torch.save`.370 save_on_each_node (`bool`, *optional*, defaults to `False`):371 When doing multi-node distributed training, whether to save models and checkpoints on each node, or only on372 the main one.373 374 This should not be activated when the different nodes use the same storage as the files will be saved with375 the same names for each node.376 save_only_model (`bool`, *optional*, defaults to `False`):377 When checkpointing, whether to only save the model, or also the optimizer, scheduler & rng state.378 Note that when this is true, you won't be able to resume training from checkpoint.379 This enables you to save storage by not storing the optimizer, scheduler & rng state.380 You can only load the model using `from_pretrained` with this option set to `True`.381 restore_callback_states_from_checkpoint (`bool`, *optional*, defaults to `False`):382 Whether to restore the callback states from the checkpoint. If `True`, will override383 callbacks passed to the `Trainer` if they exist in the checkpoint."384 use_cpu (`bool`, *optional*, defaults to `False`):385 Whether or not to use cpu. If set to False, we will use cuda or mps device if available.386 seed (`int`, *optional*, defaults to 42):387 Random seed that will be set at the beginning of training. To ensure reproducibility across runs, use the388 [`~Trainer.model_init`] function to instantiate the model if it has some randomly initialized parameters.389 data_seed (`int`, *optional*):390 Random seed to be used with data samplers. If not set, random generators for data sampling will use the391 same seed as `seed`. This can be used to ensure reproducibility of data sampling, independent of the model392 seed.393 jit_mode_eval (`bool`, *optional*, defaults to `False`):394 Whether or not to use PyTorch jit trace for inference.395 bf16 (`bool`, *optional*, defaults to `False`):396 Whether to use bf16 16-bit (mixed) precision training instead of 32-bit training. Requires Ampere or higher397 NVIDIA architecture or Intel XPU or using CPU (use_cpu) or Ascend NPU.398 fp16 (`bool`, *optional*, defaults to `False`):399 Whether to use fp16 16-bit (mixed) precision training instead of 32-bit training.400 fp16_opt_level (`str`, *optional*, defaults to 'O1'):401 For `fp16` training, Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']. See details on402 the [Apex documentation](https://nvidia.github.io/apex/amp).403 fp16_backend (`str`, *optional*, defaults to `"auto"`):404 This argument is deprecated. Use `half_precision_backend` instead.405 half_precision_backend (`str`, *optional*, defaults to `"auto"`):406 The backend to use for mixed precision training. Must be one of `"auto", "apex", "cpu_amp"`. `"auto"` will407 use CPU/CUDA AMP or APEX depending on the PyTorch version detected, while the other choices will force the408 requested backend.409 bf16_full_eval (`bool`, *optional*, defaults to `False`):410 Whether to use full bfloat16 evaluation instead of 32-bit. This will be faster and save memory but can harm411 metric values.412 fp16_full_eval (`bool`, *optional*, defaults to `False`):413 Whether to use full float16 evaluation instead of 32-bit. This will be faster and save memory but can harm414 metric values.415 tf32 (`bool`, *optional*):416 Whether to enable the TF32 mode, available in Ampere and newer GPU architectures. The default value depends417 on PyTorch's version default of `torch.backends.cuda.matmul.allow_tf32`. For more details please refer to418 the [TF32](https://huggingface.co/docs/transformers/perf_train_gpu_one#tf32) documentation. This is an419 experimental API and it may change.420 local_rank (`int`, *optional*, defaults to -1):421 Rank of the process during distributed training.422 ddp_backend (`str`, *optional*):423 The backend to use for distributed training. Must be one of `"nccl"`, `"mpi"`, `"ccl"`, `"gloo"`, `"hccl"`.424 tpu_num_cores (`int`, *optional*):425 When training on TPU, the number of TPU cores (automatically passed by launcher script).426 dataloader_drop_last (`bool`, *optional*, defaults to `False`):427 Whether to drop the last incomplete batch (if the length of the dataset is not divisible by the batch size)428 or not.429 eval_steps (`int` or `float`, *optional*):430 Number of update steps between two evaluations if `eval_strategy="steps"`. Will default to the same431 value as `logging_steps` if not set. Should be an integer or a float in range `[0,1)`. If smaller than 1,432 will be interpreted as ratio of total training steps.433 dataloader_num_workers (`int`, *optional*, defaults to 0):434 Number of subprocesses to use for data loading (PyTorch only). 0 means that the data will be loaded in the435 main process.436 past_index (`int`, *optional*, defaults to -1):437 Some models like [TransformerXL](../model_doc/transformerxl) or [XLNet](../model_doc/xlnet) can make use of438 the past hidden states for their predictions. If this argument is set to a positive int, the `Trainer` will439 use the corresponding output (usually index 2) as the past state and feed it to the model at the next440 training step under the keyword argument `mems`.441 run_name (`str`, *optional*, defaults to `output_dir`):442 A descriptor for the run. Typically used for [trackio](https://github.com/gradio-app/trackio),443 [wandb](https://www.wandb.com/), [mlflow](https://www.mlflow.org/), [comet](https://www.comet.com/site) and444 [swanlab](https://swanlab.cn) logging. If not specified, will be the same as `output_dir`.445 disable_tqdm (`bool`, *optional*):446 Whether or not to disable the tqdm progress bars and table of metrics produced by447 [`~notebook.NotebookTrainingTracker`] in Jupyter Notebooks. Will default to `True` if the logging level is448 set to warn or lower (default), `False` otherwise.449 remove_unused_columns (`bool`, *optional*, defaults to `True`):450 Whether or not to automatically remove the columns unused by the model forward method.451 label_names (`list[str]`, *optional*):452 The list of keys in your dictionary of inputs that correspond to the labels.453 454 Will eventually default to the list of argument names accepted by the model that contain the word "label",455 except if the model used is one of the `XxxForQuestionAnswering` in which case it will also include the456 `["start_positions", "end_positions"]` keys.457 458 You should only specify `label_names` if you're using custom label names or if your model's `forward` consumes multiple label tensors (e.g., extractive QA).459 load_best_model_at_end (`bool`, *optional*, defaults to `False`):460 Whether or not to load the best model found during training at the end of training. When this option is461 enabled, the best checkpoint will always be saved. See462 [`save_total_limit`](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.save_total_limit)463 for more.464 465 <Tip>466 467 When set to `True`, the parameters `save_strategy` needs to be the same as `eval_strategy`, and in468 the case it is "steps", `save_steps` must be a round multiple of `eval_steps`.469 470 </Tip>471 472 metric_for_best_model (`str`, *optional*):473 Use in conjunction with `load_best_model_at_end` to specify the metric to use to compare two different474 models. Must be the name of a metric returned by the evaluation with or without the prefix `"eval_"`.475 476 If not specified, this will default to `"loss"` when either `load_best_model_at_end == True`477 or `lr_scheduler_type == SchedulerType.REDUCE_ON_PLATEAU` (to use the evaluation loss).478 479 If you set this value, `greater_is_better` will default to `True` unless the name ends with "loss".480 Don't forget to set it to `False` if your metric is better when lower.481 greater_is_better (`bool`, *optional*):482 Use in conjunction with `load_best_model_at_end` and `metric_for_best_model` to specify if better models483 should have a greater metric or not. Will default to:484 485 - `True` if `metric_for_best_model` is set to a value that doesn't end in `"loss"`.486 - `False` if `metric_for_best_model` is not set, or set to a value that ends in `"loss"`.487 ignore_data_skip (`bool`, *optional*, defaults to `False`):488 When resuming training, whether or not to skip the epochs and batches to get the data loading at the same489 stage as in the previous training. If set to `True`, the training will begin faster (as that skipping step490 can take a long time) but will not yield the same results as the interrupted training would have.491 fsdp (`bool`, `str` or list of [`~trainer_utils.FSDPOption`], *optional*, defaults to `None`):492 Use PyTorch Distributed Parallel Training (in distributed training only).493 494 A list of options along the following:495 496 - `"full_shard"`: Shard parameters, gradients and optimizer states.497 - `"shard_grad_op"`: Shard optimizer states and gradients.498 - `"hybrid_shard"`: Apply `FULL_SHARD` within a node, and replicate parameters across nodes.499 - `"hybrid_shard_zero2"`: Apply `SHARD_GRAD_OP` within a node, and replicate parameters across nodes.500 - `"offload"`: Offload parameters and gradients to CPUs (only compatible with `"full_shard"` and501 `"shard_grad_op"`).502 - `"auto_wrap"`: Automatically recursively wrap layers with FSDP using `default_auto_wrap_policy`.503 fsdp_config (`str` or `dict`, *optional*):504 Config to be used with fsdp (Pytorch Distributed Parallel Training). The value is either a location of505 fsdp json config file (e.g., `fsdp_config.json`) or an already loaded json file as `dict`.506 507 A List of config and its options:508 - min_num_params (`int`, *optional*, defaults to `0`):509 FSDP's minimum number of parameters for Default Auto Wrapping. (useful only when `fsdp` field is510 passed).511 - transformer_layer_cls_to_wrap (`list[str]`, *optional*):512 List of transformer layer class names (case-sensitive) to wrap, e.g, `BertLayer`, `GPTJBlock`,513 `T5Block` .... (useful only when `fsdp` flag is passed).514 - backward_prefetch (`str`, *optional*)515 FSDP's backward prefetch mode. Controls when to prefetch next set of parameters (useful only when516 `fsdp` field is passed).517 518 A list of options along the following:519 520 - `"backward_pre"` : Prefetches the next set of parameters before the current set of parameter's521 gradient computation.522 - `"backward_post"` : This prefetches the next set of parameters after the current set of523 parameter's gradient computation.524 - forward_prefetch (`bool`, *optional*, defaults to `False`)525 FSDP's forward prefetch mode (useful only when `fsdp` field is passed).526 If `"True"`, then FSDP explicitly prefetches the next upcoming all-gather while executing in the527 forward pass.528 - limit_all_gathers (`bool`, *optional*, defaults to `False`)529 FSDP's limit_all_gathers (useful only when `fsdp` field is passed).530 If `"True"`, FSDP explicitly synchronizes the CPU thread to prevent too many in-flight531 all-gathers.532 - use_orig_params (`bool`, *optional*, defaults to `True`)533 If `"True"`, allows non-uniform `requires_grad` during init, which means support for interspersed534 frozen and trainable parameters. Useful in cases such as parameter-efficient fine-tuning. Please535 refer this536 [blog](https://dev-discuss.pytorch.org/t/rethinking-pytorch-fully-sharded-data-parallel-fsdp-from-first-principles/1019537 - sync_module_states (`bool`, *optional*, defaults to `True`)538 If `"True"`, each individually wrapped FSDP unit will broadcast module parameters from rank 0 to539 ensure they are the same across all ranks after initialization540 - cpu_ram_efficient_loading (`bool`, *optional*, defaults to `False`)541 If `"True"`, only the first process loads the pretrained model checkpoint while all other processes542 have empty weights. When this setting as `"True"`, `sync_module_states` also must to be `"True"`,543 otherwise all the processes except the main process would have random weights leading to unexpected544 behaviour during training.545 - activation_checkpointing (`bool`, *optional*, defaults to `False`):546 If `"True"`, activation checkpointing is a technique to reduce memory usage by clearing activations of547 certain layers and recomputing them during a backward pass. Effectively, this trades extra548 computation time for reduced memory usage.549 - xla (`bool`, *optional*, defaults to `False`):550 Whether to use PyTorch/XLA Fully Sharded Data Parallel Training. This is an experimental feature551 and its API may evolve in the future.552 - xla_fsdp_settings (`dict`, *optional*)553 The value is a dictionary which stores the XLA FSDP wrapping parameters.554 555 For a complete list of options, please see [here](556 https://github.com/pytorch/xla/blob/master/torch_xla/distributed/fsdp/xla_fully_sharded_data_parallel.py).557 - xla_fsdp_grad_ckpt (`bool`, *optional*, defaults to `False`):558 Will use gradient checkpointing over each nested XLA FSDP wrapped layer. This setting can only be559 used when the xla flag is set to true, and an auto wrapping policy is specified through560 fsdp_min_num_params or fsdp_transformer_layer_cls_to_wrap.561 deepspeed (`str` or `dict`, *optional*):562 Use [Deepspeed](https://github.com/deepspeedai/DeepSpeed). This is an experimental feature and its API may563 evolve in the future. The value is either the location of DeepSpeed json config file (e.g.,564 `ds_config.json`) or an already loaded json file as a `dict`"565 566 <Tip warning={true}>567 If enabling any Zero-init, make sure that your model is not initialized until568 *after* initializing the `TrainingArguments`, else it will not be applied.569 </Tip>570 571 accelerator_config (`str`, `dict`, or `AcceleratorConfig`, *optional*):572 Config to be used with the internal `Accelerator` implementation. The value is either a location of573 accelerator json config file (e.g., `accelerator_config.json`), an already loaded json file as `dict`,574 or an instance of [`~trainer_pt_utils.AcceleratorConfig`].575 576 A list of config and its options:577 - split_batches (`bool`, *optional*, defaults to `False`):578 Whether or not the accelerator should split the batches yielded by the dataloaders across the devices. If579 `True` the actual batch size used will be the same on any kind of distributed processes, but it must be a580 round multiple of the `num_processes` you are using. If `False`, actual batch size used will be the one set581 in your script multiplied by the number of processes.582 - dispatch_batches (`bool`, *optional*):583 If set to `True`, the dataloader prepared by the Accelerator is only iterated through on the main process584 and then the batches are split and broadcast to each process. Will default to `True` for `DataLoader` whose585 underlying dataset is an `IterableDataset`, `False` otherwise.586 - even_batches (`bool`, *optional*, defaults to `True`):587 If set to `True`, in cases where the total batch size across all processes does not exactly divide the588 dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among589 all workers.590 - use_seedable_sampler (`bool`, *optional*, defaults to `True`):591 Whether or not use a fully seedable random sampler ([`accelerate.data_loader.SeedableRandomSampler`]). Ensures592 training results are fully reproducible using a different sampling technique. While seed-to-seed results593 may differ, on average the differences are negligible when using multiple different seeds to compare. Should594 also be ran with [`~utils.set_seed`] for the best results.595 - use_configured_state (`bool`, *optional*, defaults to `False`):596 Whether or not to use a pre-configured `AcceleratorState` or `PartialState` defined before calling `TrainingArguments`.597 If `True`, an `Accelerator` or `PartialState` must be initialized. Note that by doing so, this could lead to issues598 with hyperparameter tuning.599 parallelism_config (`ParallelismConfig`, *optional*):600 Parallelism configuration for the training run. Requires Accelerate `1.10.1`601 label_smoothing_factor (`float`, *optional*, defaults to 0.0):602 The label smoothing factor to use. Zero means no label smoothing, otherwise the underlying onehot-encoded603 labels are changed from 0s and 1s to `label_smoothing_factor/num_labels` and `1 - label_smoothing_factor +604 label_smoothing_factor/num_labels` respectively.605 debug (`str` or list of [`~debug_utils.DebugOption`], *optional*, defaults to `""`):606 Enable one or more debug features. This is an experimental feature.607 608 Possible options are:609 610 - `"underflow_overflow"`: detects overflow in model's input/outputs and reports the last frames that led to611 the event612 - `"tpu_metrics_debug"`: print debug metrics on TPU613 614 The options should be separated by whitespaces.615 optim (`str` or [`training_args.OptimizerNames`], *optional*, defaults to `"adamw_torch"` (for torch>=2.8 `"adamw_torch_fused"`)):616 The optimizer to use, such as "adamw_torch", "adamw_torch_fused", "adamw_apex_fused", "adamw_anyprecision",617 "adafactor". See `OptimizerNames` in [training_args.py](https://github.com/huggingface/transformers/blob/main/src/transformers/training_args.py)618 for a full list of optimizers.619 optim_args (`str`, *optional*):620 Optional arguments that are supplied to optimizers such as AnyPrecisionAdamW, AdEMAMix, and GaLore.621 group_by_length (`bool`, *optional*, defaults to `False`):622 Whether or not to group together samples of roughly the same length in the training dataset (to minimize623 padding applied and be more efficient). Only useful if applying dynamic padding.624 length_column_name (`str`, *optional*, defaults to `"length"`):625 Column name for precomputed lengths. If the column exists, grouping by length will use these values rather626 than computing them on train startup. Ignored unless `group_by_length` is `True` and the dataset is an627 instance of `Dataset`.628 report_to (`str` or `list[str]`, *optional*, defaults to `"all"`):629 The list of integrations to report the results and logs to. Supported platforms are `"azure_ml"`,630 `"clearml"`, `"codecarbon"`, `"comet_ml"`, `"dagshub"`, `"dvclive"`, `"flyte"`, `"mlflow"`, `"neptune"`,631 `"swanlab"`, `"tensorboard"`, `"trackio"` and `"wandb"`. Use `"all"` to report to all integrations632 installed, `"none"` for no integrations.633 project (`str`, *optional*, defaults to `"huggingface"`):634 The name of the project to use for logging. Currently, only used by Trackio.635 trackio_space_id (`str` or `None`, *optional*, defaults to `"trackio"`):636 The Hugging Face Space ID to deploy to when using Trackio. Should be a complete Space name like637 `'username/reponame'` or `'orgname/reponame' `, or just `'reponame'` in which case the Space will be638 created in the currently-logged-in Hugging Face user's namespace. If `None`, will log to a local directory.639 Note that this Space will be public unless you set `hub_private_repo=True` or your organization's default640 is to create private Spaces."641 ddp_find_unused_parameters (`bool`, *optional*):642 When using distributed training, the value of the flag `find_unused_parameters` passed to643 `DistributedDataParallel`. Will default to `False` if gradient checkpointing is used, `True` otherwise.644 ddp_bucket_cap_mb (`int`, *optional*):645 When using distributed training, the value of the flag `bucket_cap_mb` passed to `DistributedDataParallel`.646 ddp_broadcast_buffers (`bool`, *optional*):647 When using distributed training, the value of the flag `broadcast_buffers` passed to648 `DistributedDataParallel`. Will default to `False` if gradient checkpointing is used, `True` otherwise.649 dataloader_pin_memory (`bool`, *optional*, defaults to `True`):650 Whether you want to pin memory in data loaders or not. Will default to `True`.651 dataloader_persistent_workers (`bool`, *optional*, defaults to `False`):652 If True, the data loader will not shut down the worker processes after a dataset has been consumed once.653 This allows to maintain the workers Dataset instances alive. Can potentially speed up training, but will654 increase RAM usage. Will default to `False`.655 dataloader_prefetch_factor (`int`, *optional*):656 Number of batches loaded in advance by each worker.657 2 means there will be a total of 2 * num_workers batches prefetched across all workers.658 skip_memory_metrics (`bool`, *optional*, defaults to `True`):659 Whether to skip adding of memory profiler reports to metrics. This is skipped by default because it slows660 down the training and evaluation speed.661 push_to_hub (`bool`, *optional*, defaults to `False`):662 Whether or not to push the model to the Hub every time the model is saved. If this is activated,663 `output_dir` will begin a git directory synced with the repo (determined by `hub_model_id`) and the content664 will be pushed each time a save is triggered (depending on your `save_strategy`). Calling665 [`~Trainer.save_model`] will also trigger a push.666 667 <Tip warning={true}>668 669 If `output_dir` exists, it needs to be a local clone of the repository to which the [`Trainer`] will be670 pushed.671 672 </Tip>673 674 resume_from_checkpoint (`str`, *optional*):675 The path to a folder with a valid checkpoint for your model. This argument is not directly used by676 [`Trainer`], it's intended to be used by your training/evaluation scripts instead. See the [example677 scripts](https://github.com/huggingface/transformers/tree/main/examples) for more details.678 hub_model_id (`str`, *optional*):679 The name of the repository to keep in sync with the local *output_dir*. It can be a simple model ID in680 which case the model will be pushed in your namespace. Otherwise it should be the whole repository name,681 for instance `"user_name/model"`, which allows you to push to an organization you are a member of with682 `"organization_name/model"`. Will default to `user_name/output_dir_name` with *output_dir_name* being the683 name of `output_dir`.684 685 Will default to the name of `output_dir`.686 hub_strategy (`str` or [`~trainer_utils.HubStrategy`], *optional*, defaults to `"every_save"`):687 Defines the scope of what is pushed to the Hub and when. Possible values are:688 689 - `"end"`: push the model, its configuration, the processing class e.g. tokenizer (if passed along to the [`Trainer`]) and a690 draft of a model card when the [`~Trainer.save_model`] method is called.691 - `"every_save"`: push the model, its configuration, the processing class e.g. tokenizer (if passed along to the [`Trainer`]) and692 a draft of a model card each time there is a model save. The pushes are asynchronous to not block693 training, and in case the save are very frequent, a new push is only attempted if the previous one is694 finished. A last push is made with the final model at the end of training.695 - `"checkpoint"`: like `"every_save"` but the latest checkpoint is also pushed in a subfolder named696 last-checkpoint, allowing you to resume training easily with697 `trainer.train(resume_from_checkpoint="last-checkpoint")`.698 - `"all_checkpoints"`: like `"checkpoint"` but all checkpoints are pushed like they appear in the output699 folder (so you will get one checkpoint folder per folder in your final repository)700 701 hub_token (`str`, *optional*):702 The token to use to push the model to the Hub. Will default to the token in the cache folder obtained with703 `hf auth login`.704 hub_private_repo (`bool`, *optional*):705 Whether to make the repo private. If `None` (default), the repo will be public unless the organization's706 default is private. This value is ignored if the repo already exists. If reporting to Trackio with707 deployment to Hugging Face Spaces enabled, the same logic determines whether the Space is private.708 hub_always_push (`bool`, *optional*, defaults to `False`):709 Unless this is `True`, the `Trainer` will skip pushing a checkpoint when the previous push is not finished.710 hub_revision (`str`, *optional*):711 The revision to use when pushing to the Hub. Can be a branch name, a tag, or a commit hash.712 gradient_checkpointing (`bool`, *optional*, defaults to `False`):713 If True, use gradient checkpointing to save memory at the expense of slower backward pass.714 gradient_checkpointing_kwargs (`dict`, *optional*, defaults to `None`):715 Key word arguments to be passed to the `gradient_checkpointing_enable` method.716 include_inputs_for_metrics (`bool`, *optional*, defaults to `False`):717 This argument is deprecated. Use `include_for_metrics` instead, e.g, `include_for_metrics = ["inputs"]`.718 include_for_metrics (`list[str]`, *optional*, defaults to `[]`):719 Include additional data in the `compute_metrics` function if needed for metrics computation.720 Possible options to add to `include_for_metrics` list:721 - `"inputs"`: Input data passed to the model, intended for calculating input dependent metrics.722 - `"loss"`: Loss values computed during evaluation, intended for calculating loss dependent metrics.723 eval_do_concat_batches (`bool`, *optional*, defaults to `True`):724 Whether to recursively concat inputs/losses/labels/predictions across batches. If `False`,725 will instead store them as lists, with each batch kept separate.726 auto_find_batch_size (`bool`, *optional*, defaults to `False`)727 Whether to find a batch size that will fit into memory automatically through exponential decay, avoiding728 CUDA Out-of-Memory errors. Requires accelerate to be installed (`pip install accelerate`)729 full_determinism (`bool`, *optional*, defaults to `False`)730 If `True`, [`enable_full_determinism`] is called instead of [`set_seed`] to ensure reproducible results in731 distributed training. Important: this will negatively impact the performance, so only use it for debugging.732 torchdynamo (`str`, *optional*):733 If set, the backend compiler for TorchDynamo. Possible choices are `"eager"`, `"aot_eager"`, `"inductor"`,734 `"nvfuser"`, `"aot_nvfuser"`, `"aot_cudagraphs"`, `"ofi"`, `"fx2trt"`, `"onnxrt"` and `"ipex"`.735 ray_scope (`str`, *optional*, defaults to `"last"`):736 The scope to use when doing hyperparameter search with Ray. By default, `"last"` will be used. Ray will737 then use the last checkpoint of all trials, compare those, and select the best one. However, other options738 are also available. See the [Ray documentation](739 https://docs.ray.io/en/latest/tune/api_docs/analysis.html#ray.tune.ExperimentAnalysis.get_best_trial) for740 more options.741 ddp_timeout (`int`, *optional*, defaults to 1800):742 The timeout for `torch.distributed.init_process_group` calls, used to avoid GPU socket timeouts when743 performing slow operations in distributed runnings. Please refer the [PyTorch documentation]744 (https://pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group) for more745 information.746 use_mps_device (`bool`, *optional*, defaults to `False`):747 This argument is deprecated.`mps` device will be used if it is available similar to `cuda` device.748 torch_compile (`bool`, *optional*, defaults to `False`):749 Whether or not to compile the model using PyTorch 2.0750 [`torch.compile`](https://pytorch.org/get-started/pytorch-2.0/).751 752 This will use the best defaults for the [`torch.compile`753 API](https://pytorch.org/docs/stable/generated/torch.compile.html?highlight=torch+compile#torch.compile).754 You can customize the defaults with the argument `torch_compile_backend` and `torch_compile_mode` but we755 don't guarantee any of them will work as the support is progressively rolled in in PyTorch.756 757 This flag and the whole compile API is experimental and subject to change in future releases.758 torch_compile_backend (`str`, *optional*):759 The backend to use in `torch.compile`. If set to any value, `torch_compile` will be set to `True`.760 761 Refer to the PyTorch doc for possible values and note that they may change across PyTorch versions.762 763 This flag is experimental and subject to change in future releases.764 torch_compile_mode (`str`, *optional*):765 The mode to use in `torch.compile`. If set to any value, `torch_compile` will be set to `True`.766 767 Refer to the PyTorch doc for possible values and note that they may change across PyTorch versions.768 769 This flag is experimental and subject to change in future releases.770 include_tokens_per_second (`bool`, *optional*, defaults to `False`):771 Whether or not to compute the number of tokens per second per device for training speed metrics.772 773 This will iterate over the entire training dataloader once beforehand,774 and will slow down the entire process.775 776 include_num_input_tokens_seen (`bool`, *optional*):777 Whether or not to track the number of input tokens seen throughout training.778 779 May be slower in distributed training as gather operations must be called.780 781 neftune_noise_alpha (`Optional[float]`):782 If not `None`, this will activate NEFTune noise embeddings. This can drastically improve model performance783 for instruction fine-tuning. Check out the [original paper](https://huggingface.co/papers/2310.05914) and the784 [original code](https://github.com/neelsjain/NEFTune). Support transformers `PreTrainedModel` and also785 `PeftModel` from peft. The original paper used values in the range [5.0, 15.0].786 optim_target_modules (`Union[str, list[str]]`, *optional*):787 The target modules to optimize, i.e. the module names that you would like to train.788 Currently used for the GaLore algorithm (https://huggingface.co/papers/2403.03507) and APOLLO algorithm (https://huggingface.co/papers/2412.05270).789 See GaLore implementation (https://github.com/jiaweizzhao/GaLore) and APOLLO implementation (https://github.com/zhuhanqing/APOLLO) for more details.790 You need to make sure to pass a valid GaLore or APOLLO optimizer, e.g., one of: "apollo_adamw", "galore_adamw", "galore_adamw_8bit", "galore_adafactor" and make sure that the target modules are `nn.Linear` modules only.791 792 batch_eval_metrics (`bool`, *optional*, defaults to `False`):793 If set to `True`, evaluation will call compute_metrics at the end of each batch to accumulate statistics794 rather than saving all eval logits in memory. When set to `True`, you must pass a compute_metrics function795 that takes a boolean argument `compute_result`, which when passed `True`, will trigger the final global796 summary statistics from the batch-level summary statistics you've accumulated over the evaluation set.797 798 eval_on_start (`bool`, *optional*, defaults to `False`):799 Whether to perform a evaluation step (sanity check) before the training to ensure the validation steps works correctly.800 801 eval_use_gather_object (`bool`, *optional*, defaults to `False`):802 Whether to run recursively gather object in a nested list/tuple/dictionary of objects from all devices. This should only be enabled if users are not just returning tensors, and this is actively discouraged by PyTorch.803 804 use_liger_kernel (`bool`, *optional*, defaults to `False`):805 Whether enable [Liger](https://github.com/linkedin/Liger-Kernel) Kernel for LLM model training.806 It can effectively increase multi-GPU training throughput by ~20% and reduces memory usage by ~60%, works out of the box with807 flash attention, PyTorch FSDP, and Microsoft DeepSpeed. Currently, it supports llama, mistral, mixtral and gemma models.808 809 liger_kernel_config (`Optional[dict]`, *optional*):810 Configuration to be used for Liger Kernel. When use_liger_kernel=True, this dict is passed as keyword arguments to the811 `_apply_liger_kernel_to_instance` function, which specifies which kernels to apply. Available options vary by model but typically812 include: 'rope', 'swiglu', 'cross_entropy', 'fused_linear_cross_entropy', 'rms_norm', etc. If `None`, use the default kernel configurations.813 814 average_tokens_across_devices (`bool`, *optional*, defaults to `True`):815 Whether or not to average tokens across devices. If enabled, will use all_reduce to synchronize816 num_tokens_in_batch for precise loss calculation. Reference:817 https://github.com/huggingface/transformers/issues/34242818 """819 820 # Sometimes users will pass in a `str` repr of a dict in the CLI821 # We need to track what fields those can be. Each time a new arg822 # has a dict type, it must be added to this list.823 # Important: These should be typed with Optional[Union[dict,str,...]]824 _VALID_DICT_FIELDS = [825 "accelerator_config",826 "fsdp_config",827 "deepspeed",828 "gradient_checkpointing_kwargs",829 "lr_scheduler_kwargs",830 ]831 framework = "pt"832 833 output_dir: Optional[str] = field(834 default=None,835 metadata={836 "help": "The output directory where the model predictions and checkpoints will be written. Defaults to 'trainer_output' if not provided."837 },838 )839 overwrite_output_dir: bool = field(840 default=False,841 metadata={842 "help": (843 "Overwrite the content of the output directory. "844 "Use this to continue training if output_dir points to a checkpoint directory."845 )846 },847 )848 849 do_train: bool = field(default=False, metadata={"help": "Whether to run training."})850 do_eval: bool = field(default=False, metadata={"help": "Whether to run eval on the dev set."})851 do_predict: bool = field(default=False, metadata={"help": "Whether to run predictions on the test set."})852 eval_strategy: Union[IntervalStrategy, str] = field(853 default="no",854 metadata={"help": "The evaluation strategy to use."},855 )856 prediction_loss_only: bool = field(857 default=False,858 metadata={"help": "When performing evaluation and predictions, only returns the loss."},859 )860 861 per_device_train_batch_size: int = field(862 default=8, metadata={"help": "Batch size per device accelerator core/CPU for training."}863 )864 per_device_eval_batch_size: int = field(865 default=8, metadata={"help": "Batch size per device accelerator core/CPU for evaluation."}866 )867 868 per_gpu_train_batch_size: Optional[int] = field(869 default=None,870 metadata={871 "help": (872 "Deprecated, the use of `--per_device_train_batch_size` is preferred. "873 "Batch size per GPU/TPU core/CPU for training."874 )875 },876 )877 per_gpu_eval_batch_size: Optional[int] = field(878 default=None,879 metadata={880 "help": (881 "Deprecated, the use of `--per_device_eval_batch_size` is preferred. "882 "Batch size per GPU/TPU core/CPU for evaluation."883 )884 },885 )886 887 gradient_accumulation_steps: int = field(888 default=1,889 metadata={"help": "Number of updates steps to accumulate before performing a backward/update pass."},890 )891 eval_accumulation_steps: Optional[int] = field(892 default=None,893 metadata={"help": "Number of predictions steps to accumulate before moving the tensors to the CPU."},894 )895 896 eval_delay: float = field(897 default=0,898 metadata={899 "help": (900 "Number of epochs or steps to wait for before the first evaluation can be performed, depending on the"901 " eval_strategy."902 )903 },904 )905 906 torch_empty_cache_steps: Optional[int] = field(907 default=None,908 metadata={909 "help": "Number of steps to wait before calling `torch.<device>.empty_cache()`."910 "This can help avoid CUDA out-of-memory errors by lowering peak VRAM usage at a cost of about [10% slower performance](https://github.com/huggingface/transformers/issues/31372)."911 "If left unset or set to None, cache will not be emptied."912 },913 )914 915 learning_rate: float = field(default=5e-5, metadata={"help": "The initial learning rate for AdamW."})916 weight_decay: float = field(default=0.0, metadata={"help": "Weight decay for AdamW if we apply some."})917 adam_beta1: float = field(default=0.9, metadata={"help": "Beta1 for AdamW optimizer"})918 adam_beta2: float = field(default=0.999, metadata={"help": "Beta2 for AdamW optimizer"})919 adam_epsilon: float = field(default=1e-8, metadata={"help": "Epsilon for AdamW optimizer."})920 max_grad_norm: float = field(default=1.0, metadata={"help": "Max gradient norm."})921 922 num_train_epochs: float = field(default=3.0, metadata={"help": "Total number of training epochs to perform."})923 max_steps: int = field(924 default=-1,925 metadata={"help": "If > 0: set total number of training steps to perform. Override num_train_epochs."},926 )927 lr_scheduler_type: Union[SchedulerType, str] = field(928 default="linear",929 metadata={"help": "The scheduler type to use."},930 )931 lr_scheduler_kwargs: Union[dict[str, Any], str] = field(932 default_factory=dict,933 metadata={934 "help": (935 "Extra parameters for the lr_scheduler such as {'num_cycles': 1} for the cosine with hard restarts."936 )937 },938 )939 warmup_ratio: float = field(940 default=0.0, metadata={"help": "Linear warmup over warmup_ratio fraction of total steps."}941 )942 warmup_steps: int = field(default=0, metadata={"help": "Linear warmup over warmup_steps."})943 944 log_level: str = field(945 default="passive",946 metadata={947 "help": (948 "Logger log level to use on the main node. Possible choices are the log levels as strings: 'debug',"949 " 'info', 'warning', 'error' and 'critical', plus a 'passive' level which doesn't set anything and"950 " lets the application set the level. Defaults to 'passive'."951 ),952 "choices": trainer_log_levels.keys(),953 },954 )955 log_level_replica: str = field(956 default="warning",957 metadata={958 "help": "Logger log level to use on replica nodes. Same choices and defaults as ``log_level``",959 "choices": trainer_log_levels.keys(),960 },961 )962 log_on_each_node: bool = field(963 default=True,964 metadata={965 "help": (966 "When doing a multinode distributed training, whether to log once per node or just once on the main"967 " node."968 )969 },970 )971 logging_dir: Optional[str] = field(default=None, metadata={"help": "Tensorboard log dir."})972 logging_strategy: Union[IntervalStrategy, str] = field(973 default="steps",974 metadata={"help": "The logging strategy to use."},975 )976 logging_first_step: bool = field(default=False, metadata={"help": "Log the first global_step"})977 logging_steps: float = field(978 default=500,979 metadata={980 "help": (981 "Log every X updates steps. Should be an integer or a float in range `[0,1)`. "982 "If smaller than 1, will be interpreted as ratio of total training steps."983 )984 },985 )986 logging_nan_inf_filter: bool = field(default=True, metadata={"help": "Filter nan and inf losses for logging."})987 save_strategy: Union[SaveStrategy, str] = field(988 default="steps",989 metadata={"help": "The checkpoint save strategy to use."},990 )991 save_steps: float = field(992 default=500,993 metadata={994 "help": (995 "Save checkpoint every X updates steps. Should be an integer or a float in range `[0,1)`. "996 "If smaller than 1, will be interpreted as ratio of total training steps."997 )998 },999 )1000 save_total_limit: Optional[int] = field(1001 default=None,1002 metadata={1003 "help": (1004 "If a value is passed, will limit the total amount of checkpoints. Deletes the older checkpoints in"1005 " `output_dir`. When `load_best_model_at_end` is enabled, the 'best' checkpoint according to"1006 " `metric_for_best_model` will always be retained in addition to the most recent ones. For example,"1007 " for `save_total_limit=5` and `load_best_model_at_end=True`, the four last checkpoints will always be"1008 " retained alongside the best model. When `save_total_limit=1` and `load_best_model_at_end=True`,"1009 " it is possible that two checkpoints are saved: the last one and the best one (if they are different)."1010 " Default is unlimited checkpoints"1011 )1012 },1013 )1014 save_safetensors: bool = field(1015 default=True,1016 metadata={1017 "help": "Use safetensors saving and loading for state dicts instead of default torch.load and torch.save."1018 },1019 )1020 save_on_each_node: bool = field(1021 default=False,1022 metadata={1023 "help": (1024 "When doing multi-node distributed training, whether to save models and checkpoints on each node, or"1025 " only on the main one"1026 )1027 },1028 )1029 save_only_model: bool = field(1030 default=False,1031 metadata={1032 "help": (1033 "When checkpointing, whether to only save the model, or also the optimizer, scheduler & rng state."1034 "Note that when this is true, you won't be able to resume training from checkpoint."1035 "This enables you to save storage by not storing the optimizer, scheduler & rng state."1036 "You can only load the model using from_pretrained with this option set to True."1037 )1038 },1039 )1040 restore_callback_states_from_checkpoint: bool = field(1041 default=False,1042 metadata={1043 "help": "Whether to restore the callback states from the checkpoint. If `True`, will override callbacks passed to the `Trainer` if they exist in the checkpoint."1044 },1045 )1046 no_cuda: bool = field(1047 default=False,1048 metadata={"help": "This argument is deprecated. It will be removed in version 5.0 of 🤗 Transformers."},1049 )1050 use_cpu: bool = field(1051 default=False,1052 metadata={1053 "help": "Whether or not to use cpu. If left to False, we will use the available torch device/backend (cuda/mps/xpu/hpu etc.)"1054 },1055 )1056 use_mps_device: bool = field(1057 default=False,1058 metadata={1059 "help": "This argument is deprecated. `mps` device will be used if available similar to `cuda` device."1060 " It will be removed in version 5.0 of 🤗 Transformers"1061 },1062 )1063 seed: int = field(default=42, metadata={"help": "Random seed that will be set at the beginning of training."})1064 data_seed: Optional[int] = field(default=None, metadata={"help": "Random seed to be used with data samplers."})1065 jit_mode_eval: bool = field(1066 default=False, metadata={"help": "Whether or not to use PyTorch jit trace for inference"}1067 )1068 bf16: bool = field(1069 default=False,1070 metadata={1071 "help": (1072 "Whether to use bf16 (mixed) precision instead of 32-bit. Requires Ampere or higher NVIDIA"1073 " architecture or using CPU (use_cpu) or Ascend NPU. This is an experimental API and it may change."1074 )1075 },1076 )1077 fp16: bool = field(1078 default=False,1079 metadata={"help": "Whether to use fp16 (mixed) precision instead of 32-bit"},1080 )1081 fp16_opt_level: str = field(1082 default="O1",1083 metadata={1084 "help": (1085 "For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']. "1086 "See details at https://nvidia.github.io/apex/amp.html"1087 )1088 },1089 )1090 half_precision_backend: str = field(1091 default="auto",1092 metadata={1093 "help": "The backend to be used for half precision.",1094 "choices": ["auto", "apex", "cpu_amp"],1095 },1096 )1097 bf16_full_eval: bool = field(1098 default=False,1099 metadata={1100 "help": (1101 "Whether to use full bfloat16 evaluation instead of 32-bit. This is an experimental API and it may"1102 " change."1103 )1104 },1105 )1106 fp16_full_eval: bool = field(1107 default=False,1108 metadata={"help": "Whether to use full float16 evaluation instead of 32-bit"},1109 )1110 tf32: Optional[bool] = field(1111 default=None,1112 metadata={1113 "help": (1114 "Whether to enable tf32 mode, available in Ampere and newer GPU architectures. This is an experimental"1115 " API and it may change."1116 )1117 },1118 )1119 local_rank: int = field(default=-1, metadata={"help": "For distributed training: local_rank"})1120 ddp_backend: Optional[str] = field(1121 default=None,1122 metadata={1123 "help": "The backend to be used for distributed training",1124 "choices": ["nccl", "gloo", "mpi", "ccl", "hccl", "cncl", "mccl"],1125 },1126 )1127 tpu_num_cores: Optional[int] = field(1128 default=None, metadata={"help": "TPU: Number of TPU cores (automatically passed by launcher script)"}1129 )1130 tpu_metrics_debug: bool = field(1131 default=False,1132 metadata={1133 "help": (1134 "Deprecated, the use of `--debug tpu_metrics_debug` is preferred. TPU: Whether to print debug metrics"1135 )1136 },1137 )1138 debug: Union[str, list[DebugOption]] = field(1139 default="",1140 metadata={1141 "help": (1142 "Whether or not to enable debug mode. Current options: "1143 "`underflow_overflow` (Detect underflow and overflow in activations and weights), "1144 "`tpu_metrics_debug` (print debug metrics on TPU)."1145 )1146 },1147 )1148 1149 dataloader_drop_last: bool = field(1150 default=False, metadata={"help": "Drop the last incomplete batch if it is not divisible by the batch size."}1151 )1152 eval_steps: Optional[float] = field(1153 default=None,1154 metadata={1155 "help": (1156 "Run an evaluation every X steps. Should be an integer or a float in range `[0,1)`. "1157 "If smaller than 1, will be interpreted as ratio of total training steps."1158 )1159 },1160 )1161 dataloader_num_workers: int = field(1162 default=0,1163 metadata={1164 "help": (1165 "Number of subprocesses to use for data loading (PyTorch only). 0 means that the data will be loaded"1166 " in the main process."1167 )1168 },1169 )1170 dataloader_prefetch_factor: Optional[int] = field(1171 default=None,1172 metadata={1173 "help": (1174 "Number of batches loaded in advance by each worker. "1175 "2 means there will be a total of 2 * num_workers batches prefetched across all workers. "1176 )1177 },1178 )1179 past_index: int = field(1180 default=-1,1181 metadata={"help": "If >=0, uses the corresponding part of the output as the past state for next step."},1182 )1183 1184 run_name: Optional[str] = field(1185 default=None,1186 metadata={1187 "help": (1188 "An optional descriptor for the run. Notably used for trackio, wandb, mlflow comet and swanlab "1189 "logging."1190 )1191 },1192 )1193 disable_tqdm: Optional[bool] = field(1194 default=None, metadata={"help": "Whether or not to disable the tqdm progress bars."}1195 )1196 1197 remove_unused_columns: bool = field(1198 default=True, metadata={"help": "Remove columns not required by the model when using an nlp.Dataset."}1199 )1200 label_names: Optional[list[str]] = field(