s123hree/green-code-optimizer-a100
0
1"""22026.4.932026.4.844.57.250.23.06__UNSLOTH_VERSIONING__7"""8 9# Unsloth auto generated code10# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.11#12# This program is free software: you can redistribute it and/or modify13# it under the terms of the GNU Lesser General Public License as published by14# the Free Software Foundation, either version 3 of the License, or15# (at your option) any later version.16#17# This program is distributed in the hope that it will be useful,18# but WITHOUT ANY WARRANTY; without even the implied warranty of19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the20# GNU General Public License for more details.21#22# You should have received a copy of the GNU Lesser General Public License23# along with this program. If not, see <https://www.gnu.org/licenses/>.24 25from torch import Tensor26import torch27import torch.nn as nn28from torch.nn import functional as F29from unsloth_zoo.temporary_patches.common import torch_compile30from typing import Any, List, Optional, Tuple, Union, Dict, Set, Callable31from trl.trainer.sft_trainer import (Any, AutoConfig, AutoProcessor, Callable, DataCollator, DataCollatorForLanguageModeling, DataCollatorForVisionLanguageModeling, Dataset, EvalPrediction, IterableDataset, Optional, Path, PeftConfig, PreTrainedModel, PreTrainedTokenizerBase, ProcessorMixin, SFTConfig, SFTTrainer, Trainer, TrainerCallback, TrainingArguments, Union, apply_chat_template, clone_chat_template, contextlib, dataclass, defaultdict, dft_loss, generate_model_card, get_act_offloading_ctx_manager, get_comet_experiment_url, is_conversational, is_wandb_available, logger, logging, nn, os, pack_dataset, pad, selective_log_softmax, torch, transformers, Any, AutoConfig, AutoProcessor, Callable, DataCollator, DataCollatorForLanguageModeling, DataCollatorForVisionLanguageModeling, Dataset, EvalPrediction, IterableDataset, Optional, PeftConfig, PreTrainedModel, PreTrainedTokenizerBase, ProcessorMixin, SFTConfig, SFTTrainer, Trainer, TrainerCallback, TrainingArguments, Union, clone_chat_template, contextlib, defaultdict, dft_loss, get_act_offloading_ctx_manager, is_conversational, logger, nn, os, pad, torch, transformers, Callable, DataCollator, DataCollatorForLanguageModeling, Dataset, IterableDataset, Optional, Union, apply_chat_template, is_conversational, os, pack_dataset, pad, transformers, Optional, PreTrainedModel, Trainer, logger, os, torch, os)32 33 34import os35import math36import logging37from typing import *38from dataclasses import dataclass, field39from packaging.version import Version40import torch41import numpy as np42from contextlib import nullcontext43from torch.nn import functional as F44import inspect45from transformers import DataCollatorForSeq2Seq, DataCollatorForLanguageModeling as TransformersDataCollatorForLanguageModeling46from transformers.training_args import ParallelMode47from unsloth_zoo.device_type import DEVICE_TYPE, device_synchronize48 49# Wrap trainer with padding to right and enable training mode50import functools51from types import MethodType52try:53 from unsloth_zoo.gradient_checkpointing import reset_unsloth_gradient_checkpointing_buffers54except:55 def reset_unsloth_gradient_checkpointing_buffers(): pass56def prepare_for_training_mode(f):57 @functools.wraps(f)58 def wrapper(self, *args, **kwargs):59 # Finish the previous W&B run if this is a subsequent train() call.60 # We do this at the START of train() (not the end) so that61 # evaluate() / log() still work after train() completes.62 # HF's WandbCallback.setup() will call wandb.init() for the new run.63 # See: https://github.com/unslothai/unsloth/issues/395464 if getattr(self, '_unsloth_training_completed', False):65 try:66 import wandb67 if wandb.run is not None:68 wandb.finish()69 # Reset HF's WandbCallback so it calls wandb.init() for the new run70 for cb in self.callback_handler.callbacks:71 if type(cb).__name__ == 'WandbCallback':72 cb._initialized = False73 break74 except:75 pass76 # Enable training mode77 _was_training = None78 # Get gradient checkpointing setting from training arguments79 use_gc = getattr(self.args, 'gradient_checkpointing', True)80 if hasattr(self, 'model') and hasattr(self.model, "training"):81 _was_training = self.model.training82 if hasattr(self, 'model') and hasattr(self.model, "for_training"):83 self.model.for_training(use_gradient_checkpointing=use_gc)84 output = f(self, *args, **kwargs)85 # Restore previous mode when possible86 if hasattr(self, 'model') and hasattr(self.model, "for_inference"):87 if _was_training is False:88 self.model.for_inference()89 elif _was_training is True and hasattr(self.model, "for_training"):90 self.model.for_training(use_gradient_checkpointing=use_gc)91 # Reset gradient checkpointing buffers to free memory while staying ready for next run92 try:93 reset_unsloth_gradient_checkpointing_buffers()94 except:95 pass96 # Mark that training completed so the next train() call can97 # finish this W&B run before starting a new one98 self._unsloth_training_completed = True99 return output100 return wrapper101pass102 103torch_compile_options = {104 "epilogue_fusion" : True,105 "max_autotune" : False,106 "shape_padding" : True,107 "trace.enabled" : False,108 "triton.cudagraphs" : False,109}110 111@torch.compile(dynamic = True, fullgraph = True, options = torch_compile_options,)112def chunked_hidden_states_selective_log_softmax(113 hidden_states: torch.Tensor,114 lm_head: torch.Tensor,115 index: torch.Tensor,116 chunks: int = 4,117 logit_scale_multiply: float = 0.0,118 logit_scale_divide: float = 0.0,119 logit_softcapping: float = 0.0,120 temperature: float = 1.0,121) -> torch.Tensor:122 # All Unsloth Zoo code licensed under AGPL3123 flat_hidden_states = hidden_states.reshape(-1, hidden_states.shape[-1])124 flat_index = index.reshape(-1)125 126 chunked_hidden_states = torch.chunk(flat_hidden_states, chunks=chunks, dim=0)127 chunked_index = torch.chunk(flat_index, chunks=chunks, dim=0)128 129 all_per_token_logps = []130 131 for chunk_hidden_states, chunk_index in zip(chunked_hidden_states, chunked_index):132 chunk_logits = chunk_hidden_states.to(lm_head.dtype) @ lm_head.t()133 134 if logit_scale_multiply != 0.0:135 chunk_logits = chunk_logits * logit_scale_multiply136 if logit_scale_divide != 0.0:137 chunk_logits = chunk_logits / logit_scale_divide138 if logit_softcapping != 0.0:139 chunk_logits = logit_softcapping * torch.tanh(chunk_logits / logit_softcapping)140 141 chunk_logits = chunk_logits.to(torch.float32)142 143 if temperature != 1.0:144 chunk_logits = chunk_logits / temperature145 146 selected_logits = torch.gather(chunk_logits, dim=-1, index=chunk_index.unsqueeze(-1)).squeeze(-1)147 logsumexp_values = torch.logsumexp(chunk_logits, dim=-1)148 per_token_logps = selected_logits - logsumexp_values149 all_per_token_logps.append(per_token_logps)150 151 all_per_token_logps = torch.concat(all_per_token_logps)152 153 all_per_token_logps = all_per_token_logps.reshape((hidden_states.shape[0], hidden_states.shape[1]))154 return all_per_token_logps155 156@torch.compile(dynamic = True, fullgraph = True, options = torch_compile_options,)157def chunked_selective_log_softmax(logits, index, temperature: float = 1.0):158 # Split into 4 chunks only159 chunked_logits = torch.chunk(logits.reshape(-1, logits.shape[-1]), chunks = 4, dim = 0)160 chunked_index = torch.chunk(index.reshape(-1), chunks = 4, dim = 0)161 all_per_token_logps = []162 # Below loop does the same as selective_log_softmax(chunk_logits, chunk_index)163 for chunk_logits, chunk_index in zip(chunked_logits, chunked_index):164 chunk_logits = chunk_logits.to(torch.float32)165 if temperature != 1.0:166 chunk_logits = chunk_logits / temperature167 selected_logits = torch.gather(chunk_logits, dim = -1, index = chunk_index.unsqueeze(-1)).squeeze(-1)168 logsumexp_values = torch.logsumexp(chunk_logits, dim = -1)169 per_token_logps = selected_logits - logsumexp_values170 all_per_token_logps.append(per_token_logps)171 pass172 all_per_token_logps = torch.concat(all_per_token_logps)173 all_per_token_logps = all_per_token_logps.reshape((logits.shape[0], logits.shape[1]))174 return all_per_token_logps175 176def calculate_pad_tokens_in_prompt(177 input_ids: torch.Tensor,178 logits_to_keep: int,179 pad_token_id: int180) -> torch.Tensor:181 """182 Given prompt tensor, it returns all the left padded tokens in that sequence. so [pad, pad, pad, cat] = 3 tokens183 """184 if logits_to_keep >= input_ids.shape[1]:185 raise ValueError("logits_to_keep must be smaller than the sequence length.")186 187 prompt_section = input_ids[:, :-logits_to_keep]188 189 padding_mask = (prompt_section == pad_token_id)190 191 pad_token_counts = padding_mask.sum(dim=1)192 193 return pad_token_counts194 195def create_completion_attention_mask(196 completion_input_ids: torch.Tensor,197 left_pad_tokens_per_prompt: torch.Tensor,198 max_left_pad: int,199 pad_token_id: int200) -> torch.Tensor:201 """202 Given that we have a sequence, [p,p,p,c,c,c,pad,pad,pad]203 204 Where p are extra prompt tokens we got from slicing the torch tensor, c is completion tokens205 and pad are pad tokens, this function would make a completion mask that would 0 out the pad206 and p tokens. so in this example [0,0,0,1,1,1,0,0,0]207 """208 batch_size, completion_len = completion_input_ids.shape209 device = completion_input_ids.device210 211 num_tokens_to_mask = max_left_pad - left_pad_tokens_per_prompt212 213 indices = torch.arange(completion_len, device=device).unsqueeze(0)214 shift_mask = indices >= num_tokens_to_mask.unsqueeze(1)215 216 non_padding_mask = (completion_input_ids != pad_token_id)217 218 final_mask = shift_mask & non_padding_mask219 220 return final_mask221 222def left_pack_padding(tensor: torch.Tensor, pad_id: int) -> torch.Tensor:223 """224 Moves all padding tokens in each sequence of a batch to the right.225 """226 mask = (tensor != pad_id)227 # Must do stable=True since binary mark is unordered228 sorted_indices = torch.argsort(mask, dim=1, descending=True, stable=True)229 packed_tensor = torch.gather(tensor, 1, sorted_indices)230 return packed_tensor231 232def align_logprobs_with_mask(233 logprob_tensor: torch.Tensor,234 attention_mask: torch.Tensor,235 pad_value: float = 0.0236) -> torch.Tensor:237 """238 Aligns a log probability tensor with a given attention mask.239 """240 241 device = logprob_tensor.device242 batch_size, logprob_seq_len = logprob_tensor.shape243 mask_seq_len = attention_mask.shape[1]244 245 padded_logprobs = torch.full(246 attention_mask.shape,247 fill_value=pad_value,248 dtype=logprob_tensor.dtype,249 device=device250 )251 252 left_pad_counts = torch.argmax(attention_mask, dim=1)253 254 cols = torch.arange(logprob_seq_len, device=device)255 dest_indices = left_pad_counts.unsqueeze(1) + cols256 257 # Create destination row indices258 # Shape: [batch_size, logprob_seq_len]259 row_indices = torch.arange(batch_size, device=device).unsqueeze(1).expand_as(dest_indices)260 261 # --- 4. Filter out-of-bounds indices and perform assignment ---262 # Create a mask to identify only the indices that are within the bounds263 # of the target tensor's sequence length.264 valid_mask = dest_indices < mask_seq_len265 266 # Use this mask to select only the valid row indices, column indices,267 # and the corresponding values from the logprob tensor.268 # This flattens the selected elements into 1D tensors.269 valid_rows = row_indices[valid_mask]270 valid_cols = dest_indices[valid_mask]271 valid_vals = logprob_tensor[valid_mask]272 273 # Place the valid values into their correct positions in the padded tensor274 # using a single, efficient advanced indexing operation.275 padded_logprobs[valid_rows, valid_cols] = valid_vals276 277 return padded_logprobs278 279def autotune_batch_and_chunks(280 total_input_rows,281 seq_len,282 hidden_size,283 vocab_size,284 dtype_bytes=16,285 multiplier=None286):287 if multiplier is None:288 final_m = max(4, seq_len // 4096)289 else:290 final_m = multiplier291 292 if torch.cuda.is_available():293 free_bytes, _ = torch.cuda.mem_get_info()294 limit_gb = (free_bytes / (1024**3))*.80295 elif hasattr(torch, "xpu") and torch.xpu.is_available():296 # For XPU: estimate free memory from total - reserved297 total_mem = torch.xpu.get_device_properties(0).total_memory298 reserved_mem = torch.xpu.memory_reserved()299 free_bytes = total_mem - reserved_mem300 limit_gb = (free_bytes / (1024**3)) * 0.80301 else:302 # Fallback: assume 8GB available303 limit_gb = 8.0304 305 bytes_to_gb = 1024**3306 307 b_vals = torch.arange(total_input_rows, 0, -1, device='cpu', dtype=torch.float32)308 309 hidden_gb = (b_vals * seq_len * hidden_size * dtype_bytes) / bytes_to_gb310 311 base_logits = ((b_vals/total_input_rows) * b_vals * seq_len * vocab_size * dtype_bytes) / bytes_to_gb312 logits_gb = base_logits / final_m313 314 total_mem_gb = hidden_gb + logits_gb315 316 valid_mask = total_mem_gb <= limit_gb317 valid_indices = torch.nonzero(valid_mask, as_tuple=False)318 319 if valid_indices.shape[0] == 0:320 #This means your GPU will OOM321 return 4, final_m322 323 best_idx = valid_indices[0].item()324 final_b = int(b_vals[best_idx].item())325 326 return final_b, final_m327 328def sanitize_logprob(logprob):329 """Local port of trl.scripts.vllm_serve.sanitize_logprob.330 Filters NaN logprobs from vLLM outputs."""331 value = logprob.logprob332 if math.isnan(value):333 logging.getLogger(__name__).warning(334 f"Generated NaN logprob, token logprob '{logprob}' will be ignored"335 )336 return None337 return value338@dataclass339class UnslothSFTConfig(SFTConfig):340 """341 342 Configuration class for the [`SFTTrainer`].343 344 This class includes only the parameters that are specific to SFT training. For a full list of training arguments,345 please refer to the [`~transformers.TrainingArguments`] documentation. Note that default values in this class may346 differ from those in [`~transformers.TrainingArguments`].347 348 Using [`~transformers.HfArgumentParser`] we can turn this class into349 [argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the350 command line.351 352 Parameters:353 > Parameters that control the model354 355 model_init_kwargs (`dict[str, Any]` or `None`, *optional*, defaults to `None`):356 Keyword arguments for [`~transformers.AutoModelForCausalLM.from_pretrained`], used when the `model`357 argument of the [`SFTTrainer`] is provided as a string. If you're training a MoE architecture and want to358 include the load balancing/auxilliary loss as a part of the final loss, remember to set359 `output_router_logits=True` in this dictionary.360 chat_template_path (`str` or `None`, *optional*, defaults to `None`):361 If specified, sets the model's chat template. This can either be the path to a tokenizer (local directory362 or Hugging Face Hub model) or a direct path to a Jinja template file. When using a Jinja file, you must363 ensure that any special tokens referenced in the template are added to the tokenizer and that the model's364 embedding layer is resized accordingly.365 366 > Parameters that control the data preprocessing367 368 dataset_text_field (`str`, *optional*, defaults to `"text"`):369 Name of the column that contains text data in the dataset.370 dataset_kwargs (`dict[str, Any]` or `None`, *optional*, defaults to `None`):371 Dictionary of optional keyword arguments for the dataset preparation. The only supported key is372 `skip_prepare_dataset`. When the model is a VLM, `skip_prepare_dataset` is automatically treated as `True`373 regardless of the provided value, since preprocessing is done on the fly.374 dataset_num_proc (`int` or `None`, *optional*, defaults to `None`):375 Number of processes to use for processing the dataset.376 eos_token (`str` or `None`, *optional*, defaults to `None`):377 Token used to indicate the end of a turn or sequence. If `None`, it defaults to378 `processing_class.eos_token`.379 pad_token (`int` or `None`, *optional*, defaults to `None`):380 Token used for padding. If `None`, it defaults to `processing_class.pad_token`, or if that is also `None`,381 it falls back to `processing_class.eos_token`.382 max_length (`int` or `None`, *optional*, defaults to `1024`):383 Maximum length of the tokenized sequence. Sequences longer than `max_length` are truncated from the right.384 If `None`, no truncation is applied. When packing is enabled, this value sets the sequence length.385 packing (`bool`, *optional*, defaults to `False`):386 Whether to group multiple sequences into fixed-length blocks to improve computational efficiency and reduce387 padding. Uses `max_length` to define sequence length.388 packing_strategy (`str`, *optional*, defaults to `"bfd"`):389 Strategy for packing sequences. Can be either `"bfd"` (best-fit decreasing, default), or `"wrapped"`.390 padding_free (`bool`, *optional*, defaults to `False`):391 Whether to perform forward passes without padding by flattening all sequences in the batch into a single392 continuous sequence. This reduces memory usage by eliminating padding overhead. Currently, this is only393 supported with the FlashAttention 2 or 3, which can efficiently handle the flattened batch structure. When394 packing is enabled with strategy `"bfd"`, padding-free is enabled, regardless of the value of this395 parameter.396 pad_to_multiple_of (`int` or `None`, *optional*, defaults to `None`):397 If set, the sequences will be padded to a multiple of this value.398 eval_packing (`bool` or `None`, *optional*, defaults to `None`):399 Whether to pack the eval dataset. If `None`, uses the same value as `packing`.400 401 > Parameters that control the training402 403 completion_only_loss (`bool` or `None`, *optional*, defaults to `None`):404 Whether to compute loss only on the completion part of the sequence. If set to `True`, loss is computed405 only on the completion, which is supported only for [prompt-completion](#prompt-completion) datasets. If406 `False`, loss is computed on the entire sequence. If `None` (default), the behavior depends on the dataset:407 loss is computed on the completion for [prompt-completion](#prompt-completion) datasets, and on the full408 sequence for [language modeling](#language-modeling) datasets.409 assistant_only_loss (`bool`, *optional*, defaults to `False`):410 Whether to compute loss only on the assistant part of the sequence. If set to `True`, loss is computed only411 on the assistant responses, which is supported only for [conversational](#conversational) datasets. If412 `False`, loss is computed on the entire sequence.413 loss_type (`str`, *optional*, defaults to `"nll"`):414 Type of loss to use. Possible values are `"nll"` (negative log-likelihood, default) and `"dft"` (Dynamic415 Fine-Tuning, as described in [this paper](https://huggingface.co/papers/2508.05629)).416 activation_offloading (`bool`, *optional*, defaults to `False`):417 Whether to offload the activations to the CPU.418 419 """420 vllm_sampling_params: Optional[Any] = field(421 default = None,422 metadata = {'help': 'vLLM SamplingParams'},423 )424 unsloth_num_chunks : Optional[int] = field(425 default = -1,426 metadata = {'help': 'Chunk size to reduce memory usage. -1 is most efficient.'},427 )428 unsloth_logit_chunk_multiplier : Optional[int] = field(429 default = None,430 metadata = {'help': 'Multiplier for chunked logit computations.'},431 )432 unsloth_grpo_mini_batch : Optional[int] = field(433 default = None,434 metadata = {'help': 'Mini batch size for GRPO hidden state accumulation. Default is None unless user defines it.'},435 )436 max_seq_length : Optional[int] = field(437 default = None,438 metadata = {'help': 'Maximum sequence length to truncate to.'},439 )440 def __init__(441 self,442 output_dir = None,443 overwrite_output_dir = None,444 do_train = False,445 do_eval = False,446 do_predict = False,447 eval_strategy = 'no',448 prediction_loss_only = False,449 per_device_train_batch_size = 4,450 per_device_eval_batch_size = 4,451 per_gpu_train_batch_size = None,452 per_gpu_eval_batch_size = None,453 gradient_accumulation_steps = 2,454 eval_accumulation_steps = 2,455 eval_delay = 0,456 torch_empty_cache_steps = 250,457 learning_rate = 5e-05,458 weight_decay = 0.01,459 adam_beta1 = 0.9,460 adam_beta2 = 0.999,461 adam_epsilon = 1e-08,462 max_grad_norm = 1.0,463 num_train_epochs = 3.0,464 max_steps = -1,465 lr_scheduler_type = 'linear',466 warmup_ratio = 0.1,467 warmup_steps = 0,468 log_level = 'passive',469 log_level_replica = 'warning',470 log_on_each_node = True,471 logging_dir = None,472 logging_strategy = 'steps',473 logging_first_step = False,474 logging_steps = 1,475 logging_nan_inf_filter = False,476 save_strategy = 'steps',477 save_steps = 500,478 save_total_limit = None,479 save_safetensors = True,480 save_on_each_node = False,481 save_only_model = False,482 restore_callback_states_from_checkpoint = False,483 no_cuda = False,484 use_cpu = False,485 use_mps_device = False,486 seed = 3407,487 data_seed = 3407,488 jit_mode_eval = False,489 bf16 = False,490 fp16 = False,491 fp16_opt_level = 'O1',492 half_precision_backend = 'auto',493 bf16_full_eval = False,494 fp16_full_eval = False,495 tf32 = None,496 local_rank = -1,497 ddp_backend = None,498 tpu_num_cores = None,499 tpu_metrics_debug = False,500 debug = '',501 dataloader_drop_last = False,502 eval_steps = None,503 dataloader_num_workers = 0,504 dataloader_prefetch_factor = None,505 past_index = -1,506 run_name = None,507 disable_tqdm = None,508 remove_unused_columns = True,509 label_names = None,510 load_best_model_at_end = False,511 metric_for_best_model = None,512 greater_is_better = None,513 ignore_data_skip = False,514 fsdp = None,515 fsdp_min_num_params = 0,516 fsdp_config = None,517 fsdp_transformer_layer_cls_to_wrap = None,518 accelerator_config = None,519 parallelism_config = None,520 deepspeed = None,521 label_smoothing_factor = 0.0,522 optim = 'adamw_8bit',523 optim_args = None,524 adafactor = False,525 group_by_length = False,526 length_column_name = 'length',527 report_to = 'none',528 project = 'huggingface',529 trackio_space_id = 'trackio',530 ddp_find_unused_parameters = None,531 ddp_bucket_cap_mb = None,532 ddp_broadcast_buffers = None,533 dataloader_pin_memory = True,534 dataloader_persistent_workers = False,535 skip_memory_metrics = True,536 use_legacy_prediction_loop = False,537 push_to_hub = False,538 resume_from_checkpoint = None,539 hub_model_id = None,540 hub_strategy = 'every_save',541 hub_token = None,542 hub_private_repo = None,543 hub_always_push = False,544 hub_revision = None,545 gradient_checkpointing = True,546 gradient_checkpointing_kwargs = None,547 include_inputs_for_metrics = False,548 eval_do_concat_batches = True,549 fp16_backend = 'auto',550 push_to_hub_model_id = None,551 push_to_hub_organization = None,552 push_to_hub_token = None,553 mp_parameters = '',554 auto_find_batch_size = False,555 full_determinism = False,556 torchdynamo = None,557 ray_scope = 'last',558 ddp_timeout = 1800,559 torch_compile = False,560 torch_compile_backend = None,561 torch_compile_mode = None,562 include_tokens_per_second = False,563 include_num_input_tokens_seen = False,564 neftune_noise_alpha = None,565 optim_target_modules = None,566 batch_eval_metrics = False,567 eval_on_start = False,568 use_liger_kernel = False,569 liger_kernel_config = None,570 eval_use_gather_object = False,571 average_tokens_across_devices = True,572 model_init_kwargs = None,573 chat_template_path = None,574 dataset_text_field = 'text',575 dataset_kwargs = None,576 dataset_num_proc = None,577 eos_token = None,578 pad_token = None,579 max_length = 1024,580 packing = False,581 packing_strategy = 'bfd',582 padding_free = None,583 pad_to_multiple_of = None,584 eval_packing = None,585 completion_only_loss = None,586 assistant_only_loss = False,587 loss_type = 'nll',588 activation_offloading = False,589 vllm_sampling_params = None,590 unsloth_num_chunks = -1,591 unsloth_logit_chunk_multiplier = None,592 unsloth_grpo_mini_batch = None,593 max_seq_length = None,594 **kwargs,595 ):596 if learning_rate < 1e-7: print(f'Unsloth: Your learning rate of `{learning_rate}` is too small and less than 1e-7! Consider increasing it, otherwise gradient updates will be close to 0!')597 if learning_rate > 1: print(f'Unsloth: Your learning rate of `{learning_rate}` is way too larger > 1! Consider decreasing it to 1e-1, otherwise gradient updates will explode!')598 if num_train_epochs is None:599 num_train_epochs = 3.0 # Default to 3 epochs if None, max_steps will override600 if output_dir is None and save_strategy == 'steps' and save_steps == 500:601 output_dir = 'unsloth_training_checkpoints'602 save_strategy = 'no'603 import multiprocessing as _mp604 if dataset_num_proc is None:605 if _mp.get_start_method() != 'fork':606 dataset_num_proc = None607 else:608 import psutil609 dataset_num_proc = min(max((psutil.cpu_count() or 1)+4, 2), 64)610 memory_gb_left = psutil.virtual_memory().available / (1024**3)611 if memory_gb_left <= 2: dataset_num_proc = 1612 else: dataset_num_proc = min(dataset_num_proc, int(memory_gb_left))613 if os.environ.get('UNSLOTH_ENABLE_FLEX_ATTENTION', '0') == '1':614 from unsloth_zoo.flex_attention import HAS_FLEX_ATTENTION615 if HAS_FLEX_ATTENTION and pad_to_multiple_of is None:616 from unsloth_zoo.flex_attention import FLEX_ATTENTION_BLOCK_SIZE617 pad_to_multiple_of = FLEX_ATTENTION_BLOCK_SIZE618 619 620 super().__init__(621 output_dir = output_dir,622 overwrite_output_dir = overwrite_output_dir,623 do_train = do_train,624 do_eval = do_eval,625 do_predict = do_predict,626 eval_strategy = eval_strategy,627 prediction_loss_only = prediction_loss_only,628 per_device_train_batch_size = per_device_train_batch_size,629 per_device_eval_batch_size = per_device_eval_batch_size,630 per_gpu_train_batch_size = per_gpu_train_batch_size,631 per_gpu_eval_batch_size = per_gpu_eval_batch_size,632 gradient_accumulation_steps = gradient_accumulation_steps,633 eval_accumulation_steps = eval_accumulation_steps,634 eval_delay = eval_delay,635 torch_empty_cache_steps = torch_empty_cache_steps,636 learning_rate = learning_rate,637 weight_decay = weight_decay,638 adam_beta1 = adam_beta1,639 adam_beta2 = adam_beta2,640 adam_epsilon = adam_epsilon,641 max_grad_norm = max_grad_norm,642 num_train_epochs = num_train_epochs,643 max_steps = max_steps,644 lr_scheduler_type = lr_scheduler_type,645 warmup_ratio = warmup_ratio,646 warmup_steps = warmup_steps,647 log_level = log_level,648 log_level_replica = log_level_replica,649 log_on_each_node = log_on_each_node,650 logging_dir = logging_dir,651 logging_strategy = logging_strategy,652 logging_first_step = logging_first_step,653 logging_steps = logging_steps,654 logging_nan_inf_filter = logging_nan_inf_filter,655 save_strategy = save_strategy,656 save_steps = save_steps,657 save_total_limit = save_total_limit,658 save_safetensors = save_safetensors,659 save_on_each_node = save_on_each_node,660 save_only_model = save_only_model,661 restore_callback_states_from_checkpoint = restore_callback_states_from_checkpoint,662 no_cuda = no_cuda,663 use_cpu = use_cpu,664 use_mps_device = use_mps_device,665 seed = seed,666 data_seed = data_seed,667 jit_mode_eval = jit_mode_eval,668 bf16 = bf16,669 fp16 = fp16,670 fp16_opt_level = fp16_opt_level,671 half_precision_backend = half_precision_backend,672 bf16_full_eval = bf16_full_eval,673 fp16_full_eval = fp16_full_eval,674 tf32 = tf32,675 local_rank = local_rank,676 ddp_backend = ddp_backend,677 tpu_num_cores = tpu_num_cores,678 tpu_metrics_debug = tpu_metrics_debug,679 debug = debug,680 dataloader_drop_last = dataloader_drop_last,681 eval_steps = eval_steps,682 dataloader_num_workers = dataloader_num_workers,683 dataloader_prefetch_factor = dataloader_prefetch_factor,684 past_index = past_index,685 run_name = run_name,686 disable_tqdm = disable_tqdm,687 remove_unused_columns = remove_unused_columns,688 label_names = label_names,689 load_best_model_at_end = load_best_model_at_end,690 metric_for_best_model = metric_for_best_model,691 greater_is_better = greater_is_better,692 ignore_data_skip = ignore_data_skip,693 fsdp = fsdp,694 fsdp_min_num_params = fsdp_min_num_params,695 fsdp_config = fsdp_config,696 fsdp_transformer_layer_cls_to_wrap = fsdp_transformer_layer_cls_to_wrap,697 accelerator_config = accelerator_config,698 parallelism_config = parallelism_config,699 deepspeed = deepspeed,700 label_smoothing_factor = label_smoothing_factor,701 optim = optim,702 optim_args = optim_args,703 adafactor = adafactor,704 group_by_length = group_by_length,705 length_column_name = length_column_name,706 report_to = report_to,707 project = project,708 trackio_space_id = trackio_space_id,709 ddp_find_unused_parameters = ddp_find_unused_parameters,710 ddp_bucket_cap_mb = ddp_bucket_cap_mb,711 ddp_broadcast_buffers = ddp_broadcast_buffers,712 dataloader_pin_memory = dataloader_pin_memory,713 dataloader_persistent_workers = dataloader_persistent_workers,714 skip_memory_metrics = skip_memory_metrics,715 use_legacy_prediction_loop = use_legacy_prediction_loop,716 push_to_hub = push_to_hub,717 resume_from_checkpoint = resume_from_checkpoint,718 hub_model_id = hub_model_id,719 hub_strategy = hub_strategy,720 hub_token = hub_token,721 hub_private_repo = hub_private_repo,722 hub_always_push = hub_always_push,723 hub_revision = hub_revision,724 gradient_checkpointing = gradient_checkpointing,725 gradient_checkpointing_kwargs = gradient_checkpointing_kwargs,726 include_inputs_for_metrics = include_inputs_for_metrics,727 eval_do_concat_batches = eval_do_concat_batches,728 fp16_backend = fp16_backend,729 push_to_hub_model_id = push_to_hub_model_id,730 push_to_hub_organization = push_to_hub_organization,731 push_to_hub_token = push_to_hub_token,732 mp_parameters = mp_parameters,733 auto_find_batch_size = auto_find_batch_size,734 full_determinism = full_determinism,735 torchdynamo = torchdynamo,736 ray_scope = ray_scope,737 ddp_timeout = ddp_timeout,738 torch_compile = torch_compile,739 torch_compile_backend = torch_compile_backend,740 torch_compile_mode = torch_compile_mode,741 include_tokens_per_second = include_tokens_per_second,742 include_num_input_tokens_seen = include_num_input_tokens_seen,743 neftune_noise_alpha = neftune_noise_alpha,744 optim_target_modules = optim_target_modules,745 batch_eval_metrics = batch_eval_metrics,746 eval_on_start = eval_on_start,747 use_liger_kernel = use_liger_kernel,748 liger_kernel_config = liger_kernel_config,749 eval_use_gather_object = eval_use_gather_object,750 average_tokens_across_devices = average_tokens_across_devices,751 model_init_kwargs = model_init_kwargs,752 chat_template_path = chat_template_path,753 dataset_text_field = dataset_text_field,754 dataset_kwargs = dataset_kwargs,755 dataset_num_proc = dataset_num_proc,756 eos_token = eos_token,757 pad_token = pad_token,758 max_length = max_length,759 packing = packing,760 packing_strategy = packing_strategy,761 padding_free = padding_free,762 pad_to_multiple_of = pad_to_multiple_of,763 eval_packing = eval_packing,764 completion_only_loss = completion_only_loss,765 assistant_only_loss = assistant_only_loss,766 loss_type = loss_type,767 activation_offloading = activation_offloading,**kwargs)768 self.vllm_sampling_params = vllm_sampling_params769 self.unsloth_num_chunks = unsloth_num_chunks770 if unsloth_grpo_mini_batch is not None:771 if self.generation_batch_size >= unsloth_grpo_mini_batch:772 self.unsloth_grpo_mini_batch = unsloth_grpo_mini_batch773 else:774 raise ValueError(775 f"Unsloth GRPO mini batch size needs to be less than or equal to the effective generation batch size, "776 f"which is self.per_device_train_batch_size * gradient_accumulation_steps."777 )778 self.unsloth_logit_chunk_multiplier = unsloth_logit_chunk_multiplier779 self.max_seq_length = max_seq_length780 781pass782 783class _UnslothSFTTrainer(Trainer):784 """"""785 786 _tag_names = ["trl", "sft"]787 788 def __init__(789 self,790 model: Union[str, nn.Module, PreTrainedModel],791 args: Optional[Union[SFTConfig, TrainingArguments]] = None,792 data_collator: Optional[DataCollator] = None, # type: ignore793 train_dataset: Optional[Union[Dataset, IterableDataset]] = None,794 eval_dataset: Optional[Union[Dataset, dict[str, Dataset]]] = None,795 processing_class: Optional[Union[PreTrainedTokenizerBase, ProcessorMixin]] = None,796 compute_loss_func: Optional[Callable] = None,797 compute_metrics: Optional[Callable[[EvalPrediction], dict]] = None,798 callbacks: Optional[list[TrainerCallback]] = None,799 optimizers: tuple[Optional[torch.optim.Optimizer], Optional[torch.optim.lr_scheduler.LambdaLR]] = (None, None),800 optimizer_cls_and_kwargs: Optional[tuple[type[torch.optim.Optimizer], dict[str, Any]]] = None,801 preprocess_logits_for_metrics: Optional[Callable[[torch.Tensor, torch.Tensor], torch.Tensor]] = None,802 peft_config: Optional["PeftConfig"] = None,803 formatting_func: Optional[Callable[[dict], str]] = None,804 ):805 # Args806 if args is None:807 model_name = model if isinstance(model, str) else model.config._name_or_path808 model_name = model_name.split("/")[-1]809 args = SFTConfig(f"{model_name}-SFT")810 elif isinstance(args, TrainingArguments) and not isinstance(args, SFTConfig):811 dict_args = args.to_dict()812 dict_args["hub_token"] = args.hub_token # to_dict hides the hub_token813 dict_args.pop("push_to_hub_token", None)814 args = SFTConfig(**dict_args)815 816 # Model817 model_init_kwargs = args.model_init_kwargs or {}818 if isinstance(model, str):819 model_id = model820 dtype = model_init_kwargs.get("dtype")821 if isinstance(dtype, torch.dtype) or dtype == "auto" or dtype is None:822 pass # dtype is already a torch.dtype or "auto" or None823 elif isinstance(dtype, str) and dtype in ["bfloat16", "float16", "float32"]:824 dtype = getattr(torch, dtype)825 model_init_kwargs["dtype"] = dtype826 else:827 raise ValueError(828 "Invalid `dtype` passed to `SFTConfig`. Expected either 'auto' or a string representing "829 f"a valid `torch.dtype` (e.g., 'float32'), but got {dtype}."830 )831 config = AutoConfig.from_pretrained(model_id)832 architecture = getattr(transformers, config.architectures[0])833 model = architecture.from_pretrained(model_id, **model_init_kwargs)834 else:835 model_id = model.config._name_or_path836 if args.model_init_kwargs is not None:837 logger.warning(838 "You passed `model_init_kwargs` to the `SFTConfig`, but your model is already instantiated. "839 "The `model_init_kwargs` will be ignored."840 )841 842 # Processing class843 if processing_class is None:844 processing_class = AutoProcessor.from_pretrained(model_id)845 846 # Handle pad token for processors or tokenizers847 if isinstance(processing_class, ProcessorMixin):848 tokenizer = processing_class.tokenizer849 self._is_vlm = True850 elif isinstance(processing_class, PreTrainedTokenizerBase):851 tokenizer = processing_class852 self._is_vlm = False853 else:854 raise TypeError("The `processing_class` must be either a `PreTrainedTokenizerBase` or a `ProcessorMixin`")855 856 if args.eos_token is not None:857 eos_token = args.eos_token858 eos_token_id = tokenizer.convert_tokens_to_ids(eos_token)859 if eos_token_id is None:860 raise ValueError(861 f"The specified `eos_token` ('{eos_token}') is not found in the vocabulary of the given "862 f"`processing_class` ({processing_class.__class__.__name__}). Ensure that the `eos_token` exists "863 "in the vocabulary before using it as an EOS token."864 )865 tokenizer.eos_token_id = eos_token_id866 867 if args.chat_template_path is not None:868 if os.path.isfile(args.chat_template_path) and args.chat_template_path.endswith((".jinja", ".j2")):869 with open(args.chat_template_path, encoding="utf-8") as chat_template_file:870 processing_class.chat_template = chat_template_file.read()871 added_tokens = []872 else:873 model, processing_class, added_tokens = clone_chat_template(874 model, processing_class, args.chat_template_path875 )876 else:877 added_tokens = []878 879 # Catch some wrong configurations related to VLMs880 if self._is_vlm and args.packing:881 raise ValueError(882 "Packing is not supported for vision-language models. Please set `packing=False` in the SFTConfig."883 )884 if self._is_vlm and args.padding_free:885 raise ValueError(886 "Padding-free training is yet not supported for vision-language models. Please set "887 "`padding_free=False` in the `SFTConfig`."888 )889 if self._is_vlm and args.assistant_only_loss:890 raise ValueError(891 "Assistant-only loss is not yet supported for vision-language models. Please set "892 "`assistant_only_loss=False` in the `SFTConfig`."893 )894 895 # PEFT configuration and model wrapping896 if False:897 if added_tokens:898 # Ensure that the added tokens are trainable899 if peft_config.trainable_token_indices is None:900 peft_config.trainable_token_indices = {"embed_tokens": added_tokens}901 elif "embed_tokens" not in peft_config.trainable_token_indices:902 peft_config.trainable_token_indices["embed_tokens"] = added_tokens903 else:904 peft_config.trainable_token_indices["embed_tokens"].extend(added_tokens)905 906 # Ensure that the lm_head is trainable907 if peft_config.modules_to_save is None or "lm_head" not in peft_config.modules_to_save:908 logger.warning(909 "Cloning chat template added new tokens to the tokenizer, but 'lm_head' is not in PEFT's "910 "`modules_to_save`. As a result, the model may not learn to generate outputs with these new "911 "tokens, leading to degraded generation quality. To fix this, add "912 "`modules_to_save=['lm_head']` to your PEFT configuration."913 )914 915 if peft_config.modules_to_save is None:916 peft_config.modules_to_save = ["lm_head"]917 else:918 peft_config.modules_to_save.append("lm_head")919 920 # In Prompt Tuning a small set of trainable virtual tokens [continuous prompt embeddings] is prepended to the921 # input. We store the number of these tokens so we can account for them correctly when calculating accuracy.922 self.num_virtual_tokens = 0923 924 if False:925 pass926 if model.active_adapter in model.peft_config:927 peft_model_config = model.peft_config[model.active_adapter]928 self.num_virtual_tokens = getattr(peft_model_config, "num_virtual_tokens", 0)929 930 # Data collator931 # BFD packing requires padding-free mode; otherwise, the collator outputs padded attention masks, causing932 # FlashAttention to ignore position_ids and recompute them incorrectly from the padded attention mask.933 self.padding_free = args.padding_free or (args.packing and args.packing_strategy == "bfd")934 use_flash_attention = model.config._attn_implementation in [935 "flash_attention_2",936 "flash_attention_3",937 "kernels-community/vllm-flash-attn3",938 ]939 if self.padding_free:940 if data_collator is not None:941 raise ValueError("Passing a custom data collator is not supported when using padding-free.")942 if args.packing and args.packing_strategy == "wrapped":943 logger.warning(944 "You are passing `padding_free=True` with the 'wrapped' packing strategy, which is not "945 "recommended. Please refer to the documentation to understand why this is not recommended."946 )947 if not use_flash_attention:948 logger.warning(949 "Padding-free training is enabled, but the attention implementation is not set to "950 "'flash_attention_2'. Padding-free training flattens batches into a single sequence, and "951 "'flash_attention_2' is the only known attention mechanism that reliably supports this. Using "952 "other implementations may lead to unexpected behavior. To ensure compatibility, set "953 "`attn_implementation='flash_attention_2'` in the model configuration, or verify that your "954 "attention mechanism can handle flattened sequences."955 )956 957 # Decide whether to use completion-only loss: if not specified, then it is set to True if the dataset format958 # is prompt-completion, and False if the dataset format is language modeling.959 dataset_sample = next(iter(train_dataset))960 if args.completion_only_loss is None:961 self.completion_only_loss = "prompt" in dataset_sample and "completion" in dataset_sample962 else:963 self.completion_only_loss = args.completion_only_loss964 965 if data_collator is None and not self._is_vlm:966 # Get the pad token: if not provided, use the one from the processing class or the eos token967 # if the processing class does not have a pad token.968 pad_token = args.pad_token or tokenizer.pad_token or tokenizer.eos_token969 pad_token_id = tokenizer.convert_tokens_to_ids(pad_token)970 if pad_token_id is None:971 raise ValueError(972 f"The specified `pad_token` ('{pad_token}') is not found in the vocabulary of the given "973 f"`processing_class` ({processing_class.__class__.__name__}). Ensure that the `pad_token` exists "974 "in the vocabulary before using it as a padding token."975 )976 data_collator = DataCollatorForLanguageModeling(977 pad_token_id=pad_token_id,978 completion_only_loss=self.completion_only_loss,979 padding_free=self.padding_free,980 # Using position_ids without flash_attn hurts the training981 return_position_ids=use_flash_attention,982 pad_to_multiple_of=args.pad_to_multiple_of,983 )984 elif data_collator is None and self._is_vlm:985 data_collator = DataCollatorForVisionLanguageModeling(986 processor=processing_class,987 max_length=args.max_length,988 completion_only_loss=self.completion_only_loss,989 pad_to_multiple_of=args.pad_to_multiple_of,990 dataset_text_field=args.dataset_text_field,991 )992 993 if args.packing and args.packing_strategy == "bfd" and not use_flash_attention:994 logger.warning(995 "You are using packing, but the attention implementation is not set to 'flash_attention_2' or "996 "'kernels-community/vllm-flash-attn3'. Packing flattens batches into a single sequence, and Flash "997 "Attention is the only known attention mechanisms that reliably support this. Using other "998 "implementations may lead to cross-contamination between batches. To avoid this, either disable "999 "packing by setting `packing=False`, or set `attn_implementation='flash_attention_2'` or "1000 "`attn_implementation='kernels-community/vllm-flash-attn3'` in the model configuration."1001 )1002 if args.assistant_only_loss and not is_conversational(dataset_sample):1003 raise ValueError(1004 "You set `assistant_only_loss=True`, but the dataset is not conversational. This option is only "1005 "supported for conversational datasets."1006 )1007 1008 # Dataset1009 # Skip dataset preparation if `skip_prepare_dataset=True` in `dataset_kwargs`, or if it's a VLM, where1010 # preprocessing [e.g., image-to-pixel conversion] is too costly and done on the fly instead.1011 skip_prepare_dataset = (1012 args.dataset_kwargs is not None and args.dataset_kwargs.get("skip_prepare_dataset", False) or self._is_vlm1013 )1014 if not skip_prepare_dataset:1015 if self.completion_only_loss and formatting_func:1016 raise ValueError(1017 "A formatting function was provided while `completion_only_loss=True`, which is incompatible. "1018 "Using a formatter converts the dataset to a language modeling type, conflicting with "1019 "completion-only loss. To resolve this, apply your formatting function before passing the "1020 "dataset, or disable `completion_only_loss` in `SFTConfig`."1021 )1022 self._unsloth_model_ref = model1023 train_dataset = self._prepare_dataset(1024 train_dataset, processing_class, args, args.packing, formatting_func, "train"1025 )1026 if eval_dataset is not None:1027 packing = args.packing if args.eval_packing is None else args.eval_packing1028 if isinstance(eval_dataset, dict):1029 eval_dataset = {1030 key: self._prepare_dataset(dataset, processing_class, args, packing, formatting_func, key)1031 for key, dataset in eval_dataset.items()1032 }1033 else:1034 eval_dataset = self._prepare_dataset(1035 eval_dataset, processing_class, args, packing, formatting_func, "eval"1036 )1037 1038 # Loss function1039 if args.loss_type == "nll":1040 pass # use the default loss1041 elif args.loss_type == "dft":1042 if compute_loss_func is not None:1043 raise ValueError(1044 "You passed a `compute_loss_func` together with `loss_type='dft'` to the `SFTTrainer`. "1045 "When using `loss_type='dft'`, the loss function is internally set to the DFT loss, so passing a "1046 "`compute_loss_func` is not allowed."1047 )1048 compute_loss_func = dft_loss1049 else:1050 raise ValueError(f"Invalid `loss_type` {args.loss_type} passed. Supported values are 'nll' and 'dft'.")1051 1052 # Initialize the metrics1053 self._metrics = {"train": defaultdict(list), "eval": defaultdict(list)}1054 self._total_train_tokens = 01055 1056 # Initialize the Trainer. Parent class will handle:1057 # - DeepSpeed configuration [through create_accelerator_and_postprocess]1058 # - FSDP setup1059 # - Distributed training setup1060 # - Optimizer and scheduler creation1061 1062 super().__init__(1063 model=model,1064 args=args,1065 data_collator=data_collator,1066 train_dataset=train_dataset,1067 eval_dataset=eval_dataset,1068 processing_class=processing_class,1069 compute_loss_func=compute_loss_func,1070 compute_metrics=compute_metrics,1071 callbacks=callbacks,1072 optimizers=optimizers,1073 optimizer_cls_and_kwargs=optimizer_cls_and_kwargs,1074 preprocess_logits_for_metrics=preprocess_logits_for_metrics,1075 )1076 1077 # Initialize activation offloading context1078 if self.args.activation_offloading:1079 self.maybe_activation_offload_context = get_act_offloading_ctx_manager(model=self.model)1080 else:1081 self.maybe_activation_offload_context = contextlib.nullcontext()1082 1083 # Add tags for models that have been loaded with the correct transformers version1084 if hasattr(self.model, "add_model_tags"):1085 self.model.add_model_tags(self._tag_names)1086 1087 self.aux_loss_enabled = getattr(model.config, "output_router_logits", False)1088 self.aux_loss_coef = getattr(model.config, "router_aux_loss_coef", 0.0)1089 if self.aux_loss_enabled and self.aux_loss_coef == 0.0:1090 logger.warning(1091 "You set `output_router_logits` to `True` in the model config, but `router_aux_loss_coef` is set to "1092 "`0.0`, meaning the auxiliary loss will not be used. Either set `router_aux_loss_coef` to a value "1093 "greater than `0.0`, or set `output_router_logits` to `False` if you don't want to use the auxiliary "1094 "loss.",1095 )1096 1097 def _prepare_dataset(1098 self,1099 dataset: Union[Dataset, IterableDataset],1100 processing_class,1101 args,1102 packing: bool,1103 formatting_func: Optional[Callable[[dict], str]],1104 dataset_name: str,1105 ) -> Union[Dataset, IterableDataset]:1106 # All Unsloth Zoo code licensed under LGPLv31107 try:1108 if isinstance(dataset, ConstantLengthDataset): return dataset1109 except:1110 pass1111 1112 map_kwargs = {}1113 use_desc = isinstance(dataset, Dataset)1114 is_vlm = hasattr(processing_class, "tokenizer")1115 tokenizer = processing_class1116 if is_vlm: tokenizer = processing_class.tokenizer1117 1118 # Dynamic detection: check if model's module defines a function1119 # that requires token_type_ids when is_training=True1120 import sys as _sys1121 _needs_token_type_ids = False1122 # Split to avoid compiler substring match on masking_utils names1123 _ccm = 'create_' + 'causal_mask_mapping'1124 _model = getattr(self, '_unsloth_model_ref', None) or getattr(self, 'model', None)1125 if _model is not None:1126 for _m in (_model, getattr(_model, 'model', None)):1127 if _m is None: continue1128 _mod = _sys.modules.get(type(_m).__module__)1129 if _mod is not None and hasattr(_mod, _ccm):1130 _needs_token_type_ids = True1131 break1132 1133 if not _needs_token_type_ids:1134 # Fallback: model not yet available, check processor class MRO1135 for _base in type(processing_class).__mro__:1136 _base_mod = getattr(_base, '__module__', '')1137 if 'transformers.models.' in _base_mod:1138 _modeling_mod = _base_mod.replace('.processing_', '.modeling_')1139 _mod = _sys.modules.get(_modeling_mod)1140 if _mod is not None and hasattr(_mod, _ccm):1141 _needs_token_type_ids = True1142 break1143 if _needs_token_type_ids and hasattr(args, 'remove_unused_columns'):1144 args.remove_unused_columns = False1145 1146 # Get max length1147 max_seq_length = getattr(args, "max_length", 0)1148 if max_seq_length == 0: max_seq_length = getattr(args, "max_seq_length", 0)1149 if max_seq_length == 0: max_seq_length = getattr(self, "max_seq_length", 0)1150 if max_seq_length == 0: max_seq_length = getattr(self, "max_seq", 0)1151 if max_seq_length == 0: raise RuntimeError("Unsloth: max_seq_length is 0! Please specify one!")1152 dataset_text_field = getattr(args, "dataset_text_field", "text")1153 do_truncation = max_seq_length != 01154 do_formatting_func = False1155 do_tokenize = True1156 do_prompt_completion = False1157 1158 # Get correct column names1159 column_names = set(next(iter(dataset)).keys())1160 used_column_names = ["input_ids"]1161 if "attention_mask" in column_names:1162 used_column_names.append("attention_mask")1163 if _needs_token_type_ids:1164 used_column_names.append("token_type_ids")1165 1166 # Check if already tokenized so skip1167 from transformers import DataCollatorForSeq2Seq, DataCollatorForLanguageModeling1168 if "labels" in column_names:1169 # Most likely forgot data collator!1170 if is_vlm and not hasattr(tokenizer, "pad"):1171 # Check if processing_class has a .pad, if not, use tokenizer.tokenizer1172 raise RuntimeError(f"Unsloth: {processing_class.__class__} does not have .pad!")1173 self.data_collator = DataCollatorForSeq2Seq(tokenizer)1174 used_column_names.append("labels")1175 do_tokenize = False1176 elif "input_ids" in column_names:1177 # Skip dataset prep, and set data collator1178 if is_vlm and not hasattr(tokenizer, "pad"):1179 # Check if processing_class has a .pad, if not, use tokenizer.tokenizer1180 raise RuntimeError(f"Unsloth: {processing_class.__class__} does not have .pad!")1181 self.data_collator = DataCollatorForLanguageModeling(tokenizer, mlm = False)1182 do_tokenize = False1183 elif "prompt" in column_names and "completion" in column_names:1184 # Prompt/completion dataset (used with completion_only_loss).1185 # TRL's __init__ already set self.data_collator for completion_only_loss1186 # before calling us -- we must NOT overwrite it here.1187 do_prompt_completion = True1188 used_column_names.append("completion_mask")1189 elif dataset_text_field not in column_names:1190 do_formatting_func = True1191 if formatting_func is None:1192 raise RuntimeError("Unsloth: You must specify a `formatting_func`")1193 pass1194 1195 if do_tokenize:1196 # Check double BOS tokens1197 if do_formatting_func:1198 test_text = formatting_func(next(iter(dataset)))1199 if not isinstance(test_text, list):1200 raise ValueError(