o h@sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlmZddlmZddlmZmZmZmZmZmZmZmZddlmZm Z ddl!m"Z#ddl$Z%ddl&Z&ddl'm(Z)ddl*m+Z+m,Z,m-Z-dd l.m/Z/dd l&m0Z0dd l1m2Z2m3Z3m4Z4m5Z5dd l6m7Z7dd l8m9Z9ddl:m;Z;mm?Z?m@Z@ddlAmBZBmCZCddlDmEZEmFZFmGZGddlHmIZIddlJmKZKmLZLmMZMddlNmOZOmPZPddlQmRZRmSZSddlTmUZUmVZVddlWmXZXddlYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`ddlambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsddltmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZmZmZmZmZmZmZmZmZmZmZmZddlmZmZmZddl"mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddlmZe[gZe]ZerddlmZeZerddlmZerddlZed d!rddlmmZddlmmZerddlmm&Zdd lm7Ze/ee/d"kZdd#lamZmZmZmZnd ZerddlZer'dd$lmZerldd%lmZmZdd lm7Zdd&lmZmZmZmZmZmZe4gZe/e͡e/d'krbdd(lmZeeg7ZeGrldd)lmZersddlZeeۡZd*Zd+Zd,Zd-Zd.Zd/Zd0ZGd1d2d2ZdS)3uc The Trainer class, to easily train a 🤗 Transformers from scratch or finetune it on a new task. N)Mapping)Path) TYPE_CHECKINGAnyCallableDictListOptionalTupleUnion)#get_reporting_integration_callbacks hp_params) ModelCard create_repo upload_folder)version)nn) DataLoaderDataset RandomSamplerSequentialSampler) __version__)PretrainedConfig) DataCollatorDataCollatorWithPaddingdefault_data_collator) DebugOptionDebugUnderflowOverflow)"ALL_HYPERPARAMETER_SEARCH_BACKENDSdefault_hp_search_backend)deepspeed_initdeepspeed_load_checkpointis_deepspeed_available)TrainingSummary)PreTrainedModelload_sharded_checkpoint unwrap_model)!MODEL_FOR_CAUSAL_LM_MAPPING_NAMESMODEL_MAPPING_NAMES) Adafactor get_scheduler)ALL_LAYERNORM_LAYERSis_torch_less_than_1_11)PreTrainedTokenizerBase)CallbackHandlerDefaultFlowCallbackPrinterCallbackProgressCallbackTrainerCallbackTrainerControl TrainerState)DistributedTensorGathererIterableDatasetShard LabelSmootherLengthGroupedSamplerSequentialDistributedSamplerdistributed_broadcast_scalarsdistributed_concatfind_batch_sizeget_dataloader_samplerget_model_param_countget_module_class_from_nameget_parameter_names nested_concat nested_detachnested_numpifynested_xla_mesh_reducereissue_pt_warningsremove_dummy_checkpoint)PREFIX_CHECKPOINT_DIRBestRunEvalLoopOutputEvalPredictionHPSearchBackend HubStrategyIntervalStrategyPredictionOutputRemoveColumnsCollatorTrainerMemoryTracker TrainOutputdefault_compute_objectivedenumpify_detensorizeenable_full_determinismfind_executable_batch_sizeget_last_checkpoint has_lengthneftune_post_forward_hooknumber_of_arguments seed_workerset_seed speed_metrics)OptimizerNames ParallelModeTrainingArguments)ADAPTER_CONFIG_NAMEADAPTER_SAFE_WEIGHTS_NAMEADAPTER_WEIGHTS_NAME CONFIG_NAMESAFE_WEIGHTS_INDEX_NAMESAFE_WEIGHTS_NAMEWEIGHTS_INDEX_NAME WEIGHTS_NAMEPushInProgresscan_return_loss find_labelsis_accelerate_availableis_apex_availableis_bitsandbytes_availableis_datasets_availableis_in_notebookis_ipex_availableis_peft_availableis_safetensors_availableis_sagemaker_dp_enabledis_sagemaker_mp_enabledis_torch_compile_availableis_torch_neuroncore_availableis_torch_npu_availableis_torch_tpu_availablelogging strtobool)QuantizationMethod)NotebookProgressCallback)ampF) check_devicez1.10)smp_forward_backwardsmp_forward_only smp_gathersmp_nested_concat) PeftModel) Acceleratorskip_first_batches)DistributedDataParallelKwargsGradientAccumulationPluginload_fsdp_modelload_fsdp_optimizersave_fsdp_modelsave_fsdp_optimizer0.23.0)SeedableRandomSampler)DeepSpeedSchedulerWrapperztraining_args.binztrainer_state.jsonz optimizer.ptz optimizer.binz scheduler.ptz scaler.ptpytorch_model_fsdpc@seZdZdZddlmZmZmZmZm Z            dde e e j fdedeed eed ee eeeeffd eed eege fd eeegefdeeedeejjejjjfdeeejejgejffddZ ddZ!ddZ"ddZ#ddZ$ddZ%ddZ&dd Z'dd!d"d#eefd$d%Z( dded#eed&efd'd(Z)d&eej*j+j,fd)d*Z-d&e.fd+d,Z/d ed&eej*j+j,fd-d.Z0dd eed&e.fd/d0Z1d1ed&e.fd2d3Z2d4e3fd5d6Z4d&eefd7d8Z5d9d:Z6e7ded&ee8e8ffd;d<Z9dd4e3d=ejjfd>d?Z:d@e.d&e3fdAdBZ;ddCe.dDee3d&e3fdEdFZffdMdNZ?dOefdPdQZ@ddRdSZAddUdVZBdTejCfdWdXZDddZd[ZE   dd\ee eeFfdGe dHeee8ffd]eeefd^d_ZG dd`daZHdbdcZIddddeZJdfdgZKdhdiZLdjdkZMdldmZNddndoZOdpdqZPdrdsZQdtduZR   v w  ddxeedHgeee>ffdyeeeee>fge>fdze3d{e eeefd|ee d}eSfd~eedHgefd&e eTeeTffddZUdeee>fd&dfddZVde eje8fd&e eje8ffddZWdeee eje8ffd&eee eje8fffddZXddZYddeeFfddZZde j deee eje8ffd&ejfddZ[dddZ\d&eFfddZ]d&eFfddZ^ddeedeFfddZ_ddeefddZ`ddeefddZaddZbdecdTfd&eefddZdddddZe   dd eedeeeded&eee>ffddZf dd1edeeeded&egfddZh   dd@e.d#edeeFdeeeded&eif ddZjdddZk dde j deee eje8ffdeFdeeed&eeejeejeejff ddZldeee eje8fffddZmddZn         ddeedeede eeedfdeedeede eeedfde eeedfd!e eeedfde eeedffddZoddÄZpddńZqddeedeFd&efddʄZr   dd@e.d#edeeFdeeeded&eif dd̄Zsdd΄ZtdddЄZudd҄ZvdS)Trainerul Trainer is a simple but feature-complete training and eval loop for PyTorch, optimized for 🤗 Transformers. Args: model ([`PreTrainedModel`] or `torch.nn.Module`, *optional*): The model to train, evaluate or use for predictions. If not provided, a `model_init` must be passed. [`Trainer`] is optimized to work with the [`PreTrainedModel`] provided by the library. You can still use your own models defined as `torch.nn.Module` as long as they work the same way as the 🤗 Transformers models. args ([`TrainingArguments`], *optional*): The arguments to tweak for training. Will default to a basic instance of [`TrainingArguments`] with the `output_dir` set to a directory named *tmp_trainer* in the current directory if not provided. data_collator (`DataCollator`, *optional*): The function to use to form a batch from a list of elements of `train_dataset` or `eval_dataset`. Will default to [`default_data_collator`] if no `tokenizer` is provided, an instance of [`DataCollatorWithPadding`] otherwise. train_dataset (`torch.utils.data.Dataset` or `torch.utils.data.IterableDataset`, *optional*): The dataset to use for training. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in a distributed fashion, your iterable dataset should either use a internal attribute `generator` that is a `torch.Generator` for the randomization that must be identical on all processes (and the Trainer will manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that internally sets the seed of the RNGs used. eval_dataset (Union[`torch.utils.data.Dataset`, Dict[str, `torch.utils.data.Dataset`]), *optional*): The dataset to use for evaluation. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. If it is a dictionary, it will evaluate on each dataset prepending the dictionary key to the metric name. tokenizer ([`PreTrainedTokenizerBase`], *optional*): The tokenizer used to preprocess the data. If provided, will be used to automatically pad the inputs to the maximum length when batching inputs, and it will be saved along the model to make it easier to rerun an interrupted training or reuse the fine-tuned model. model_init (`Callable[[], PreTrainedModel]`, *optional*): A function that instantiates the model to be used. If provided, each call to [`~Trainer.train`] will start from a new instance of the model as given by this function. The function may have zero argument, or a single one containing the optuna/Ray Tune/SigOpt trial object, to be able to choose different architectures according to hyper parameters (such as layer count, sizes of inner layers, dropout probabilities etc). compute_metrics (`Callable[[EvalPrediction], Dict]`, *optional*): The function that will be used to compute metrics at evaluation. Must take a [`EvalPrediction`] and return a dictionary string to metric values. callbacks (List of [`TrainerCallback`], *optional*): A list of callbacks to customize the training loop. Will add those to the list of default callbacks detailed in [here](callback). If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method. optimizers (`Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`, *optional*, defaults to `(None, None)`): A tuple containing the optimizer and the scheduler to use. Will default to an instance of [`AdamW`] on your model and a scheduler given by [`get_linear_schedule_with_warmup`] controlled by `args`. preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`, *optional*): A function that preprocess the logits right before caching them at each evaluation step. Must take two tensors, the logits and the labels, and return the logits once processed as desired. The modifications made by this function will be reflected in the predictions received by `compute_metrics`. Note that the labels (second parameter) will be `None` if the dataset does not have them. Important attributes: - **model** -- Always points to the core model. If using a transformers model, it will be a [`PreTrainedModel`] subclass. - **model_wrapped** -- Always points to the most external model in case one or more other modules wrap the original model. This is the model that should be used for the forward pass. For example, under `DeepSpeed`, the inner model is wrapped in `DeepSpeed` and then again in `torch.nn.DistributedDataParallel`. If the inner model hasn't been wrapped, then `self.model_wrapped` is the same as `self.model`. - **is_model_parallel** -- Whether or not a model has been switched to a model parallel mode (different from data parallelism, this means some of the model layers are split on different GPUs). - **place_model_on_device** -- Whether or not to automatically place the model on the device - it will be set to `False` if model parallel or deepspeed is used, or if the default `TrainingArguments.place_model_on_device` is overridden to return `False` . - **is_in_train** -- Whether or not a model is currently running `train` (e.g. when `evaluate` is called while in `train`) r )_get_learning_rate log_metricsmetrics_format save_metrics save_stateNNNmodelargs data_collator train_dataset eval_dataset tokenizer model_initcompute_metrics callbacks optimizerspreprocess_logits_for_metricsc CsB|durd} td| dt| d}||_|jjr!t|jjnt|jjd|_d|_ d|_ | t |jj |_|j|} t| |j|dur`|dur\||_|}ntd|durjtdt||_|jjtvr}td|jjd t|d r|jr|j rd |_!nd|_!t"|d ddurd dt#|j$%D}t&|dkrd |_!nt&|dkr|jj't('|dk|_!nd|_!|j!rtdt)ot*|t+}t"|ddot"|dd }|r|std|rt"|ddstd|j,d|_-t&|j.dkr|j/r td|j,ds|j0t1j2krtd|j3|_3|j!s;|j/s;|j4s/|j5r3|j6r;|j-s;|j7r>d|_3|durEt8nt9|}|durP|n||_:||_;||_<||_=|j3rqt"|ddt>j?ksq|@||j'|j!ryd|j_A||_B||_C|jD|_D||_E| |_F| \|_G|_H|dur|jGdus|jHdurtdtIr|jGdur|jCJD]}|j'}|jGjKD]}t&|ddkr|ddj'}nq||krtd|j/s|j-s|j7r|jGdus|jHdurtdtLtM|jjN}| dur|n|| } tO| |jC|j=|jG|jH|_P|Q|jjRr!tSntTd|_Ud|_V|jjWr3|X|jjYrAtZj[|jj\d dt]|j:sUt]t"|j:d drUtd!|j^dkr`td"|durtt_|st|j^dkrttd#|durt*|t(j`jajbr|jcrtd$d|_dd|_ed|_ftgr|jhrtd%tir|jjtkjljmjjkrtnd&tkjljmjjd'|jjd(tkjljmjjtkjljmjj|_jnttkjljmd)rtnd&tkjljmjjd*|jjs|jhr|jod+kr|j't('d,kr|jjrtd-d.|_otd/|jod0|jjs|jhr6|j/s6tgs6|jod.kr%d |_ft(jp|_qn|jod1kr6trs3tsd2d |_e|jjtdkrFtu|jjtd3|_vnd|_vtw|x|yd4|_ltz|_{d|_|d|_}t~|jCj}|jjdurm|n|jj|_t|jCj|_|jP|j|jl|j{|_{|j|_d|_|j|jrtstd5dSdS)6N tmp_trainerz1No `TrainingArguments` passed, using `output_dir=z`.) output_dirFz<`Trainer` requires either a `model` or `model_init` argumentz`Trainer` requires either a `model` or `model_init` argument, but not both. `model_init` will overwrite your model when calling the `train` method. This will become a fatal error in the next release.zThe model you have picked (a) cannot be used as is for training: it only computes hidden states and does not accept any labels. You should choose a model with a head suitable for your task like any of the `AutoModelForXxx` listed at https://huggingface.co/docs/transformers/model_doc/autois_parallelizableT hf_device_mapcSsg|]}|dvr|qS))cpudisk).0devicerrJ/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/trainer.py z$Trainer.__init__..r rzYou have loaded a model on multiple GPUs. `is_model_parallel` attribute will be force-set to `True` to avoid any unexpected behavior such as device placement mismatching. is_quantized_hf_peft_config_loadedzYou cannot perform fine-tuning on purely quantized models. Please attach trainable adapters on top of the quantized model to correctly perform fine-tuning. Please see: https://huggingface.co/docs/transformers/peft for more details_is_quantized_training_enabledzThe model you want to train is loaded in 8-bit precision. if you want to fine-tune an 8-bit model, please make sure that you have installed `bitsandbytes>=0.37.0`. xlazZUsing --fsdp xxx together with --deepspeed is not possible, deactivate one of those flags.z.Using fsdp only works in distributed training.quantization_methodzPassing a `model_init` is incompatible with providing the `optimizers` argument. You should subclass `Trainer` and override the `create_optimizer_and_scheduler` method.paramsa[The model and the optimizer parameters are not on the same device, which probably means you created an optimizer around your model **before** putting on the device and passing it to the `Trainer`. Make sure the lines `import torch_xla.core.xla_model as xm` and `model.to(xm.xla_device())` is performed before the optimizer creation in your script.zPassing `optimizers` is not allowed if Deepspeed or PyTorch FSDP is enabled. You should subclass `Trainer` and override the `create_optimizer_and_scheduler` method.exist_ok collate_batchzRThe `data_collator` should be a simple callable (function, class with `__call__`).zHmax_steps is given, it will override any value given in num_train_epochszThe train_dataset does not implement __len__, max_steps has to be specified. The number of steps needs to be known in advance for the learning rate scheduler.zTthe `--group_by_length` option is only available for `Dataset`, not `IterableDatasetzOSageMaker Model Parallelism does not support BF16 yet. Please use FP16 instead z(FP16 provided in SM_HP_MP_PARAMETERS is z+, but FP16 provided in trainer argument is z , setting to fp16zJ, but SageMaker Model Parallelism < 1.10 does not support FP16 in trainer.autorz2Tried to use `fp16` but it is not supported on cpucpu_ampzUsing z half precision backendapexzcUsing FP16 with APEX but APEX is not installed, please refer to https://www.github.com/nvidia/apex.)epsilon)is_local_process_zerois_world_process_zeroz3Using torch.compile requires PyTorch 2.0 or higher.)loggerinfor`rfull_determinismrUseedr\hp_name deepspeed is_in_train"create_accelerator_and_postprocessrQskip_memory_metrics_memory_trackerstartget_process_log_levelrz set_verbosity_setup_devicesrcall_model_init RuntimeErrorwarningswarn FutureWarning __class____name__r) ValueErrorhasattrrmodel_parallelis_model_parallelgetattrsetrvalueslenrtorchrr isinstancer fsdp_configis_fsdp_xla_enabledfsdpis_deepspeed_enabled parallel_moder_ DISTRIBUTEDplace_model_on_devicefp16_full_evalbf16_full_evaldo_trainis_fsdp_enabledrrrrrrr|BITS_AND_BYTES_move_model_to_device_n_gpu model_wrappedrneftune_noise_alpharr optimizer lr_schedulerry parameters param_groupsDEFAULT_CALLBACKSr report_tor/callback_handler add_callback disable_tqdmr1DEFAULT_PROGRESS_CALLBACK_loggers_initialized hub_model_id push_to_hub init_hf_repo should_saveosmakedirsrcallable max_stepsrXutilsdataIterableDatasetgroup_by_length_signature_columnsuse_apex use_cpu_amprubf16IS_SAGEMAKER_MP_POST_1_10rsmpstatecfgwarninghalf_precision_backendbfloat16 amp_dtyperm ImportErrorlabel_smoothing_factorr8label_smootherr5rrr4control current_floshp_search_backendrk label_namesrj on_init_endtrain_batch_size_train_batch_size_created_lr_schedulerstop_and_update_metrics torch_compilerv)selfrrrrrrrrrrrr log_leveldevices_is_peft_model_is_quantized_and_base_modeldefault_collatorparam model_device param_groupoptimizer_devicedefault_callbacksdefault_label_namesrrr__init__>s          "                zTrainer.__init__cCsLt|}trt|tr|jj}n|}~|j|_|t }||_ |S)z Activates the neftune as presented in this code: https://github.com/neelsjain/NEFTune and paper: https://arxiv.org/abs/2310.05914 ) r'rrrr base_modelrget_input_embeddingsrregister_forward_hookrYneftune_hook_handle)rrunwrapped_model embeddings hook_handlerrr_activate_neftunegs zTrainer._activate_neftunecCsTt|ds tdt|}trt|tr|jj}n|}|j |` ~dS)z^ Deactivates the neftune method. Make sure to call `_activate_neftune` first. r.zNNeftune is not activated make sure to call `trainer._activate_neftune()` firstN) rrr'rrrrr+rr,r.remover)rrr/r0rrr_deactivate_neftunezs   zTrainer._deactivate_neftunecC|j|dS)ag Add a callback to the current list of [`~transformers.TrainerCallback`]. Args: callback (`type` or [`~transformers.TrainerCallback`]): A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the first case, will instantiate a member of that class. N)rrrcallbackrrrr zTrainer.add_callbackcCs |j|S)aK Remove a callback from the current list of [`~transformers.TrainerCallback`] and returns it. If the callback is not found, returns `None` (and no error is raised). Args: callback (`type` or [`~transformers.TrainerCallback`]): A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the first case, will pop the first member of that class found in the list of callbacks. Returns: [`~transformers.TrainerCallback`]: The callback removed, if found. )r pop_callbackr6rrrr9s zTrainer.pop_callbackcCr5)a Remove a callback from the current list of [`~transformers.TrainerCallback`]. Args: callback (`type` or [`~transformers.TrainerCallback`]): A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the first case, will remove the first member of that class found in the list of callbacks. N)rremove_callbackr6rrrr:r8zTrainer.remove_callbackcCs6||}|jjtjkrt|dr|dSdSdS)N tie_weights)torrr_TPUrr;)rrrrrrrs  zTrainer._move_model_to_devicecCsP|jdur&t|jj}t|j|_|jttddg|j 7_dSdS)Nlabel label_ids) rinspect signaturerforwardlistrkeysrr)rrArrr _set_signature_columns_if_neededs $z(Trainer._set_signature_columns_if_neededdatasetzdatasets.Dataset descriptionc s|jjsS||j}ttjt|}t|dkrK|dur$dnd|d}t d|d|j j j dd |d d |d |j j j d fd d |D}ttjtdkrojjd|jddS|S)Nrzin the z setzThe following columns z) don't have a corresponding argument in `z!.forward` and have been ignored: , z. If z are not expected by `z/.forward`, you can safely ignore this message.csg|] }|jvr|qSr) column_namesrkrFrrrz2Trainer._remove_unused_columns..z1.4.0type format_kwargs)rOcolumnsrP)rremove_unused_columnsrErrCrrJrrrrrrjoinrparsedatasetsr set_formatformatremove_columns)rrFrGsignature_columnsignored_columnsdset_descriptionrQrrMr_remove_unused_columnss2  zTrainer._remove_unused_columnsreturncCs6|jjs|S||j}t||t||jjjd}|S)z=Wrap the data collator in a callable removing unused columns.)rrYrrG model_name) rrRrErrPrrrr)rrrGrYremove_columns_collatorrrr"_get_collator_with_removed_columnssz*Trainer._get_collator_with_removed_columnscCs|jdus t|js dS|jjrJtr,t|jtjr,|jj|jj vr)|j|jjnd}nd}|j dur9|j j dnd}t |jj |jj|j||dSt|jS)Nr)rFlengthsmodel_input_name)rrXrrrorrUrlength_column_namerJrmodel_input_namesr9rgradient_accumulation_stepsr)rrarbrrr_get_train_samplers" zTrainer._get_train_samplercCs|jdur td|j}|j}tr t|tjr |j|dd}n|j|dd}|j ||j j |j j |j j d}t|tjjjsN||d<|j j|d<t|d<|jt|fi|S) a@ Returns the training [`~torch.utils.data.DataLoader`]. Will use no sampler if `train_dataset` does not implement `__len__`, a random sampler (adapted to distributed training if necessary) otherwise. Subclass and override this method if you want to inject some custom behavior. Nz+Trainer: training requires a train_dataset.trainingrG batch_size collate_fn num_workers pin_memorypersistent_workerssampler drop_lastworker_init_fn)rrrrorrUrr\r`rrdataloader_num_workersdataloader_pin_memorydataloader_persistent_workersrrrrrfdataloader_drop_lastr[ acceleratorpreparer)rrrdataloader_paramsrrrget_train_dataloaders$   zTrainer.get_train_dataloadercCsf|jjr'trt|ttdStr#t|t t |jj dSt |S|jj dkr1t |SdS)N) num_replicasrank)rzr{rjr )ruse_legacy_prediction_loopryr:xmxrt_world_size get_ordinalrur dp_sizedp_rankper_device_eval_batch_sizer world_size)rrrrr_get_eval_sampler&s  zTrainer._get_eval_samplercCs|dur |jdur td|dur|n|j}|j}tr*t|tjr*|j|dd}n|j|dd}|j j ||j j |j j |j j d}t|tjjjsV|||d<|j j|d<|jt|fi|S)a Returns the evaluation [`~torch.utils.data.DataLoader`]. Subclass and override this method if you want to inject some custom behavior. Args: eval_dataset (`torch.utils.data.Dataset`, *optional*): If provided, will override `self.eval_dataset`. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. It must implement `__len__`. Nz-Trainer: evaluation requires an eval_dataset. evaluationrhrirorp)rrrrorrUrr\r`reval_batch_sizerrrsrtrrrrrrurvrwr)rrrrxrrrget_eval_dataloader<s"  zTrainer.get_eval_dataloader test_datasetcCs|j}trt|tjr|j|dd}n|j|dd}|jj||jj |jj |jj d}t|t j jjs@|||d<|jj|d<|jt|fi|S)a Returns the test [`~torch.utils.data.DataLoader`]. Subclass and override this method if you want to inject some custom behavior. Args: test_dataset (`torch.utils.data.Dataset`, *optional*): The test dataset to use. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. It must implement `__len__`. testrhrirorp)rrorrUrr\r`rrrrrsrtrrrrrrurvrwr)rrrrxrrrget_test_dataloader_s  zTrainer.get_test_dataloadernum_training_stepscCs8|trtjjjr|jj}n|j}|j||ddS)aZ Setup the optimizer and the learning rate scheduler. We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the Trainer's init through `optimizers`, or subclass and override this method (or `create_optimizer` and/or `create_scheduler`) in a subclass. )rrN)create_optimizerr r r r rrcreate_schedulerrrrrrrcreate_optimizer_and_schedulers  z&Trainer.create_optimizer_and_schedulercCst|t}dd|D}|S)a! Get all parameter names that weight decay will be applied to Note that some models implement their own layernorm instead of calling nn.LayerNorm, weight decay could still apply to those modules since this function only filter out instance of nn.LayerNorm cSsg|]}d|vr|qS)biasrrnamerrrrrz5Trainer.get_decay_parameter_names..)rAr,)rrdecay_parametersrrrget_decay_parameter_namess z!Trainer.get_decay_parameter_namesc sNtr|jn|j}|jdur||fdd|D|jjdfdd|Dddg}t |j\}}||fi||_|j dkrddl }|j j }d}|D]7}t|tjr|td d |D7}td |d |d d||ddditd|dqWtd|d dtrt|j|_|jS)a Setup the optimizer. We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the Trainer's init through `optimizers`, or subclass and override this method in a subclass. Ncs"g|] \}}|vr|jr|qSr requires_gradrnprrrrz,Trainer.create_optimizer..)r weight_decaycs"g|] \}}|vr|jr|qSrrrrrrrrAdam8bitrcSsi|] }||qSr)data_ptrnumel)rrrrr rNz,Trainer.create_optimizer..zskipped z: izM paramsweight optim_bits zbitsandbytes: will optimize z in fp32z skipped: )rurrrrnamed_parametersrrrget_optimizer_cls_and_kwargsr bitsandbytesoptimGlobalOptimManager get_instancemodulesrr Embeddingsumrrrrregister_module_overridedebugr DistributedOptimizer) r opt_modeloptimizer_grouped_parameters optimizer_clsoptimizer_kwargsrmanagerskippedmodulerrrrs>        zTrainer.create_optimizerc Csi}|jr|jdddD] }|d\}}|||<qd|ji}|j|jf|jd}|jtj kr@t }| ddd||fS|jtj krWd d l m}|}| |||fS|jtjtjfvr~d d lm}|}| ||jtjkrz| d d i||fS|jtjkrzd d lm}|}| |W||fStytdw|jtjkrzd dlm} | }| |W||fStytdw|jtjkrzd dlm} | }| |W||fStytdw|jtjtjtjtjtj tj!tj"tj#fvrszJd dl$m}m%} d} d} d}|}d|jvrd } d|jvr!d} d|jvr*|}nd|jvr:| }d|j|jfi}| | d}| || |Wn tyVtdwt&rot'(t)j*'dt'(dkrot+,d ||fS|jtj-krz7d d!l.m/}|}| || t0|1d"d#t2t3|1d$d%t2t3|1d&d%t2t3|1d'd(d)W||fStytd*w|jtj4krt3jj4}||fS|jtj5krt3jj6}||fS|jtj7krt3jj8}||fStd+|j),z Returns the optimizer class and optimizer parameters based on the training arguments. Args: args (`transformers.training_args.TrainingArguments`): The training arguments for the training session.  rH,=lr)betasepsF)scale_parameter relative_stepr )AdamWrfusedTz7Trainer failed to import syncfree AdamW from torch_xla.) NpuFusedAdamWz3Trainer failed to import FusedAdamW from torch_npu.) FusedAdamzFTrainer tried to instantiate apex FusedAdam but apex is not installed!)rLionrNpaged8bitadamlionr)is_pagedrzDTrainer tried to instantiate bnb optimizer but bnb is not installed!rz0.41.1zYou are using 8-bit optimizers with a version of `bitsandbytes` < 0.41.1. It is recommended to update your version as a major bug has been fixed in 8-bit optimizers.)AnyPrecisionAdamWuse_kahan_summationFalsemomentum_dtypefloat32variance_dtypecompensation_buffer_dtyper)rrrrz4Please install https://github.com/pytorch/torchdistxz2Trainer cannot instantiate unsupported optimizer: )9 optim_argsreplacesplit learning_rate adam_beta1 adam_beta2 adam_epsilonrr^ ADAFACTORr*updateADAMW_HF optimizationr ADAMW_TORCHADAMW_TORCH_FUSED torch.optimADAMW_TORCH_XLAtorch_xla.amp.syncfreerrADAMW_TORCH_NPU_FUSEDtorch_npu.optimrADAMW_APEX_FUSEDapex.optimizersr ADAMW_BNB ADAMW_8BIT PAGED_ADAMWPAGED_ADAMW_8BITLION LION_8BIT PAGED_LIONPAGED_LION_8BITbitsandbytes.optimrrnrrT importlibmetadatarr ADAMW_ANYPRECISIONtorchdistx.optimizersrr{getrrSGDADAGRADAdagradRMSPROPRMSprop)rrmappingkeyvaluer adam_kwargsrrrrrrradditional_optim_kwargs bnb_kwargsrrrrrs     h  c   \   V    N    F              z$Trainer.get_optimizer_cls_and_kwargsrcCsH|jdur!t|jj|dur|jn||j|||jjd|_d|_|jS)z Setup the scheduler. The optimizer of the trainer must have been set up either before this method is called or passed as an argument. Args: num_training_steps (int): The number of training steps to do. N)rnum_warmup_stepsrscheduler_specific_kwargsT)rr+rlr_scheduler_typerget_warmup_stepslr_scheduler_kwargsrrrrrrOs  zTrainer.create_scheduler dataloaderc CsTz|j}t|trt|jjWSt|jWStttfy)t||jjYSw)z Helper to get number of samples in a [`~torch.utils.data.DataLoader`] by accessing its dataset. When dataloader.dataset does not exist or has no length, estimates as best it can ) rFrr7r NameErrorAttributeError TypeErrorrper_device_train_batch_size)rrrFrrr num_examplesbs  zTrainer.num_examplestrain_dlrcCshd}z!t|D]\}}|d}|dur||WS||7}q|WSty3td|YSw)zq Helper to get number of tokens in a [`~torch.utils.data.DataLoader`] by enumerating dataloader. r input_idsNz%Cannot get num_tokens from dataloader) enumeraterKeyErrorrr )rr r train_tokensstepbatchtokensrrr num_tokensps    zTrainer.num_tokenstrialz optuna.TrialcCs||_|jdus |durdS|jtjkr||}n(|jtjkr)|}|ddn|jtjkr:dd|j D}n|jtj krB|}| D],\}}t |j |sZt d|dqFt|j |d}|durkt||}t|j ||qF|jtjkrt d|j|jtjkrt d|j|jtj krt d ||jr|j jdurtd d d lm}d d lm}||j j|j _|j j|j ||j jd|j _|dS)zHP search setup codeNwandbcSs(i|]\}}|t|trt|n|qSr)rstrintrrLvrrrrs(z,Trainer._hp_search_setup..zTrying to set zY in the hyperparameter search but there is no corresponding field in `TrainingArguments`.zTrial: zSigOpt Assignments: zW&B Sweep parameters: z7For sweeps with deepspeed, `args.deepspeed` must be setr)DeepSpeedPluginHfTrainerDeepSpeedConfig) hf_ds_config)_trialrrLOPTUNAhp_spaceRAYpopSIGOPT assignmentsitemsWANDBrrrr rrOsetattrrrrrraccelerate.utilsr#transformers.integrations.deepspeedrhf_deepspeed_configtrainer_config_processdeepspeed_pluginr)rrrrrold_attrrrrrr_hp_search_setupsJ               zTrainer._hp_search_setuprmetricscCs |jdus |dur dS|}|||_|jtjkrBddl}|js>| |j|| r@|j |j |j|j|dSdS|jtjkrddl}t)}d}|jjrd|j|d|jj|}|j|d<|jj ||dWddS1s|wYdSdS)Nr)checkpoint_dir objective checkpoint)rcopycompute_objectiver0rLroptunastudy_is_multi_objectivereport should_pruner on_train_endrr r TrialPrunedr  ray.traintempfileTemporaryDirectoryr_tune_save_checkpointtrain Checkpointfrom_directory)rrrr.r5raytemp_checkpoint_dirr2rrr_report_to_hp_searchs0       "zTrainer._report_to_hp_searchr/cCstj|td|jj}|j|dd|jjrB|j tj|t t |j tj|tt |jtj|tdSdS)N-T_internal_call)rpathrSrHr  global_step save_modelrr save_to_jsonTRAINER_STATE_NAMErsaver state_dictOPTIMIZER_NAMErSCHEDULER_NAME)rr/rrrrr?s zTrainer._tune_save_checkpointcCsLt|j}|dkr|}n|dkr||}ntd|dur$td|S)Nrr z'model_init should have 0 or 1 argument.z"model_init should not return None.)rZrr)rrmodel_init_argcountrrrrrs   zTrainer.call_model_initFc s|s|dur td|Stt||zt|}||jdd}|r.||_ |j j ddkt Wttt jjtdkrjttrYt jj|dd}n1t jj|fddDdd}n g}D]}t |}||qnt|}t jj||dd }Wdn1swYWdn1swYt j|}t |d i|d iWdn1swY|}d|_W|Stttttfy} ztd | d WYd} ~ |Sd} ~ ww|S) NzAfailed to use PyTorch jit mode due to current dataloader is none._original_forwardF) cache_enabledz2.0.0)example_kwarg_inputsstrictcsi|]}||qSrr)rr example_batchrrrsz0Trainer.torch_jit_model_eval..rVz'failed to use PyTorch jit mode due to: .r) rr nextiter_prepare_inputsr3eval__dict__r!rBrvautocastrno_gradrrTr base_versionrdictjittrace ones_likeappendtuplefreezerrrrr IndexError) rrrrg jit_modeloriginal_forward jit_inputsrexample_tensorerrWrtorch_jit_model_evalsZ        zTrainer.torch_jit_model_evalcCststdddl}|s+||js|jjrtjn|}|j ||dd|j d}|S|j s2| |j |||j ddd\}|_ |S)NzUsing IPEX but IPEX is not installed or IPEX's version does not match current PyTorch, please refer to https://github.com/intel/intel-extension-for-pytorch.rO1F)dtypelevelconv_bn_foldinginplaceT)rrrrurs) rqrintel_extension_for_pytorchr^rrrrroptimizergr@r)rrrgrripexrrripex_optimize_model s   zTrainer.ipex_optimize_modelTcs|jjr|jr tjntj}|j|||d}tr,t|j t j j r#|j St j ||jj dSt||ur4|S|jrG|rGtj||j|jjd\}|_|jjdkrXt|ddsXt|}|jjrqt}||||}tt|d|_|su|S|jrzdd lmdd lm dd l!m"}m#}Wn t$yt$d wd}d} t|d d} |jj%&d| } |jj%ddkrt'j(||jj%dd}n%| durt)} | D]} t*|| }|durt+d| ,|qt'j(|| d}|jj-}|jj%drfdd} |f|| d||_ }difdd}|t._/|St0r%tj1j2|t3t45dgd}|S|jj6t7j8krwt9r3|Si}|jj:durC|jj:|d<nt|t;rP|j< |d<nd|d<|jj=dura|jj=|d<|jj>durn|jj>|d<t?di||j@_A|S) N)rr)backward_passes_per_step) opt_levelr is_loaded_in_8bitFr)XlaFullyShardedDataParallel)checkpoint_module)size_based_auto_wrap_policytransformer_auto_wrap_policyzJMissing XLA FSDP related module; please make sure to use torch-xla >= 2.0._no_split_modulestransformer_layer_cls_to_wrapmin_num_params)rz@Could not find the transformer layer class to wrap in the model.)transformer_layer_clsxla_fsdp_grad_ckptcs|g|Ri|SNr)mrkwargsFSDPrrrauto_wrapper_callableisz2Trainer._wrap_model..auto_wrapper_callable)auto_wrap_policyrcSs |jdi|}|rt|S)Nr)rr} mark_step)rbarrieroptimizer_argslossrrrpatched_optimizer_stepvsz3Trainer._wrap_model..patched_optimizer_stepSMDATAPARALLEL_LOCAL_RANK) device_idsfind_unused_parametersT bucket_cap_mbbroadcast_buffersr)Bruse_ipexrrrrryrurrr rDistributedModelrer'rr~ initializerfp16_opt_leveln_gpurr DataParallel jit_mode_evaltimerproundjit_compilation_timertorch_xla.distributed.fsdpr~rtorch_xla.distributed.fsdp.wraprrrrr functoolspartialrr@ Exceptionaddxla_fsdp_configr}optimizer_steprtparallelDistributedDataParallelrrgetenvrr_rrwddp_find_unused_parametersr%is_gradient_checkpointingddp_bucket_cap_mbddp_broadcast_buffersrrv ddp_handler)rrrgrrr start_timerrrr%default_transformer_cls_names_to_wrap"fsdp_transformer_layer_cls_to_wraptransformer_cls_to_wrap layer_classtransformer_cls fsdp_kwargsrrrrr _wrap_model s                zTrainer._wrap_modelresume_from_checkpointignore_keys_for_evalc Ks|durd}|j|j}d|_|jdur||j|_|js#|jr.|j s.| |j|j d|vr=| d}t dtt|dkrRtddt|d |||jj|_d}|jdur|jjrmt|jjnt|jj|||_d}d \|_|_t |t!r|rt"|j#}|durt$d |j#d |durt%s|j&s|j's|(|t)*t+j,|t-}|jdur|j|_|r|j.r| |j|j |j|_/t0|j1|j|j2}|j3rzt45|||||d Wt46St46w|||||d S)a Main training entry point. Args: resume_from_checkpoint (`str` or `bool`, *optional*): If a `str`, local path to a saved checkpoint as saved by a previous instance of [`Trainer`]. If a `bool` and equals `True`, load the last checkpoint in *args.output_dir* as saved by a previous instance of [`Trainer`]. If present, training will resume from the model/optimizer/scheduler states loaded here. trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): The trial run or the hyperparameter dictionary for hyperparameter search. ignore_keys_for_eval (`List[str]`, *optional*) A list of keys in the output of your model (if it is a dictionary) that should be ignored when gathering predictions for evaluation during the training. kwargs (`Dict[str, Any]`, *optional*): Additional keyword arguments used to hide deprecated arguments FNT model_pathzi`model_path` is deprecated and will be removed in a future version. Use `resume_from_checkpoint` instead.rz3train() received got unexpected keyword arguments: rIrZrz/No valid checkpoint found in output directory ())rrrr)7rrrrrr2rrrrrrr!rrrrrrSrCrDr-rrrrrUrr\rrrrboolrWrrrurr_load_from_checkpointr5load_from_jsonrrIrMrrrV_inner_training_loopauto_find_batch_sizer hf_hub_utilsdisable_progress_barsenable_progress_bars) rrrrrrmodel_reloadedr inner_training_looprrrr@s             z Trainer.trainc/ Cs |j||_|jjr|j|j_td|j| }|j|j |j }d}d} t |rt |}||j } t| d} ||} |jdkrk|j} |j| t|j| dk} |j|}|jrj|||j|j } nOt|j| } t|j} |||j}|jr|||j} n-|jdkr|j} tj} | } ||j} |j|}|jr|||j|j } ntd|jtj|jjvr|jjdkrtdt|j}tp|j p|j!}|j"rd|_#d|_"|j$rt%|| d\|_&|_#|s|j'| dt(|_|du|j_)|j|j_|j*dur"|j*dkrt| |j*|j_*n|j*|j_*|j+dur>|j+dkr9t| |j+|j_+n|j+|j_+|j,durZ|j,dkrUt| |j,|j_,n|j,|j_,|j-rq|j.durgi}n|j.}|jj/|d|0|j1}||jurd nd}|r|j'| d|r|j2t3|j#d r|j4r|j5|j}n|j5|j|j&\}|_&n|j5|j|j&|j#\}|_&|_#|j!r||_|_1||jur||_1|j$r|j1|_6|dur|j$rt7|j1|nts|j!r|8||j1|9|t:d t:d | d t:d| d t:d|jj;d |jj;|jkr5t:d|jd t:d|d t:d|j t:d| d t:dt<|d dd d|j_=t>>}d}d}d}|durt?j@At?j@B|tCrt(Dt?j@B|tC|_|jjE| }|jFs|jjE| }||j 9}nd}t:dt:d|t:d|jjE|jFst:d|d|d|j|jG_|j&|jG_&|j#|jG_#||jG_H|jIdur|jJdur|I|jJ|j_K|dur|jLtMjNkr|jOn|}tP||j_Qnd|j_Q| |j_| |j_|R|j_R|S|j_StTUdV|jW}d|_X|jjE|_Y|Z|jG[||j|j\|_\|jFst]|D];}t^|}t_g}t`atbt`adkr^|ctdte|tf|}tgsk|sr|D]}qE|dury|ng}th|}qEd}t]|| D],}|} t3| dr| i||jjdkrd|_k|durt | n|j|j }!|jGl||j|j\|_\||kr|dur|dkr|m|d}"d}#|dkrtn| |} |}#d}d }"d}$to| D]~\}$}%|d7}|jjprtq|jd d!}&|&|%vr trd"n|jjs|jt|%|&u7_s|"r#|m|d}"|dkrB|d8}|dur6|vd|dkr@|m|q|durM|wd}|$|j dkr`|jGx||j|j\|_\|jy||z||%}'Wdn 1sxwY|j{rt|stT}|'stT~|'r||d|jjE|jY7}n||'7}|jt||%7_|!|j ko|$d|!k}(||j dks|(rP|(r|jjd |jdur|jdkrtr|jr|j&|jn|j4rtjt|j&|jn |j||j|j&|jj })|)r#te|j#tTjj#js#|j#|Z|jjEd7_E||$d|#|!|j_=|jG||j|j\|_\||||||n |jG||j|j\|_\|j\jse|j\jrgnq|$dkrtrd#|jjEd$| d%d |j\_|jG||j|j\|_\||||||tj|jjvrt|rttntrd&|j\jrnq|jjrt3|d'rt|d't:d(|jr|jjdurt|rtd)n|jtjkrtntrt||jX|7_X|jX|jjE}*td*|||jj| d+}+||jj|+d,<|*|+d-<d|_|j|+||+||},|jd|,d.}-|jjrk|jjdurk|jjdkrk|-D]}.t?j@|.|jjsit:d/|.d0t|.qO|jG||j|j\|_\||jdur||jt|jjE|*|+S)1Nz)Currently training with a batch size of: r rzYargs.max_steps must be set to a positive value if dataloader does not have a length, was zCurrently --debug underflow_overflow is not supported under DP. Please use DDP (torchrun or torch.distributed.launch (deprecated)).F)r)gradient_checkpointing_kwargsTrz***** Running training ***** Num examples = rz Num Epochs = z( Instantaneous batch size per device = zA Training with DataParallel so batch size has been adjusted to: zE Total train batch size (w. parallel, distributed & accumulation) = z Gradient Accumulation steps = z Total optimization steps = z# Number of trainable parameters = )trainable_onlyzE Continuing training from checkpoint, will skip to saved global_stepz! Continuing training from epoch z' Continuing training from global step z Will skip the first z epochs then the first z batches in the first epoch.rr set_epochmain_input_namer zTried to track the number of tokens seen, however the current model is not configured properly to know what item is the input. To fix this, add a `main_input_name` attribute to the model class you are using.zXThere seems to be not a single sample in your epoch_iterator, stopping training at step zI! This is expected if you're using an IterableDataset and set num_steps (z.) higher than the number of available samples.zYou enabled PyTorch/XLA debug metrics but you don't have a TPU configured. Check your training configuration if this is unexpected._pastzU Training completed. Do not forget to share your model on huggingface.co/models =) load_best_model_at_endr@) num_samples num_stepsr total_flos train_loss use_mtimerDeleting older checkpoint [] due to args.save_total_limit)rv free_memoryrrrr rrrryrerrXrmaxr rrinclude_tokens_per_secondrmathceilnum_train_epochssysmaxsizerrUNDERFLOW_OVERFLOWrrrrurrrrrr!rrr5is_hyper_param_search logging_steps eval_steps save_stepsgradient_checkpointingrgradient_checkpointing_enablerrr@rrrwrr"r_load_optimizer_and_schedulerrrr?epochrrrIisfilerSrMrrJignore_data_skiprtrain_dataloaderrr trial_namerrLr"r#r trial_paramsrrrtensorr<r_total_loss_scalar_globalstep_last_logged zero_gradon_train_beginrranger>rrrTaccelerate_versionrgrrrhr-rCr past_indexron_epoch_begin_load_rng_staterr include_num_input_tokens_seenrr num_input_tokens_seengatherrrclose on_step_begin accumulate training_steplogging_nan_inf_filterryisnanisinfrfloatfloating_point_opsgradient_state_set_sync_gradients max_grad_normrclip_master_gradsrrclip_grad_norm_r~ master_paramsrroptimizer_step_was_skippedrReduceLROnPlateau on_step_end_maybe_log_save_evaluateon_substep_endshould_epoch_stopshould_training_stop on_epoch_endTPU_METRICS_DEBUGr} master_printmetmetrics_reportdelattrrbest_model_checkpoint rendezvousrr_rdistrr _load_best_modelitemr] store_flosrrrrlog_get_output_dir_sorted_checkpointsrsave_total_limitsamefileshutilrmtreer:_finish_current_pushrr4rR)/rrjrrrrrtotal_train_batch_sizelen_dataloadernum_train_tokensnum_update_steps_per_epochr rrnum_train_samplesdebug_overflowdelay_optimizer_creationrruse_accelerator_preparerepochs_trainedsteps_trained_in_current_epochsteps_trained_progress_barr#tr_lossrro sampler_kindsis_random_sampler_total_batched_samplesepoch_iteratorsteps_in_epoch rng_to_sync steps_skippedrinputsr tr_loss_step)is_last_step_and_steps_less_than_grad_accoptimizer_was_runrr.run_dircheckpoints_sortedr2rrrrs                                                                                     &   zTrainer._inner_training_loopcCs|jdurW|durW|jtjkr|j}n*|jtjkr%ddl}|j}n|jtj kr/|j }n|jtj kr=ddl }|j j }|jdurG||nd|}tj|jj|}|S|jj}|S)Nrzrun-)rrLrnumberr r<r@ get_context get_trial_idr"idr%rrunrrrIrSrr)rrrun_idrCrrun_namer=rrrrs     zTrainer._get_output_dircs|dur|j}tjt}tjt}tjt}tjt}tjt}tjt }tjt } tj oKt fddt D} | rY|jsYtddt dd|||| ||fDsq| sqtdtddtj|rt|} | j} | dur| tkrtd | d td tj|stj|s| r$trtjtjd rtjtd d ddSt|jdr|jjdurtdtj|dd} d | d<|j | dd}~ dS|jrt!|j"j#j$|j"|dS|jj%rtj|rt&jj'|dd} ntj|dd} | | d }~ |(|dSt)r^t*|t+r^t|drWt|drWtj,rL|j-|j.dddStdtddStddSt/|t|jj%d}tst|(|dSdS)Nc3s.|]}tjtj|rt|vVqdSr)rrIisdirrSFSDP_MODEL_NAMEr folder_namerrr s z0Trainer._load_from_checkpoint..zCheckpoint found at z* is only supported when using PyTorch FSDPcss|] }tj|VqdSr)rrIr)rfrrrrKs  z!Can't find a valid checkpoint at zLoading model from rZz9You are resuming training from a checkpoint trained with z- of Transformers but your current version is zJ. This is not recommended and could yield to errors or unwanted behaviors.user_content.ptFrItagrload_optimizerrTzPEnabling FP16 and loading from smp < 1.10 checkpoint together is not suppported.r map_location_smp_is_partialrYractive_adapter load_adapter) is_trainablejThe intermediate checkpoints of PEFT may not be saved correctly, consider using a custom callback to save i in corresponding saving folders. Check some examples here: https://github.com/huggingface/peft/issues/96GCould not load adapter model, make sure to have `peft>=0.3.0` installed)rV prefer_safe)0rrrIrSrdrcrbrhrgrfrerFanylistdirrrrrrrfrom_json_filetransformers_versionrr rur rrrrrloadload_state_dictrrvr  fsdp_pluginsave_safetensors safetensors load_file_issue_warnings_after_loadrrrrexistsrVrUr&)rrr config_fileadapter_weights_fileadapter_safe_weights_file weights_fileweights_index_filesafe_weights_filesafe_weights_index_file is_fsdp_ckptconfigcheckpoint_versionrO load_resultrrJrrs      zTrainer._load_from_checkpointc Cstd|jjd|jjdtj|jjt}tj|jjt }tj|jjt }tj|jjt }t r:|j n|j}|jrJt|j |jjdS|jr\t|jjj|j||jj}dStj|sutj|sutj|sutj|r8d}t rtjtj|jjdrtj|jjtddddS|jjrtj|rtjj|dd }ntj|dd }d|d <|j|dd }dSt rt!|t"rt#|d rt#|drtj|stj|r|$|jj|j%ddl&m'} | gg}n6t(dt dd}n*t(dd}n"|jjrtj|rtjj|dd }ntj|dd }||d}t s4|r6|)|dSdSdStjtj|jjt*r]t+||jjt d }t s[|)|dSdSt(d|ddS)NzLoading best model from z (score: z).TrMFrNrrTrQrSrYrUrVr)_IncompatibleKeysrXrYrZz#Could not locate the best model at zi, if you are running a distributed training on multiple nodes, you should activate `--save_on_each_node`.),rrr r best_metricrrIrSrhrfrcrbrurrrr"rrrvrbrgrr rrrcrdrrer`rarrrrrrVrUtorch.nn.modules.modulersr rfrgr&) rbest_model_pathbest_safe_model_pathbest_adapter_model_pathbest_safe_adapter_model_pathrrrhas_been_loadedrOrsrrrrRs             zTrainer._load_best_modelcCszt|jdkr(|jjdurt|jt|jjkr|jn td|jdt|jdkr;td|jddSdS)Nrz8There were missing keys in the checkpoint model loaded: rZz;There were unexpected keys in the checkpoint model loaded: ) r missing_keysr_keys_to_ignore_on_saverr;rr unexpected_keys)rrrrrrrfs  z"Trainer._issue_warnings_after_loadc Csl|jjrCtr ti}||}||8}t||j j |j d|d<| |d<|j |7_ |j j |_ |||d}|jjrt|jtrli}|jD]\} } |j| |d| d} || qVn|j|d}|||j j |t|jtjjjr|jj} | dsd| } |j|| |jj r|j!|||d|j"#|j|j |j|_dSdS)Nr}rreval_)r ignore_keysmetric_key_prefixr)r.)$r should_logryr}r_nested_gathermeanrrr rJrrrrrshould_evaluaterrrcr$evaluaterrErrrr rmetric_for_best_model startswithrr_save_checkpointron_save) rr0rrrrlogstr_loss_scalarr.eval_dataset_namerdataset_metricsmetric_to_checkrrrr sF        z Trainer._maybe_log_save_evaluatec Cs|durdS|jjdkr-|jj}tj|d|d}tj|s,td|ddSntj|d}tj|sAtddSt |}t |dt j |d t j |d t jr|jjtjkrst jj |d n'z t jj |d Wnty}ztd |d WYd}~nd}~wwtrt|dtr|jjtjkrt jj |ddSz t jj |dWdSty}ztd|d WYd}~dSd}~wwdS)Nr rng_state_.pthz$Didn't find an RNG file for process zr, if you are resuming a training that wasn't launched in a distributed fashion, reproducibility is not guaranteed. rng_state.pthzDidn't find an RNG file, if you are resuming a training that was launched in a distributed fashion, reproducibility is not guaranteed.pythonnumpyrcudazUDidn't manage to set back the RNG states of the GPU because of the following error: zO This won't yield the same results as if the training had not been interrupted.rnpuzUDidn't manage to set back the RNG states of the NPU because of the following error: )rr process_indexrrIrSrrrrr`randomsetstatenp set_state set_rng_stater is_availablerr_rset_rng_state_allrryr}rxr)rr2rrng_filecheckpoint_rng_staterorrrrs\        zTrainer._load_rng_statec Cstd|jj}|jdur|dur||j|d}tj||}tj |r>t t |dkr>t d|d|}n tj|d|}|j|dd|jjs]|||||dur|jjdur|jj}|d sud |}||} |jjrtjntj} |jjdus|jjdus| | |jjr| |j_||j_|jjr|jtj|t|jjr||||krt |||jjr|j!d|d dSdS) NrF)rrz!Checkpoint destination directory zV already exists and is non-empty.Saving will proceed but saved results may be invalid.ztmp-TrGr~r)"rHr rJrrrrrIrSrgrr]rr rKrsave_only_model_save_optimizer_and_scheduler_save_rng_staterrgreater_is_betterrgreaterlessrtrrrLrMr_push_from_checkpointrename_rotate_checkpoints) rrrr.checkpoint_folderr=rstaging_output_dirr metric_valueoperatorrrrr sH           zTrainer._save_checkpointc Csttjtjd}tjr+|jj t j kr#tjj |d<ntjj|d<t r4t|d<trO|jj t j krGtjj |d<ntjj|d<tj|dd|jjdkrit|tj|ddSt|tj|d |jjd dS) N)rrrrrrTrr rrr)rgetstater get_stater get_rng_staterrrrr_rget_rng_state_allryr}rxrrrrrNrIrSr)rr rng_statesrrrrU s$   &zTrainer._save_rng_statecCstr?tdt|jtj|t t j dd}t|j tj|t t|Wdn1s9wYnetrj|jjdd}ttdksXtjjjritj|tj|t dtjjjdn:|jrt|j|n0|jrt|jjj|j|j|t|jjj|j|j|j|n|j j!rt"|jtj|t |jot#|j t$ }|j j!r|jr|rtst j dd}t"|j tj|t Wdn1swYt|dSdSdSdS)Nsaving_optimizer_statesTrecordF)gather_if_shardr)rv3)%ryr}rrNrrOrrIrSrPrcatch_warningsrrQrFrulocal_state_dictr rrdp_rankr r shard_optimizer_staterrsave_checkpointrrrvrbrrrrrrr)rrcaught_warningsopt_state_dictis_deepspeed_custom_schedulerrrrru sX     z%Trainer._save_optimizer_and_schedulerc sdurdS|jr9t|jts7tjdd}|jtt j t Wdn1s.wYt |dStrHtt j tdn)t j t j tpqt j t j tpqt j oqtfddt D}|rQt j t j t rStrtjt j tdd}tjdd}tjt j t dd}Wdn1swYt |t||jjt||jj|j||j|dStrt j t j d rfd d }nfd d }|j|n0|jjd kr|jjnd}|jrt |j!j"j#|j!|j|j$n|jtjt j t|dtjdd}|jtt j t Wdn 1sFwYt |dSdSdS)z3If optimizer and scheduler states exist, load them.NTr_*c3s8|]}tjtj|rtdd|vVqdS)rZrN)rrIrFrSOPTIMIZER_NAME_BINrrHr1rrrK s z8Trainer._load_optimizer_and_scheduler..rrQrMcs"|tjtjtdddS)NTr)rar r`rrIrSrPmodoptr1rr opt_load_hook s"z.opt_load_hookcsJtr|tjtjtddddS|tjtjtdddS)NT)r back_compatr)r rar r`rrIrSrPrr1rrr s "r )%rrrrrrrarr`rrIrSrQrFruglobrPrrrFr\r]ryr}send_cpu_data_to_devicerrrrregister_post_step_hookrrrrvr rbr)rr2rcheckpoint_file_existsoptimizer_statelr_scheduler_staterrRrr1rr sh      z%Trainer._load_optimizer_and_schedulerminimizerr4n_trials directionbackendrrc Ks|durt}t|}t|}|||_|jdur td|dur'|jn||_||_ |dur3t n||_ |j |||fi|} d|_| S)aD Launch an hyperparameter search using `optuna` or `Ray Tune` or `SigOpt`. The optimized quantity is determined by `compute_objective`, which defaults to a function returning the evaluation loss when no metric is provided, the sum of all metrics otherwise. To use this method, you need to have provided a `model_init` when initializing your [`Trainer`]: we need to reinitialize the model at each new run. This is incompatible with the `optimizers` argument, so you need to subclass [`Trainer`] and override the method [`~Trainer.create_optimizer_and_scheduler`] for custom optimizer/scheduler. Args: hp_space (`Callable[["optuna.Trial"], Dict[str, float]]`, *optional*): A function that defines the hyperparameter search space. Will default to [`~trainer_utils.default_hp_space_optuna`] or [`~trainer_utils.default_hp_space_ray`] or [`~trainer_utils.default_hp_space_sigopt`] depending on your backend. compute_objective (`Callable[[Dict[str, float]], float]`, *optional*): A function computing the objective to minimize or maximize from the metrics returned by the `evaluate` method. Will default to [`~trainer_utils.default_compute_objective`]. n_trials (`int`, *optional*, defaults to 100): The number of trial runs to test. direction (`str` or `List[str]`, *optional*, defaults to `"minimize"`): If it's single objective optimization, direction is `str`, can be `"minimize"` or `"maximize"`, you should pick `"minimize"` when optimizing the validation loss, `"maximize"` when optimizing one or several metrics. If it's multi objectives optimization, direction is `List[str]`, can be List of `"minimize"` and `"maximize"`, you should pick `"minimize"` when optimizing the validation loss, `"maximize"` when optimizing one or several metrics. backend (`str` or [`~training_utils.HPSearchBackend`], *optional*): The backend to use for hyperparameter search. Will default to optuna or Ray Tune or SigOpt, depending on which one is installed. If all are installed, will default to optuna. hp_name (`Callable[["optuna.Trial"], str]]`, *optional*): A function that defines the trial/run name. Will default to None. kwargs (`Dict[str, Any]`, *optional*): Additional keyword arguments passed along to `optuna.create_study` or `ray.tune.run`. For more information see: - the documentation of [optuna.create_study](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.create_study.html) - the documentation of [tune.run](https://docs.ray.io/en/latest/tune/api_docs/execution.html#tune-run) - the documentation of [sigopt](https://app.sigopt.com/docs/endpoints/experiments/create) Returns: [`trainer_utils.BestRun` or `List[trainer_utils.BestRun]`]: All the information about the best run or best runs for multi-objective optimization. Experiment summary can be found in `run_summary` attribute for Ray backend. NzXTo use hyperparameter search, you need to pass your model through a model_init function.) r rLrensure_availablerrrdefault_hp_spacerrrSr4rC) rrr4rrrrr backend_objbest_runrrrhyperparameter_search s ;  zTrainer.hyperparameter_searchrcCsr|jjdurt|jjd|d<|jjr|jj|d<i|d|jji}|jj||j |j|j|j ||_ dS)z Log `logs` on the various objects watching training. Subclass and override this method to inject custom behavior. Args: logs (`Dict[str, float]`): The values to log. Nrrr) r rrrrrrJ log_historyrgron_logr)rroutputrrrrB s  z Trainer.logrcst|trt|fdd|DSt|ttfr(t|fdd|DSt|tjrVdjj i}j rNt |sAt |rN| djjjji|jdi|S|S) z| Prepares one `data` before feeding it to the model, be it a tensor or a nested list/dictionary of tensors. csi|] \}}||qSr_prepare_inputrrrrrZ sz*Trainer._prepare_input..c3|]}|VqdSrr)rrrrrrK\ z)Trainer._prepare_input..rrrNr)rrrOr$rhrCrTensorrrris_floating_point is_complexrrvr r+rrrr<)rrrrrrrU s   zTrainer._prepare_inputr9cCsR||}t|dkrtdd|jd|jjdkr'|jdur'|j|d<|S)z Prepare `inputs` before feeding them to the model, converting them to tensors if they are not already and handling potential state. rzThe batch received was empty, your model won't be able to train on it. Double-check that your training dataset contains keys expected by the model: rrZNmems)rrrrSrrrrrr9rrrr]g s    zTrainer._prepare_inputscCs|S)zF A helper wrapper to group together context managers. )autocast_smart_context_managerrrrrcompute_loss_context_managerw sz$Trainer.compute_loss_context_managerrTcCs*|jrtjjj||jd}|St}|S)z A helper wrapper that creates an appropriate context manager for `autocast` while feeding it the desired arguments, depending on the situation. )rTrr)rrrr~r`r contextlib nullcontext)rrT ctx_managerrrrr} s z&Trainer.autocast_smart_context_managercCs|||}trt|||jj}||jj S| | ||}Wdn1s4wY|jj dkrC| }|jrbt||j }|Wdn1s\wYn|j|||jjS)aq Perform a training step on a batch of inputs. Subclass and override to inject custom behavior. Args: model (`nn.Module`): The model to train. inputs (`Dict[str, Union[torch.Tensor, Any]]`): The inputs and targets of the model. The dictionary will be unpacked before being fed to the model. Most models expect the targets under the argument `labels`. Check your model's documentation for all accepted arguments. Return: `torch.Tensor`: The tensor with training loss on this batch. Nr )r@r]rurrre reduce_meandetachr<rr compute_lossrrrr~ scale_lossrbackwardrv)rrr9loss_mbr scaled_lossrrrr s"     zTrainer.training_stepc Cs|jdurd|vr|d}nd}|d i|}|jjdkr%||jj|_|durVt|}tr.-*z.*z -([0-9]+)rcSsg|]}|dqS)r r)rr2rrrrv srr )rrrgrrIgetmtimerematchgroupsrsortedr rrindexrr) rrcheckpoint_prefixrordering_and_checkpoint_pathglob_checkpointsrI regex_matchr>best_model_indexirrrrf s" $zTrainer._sorted_checkpointscCs|jjdus |jjdkrdS|j||d}t||jjkrdS|jj}|jjdur9|jjdkr9|d|jjkr9d}tdt||}|d|}|D]}td|dt j |dd qJdS) Nrrr rrrrT) ignore_errors) rr rrr rrrrr"r#)rrrr>r number_of_checkpoints_to_deletecheckpoints_to_be_deletedr2rrrr s    zTrainer._rotate_checkpointsr^rrc Cs|j||}t}|jjr|jn|j}||d|jdur"dnd||d}|jj |jj }|d|j vrB||j |d7}|j t |||jt|j|d||j tj|jjvritt|j|j|j|j|j |_|j|j |j S)a' Run evaluation and returns metrics. The calling script will be responsible for providing a method to compute metrics, as they are task-dependent (pass it to the init `compute_metrics` argument). You can also subclass and override this method to inject custom behavior. Args: eval_dataset (`Dataset`, *optional*): Pass a dataset if you wish to override `self.eval_dataset`. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed. It must implement the `__len__` method. ignore_keys (`List[str]`, *optional*): A list of keys in the output of your model (if it is a dictionary) that should be ignored when gathering predictions. metric_key_prefix (`str`, *optional*, defaults to `"eval"`): An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named "eval_bleu" if the prefix is "eval" (default) Returns: A dictionary containing the evaluation loss and the potential metrics computed from the predictions. The dictionary also contains the epoch number which comes from the training state. EvaluationNT)rGprediction_loss_onlyrr_jit_compilation_timerr)rrrrrr|prediction_loopevaluation_looprrrr.rr]rrrrrrrr}rrrr on_evaluater rr) rrrreval_dataloaderr eval_looprtotal_batch_sizerrrr s8    zTrainer.evaluaterc Cs|j||}t}|jjr|jn|j}||d||d}|jj|jj }|d|j vr:||j |d7}|j t |||j t|j |d|j|j|j|j|j |_|j|j t|j|j|j dS)a Run prediction and returns predictions and potential metrics. Depending on the dataset and your use case, your test dataset may contain labels. In that case, this method will also return metrics, like in `evaluate()`. Args: test_dataset (`Dataset`): Dataset to run the predictions on. If it is an `datasets.Dataset`, columns not accepted by the `model.forward()` method are automatically removed. Has to implement the method `__len__` ignore_keys (`List[str]`, *optional*): A list of keys in the output of your model (if it is a dictionary) that should be ignored when gathering predictions. metric_key_prefix (`str`, *optional*, defaults to `"test"`): An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named "test_bleu" if the prefix is "test" (default) If your predictions or labels have different sequence length (for instance because you're doing dynamic padding in a token classification task) the predictions will be padded (on the right) to allow for concatenation into one array. The padding index is -100. Returns: *NamedTuple* A namedtuple with the following keys: - predictions (`np.ndarray`): The predictions on `test_dataset`. - label_ids (`np.ndarray`, *optional*): The labels (if the dataset contained some). - metrics (`Dict[str, float]`, *optional*): The potential dictionary of metrics (if the dataset contained labels). Prediction)rGrrrr) predictionsr?r.)rrrrrr|rr rrr.rr]rrrr on_predictr rrrOr&r?) rrrrtest_dataloaderrr#rr$rrrpredict s* $  zTrainer.predictrc Csh|j}|dur |n|j}|jr|jdurt|ddd\}}|j|jd|d}t|jj dkrY||jurY|jr<|j |n|jj |dd}|j rJ||_||jurR||_ |jrY|j |_|jsu|jri|jtj|jd}n |jru|jtj|jd}|jj} td |d t|rtd ||ntd td | |||j_t|dd} |jdkrd|_ d} d} d} d}d}d}d}d}d}t!|D]K\}}t"|}|dur||7}| dur|} |j#||||d\}}}t|jdd}|j$r|%||nd}t&r t'(|dur$|)|*| }| dur|nt+| |dd} |dur2|jj,|ddd}|durS|jj,|ddd}|)|}|durL|nt+||dd}|dur|jj,|ddd}|j-durm|-||}|)|}| dury|nt+| |dd} |dur|)|}| dur|nt+| |dd} |j.||j/|j0|_0|j1dur|d|j1dkr| durt2| }|dur|nt3j4||fdd}| durt2| }|dur|nt+||dd}|durt2|}|dur|nt+||dd}| durt2| }|dur |nt+||dd}d\} } }} q|jj5|_)|jr,t6|dr,t7|d| durEt2| }|dur<|nt3j4||fdd}| dur\t2| }|durU|nt+||dd}|durst2|}|durl|nt+||dd}| durt2| }|dur|nt+||dd}t| rt| }n t8| t9rt| dddkr| j}n t|r||}n|}|dkr|dkr|}|j:dur|dur|dur|j$r|:t;|||d}n |:t;||d}ni}t<|}|dur|=>||d<t6|dr |j?||d<t@|AD]}|B|ds*|C|||d|<qtD||||d S)! Prediction/evaluation loop, shared by `Trainer.evaluate()` and `Trainer.predict()`. Works both with or without labels. NrTr inferenceFrgrevaluation_moderrr***** Running  *****rz Num examples: Unknown Batch size = rFrrr  padding_indexr )dim pad_index)axisNNNNrr r&r?r9r&r?_lossrrr3r&r?r.r)Errrrr!rrrrv_modelsrw prepare_modelrrrrr<rfloat16rrrrrrrXr r^rr"rrrr r=prediction_stepinclude_inputs_for_metricsrryr}rgather_functionrepeatrBpad_across_processesron_prediction_stepr reval_accumulation_stepsrDr concatenategather_for_metricsrrrr7rrKrTrrrrCrDrr!rJ) rrrGrrrrr3rrjr losses_host preds_host labels_host inputs_host all_losses all_preds all_labels all_inputsobserved_num_examplesrr9observed_batch_sizerlogitsrr inputs_decodelossesrr.rrrrr  s                                         zTrainer.evaluation_loopcCs||durdStr|durd}t||}|Strt|}|S|jjdur,|jjjdks8|jjdur<|jjdkr. return_lossNTrpkeys_to_ignore_at_inferencec3rrr`rrarrrK8 rr rc3&|]\}}|dgvr|VqdSrNrrrrrrKD $c3 |] \}}|vr|VqdSrrrrrrrKN )rc3rdrerrrrrrKY rfc3rgrrrrrrrKa rhr)rrallrrjr]rrrrprCrhrrarurrrcr$rrrrrrrrrr)rrr9rr has_labelsrbloss_without_labelsr raw_outputsr logits_mbrrUrr)rr9rrB sr*                 *  zTrainer.prediction_stepcCst|jdr |j|SdS)a For models that inherit from [`PreTrainedModel`], uses that method to compute the number of floating point operations for every backward + forward pass. If using another model, either implement such a method in the model or subclass and override this method. Args: inputs (`Dict[str, Union[torch.Tensor, Any]]`): The inputs and targets of the model. Returns: `int`: The number of floating-point operations. rr)rrrrrrrrq s zTrainer.floating_point_opscCs^|sdS|jjdurt|jjj}n|jj}t||jj|jj dd}|j |_d|_ dS)zE Initializes a git repo in `self.args.hub_model_id`. NT)tokenprivater) rrrrrabsoluterr hub_tokenhub_private_reporepo_idpush_in_progress)r repo_namerepo_urlrrrr s  zTrainer.init_hf_repolanguagelicensetagsr^finetuned_fromtasks dataset_tags dataset_argsc  Cs|sdStj|jjd} d} tj| r$t| j d} | dk} t j |||||||||| d } | }t| d }||Wdn1sLwY| r_t|j|jjdSdS)a Creates a draft of a model card using the information available to the `Trainer`. Args: language (`str`, *optional*): The language of the model (if applicable) license (`str`, *optional*): The license of the model. Will default to the license of the pretrained model used, if the original model given to the `Trainer` comes from a repo on the Hub. tags (`str` or `List[str]`, *optional*): Some tags to be included in the metadata of the model card. model_name (`str`, *optional*): The name of the model. finetuned_from (`str`, *optional*): The name of the model used to fine-tune this one (if applicable). Will default to the name of the repo of the original model given to the `Trainer` (if it comes from the Hub). tasks (`str` or `List[str]`, *optional*): One or several task identifiers, to be included in the metadata of the model card. dataset_tags (`str` or `List[str]`, *optional*): One or several dataset tags, to be included in the metadata of the model card. dataset (`str` or `List[str]`, *optional*): One or several dataset identifiers, to be included in the metadata of the model card. dataset_args (`str` or `List[str]`, *optional*): One or several dataset arguments, to be included in the metadata of the model card. Nz README.mdF library_namepeft) rwrxryr^rzr{r|rFr}w)rrrIrSrrrgrr`rrr$ from_trainer to_model_cardopenwriter'rcreate_or_update_model_card)rrwrxryr^rzr{r|rFr}model_card_filepathis_peft_libraryr~training_summary model_cardrLrrrcreate_model_card s4%   zTrainer.create_model_cardc Cs|r |jjtjkr dS|jjs|jdur|jsdS|jj}t t t g}t r1| tttg|D]}tjtj||rPttj||tj||q3|jdur\|j|t|jtj|t|jjtjkrwd|jj }n dt!|jj"}t#|j$|||jj%ddt&dgd}|g}|jjtj'tj(fvr|jjtj'krdnt)|j*}t#|j$|||d|jj%dd } |+| |jdus|jrt,||_dS|jj- |dS) NzTraining in progress, step zTraining in progress, epoch Trr rs folder_pathrrn run_as_futureignore_patternszlast-checkpointz , checkpoint)rsr path_in_reporrnr).rr hub_strategyrMENDhub_always_pushrtis_donerrdrhrfrrextendrarcrbrrIrrSr"r3rrrrNr save_strategyrNSTEPSr rJrrrrrqrH CHECKPOINTALL_CHECKPOINTSrrrgrijobs) rrrmodeling_files modeling_filermodel_push_job push_jobsrcheckpoint_pushrrrr sT       zTrainer._push_from_checkpointcCsBt|dsdS|jdur|jstd|jdSdSdS)Nrtz\Waiting for the current checkpoint push to be finished, this might take a couple of minutes.)rrtrrrwait_until_donerrrrr$s  zTrainer._finish_current_pushEnd of trainingrblockingc Ks|dd}|dur%|jjr%|jjdurt|jjj}n |jjdd}|jdur.||j dd| s:dS|j d d|i|| t |j|jj||jj| dtdgd S) u Upload `self.model` and `self.tokenizer` to the 🤗 model hub on the repo `self.args.hub_model_id`. Parameters: commit_message (`str`, *optional*, defaults to `"End of training"`): Message to commit while pushing. blocking (`bool`, *optional*, defaults to `True`): Whether the function should return only when the `git push` has finished. kwargs (`Dict[str, Any]`, *optional*): Additional keyword arguments passed along to [`~Trainer.create_model_card`]. Returns: The URL of the repository where the model was pushed if `blocking=False`, or a `Future` object tracking the progress of the commit if `blocking=True`. r^N/rTrGrr rr)r!rrrrrrrrrKrrr$rrqrH)rrrrr^rrrrs(     zTrainer.push_to_hubc#Cs|j}t|s td|dur|n|j}|jr%|jdur%t|ddd\}}|j|jd|d}t |j j dkra||jura|jrD|j |n|j j |dd}|jrR||_||jurZ||_|jra|j|_|js}|jrq|jtj|jd }n |jr}|jtj|jd }|j} ||} td |d td | td | d} d} d} d}td|j}t|| | d}|sd}t|drt |j!t"r|j!j}t|| |d}t|| |d}t|| |d}|#|j$dkrd|_%||j&_'t(|D]\}}|j)||||d\}}}t*|jdd}|j+r|,||nd}|dur-|-| }| dur$|ntj.| |fdd} |dur@| dur9|nt/| |dd} |durS| durL|nt/| |dd} |durf|dur_|nt/||dd}|j&0||j1|j2|_2|j3dur|d|j3dkr|4|5| d|s|4|5| d|4|5| d|4|5|dd\} } } }q|j$rt|drt6|d|4|5| d|s|4|5| d|4|5| d|4|5|d|7}|s|7nd}|s|7nd}|s|7nd} |j8dur-|dur-|dur-|j+r#|8t9||| d}!n |8t9||d}!ni}!t:|!}!|durC|;<|!|d<t=|!>D]}"|"?|d s`|!@|"|!|d |"<qItA|||!| d!S)"r*z+dataloader must implement a working __len__NrTr+Fr-r.r0r1r2rr3r )make_multiple_ofrorrr )r7r4r5 eval_losses eval_predseval_label_idseval_inputs_idsr:rr;r<r=r3r>)BrrXrrrrr!rrrrvr?rwr@rrrrr<rrArrrrjr rrrrr6rrror:r^rrrr"r rBrrCrrEcatrBrGr rrH add_arrays_gather_and_numpifyrfinalizerrKrTrrrCrDrr!rJ)#rrrGrrrrr3rrjr rKrLrMrNreval_losses_gathererrpreds_gathererlabels_gathererinputs_gathererrr9rrUrrrVrW eval_losspredsr? inputs_idsr.rrrrrIs                   zTrainer.prediction_loopcCsX|durdStrt||}t |Strt|}t |S|jjtjkr(t|}t |S)rXN) ryrErurrrr_rr<rDr^rrrrs zTrainer._gather_and_numpifycCsD|sdSddg}tjtj|jjdr8ttj|jjdd }|}Wdn1s2wYnd}|}|D]}||vrU| drN||7}q>|d|7}q>||krttj|jjdd}t d || |Wdn1s~wY|j dtd |js|jd |jdSdS) z8Add SageMaker Checkpointing patterns to .gitignore file.Nz*.sagemaker-uploadingz*.sagemaker-uploadedz .gitignorerrH rz"Writing .gitignore file. Content: g?z'Add *.sagemaker patterns to .gitignore.)rrrIrgrSrepo local_dirrreadendswithrrrgit_addrsleep is_repo_clean git_commitgit_push)rpatternsrLcurrent_contentcontentpatternrrr_add_sm_patterns_to_gitignores6        z%Trainer._add_sm_patterns_to_gitignorecCs,d|jji}d|d<tdi|}t|jj|jj|jj|d|_|jj|_ t |jj dddu|_ t |jj dddu|_ |j rf|jj j}|jjd|j|_tdrf|jjd |j|_|jrf|jjrftd |j rt |jd ddurd d lm}|jj j}||jj|_|jj|_|j|jdSdSdS)NrFsync_with_dataloader)dispatch_batches split_batchesr+gradient_accumulation_pluginr+rblimit_all_gathersractivation_checkpointingzThe activation_checkpointing in FSDP config and the gradient_checkpointing in training arg can't be set to True simultaneously. Please use FSDP's activation_checkpointing logic when using FSDP.r)rrr)rrerrrrr+rvrJrDrr rrrbrrrrlrrrr(rrrpdeepspeed_configr*)rgrad_acc_kwargsrrbr ds_pluginrrrrsD      z*Trainer.create_accelerator_and_postprocess) NNNNNNNNNrNr)F)TN)NNN)NNNNN)NNrrNN)T)NF)FN)r]N)NNr^)Nr) NNNNNNNNN)rT)wr __module__ __qualname____doc__trainer_pt_utilsrrrrrr r%rModuler`r rrrrr.rrKrr3r rr OptimizerrLambdaLRrr*r2r4rr9r:rrEr\r`rrSamplerrfrryrrrrrrr staticmethodrrrr rr-rrEr?rrprryrrr@rrrrrfr rrrrrrLrIrrrr]rrrrrrrKrrrrHrrrrOr)rJr rrBrrrrr$rrrrrrrrrrsS     +   "#! 0.1  * z  q M lV 0 4< ,T O&6, '% 3"     H  B   W  i  A98    'r)rrr3rrimportlib.metadatarr@rrrr r"rr=rrcollections.abcrpathlibrtypingrrrrrr r r integrationsr rhuggingface_hub.utilsrrrrrtorch.distributed distributedrhuggingface_hubrrr packagingrrtorch.utils.datarrrrrHrconfiguration_utilsrdata.data_collatorrrr debug_utilsrrrrr integrations.deepspeedr!r"r# modelcardr$modeling_utilsr%r&r'models.auto.modeling_autor(r)rr*r+ pytorch_utilsr,r-tokenization_utils_baser.trainer_callbackr/r0r1r2r3r4r5rr6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrG trainer_utilsrHrIrJrKrLrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r] training_argsr^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrrsrtrurvrwrxryrzr{utils.quantization_configr|rrutils.notebookr}rr~rUtorch_xla.core.xla_modelcore xla_modelr}torch_xla.debug.metricsrr.r!smdistributed.modelparallel.torch modelparallelr smdistributed.modelparallel SMP_VERSIONrTr rrrrsafetensors.torchrdrr acceleraterrrr'rrrrrr DATA_SAMPLERSaccelerate.data_loaderrrr5 get_loggerrrrrMrPrrQ SCALER_NAMErGrrrrrs  (        $P `t