o h`+ @s"dZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZddlmZmZmZmZmZmZddlZddlmZddlmZmZmZm Z m!Z!e!"e#Z$e rjddl%Z%ej&'d duo{e(d d )d kZ*e*rz%ddl+Z+e,e+d re+j-.drdZ*ne(d d )d kre$/ddZ*Wn e0e1fydZ*Ynwej&'ddupej&'dduZ2ere2rzej3dZ4e$5de4dWn,ej3j6yzej3dZ4e$5de4dWnej3j6ydZ2YnwYnwddl7m8Z8m9Z9ddl:m;Z;mm?Z?ddlm@Z@mAZAddZBddZCdd ZDd!d"ZEd#d$ZFd%d&ZGd'd(ZHd)d*ZId+d,ZJd-d.ZKd/d0ZLd1d2ZMd3d4ZNd5d6ZOd7d8ZPd9d:ZQd;d<ZRd=eSd>eTd?eeTd?eeTd?eeTd?ecCr,)Nsigoptr.r)r)r)r*is_sigopt_availablesr0rAcCs8tjddur dStjddurdStjdduS)NazuremlFz azureml.corezazureml.core.runr.r)r)r)r*is_azureml_availablews rCcCs(tdddkr dStjdduS)NDISABLE_MLFLOW_INTEGRATIONFALSETRUEFmlflow)r!r"r#r&r'r(r)r)r)r*is_mlflow_availablesrHcCsdtjdtjdfvS)NdagshubrGr.r)r)r)r*is_dagshub_availablesrJcCr1r2) _has_neptuner)r)r)r*is_neptune_availabler5rLcCr,)N codecarbonr.r)r)r)r*is_codecarbon_availabler0rNcCr,)Nflytekitr.r)r)r)r*is_flytekit_availabler0rPcCr=)NFzflytekitplugins.deck)rPr&r'r(r)r)r)r* is_flyte_deck_standard_availabler?rQcCr,)Ndvcliver.r)r)r)r*is_dvclive_availabler0rScCsltrddl}t||jr|jStrt|tr|Str$t|tr$|Str.t|tr.|St d|j )NrzUnknown type for trial ) r:r9 isinstanceTrialparamsr>dictrAr+ RuntimeError __class__)trialr9r)r)r* hp_paramss    r[n_trials directionreturnc stddl}jjdkr^dfdd }|dd}|dd}t|tr$|nd}|dur,dn|}|jd||d|} | j||||d| sT| j } t t | j | j | jS| j} d d | DSt|D]U} d_ttj} jjtjkrztd tjj| dd tt| }t|D]\}}|d krtj||qj ddt!dddur"}#|_qbdS)Nrcsd}|rt|D]}|trtj||}q d_jjdkrDjj t j kr,t d |tjjtjddj|dnj||dtdddur]}|_jS)NConly support DDP optuna HPO for ParallelMode.DISTRIBUTED currently.rsrcresume_from_checkpointrdrZ objective)r!listdir startswithrpathjoinrfargs world_size parallel_moder DISTRIBUTEDrX_hp_search_setuptorch distributedbroadcast_object_listpickledumpstraingetattrevaluatecompute_objective)rZcheckpoint_dir checkpointsubdirmetricstrainerr)r* _objectives$    z(run_hp_search_optuna.._objectivetimeoutn_jobsr_)r] directions)r\rrcSs"g|] }tt|j|j|jqSr))rstrnumbervaluesrV).0bestr)r)r* s"z(run_hp_search_optuna..r`ra local_rankrcrfr2r))$r9rk process_indexpoprTlist create_studyoptimize_is_multi_objective best_trialrrrvaluerV best_trialsrangerfrsrtrmrrnrXrprqrrloadsbytesritemssetattrrurvrwrx)r~r\r]kwargsr9rrrrstudyrriargs_main_rankrkkeyrr|r)r}r*run_hp_search_optunas>      rc sddlddldtffdd }|jjs&ddlm}td|dd |_| t }d|_ d |vrYd d i|d <|j j dkrDd |d d <d|j j dkrMdnd}td|d|d d d}||j _d|vrwddlm} | dgd|d<d|vrddlm} m} m} m} t|d| | | | fr|j jr|j jtjkrtdjt|dj dj!j"||dt#$fdd}t%drƈj&|_&j!j'|f|(d|d|}|j)d|dd |j j*d!}t+|j,|j-d|j.|}|dur|/||S)"NrrZcsDzddlm}||r|tWn tyYnwd|_j}|rSd|_| }t t | t d}|j||dWdn1sMwYn|j|dt|dddur|}|||_||jddt}|j|d jj|}jj||d WddS1swYdSdS) Nr)NotebookProgressCallbackrf*re)rZT)rfdone)ry)rz)transformers.utils.notebookr pop_callback add_callbackrModuleNotFoundErrorrfruget_checkpoint as_directorynextrglobras_posixrvrwrxupdatetempfileTemporaryDirectory_tune_save_checkpoint Checkpointfrom_directoryreport)rZ local_trainerrrzrycheckpoint_pathr|temp_checkpoint_dir)r;r)r*rs8          "z%run_hp_search_ray.._objectiver )TrainerMemoryTrackerzMemory tracking for your Trainer is currently enabled. Automatically disabling the memory tracker since the memory tracker is not serializable.T)skip_memory_metricsresources_per_trialcpur_gpuz1 CPUz and 1 GPUrzgNo `resources_per_trial` arg was passed into `hyperparameter_search`. Setting it to a default value of z for each trial.progress_reporter) CLIReporterrf)metric_columns scheduler) ASHASchedulerHyperBandForBOHBMedianStoppingRulePopulationBasedTrainingaYou are using {cls} as a scheduler but you haven't enabled evaluation during training. This means your trials will not report intermediate results to Ray Tune, and can thus not be stopped early or used to exploit other trials parameters. If this is what you want, do not use {cls}. If you would like to use {cls}, make sure you pass `do_eval=True` and `evaluation_strategy='steps'` in the Trainer `args`.)cls)rcsbtr*ddl}tj|jd}tj d|}tj |}|t j |j <|j||i|S)a" Wrapper around `tune.with_parameters` to ensure datasets_modules are loaded on each Actor. Without this, an ImportError will be thrown. See https://github.com/huggingface/transformers/issues/11565. Assumes that `_objective`, defined above, is a function. rNz __init__.pydatasets_modules)r datasets.loadr!rirjloadinit_dynamic_modulesr&r'spec_from_file_locationmodule_from_specsysmodulesnameloader exec_module)rkrdatasetsdynamic_modules_pathspecr) trainabler)r* dynamic_modules_import_trainableNs    z;run_hp_search_ray..dynamic_modules_import_trainable __mixins__)r num_samples)metricmodescope)0r; ray.trainrW_memory_trackerr trainer_utilsrr$r%rTensorBoardCallbackmodelrkn_gpuinfoget_n_gpuray.tunerray.tune.schedulersrrrrrTdo_evalevaluation_strategyrNOrXformattype__name__tunewith_parameters functoolswrapshasattrrrunhp_spaceget_best_trial ray_scopertrial_id last_resultrr)r~r\r]rrr _tb_writer resource_msggpus_per_trialrrrrrranalysisrbest_runr))r;rr*run_hp_search_raysr&          rc Ksddl}|jjdkrtjddkr|d|jdd|dd|dd gd |d }t d |j | D]b}|Vd|_ |jjd krj|jjtjkrQtd ||jtjjt|jdd|jddn|jd|jdt|dddur|}|||_ |d|j Wdn1swYq8t| d}t!|j |j"dj#|j$} | Sddlm%} | } |&dd} | dur| '| | (j)d|dd|dd gd |dd}t d |j |j*j+|j,krs| (|j -)} d|_ |jjd kr'|jjtjkrtd || tjjt|jdd|jddn|jd| dt|dddurA|}|||_ d|j dg}| (|j .j)| j |d}t d| j d|j d| (|j /}|j*j+|j,kst| (|j 0/1d}t!|j |j#|j$} | St2|D]Z}d|_ tt|j}|jjtjkrtd tjj|ddt3t4|}t5|6D]\}}|dkrt7|j||q|jddt|dddur|}|||_ qdS)Nrr@z8.0.0 huggingfacezhuggingface-tuneofflinerfr)rrfstrategyr_)rr parametersr|parallel_bandwidthbudgetz6created experiment: https://app.sigopt.com/experiment/zConly support DDP Sigopt HPO for ParallelMode.DISTRIBUTED currently.rarcre) Connectionproxies)rrr|robservation_budgetprojectrr) suggestionrz"[suggestion_id, observation_id]: [, ]r)8r@rkrr&metadataversion set_projectcreate_experimentrr$ridlooprfrlrmrrnrXrorrprqrrrsrtrurvrwrx log_metricr get_best_runsrrr assignmentsrr set_proxies experimentscreateprogressobservation_countr suggestions observationsfetchbest_assignments iterate_pagesrrrrrr)r~r\r]rr@ experimentrr|rrrconnrrrobsrrrkrrr)r)r*run_hp_search_sigoptss      *           rc  s:ddlm}|s tdddld}jjD] }t|tr"d}nq|s+tdgj _ dddd| d d}| d d}| d d} | d d} | d d d} | d d<| d d <| rk| | d <fdd} |sj | || dn|}td|j|| |dtdddS)Nr )r+z8This function needs wandb installed: `pip install wandb`rFTr )run_idrfhyperparameterssweep_idrrentityrz eval/lossgoalcsjrjn}|jj_|jidj}d_jdt |ddt dddurP } |_t |}|vrPtdd|d}ddurod krdjdk}n d krojdk}|swddur|jd<jd<t|d <jS) N)rr_itemsrerfzProvided metric zU not found. This might result in unexpected sweeps charts. The available metrics are Frminimizemaximizer )rinitrstate trial_namerrrfruvarsrvrwrx rewrite_logsr$r%keysr rW)rrr|format_metrics best_scorerr]rr~r r)r*rs6      z'run_hp_search_wandb.._objective)rr"zwandb sweep id - )functioncountrrfr ) integrationsr+ ImportErrorr callback_handler callbacksrT WandbCallbackrrk report_torrsweepr$ragentr) r~r\r]rr+reporting_to_wandbcallbackr!rrr" sweep_configrr)r/r*run_hp_search_wandbs:              !r=cCsg}tr ts |dtr|dtr|dtr%|dtr-|dtr5|dtr=|dtrE|dt rM|d t rU|d |S) Nazure_mlrrIrRrGrr6r rMr-) rCrHappendr4rJrSrLr8r+rNr/)r2r)r)r*$get_available_reporting_integrationss,           r@cCs~i}d}t|}d}t|}|D]*\}}||r&||d||d<q||r6||d||d<q||d|<q|S)Neval_test_eval/ztest/ztrain/)lenrrh)dnew_d eval_prefixeval_prefix_len test_prefixtest_prefix_lenkvr)r)r*r+3s  r+c@s>eZdZdZd ddZd ddZddZd d d Zd d ZdS)rz A [`TrainerCallback`] that sends the logs to [TensorBoard](https://www.tensorflow.org/tensorboard). Args: tb_writer (`SummaryWriter`, *optional*): The writer to use. Will instantiate one if not set. Nc Cst}|s td|r8z ddlm}||_Wn$ty7z ddlm}||_Wn ty4d|_YnwYnwd|_||_dS)NzuTensorBoardCallback requires tensorboard to be installed. Either update your PyTorch version or install tensorboardX.r) SummaryWriter)r8rXtorch.utils.tensorboardrM_SummaryWriterr3r7 tb_writer)selfrPhas_tensorboardrMr)r)r*__init__Ls(        zTensorBoardCallback.__init__cCs*|p|j}|jdur|j|d|_dSdS)N)log_dir) logging_dirrOrP)rQrkrTr)r)r*_init_summary_writercs  z(TensorBoardCallback._init_summary_writerc Ks|jsdSd}|jr|j}|durtj|j|}|jdur$||||jdurR|j d| d|vrT|d}t |drV|j durX|j }|j d|dSdSdSdSdS)Nrkrr model_config) is_world_process_zerois_hyper_param_searchr)r!rirjrUrPrVadd_textto_json_stringrr) rQrkr(controlrrTr)rmodel_config_jsonr)r)r*on_train_beginhs&    z"TensorBoardCallback.on_train_beginc Ks|jsdS|jdur|||jdurJt|}|D]&\}}t|ttfr1|j|||j qt d|dt |d|dq|j dSdS)N)Trainer is attempting to log a value of " " of type for key "zn" as a scalar. This invocation of Tensorboard's writer.add_scalar() is incorrect so we dropped this attribute.)rXrPrVr+rrTintfloat add_scalar global_stepr$r%rflushrQrkr(r\logsrrKrLr)r)r*on_log~s(   zTensorBoardCallback.on_logcKs|jr |jd|_dSdSr2)rPcloserQrkr(r\rr)r)r* on_train_ends  z TensorBoardCallback.on_train_endr2) r __module__ __qualname____doc__rSrVr^rirlr)r)r)r*rCs    rc@sFeZdZdZddZddZdddZdd d Zdd d Zd dZ dS)r6z{ A [`TrainerCallback`] that logs metrics, media, model checkpoints to [Weight and Biases](https://www.wandb.com/). cCst}|s td|rddl}||_d|_tddt dhvr?t dtddt d tdd d |_ dStdd |_ dS) NzFWandbCallback requires wandb to be installed. Run `pip install wandb`.rFWANDB_LOG_MODELrErFzSetting `WANDB_LOG_MODEL` as zn is deprecated and will be removed in version 5 of transformers. Use one of `'end'` or `'checkpoint'` instead.zSetting `WANDB_LOG_MODEL` from z to `end` insteadendfalse)r+rXr _wandb _initializedr!r"r#runionDeprecationWarningr$r _log_modellower)rQ has_wandbr r)r)r*rSs zWandbCallback.__init__c KsT|jdurdSd|_|jrtdi|}t|dr-|jdur-|j}i||}|j}i}|dur@||d<|j |d<n|j dusP|j |j ksP|j |d<|jj dure|jj ddt dd i||jjj|dd t|jd dr|jd |jjd d ddt dd} ts| dvr|jj|| td|jd|jj jdddSdS)u Setup the optional Weights & Biases (*wandb*) integration. One can subclass and override this method to customize the setup if needed. Find more information [here](https://docs.wandb.ai/guides/integrations/huggingface). You can also override the following environment variables: Environment: - **WANDB_LOG_MODEL** (`str`, *optional*, defaults to `"false"`): Whether to log model and checkpoints during training. Can be `"end"`, `"checkpoint"` or `"false"`. If set to `"end"`, the model will be uploaded at the end of training. If set to `"checkpoint"`, the checkpoint will be uploaded every `args.save_steps` . If set to `"false"`, the model will not be uploaded. Use along with [`~transformers.TrainingArguments.load_best_model_at_end`] to upload best model. Setting `WANDB_LOG_MODEL` as `bool` will be deprecated in version 5 of 🤗 Transformers. - **WANDB_WATCH** (`str`, *optional* defaults to `"false"`): Can be `"gradients"`, `"all"`, `"parameters"`, or `"false"`. Set to `"all"` to log gradients and parameters. - **WANDB_PROJECT** (`str`, *optional*, defaults to `"huggingface"`): Set this to a custom string to store results in a different project. - **WANDB_DISABLED** (`bool`, *optional*, defaults to `False`): Whether to disable wandb entirely. Set `WANDB_DISABLED=true` to disable. NTz`Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"rrgroupr WANDB_PROJECTr)allow_val_change define_metrictrain/global_stepr) step_metric step_sync WANDB_WATCHrr)allr gradientsd)loglog_freqtransformers_trainer)coder))rsrtrXr$rto_dictrrr)run_name output_dirrr'r!r"rrvr}rwatchmax logging_steps_label) rQrkr(rr combined_dictrWr) init_args _watch_modelr)r)r*setupsB           zWandbCallback.setupNcKsR|jdurdS|j}|r|jd|_d|_|js'|j|||fi|dSdSNF)rsrYfinishrtrr)rQrkr(r\rr hp_searchr)r)r*r^s  zWandbCallback.on_train_beginc Kst|jdurdS|jdvr|jr|jrddlm}||||d}t} || |j s9ddt |jj  Dn d|j |jd|ji} td |jdusU|j|jkr]d |jjjnd |jjj} |jj| d | d } t| d D]%} | r| j| jdd}|| Wdn1swYqu|jj| WddS1swYdSdSdSdS)N)rqrzr Trainerrkr tokenizercS,i|]\}}t|tjr|ds||qS_rTnumbersNumberrhrrKrLr)r)r*  z.WandbCallback.on_train_end..rCztrain/total_flosszLogging model artifacts. ...zmodel-rrrrrwb)r)rsrwrtrXr~rrr save_modelload_best_model_at_endrWsummaryrmetric_for_best_model best_metric total_flosr$rrrrr rArtifactrris_filenew_filewrite read_bytes log_artifact)rQrkr(r\rrrr fake_trainertemp_dirr model_nameartifactffar)r)r*rls<     " zWandbCallback.on_train_endcKsR|jdurdS|js|||||jr't|}|ji|d|jidSdS)Nr~)rsrtrrXr+rre)rQrkr(r\rrhrr)r)r*ri#s zWandbCallback.on_logc Ks|jdkrg|jri|jrkddt|jjD}d|j}tj |j |}t d|d|jdus:|j|j krBd|jjjnd|jjj}|jj|d|d} | ||jj| d|jgd dSdSdSdS) NrzcSrrrrr)r)r*r.rz)WandbCallback.on_save.. checkpoint- Logging checkpoint artifacts in z. ...rr)aliases)rwrtrXrWrsrrrer!rirjrr$rrrr rradd_dirr) rQrkr(r\rcheckpoint_metadatackpt_dir artifact_pathcheckpoint_namerr)r)r*on_save,s   zWandbCallback.on_saver2NN rrmrnrorSrr^rlrirr)r)r)r*r6s D " r6c@s<eZdZdZddZddZd ddZdd d Zd d ZdS) CometCallbackz^ A [`TrainerCallback`] that sends the logs to [Comet ML](https://www.comet.ml/site/). cCststdd|_d|_dS)NzLCometCallback requires comet-ml to be installed. Run `pip install comet-ml`.F)r3rXrt _log_assetsrQr)r)r*rSFs zCometCallback.__init__cCsd|_tdd}|dvrd|_|jrtdd}d}dtd d i}|dkr?tjdi|}|d d t d n|dkr^tdd|d<tj di|}|d d t d|dur|j |d d|j |dd dt|dr|j |jdd ddSdSdSdS)a Setup the optional Comet.ml integration. Environment: - **COMET_MODE** (`str`, *optional*, defaults to `ONLINE`): Whether to create an online, offline experiment or disable Comet logging. Can be `OFFLINE`, `ONLINE`, or `DISABLED`. - **COMET_PROJECT_NAME** (`str`, *optional*): Comet project name for experiments. - **COMET_OFFLINE_DIRECTORY** (`str`, *optional*): Folder to use for saving offline experiments when `COMET_MODE` is `OFFLINE`. - **COMET_LOG_ASSETS** (`str`, *optional*, defaults to `TRUE`): Whether or not to log training assets (tf event logs, checkpoints, etc), to Comet. Can be `TRUE`, or `FALSE`. For a number of configurable items in the environment, see [here](https://www.comet.ml/docs/python-sdk/advanced/#comet-configuration-variables). TCOMET_LOG_ASSETSrE>1rFrONLINEN project_nameCOMET_PROJECT_NAMErz Created from transformersz)Automatic Comet.ml online logging enabledOFFLINECOMET_OFFLINE_DIRECTORYz./offline_directoryzLAutomatic Comet.ml offline logging enabled; use `comet upload` when finished) frameworkzargs/)prefixrrzconfig/r))rtr!r"r#rrXr Experiment log_otherr$rOfflineExperiment_set_model_graph_log_parametersrr)rQrkr(r log_assets comet_moderexperiment_kwargsr)r)r*rLs2      zCometCallback.setupNcK|js ||||dSdSr2rtrrQrkr(r\rrr)r)r*r^vzCometCallback.on_train_begincKsN|js |||||jr#tj}|dur%|j||j|jdddSdSdS)Nr)stepepochr) rtrrXrrget_global_experiment _log_metricsrer)rQrkr(r\rrhrrr)r)r*rizs zCometCallback.on_logcKs`|jr*|jr,tj}|dur.|jdur$td|j|j dd|j d| dSdSdSdS)NTz(Logging checkpoints. This may take time.) recursive log_file_namer) rtrXrrrrr$rlog_asset_folderrrerq)rQrkr(r\rrr)r)r*rls      zCometCallback.on_train_endr2r) rrmrnrorSrr^rirlr)r)r)r*rAs *  rc@s,eZdZdZd ddZddZd ddZdS) AzureMLCallbackzh A [`TrainerCallback`] that sends the logs to [AzureML](https://pypi.org/project/azureml-sdk/). NcCststd||_dS)NzPAzureMLCallback requires azureml to be installed. Run `pip install azureml-sdk`.)rCrX azureml_run)rQrr)r)r*rSs zAzureMLCallback.__init__cKs2ddlm}|jdur|jr||_dSdSdS)NrRun)azureml.core.runrrrX get_context)rQrkr(r\rrr)r)r* on_init_ends zAzureMLCallback.on_init_endcKsJ|jr|jr!|D]\}}t|ttfr|jj|||dq dSdSdS)N) description)rrXrrTrbrcrrgr)r)r*ris zAzureMLCallback.on_logr2)rrmrnrorSrrir)r)r)r*rs  rc@sLeZdZdZddZddZdddZdd d Zd d Zd dZ ddZ dS)MLflowCallbackz A [`TrainerCallback`] that sends the logs to [MLflow](https://www.mlflow.org/). Can be disabled by setting environment variable `DISABLE_MLFLOW_INTEGRATION = TRUE`. cCsJtstdddl}|jjj|_|jjj|_d|_ d|_ d|_ ||_ dS)NzIMLflowCallback requires mlflow to be installed. Run `pip install mlflow`.rF) rHrXrGutils validationMAX_PARAM_VAL_LENGTH_MAX_PARAM_VAL_LENGTHMAX_PARAMS_TAGS_PER_BATCH_MAX_PARAMS_TAGS_PER_BATCHrt _auto_end_run_log_artifacts_ml_flow)rQrGr)r)r*rSs   zMLflowCallback.__init__c Cstddtv|_tddtv|_tdd|_tddtv|_tdd|_t d|jd |j d |jd |j|j r|j dusT|jsT|jrx|jr^|j |j|j j|j |jd t d |j jjd|_|}t|dr|jdur|j}i||}|jrt|n|}t|D]\}}tt||jkrt d|d|d||=qt|}tdt||jD]} |j t!|| | |jqtdd} | rt"#| } |j $| d|_%dS)at Setup the optional MLflow integration. Environment: - **HF_MLFLOW_LOG_ARTIFACTS** (`str`, *optional*): Whether to use MLflow `.log_artifact()` facility to log artifacts. This only makes sense if logging to a remote server, e.g. s3 or GCS. If set to `True` or *1*, will copy each saved checkpoint on each save in [`TrainingArguments`]'s `output_dir` to the local or remote artifact storage. Using it without a remote storage will just copy the files to your artifact location. - **MLFLOW_EXPERIMENT_NAME** (`str`, *optional*, defaults to `None`): Whether to use an MLflow experiment_name under which to launch the run. Default to `None` which will point to the `Default` experiment in MLflow. Otherwise, it is a case sensitive name of the experiment to be activated. If an experiment with this name does not exist, a new experiment with this name is created. - **MLFLOW_TAGS** (`str`, *optional*): A string dump of a dictionary of key/value pair to be added to the MLflow run as tags. Example: `os.environ['MLFLOW_TAGS']='{"release.candidate": "RC1", "release.version": "2.2.0"}'`. - **MLFLOW_NESTED_RUN** (`str`, *optional*): Whether to use MLflow nested runs. If set to `True` or *1*, will create a nested run inside the current run. - **MLFLOW_RUN_ID** (`str`, *optional*): Allow to reattach to an existing run which can be usefull when resuming training from a checkpoint. When `MLFLOW_RUN_ID` environment variable is set, `start_run` attempts to resume a run with the specified run ID and other parameters are ignored. - **MLFLOW_FLATTEN_PARAMS** (`str`, *optional*, defaults to `False`): Whether to flatten the parameters dictionary before logging. HF_MLFLOW_LOG_ARTIFACTSrEMLFLOW_NESTED_RUNMLFLOW_EXPERIMENT_NAMENMLFLOW_FLATTEN_PARAMS MLFLOW_RUN_IDzMLflow experiment_name=z , run_name=z , nested=z, tags=)rnestedzMLflow run started with run_id=Trr_z " for key "z" as a parameter. MLflow's log_param() only accepts values no longer than 250 characters so we dropped this attribute. You can use `MLFLOW_FLATTEN_PARAMS` environment variable to flatten the parameters and avoid this message.r MLFLOW_TAGS)&r!r"r#rr _nested_run_experiment_name_flatten_params_run_idr$debugrrXr active_runset_experiment start_runrrrrrrr rrrDrrr%rr log_paramsrWjsonrset_tagsrt) rQrkr(rrrWrrcombined_dict_itemsr mlflow_tagsr)r)r*rsJ        zMLflowCallback.setupNcKrr2rrr)r)r*r^rzMLflowCallback.on_train_beginc Ks|js |||||jr@i}|D]!\}} t| ttfr#| ||<qtd| dt | d|dq|j j ||j ddSdS)Nr_r`razc" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.)r|r) rtrrXrrTrbrcr$r%rr log_metricsre) rQrkr(r\rhrrr|rKrLr)r)r*ris zMLflowCallback.on_logcKs:|jr|jr|jr|jr|jdSdSdSdSdSr2)rtrXrrrend_runrkr)r)r*rls zMLflowCallback.on_train_endcKsp|jr2|jr4|jr6d|j}tj|j|}t d|d|j j j |d|i|j j ddSdSdSdS)Nrr. This may take time. model_path) artifacts python_model)rtrXrrer!rirjrr$rrpyfunc log_model PythonModelrQrkr(r\rrrr)r)r*rs   zMLflowCallback.on_savecCs@|jrtt|jddr|jdur|jdSdSdSdS)Nr)rcallablervrrr rr)r)r*__del__!szMLflowCallback.__del__r2) rrmrnrorSrr^rirlrrr)r)r)r*rs F  rcs4eZdZdZfddZfddZddZZS)DagsHubCallbackzh A [`TrainerCallback`] that logs to [DagsHub](https://dagshub.com/). Extends [`MLflowCallback`] cs.tts tdddlm}||_dS)NzLDagsHubCallback requires dagshub to be installed. Run `pip install dagshub`.r)Repo)superrSrJr3dagshub.uploadr)rQrrYr)r*rS1s   zDagsHubCallback.__init__cstddtv|_tdpd|_td|_|j|jtj d|jtj ddd td p6dd |_ t d |_ |jd urHt dtj|i|d S)z Setup the DagsHub's Logging integration. Environment: - **HF_DAGSHUB_LOG_ARTIFACTS** (`str`, *optional*): Whether to save the data and model artifacts for the experiment. Default to `False`. HF_DAGSHUB_LOG_ARTIFACTSrEHF_DAGSHUB_MODEL_NAMEmainMLFLOW_TRACKING_URI.rBRANCH)ownerrbranchrNzpDagsHubCallback requires the `MLFLOW_TRACKING_URI` environment variable to be set. Did you run `dagshub.init()`?)r!r"r#r log_artifactsrremotersplitsepreporrirXrr)rQrkrrr)r*r:s     zDagsHubCallback.setupcKsP|jr&t|ddrt|jjtj|j d|j t |j |j dSdS)Ntrain_dataloaderz dataset.pt)r%rvrpsaver*datasetr!rirjrr) directoryrrrkr)r)r*rlUs  zDagsHubCallback.on_train_end)rrmrnrorSrrl __classcell__r)r)rr*r,s   rcseZdZfddZZS)NeptuneMissingConfigurationcstddS)NaA ------ Unsupported ---- We were not able to create new runs. You provided a custom Neptune run to `NeptuneCallback` with the `run` argument. For the integration to work fully, provide your `api_token` and `project` by saving them as environment variables or passing them to the callback. )rrSrrr)r*rS^sz$NeptuneMissingConfiguration.__init__)rrmrnrSr.r)r)rr*r/]sr/c@s<eZdZdZdZdZdZdZdZdhZ dddd dd dd d e e d e e de e de de de e f ddZ ddZddZddZddZddZeddZed d!Zd"d#Zd$d%Zd&d'Zd(d)Zd*e d+e fd,d-Zd.d/Zd?d0d1Zd2d3Zd4d5Zd6d7Zd?d8d9Z e!d:d;Z"d?dZ%dS)@NeptuneCallbackaTrainerCallback that sends the logs to [Neptune](https://app.neptune.ai). Args: api_token (`str`, *optional*): Neptune API token obtained upon registration. You can leave this argument out if you have saved your token to the `NEPTUNE_API_TOKEN` environment variable (strongly recommended). See full setup instructions in the [docs](https://docs.neptune.ai/setup/installation). project (`str`, *optional*): Name of an existing Neptune project, in the form "workspace-name/project-name". You can find and copy the name in Neptune from the project settings -> Properties. If None (default), the value of the `NEPTUNE_PROJECT` environment variable is used. name (`str`, *optional*): Custom name for the run. base_namespace (`str`, optional, defaults to "finetuning"): In the Neptune run, the root namespace that will contain all of the metadata logged by the callback. log_parameters (`bool`, *optional*, defaults to `True`): If True, logs all Trainer arguments and model parameters provided by the Trainer. log_checkpoints (`str`, *optional*): If "same", uploads checkpoints whenever they are saved by the Trainer. If "last", uploads only the most recently saved checkpoint. If "best", uploads the best checkpoint (among the ones saved by the Trainer). If `None`, does not upload checkpoints. run (`Run`, *optional*): Pass a Neptune run object if you want to continue logging to an existing run. Read more about resuming runs in the [docs](https://docs.neptune.ai/logging/to_existing_object). **neptune_run_kwargs (*optional*): Additional keyword arguments to be passed directly to the [`neptune.init_run()`](https://docs.neptune.ai/api/neptune#init_run) function when a new run is created. For instructions and examples, see the [Transformers integration guide](https://docs.neptune.ai/integrations/transformers) in the Neptune documentation. z%source_code/integrations/transformersmodel_parametersrZ trial_paramstrainer_parametersz train/epochN finetuningT) api_tokenrrbase_namespacerlog_parameterslog_checkpointsr5rrr6r7r8c Ks^tstdzddlm} ddlm} Wnty*ddlm} ddlm} Ynw| d|t t df| d|t t df| d|t t df| d|t | d || t df| d |t | d |t t df||_ ||_ ||_||_d|_d |_d|_d |_|||d ||_d|_|jdu|_d|_|jdvrd|j|_d|_dSd|_d |_dS)NzwNeptuneCallback requires the Neptune client library to be installed. To install the library, run `pip install neptune`.rr) verify_typer5rrr6rr7r8F)r5rr>rlastz checkpoints/T checkpoints)rL ValueErrorrrneptune.internal.utilsr9r3neptune.new.internal.utils#neptune.new.metadata_containers.runrrbool_base_namespace_pathr_log_checkpoints _initial_run_run_is_monitoring_runr_force_reset_monitoring_run_init_run_kwargs_volatile_checkpoints_dir_should_upload_checkpoint_recent_checkpoint_path_target_checkpoints_namespace*_should_clean_recently_uploaded_checkpoint) rQr5rrr6rr7r8neptune_run_kwargsrr9r)r)r*rSsF          zNeptuneCallback.__init__cCs"|jr|j|`d|_dSdSr2)rDstoprr)r)r*_stop_run_if_existss   z#NeptuneCallback._stop_run_if_existsc Kszddlm}ddlm}m}Wnty'ddlm}ddlm}m}Ynw|z|di|j ||_ |j d |_ WdS||fyT}zt |d}~ww)Nr)init_run)NeptuneMissingApiTokenException"NeptuneMissingProjectNameExceptionsys/idr))rrPneptune.exceptionsrQrRr3 neptune.newneptune.new.exceptionsrOrGrDrrr/)rQadditional_neptune_kwargsrPrQrRer)r)r*_initialize_runs   zNeptuneCallback._initialize_runcCs(|j|_d|_|jd|_d|_dS)NTrS)rCrDrErrrr)r)r*_use_initial_runs z NeptuneCallback._use_initial_runcCsb|jdur |dS|js|jrdS|jr(|js(|js(|j|jdd|_dS|d|_dS)N)with_idTF)rCrZrFrErDrYrrr)r)r*_ensure_run_with_monitorings     z+NeptuneCallback._ensure_run_with_monitoringcCs@|jdur |dS|js|j|jdddddd|_dSdS)NF)r[capture_stdoutcapture_stderrcapture_hardware_metricscapture_traceback)rCrZrDrYrrErr)r)r*'_ensure_at_least_run_without_monitorings   z7NeptuneCallback._ensure_at_least_run_without_monitoringcCs|jdur ||jSr2)rDrarr)r)r*rs zNeptuneCallback.runcCs |j|jSr2)rrArr)r)r*_metadata_namespaces z#NeptuneCallback._metadata_namespacecCst|jtj<dSr2)r rr0integration_version_keyrr)r)r*_log_integration_versionr0z(NeptuneCallback._log_integration_versioncCs||jtj<dSr2)to_sanitized_dictrbr0trainer_parameters_key)rQrkr)r)r*_log_trainer_parameterssz'NeptuneCallback._log_trainer_parameterscCsJddlm}|rt|dr!|jdur#||j|jtj<dSdSdSdS)Nr)stringify_unsupportedr) neptune.utilsrhrrrrbr0model_parameters_key)rQrrhr)r)r*_log_model_parameters s  z%NeptuneCallback._log_model_parameterscCsR|rt|dr|j|jtj<|r#t|dr%|jdur'|j|jtj<dSdSdSdS)Nr)r2)rr)rbr0trial_name_keyr2trial_params_key)rQr(r)r)r*"_log_hyper_param_search_parameterss  z2NeptuneCallback._log_hyper_param_search_parameterssource_directoryrzc Cstj||}}|jdurMtj|j|}z|ddtjj}tj||}t|||}Wnt yL}zt d |WYd}~nd}~ww|j |j||jrh|jdurh|j |j|j||_dS)Nz..rzpNeptuneCallback was unable to made a copy of checkpoint due to I/O exception: '{}'. Could fail trying to upload.)r!rirjrHreplacelstripr(shutilcopytreeIOErrorr$r%rrbrK upload_filesrLrJ delete_files) rQrorz target_path relative_pathconsistent_checkpoint_path cpkt_path copy_pathrXr)r)r*_log_model_checkpoints&   z%NeptuneCallback._log_model_checkpointcKsHd|_|jr|js|jdurtj|_|jdkr |js"tddSdS)NrzWTo save the best model checkpoint, the load_best_model_at_end argument must be enabled.) rHrBoverwrite_output_dirsave_total_limitrrrrr<rkr)r)r*r2s  zNeptuneCallback.on_init_endcKsR|jsdS|d|_||jr|||||jr'||dSdS)NT) rXr\rFrdrrgrkrYrnrr)r)r*r^:s  zNeptuneCallback.on_train_begincKs |dSr2)rOrkr)r)r*rlIs zNeptuneCallback.on_train_endcCs&|jdur tj|jdd|dS)NT) ignore_errors)rHrrrmtreerOrr)r)r*rLs  zNeptuneCallback.__del__cKs$|jr||jd|jdSdS)Nr)rIr|rrerkr)r)r*rRszNeptuneCallback.on_savec Ks`|jdkr.|j}|dsd|}||}|jrtjntj}|jdup*|||j|_ dSdS)NrrA) rBrrhrgreater_is_betternpgreaterlessrrI) rQrkr(r\r|rbest_metric_name metric_valueoperatorr)r)r* on_evaluateVs    zNeptuneCallback.on_evaluatecCs*|jjD] }t||r|jSqtd)Nz6The trainer doesn't have a NeptuneCallback configured.)r4r5rTr Exception)rr~r;r)r)r*get_runbs   zNeptuneCallback.get_runrhcKsj|jsdS|dur1t|D]#\}}t|ttfr0|tjvr%||j|<q|j|j ||j dqdSdS)N)r) rXr+rrTrbrcr0 flat_metricsrbrre)rQrkr(r\rhrrrr)r)r*rijs  zNeptuneCallback.on_logr2)&rrmrnrorcrjrlrmrfrrrr@rSrOrYrZr\rapropertyrrbrdrgrkrnr|rr^rlrrr classmethodrrrcrir)r)r)r*r0hsf 7      r0c@s2eZdZdZddZddZd ddZd d ZdS) CodeCarbonCallbackzI A [`TrainerCallback`] that tracks the CO2 emission of training. cCs&tstdddl}||_d|_dS)NzWCodeCarbonCallback requires `codecarbon` to be installed. Run `pip install codecarbon`.r)rNrXrM _codecarbontracker)rQrMr)r)r*rS|s zCodeCarbonCallback.__init__cKs.|jdur|jr|jj|jd|_dSdSdS)N)r)ris_local_process_zerorEmissionsTrackerrrkr)r)r*rszCodeCarbonCallback.on_init_endNcK"|jr |jr|jdSdSdSr2)rrstartrr)r)r*r^ z!CodeCarbonCallback.on_train_begincKrr2)rrrNrkr)r)r*rlrzCodeCarbonCallback.on_train_endr2)rrmrnrorSrr^rlr)r)r)r*rws   rc@sFeZdZdZddZddZdddZdd d Zdd d Zd dZ dS)ClearMLCallbacka A [`TrainerCallback`] that sends the logs to [ClearML](https://clear.ml/). Environment: - **CLEARML_PROJECT** (`str`, *optional*, defaults to `HuggingFace Transformers`): ClearML project name. - **CLEARML_TASK** (`str`, *optional*, defaults to `Trainer`): ClearML task name. - **CLEARML_LOG_MODEL** (`bool`, *optional*, defaults to `False`): Whether to log models as artifacts during training. cCsRtr ddl}||_ntdd|_d|_d|_tdd t dhv|_ dS)NrzNClearMLCallback requires 'clearml' to be installed. Run `pip install clearml`.FCLEARML_LOG_MODELrErF) r/r-_clearmlrXrt_initialized_externally _clearml_taskr!r"r#rrurw)rQr-r)r)r*rSs"zClearMLCallback.__init__cKs|jdurdS|jr dS|jrltd|jdurQ|jjr2|jj|_d|_d|_tdn|jjj t ddt ddddd dd |_d|_td |j |d t |d rn|jdurp|j |jddSdSdSdS)Nz"Automatic ClearML logging enabled.Tz)External ClearML Task has been connected.CLEARML_PROJECTzHuggingFace Transformers CLEARML_TASKrF)r6pytorch)r task_nameauto_connect_frameworks output_uriz"ClearML Task has been initialized.ArgsrzModel Configuration)rrtrXr$rrTask current_taskrr'r!r"connectrr)rQrkr(rrrr)r)r*rs2        zClearMLCallback.setupNcKs@|jdurdS|jr d|_|js|j||||fi|dSdSr)rrYrtr)rQrkr(r\rrrr)r)r*r^s zClearMLCallback.on_train_beginc Ks:|jdurdS|jr|jr|js|jdSdSdSdSr2)rrrXrrj) rQrkr(r\rrr|rhrr)r)r*rls  zClearMLCallback.on_train_endc Ks2|jdurdS|js|j||||fi||jrd}t|} d} t| } gd} |D]i\} }t|ttfr| | vrG|j j | |dq-| |r^|j j | | dd||jdq-| | ru|j j | | dd||jdq-|j j | d||jdq-td |d t|d | d q-dSdS) NrArB) train_runtimetrain_samples_per_secondtrain_steps_per_second train_lossrrreval)titleseriesr iterationtestrur_r`razn" as a scalar. This invocation of ClearML logger's report_scalar() is incorrect so we dropped this attribute.)rrtrrXrDrrTrbrcr get_loggerreport_single_valuerh report_scalarrer$r%r)rQrkr(r\rrrhrrGrHrIrJsingle_value_scalarsrKrLr)r)r*risH        zClearMLCallback.on_logcKsd|jr,|jr.|jr0d|j}tj|j|}t d|d|jj ||jdddSdSdSdS)Nrrr F)rauto_delete_file) rwrrXrer!rirjrr$rupdate_output_modelrr)r)r*rs   zClearMLCallback.on_saver)NNNN)NNNrr)r)r)r*rs     *rcs<eZdZdZd dedeffdd ZddZd d ZZS) FlyteCallbacka^A [`TrainerCallback`] that sends the logs to [Flyte](https://flyte.org/). NOTE: This callback only works within a Flyte task. Args: save_log_history (`bool`, *optional*, defaults to `True`): When set to True, the training logs are saved as a Flyte Deck. sync_checkpoints (`bool`, *optional*, defaults to `True`): When set to True, checkpoints are synced with Flyte and can be used to resume training in the case of an interruption. Example: ```python # Note: This example skips over some setup steps for brevity. from flytekit import current_context, task @task def train_hf_transformer(): cp = current_context().checkpoint trainer = Trainer(..., callbacks=[FlyteCallback()]) output = trainer.train(resume_from_checkpoint=cp.restore()) ``` Tsave_log_historysync_checkpointscsXtts tdtrtstdd}ddlm }|j |_ ||_ ||_ dS)NzLFlyteCallback requires flytekit to be installed. Run `pip install flytekit`.zSyncing log history requires both flytekitplugins-deck-standard and pandas to be installed. Run `pip install flytekitplugins-deck-standard pandas` to enable this feature.Fr)current_context)rrSrPr3rQrr$r%rOrrzcprr)rQrrrrr)r*rS(s     zFlyteCallback.__init__cKsR|jr%|jr'd|j}tj|j|}td|d|j |dSdSdS)NrzSyncing checkpoint in z to Flyte. This may take time.) rrXrer!rirjrr$rrr+rr)r)r*r:s  zFlyteCallback.on_savec KsL|jr$ddl}ddlm}ddlm}||j}|d||dSdS)Nr)Deck) TableRendererz Log History) rpandasrOrflytekitplugins.deck.rendererr DataFrame log_historyto_html) rQrkr(r\rpdrrlog_history_dfr)r)r*rlBs   zFlyteCallback.on_train_end)TT) rrmrnror@rSrrlr.r)r)rr*r s rc@sheZdZdZ  ddeedeeedeffddZ dd Z dd d Z dd d Z ddZ ddZdS)DVCLiveCallbackaW A [`TrainerCallback`] that sends the logs to [DVCLive](https://www.dvc.org/doc/dvclive). Use the environment variables below in `setup` to configure the integration. To customize this callback beyond those environment variables, see [here](https://dvc.org/doc/dvclive/ml-frameworks/huggingface). Args: live (`dvclive.Live`, *optional*, defaults to `None`): Optional Live instance. If None, a new instance will be created using **kwargs. log_model (Union[Literal["all"], bool], *optional*, defaults to `None`): Whether to use `dvclive.Live.log_artifact()` to log checkpoints created by [`Trainer`]. If set to `True`, the final checkpoint is logged at the end of training. If set to `"all"`, the entire [`TrainingArguments`]'s `output_dir` is logged at each checkpoint. NliverrcKsdtstdddlm}||_d|_d|_t||r#||_d|_dS|dur0td|jddS)NzLDVCLiveCallback requires dvclive to be installed. Run `pip install dvclive`.rLiveFTz Found class z for live, expected dvclive.Live) rSrXrRrrwrtrrTrY)rQrrrrr)r)r*rS\s   zDVCLiveCallback.__init__cCsxddlm}d|_|jdur&td}|tvrd|_n |dkr&d|_|j r:|j s0||_ |j | dSdS)a@ Setup the optional DVCLive integration. To customize this callback beyond the environment variables below, see [here](https://dvc.org/doc/dvclive/ml-frameworks/huggingface). Environment: - **HF_DVCLIVE_LOG_MODEL** (`str`, *optional*): Whether to use `dvclive.Live.log_artifact()` to log checkpoints created by [`Trainer`]. If set to `True` or *1*, the final checkpoint is logged at the end of training. If set to `all`, the entire [`TrainingArguments`]'s `output_dir` is logged at each checkpoint. rrTNHF_DVCLIVE_LOG_MODELr) rRr _initalizedrwr!r"r#rrxrXrrr)rQrkr(rr log_model_envr)r)r*rps     zDVCLiveCallback.setupcKrr2rrr)r)r*r^rzDVCLiveCallback.on_train_beginc Ks|js |||||jrJddlm}ddlm}|D]%\} } || r1|j || d| qt d| dt | d| dq|j dSdS) Nr)Metric)standardize_metric_namezdvclive.huggingfacer_r`razh" as a scalar. This invocation of DVCLive's Live.log_metric() is incorrect so we dropped this attribute.)rtrrX dvclive.plotsr dvclive.utilsrr could_logrrr$r%r next_step) rQrkr(r\rrhrrrrrr)r)r*ris&   zDVCLiveCallback.on_logcKs4|jdkr|jr|jr|j|jdSdSdSdS)Nr)rwrtrXrrrrkr)r)r*rs zDVCLiveCallback.on_savec Ks|jrC|jrEddlm}|jdur<|||d|dd}|jr#dnd}tj |j |}| ||j j ||ddd |j dSdSdS) NrrTrrrrr:)rrcopy)rtrXtransformers.trainerrrwrrr!rirjrrrrrq) rQrkr(r\rrrrrr)r)r*rls    zDVCLiveCallback.on_train_endrr2)rrmrnrorrr rr@rSrr^rirrlr)r)r)r*rLs    r) r>rrGrr6r rMr-rIflyterRcCs>|D]}|tvrt|ddtdqdd|DS)Nz is not supported, only rz are supported.cSsg|]}t|qSr))INTEGRATION_TO_CALLBACK)r integrationr)r)r*rsz7get_reporting_integration_callbacks..)rr<rjr,)r7rr)r)r*#get_reporting_integration_callbackssr)jrorimportlib.metadatar&importlib.utilrrr!rsrrrr dataclassesrpathlibrtypingrrrrrr numpyrrr r rr r rrrrrr$rpr'r(r"r#r3rrr get_configr%r3r<rKr_neptune_versionrPackageNotFoundErrortrainer_callbackrrrrrr training_argsrrrr+r/r4r8r:r<r>rArCrHrJrLrNrPrQrSr[rbrrrrr=r@r+rr6rrrrrr/r0rrrrrrr)r)r)r*s      $      9bEV)M1 y?f