o
    ha                     @   s  d dl Z d dlZd dlZd dlmZ d dlmZmZmZm	Z	 d dl
Z
ddlmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZ d	d
lmZmZ g fddZde jde	ee eeef f fddZde jdeeef fddZde jde	ee eeef f fddZ 	d$de jdeeef de!de	e jeeef f fddZ"dee deeef fddZ#dede jde	e jeeef f fddZ$d d! Z%G d"d# d#Z&dS )%    Nliteral_eval)AnyDictListTuple   )SageMakerConfig)DynamoBackendPrecisionTypeis_ipex_availableis_npu_availableis_xpu_available)DEEPSPEED_MULTINODE_LAUNCHERS)is_port_in_usemerge_dicts   )DistributedTypeSageMakerDistributedTypec                 C   sD   | |\}}t|  D ]\}}|t| v rt||| q|S )z4
    Filters out all `accelerate` specific args
    )parse_known_argsvarsitemskeyssetattr)argsparserdefault_argsnew_args_keyvalue r!   M/var/www/html/ai/venv/lib/python3.10/site-packages/accelerate/utils/launch.py_filter_args$   s   r#   r   returnc              	   C   s$  g }| j r| jrtd| j s|tj | jr|d || j || j t	j
 }t| jp4| j|d< | jr?d|d< | jdkr`| jdur`t rR| j|d< nt r[| j|d	< n| j|d
< | jdkrr| j|d< t| j|d< n| jdkr| jdur| jnd|d< | jdurt| jnd|d< z	t| j }W n ty   td| j  dt  dw t||d< z	t| j }W n ty   td| j  dt  dw |j|d< | j|d< t| j |d< t| j!|d< t| j"|d< t# rt| j$ |d< t| j% |d< ||fS )zz
    Prepares and returns the command list and an environment with the correct simple launcher environment variables.
    0--module and --no_python cannot be used togetherz-mACCELERATE_USE_CPUtrueACCELERATE_DEBUG_MODEallNZE_AFFINITY_MASKASCEND_RT_VISIBLE_DEVICESCUDA_VISIBLE_DEVICESr   MASTER_ADDRMASTER_PORTz	127.0.0.129500Unknown mixed_precision mode: . Choose between .ACCELERATE_MIXED_PRECISIONUnknown dynamo backend: ACCELERATE_DYNAMO_BACKENDACCELERATE_DYNAMO_MODEACCELERATE_DYNAMO_USE_FULLGRAPHACCELERATE_DYNAMO_USE_DYNAMICOMP_NUM_THREADSACCELERATE_USE_IPEXACCELERATE_USE_XPU)&	no_pythonmodule
ValueErrorappendsys
executabletraining_scriptextendtraining_script_argsosenvironcopystrcpuuse_cpudebuggpu_idsr   r   num_machinesmain_process_ipmain_process_portnum_processesr   mixed_precisionlowerlistr
   dynamo_backendupperr    dynamo_modedynamo_use_fullgraphdynamo_use_dynamicnum_cpu_threads_per_processr   ipexuse_xpu)r   cmdcurrent_envrQ   rT   r!   r!   r"   prepare_simple_launcher_cmd_env/   sb   







r^   c              	   C   sD  t | d}t | d}t | d}t | d}|dkrVt| dt||  t| dt| t| dt| j t | d	d
rJt| dt| t| dt| n t| d| d|  nt| dt| |durjt| dt| |du rpd}|dkpzt| jdk}|rt|rtd| d| jr| jrt	d| jrt| dd n	| jrt| dd t
j }| jrd|d< t | dd}|dkr| jdurt r||d< nt r||d< n||d< | j }zt|}W n t	y   t	d| d t  d!w t||d"< z	t| j }	W n t	y   t	d#| j  d t  d!w |	j|d$< | j|d%< t| j|d&< t| j|d'< | jrd|d(< | jrI| jsIt	d)t| j|d*< t| j  |d+< t| j!|d,< | j"durmt| j"|d-< | j#durzt| j#|d.< | j$durt| j$|d/< | j%durt| j%|d0< t| j& |d1< t| j' |d2< t| j |d3< t| j |d4< | j(rd5}
d|d6< t| j)||
d7 < t| j*||
d8 < t| j+||
d9 < | j,durt| j,||
d: < | j-durt| j-||
d; < | j.dur
t| j.||
d< < | j/durt| j/||
d= < t| j0|d>< |S )?z_
    Prepares and returns an environment with the correct multi-GPU environment variables.
    rP   rM   rN   rO   r   nproc_per_nodennodes	node_ranksame_networkFmaster_addrmaster_portrdzv_endpoint:N<s  r   3Tried to launch distributed communication on port `&  `, but another process is utilizing it. Please specify a different port (such as using the `----main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.r%   r=   Tr<   r'   r(   rL   r)   r*   r+   r,   r0   r1   r2   r3   r4   r5   r6   r7   r8   ACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`FSDP_SHARDING_STRATEGYFSDP_OFFLOAD_PARAMSFSDP_MIN_NUM_PARAMSFSDP_AUTO_WRAP_POLICYFSDP_TRANSFORMER_CLS_TO_WRAPFSDP_BACKWARD_PREFETCHFSDP_STATE_DICT_TYPEFSDP_FORWARD_PREFETCHFSDP_USE_ORIG_PARAMSFSDP_CPU_RAM_EFFICIENT_LOADINGFSDP_SYNC_MODULE_STATESMEGATRON_LM_ACCELERATE_USE_MEGATRON_LM	TP_DEGREE	PP_DEGREEGRADIENT_CLIPPINGNUM_MICRO_BATCHESSEQUENCE_PARALLELISMRECOMPUTE_ACTIVATIONSUSE_DISTRIBUTED_OPTIMIZERr9   )1getattrr   rH   intmachine_rankr   ConnectionErrorr=   r<   r>   rE   rF   rG   rK   rL   r   r   rQ   rR   r   rS   r
   rT   rU   r    rV   rW   rX   use_fsdpfsdp_cpu_ram_efficient_loadingfsdp_sync_module_statesfsdp_sharding_strategyfsdp_offload_paramsfsdp_min_num_paramsfsdp_auto_wrap_policy"fsdp_transformer_layer_cls_to_wrapfsdp_backward_prefetch_policyfsdp_state_dict_typefsdp_forward_prefetchfsdp_use_orig_paramsuse_megatron_lmmegatron_lm_tp_degreemegatron_lm_pp_degreemegatron_lm_gradient_clippingmegatron_lm_num_micro_batches megatron_lm_sequence_parallelism!megatron_lm_recompute_activations%megatron_lm_use_distributed_optimizerrY   )r   rP   rM   rN   rO   need_port_checkr]   rL   rQ   rT   prefixr!   r!   r"   prepare_multi_gpu_envj   s   










r   c           
   	   C   sT  t | d}t | d}t | d}t | d}d}| jdu r#t| dtd  |dkr| jtd krd	d
g}|dt| jdt| jg | jdurQ|dt| jg n| jdura|dt| jg n|dt| j	| j
 g |dt|g | jr| jrtd| jr|d n| jr|d || j || j n]|dkr| jtd krt| dt||  t| dt| t| dt| j t | ddrt| dt| t| dt| n t| d| d|  nt| dt| |durt| dt| |du rd}|dkpt| jdk}|rt|rtd| d| jr+| jr+td| jr6t| d d! n
| jr@t| d"d! tj }| jrMd#|d$< t | d%d&}|d&krt| jdurtt rg||d'< nt rp||d(< n||d)< z	t| j }	W n ty   td*| j  d+t  d,w t d-tj!"d,|d-< t|	|d.< t| j# |d/< d#|d0< | j$durt| j$|d1< | j%durt| j%|d2< | j&durt| j& |d3< | j'durt| j' |d4< | j(durt| j( |d5< | j)dur
t| j) |d6< | j*durt| j* |d7< | j+dur&t| j+|d8< ||fS )9zt
    Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
    rP   rM   rN   rO   Ndeepspeed_multinode_launcherr   r   	deepspeedz--no_local_rankz
--hostfilez
--launcherz	--excludez	--includez
--num_gpusz--master_portr%   z--modulez--no_pythonr_   r`   ra   rb   Frc   rd   re   rf   rg   rh   ri   r=   Tr<   r'   r(   rL   r)   r*   r+   r,   r0   r1   r2   
PYTHONPATHr3   ACCELERATE_CONFIG_DS_FIELDSACCELERATE_USE_DEEPSPEEDACCELERATE_DEEPSPEED_ZERO_STAGE&ACCELERATE_GRADIENT_ACCUMULATION_STEPSACCELERATE_GRADIENT_CLIPPING-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEACCELERATE_DEEPSPEED_ZERO3_INIT+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL ACCELERATE_DEEPSPEED_CONFIG_FILE),r   r   r   r   rC   rH   deepspeed_hostfiledeepspeed_exclusion_filterdeepspeed_inclusion_filterrP   rM   r=   r<   r>   r?   rB   rD   r   r   r   r   rE   rF   rG   rK   rL   r   r   r   rQ   rR   rS   env_var_path_addpathabspath'deepspeed_fields_from_accelerate_config
zero_stagegradient_accumulation_stepsgradient_clippingoffload_optimizer_deviceoffload_param_devicezero3_init_flagzero3_save_16bit_modeldeepspeed_config_file)
r   rP   rM   rN   rO   r\   r   r]   rL   rQ   r!   r!   r"   prepare_deepspeed_cmd_env   s   












r   Fr]   podc                 C   sL   | j dkr| jrd|d< nd|d< | jrd|d< |r"| j| _| j| _| |fS )zY
    Prepares and returns an environment with the correct TPU environment variables.
    bf161XLA_DOWNCAST_BF16XLA_USE_BF16r'   r(   )rQ   downcast_bf16rK   tpu_vmvmtpu_nametpu)r   r]   r   r!   r!   r"   prepare_tpuP  s   

r   nargsc                 C   s   t | dk ri S dd }t }|| \}}t|D ]8\}}|drSd }|d t |k r<||d  dr;tdntd|d u rL|j||d q|j||d qd	d
 || j	
 D S )Nr   c                 S   s<   zt | } | d | krt| W S | W S  ty   |  Y S w )Nr   )floatr   r>   )sr!   r!   r"   _infer_typei  s   
z+_convert_nargs_to_dict.<locals>._infer_type)-z--r   ul   SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)type)actionc                 S   s&   i | ]\}}||d v rt |n|qS ))TrueFalser   ).0r   r    r!   r!   r"   
<dictcomp>  s    z*_convert_nargs_to_dict.<locals>.<dictcomp>)lenargparseArgumentParserr   	enumerate
startswithr>   add_argument
parse_args__dict__r   )r   r   r   r   unknownindexargumentr   r!   r!   r"   _convert_nargs_to_dictd  s0   

r   sagemaker_configc                 C   s$  t d | jtjd< | jd ur| jtjd< n|jd ur-|jd ur-|jtjd< |jtjd< ntdtj	|j
}|s<d}tj|j
}|dsPtd	| d
t d t|j}z	t|j }W n tyy   td|j  dt  dw z	t|j }W n ty   td|j  dt  dw dt||j|jt|jt|j| jjd}d }| jtjkrddddiii}d }	| jd urt d| j d i }	t | j%}
t!|
D ]\}}|dkrq|"d}|d # |	|d < qW d    n	1 sw   Y  t d|	  d }| j$d urdt d| j$ d g }t | j$.}
t!|
D ]!\}}|dkr6q+|"d}|d |d # d}|%| q+W d    n	1 sXw   Y  t d|  t d | j&||| j'| j(| j)| j*| j+| j,| j-d||||d }| j.d urt/| j.|}||	fS )!Nz(Configuring Amazon SageMaker environmentAWS_DEFAULT_REGIONAWS_PROFILEAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profiler2   z.pyz8Your training script should be a python script and not ""z'Converting Arguments to Hyperparametersr0   r1   r4   r'   )ACCELERATE_USE_SAGEMAKERr3   r5   r6   r7   r8   %ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPEsmdistributeddataparallelenabledTzLoading SageMaker Inputs from z filer   	r   zLoaded SageMaker Inputs: zLoading SageMaker Metrics from )NameRegexzLoaded SageMaker Metrics: zCreating EstimatorF)	image_urientry_point
source_dirroletransformers_versionpytorch_version
py_versionbase_job_nameinstance_countinstance_typedebugger_hook_configdistributionhyperparametersenvironmentmetric_definitions)0printregionrE   rF   profileaws_access_key_idaws_secret_access_keyEnvironmentErrorr   dirnamerB   basenameendswithr>   r   rD   r   rQ   rR   rS   r
   rT   rU   rH   r    rV   rW   rX   distributed_typer   DATA_PARALLELsagemaker_inputs_fileopenr   splitstripsagemaker_metrics_filer?   r   iam_role_namer   r   r   r   rM   ec2_instance_typeadditional_argsr   )r   r   r   r   r   rQ   rT   r   r   sagemaker_inputsfileilinelsagemaker_metricsmetric_dictr!   r!   r"   prepare_sagemager_args_inputs  s   








r  c                 C   s6   dd t j| ddD }|t| d|S )z
    Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
    caller to set it in os.environ.
    c                 S   s   g | ]
}t |d kr|qS )r   )r   )r   pr!   r!   r"   
<listcomp>  s    z$env_var_path_add.<locals>.<listcomp> rf   )rE   rF   getr  r?   rH   join)env_var_namepath_to_addpathsr!   r!   r"   r     s   
r   c                   @   s"   e Zd ZdZd	ddZdd ZdS )
PrepareForLaunchai  
    Prepare a function that will launched in a distributed setup.

    Args:
        launcher (`Callable`):
            The function to launch.
        distributed_type ([`~state.DistributedType`]):
            The distributed type to prepare for.
        debug (`bool`, *optional*, defaults to `False`):
            Whether or not this is a debug launch.
    NOFc                 C   s   || _ t|| _|| _d S )N)launcherr   r   rK   )selfr  r   rK   r!   r!   r"   __init__  s   

zPrepareForLaunch.__init__c                 G   s   | j r!ttjd}tjd}tjjd|tj|||d n1| j	t
jt
jt
jt
jfv rRt|tjd< ttjdd}ttjdd	}t|| | tjd
< tdtjd< | j|  d S )N
WORLD_SIZEACCELERATE_DEBUG_RDV_FILEgloo)rankstore
world_size
LOCAL_RANKNPROCr   	NODE_RANKr   RANKFORK_LAUNCHED)rK   r   rE   rF   r  torchdistributedinit_process_group	FileStorer   r   	MULTI_GPU	MULTI_NPU	MULTI_XPU	MULTI_CPUrH   r  )r  r   r   r#  rdv_filenprocra   r!   r!   r"   __call__  s*   zPrepareForLaunch.__call__N)r  F)__name__
__module____qualname____doc__r  r3  r!   r!   r!   r"   r  	  s    
r  )F)'r   rE   r@   astr   typingr   r   r   r   r)  commands.config.config_argsr	   utilsr
   r   r   r   r   utils.constantsr   utils.otherr   r   dataclassesr   r   r#   	NamespacerH   r^   r   r   boolr   r   r  r   r  r!   r!   r!   r"   <module>   sF   (;(pw

*
q
