o
    ha&                     @   s   d dl Z d dlZddlmZ ddlmZmZmZ ddlm	Z	 ddl
mZ ederOe	 rOd dlm  mZ d d	lmZmZ d d
lmZ d dlmZ d dlmZ eeZdddZdddZdddZdddZdS )    N   )
get_logger   )FSDP_MODEL_NAMEFSDP_PYTORCH_VERSIONOPTIMIZER_NAME)is_torch_distributed_available)is_torch_versionz>=)DefaultLoadPlannerDefaultSavePlanner)!load_sharded_optimizer_state_dict)FullyShardedDataParallel)StateDictTypec           
      C   s  t j|dd | jtjkr|jdk}|| j_|| j_t	|| j| j| j
 | }| jtjkrd|dkr9t dnt d| d}t j||}|jdkrctd|  t|| td|  n~| jtjkr|dkrwt d	|j dnt d| d	|j d}t j||}td|  t|| td|  nQ| jtjkrt j|t d| }	t j|	dd td|	  d
|i}tj|t|	t d td|	  W d    d S W d    d S W d    d S W d    d S 1 sw   Y  d S )NTexist_okr   r   .bin_zSaving model to zModel saved to _rankmodel
state_dictstorage_writerplanner)osmakedirsstate_dict_typer   FULL_STATE_DICTnum_processesstate_dict_configoffload_to_cpu
rank0_onlyFSDPoptim_state_dict_configr   r   pathjoinprocess_indexloggerinfotorchsaveLOCAL_STATE_DICTSHARDED_STATE_DICTdist_cpsave_state_dictFileSystemWriterr   )
fsdp_pluginacceleratorr   
output_dirmodel_indexis_multi_processr   weights_nameoutput_model_fileckpt_dir r7   Q/var/www/html/ai/venv/lib/python3.10/site-packages/accelerate/utils/fsdp_utils.pysave_fsdp_model#   sX   
"
 "r9   c                 C   s  |   | jtjkr|jdk}|| j_|| j_t|| j| j| j	 | jtjkrrt
|tkrD|jdkrD| js;td	 W d    d S |dkrMt dnt d| d}tj||}td|  t|}td|  n{| jtjkr|dkrt d|j dnt d| d|j d}tj||}td|  t|}td|  n?| jtjkrt |vrtj|t d| n|}	td|	  d	| i}tj|t|	t d
 |d	 }td|	  ||}
W d    |
S 1 sw   Y  |
S )Nr   r   zzSet the `sync_module_states` flag to `True` so that model states are synced across processes when initializing FSDP objectr   r   zLoading model from zModel loaded from r   r   )r   storage_readerr   )wait_for_everyoner   r   r   r   r   r   r    r!   r"   typer%   sync_module_states
ValueErrorr   r   r#   r$   r&   r'   r(   loadr*   r+   r   r,   load_state_dictFileSystemReaderr
   )r/   r0   r   	input_dirr2   r3   r4   input_model_filer   r6   load_resultr7   r7   r8   load_fsdp_modelP   sb   
"



**rE   c           
      C   s<  t j|dd t|| j| j| j t||}| jtjkrS|j	dkrR|dkr-t
 dnt
 d| d}t j||}td|  t|| td|  n9t j|t
 d| }	t j|	dd td|	  tjd|it|	t d	 td|	  W d    d S W d    d S 1 sw   Y  d S )
NTr   r   r   r   zSaving Optimizer state to zOptimizer state saved in 	optimizerr   )r   r   r!   r   r   r"   optim_state_dictr   r   r%   r   r#   r$   r&   r'   r(   r)   r,   r-   r.   r   )
r/   r0   rF   r   r1   optimizer_indexoptim_stateoptim_state_nameoutput_optimizer_filer6   r7   r7   r8   save_fsdp_optimizer   s4   
 " rL   c                 C   sB  |   t|| j| j| j | jtjkrOd }|jdks!| jjsN|dkr*t	 dnt	 d| d}t
j||}td|  t|}td|  n3t	 |vr`t
j|t	 d| n|}	td|	  t| dt|	d}|d }td	|	  tj|||d
}
||
 W d    d S 1 sw   Y  d S )Nr   r   r   zLoading Optimizer state from zOptimizer state loaded from zLoading Optimizer from rF   )model_state_dictoptimizer_keyr:   zOptimizer loaded from )r   optimrG   )r;   r!   r   r   r"   r   r   r%   r    r   r   r#   r$   r&   r'   r(   r?   r   r   r,   rA   optim_state_dict_to_loadr@   )r/   r0   rF   r   rB   rH   rI   optimizer_nameinput_optimizer_filer6   flattened_osdr7   r7   r8   load_fsdp_optimizer   s:    

"rT   )r   )r   r(   loggingr   	constantsr   r   r   importsr   versionsr	   torch.distributed.checkpointdistributed
checkpointr,   ,torch.distributed.checkpoint.default_plannerr
   r   &torch.distributed.checkpoint.optimizerr   2torch.distributed.fsdp.fully_sharded_data_parallelr   r!   r   __name__r&   r9   rE   rL   rT   r7   r7   r7   r8   <module>   s"   

-
5