o
    hy                     @   s   d dl Z d dlmZmZmZmZmZmZmZ d dl	Z	d dl
mZ d dlmZ ddlmZmZ dZe 	ddejdeee	jj  d	ejfd
dZdejd	efddZG dd dZdS )    N)AnyDictIterableListOptionalSetTuple)DistributedDataParallel   )_get_registrycontract moduleignored_modulesreturnc                 K   sZ   t jd d|v rt|d tt jfstdt|d  t|dj	| fi | | S )zReplicates a module

    Args:
        module (torch.nn.Module): module to replicate

    Example::
        >>> # xdoctest: +REQUIRES(module:torch._C._distributed_c10d)
        >>> module = nn.Linear(3, 3)
        >>> replicate(module)
    ztorch.distributed.replicate	device_idz6Expected device_id to be int or torch.device, but got )r   )
torch_C_log_api_usage_once
isinstanceintdeviceRuntimeErrortype_ReplicateStatemark_module)r   r   kwargs r   ]/var/www/html/ai/venv/lib/python3.10/site-packages/torch/distributed/_composable/replicate.py	replicate   s   r   c                 C   s   dt | v S )z+Check if module is marked with fully_shard.fully_shard)r   )r   r   r   r   _is_fully_sharded(   s   r!   c                   @   s   e Zd Zdeeejj  ddfddZdejddfddZ	e
fdejd	eddfd
dZdddZdejdeedf deeef defddZdejdeej dejdejfddZdS )r   r   r   Nc                 C   sP   d | _ d| _t | _i | _|d urt|nt | _dd | jD | _g | _	d S )NFc                 S   s   h | ]}|  D ]}|qqS r   )
parameters).0mpr   r   r   	<setcomp>6   s
    z+_ReplicateState.__init__.<locals>.<setcomp>)
r   has_initializednnParameterList_param_listr   setr   ignored_params_param_names)selfr   r   r   r   __init__.   s   

z_ReplicateState.__init__r   c                 K   sH   t |rtd|| _dt|_|j| jdd || j	 || _
d S )NzGCannot apply `replicate()` on a Module already managed by `fully_shard`FT)with_kwargs)r!   AssertionErrorr   r   state_params_collectedregister_forward_pre_hookforward_pre_hookregister_forward_hookforward_post_hookr   )r.   r   r   r   r   r   r   <   s   
z_ReplicateState.mark_moduleprefixc                 C   s   t |rd S || jv rd S |tkr| dnt}|jddD ]\}}|| jvr7| j| | j| |  q| D ]\}}| j	|| | d q<d S )N.F)recurse)r   r8   )
r!   r   _ROOT_MODULE_PREFIXnamed_parametersr,   r*   appendr-   named_children_collect_params)r.   r   r8   recurse_prefixnr%   namechild_moduler   r   r   r?   H   s   

z_ReplicateState._collect_paramsc                 C   s   | j rd S d| _ | | j | jt| j_d| jv rJ| jd d ur?| jd }t|t	j
r8|jdkr8d | jd< n|g| jd< nd | jd< | jd t| jfi | j| _t| jt| j_d S )NTr   cpu
device_ids)r'   r?   r   r-   r   r2   _replicate_param_namesr   r   r   r   r   popr	   r*   _ddpweakrefref_ddp_weakref)r.   r   r   r   r   init_helper^   s   


z_ReplicateState.init_helperargs.r   c                 C   s&   |    | jj|i |\}}||fS N)rL   rH   _pre_forward)r.   r   rM   r   r   r   r   r5   }   s   z _ReplicateState.forward_pre_hookinputoutputc                 C   s   | j |S rN   )rH   _post_forward)r.   r   rP   rQ   r   r   r   r7      s   z!_ReplicateState.forward_post_hook)r   N)__name__
__module____qualname__r   r   r   r(   Moduler/   r   r;   strr?   rL   r   r   r   r5   Tensorr7   r   r   r   r   r   -   s<    




r   rN   )rI   typingr   r   r   r   r   r   r   r   torch.nnr(   torch.nn.parallelr	   r   r   r;   rV   r   boolr!   r   r   r   r   r   <module>   s$    $