o
    h<#                  	   @   s  U d dl mZmZ d dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZmZ dd	lmZmZmZmZmZmZ dd
lmZmZmZmZmZmZ ddlmZmZ dgZ ee! e"d< dej#defddZ$dedefddZ%dededefddZ&de!dedefddZ'de!dededefddZ(de!dej#defddZ)de!defd d!Z*d"d# Z+d$d% Z,de!d&ed'ee dee fd(dZ-d)edefd*d+Z.de!d,edee fd-d.Z/dedefd/d0Z0de!d1ed2edee fd3d4Z1dS )5    )AnyListN)ShardMetadata)ShardedTensor)TensorProperties)DTensor)compute_local_shapecompute_local_offset   )LoadItemTypeSavePlanReadItem	WriteItemWriteItemTypeTensorWriteData)BytesStorageMetadataChunkStorageMetadataTensorStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPES)+_shards_get_overlap_region_wrt_saved_tensor"_check_shard_metadata_pair_overlap create_read_items_for_chunk_list__all__tensorreturnc                 C   s$   t tdgt|   |  dS )Nr   offsetssizes)r   torchSizelensize)r    r$   b/var/www/html/ai/venv/lib/python3.10/site-packages/torch/distributed/checkpoint/planner_helpers.py_create_chunk_from_tensor%   s   r&   shard_mdc                 C   s   t t| jt| jdS )Nr   )r   r    r!   shard_offsetsshard_sizes)r'   r$   r$   r%   _chunk_for_shard,   s   

r*   sharded_tensorc                 C   s   t t||  j|  jdS )Nchunk
propertiesr#   )r   r*   metadatatensor_propertiesr#   )r+   r'   r$   r$   r%   _sharded_tensor_metadata3   s
   r1   fqnc              	   C   sz   |j }|jdksJ dtt|j||j}tt|j||j}tt	| |t
jtt||dt| | ddS )Nr
   .Only 1D DeviceMeshes can currently be handled.r   r,   indextypetensor_data)device_meshndimr    r!   r   shape
placementsr	   r   r   r   SHARDr   r   r   create_from_tensorto_localr#   )r2   r   r8   r   r   r$   r$   r%   _create_write_items_for_dtensor=   s$   r?   c                 C   s(   t |j}tt| |tjt||dS )Nr4   )r    r!   r(   r   r   r   r<   r1   )r2   r+   r'   r   r$   r$   r%   _create_write_item_for_shardU   s   r@   c                 C   sN   t dgt|  }tt| |tjtt	|| dt
|| ddS )Nr   r   r,   r4   )r    r!   r"   r#   r   r   r   TENSORr   r   r   r=   )r2   r   r   r$   r$   r%   _create_write_item_for_tensor`   s   rB   bytesc                 C   s   t t| tjdS )N)r5   r6   )r   r   r   BYTE_IO)r2   rC   r$   r$   r%   _create_write_item_for_bytesiom   s   rE   c              	   C   s.   t tj| t|f|t|ft|fdS N)r6   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r   rD   r    r!   rG   dest_offsetrI   storage_offsetlengthr$   r$   r%   _create_read_item_for_byteiot   s   


rP   c              	   C   s(   t tj| t||t|t|dS rF   )r   r   rA   r    r!   rG   rH   rI   rJ   rK   r$   r$   r%   _create_read_item_for_tensor   s   rR   checkpoint_mdlocal_chunksc                 C   s   g }t |D ]L\}}t |jD ]B\}}t||sqg }g }	g }
t||dD ]\}}}}|| |	| |
| q%|tt| |j||	t| |j|||
d qq|S )aX  
    Creates a list of ``ReadItem`` based on the checkpoint and local chunks.

    This applies the resharding algorithm and computes the reads needed
    to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

    Args:
        fqn (str) : The state_dict FQN to pass to ``ReadItem``.
        checkpoint_md (TensorStorageMetadata): metadata for a given tensor
            from a checkpoint.
        local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
            loaded.

    Returns:
        A list of ``ReadItem`` that will satisfy all input chunks.
    )saved_shardcurrent_shardrQ   )	enumeratechunksr   r   appendrR   r   r   )r2   rS   rT   
read_itemsidxshardstorage_idx
storage_mdrJ   rH   rK   dimoffset_for_saved_tensoroffset_for_current_tensorrO   r$   r$   r%   r      sH   

"
state_dictc                 C   s   g }|   D ]@\}}t|tr|t|| qt|tr/| jD ]}|t||| q"qt|t	j
r>|t|| q|t|| qt|S )N)items
isinstancer   rY   r?   r   r/   shards_metadatar@   r    TensorrB   rE   r   )rb   requestsr2   objr'   r$   r$   r%   "_create_default_metadata_only_plan   s   


ri   objectc                    s\   t trt gS t tr fdd D S t tjr(t gS t gS )Nc                    s   g | ]	}t  |jqS r$   )r@   r/   .0r\   r2   rj   r$   r%   
<listcomp>   s    z'_create_write_items.<locals>.<listcomp>)	rd   r   r?   r   local_shardsr    rf   rB   rE   rm   r$   rm   r%   _create_write_items   s   

rp   c                 C   sP   | j }|jdksJ dtt| j|| j}tt| j|| j}t||dS )Nr
   r3   r   )	r8   r9   r    r!   r   r:   r;   r	   r   )r   r8   r   r   r$   r$   r%   _create_chunk_from_dtensor   s   rq   mdrh   c                 C   s   t |ts@t |trt|g}n*t |trdd | D }nt |tjr+t|g}nt	d|  ddt
|  t| ||S tt| dt| dddgS )Nc                 S   s   g | ]}t |jqS r$   )r*   r/   rk   r$   r$   r%   rn      s    z&_create_read_items.<locals>.<listcomp>z Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   rL   )rd   r   r   rq   r   ro   r    rf   r&   
ValueErrorr6   r   rP   r   )r2   rr   rh   rT   r$   r$   r%   _create_read_items   s*   



rt   )2typingr   r   r    !torch.distributed._shard.metadatar   'torch.distributed._shard.sharded_tensorr   0torch.distributed._shard.sharded_tensor.metadatar   torch.distributed._tensorr    torch.distributed._tensor._utilsr   r	   plannerr   r   r   r   r   r   r/   r   r   r   r   r   r   
reshardingr   r   r   str__annotations__rf   r&   r*   r1   r?   r@   rB   rE   rP   rR   r   ri   rp   rq   rt   r$   r$   r$   r%   <module>   s^      		



="