o h9 @s0UddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z m Z mZddlZddlmZddlmZddlmZmZmZmZmZmZmZddlmZmZmZm Z m!Z!m"Z"m#Z#dd l$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,dd l-m.Z.dd l/m0Z0ddl1m2Z2e3e4Z5ej6e7d<gdZ8GdddeZ9GdddeZ:de e;e fde!defddZdefddZ? d1de ed!e>de e ee!ffd"d#Z@de"de!fd$d%ZAd&ed'ede>fd(d)ZBd*ejCd+ede>fd,d-ZDd.e ede!de>fd/d0ZEdS)2N)ChainMap)reduce)ListTupleDictAnyUnioncast)narrow_tensor_by_index)DTensor) SavePlanner LoadPlannerSavePlanLoadPlanReadItem WriteItem WriteItemType)BytesStorageMetadataChunkStorageMetadataTensorStorageMetadata MetadataIndexMetadataSTATE_DICT_TYPE STORAGE_TYPES)_create_read_items_create_write_items"_create_default_metadata_only_plan)FLATTEN_MAPPINGflatten_state_dict)_flatten_sharded_tensors) dedup_tensors)find_state_dict_object) set_elementlogger)DefaultSavePlannerDefaultLoadPlannercreate_default_local_load_plancreate_default_global_load_plancreate_default_local_save_plancreate_default_global_save_planc @seZdZUeed<   ddedededdfdd Zd ed eddfd d Zde fddZ de e de e e e ffddZde de fddZdedeejejffddZdedefddZdedefddZdS) r$mappingsTrflatten_sharded_tensorsdedup_replicated_tensorsreturnNcCs||_||_||_i|_dSN)rr+r,r*)selfrr+r,r0b/var/www/html/ai/venv/lib/python3.10/site-packages/torch/distributed/checkpoint/default_planner.py__init__Gs zDefaultSavePlanner.__init__ state_dictis_coordinatorcCs2|jr t|\}|_|jrt|}||_||_dSr.)rr*r+rr3r4)r/r3r4r0r0r1set_up_plannerRs  z!DefaultSavePlanner.set_up_plannercCs0t|j|j}|jrtj||jd}||_|jS)N planner_data)r(r3r4r dataclassesreplacer*plan)r/r:r0r0r1create_local_plan\sz$DefaultSavePlanner.create_local_plan all_planscCsr|jrt|}t|\}}|jr$dd|D}tt|}tj||d}t||s-t d||_ ||_ |j |j fS)NcSsg|]}|jqSr0r6).0pr0r0r1 ssz9DefaultSavePlanner.create_global_plan..r6zFailed to validate global plan) r,r r)rdictrr8r9_validate_global_plan ValueError global_planmetadata)r/r<rCrDplanner_data_dictmerged_mappingsr0r0r1create_global_planfs    z%DefaultSavePlanner.create_global_plannew_plancCs ||_|Sr.)r:r/rHr0r0r1 finish_planszDefaultSavePlanner.finish_plan write_itemcC||j}|||Sr.) lookup_objectindextransform_object)r/rKobjectr0r0r1 resolve_datas  zDefaultSavePlanner.resolve_datarNcC t|j|Szo This is an extension from the planner interface to make it easy to extend the default planner r!r3r/rNr0r0r1rM z DefaultSavePlanner.lookup_objectrPcCs(|jtjkrt}t|||}|SrS)typerBYTE_IOioBytesIOtorchsave)r/rKrPbytesr0r0r1rOs  z#DefaultSavePlanner.transform_object)TTT)__name__ __module__ __qualname__r__annotations__boolr2rr5rr;rrrrGrJrrr[TensorrYrZrQrrrMrOr0r0r0r1r$DsF        r$c@seZdZUdZeed<eed<  d&dededdfd d Zd ed e d eddfddZ de fddZ de e de e fddZde de fddZdedejddfddZdefddZdedejddfdd Zd!edejfd"d#Zdedejfd$d%ZdS)'r%z DefaultLoadPlanner that adds multiple features on top of LoadPlanner. In particular it adds the following: flatten_state_dict: Handle state_dict with nested dicts flatten_sharded_tensors: For FSDP in 2D parallel mode original_state_dictr*Trr+r-NcCs||_||_i|_i|_dSr.)rr+rdr*)r/rr+r0r0r1r2s zDefaultLoadPlanner.__init__r3rDr4cCs>||_|jr t|}|jrt|\}|_||_||_||_dSr.)rdr+rrr*r3rDr4)r/r3rDr4r0r0r1r5s z!DefaultLoadPlanner.set_up_plannercCst|j|jSr.)r&r3rD)r/r0r0r1r;sz$DefaultLoadPlanner.create_local_planrCcCst|Sr.)r')r/rCr0r0r1rGsz%DefaultLoadPlanner.create_global_planrHcCs|Sr.r0rIr0r0r1rJzDefaultLoadPlanner.finish_plan read_itemvaluecCs@|jrt|j|j|jjt|dSt||j|jj<dSr.) rr"rdr* dest_indexfqnr[loadr3)r/rfrgr0r0r1 load_bytess zDefaultLoadPlanner.load_bytescCrLr.) lookup_tensorrhtransform_tensorr/rftensorr0r0r1resolve_tensors  z!DefaultLoadPlanner.resolve_tensorrocCsdSr.r0rnr0r0r1 commit_tensorrez DefaultLoadPlanner.commit_tensorrNcCrRrSrTrUr0r0r1rlrVz DefaultLoadPlanner.lookup_tensorcCst||j|jSrS)r dest_offsetslengthsrnr0r0r1rms z#DefaultLoadPlanner.transform_tensor)TT)r^r_r`__doc__rrarrbr2rr5rr;rrGrJrrYrZrkrpr[rcrqrrlrmr0r0r0r1r%s<      r%r3rDr-cCsdg} |D]&\}}|j|}t|tr%|jdur$|t|||7}q|t|||7}qt|Sr.)itemsstate_dict_metadata isinstancer device_meshget_coordinaterr)r3rDrequestsriobjmdr0r0r1r&s  r&r<cCs|S)z Create global load plan used by DefaultLoadPlanner. The default load behavior involved no global coordination and this function currently doesn't change the local plans. r0)r<r0r0r1r' s r'r4cCsdg}|D]'\}}t|tr|jdur|t||7}qt|tjs&|r-|t||7}qt|S)a Create the ``SavePlan`` used by DefaultSavePlanner. On non-coordinator ranks, this function ignores tensors and non-tensor objects, only producing writes for ShardedTensor objects. On the coordinator rank, produce writes for all values. N) rurwr rxryrr[rcr)r3r4rzrir{r0r0r1r(s  r(Trewrite_index_hintsc Cs i}g}|D]}g}|jD]q}|jtjks|jj|vsJ|jtjkr0t||jj<||q |j dus7Jt t | |jjt |j j |j jgd}|}|ratj|jt|jd} tj|| d}|||j jdusvJd|jjd|j|j jq |tj||dq|t|fS)a6 Create the global plan and metadata used by DefaultSavePlanner. Metadata is produced by concatenating the metadata of all ``WriteItem`` from the supplied plans. The only global planning change is to update index hints in all ``MetadataIndex`` objects if ``rewrite_index_hints`` is True. N) propertiessizechunks)rNzZ Cannot create MD for tensor without bounds. FQN: z )ru)rurWrSHARDrNrirXrappend tensor_datar r setdefaultr~rr8r9lenrchunkr) r<r}r| new_plansr: new_itemsitem tensor_mdnew_item new_indexr0r0r1r).sJ         r)cCst|}t|g\}}|S)z^ Return the ``Metadata`` if DefaultSavePlanner was used to checkpoint ``state_dict``. )rr))r3r:_r|r0r0r1_create_default_local_metadataesrbox0box1cCsdt|j}t|D]&}|j||j||j|krdS|j||j||j|kr/dSq dS)zC Checks if two boxes overlap. Tuples are (offset, lengths) FT)roffsetsrangesizes)rrndimsir0r0r1_check_box_overlapns router_box_size inner_boxcCs`tt|D]'}|j|dkrdS|j|dkrdS|j||j|||kr-dSqdS)NrFT)rrrr)rrrr0r0r1_check_box_boundssrrCc Csd}|jD]j\}}t|trqt|jdkrqd}t|jD]:\}}t|j|s5t d||j|d}|t t j |jd7}|j|ddD]}t||rYt d|||d}qHq t t j |jd} || krqt d|| |d}q|S)NTrz~ key:%s has out of bounds chunk: tensor-size:%s chunk: %s Fz$key:%s has overlapping chunks: %s %szq key:%s invalid fill tensor-volume: %s chunks-volume: %s )rvrurwrrr enumeraterrr#warningroperatormulrr) rCrDall_goodkeyrg chunks_volume chunk_idxchunk0chunk1 tensor_volumer0r0r1rAs@   rA)T)Fr8rYloggingr collectionsr functoolsrtypingrrrrrr r[torch.distributed._shard._utilsr torch.distributed._tensorr $torch.distributed.checkpoint.plannerr r rrrrr%torch.distributed.checkpoint.metadatarrrrrrr,torch.distributed.checkpoint.planner_helpersrrr)torch.distributed.checkpoint._nested_dictrr2torch.distributed.checkpoint._sharded_tensor_utilsr+torch.distributed.checkpoint._dedup_tensorsr "torch.distributed.checkpoint.utilsr!&torch.distributed.checkpoint._traverser" getLogger__file__r#Loggerra__all__r$r%strr&r'rbr(r)rrSizerrAr0r0r0r1s      $$       XR      7