o h@sUddlZddlZddlmZddlmZddlmZddlm Z m Z GdddeZ Gd d d eZ Gd d d eZ Gd ddeZdae e e ejed<dejfddZdS)N)comm)Function_get_device_index)ListOptionalc@$eZdZeddZeddZdS) BroadcastcGstdd|Ds Jddd|D}||_t|dkr tSt||_|d|_t||j}g}t |j ddD]\}}|sP|D] }| ||qFq>|j |tdd|DS) Ncs|] }|jjdkVqdScpuNdevicetype.0irR/var/www/html/ai/venv/lib/python3.10/site-packages/torch/nn/parallel/_functions.py z$Broadcast.forward..z2Broadcast function not implemented for CPU tensorscSg|]}t|dqSTrrxrrr z%Broadcast.forward..rrcSsg|] }|D]}|qqSrr)rtensorstrrrrs) all target_gpuslentuple num_inputs get_device input_devicerbroadcast_coalesced enumerateneeds_input_gradappendmark_non_differentiable)ctxr!inputsoutputsnon_differentiablesidxinput_requires_gradoutputrrrforward s$   zBroadcast.forwardcGsdtj|j|jg|RS)NN)ReduceAddCoalescedapplyr&r$r, grad_outputsrrrbackward szBroadcast.backwardN__name__ __module__ __qualname__ staticmethodr3r9rrrrr s  r c@r )r5csLfddtdtD|_fddtdtD}t||S)Ncsg|]}|qSrr%r)gradsrrr)sz.ReduceAddCoalesced.forward..rcsg|] }||qSrrrr@r$rrr+s)ranger"r!rreduce_add_coalesced)r, destinationr$r@grads_rrArr3's   zReduceAddCoalesced.forwardcGsdtj|jg|RS)NNN)r r6r!r7rrrr9/szReduceAddCoalesced.backwardNr:rrrrr5%s  r5c@r )Gathercstdd|Ds Jd|dkrd_nt|d}|_|_tdd|D_tdd|DrI|dkrItd d|D}td d_nd _tfd d|D_ t |jjS) Ncsr r rrrrrr8rz!Gather.forward..z/Gather function not implemented for CPU tensorsr Tcss|]}|VqdSr4r?rrrrrAcss|] }|dkVqdSrN)dimrrrrrrBrrcss|]}|dVqdS)rN)viewrKrrrrCszvWas asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.Fc3s|] }|jVqdSr4)sizerJrr,rrrJr) r target_devicerrJr# input_gpuswarningswarnunsqueezed_scalar input_sizesrgather)r,rOrJr-rrNrr36s   zGather.forwardcCs6t|j|j|j|}|jrtdd|D}d|S)Ncss|]}|dVqdSrIr)rgrrrrQrHz"Gather.backward..rF)Scatterr6rPrTrJrSr#)r, grad_outputscattered_gradsrrrr9MszGather.backwardNr:rrrrrG4s  rGc@r )rWc Csdd|D}||_|jjdkr|nd|_d}tjr*|jdkr*dd|D}t ||||j|}|durjt |D]-\}}tj||tj } | ||| | Wdn1sdwYq<|S)NcSrrrrrrrrYrz#Scatter.forward..r cSsg|] }ttd|qS)cuda) _get_streamtorchr)rrrrrr_s)rJrrr%r&r]r[ is_availablerscatterr(current_stream wait_stream record_stream) r,r! chunk_sizesrJinputstreamsr.rr2 main_streamrrrr3Ws   zScatter.forwardcGs dddtj|j|jg|RfSr4)rGr6r&rJ)r,rXrrrr9js zScatter.backwardNr:rrrrrWUs  rW_streamsrcCsh|jdkrdStt|jd}|durdStdurdg|at|jdur/||jt|j<t|jS)zBGets a background stream for copying between CPU and target devicer N)rgetattrr]rg device_countindexStream)r device_modrrrr\rs  r\)rQr]rtorch.autogradr torch._utilsrtypingrrr r5rGrWrgrk__annotations__rr\rrrrs    !