o
    hE                     @   sL  d dl Z d dlZd dl mZ d dlmZmZmZmZmZm	Z	 d dl
mZmZmZmZ ddlmZ d dlmZ d dlZd dlZd dlmZmZmZmZmZ eeef Zeeeed	f f Zd
d Zde	ee  de	defddZdeeeed	f f defddZdededeg e f defddZ!dedededeee	e e	e ef fddZ"de	e de	e dedefdd Z#d!d" Z$deeeed	f f d#eded$ededefd%d&Z%d'd( Z&d#ededdfd)d*Z'defd+d,Z(d-a)da*d.d/ Z+d0d1 Z,d2d3 Z-d4d5 Z.d6d7 Z/d8d9 Z0d:d; Z1d<d= Z2ed>d? Z3ed@dA Z4dBdC Z5dDdE Z6dS )F    N)Tensor)AnyCallableOptionalTupleUnionList)tree_flattentree_unflatten_broadcast_to_and_flattenTreeSpec   )	tree_map_)partial)_add_batch_dim_remove_batch_dim_vmap_decrement_nesting_vmap_increment_nestingis_batchedtensor.c                    s    dt   fdd}|S )Nzdtorch.func transforms don't yet support saved tensor hooks. Please open an issue with your use case.c                     s@   t jj  | i |W  d    S 1 sw   Y  d S N)torchautogradgraphdisable_saved_tensors_hooks)argskwargsfmessage K/var/www/html/ai/venv/lib/python3.10/site-packages/torch/_functorch/vmap.pyfn#   s   $z.doesnt_support_saved_tensors_hooks.<locals>.fn)	functoolswraps)r   r!   r   r   r    "doesnt_support_saved_tensors_hooks   s
   r$   flat_in_dims	flat_argsreturnc                    sZ   dd t | |D  t dkrtd r)t fdd D r)td  d d S )	Nc                 S   s"   g | ]\}}|d ur| |qS r   )size.0in_dimargr   r   r    
<listcomp>.   s    z0_validate_and_get_batch_size.<locals>.<listcomp>r   z/vmap: Expected at least one Tensor to vmap overc                 3   s    | ]	}| d  kV  qdS )r   Nr   )r*   r(   batch_sizesr   r    	<genexpr>2   s    z/_validate_and_get_batch_size.<locals>.<genexpr>zTvmap: Expected all tensors to have the same size in the mapped dimension, got sizes z for the mapped dimension)ziplen
ValueErrorany)r%   r&   r   r.   r    _validate_and_get_batch_size+   s   r5   batched_outputsc                 C   s   t | tr	t| S dS )Nr   )
isinstancetupler2   )r6   r   r   r    _num_outputs9   s   
r9   valuenum_elementserror_message_lambdac                 C   s.   t | ts
| f| S t| |krt| | S r   )r7   r8   r2   r3   )r:   r;   r<   r   r   r    	_as_tupleB   s
   


r=   in_dimsr   funcc           	      C   s  t | tst | tstdt| d|  dt|  dt|dkr,tdt| dt|\}}t| |}|d u rRtdt| d|  dt| d  d	| d	t	t
||D ]~\}\}}t |tsx|d urxtdt| d|  d
| dt |trt |tstdt| d|  d
| dt| d	|d ur||  k s|| krtdt| d|  d
| d|  d|  d|  d|d ur|dk r||  ||< qYt|||||fS )Nvmap(z
, in_dims=zv, ...)(<inputs>): expected `in_dims` to be int or a (potentially nested) tuple matching the structure of inputs, got: .r   z)(<inputs>): got no inputs. Maybe you forgot to add inputs, or you are trying to vmap over a function with no inputs. The latter is unsupported.zb, ...)(<inputs>): in_dims is not compatible with the structure of `inputs`. in_dims has structure r   z but inputs has structure z, ...)(<inputs>): Got in_dim=zE for an input but in_dim must be either an integer dimension or None.z' for an input but the input is of type zT. We cannot vmap over non-Tensor arguments, please use None as the respective in_dimz> for some input, but that input is a Tensor of dimensionality z  so expected in_dim to satisfy -z <= in_dim < )r7   intr8   r3   	_get_nametyper2   r	   r   	enumerater1   r   dimr5   )	r>   r   r?   r&   	args_specr%   ir,   r+   r   r   r    _process_batched_inputsJ   sf   

"rI   
vmap_levelc                    s"    fddt | |D }t||S )Nc                    s(   g | ]\}}|d u r|nt || qS r   )r   r)   rJ   r   r    r-      s
    
z*_create_batched_inputs.<locals>.<listcomp>)r1   r
   )r%   r&   rJ   rG   batched_inputsr   rK   r    _create_batched_inputs|   s   

rM   c                 C   sp   |d u rt |tjrt|rtd|  d|  d|S t |tjs1td|  d|  dt| dt||||S )Nr@   z	, ...): `z5` can not return a BatchedTensor when out_dim is Nonez%` must only return Tensors, got type z2. Did you mean to set out_dim= to None for output?)r7   r   r   r   r3   rD   r   )namebatched_outputrJ   
batch_sizeout_dimr   r   r    _maybe_remove_batch_dim   s   
rR   out_dimsrP   c           	         s   t | \}fdd}t| tjr7ttrg}n&ttr+tdkr+}nd u r3g}n|  nt}|d u rC|   fddt||D }t	|S )Nc                
      s.   t dt  d dtd  d d	)Nr@   , ..., out_dims=z`)(<inputs>): out_dims is not compatible with the structure of `outputs`. out_dims has structure r   z but outputs has structure rA   )r3   rC   r	   r   )r?   rS   output_specr   r    incompatible_error   s   
z+_unwrap_batched.<locals>.incompatible_errorr   c                    s$   g | ]\}}t t| |qS r   )rR   rC   )r*   rO   rQ   )rP   r?   rJ   r   r    r-      s    z#_unwrap_batched.<locals>.<listcomp>)
r	   r7   r   r   rB   r8   r2   r   r1   r
   )	r6   rS   rJ   rP   r?   flat_batched_outputsrV   flat_out_dimsflat_outputsr   )rP   r?   rS   rU   rJ   r    _unwrap_batched   s"   


rZ   c                 C   s4   t | trd S | d u rd S tdt| d| d)Nr@   rT   z): `out_dims` must be an int, None or a python collection of ints representing where in the outputs the vmapped dimension should appear.)r7   rB   r3   rC   )xr?   rS   r   r   r    _check_int_or_none   s   
r\   c                 C   s&   t | trd S ttt|| d|  d S )N)r?   rS   )r7   rB   r   r   r\   )rS   r?   r   r   r    $_check_out_dims_is_int_or_int_pytree   s   
r]   c                 C   s   t | dr| jS t| S )N__name__)hasattrr^   repr)r?   r   r   r    rC      s   
rC   Fc                     s   t rd S da tjdddkrnd S tjdddaddlm	   fd	d
} | tj
jjj | tj
jjj | tj
jjj | tj
jjj | tj
jjj | tj
jjj | tj
jjj | tj
jjj d S )NTPYTORCH_JIT1atenIMPLFuncTorchBatchedr   decomposition_tablec                    s*   |  v rt |  |   d S td|  )Nz!could not find decomposition for )VMAP_DECOMPOSITIONS_LIBimplRuntimeError)decomprf   r   r    #_register_python_decomposition_vmap   s   zElazy_load_decompositions.<locals>._register_python_decomposition_vmap)DECOMPOSITIONS_LOADEDosenvirongetr   libraryLibraryrh   torch._decomprg   opsrc   mse_loss_backwarddefaultsmooth_l1_loss_backwardhuber_loss_backwardnll_loss_forwardnll_loss2d_forwardnll_loss_backwardnll_loss2d_backwardaddr)rl   r   rf   r    lazy_load_decompositions   s    r~   c                 O   sp   t   t||  t||| \}}}	}
|d ur*t|	|||}t| |||
||fi |S t| |||	|
||fi |S r   )r~   r]   rI   _get_chunked_inputs_chunked_vmap
_flat_vmap)r?   r>   rS   
randomness
chunk_sizer   r   rP   r%   r&   rG   chunks_flat_argsr   r   r    	vmap_impl   s   
r   c                 C   s4   | |  }}|g| }| | }|dkr| | |S )Nr   )append)total_elemsr   n_chunkschunk_sizes	remainderr   r   r    get_chunk_sizes  s   

r   c                    sN   |f |d urt ||}tt| t fddt| |D }t| }|S )Nc                 3   s8    | ]\}}|d ur|j  |dn|gt  V  qd S N)rF   )tensor_splitr2   )r*   tr+   
split_idxsr   r    r0     s
    "
z&_get_chunked_inputs.<locals>.<genexpr>)r   r8   	itertools
accumulater1   )r&   r%   rP   r   r   flat_args_chunksr   r   r   r    r     s   
r   c                 C   sH   g }d }| D ]}t |\}}|| |d u r|}qtt| }||fS r   )r	   r   listr1   )chunks_output_flat_chunks_outputarg_specoutputflat_output	arg_specsflat_output_chunksr   r   r    _flatten_chunks_output(  s   
r   c                 C   sX   t | |}t|t|ksJ g }t|D ]\}}|tj|| |d d ||< q|S r   )r   r2   rE   r   r   cat)rS   r   r   rX   r   idxrQ   r   r   r    _concat_chunked_outputs9  s   

r   c                 K   s   g }|dkr
t  nd }|D ]&}	t||	}
|
dkrq|d ur#t | |t| |
||	|||fi | qt|\}}~t|||}t||S )Nsamer   )	r   get_rng_stater5   set_rng_stater   r   r   r   r
   )r?   r%   r   rG   rS   r   r   chunks_outputrsr&   rP   r   r   r   r   r   r    r   G  s&   


r   c                 C   s   | dvrt d|  d S )N)error	differentr   zLOnly allowed values for randomness are 'error', 'different', or 'same'. Got )rj   )r   r   r   r    _check_randomness_argq  s   r   c                 K   sH   t ||}zt||||}	| |	i |}
t|
|||| W t  S t  w r   )r   rM   rZ   r   )r?   rP   r%   r&   rG   rS   r   r   rJ   rL   r6   r   r   r    r   v  s   
r   c                    s    fdd}|S )Nc                     s@   t  }zt| |}|i |}t||W t  S t  w r   )r   wrap_batchedunwrap_batchedr   )r   r   rJ   rL   r6   rP   r?   r>   r   r   r    inner  s   

zrestore_vmap.<locals>.innerr   )r?   r>   rP   r   r   r   r   r    restore_vmap  s   r   c                 C   s4   t | \}}t||}|d usJ t||||}|S r   )r	   r   rM   )r   bdimslevelr&   spec
flat_bdimsresultr   r   r    r     s
   
r   c                    sR   t | \}}t|dkr| dfS  fdd|D }t| \}}t||t||fS )Nr   r   c                    s0   g | ]}t |tjrtjj| n|d fqS r   )r7   r   r   _C
_functorchrZ   )r*   r,   r   r   r    r-     s
    z"unwrap_batched.<locals>.<listcomp>)r	   r2   r1   r
   )r   r   r&   r   r   r   r   r   r   r    r     s   
r   )7r   r"   r   typingr   r   r   r   r   r   torch.utils._pytreer	   r
   r   r   pytree_hacksr   r   rn   r   torch._C._functorchr   r   r   r   r   rB   	in_dims_t
out_dims_tr$   r5   r9   strr=   rI   rM   rR   rZ   r\   r]   rC   rm   rh   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    <module>   s    

""	
2
	
$
"	*

