o hg@sjUddlmZmZmZmZddlZddlmmZ ddlm Z gZ ee e d<ejjGdddZdS))DictListOptionalTupleN)Tensor__all__c@seZdZ         ddeededeeefd ed ed ed ed ededefddZdede efddZ dee efddZ dS)_FunctionalAdamMbP?g?g+?:0yE>Fparamslrbetaseps weight_decayamsgradmaximizeforeachfused_allow_empty_param_listc Csd|ks td|d|kstd|d|dkr"dks,ntd|dd|dkr8dksBntd|dd|ksMtd ||||d|d|d |_||_||_||_| |_tjt tj t t tj ffi|_ t |dkr| std d |i|_dS) Nr zInvalid learning rate: zInvalid epsilon value: rg?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: )rrbeta1beta2rz%optimizer got an empty parameter listr ) ValueErrordefaultsrrrrtorchjitannotaterrstrstatelen param_group) selfr rrrrrrrrrr$]/var/www/html/ai/venv/lib/python3.10/site-packages/torch/distributed/optim/functional_adam.py__init__s0 $z_FunctionalAdam.__init__paramgradc Csjg}g}g}g}g}g}|dur||||||jvrQi|j|<|j|} td| d<tj|tjd| d<tj|tjd| d<|jrQtj|tjd| d<|j|} || d|| d|jrn|| d|| dt2tj |||||||j|j |j d|j d |j d |j d |j d |j |j ddd WddS1swYdS)zo Similar to step, but operates on a single parameter and optionally a gradient tensor. Nr step memory_formatexp_avg exp_avg_sqmax_exp_avg_sqrrrrr rrrrrrrrr grad_scale found_inf)appendr rtensor zeros_likepreserve_formatrno_gradFadamrrrr) r#r'r(params_with_gradgradsexp_avgs exp_avg_sqsmax_exp_avg_sqs state_stepsr r$r$r% step_paramAsd         "z_FunctionalAdam.step_param gradientsc Cs|jd}g}g}g}g}g}g}t|t|kr,tddt|ddt|t|jd|D]m\} } | dur|| || | |jvr}i|j| <|j| } td| d<tj| tj d| d <tj| tj d| d <|j r}tj| tj d| d <|j| } || d || d |j r|| d || dq4t 2t j |||||||j |j|jd |jd |jd|jd|jd|j|jdddWddS1swYdS)Nr zEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: r r)r*r,r-r.rrrrrr/)r"r!rzipr2r rr3r4r5rr6r7r8rrrr) r#r@r r9r:r;r<r=r>r'gradientr r$r$r%r){sz           "z_FunctionalAdam.stepN) r r r r FFFFF) __name__ __module__ __qualname__rrfloatrboolr&rr?r)r$r$r$r%rsB    ,:r)typingrrrrrtorch.optim._functionaloptim _functionalr7rrr__annotations__rscriptrr$r$r$r%s