o hwR@sddlZddlZddlZddlZddlZddlmZddlmZddl Z ddl m Z ddl Z ddl ZddlmZdZdhddZd d Zd d ZiZde jjejjfed <de ejj fed<dddejjfed<e jjejjfed<e jjejjfed<e jjejjfed<e ejj fed<e ejjfed<ddddfed<ddddfed<e jjddfed<e ddfed <d!dd"dfed#<d$dd%dfed&<e jjd'dfed(<e jjd)dfed*<e jjd+dfed,<e d-dfed.<e d/dfed0<d1dd2dfed3<d4dd5dfed6<iZd7d8ged<d7d8ged<d7d8ged<d7ged<d7ged<d9ged<d7d8ged:<d;ged<d<d?d@ged(<d?d@ged,<d?d@ged*<dAged#<d+r/momentum_pytorchadam paged_adamw paged_adamlion paged_lioncCr&r'r*r-rrrr/4r0cCtjj|ddddSNr(r)F block_wise)bnbr+r,r-rrrr/5momentumcCr&r'r r+RMSpropr-rrrr/8r0cCr7r8)r;r+r?r-rrrr/9r<rmspropcCtjj|ddSNFr9r;r+Adam8bitr-rrrr/;r0adam8bitcCrArBr;r+Lion8bitr-rrrr/<r0lion8bitcCr&r'r*r-rrrr/>r0cCr7r8r;r+SGD8bitr-rrrr/?r< momentum8bitcCr&r'r>r-rrrr/Br0cCr7r8r;r+ RMSprop8bitr-rrrr/Cr< rmsprop8bitcCrANTr9rCr-rrrr/Fr0adam8bit_blockwisecCrArO)r;r+PagedAdamW8bitr-rrrr/Gr0paged_adamw8bit_blockwisecCrArO)r;r+ PagedAdam8bitr-rrrr/Hr0paged_adam8bit_blockwisecCrArOrFr-rrrr/Ir0lion8bit_blockwisecCrArO)r;r+ PagedLion8bitr-rrrr/Jr0paged_lion8bit_blockwisecCr&r'r*r-rrrr/Lr0cCr7Nr(r)Tr9rIr-rrrr/Mr<momentum8bit_blockwisecCr&r'r>r-rrrr/Pr0cCr7rXrLr-rrrr/Qr<rmsprop8bit_blockwise)exp_avgstate1) exp_avg_sqstate2)momentum_bufferr\lamb) square_avgr\)r[r\qmap1max1)r]r^qmap2max2lamb8bit)r[r\rbabsmax1)r]r^rdabsmax2)r_r\rbrc)r_r\rbrg)rar\rbrc)rar\rbrg) ri)r2r=r@r3r4r5r6cCg|]}dj|qSz!dim1_{}_dim2_{}_gtype_{}_optim_{}format.0valsrrr ortzdim1, dim2, gtype, optim_name)idsc Csh|tjkr |dvr t|dkr|dkrdStj||d|dd}|}|}t|d|g}t|d|g}|tjkrFd\}} n|tjkrPd\}} nd \}} t t D]} tj||d|dd } | |_ | |_ | | t |D]\} } tjj|j|| |j|| || d q}t|||| d d | t ddkr| dkrt}t|t|d~d}t|d|g}|tt|dt|t|||| d d t |D]\} } t|j|| |j|| || d dq|tjkr |j|j|_||jtj||j||dvr1|j|ddks1JqXdS)N)r=r@rlcudadevicedtype皙?rgư>h㈵>)rr(-C6?rr(rr ropt.pt)rrr)larsr` unorm_vecg)r bfloat16pytestskiprandnclonefloatstr2optimizersfloat32rangekgradstepstr2statenamesrrstaterwrr save state_dictrload_state_dictloadr#datatorzcopy_)dim1dim2gtype optim_namep1p2torch_optimizer bnb_optimizerrrigname1name2rrrrtest_optimizer32bitpsb           r)rjrirkcCrm)zdim1_{}_dim2_{}_gtype_{}rorqrrrrtruzdim1, dim2, gtypecCs|dkr |dkr dStj||d|dd}tj||d|dd}tj||d|dd}t|dk}d}d}d} d} tjjtjj|d d tjj |||g| }| }| }tj |||g| ||f| } |tj kr}d \} } nd \} } t d D]P}tj||d|ddd}tj||d|ddd}tj||d|ddd}||_||_||_| | j|djtjksJ| j|djtjksJqdS)Nrlcpurxr{r)+?r:0yE> optim_bitsr|r~2rwr\r^)r r rand_liker;r+GlobalOptimManager get_instance initializeoverride_configregister_parametersrwAdamrrrrrrzuint8)rrrrrp3maskbeta1beta2lrepsadam2rrrg1g2g3rrrtest_global_configsF     r)rErHrKrNrPrUrYrZcCrmrnrorqrrrrt c Cs|tjkr |dvr t|dkr|dkrdStj||d|dd}|}|}d}t|d|g}t|d|g}|tjkrLd\} } d \} } n|tjkrZd\} } d \} } nd\} } d \} } g} g}t d D]}tj||d|dd }||_ ||_ | | t ||| | d dg}t |D]Q\}}}}d|vrtj|j|||j|||j|||d}ntj|j|||j|||j||d}tj|j|||| | ddk}||qt||}|t|d}|jtjkr|dksJ|dksJn|dks"J|dks+J| ||||ddkr|dkrtt ||D]\\}}}}}|}|j||}|j||}t}t|t|d~d}t|d|g}|tt|dt|tj !||j||tj !||j||d|vrtj|j|||j|||j|||d}ntj|j|||j|||j||d}tj !||tj|j|||| | ddk}|"dksJqPt ||| | d d|j#$||_#|%|j#tj !|$||tt ||D]\\}}}}}|j||%|j#q3qjdS)N)rPrUrlrwrxr{r)g~jth?r)r}r)rr(dr(rr blockwise)codeabsmaxA blocksize)rrrrg& .>ga2U0*#?g-C6Z?giUMu?ga2U0*S?rrr)&r rrrrrrrrrrrrrFdequantize_blockwiser dequantizer appendabsrzmeanr zipr rrrrrr#rrr rrr)rrrrrrrrrrrpatolprtolerrors relerrorsrrdequant_statesrrqmapmax_vals1 num_not_closeerrrelerrss1cpyraws1cpyrbrrrrtest_optimizer8bits                    rrjrcCrm)z&dim1_{}_dim2_{}_gtype_{}_optim_bits_{}rorqrrrrtszdim1, dim2, gtype, optim_bitsc Cs|dkr |dkr dStj||d|dd}d}d}d}d}|}|} tjj|g|||f||d } tjj| g|||f||d d } td } d } tdD]}| d7} tj||d|ddd|}|}|| _ t || | d \}}}| | |}||_ | | |dkrtj|| tjj| j|d| j| ddddtjj| j|d| j| ddddnU|dkrtjj|| dddtjj| j|d| j| ddddtjj| j|d| j| dddd| j|d| j| d| j|d| j| d|dd krJ|d krJt}t| t|d~ d} tjj| g|||f||d d } | tt|dqOdS)Nrlrrxr{r)rrr)rr)rpercentile_clippingrrrrwr(rjr\g-C6 ?rrr^rrr)r rrwrr;r+rzerosrrrrrrrrrrrr rrrrr)rrrrrrrrrradam1r gnorm_vecrrrr current_gnormclip_val gnorm_scalerrrrtest_adam_percentile_clippings            ri)rPrTrRrWcCrmrnrorqrrrrtrc Cs|dkr |dkr dStj||d|dd}t|d|g}tj||d|dd}||_ttD]}|tdkrAtjt}| q0tjt|} t dttd||} t ||| | dS)Nrlrwrxr{r(r) r rrrrrrw synchronizetimerr ) rrrrrrrrt0rparamsrrrtest_benchmark_blockwises       rrr;cCrm)z%dim1_{0}_gtype_{1}_optim_{2}_mode_{3}rorqrrrrtruzdim1, gtype, optim_name, modecsRtjjtjfddtdD}||}|}d}|dkr-t|d|}nt|d|}tj t dfdd }tj t d d }tj|d dd |}tjdd|d fd } t|D]/} t| || } | d krtj t } || } tjj| | | }||qitj t|t | dS)Ncsg|] }tjqSr)r nnLinear)rrrrrrrtsz/test_stream_optimizer_bench..rr rrlgЈArw)ryr)sizer)r r Sequential ModuleListrrrwr parametersemptyintrrsleeprrandintr functional cross_entropyrbackwardr)rrrmodelayers1 large_tensorr+ num_batchesbatcheslblsrrrout1loss1rrrtest_stream_optimizer_benchs2&       r)rrr)2ctypesrr!rr itertoolsros.pathrrr%rr bitsandbytesr;bitsandbytes.functionalrrrrr r#rr+rAdamW PagedAdamW PagedAdam PagedLionrrrrfloat16rroptimizer_nameslistvaluesnamesmark parametrizerrrrrrrrrrrrs                          E  -  } W