o hN+@snddlZddlZddlZddlZddlZddlZddlZddlmZddl Z ddl m Z ddl mZmZmZddlmZddlmZmZddlmZmZd d ZiZer iZejejejfed <ejejfed <ej ej!fed <ej"ej#ej$fed<ej%ej&fed<iZ'ej(ej)fe'd <ej*ej+fe'd <ej,ej-fe'd <ej.ej/fe'd<ej(ej)fe'd<ej*ej+fe'd<iZ0ej1ej2ej3fe0d <ej4ej5fe0d <ej6ej7fe0d <ej8ej9ej:fe0d<ej;ejGdddZ?iZ@de@ejA<de@ejB<de@ejC<de@ejD<de@ejE<ejAejFddddddZGdd d!ZHdd#d$ZIdd%d&ZJdd'd(ZKdd)d*ZLdd,d-ZMdd/d0ZNdd2d3ZOdd5d6ZPdd7d8ZQd9d:ZRd;d<ZSd=ed>ejTfd?d@ZUdAdBZVdCdDZWddEdFZX ddHdIZY G    ddJdKZZdd=edNedOe[d>efdPdQZ\GdRdSdSZ]dd=edUedVedNed>ef dWdXZ^     T dd=edYe]dVedUedNedZe_d>efd[d\Z`dd^d_Zadd=edVedNefd`daZbdd=edVedNefdbdcZcdd=edVedNed>efdedfZddd=edYe]dVedNedZe_d>ef dgdhZedd=edYe]dVedNedZe_d>ef didjZfdd=edYe]dVedNedZe_d>ef dkdlZgdd=edUedNed>efdmdnZh    dd=edoeeefdVedUedNed>ef dpdqZidd=edUedNed>efdrdsZjdd=edUedNed>efdtduZk  v v w  v ddxeldyedzed{ed|e[d}e[d~e_de[dede[de[de[dede[d>dfddZm v w  vddxeldyedzed{eded|e[de[d}e[d~e_de[dedededededede[de[dede[d>df*ddZn v w ddxeldyedzed{eded|e[de[d}e[d~e_de[dedededede[de[d>df"ddZo 1ddeded~e_de_fddZpdedededefddZqejEfddZr    dd=ededNefddZs   dd=ededNefddZt   dd=ededNefddZuddejvfddZw    dddZx vdddZyGdddZzGdddZ{GdddZ|ddZ}ddZ~ejfddZ vdddZdddZdddZdddZdZdddńZdddDŽZejdfddɄZejfdd˄Zdd̈́ZddτZdS)N)norm)reduce)TupleAnyDict)Tensor)pack_dict_to_tensorunpack_tensor_to_dict)COMPILED_WITH_CUDAlibcCsttj|dSNr )roperatormul)iterablerM/var/www/html/ai/venv/lib/python3.10/site-packages/bitsandbytes/functional.pyprodsradammomentumrmsproplionadagradlamblarsc@s6eZdZdZddZddZeddZd d d ZdS) GlobalPageManagerNcCtdNzCall get_instance() instead RuntimeErrorselfrrr__init__dzGlobalPageManager.__init__cCs g|_dSN) paged_tensorsr rrr initializeg zGlobalPageManager.initializecC&|jdur|||_|j|jSr$ _instance__new__r&clsrrr get_instancej   zGlobalPageManager.get_instanceFcCs$|jdddD]}t||qdS)N)r%prefetch_tensor)r!to_cputrrr prefetch_allqs zGlobalPageManager.prefetch_allF) __name__ __module__ __qualname__r*r"r& classmethodr.r4rrrrras rc@s4eZdZdZddZddZeddZdd ZdS) CUBLAS_ContextNcCrrrr rrrr"}r#zCUBLAS_Context.__init__cCs i|_dSr$)contextr rrrr&r'zCUBLAS_Context.initializecCr(r$r)r,rrrr.r/zCUBLAS_Context.get_instancecCsP|j|jvr"tj}tj|tt |j|j<tj||j|jSr$) indexr;torchcudacurrent_device set_devicectc_void_pr get_context)r!device prev_devicerrrrCs     zCUBLAS_Context.get_context) r6r7r8r*r"r&r9r.rCrrrrr:zs  r:c@s,eZdZdZddZddZeddZdS)Cusparse_ContextNcCrrrr rrrr"r#zCusparse_Context.__init__cCstt|_dSr$)rArBr get_cusparser;r rrrr&szCusparse_Context.initializecCr(r$r)r,rrrr.r/zCusparse_Context.get_instance)r6r7r8r*r"r&r9r.rrrrrFs rFr>)r<dtyperDcGspt|t|}tt|}t|ttj}t j j ||d}t j ||t|d|}d|_|j|_|S)Nshape)rKcountT) dtype2bytesrr cget_managed_ptrrAc_size_tcastPOINTERc_intnp ctypeslibas_arrayr= frombufferviewis_pagedr< page_deviceid)rKrDrM num_bytescuda_ptrc_ptr new_arrayoutrrr get_pagedsraFcCsR|jsJd|r d}n|j}t|j|}tt|t |t |dS)Nz%Only paged tensors can be prefetched!r0) rZr[rOrKnumelr cprefetchget_ptrrArQc_int32)Ar2deviceidr\rrrr1s "r1TcCsd}|jtjkrttd|dd}t|}n|jtjkr-ttd|dd}t|}|dur8t d|t|dd}|rN|rNt ||durNt ||t |t ||t | |jsd|jrktjdSdS)Nc_fp32_uint8zFunction not implemented: is_managedF)rKr=float32getattrr rAc_floatuint8c_uint8NotImplementedErrorr1rdc_int64rbrZr> synchronize) func_namerfBvalueprefetchfunccvaluerkrrrelementwise_funcs        rzcCstd|d|dS)Nfillrz)rfrvrDrwrrrr{r{cCstd|dddS)Narangerr|)rfrDrrrr~r}r~cCstd||ddS)N_mulrr|)rfrurDrrrrr}rcCs|rdnd}d|}|s|dkr|sd|nd|d}t|d|}d|}|dkr/|S|d}t|d|dg|||dS) NrIrr ?r)r=linspacerbrtolist)signed total_bitsadd_zerosign total_valuesvaluesgaplrrrcreate_linear_maps    0r+ew?cCs|r)tt|dddd}dgd}tt|dddd }n&tt|dddd}dgd}tt|dddd }|||}t|}|j}||}| dksmJ|S) Ng? r0rrr) rppfr=rrrsortrmaxrb)offsetuse_extra_valuev1v2v3vrrrrcreate_normal_maps  $  "    rcCs|}|}|rdnd}||||ksJg}g}ttd|| d||dD] \} } |d| q)g} ttjddg|d} d|d} td|D]I}| D]D}|dkrZdnd}tt|D]\} }||d| d 7}qb|dkr~|d| }n |d|| d }| ||r| | qRqNt| d|ksJ| |dkrdt| }t|D]} | dq| t | }|| }|S)Nr rrI)repeatrr) enumeraterangeappendlist itertoolsproductlenrr=rr)r exponent_bitsprecision_bitsrephas_signevaluespvaluesivalrlstbiasevalue bit_patternrvpvalrcoderrrcreate_fp8_mapsD *        rc Csg}||rdnd}d||d}t|D]R}t|r%d|||dn d|||dd}tdd|}|dd|ddd} |d|d || 7}|rh|d|d | | 7}q|dkrtdd|d}|dd|ddd} |d|d || 7}|r|d|d | | 7}|d|d t|d|ksJd t|} t| D]}|dq|t|S) a+ Creates the dynamic quantiztion map. The dynamic data type is made up of a dynamic exponent and fraction. As the exponent increase from 0 to -7 the number of bits available for the fraction shrinks. This is a generalization of the dynamic type where a certain number of the bits and be reserved for the linear quantization region (the fraction). n determines the maximum number of exponent bits. For more details see (8-Bit Approximations for Parallelism in Deep Learning)[https://arxiv.org/abs/1511.04561] r rIg?Nr0@ rrr) rintr=rrrrrr) rmax_exponent_bitsrdata non_sign_bitsadditional_itemsrfraction_items boundariesmeansrrrrcreate_dynamic_map1s2 4       rcCsnt|d|dd}|}|ddt|}t|D]}|dq|t|}||}|S)NrIr ) num_quantilesrr) estimate_quantilesrrrrrrabsr)rfrqrrrrrcreate_quantile_mapcs    rcCs8tjsdStj\}}|dkrdS|dkrdSdS)N col_turingrr col_ampere)r=r> is_availableget_device_capability)major_minorrrrget_special_format_strrsrcCsd}t}|D] }|durqt|dd}||jjdkp|M}|s'||jjq|s6tddd|Dt|dkrHtd d d|D|S) NTrZFr>zZAll input tensors need to be on the same GPU, but found some tensors to not be on a GPU: cSg|]}|j|jfqSrrMrD.0r3rrr zis_on_gpu..r zcInput tensors need to be on the same GPU, but found the following tensor and device combinations: cSrrrrrrrrr)setrmrDtypeaddr< TypeErrorr)tensorson_gpugpu_idsr3rZrrr is_on_gpu}s   rrfreturncCs|durdSt|jS)z Get the ctypes pointer from a PyTorch Tensor. Parameters ---------- A : torch.tensor The PyTorch tensor. Returns ------- ctypes.c_void_p N)rArBrdata_ptr)rfrrrrds rdcCstj}tj||Sr$)r=r>r?r@)rDrErrrpre_calls  rcCstj|dSr$)r=r>r@)rErrr post_callsrc Csnd|tjkrdndd|d|d|rdnd}tt|s2t|td|d |d |d |tt|S) N ctransform_r __to_r3nz"Transform function not supported:  to z for data type z and transpose=)r=int8hasattrr print ValueErrorrm)rKorderAorderOut transposenamerrrget_transform_funcs2  rrowc CsDtj}t|}|dkr|d}n |dkr|d|d}|d} ||f} |r5|} | }| } |ddd|f} |dks=|dkrF||||d| fS|d kr]d | d d } ||| f||d| fS|d kr|d | d d } d |dd }||| f||d| fS|dkrd | d d } d |d d }||| f||d| fStd|)NrIrr r0rcolrJcol32rrrrrzTo_order not supported: )r=zerosrrq) rMrKrDto_order from_orderr init_funcdimsrowscolsstatetmprrrget_transform_buffers6 rc s(|dur |j|f}n|d}|dur#t|d|j|j||d\}}n|d|f}t|j|||}|dtdkrJtd} td} n2|durjt} tfdd|D} t| | } t| } ntdd} td} t |j} || t |t || | ||fS)Nr rrIcsg|]}|qSrr)rrrLrrrsz$nvidia_transform..) rMrrKrDrrrArerr:r.rCrd) rfrrr`rrld new_staterxdim1dim2rptrrrLrnvidia_transforms.    r`?rr`rc CsB|dkrtd|d|dkrtd||dkr)|dkr)dd|}|dur7tjd tj|jd }t||gt|j}|jtjkr]t t |t |t |t |n#|jtjkrxt t |t |t |t |ntd |jt||dkrtd|}td d ||j}||}|S)a Estimates 256 equidistant quantiles on the input tensor eCDF. Uses SRAM-Quantiles algorithm to quickly estimate 256 equidistant quantiles via the eCDF of the input tensor `A`. This is a fast but approximate algorithm and the extreme quantiles close to 0 and 1 have high variance / large estimation errors. These large errors can be avoided by using the offset variable which trims the distribution. The default offset value of 1/512 ensures minimum entropy encoding -- it trims 1/512 = 0.2% from each side of the distrivution. An offset value of 0.01 to 0.02 usually has a much lower error but is not a minimum entropy encoding. Given an offset of 0.02 equidistance points in the range [0.02, 0.98] are used for the quantiles. Parameters ---------- A : torch.Tensor The input tensor. Any shape. out : torch.Tensor Tensor with the 256 estimated quantiles. offset : float The offset for the first and last quantile from 0 and 1. Default: 1/(2*num_quantiles) num_quantiles : int The number of equally spaced quantiles. Returns ------- torch.Tensor: The 256 quantiles in float32 datatype. rzQQuantile estimation needs at least 256 values in the Tensor, but Tensor had only z values.zgCurrently only a maximum of 256 equally spaced quantiles are supported, but the argument num_quantiles=rr rIN)rrJzNot supported data type r)rbrqr=rrlrDrrrKr cestimate_quantiles_fp32rdrArnrTfloat16cestimate_quantiles_fp16rroundrlongto)rfr`rrrDstepidxrrrrs$     * * rc@sreZdZdZdZddeDZgdZdddZd d Ze d e e e fd e jd dfddZdddZddZdS) QuantStatezVcontainer for quantization state components to work with Params4bit and similar clasesfp4nf4cCsg|]}d|qS)bitsandbytes__r)rxrrrr>szQuantState.) absmax quant_map nested_absmaxnested_quant_map quant_state quant_type blocksizerKrMnested_blocksize nested_dtype nested_offsetNc Cs>||_||_||_||_||_||_||_||_|du|_dSr$) rrMrrKrrrstate2nested) r!rrMrrrrKrrrrrr"BszQuantState.__init__cCsR|jr|j|j|j|j|j|jg|jg}||S|j|j|j|jd|jg}||S)a$ ensures compatibility with older quant state scheme with nested lists. assumes the following layout: state = [qabsmax, input_shape, A.dtype, blocksize, [offset, state2], quant_type] state2 = [absmax, input_shape, A.dtype, blocksize, None, quant_type] N)r rrMrKrrrr)r!r list_reprrrr __get_item__Ms "zQuantState.__get_item__qs_dictrDrc Cspdd|D}t|sd|vrtdt|dks'|ddd|jvr3td |jd |dt|dkrG|d}|t||d d |D}t|  |j s\Jd |vrt t|d|}||d ||d|d|tt |dd}nd\}}||d|d||d|d|tt |d|ddurt |dnd||d}|S)aW unpacks components of state_dict into QuantState where necessary, convert into strings, torch.dtype, ints, etc. qs_dict: based on state_dict, with only relevant keys, striped of prefixes. item with key `quant_state.bitsandbytes__[nf4/fp4]` may contain minor and non-tensor quant state items. cSs(g|]\}}d|vrt|tjr|qS)r isinstancer=rrkrrrrrfs(z(QuantState.from_dict..rzqs z(QuantState.from_dict..rrrrr)rrrrKNNrrrrKrMN)rrrrrKrMrr)itemsrrr)valid_qs_type_keysupdater poprkeysissubset valid_qs_keysr=tensorfloatr rmSize)r-r#rDqs_keyrrrrrr from_dictZs< $        zQuantState.from_dictFcCs|j|j|j|jt|jdt|jd}|j r6| |j j|j j|j j t|j jd|j d|s:|Sdd|D}dd|D}t||d|j<|S)z returns dict of tensors and strings to use in serialization via _save_to_state_dict() param: packed -- returns dict[str, torch.Tensor] for state_dict fit for safetensors saving ztorch.)rrrrrKrM)rrrrrcSs"i|] \}}t|tjr||qSrr$r&rrrr*"z&QuantState.as_dict..cSs"i|] \}}t|tjs||qSrr$r&rrrr*r8zquant_state.bitsandbytes__)rrrrstrrKstriptuplerMr r.rcloneritemr,r)r!packedr#qs_packed_dictnon_tensor_dictrrras_dicts* zQuantState.as_dictcCsN|j||_|jr%|j||_|jj||j_|jj||j_dSdSr$)rr r rrr)r!rDrrrr s z QuantState.to)NNNNNNNr5)r6r7r8__doc__valid_quant_typesr-r2r"r"r9rr9rr=rDr7rAr rrrrr;s   " 0 rrrc Cs<|durdtvrt|jtd<td}|dur8|}||}|||dkr*dnd7}tj|f|jtjd}|durDtj|tj d}|jj dkr|dvsPJt |}t |j} ||j}t||||g|jtjkrtt|t|t|t||t |nD|jtjkrtt|t|t|t||t |n&|jtjkrtt|t|t|t||t |ntd |jt|jn|}tt|t|t|t|t |t ||r|} || 8}t||d d \} } t| |||j| | d } || fSt||||jd } || fS)a Quantize tensor A in blocks of size 4096 values. Quantizes tensor A by dividing it into blocks of 4096 values. Then the absolute maximum value within these blocks is calculated for the non-linear quantization. Parameters ---------- A : torch.Tensor The input tensor. code : torch.Tensor The quantization map. absmax : torch.Tensor The absmax values. out : torch.Tensor The output tensor (8-bit). Returns ------- torch.Tensor: The 8-bit tensor. tuple(torch.Tensor, torch.Tensor): The quantization state to undo the quantization. Ndynamicrr rDrKrKcpurDr@?Blockwise quantization only supports 16/32-bit floats, but got F)rr )rrrrKrrrrrrK) name2qmaprr rDrbr=rrl zeros_likerorrArerrrKr cquantize_blockwise_fp32rdrTrcquantize_blockwise_fp16bfloat16cquantize_blockwise_bf16rrrHcquantize_blockwise_cpu_fp32 c_longlongmeanquantize_blockwiser)rfrrr`rr rblocks cblocksizerErqabsmaxrrrrrrZsF      0 0 0 4rZrrc Cs:|dus |dus J|dur#|dur#dtvrt|jtd<td}|dur0t|||tjd}|j}|jrLt |j|j }||j 7}|j tjkrL| }|dur[tj|j|j |jd}|jjdkrt|j}|j|j}|jdvr{td|jdt|||g|j tjkrtt|jt|t|t|t|jt|nN|j tjkrtt|jt|t|t|t|jt|n+|j tjkrtt|jt|t|t|t|jt|ntd |j t |j|S|j!}t"t|t|t|jt|t#|jt#||S) a[ Dequantizes blockwise quantized values. Dequantizes the tensor A with maximum absolute values absmax in blocks of size 4096. Parameters ---------- A : torch.Tensor The input 8-bit tensor. quant_state : QuantState Object with code, absmax and other quantization state components. absmax : torch.Tensor The absmax values. code : torch.Tensor The quantization map. out : torch.Tensor Dequantized output tensor (default: float32) Returns ------- torch.Tensor: Dequantized tensor (default: float32) NrErPrJrHrJrDrKrLrrMrNThe blockwise of J is not supported. Supported values: [2048, 4096, 1024, 512, 256, 128, 64]rO)$rQrr rDrr=rlrr dequantize_blockwiserrrKr4emptyrMrrrrrrr cdequantize_blockwise_fp32rdrArTrbrcdequantize_blockwise_fp16rUcdequantize_blockwise_bf16rrHcdequantize_blockwise_cpu_fp32rX)rfrrrr`rr rDrrrras@"     : : :  8rarNcCs|durd}d}|dkr gd}n(|dkrgd}n|dkr$gd}n|dkr:|d kr6gd ddd }ntd |durFtd |dt|}||}|dksZJ||S)Nr>r)rg 6GgfgTFٿgI4ҿg০ǿg Org__?g`\?g?g@g?g4?g`?g`v"?rr)rg?g @g(@g@g@rg@rgg g(ggggint4)rrrHrrIr rrr0iiaf4rN)rg|8geg:Kڞ׿gH2퓊cпg}Yu-ÿgQ #(DrgF?g`_?g 0E?gL_߹E?gƶ=?ga@?gкv -?rr0z94-bit AbnormalFloats currently only support blocksize 64.z Typename z not supported)rqrrrrbr )typenamerDrrrrr get_4bit_typeDs*     rpcCt|||||dSNr quantize_4bitrfrr`rcompress_statisticsrrr quantize_fp4wrwcCrqNrrsrurrr quantize_nf4zrxrzrc Cs|jjdkrtd|jj|dvrtd|d|}|j}|durA||}|||dkr3dnd7}tj|f|jtjd }|durUtj|dd dftj|jd }|d vs[Jt |j} t |||g|j tjkr|d krt tdt|t|t|t|t|nt tdt|t|t|t|t|n|j tjkr|d krt tdt|t|t|t|t|n_t tdt|t|t|t|t|nF|j tjkr|d krt tdt|t|t|t|t|n!t tdt|t|t|t|t|ntd|j t|jt||jd} |rV|} || 8}t|dd\} } ~t| ||j || || | d}||fSt|||j || |d}||fS)a Quantize tensor A in blocks of 4-bit values. Quantizes tensor A by dividing it into blocks which are independently quantized to FP4. Parameters ---------- A : torch.Tensor The input tensor. absmax : torch.Tensor The absmax values. out : torch.Tensor The output tensor (8-bit). blocksize : int The blocksize used in quantization. quant_type : str The 4-bit quantization data type {fp4, nf4} Returns ------- torch.Tensor: The 8-bit tensor with packed 4-bit values. tuple(torch.Tensor, torch.Size, torch.dtype, int): The quantization state to undo the quantization. r>z0Device type not supported for FP4 quantization: r4-bit quantization data type  is not implemented.Nrr rFrIrJrIrrO)rDr)r)rrMrKrrrrr)rrMrKrrr)rDrrqrbrMr=rrlrorrrKr cquantize_blockwise_fp32_fp4rdrArerTcquantize_blockwise_fp32_nf4rcquantize_blockwise_fp16_fp4cquantize_blockwise_fp16_nf4rUcquantize_blockwise_bf16_fp4cquantize_blockwise_bf16_nf4rrrprYrZr)rfrr`rrvrr input_shaper[rErrr]rrrrrrt}sN     22 22 22 rtcCrqrrdequantize_4bitrfrrr`rrrrdequantize_fp4rxrcCrqryrrrrrdequantize_nf4rxrc Cs|dvr td|d|dvrtd|d|dur2|dur$|dus&Jt||j|j||d}n|j}|jrNt|j|j}||j 7}|jt j krN| }|dur]t j |j|j|jd }|}t|j}t|||g|jt j kr|jd krttdt|t|t|t|jt|nttdt|t|t|t|jt|n|jt jkr|jd krttdt|t|t|t|jt|ncttdt|t|t|t|jt|nI|jt jkr,|jd krttdt|t|t|t|jt|n"ttdt|t|t|t|jt|ntd |jt |j|jd d krCdnd}|rL|!S|S)a Dequantizes FP4 blockwise quantized values. Dequantizes the tensor A with maximum absolute values absmax in blocks of size blocksize. Parameters ---------- A : torch.Tensor The input 8-bit tensor (packed 4-bit values). quant_state : QuantState object with quantisation stats, incl. absmax values, original tensor shape and original dtype. absmax : torch.Tensor The absmax values. out : torch.Tensor Dequantized output tensor. blocksize : int The blocksize used in quantization. quant_type : str The 4-bit quantization data type {fp4, nf4} Returns ------- torch.Tensor: Dequantized tensor. r^r_r`rr{r|N)rrMrKrrrJrrOrr TF)"rrqrrMrKrr rarrr=rlr4rbrDrbrrrr cdequantize_blockwise_fp32_fp4rdrArTrcdequantize_blockwise_fp32_nf4rcdequantize_blockwise_fp16_fp4cdequantize_blockwise_fp16_nf4rUcdequantize_blockwise_bf16_fp4cdequantize_blockwise_bf16_nf4rr3) rfrrr`rrrrD is_transposedrrrrsD    44  44 44 rcCsx|durdtvrt|jtd<td}||j}t|}|jtjkr,| }||}t |||}|||ffS)NrE) rQrr rDr=rrrKrlr4quantize_no_absmax)rfrr`rinprrrquantize s   rrcCs~|dus |dus J|dur)|dur)dtvrt|jtd<td}||j}|dur1||f}t||d|}||dS)NrEr r)rQrr rDdequantize_no_absmax)rfrrrr`rrr dequantize.s  rc Cs`t|j}|durtj|tjd}t||gtt|t|t|t | t ||S)a Quantizes input tensor to 8-bit. Quantizes the 32-bit input tensor `A` to the 8-bit output tensor `out` using the quantization map `code`. Parameters ---------- A : torch.Tensor The input tensor. code : torch.Tensor The quantization map. out : torch.Tensor, optional The output tensor. Needs to be of type byte. Returns ------- torch.Tensor: Quantized 8-bit tensor. NrG) rrDr=rRrorr cquantizerdrArTrbrrfrr`rErrrrBs  &rc Csbt|j}|durtj|tjd}t|||gtt|t|t|t | t ||S)a Dequantizes the 8-bit tensor to 32-bit. Dequantizes the 8-bit tensor `A` to the 32-bit tensor `out` via the quantization map `code`. Parameters ---------- A : torch.Tensor The 8-bit input tensor. code : torch.Tensor The quantization map. out : torch.Tensor The 32-bit output tensor. Returns ------- torch.Tensor: 32-bit output tensor. NrG) rrDr=rRrlrr cdequantizerdrArTrbrrrrrr_s &rrroptimizer_namegrstate1beta1epsr lrrbeta2 weight_decay gnorm_scale unorm_vec max_unormcCs>d}| dkrt|j}d}|jtjkrt|d}n.|jtjkr*t|d}n!|jtjkr?t t|dkr?t|d}n t d|jd|jt ||||| gt |j }|t|t|t|t|t| t| t|t|t| t|t| t|t|t| t|t|t|dS) a$ Performs an inplace optimizer update with one or two optimizer states. Universal optimizer update for 32-bit state and 32/16-bit gradients/weights. Parameters ---------- optimizer_name : str The name of the optimizer: {adam}. g : torch.Tensor Gradient tensor. p : torch.Tensor Parameter tensor. state1 : torch.Tensor Optimizer state 1. beta1 : float Optimizer beta1. eps : float Optimizer epsilon. weight_decay : float Weight decay. step : int Current optimizer step. lr : float The learning rate. state2 : torch.Tensor Optimizer state 2. beta2 : float Optimizer beta2. gnorm_scale : float The factor to rescale the gradient to the max clip value. unorm_vec : torch.Tensor The tensor for the update norm. max_unorm : float The maximum update norm relative to the weight norm. skip_zeros : bool Whether to skip zero-valued gradients or not (default: False). rNrr rrIAGradient+optimizer bit data type combination not supported: grad , optimizer )r=rrr4rKrlstr2optimizer32bitrrUrrrrrDrdrArnrec_boolrbr)rrrrrrr rrrrrrr skip_zeros param_norm optim_funcrErrroptimizer_update_32bit|s@8     rqmap1qmap2max1max2new_max1new_max2cCsd}|dkrt|j}t|j}t|||||| | | | ||g |jtjkr|jtj krt |dt |t |t |t |t |t |t |t |t |t |t |t | t | t | t | t | t |t |t |t |t |nk|jtjkr|jtj krt |dt |t |t |t |t |t |t |t |t |t |t |t | t | t | t | t | t |t |t |t |t |n td|jd|jt|dS)a Performs an inplace Adam update. Universal Adam update for 32/8-bit state and 32/16-bit gradients/weights. Uses AdamW formulation if weight decay > 0.0. Parameters ---------- optimizer_name : str The name of the optimizer. Choices {adam, momentum} g : torch.Tensor Gradient tensor. p : torch.Tensor Parameter tensor. state1 : torch.Tensor Adam state 1. state2 : torch.Tensor Adam state 2. beta1 : float Adam beta1. beta2 : float Adam beta2. eps : float Adam epsilon. weight_decay : float Weight decay. step : int Current optimizer step. lr : float The learning rate. qmap1 : torch.Tensor Quantization map for first Adam state. qmap2 : torch.Tensor Quantization map for second Adam state. max1 : torch.Tensor Max value for first Adam state update. max2 : torch.Tensor Max value for second Adam state update. new_max1 : torch.Tensor Max value for the next Adam update of the first state. new_max2 : torch.Tensor Max value for the next Adam update of the second state. gnorm_scale : float The factor to rescale the gradient to the max clip value. unorm_vec : torch.Tensor The tensor for the update norm. max_unorm : float The maximum update norm relative to the weight norm. rrr rrN)r=rrr4rrDrrKrlrostr2optimizer8bitrdrArnrerbrrr)rrrrrrrrr rrrrrrrrrrrrrErrroptimizer_update_8bitsrH      rabsmax1absmax2cCsxd}t|j}t||||| | | | g|jtjkr&|jtjkr&t|d}n:|jtjkr9|jtjkr9t|d}n'|jtj krT|jtjkrTt t|dkrTt|d}n t d|jd|jt |t||||| | | | gt|j}|t |t |t |t |t|t|t|t|t| t | t | t | t | t|t|t|t|t |dS)Nrr rrIrr)rrDrrKr=rlrostr2optimizer8bit_blockwiserrUrrrrdrArnrerrb)rrrrrrrrr rrrrrrrrrrErrroptimizer_update_8bit_blockwise^sH    rgrad gnorm_vec percentilec Cst|j}t||g|jtjkr&tt|t|t |t | n$|jtj krAt t|t|t |t | n td|jdt|t||d}t|\}}t||}d} ||kro||} ||| fS)aApplies percentile clipping grad: torch.Tensor The gradient tensor. gnorm_vec: torch.Tensor Vector of gradient norms. 100 elements expected. step: int The current optimiation steps (number of past gradient norms). zGradient type z not supported!dr)rrDrrKr=rlr cpercentile_clipping_g32rdrArerbrcpercentile_clipping_g16rrsqrtr) rrr rrE current_gnormvalsr clip_valuerrrrpercentile_clippings2      r histogramindex1index2sourcecCst|jdks J|jtjksJ|jtjksJ|jtjks!J|jtjks)J|jjdks1J|jjdks9J|jjdksAJ|jjdksIJt |jd}t | }t ||||gt t|t|t|t|||dS)NrIr>r)rrMrKr=rlint32rDrrArerbrr chistogram_scatter_add_2drd)rrrrmaxdim1rrrrhistogram_scatter_add_2ds(rc Cstjs tj|j|ks|j|kr td|jd|j|j}|j}|}|} d} t|dkr|t|dkr||sI| sI|jd|jdkrId} n|rZ| sZ|jd|jdkrZd} n|rk| rk|jd|jdkrkd} n|s{| r{|jd|jdkr{d} nt|dkrt|dkr|s| s|jd|jdkrd} n|r| s|jd|jdkrd} ny|r| r|jd|jdkrd} nh|s| r|jd|jdkrd} nWt|dkr#t|dkr#|s| s|jd|jdkrd} n8|r| s|jd|jdkrd} n'|r| r|jd|jdkrd} n|s#| r#|jd|jdkr#d} |durc|j} | sbt|dkrbt|dkrb| d|dkrb| d|dkrb|d|dkrb|d|dkrbd} nt|dkrt|dkr|s| s|d|df} n|r| r|d|df} n|r| s|d|df} n|s| r|d|df} nt|dkrt|dkr|s| s|d|d|df} n|r| r|d|d|df} ny|r| s|d|d|df} ng|s| r|d|d|df} nUt|dkrXt|dkrX|s#| s#|d|d|df} n5|r5| r5|d|d|df} n#|rG| sG|d|d|df} n|sX| rX|d|d|df} | sltd |d |d |d | d | S) Nz3Expected torch.int8 input tensors A and B, but got  and TrIr rFrz?Tensor dimensions incorrect for matrix mulitiplication: A x B:  x z with transpose for A x B: r() r=r>is_initializedinitrKrrMrr) rfrur` transposed_A transposed_B expected_typesAsBtAtBcorrectsoutrrr check_matmuls"" "            rrucCsht|j}|dur td||jdkrtd|j}|d}|j} |jr3t|j|j} | |j 7} |durat |jdkrRt j |jd|jd|f|j |jd}nt j |jd|f|j |jd}d} |d} |d} |d} |d}|jddd}t|||| |jgt| } t| } t| } t| } t|}t|}|j t jkr&|j t jkrt| | | t|t|t| t|jt|| ||t|j n]|j t jkrt| | | t|t|t| t|jt|| ||t|j n7|j t jkrt| | | t|t|t| t|jt|| ||t|j ntd |j td |j t||S) NzGstate cannot None. gem_4bit( ) requires the state from quantize_4bit( )r0zcDimensions of A are invalid. Must be a vector with the leading dimensions of "1", e.g. [1, 1, 2048]rrr sizerKrDrIz%Matmul not implemented for data type )rrDrrbrMrr rarrrr=rbrKrrrArerorr cgemm_4bit_inference_naive_fp16rdrrUcgemm_4bit_inference_naive_bf16rlcgemm_4bit_inference_naive_fp32rqr)rfrur`rrrrEBshapeboutrrmr'ldaldcldbrrr gemv_4bit2sN  (       @ @@rcCsvt|||||}|durtj|tj|jd}t|jdkr>t|jdkr>|jd|jdkr>|jd|jdkr>t|||S|j}|j}|rUt|dkrU|d|df}n|rht|dkrh|d|d|df}|ryt|dkry|d|df}n|rt|dkr|d|d|df}t|dkr4|d|jdkrd}n |d|jdkrd}t|jdkr|d|jdkrd}n)|d|jdkrd}n|d|jdkrd}n |d|jdkrd}t|dkr|d}||rdnd} nt|dkrt|dkr|d|d}|d} |d} |d} ||r,dnd} |d} nHt|dkr|t|dksDJ|d|dkrV|d|dks`t d|d |d}d}|d} |d}|d|d} | } |d} | } t |j}t |||gt|t|t|t| t|t| t|t|t|t| t| t|  |S) NrrrrIr FTzMOnly bsi,bso->io supported for tensor contractions, but dims for A x B were: r)rr=rrrDrrM batched_igemmstriderr:r.rCrr cigemmrArrerd)rfrur`rrrrrrrrr'rrrrrrigemmnsz(   $.*rcCst|jdkrt|jdkstd|jd|jt|||||}|dur0tj|tj|jd}|r=| d}d}nV| }|d|jdkrU| }| d}n>|d|jdkrgd }| d}n,|ddkrx| }| d}n|ddkr| }| d}n | }| d}|r| d}d}n8| }|d|jdkr| }| d}d}n|d|jdkr| d}d }n | }| d}d}|jd} |jd} |jd} |jd} | } |jd|jd}|jd|jd}|jd|jd}t |j}t|||gt|t|t|t| t| t| t|t|t|t|t|t| t|t|t|t| |S) Nrz@Expected 3-dimensional tensors for bmm, but got shapes A and B: rrr FrrIT)rrMrrr=rrrD is_contiguousr contiguousr:r.rCrr cbatched_igemmrArrerdc_longc_uint32)rfrur`rrrrsr num_batchrrr'rstrideAstrideBstrideCrrrrrsl           .* rc Cs|d}|d}t|} t|} | dksJd| dkr!|d} n | dkr-|d|d} |d} } tt|dksBJd||ddkrZ| dkrZtjd|df|jtjdS|ddkry| dkrytjt|dd|dg|jtjdS| dkr|durt|d|df||jdd \}}n| dkr|durt|d|d|df||jdd \}}| dksJd |jj d ksJ|jj d ksJ|j tj ksJ|j tj ksJ|j |ksJ|ddksJ|dd vsJ|ddksJ|d |d ks Jd|d||d}|j}tj |jt|j}t|}t|}t|}|d }t| d}|dkrIt| dddd}n t| dddd}t| d}t| } t| } t|}d}td}t|||g|dkr|tjkrt|| | |||||||| }n:t|| | |||||||| }n*|dkr|tjkrt|| | |||||||| }nt|| | |||||||| }|dkrtd|d|d|dd|||fd| | |f tdtj |||fS)NrrIz:Only two dimensional matrices are supported for argument Brr z(Input tensor dimensions need to be > 0: rFrrzlen(B.shape)==3 not supportedr>rrr0zNMatmullt only supports A @ B^T. Inner matrix dimensions do not match: A @ B = z @ rrrrrrzA: z, B: z, C: z; (lda, ldb, ldc): z ; (m, n, k): zcublasLt ran into an error!)rrrr=rbrDrr;rrrKrr>r@r:r.rCrdrArerrr cigemmlt_turing_32cigemmlt_turing_8cigemmlt_ampere_32cigemmlt_ampere_8r Exception)rfruSASBr`SoutrKshapeAshapeBdimsAdimsBrrrformatBrErptrAptrBptrCr'rrr has_error ptrRowScalerrrigemmlts  *          6 rc Cs|jtjksJ|dur|jtjksJ|d}t|dkr*|d|d|df}|dur8tj|tj|jd}|durHtj|dtj|jd}|durXtj|dtj|jd}|jd|jdksmJ|jd|j|jd|jdksJ|jd|jt |j} t |} t |} t |} t |} t |}t |}t |}t |d}t |d}t |||||||gt| | | | ||||| t| |S)Nrrr rIrJ vs )rKr=rrrrbrDrlrMrrdrArerr cdequant_mm_int32_fp16r)rfr row_stats col_statsr` new_row_stats new_col_statsr out_shaperErptrOut ptrRowStats ptrColStatsptrNewRowStatsptrNewColStatsptrBiasnumRowsnumColsrrr mm_dequant{sF   rc Csh|jtjksJ|j}|jd}t|jdkr"|jd|jd}n|jd}|dd}|ddd} |durFtj|ftj|d d }|durWtj|ftj|d d }|durm|d krmtj | |dftj |d }t |} t |} t |} t |} t |}t |}t|j}t||||gt| | | | t |||t||d kr|d|||fS) Nr0rrr rrrnrJgjr)rKr=rrDrMrrbrlfill_rrrdrArerrr cget_col_row_statsrnrcumsum_)rfrr  nnz_block_ptr thresholdrDrr col_tiles tiled_rowsrrr ptrNnzrowsrErrrget_colrow_absmaxsJ          rc@eZdZddZdS)COOSparseTensorcCs|jtjksJ|jtjksJ|jtjksJ||ks J||ks(J||ks0J||_||_||_||_||_ ||_ dSr$) rKr=rrrbrrnnzrowidxcolidxr)r!rrr"r#r$rrrrr"s zCOOSparseTensor.__init__Nr6r7r8r"rrrrr! r!c@r )CSRSparseTensorcCs|jtjksJ|jtjksJ|jtjksJ||ks J||ks(J||dks2J||_||_||_||_||_ ||_ dSr ) rKr=rrrbrrr"rowptrr$r)r!rrr"r(r$rrrrr" zCSRSparseTensor.__init__Nr%rrrrr'r&r'c@r )CSCSparseTensorcCs|jtjksJ|jtjksJ|jtjksJ||ks J||ks(J||dks2J||_||_||_||_||_ ||_ dSr ) rKr=rrrbrrr"colptrr#r)r!rrr"r+r#rrrrr"r)zCSCSparseTensor.__init__Nr%rrrrr*r&r*cCsztj|jdd\}}|dtj|jdftj|jjd}|j| | dd| dt |j|j |j||j|jSNT return_countsr rJr)r<srcdim)r=uniquer#add_rrrrDscatter_r rrr'rr"r$r)cooArcountsr(rrrcoo2csrs  r6cCst|j\}}|j|}|j|}tj|dd\}}|dtj|jdftj |jj d}|j | | dd|dt|j|j|j|||Sr,)r=rr$r#rr1r2rrrrDr3r rrr*rr")r4r col2rowidxr#r colvaluesr5r+rrrcoo2cscs    r9cCsLtj|ftj|d}tj|ftj|d}tj|f||d}t||||||S)NrJ)r=rrr!)rrr"rDrKr#r$rrrr coo_zeros,sr:cCs@|j}|jtjks J|jdksJt|j}|jd}t|jdkr.|jd|jd} n|jd} |dus;|durDt||d\}}} |durRtj |j|tj d}|dur`tj |j|tj d}d} t |} t |} t |}t |}t |}t |||||g|dkr| d }|dkrt|jd|jd| d |} t | j}t | j}t | j}t | }t| || ||||||t|t| t| t| j\}}|| _| j|| _| j|| _n3t| || ||ddddtdt| t| nt| || ||ddddt|t| t| t|||||| fS) Nr>r0rrr )rrFr)rDrKr=halfrrrMrrrrrdrr=r:r#r$rr cdouble_rowcol_quantrArnrerr)rfr rout_colout_rowrrDrErr nnz_row_ptr coo_tensorrrr ptrOutCol ptrOutRowr" ptrRowIdx ptrColIdxptrVal ptrRowPtrrrrrr double_quant3s         rGc Cst|j}|dur|j|f}n|d}|dur)t|d|j|j||d|\}}n|d|f}|d} t| dkrHt| d} t| d} nt| d| d} t| d} t||g|dkr|rst t |t || | n}t t |t || | np|dkr|rt t |t || | n]t t |t || | nP|dkr|rt t |t || | n=t t |t || | n0|dkr|dkrt t |t || | n|dkrt t |t || | n td|d |t|||fS) Nr rrIrrrrz)Transform function not implemented: From r)rrDrMrrKrrArerr ctransform_row2col32Trdctransform_row2col32ctransform_row2turingTctransform_row2turingctransform_row2ampereTctransform_row2amperectransform_turing2rowctransform_ampere2rowrqr) rfrrr`rrrrErrMrrrrr transformsB ,   rPcCsj|durtj|j|jdf|j|jd}|j}|j|ks J|j |ks)J|j |ks2J|j |jdkstjt||dd}t|t|tj}||fS|dkru|j }|}|| }|dkrWd}d |}| }t||} t||| | }||fS|d vr|j }|}tj||ddtj ||dd}d||dk<d |}tj ||dd}t||} t||| | }||fS|d krt Bt|} tj| |dd}|d }| | | k} t|| } | | | | || <t||ttj}Wd||fS1swY||fSdS) Nlinear)rjrT)r0keepdim zeropointrr go@)vector-zeropoint row-zeropointtruncated-vectorgffffff?)r=rrr4r r ramaxCrKminaminno_grad expand_asr) rr0rrxqrKdynaqxminxzpxabsxrrrrrvectorwise_quant< sZ     r~cCs$|dkr|t|tj}|SdS)Nrj)rsr r=rl)rxrrrrrrvectorwise_dequantj srcCs|dkr||tt}|||S|dkr(d||}|||S|dkrqd||}|}t|jdkrIt|jdkrI|d}t|jdkr\t|jdkr\|d}t|jdkrh||9}n||9}||S|dkr|}t|jdkrt|jdkr|d}t|jdkrt|jdkr|d}t|jdkr|d|9}n|d|9}|d|9}||S|d kr|}t|jdkrt|jdkr|d}t|jdkrt|jdkr|d}t|jdkr|||tt9}n |||tt9}||S|d vrd|}t|jdkr/t|jdkr/|d}t|jdkrDt|jdkrD|d}t|jdkrS||t9}n||t9}||t9}||SdS) NrkrnrrprrIrror)rqrj)rsr4r rrMsqueezer3)rxS1S2rKrrrrrrvectorwise_mm_dequantr sd                      rcCs|d|d|d}|}t|jdkr(t|jdkr(|d}t|jdkr8||d9}n||d9}||dd9}||7}||S)Nrr rIrrl)r4r3sumrrMrr )rxrfrurrrKrrrrrdequant_min_max s"   rc Cs|d}|d}|dvsJ|jjdksJtj|d|ftj|jd}t|}t|d}t|d}t|} t|} t|} t |j} |dkr[t | | | |||n|dkrit | | | |||t | |S)Nrr rr>rJrr)rDrr=rrbrrArerdrr cextractOutliers_turingcextractOutliers_amperer) rfrrrformatAr`idx_sizerrrptrIdxr rErrrextract_outliers s(  rcCs6t|}tt|t|t|t||Sr$)r=rRr cpipeline_testrdrArQrb)rf batch_sizer`rrr pipeline_test s (rr5)T)NTr$)TrT)rT)TrrIr)Trr)r)rF)rNFNN)Nrr)NNNrDF)NNNNrDF)NrN)NNrNF)NNrNFr)NNNrN)NNNrNrr+)NNNN)NrrrNrF)rrNr)rrF)r)NFFN)NFF)NNNr)NNNNr)r rj)rj)ctypesrArrrandomr=math scipy.statsrnumpyrU functoolsrtypingrrrrbitsandbytes.utilsrr cextensionr r rrQrcadam32bit_grad_fp32cadam32bit_grad_fp16cadam32bit_grad_bf16cmomentum32bit_grad_32cmomentum32bit_grad_16crmsprop32bit_grad_32crmsprop32bit_grad_16clion32bit_grad_fp32clion32bit_grad_fp16clion32bit_grad_bf16cadagrad32bit_grad_32cadagrad32bit_grad_16rcadam_static_8bit_grad_32cadam_static_8bit_grad_16cmomentum_static_8bit_grad_32cmomentum_static_8bit_grad_16crmsprop_static_8bit_grad_32crmsprop_static_8bit_grad_16clion_static_8bit_grad_32clion_static_8bit_grad_16rcadam_8bit_blockwise_grad_fp32cadam_8bit_blockwise_grad_fp16cadam_8bit_blockwise_grad_bf16"cmomentum_8bit_blockwise_grad_fp32"cmomentum_8bit_blockwise_grad_fp16!crmsprop_8bit_blockwise_grad_fp32!crmsprop_8bit_blockwise_grad_fp16clion_8bit_blockwise_grad_fp32clion_8bit_blockwise_grad_fp16clion_8bit_blockwise_grad_bf16!cadagrad_8bit_blockwise_grad_fp32!cadagrad_8bit_blockwise_grad_fp16rr:rFrOrlrrUrorrDrar1rzr{r~rrrrrrrrrBrdrrrrrr4rrrZrrarprwrzrtrrrrrrrr9rrrrrrrrrrrrrr!r'r*r6r9r;r:rGrPr[rirsr~rrrrrrrrrs               . 2   , (6 wK I3$P$$M  &      n            = , \ ? \ Sf 4 .  ` , $V  .;