o
    hOb                     @   s   d dl mZ d dl mZ d dlmZ d dlmZ d dlZd dl	m
Z G dd dZG dd	 d	ZG d
d dejjZG dd deZG dd deZdS )    )abc)defaultdict)deepcopy)chainNc                   @   s   e Zd Zdd ZdS )MockArgsc                 C   s   |D ]
}t | |||  qd S N)setattr)selfinitial_datakey r   R/var/www/html/ai/venv/lib/python3.10/site-packages/bitsandbytes/optim/optimizer.py__init__   s   zMockArgs.__init__N)__name__
__module____qualname__r   r   r   r   r   r      s    r   c                   @   sH   e Zd ZdZdd Zdd Zedd Zdd	 Z	dd
dZ	dd Z
dS )GlobalOptimManagerNc                 C      t d)NzCall get_instance() instead)RuntimeErrorr	   r   r   r   r         zGlobalOptimManager.__init__c                 C   s"   i | _ i | _d | _d| _g | _d S )NF)
pid2configindex2config	optimizeruses_config_overridemodule_weight_config_tripler   r   r   r   
initialize   s
   
zGlobalOptimManager.initializec                 C   s&   | j d u r| | | _ | j   | j S r   )	_instance__new__r   )clsr   r   r   get_instance"   s   

zGlobalOptimManager.get_instancec                 C   sr   t |}t|d tsd|ig}t|D ]"\}}t|d D ]\}}t|| jv r5| jt| | j||f< qqd S )Nr   params)list
isinstancedict	enumerateidr   r   )r	   r!   param_groupsgroup_indexgroupp_indexpr   r   r   register_parameters)   s   
z&GlobalOptimManager.register_parametersc                 C   s   d| _ t|tjjr|g}t|tjr|g}|dur(|dur(|du s$J ||i}|durJ|D ]}t|| jv rB| jt| | q.|| jt|< q.dS dS )a  
        Overrides initial optimizer config for specific parameters.

        The key-values of the optimizer config for the input parameters are overridden
        This can be both, optimizer parameters like "betas", or "lr" or it can be
        8-bit specific parameters like "optim_bits", "percentile_clipping".

        Parameters
        ----------
        parameters : torch.Tensor or list(torch.Tensors)
            The input parameters.
        key : str
            The hyperparamter to override.
        value : object
            The value for the hyperparamters.
        key_value_dict : dict
            A dictionary with multiple key-values to override.
        TN)	r   r#   torchnn	ParameterTensorr&   r   update)r	   
parametersr   valuekey_value_dictr+   r   r   r   override_config5   s   z"GlobalOptimManager.override_configc                 C   s   | j |||f d S r   )r   append)r	   module
param_nameconfigr   r   r   register_module_overrideZ   s   z+GlobalOptimManager.register_module_override)NNN)r   r   r   r   r   r   classmethodr    r,   r5   r:   r   r   r   r   r      s    

%r   c                       s   e Zd Zd fdd	Zdd Z fddZd	d
 Zdd Zdd Ze	
 dddZdd Zdd Zdd Ze	jfddZdd Z  ZS )Optimizer8bit    Fc                    sX   t  || d| _i | _|| _tj | _t	 | _
h d| _|dkr*|   d S d S )NF>   max1max2qmap1qmap2state1state2absmax1absmax2new_max1new_max2	gnorm_vec	unorm_vec   )superr   initialized	name2qmapis_pagedFGlobalPageManagerr    page_mngr   mngnon_castable_tensor_keys	fill_qmap)r	   r!   defaults
optim_bitsrN   	__class__r   r   r   _   s   

zOptimizer8bit.__init__c                 C   s(   t jdd| jd< t jdd| jd< d S )NT)signeddynamicFudynamic)rO   create_dynamic_maprM   r   r   r   r   rT   y   s   zOptimizer8bit.fill_qmapc                    s   t  | d S r   )rK   __setstate__)r	   staterW   r   r   r]   }   s   zOptimizer8bit.__setstate__c                    s&  t |}j}|d }t|t|krtddd |D }dd |D }tdd t||D r5tddd	 ttd
d |D tdd |D D } fdd tt	}|d 
 D ]\}}	||v rt|| }
 |
|	||
< q`|	||< q`dd fddt||D }||d dS )zLoads the optimizer state.

        Args:
            state_dict (dict): optimizer state. Should be an object returned
                from a call to :meth:`state_dict`.
        r'   z<loaded state dict has a different number of parameter groupsc                 s       | ]	}t |d  V  qdS r!   Nlen.0gr   r   r   	<genexpr>       z0Optimizer8bit.load_state_dict.<locals>.<genexpr>c                 s   r_   r`   ra   rc   r   r   r   rf      rg   c                 s   s    | ]	\}}||kV  qd S r   r   )rd   p_lens_lenr   r   r   rf      rg   z]loaded state dict contains a parameter group that doesn't match the size of optimizer's groupc                 S   s   i | ]\}}||qS r   r   )rd   old_idr+   r   r   r   
<dictcomp>   s    z1Optimizer8bit.load_state_dict.<locals>.<dictcomp>c                 s       | ]}|d  V  qdS r`   r   rc   r   r   r   rf          c                 s   rl   r`   r   rc   r   r   r   rf      rm   c                    s   t |tjr  r|jtjkr| j}|S t |tr=| D ]\}}|j	v r3| j
||< q! |||< q!|S t |tjrQt| fdd|D S |S )zBMake a deep copy of value, casting all tensors to device of param.c                 3   s    | ]} |V  qd S r   r   )rd   v)castparamr   r   rf      s    z>Optimizer8bit.load_state_dict.<locals>.cast.<locals>.<genexpr>)r#   r-   r0   is_floating_pointdtypeuint8tor$   itemsrS   devicecontainer_abcsIterabletype)rp   r3   krn   )ro   r	   )rp   r   ro      s   

z+Optimizer8bit.load_state_dict.<locals>.castr^   c                 S   s   | d |d< |S )Nr!   r   )r)   	new_groupr   r   r   update_group   s   z3Optimizer8bit.load_state_dict.<locals>.update_groupc                    s   g | ]	\}} ||qS r   r   )rd   re   ng)r|   r   r   
<listcomp>   s    z1Optimizer8bit.load_state_dict.<locals>.<listcomp>)r^   r'   N)r   r'   rb   
ValueErroranyzipr   from_iterabler   r$   ru   r]   )r	   
state_dictgroupssaved_groups
param_lens
saved_lensid_mapr^   rz   rn   rp   r'   r   )ro   r	   r|   r   load_state_dict   s>   

zOptimizer8bit.load_state_dictc           	      C   s   t | jD ];\}}t |d D ]0\}}|| jv r?| j| }| D ]\}}t|tjr>t|dd}|s>||j	| j| |< q!qqd S )Nr!   rN   F)
r%   r'   r^   ru   r#   r-   r0   getattrrt   rv   )	r	   gindexr)   pindexr+   valuesrz   rn   rN   r   r   r   to_gpu   s   

zOptimizer8bit.to_gpuc           
      C   s   | j jD ]]\}}}t||}|d usJ t|tjs"t|tjs"J d}t| jD ]7\}}|r1 n0t|d D ](\}}	|r? n!t	|	t	|kr_|| j j
t	|	< | j j
t	|	 | j j||f< d}q7q)qd S )NFr!   T)rR   r   r   r#   r-   r0   r/   r%   r'   r&   r   r   )
r	   r7   attrr9   pmodulefoundr   r)   r   r+   r   r   r   check_overrides   s,   
zOptimizer8bit.check_overridesNc           	      C   s   d}|durt   | }W d   n1 sw   Y  g }| js-|   |   d| _t| jD ]:\}}t|d D ]/\}}|jdu rFq<| j| }t	|dkrY| 
|||| | | | |||| t j  q<q2| jrut j  |S )zPerforms a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        NTr!   r   )r-   enable_gradrL   r   r   r%   r'   gradr^   rb   
init_stateprefetch_stateupdate_stepcudasynchronizerN   )	r	   closureloss	overflowsr   r)   r   r+   r^   r   r   r   step   s0   





zOptimizer8bit.stepc                 C   s   i }|d |d< |d |d< |d |d< |d |d< | j j|d< | j j|d< | j j|d< | j j|d< | j j|d	< | j j|d
< ||f| jjv rQ|	| jj||f  |S )Nbetasepsweight_decaylrrV   min_8bit_sizepercentile_clipping
block_wise	max_unorm
skip_zeros)
argsrV   r   r   r   r   r   rR   r   r1   )r	   r   r   r)   r9   r   r   r   
get_config  s   zOptimizer8bit.get_configc                 C   r   )Nz(init_state method needs to be overriddenNotImplementedErrorr	   r)   r+   r   r   r   r   r   r   (  r   zOptimizer8bit.init_statec                 C   r   )Nz-The update_step method needs to be overriddenr   r   r   r   r   r   +  s   zOptimizer8bit.update_stepc                 C   sX   | j r	| dk rtj|||jdS tj|j||jd}t|d | j	j
| |S )Ng     j@rr   rv   r   )rN   numelr-   
zeros_likerv   rO   	get_pagedshapefillrQ   paged_tensorsr6   )r	   r+   rr   buffr   r   r   get_state_buffer0  s   zOptimizer8bit.get_state_bufferc                 C   s\   | j r(| j| }|d }t|dd}|r*t|d  d|v r,t|d  d S d S d S d S )NrB   rN   FrC   )rN   r^   r   rO   prefetch_tensor)r	   r+   r^   s1rN   r   r   r   r   :  s   
zOptimizer8bit.prefetch_state)r=   Fr   )r   r   r   r   rT   r]   r   r   r   r-   no_gradr   r   r   r   float32r   r   __classcell__r   r   rW   r   r<   ^   s    M'
r<   c                       V   e Zd Z												
	
d fdd	Ze dd Ze dd Z  ZS )Optimizer2StateMbP??g+?:0yE>        r=   N   d   TFc                    s>  d|kst d| d|kst d| t|tr1|dddd d}dd	 |D }tt|D ]}d||   krEd
k sRn t d| d||  q7d|ks^t d| t||||d}t	 
|||| |d u ri }||d< d|d< |	|d< |
|d< ||d< ||d< ||d< t|| _n|| _|| _d S )Nr   Invalid learning rate: Invalid epsilon value: ( ),c                 S   s   g | ]}t |qS r   )float)rd   br   r   r   r~   ^  s    z,Optimizer2State.__init__.<locals>.<listcomp>      ? Invalid beta parameter at index : Invalid weight_decay value: r   r   r   r   rV   r   r   r   r   r   r   )r   r#   strreplacestripsplitrangerb   r$   rK   r   r   r   optimizer_namer	   r   r!   r   r   r   r   rV   r   r   r   r   r   r   rN   irU   rW   r   r   r   F  s>   

zOptimizer2State.__init__c           
      C   sj  |  |||}|d dkrtj}n|d dkrtj}n	td|d  | |d k r/tj}| j| }d|d< |tjksH|tjkr]| dk r]| j|tjd	|d
< | j|tjd	|d< n|tjkr|d dkrd| jvrr| 	  | jd 
|j| jd< | jd 
|j| jd< | j|tjd	|d
< | jd |d< | j|tjd	|d< | jd |d< |d r| }|d }	|	|d dkrdnd7 }	tj|	ftj|jd|d< tj|	ftj|jd|d< n0tjdtj|jd|d< tjdtj|jd|d< tjdtj|jd|d< tjdtj|jd|d< |d dk r tjd|jd|d< |d  d!kr3tjd|jd|d"< d S d S )#NrV   r=   rJ   (Amount of optimizer bits not supported: r   r   r   r   rr   rB   rC   rZ   r[   r@   rA   r         r   rD   rE   r   r>   rF   r?   rG   r   r   r   rv   rH   r   r   rI   r   r-   r   rs   r   r   r^   r   rM   rT   rt   rv   zeros
r	   r)   r+   r   r   r9   rr   r^   nblocksr   r   r   r   {  sr   















zOptimizer2State.init_statec                 C   sJ  | j | }|j}| |||}|d  d7  < |d }|d dk r1t||d ||d \}	}
}nd}|d jtjkrrtj| j	|||d |d d	 |d
 ||d |d |d d |d ||d dkrf|d nd |d |d d d S |d jtj
kr|d stj| j	|||d |d |d d	 |d d |d
 ||d |d |d |d |d |d |d |d ||d dkr|d nd |d d |d |d |d< |d< |d |d |d< |d< d S |d jtj
kr!|d r#tj| j	|||d |d |d d	 |d d |d
 ||d |d |d |d |d |d ||d d d S d S d S )Nr   r   r   r   rH   r   rB   r   r   r   r   rC   r   r   r   rI   r   r   r   r   r@   rA   r>   r?   rF   rG   )gnorm_scalerI   r   rD   rE   r   r   r^   r   r   rO   r   rr   r-   r   optimizer_update_32bitr   rs   optimizer_update_8bitoptimizer_update_8bit_blockwiser	   r)   r+   r   r   r^   r   r9   r   current_gnorm
clip_valuer   r   r   r   r     s   








zOptimizer2State.update_step)r   r   r   r   r=   Nr   r   Tr   FF	r   r   r   r   r-   r   r   r   r   r   r   rW   r   r   E  s$    5
Gr   c                       r   )Optimizer1Stater   r   r   r   r   r=   Nr   r   TFc                    s  d|kst d| d|kst d| tt|D ]}d||   kr*dk s7n t d| d||  qd|ksCt d| t||||d}t |||| |d u r|i }||d	< d
|d< |	|d< |
|d< ||d< ||d< ||d< t|| _n|| _|| _d S )Nr   r   r   r   r   r   r   r   rV   r   r   r   r   r   r   )	r   r   rb   r$   rK   r   r   r   r   r   rW   r   r   r     s8   
zOptimizer1State.__init__c           
      C   s  |  |||}|d dkrtj}n|d dkrtj}n	td|d  | |d k r/tj}| j| }d|d< |tjksH|tjkrS| dk rS| j|tjd	|d
< no|tjkr|d dkrsd| jvrg| 	  | jd 
|j| jd< | j|tjd	|d
< | jd |d< |d r| }|d }	|	|d dkrdnd7 }	tj|	ftj|jd|d< ntjdtj|jd|d< tjdtj|jd|d< |d dk rtjd|jd|d< |d dkrtjd|jd|d< d S d S )NrV   r=   rJ   r   r   r   r   r   r   rB   rZ   r@   r   r   r   r   rD   r   r>   rF   r   r   r   r   rH   r   r   rI   r   r   r   r   r   r   L  sT   









zOptimizer1State.init_statec                 C   s  | j | }|j}| |||}|d  d7  < |d }|d dk r1t||d ||d \}	}
}nd}|d jtjkrptj| j	|||d |d d	 |d
 ||d d |d d |d ||d dkrd|d nd |d |d d d S |d jtj
kr|d stj| j	|||d d |d d	 |d d |d
 ||d |d d |d d |d d |d ||d dkr|d nd |d d |d |d |d< |d< d S |d jtj
kr|d rtj| j	|||d d |d d	 |d d |d
 ||d |d d |d d |d ||d d d S d S d S )Nr   r   r   r   rH   r   rB   r   r   r   r   r   r   r   rI   r   r   r   r@   r>   rF   )r   rD   r   r   r   r   r   r   r     s   








zOptimizer1State.update_step)r   r   r   r   r=   Nr   r   Tr   FFr   r   r   rW   r   r     s$    1
7r   )collectionsr   rw   r   copyr   	itertoolsr   r-   bitsandbytes.functional
functionalrO   r   r   optim	Optimizerr<   r   r   r   r   r   r   <module>   s   I h V