o
    h1                     @   s  d dl Z d dlZd dlZd dlmZ d dlZd dlmZmZmZm	Z	m
Z
mZ dd Zdd Zd#d	d
ZefddZddgZddgZddgZeeeeZdd Zdd eD Ze jdeeddd Ze jjdddgddgde jjdddgdd gdd!d" ZdS )$    N)product)
AutoConfigAutoModelForCausalLMAutoTokenizerBitsAndBytesConfigGenerationConfigset_seedc                	   C   s   t ddddtjdddS )NTFg      @nf4)load_in_4bitload_in_8bitllm_int8_thresholdllm_int8_has_fp16_weightbnb_4bit_compute_dtypebnb_4bit_use_double_quantbnb_4bit_quant_type)r   torchfloat16 r   r   K/var/www/html/ai/venv/lib/python3.10/site-packages/tests/test_generation.pyget_4bit_config   s   r   c                 C   sV   | \}}t  }|dkrd|_n||_tj||ddidtjd }tj	|}||fS )N16bitFr   48GBauto)quantization_config
max_memory
device_maptorch_dtype)
r   r
   r   r   from_pretrainedr   bfloat16evaltransformersr   )configmodel_name_or_path
quant_type
bnb_configmodel	tokenizerr   r   r   get_model_and_tokenizer    s   r'   Tc                 C   s.   d}|r| d|  d}|S | d|  }|S )NzA chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.z ### Human: z ### Assistant: r   )text	add_rolesdescriptionpromptr   r   r   get_prompt_for_generation_eval2   s   r-   c                 C   s>   ||}||dd d}| j|d |d}|j|d ddS )	Nptreturn_tensorscuda:0	input_ids)inputsgeneration_configr   Tskip_special_tokens)togeneratedecode)r%   r&   r)   r4   prompt_funcr3   outputsr   r   r   r8   =   s   r8   huggyllama/llama-7bzbigscience/bloom-1b7r	   fp4Fc                 C   s   dd | D S )Nc                 S   s   g | ]}t |qS r   )str.0xr   r   r   
<listcomp>G   s    z<lambda>.<locals>.<listcomp>r   )lstr   r   r   <lambda>G   s    rD   c                 C   s   g | ]	}d  t|qS )_)joinstrfuncr?   r   r   r   rB   H       rB   session)scopeparamsidsc                 c   s$    t | j\}}| j||fV  ~d S )N)r'   param)requestr%   r&   r   r   r   model_and_tokenizerI   s   rO   DQDQ_TrueDQ_False)rL   inference_kernelinference_kernel_Trueinference_kernel_Falsec                    s  t d tj}| \}} tjddddd}d|_d}d}t|jd	r,||jj_	||jj_
|s3|g| } |d
dd}	|	d }
g }|rat|D ]}|j|
|d} j|d dd}|| qHn|j|
|d} fdd|D }t||ksyJ d}t|D ]}|| d tttj ttjks|d7 }q|d dkrdnd}||krt tj |D ]}t | qtd| d| d S )N    Tg?gffffff?)max_new_tokens	do_sampletop_ptemperature   z3.14159r   r.   r/   r1   r2   )r4   r   r5   c                    s   g | ]	} j |d dqS )Tr5   )r9   )r@   outputr&   r   r   rB   v   rH   ztest_pi.<locals>.<listcomp>   r<         zFailure count: /)printr   r   r    r   rX   hasattrr!   r   r   r   r7   ranger8   r9   appendlenr>   mathpi
ValueError)rO   rS   rP   dtypefixture_configr%   r4   n_casesr)   r3   rA   r;   ir]   textoutfailure_countfailure_maxoutr   r^   r   test_piO   sR   



$

rs   )T)pytestr   rh   	itertoolsr   r    r   r   r   r   r   r   r   r'   r-   r8   modelsdtypesr
   listvaluesrG   rL   fixturerO   markparametrizers   r   r   r   r   <module>   s*     

