o
    h`                     @   sz  d dl mZmZmZ d dlmZmZ dd Zdd Zdd Z	d	d
 Z
dd Zdd Zdd Zdd Zdd Zdd Zdd ZdddZdd Zdd Zeee	ee
eeeeeeeeed ZG d!d" d"Zeg d#d$gd%d&eg d'd(gd)d*gd&ed+d,gd-d.gd/d&eg d0d1gd2d&eg d3d$d(gd4d&ed5d6gd1gd7d&ed8gd-gd9d&eg d:d(d.gd;d&eg d<d$gd=d&eg d>d.d?id@d&edAgd1gdBd&gZedCdCgd$gdDd&edEdFgg dGd&geg dHd(gdIdJgd&eg dKg dLdLgd&ged8dMgd-gdNd&edOdPgd.gd;d&gedQdRgd1gdSd&edTdUgg dVd&gedWdXgd$gdYd&edZd[gd(gd\d&ged]d^gd1gd_d&ed`d`gg dad&edbdcgg ddd&ged8gd-gd9d&geg d:d(d.gd;d&gededfgd$gdad&edgdhgg did&edjdkgg dld&edmdngg dod&geg d>d.d?id@d&gedQdRgd1gdSd&edpdqgg drd&ggZdsdt ZddwdxZddyefdzd{Zdd|d}ZdddZdddZduS )   )BASE_PYTHON_TOOLSclean_code_for_chatclean_code_for_run)InterpretorErrorevaluatec                 C   s   d|  d| dS )NzThis is the classification of z along . )textlabelsr   r   W/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/tools/evaluate_agent.py
classifier      r   c                 C   s   d|  d| d| dS )NzThis is the translation of  from z to r   r   )r	   src_langtgt_langr   r   r   
translator   s   r   c                 C      d|  dS )Nz!This is actually a sound reading r   r   r	   r   r   r   speaker      r   c                 C   $   d| vrt d|  dd|  dS )Nsoundz	`audio` (z) is not a sound.z"This is the transcribed text from r   
ValueError)audior   r   r   transcriber"      r   c                 C   r   )Nz'This is actually an image representing r   r   )promptr   r   r   image_generator(   r   r   c                 C   r   )Nimage	`image` () is not an image.zThis is a description of r   r   )r   r   r   r   image_captioner,   r   r"   c                 C   s*   d| vrt d|  dd|  d| dS )Nr   r    r!   zThis is a transformation of z according to r   r   r   r   r   r   r   image_transformer2      r$   c                 C      d| d|  dS )NThis is the answer to r   r   r   )r	   questionr   r   r   question_answerer8   r   r)   c                 C   s*   d| vrt d|  dd| d|  dS )Nr   r    r!   r'   r   r   r   r   r(   r   r   r   image_qa<   r%   r+   c                 C   r   )NzThis is the content of r   r   )urlr   r   r   text_downloaderB   r   r-   c                 C   r   )NzThis is a summary of r   r   r   r   r   r   
summarizerF   r   r.      c                 C   s
   d|  S )NzA video of r   )r   secondsr   r   r   video_generatorJ   s   
r1   c                 C   r&   )Nr'   z from the document r   r   r*   r   r   r   document_qaN   r   r2   c                 C   s   d| d|  S )NzThis is the mask of z in r   r#   r   r   r   image_segmenterR   s   r3   )text_classifierr   text_readerr.   r   r   r"   r$   text_qar-   r+   r1   r2   r3   c                   @   s   e Zd ZdZdd ZdS )Problema#  
    A class regrouping all the information to solve a problem on which we will evaluate agents.

    Args:
        task (`str` ou `list[str]`):
            One or several descriptions of the task to perform. If a list, it should contain variations on the
            phrasing, but for the same task.
        inputs (`list[str]` or `dict[str, str]`):
            The inputs that will be fed to the tools. For this testing environment, only strings are accepted as
            values. Pass along a dictionary when you want to specify the values of each inputs, or just the list of
            inputs expected (the value used will be `<<input_name>>` in this case).
        answer (`str` or `list[str`]):
            The theoretical answer (or list of possible valid answers) to the problem, as code.
    c                 C   s   || _ || _|| _d S Ntaskinputsanswer)selfr:   r;   r<   r   r   r   __init__x   s   
zProblem.__init__N)__name__
__module____qualname____doc__r>   r   r   r   r   r7   h   s    r7   )z:Is the following `text` (in Spanish) positive or negative?zEIs the text in the variable `text` (in Spanish) positive or negative?z`Translate the following `text` from Spanish to English then tell me if its positive or negative.r	   zjtext_classifier(translator(text, src_lang="Spanish", tgt_lang="English"), labels=["positive", "negative"])r9   )z+Tell me out loud what the `image` contains.z(Describe the following `image` out loud.zDFind what is in the picture stored in `image` then read it out loud.r   z#text_reader(image_captioner(image))z>text_reader(image_qa(image, question='What is in the image?'))zkGenerate an image from the text given in `text_input`. Then transform it according to the text in `prompt`.zeUse the following `text_input` to generate an image, then transform it by using the text in `prompt`.
text_inputr   z6image_transformer(image_generator(text_input), prompt))zTDownload the content of `url`, summarize it then generate an image from its content.z<Use a summary of the web page at `url` to generate an image.zXSummarize the content of the web page at `url`, and use the result to generate an image.r,   z1image_generator(summarizer(text_downloader(url))))zUTransform the following `image` using the prompt in `text`. The prompt is in Spanish.zNUse the text prompt in `text` (in Spanish) to transform the following `image`.z]Translate the `text` from Spanish to English then use it to transform the picture in `image`.zRimage_transformer(image, translator(text, src_lang='Spanish', tgt_lang='English'))zHDownload the content of `url`, summarize it then read it out loud to me.z+Read me a summary of the web page at `url`.z-text_reader(summarizer(text_downloader(url)))z6Generate an image from the text given in `text_input`.zimage_generator(text_input))z2Replace the beaver in the `image` by the `prompt`.z7Transform the `image` so that it contains the `prompt`.z'Use `prompt` to transform this `image`.z image_transformer(image, prompt))zmProvide me the summary of the `text`, then read it to me before transcribing it and translating it in French.zXSummarize `text`, read it out loud then transcribe the audio and translate it in French.zURead me a summary of the `text` out loud. Transcribe this and translate it in French.z]translator(transcriber(text_reader(summarizer(text))), src_lang='English', tgt_lang='French'))z Generate a video of the `prompt`zAnimate a `prompt`z%Make me a short video using `prompt`.zA lobster swimmingz%video_generator('A lobster swimming')zDownload the following file `url`, summarize it in a few words and generate a video from it.Fetch the file at this `url`, summarize it, and create an animation out of it.z1video_generator(summarizer(text_downloader(url)))z7Translate the following `text` from Spanish to English.zHtranslated_text=translator(text, src_lang='Spanish', tgt_lang='English')zIs it positive or negative?z$Tell me if its positive or negative.zAtext_classifier(translated_text, labels=['positive', 'negative']))zWhat does this `image` contain?zDescribe the following `image`.z-Find what is in the picture stored in `image`z"description=image_captioner(image)z=description=image_qa(image, question='What is in the image?'))z#Now, read the description out loud.z Great! Can you read it out loud?Read it out loud.zaudio=text_reader(description)z3Use the following `text_input` to generate an imagez#image = image_generator(text_input)z/Transform it according to the text in `prompt`.z+Transform it by using the text in `prompt`.z/Download the content of `url` and summarize it.z/Summarize the content of the web page at `url`.z*summary = summarizer(text_downloader(url))z#Generate an image from its content.z-Use the previous result to generate an image.zimage_generator(summary)z)Translate this Spanish `text` in English.z-Translate the `text` from Spanish to English.zJtranslated_text = translator(text, src_lang='Spanish', tgt_lang='English')z<Transform the following `image` using the translated `text`.z;Use the previous result to transform the following `image`.z)image_transformer(image, translated_text)zDownload the content of `url`.z&Get me the text on the weg page `url`.ztext = text_downloader(url)zSummarize this text.zsummary = summarizer(text)zRead it out loud to me.zRead me the previous result.ztext_reader(summary)z%Provide me the summary of the `text`.zSummarize `text`.zRead this summary to me.rD   z%audio = text_reader(summarizer(text))z.Transcribing the previous result back in text.zTranscribe the audio.ztext = transcriber(audio)z&Translating the last result in French.zTranslate this in French.z7translator(text, src_lang='English', tgt_lang='French')zgenerate a video from it.z)Create an animation from the last result.zvideo_generator(summary)c                    s   t |ts fddtD S t | tr.t| D ]\}||  v r-fddtD   S qt| D ]\}| |krFfddtD   S q3 fddtD S )Nc                       h | ]}| v r|qS r   r   .0namecode_answerr   r   	<setcomp>      z(get_theoretical_tools.<locals>.<setcomp>c                    rE   r   r   rF   one_coder   r   rK     rL   c                    rE   r   r   rF   rM   r   r   rK     rL   c                    s   h | ]
}| d  v r|qS )    r   rF   rI   r   r   rK     s    )
isinstancelist
TEST_TOOLSdictzipvalues)agent_answertheoretical_answerrJ   
one_answerr   )rJ   rN   r   get_theoretical_tools  s   

rY   NFc           	   
   C   s   t  }t D ]\}}|| vrq|||< qt|tr | }n|d ur+dd |D }|d ur5|| n|}zt| ||W S  tyT } z
t	|W  Y d }~S d }~w t
yl } z|rat| W Y d }~d S d }~ww )Nc                 S   s   i | ]	}|d | dqS )z<<z>>r   )rG   inpr   r   r   
<dictcomp>  s    z!evaluate_code.<locals>.<dictcomp>)r   copyrR   itemsrP   rS   updater   r   str	Exceptionprint)	coder;   stateverbosereturn_interpretor_errortoolsrH   tooler   r   r   evaluate_code  s.   


ri   rd   c                    s~   |rt |   t tr n g |  v r|rt d dS t| tr7t fdd|  D r7|r5t d dS |r=t d dS )	NzPerfect!r   c                 3   s    | ]}| v V  qd S r8   r   )rG   vrW   r   r   	<genexpr>  s    zscore_code.<locals>.<genexpr>z Almsot perfect, result in state!g      ?z.Result is not the right one but code executed.g333333?)ra   rP   rQ   rS   anyrU   )rV   rW   rd   r   rk   r   
score_code  s   
$rn   c                    s  fddt D }t|||}||krd}d }	nt|| }
t|| }tddd|
  d|  }||d}	 fddt D }||krHd}d }nt|| }
t|| }tddd|
  d|  }||d}t|||d}|dk rx ||d	}nd }|||f|	||ffS )
Nc                    s    h | ]}d | d  v r|qS )`r   rF   )explanationr   r   rK     s     z&evaluate_one_result.<locals>.<setcomp>g      ?rO   g      ?)selected_toolstheoretical_toolsc                    rE   r   r   rF   )rb   r   r   rK     rL   rd   )code_produced
evaluationrW   )rR   rY   lenmaxrn   )rp   rb   rV   rW   r<   rd   tools_in_explanationrr   tool_selection_scoretool_selection_errorsmissing_toolsunexpected_toolstools_in_codetool_used_scoretool_used_errorsscorecode_errorsr   )rb   rp   r   evaluate_one_result  s:   r      c              
      s  t  j }|t tkr$t t| }t |t }td| d| dg }g }ttD ]'\}	}
t|
jt	rH|
|
j |
|	gt|
j  q,||
j ||	 q,d}d}d}|rbi }i }i }tdt||D ]}t|| t|}||| } fdd|D } j|dgd}t|D ]\}	}t|||	   |rtd	||	  d
||	  d t|\}}t|j|d}tjt	ṙfddjD }ntjj}t||||j|d\}}||d 7 }||d 7 }||d 7 }|r"|d dur|d |||	 < |d dur|d |||	 < |d dur"|d |||	 < qqjd|t|  d|t|  d|t|  d}|rE||||fS |S )z
    Evaluates a new agent on all `EVALUATION_TASKS`.

    Example:

    ```py
    agent = NewOpenAiAgent(model="text-davinci-003", api_key=your_api_key)
    bads = new_evaluate_agent(agent)
    for bad in bads:
        print(bad)
    ```
    ?Fix the test tools in the evaluate_agent module. Tools mising: . Extra tools: r   rO   c                    s   g | ]}  |qS r   )format_prompt)rG   r:   )agentr   r   
<listcomp>&  s    z"evaluate_agent.<locals>.<listcomp>zTask:stopz	====Task z====

rs   c                    s   g | ]}t | jqS r   ri   r;   )rG   r<   )problemr   r   r   2  rL   r   r/   Nd   ztool selection scoreztool used scorez
code score)settoolboxkeysrR   r   	enumerateEVALUATION_TASKSrP   r:   rQ   extendrv   appendrangemingenerate_manyra   r   ri   r;   r<   r   )r   
batch_sizerd   return_errorsagent_toolsr{   r|   
eval_taskseval_idxidxpbry   r~   
code_scorerz   r   r   	start_idxend_idxbatch_taskspromptsresultsresultrp   rb   rV   rW   scoreserrorsr   )r   r   r   evaluate_agent  st   
r   c                    s  t | j }|t tkr$t t| }|t t }td| d| dd}d}d}d}	|r4i }
i }i }tD ] t d jtrD g}n fddt	t
 d jD }|D ]}|   i }t|d jtrtdd t	t
|d jD ni }t|D ]\}|rtj |	d7 }	| jjd	d
}| j|ddgd}|| d | _t|\}}|rtd|  td|  t|j||d}j}t|trՇfddt||D }nt|j|d}t||||||d\}}||d 7 }||d 7 }||d 7 }|r(|d dur|d |
j< |d dur|d |j< |d dur(|d |j< qzqVq6d||	  d||	  d||	  d}|rF||
||fS |S )z
    Evaluates a new agent on all `EVALUATION_CHATS`.

    Example:

    ```py
    agent = NewOpenAiAgent(model="text-davinci-003", api_key=your_api_key)
    bads = new_evaluate_agent(agent)
    for bad in bads:
        print(bad)
    ```
    r   r   r   rO   c                    s   g | ]  fd dD qS )c                    s$   g | ]}t |j  |j|jd qS )r9   )r7   r:   r;   r<   )rG   r   ir   r   r   w  s   $ z2evaluate_chat_agent.<locals>.<listcomp>.<listcomp>r   )rG   )chat_problemr   r   r   v  s    z'evaluate_chat_agent.<locals>.<listcomp>c                 S   s   g | ]}i qS r   r   )rG   _r   r   r   r   ~  s    r   T)	chat_modezHuman:z=====r   r   z==Explanation from the agent==
z!
==Code generated by the agent==
)rc   rd   c                    s    g | ]\}}t | j|d qS )rc   r   )rG   arc   )step_problemr   r   r     s    r   rs   r/   Nr   r   )r   r   r   rR   r   EVALUATION_CHATSrP   r:   r_   r   rv   prepare_for_new_chatr<   rQ   r   ra   r   generate_onechat_historyr   ri   r;   rT   r   )r   rd   r   r   r{   r|   ry   r~   r   total_stepsrz   r   r   resolved_problemsr   agent_statetheoretical_statestepr   r   rp   rb   rV   r<   rW   r   r   r   )r   r   r   evaluate_chat_agentR  s   
.




2

r   )r/   )NNFF)F)r   FF)FF) agentsr   r   r   python_interpreterr   r   r   r   r   r   r   r"   r$   r)   r+   r-   r.   r1   r2   r3   rR   r7   r   r   rY   ri   boolrn   r   r   r   r   r   r   r   <module>   s  
					c
  1


+Y