o
    h*                     @   s^  d dl mZmZ G dd dedZG dd dedZG dd dedZG d	d
 d
edZG dd dedZG dd dedZG dd dedZ	G dd dedZ
G dd dedZG dd dedZG dd dedZG dd dedZG dd dedZG dd dedZG dd  d edZG d!d" d"edZG d#d$ d$edZG d%d& d&edZG d'd( d(edZG d)d* d*edZG d+d, d,edZG d-d. d.edZG d/d0 d0edZG d1d2 d2edZG d3d4 d4edZG d5d6 d6edZG d7d8 d8edZG d9d: d:edZG d;d< d<edZG d=d> d>edZ G d?d@ d@edZ!G dAdB dBedZ"G dCdD dDedZ#G dEdF dFedZ$G dGdH dHedZ%G dIdJ dJedZ&G dKdL dLedZ'G dMdN dNedZ(G dOdP dPedZ)G dQdR dRedZ*G dSdT dTedZ+G dUdV dVedZ,G dWdX dXedZ-G dYdZ dZedZ.G d[d\ d\edZ/G d]d^ d^edZ0G d_d` d`edZ1G dadb dbedZ2G dcdd ddedZ3G dedf dfedZ4G dgdh dhedZ5G didj djedZ6G dkdl dledZ7G dmdn dnedZ8G dodp dpedZ9G dqdr dredZ:G dsdt dtedZ;G dudv dvedZ<G dwdx dxedZ=G dydz dzedZ>G d{d| d|edZ?d}S )~   )DummyObjectrequires_backendsc                   @      e Zd ZdgZdd ZdS )AlbertTokenizerFast
tokenizersc                 O      t | dg d S Nr   r   selfargskwargs r   a/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/utils/dummy_tokenizers_objects.py__init__      zAlbertTokenizerFast.__init__N__name__
__module____qualname__	_backendsr   r   r   r   r   r          r   )	metaclassc                   @   r   )BartTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zBartTokenizerFast.__init__Nr   r   r   r   r   r      r   r   c                   @   r   )BarthezTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zBarthezTokenizerFast.__init__Nr   r   r   r   r   r      r   r   c                   @   r   )BertTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zBertTokenizerFast.__init__Nr   r   r   r   r   r      r   r   c                   @   r   )BigBirdTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   $   r   zBigBirdTokenizerFast.__init__Nr   r   r   r   r   r   !   r   r   c                   @   r   )BlenderbotTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   +   r   z BlenderbotTokenizerFast.__init__Nr   r   r   r   r   r   (   r   r   c                   @   r   )BlenderbotSmallTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   2   r   z%BlenderbotSmallTokenizerFast.__init__Nr   r   r   r   r   r   /   r   r   c                   @   r   )BloomTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   9   r   zBloomTokenizerFast.__init__Nr   r   r   r   r   r   6   r   r   c                   @   r   )CamembertTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   @   r   zCamembertTokenizerFast.__init__Nr   r   r   r   r   r    =   r   r    c                   @   r   )CLIPTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   G   r   zCLIPTokenizerFast.__init__Nr   r   r   r   r   r!   D   r   r!   c                   @   r   )CodeLlamaTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   N   r   zCodeLlamaTokenizerFast.__init__Nr   r   r   r   r   r"   K   r   r"   c                   @   r   )CodeGenTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   U   r   zCodeGenTokenizerFast.__init__Nr   r   r   r   r   r#   R   r   r#   c                   @   r   )ConvBertTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   \   r   zConvBertTokenizerFast.__init__Nr   r   r   r   r   r$   Y   r   r$   c                   @   r   )CpmTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   c   r   zCpmTokenizerFast.__init__Nr   r   r   r   r   r%   `   r   r%   c                   @   r   )DebertaTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   j   r   zDebertaTokenizerFast.__init__Nr   r   r   r   r   r&   g   r   r&   c                   @   r   )DebertaV2TokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   q   r   zDebertaV2TokenizerFast.__init__Nr   r   r   r   r   r'   n   r   r'   c                   @   r   )RetriBertTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   x   r   zRetriBertTokenizerFast.__init__Nr   r   r   r   r   r(   u   r   r(   c                   @   r   )DistilBertTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   z DistilBertTokenizerFast.__init__Nr   r   r   r   r   r)   |   r   r)   c                   @   r   )DPRContextEncoderTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   z'DPRContextEncoderTokenizerFast.__init__Nr   r   r   r   r   r*      r   r*   c                   @   r   )DPRQuestionEncoderTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   z(DPRQuestionEncoderTokenizerFast.__init__Nr   r   r   r   r   r+      r   r+   c                   @   r   )DPRReaderTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zDPRReaderTokenizerFast.__init__Nr   r   r   r   r   r,      r   r,   c                   @   r   )ElectraTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zElectraTokenizerFast.__init__Nr   r   r   r   r   r-      r   r-   c                   @   r   )FNetTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zFNetTokenizerFast.__init__Nr   r   r   r   r   r.      r   r.   c                   @   r   )FunnelTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zFunnelTokenizerFast.__init__Nr   r   r   r   r   r/      r   r/   c                   @   r   )GPT2TokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zGPT2TokenizerFast.__init__Nr   r   r   r   r   r0      r   r0   c                   @   r   )GPTNeoXTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zGPTNeoXTokenizerFast.__init__Nr   r   r   r   r   r1      r   r1   c                   @   r   )GPTNeoXJapaneseTokenizerr   c                 O   r   r   r	   r
   r   r   r   r      r   z!GPTNeoXJapaneseTokenizer.__init__Nr   r   r   r   r   r2      r   r2   c                   @   r   )HerbertTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zHerbertTokenizerFast.__init__Nr   r   r   r   r   r3      r   r3   c                   @   r   )LayoutLMTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zLayoutLMTokenizerFast.__init__Nr   r   r   r   r   r4      r   r4   c                   @   r   )LayoutLMv2TokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   z LayoutLMv2TokenizerFast.__init__Nr   r   r   r   r   r5      r   r5   c                   @   r   )LayoutLMv3TokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   z LayoutLMv3TokenizerFast.__init__Nr   r   r   r   r   r6      r   r6   c                   @   r   )LayoutXLMTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zLayoutXLMTokenizerFast.__init__Nr   r   r   r   r   r7      r   r7   c                   @   r   )LEDTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zLEDTokenizerFast.__init__Nr   r   r   r   r   r8      r   r8   c                   @   r   )LlamaTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zLlamaTokenizerFast.__init__Nr   r   r   r   r   r9      r   r9   c                   @   r   )LongformerTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   z LongformerTokenizerFast.__init__Nr   r   r   r   r   r:      r   r:   c                   @   r   )LxmertTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zLxmertTokenizerFast.__init__Nr   r   r   r   r   r;      r   r;   c                   @   r   )MarkupLMTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   zMarkupLMTokenizerFast.__init__Nr   r   r   r   r   r<     r   r<   c                   @   r   )MBartTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   zMBartTokenizerFast.__init__Nr   r   r   r   r   r=     r   r=   c                   @   r   )MBart50TokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   zMBart50TokenizerFast.__init__Nr   r   r   r   r   r>     r   r>   c                   @   r   )MobileBertTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   z MobileBertTokenizerFast.__init__Nr   r   r   r   r   r?     r   r?   c                   @   r   )MPNetTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r      r   zMPNetTokenizerFast.__init__Nr   r   r   r   r   r@     r   r@   c                   @   r   )MT5TokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   '  r   zMT5TokenizerFast.__init__Nr   r   r   r   r   rA   $  r   rA   c                   @   r   )MvpTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   .  r   zMvpTokenizerFast.__init__Nr   r   r   r   r   rB   +  r   rB   c                   @   r   )NllbTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   5  r   zNllbTokenizerFast.__init__Nr   r   r   r   r   rC   2  r   rC   c                   @   r   )NougatTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   <  r   zNougatTokenizerFast.__init__Nr   r   r   r   r   rD   9  r   rD   c                   @   r   )OpenAIGPTTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   C  r   zOpenAIGPTTokenizerFast.__init__Nr   r   r   r   r   rE   @  r   rE   c                   @   r   )PegasusTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   J  r   zPegasusTokenizerFast.__init__Nr   r   r   r   r   rF   G  r   rF   c                   @   r   )RealmTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   Q  r   zRealmTokenizerFast.__init__Nr   r   r   r   r   rG   N  r   rG   c                   @   r   )ReformerTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   X  r   zReformerTokenizerFast.__init__Nr   r   r   r   r   rH   U  r   rH   c                   @   r   )RemBertTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   _  r   zRemBertTokenizerFast.__init__Nr   r   r   r   r   rI   \  r   rI   c                   @   r   )RobertaTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   f  r   zRobertaTokenizerFast.__init__Nr   r   r   r   r   rJ   c  r   rJ   c                   @   r   )RoFormerTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   m  r   zRoFormerTokenizerFast.__init__Nr   r   r   r   r   rK   j  r   rK   c                   @   r   )SeamlessM4TTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   t  r   z!SeamlessM4TTokenizerFast.__init__Nr   r   r   r   r   rL   q  r   rL   c                   @   r   )SplinterTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r   {  r   zSplinterTokenizerFast.__init__Nr   r   r   r   r   rM   x  r   rM   c                   @   r   )SqueezeBertTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   z!SqueezeBertTokenizerFast.__init__Nr   r   r   r   r   rN     r   rN   c                   @   r   )T5TokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   zT5TokenizerFast.__init__Nr   r   r   r   r   rO     r   rO   c                   @   r   )WhisperTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   zWhisperTokenizerFast.__init__Nr   r   r   r   r   rP     r   rP   c                   @   r   )XGLMTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   zXGLMTokenizerFast.__init__Nr   r   r   r   r   rQ     r   rQ   c                   @   r   )XLMRobertaTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   z XLMRobertaTokenizerFast.__init__Nr   r   r   r   r   rR     r   rR   c                   @   r   )XLNetTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   zXLNetTokenizerFast.__init__Nr   r   r   r   r   rS     r   rS   c                   @   r   )PreTrainedTokenizerFastr   c                 O   r   r   r	   r
   r   r   r   r     r   z PreTrainedTokenizerFast.__init__Nr   r   r   r   r   rT     r   rT   N)@utilsr   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   r   r   r   r   <module>   s|   