o hS@sNUdZddlZddlZddlZddlZddlmZddlmZm Z m Z m Z m Z ddl mZddlmZmZddlmZdd lmZdd lmZmZmZmZmZd d lmZd dlmZd dl m!Z!m"Z"m#Z#m$Z$m%Z%ersddl&m'Z'ndZ'e(e)Z*ereZ+ee,e e e,e e,ffe-d<negderdnderdndffdderdndffdderdndffdderdnderdndffddderdndffderd nddffd!d"d#erd$nderd%ndffd&d'erd(ndffd)d*d+d,derdndffd-d.erd/ndffd0der(d1ndffd2d3er4d4ndffd5der@dndffd6d7erMd8nderTd9ndffd:d;derbdndffderzd?ndffd@d>erd?ndffdAdBerdCnderdDndffdEdFerdGndffdHdIerdJndffdKerdLnderdMndffdNdOdPdQd3erd4ndffdRdSerdTndffdUerdVnderdWndffdXdYerdZndffd[d\er d]ndffd^d_erd`ndffdader$dndffdber/dcnddffdddeder>dfndffdgdhdierLdjndffdkdldmerZdnndffdoderfdndffdperqdqnddffdrd.er~d/ndffdsd.erd/ndffdtd.erd/ndffduderdvndffdwdxd.erd/ndffdydzd>erd?ndffd{d|erd}ndffd~dd3erd4ndffdderdndffdd.erd/ndffdderdnderdndffdderdndffdderdndffdder(dndffdder4dndffdder@dndffdderLdndffderWdnder^dndffdderjdndffddervdndffderdnderdndffddderdndffderdnddffderdnddffderdnderdndffderdnderdndffdd3erd4ndffdderdndffdderdnderdndffder dnderdndffderdnddffdder(dndffdder4dndffdder@dvndffdd3erLd4ndffderWdnder^dndffdderjdndffddervdndffdderdndffderdnderdndffderdnderdndffderdnderdndffdd>erd?ndffdderdndffdd.erd/ndffdd>erd?ndffdd>erd?ndffderd'nderd(ndffderd'nderd(ndffdӑder%dnder,dndffddFer8dGndffd֑dderFdndffderQdnddffdڑdder`dndffdܑdderndndffderydnderdndffderdnderdndffdderdndffdd3erd4ndffdd3erd4ndffddderdndffdderdvndffderdnderdndffderdnderdndffderdnddffdder dnddffddderdndffder'dnder.dndffder9dnder@dndffddddderUdndffderadnderhdndffdderudndffdderdndffdddd d d erd ndffd d>erd?ndffderdnderdndffdderϐdnddffderdnderdndffderdnderdndffderdnder dndffderdnderdndffder*dnder1dndffZ+ee!e+Z.dde!/DZ0de,fddZ1       !d/d"e e,ej2fd#e e e,ej2fd$e3d%e3d&e e e,e,fd'e e e3e,fd(e e,d)e3d*e,fd+d,Z4Gd-d.d.Z5dS(0z Auto Tokenizer class.N) OrderedDict) TYPE_CHECKINGDictOptionalTupleUnion)PretrainedConfig)get_class_from_dynamic_moduleresolve_trust_remote_code)PreTrainedTokenizer)TOKENIZER_CONFIG_FILE) cached_fileextract_commit_hashis_sentencepiece_availableis_tokenizers_availablelogging)EncoderDecoderConfig)_LazyAutoMapping)CONFIG_MAPPING_NAMES AutoConfigconfig_class_to_model_typemodel_type_to_module_name!replace_list_option_in_docstrings)PreTrainedTokenizerFastTOKENIZER_MAPPING_NAMESalbertAlbertTokenizerAlbertTokenizerFastalign BertTokenizerBertTokenizerFastbark)bart) BartTokenizerBartTokenizerFastbarthezBarthezTokenizerBarthezTokenizerFast)bartpho)BartphoTokenizerNbertzbert-generationBertGenerationTokenizer)z bert-japanese)BertJapaneseTokenizerN)bertweet)BertweetTokenizerNbig_birdBigBirdTokenizerBigBirdTokenizerFastbigbird_pegasusPegasusTokenizerPegasusTokenizerFast)biogpt)BioGptTokenizerN) blenderbot)BlenderbotTokenizerBlenderbotTokenizerFast)zblenderbot-small)BlenderbotSmallTokenizerNblipzblip-2 GPT2TokenizerGPT2TokenizerFastbloomBloomTokenizerFast bridgetowerRobertaTokenizerRobertaTokenizerFastbros)byt5) ByT5TokenizerN camembertCamembertTokenizerCamembertTokenizerFast)canine)CanineTokenizerN chinese_clipclapclip CLIPTokenizerCLIPTokenizerFastclipseg)clvp) ClvpTokenizerN code_llamaCodeLlamaTokenizerCodeLlamaTokenizerFastcodegenCodeGenTokenizerCodeGenTokenizerFastconvbertConvBertTokenizerConvBertTokenizerFastcpm CpmTokenizerCpmTokenizerFast)cpmant)CpmAntTokenizerN)ctrl) CTRLTokenizerN)zdata2vec-audioWav2Vec2CTCTokenizerNz data2vec-textdebertaDebertaTokenizerDebertaTokenizerFastz deberta-v2DebertaV2TokenizerDebertaV2TokenizerFast distilbertDistilBertTokenizerDistilBertTokenizerFastdprDPRQuestionEncoderTokenizerDPRQuestionEncoderTokenizerFastelectraElectraTokenizerElectraTokenizerFasternieernie_mErnieMTokenizer)esm) EsmTokenizerNfalconr)flaubert)FlaubertTokenizerNfnet FNetTokenizerFNetTokenizerFast)fsmt) FSMTTokenizerNfunnelFunnelTokenizerFunnelTokenizerFastgitzgpt-sw3GPTSw3Tokenizergpt2 gpt_bigcodegpt_neogpt_neoxGPTNeoXTokenizerFast)gpt_neox_japanese)GPTNeoXJapaneseTokenizerNgptj)zgptsan-japanese)GPTSanJapaneseTokenizerNgroupvitherbertHerbertTokenizerHerbertTokenizerFast)hubertrfibertideficsLlamaTokenizerFast instructblip)jukebox)JukeboxTokenizerNzkosmos-2XLMRobertaTokenizerXLMRobertaTokenizerFastlayoutlmLayoutLMTokenizerLayoutLMTokenizerFast layoutlmv2LayoutLMv2TokenizerLayoutLMv2TokenizerFast layoutlmv3LayoutLMv3TokenizerLayoutLMv3TokenizerFast layoutxlmLayoutXLMTokenizerLayoutXLMTokenizerFastled LEDTokenizerLEDTokenizerFastliltllamaLlamaTokenizerllava longformerLongformerTokenizerLongformerTokenizerFastlongt5 T5TokenizerT5TokenizerFast)luke) LukeTokenizerNlxmertLxmertTokenizerLxmertTokenizerFastm2m_100M2M100TokenizermarianMarianTokenizermbartMBartTokenizerMBartTokenizerFastmbart50MBart50TokenizerMBart50TokenizerFastmegaz megatron-bert)zmgp-str)MgpstrTokenizerNmistralmixtralmlukeMLukeTokenizer mobilebertMobileBertTokenizerMobileBertTokenizerFastmpnetMPNetTokenizerMPNetTokenizerFastmptmramt5 MT5TokenizerMT5TokenizerFastmusicgenmvp MvpTokenizerMvpTokenizerFastnezhanllb NllbTokenizerNllbTokenizerFastznllb-moe nystromformer oneformerz openai-gptOpenAIGPTTokenizerOpenAIGPTTokenizerFastoptowlv2owlvitpegasus pegasus_x) perceiver)PerceiverTokenizerN persimmonphi)phobert)PhobertTokenizerN pix2structplbartPLBartTokenizer) prophetnet)ProphetNetTokenizerNqdqbert)rag) RagTokenizerNrealmRealmTokenizerRealmTokenizerFastreformerReformerTokenizerReformerTokenizerFastrembertRemBertTokenizerRemBertTokenizerFast retribertRetriBertTokenizerRetriBertTokenizerFastrobertazroberta-prelayernorm)roc_bert)RoCBertTokenizerNroformerRoFormerTokenizerRoFormerTokenizerFastrwkv seamless_m4tSeamlessM4TTokenizerSeamlessM4TTokenizerFastseamless_m4t_v2speech_to_textSpeech2TextTokenizer)speech_to_text_2)Speech2Text2TokenizerNspeecht5SpeechT5Tokenizer)splinter)SplinterTokenizerSplinterTokenizerFast squeezebertSqueezeBertTokenizerSqueezeBertTokenizerFastswitch_transformerst5)tapas)TapasTokenizerN)tapex)TapexTokenizerN)z transfo-xl)TransfoXLTokenizerNtvpumt5vilt visual_bert)vits) VitsTokenizerN)wav2vec2rf)zwav2vec2-conformerrf)wav2vec2_phoneme)Wav2Vec2PhonemeCTCTokenizerNwhisperWhisperTokenizerWhisperTokenizerFastxclipxglm XGLMTokenizerXGLMTokenizerFast)xlm) XLMTokenizerNzxlm-prophetnetXLMProphetNetTokenizerz xlm-robertazxlm-roberta-xlxlnetXLNetTokenizerXLNetTokenizerFastxmodyosocCsi|]\}}||qSr8).0kvr8r8`/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py sr= class_namec Cs|dkrtStD]'\}}||vr1t|}td|d}zt||WSty0Yq wq tj D]\}}|D]}t|dd|krM|Sq=q7td}t ||r^t||SdS)Nr.ztransformers.models__name__ transformers) rritemsr importlib import_modulegetattrAttributeErrorTOKENIZER_MAPPING_extra_contenthasattr)r> module_name tokenizersmoduleconfig tokenizer main_moduler8r8r<tokenizer_class_from_names,      rPFpretrained_model_name_or_path cache_dirforce_downloadresume_downloadproxiestokenrevisionlocal_files_only subfolderc Ks| dd} | durtdt|durtd| }| dd} t|t||||||||dd| d } | dur Passing `token=True` is required when you want to use a private model. Returns: `Dict`: The configuration of the tokenizer. Examples: ```python # Download configuration from huggingface.co and cache. tokenizer_config = get_tokenizer_config("bert-base-uncased") # This model does not have a tokenizer config so the result will be an empty dict. tokenizer_config = get_tokenizer_config("xlm-roberta-base") # Save a pretrained tokenizer locally and you can reload its config from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") tokenizer.save_pretrained("tokenizer-test") tokenizer_config = get_tokenizer_config("tokenizer-test") ```use_auth_tokenNrThe `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.V`token` and `use_auth_token` are both specified. Please set only the argument `token`. _commit_hashF) rSrTrUrVrWrXrYrZ%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errorsr^z\Could not locate the tokenizer configuration file, will try to use the model config instead.zutf-8)encoding)popwarningswarn FutureWarning ValueErrorgetrr loggerinforopenjsonload)rRrSrTrUrVrWrXrYrZkwargsr[ commit_hashresolved_config_filereaderresultr8r8r<get_tokenizer_configsD I    rrc@s6eZdZdZddZeeeddZd dd Z dS) AutoTokenizera This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when created with the [`AutoTokenizer.from_pretrained`] class method. This class cannot be instantiated directly using `__init__()` (throws an error). cCstd)Nz}AutoTokenizer is designed to be instantiated using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method.)EnvironmentError)selfr8r8r<__init__[szAutoTokenizer.__init__c Os|dd}|dur tdt|dddurtd||d<|dd}d|d<|d d}|d d}|d d}|durd} t|d} | duratd |d dddtDd| \} } |ru| durpt | } nt d| dur}t | } | durtd| d| j |g|Ri|St |fi|} d| vr| d|d<| d}d}d| vrt| dttfr| d}n| ddd}|durt|tstj |fd |i|}|j}t|drd|jvr|jd}|du}|dupt|tv}t||||}|rB|rB|r|ddur|d}n|d}t||fi|} |dd}tj|r6| | j |g|Ri|S|durd} |r[|ds[|d}t |} | durf|}t |} | durstd|d| j |g|Ri|St|trt|j t|j!urt d|j!j"d|j j"d |j!}t#t|j$}|durtt|\}}|r|s|dur|j |g|Ri|S|dur|j |g|Ri|Std!td"|j"d#dd$dtDd)%a8 Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary. The tokenizer class to instantiate is selected based on the `model_type` property of the config object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by falling back to using pattern matching on `pretrained_model_name_or_path`: List options Params: pretrained_model_name_or_path (`str` or `os.PathLike`): Can be either: - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`. - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. - A path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (like Bert or XLNet), e.g.: `./my_model_directory/vocab.txt`. (Not applicable to all derived classes) inputs (additional positional arguments, *optional*): Will be passed along to the Tokenizer `__init__()` method. config ([`PretrainedConfig`], *optional*) The configuration object used to determine the tokenizer class to instantiate. cache_dir (`str` or `os.PathLike`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download the model weights and configuration files and override the cached versions if they exist. resume_download (`bool`, *optional*, defaults to `False`): Whether or not to delete incompletely received files. Will attempt to resume the download if such a file exists. proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. subfolder (`str`, *optional*): In case the relevant files are located inside a subfolder of the model repo on huggingface.co (e.g. for facebook/rag-token-base), specify it here. use_fast (`bool`, *optional*, defaults to `True`): Use a [fast Rust-based tokenizer](https://huggingface.co/docs/tokenizers/index) if it is supported for a given model. If a fast tokenizer is not available for a given model, a normal Python-based tokenizer is returned instead. tokenizer_type (`str`, *optional*): Tokenizer type to be loaded. trust_remote_code (`bool`, *optional*, defaults to `False`): Whether or not to allow for custom models defined on the Hub in their own modeling files. This option should only be set to `True` for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. kwargs (additional keyword arguments, *optional*): Will be passed to the Tokenizer `__init__()` method. Can be used to set special tokens like `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`, `additional_special_tokens`. See parameters in the `__init__()` for more details. Examples: ```python >>> from transformers import AutoTokenizer >>> # Download vocabulary from huggingface.co and cache. >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") >>> # Download vocabulary from huggingface.co (user-uploaded) and cache. >>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased") >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*) >>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/") >>> # Download vocabulary from huggingface.co and define model-specific arguments >>> tokenizer = AutoTokenizer.from_pretrained("roberta-base", add_prefix_space=True) ```r[Nr\rWr]rMT _from_autouse_fasttokenizer_typetrust_remote_codezPassed `tokenizer_type` z3 does not exist. `tokenizer_type` should be one of z, css|]}|VqdSNr8r9cr8r8r< sz0AutoTokenizer.from_pretrained..r?zt`use_fast` is set to `True` but the tokenizer class does not have a fast version. Falling back to the slow version.zTokenizer class z is not currently imported.r^tokenizer_classauto_maprsrr code_revisionFastz- does not exist or is not currently imported.z The encoder model config class: z3 is different from the decoder model config class: z. It is not recommended to use the `AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder specific tokenizer classes.zzThis tokenizer cannot be instantiated. Please make sure you have `sentencepiece` installed in order to use this tokenizer.z!Unrecognized configuration class z8 to build an AutoTokenizer. Model type should be one of css|]}|jVqdSr{)r@r|r8r8r<r~1s)%rbrcrdrergrfrjoinkeysrPrhwarningfrom_pretrainedrr isinstancetuplelistr rrrIrtyperGr r ospathisdirregister_for_auto_classendswithrdecoderencoder __class__rr@)clsrRinputsrmr[rMrxryrzrtokenizer_class_tupletokenizer_class_nametokenizer_fast_class_nametokenizer_configconfig_tokenizer_classtokenizer_auto_maphas_remote_codehas_local_code class_ref_tokenizer_class_candidate model_typetokenizer_class_pytokenizer_class_fastr8r8r<ras O                        zAutoTokenizer.from_pretrainedNFcCs|dur |dur td|durt|trtd|dur&t|tr&td|durD|durDt|trD|j|krDtd|jd|d|tjvr[t|\}}|durU|}|dur[|}tj|||f|ddS) a Register a new tokenizer in this mapping. Args: config_class ([`PretrainedConfig`]): The configuration corresponding to the model to register. slow_tokenizer_class ([`PretrainedTokenizer`], *optional*): The slow tokenizer to register. fast_tokenizer_class ([`PretrainedTokenizerFast`], *optional*): The fast tokenizer to register. NzKYou need to pass either a `slow_tokenizer_class` or a `fast_tokenizer_classz:You passed a fast tokenizer in the `slow_tokenizer_class`.z:You passed a slow tokenizer in the `fast_tokenizer_class`.zThe fast tokenizer class you are passing has a `slow_tokenizer_class` attribute that is not consistent with the slow tokenizer class you passed (fast tokenizer has z and you passed z!. Fix one of those so they match!)exist_ok)rf issubclassrr slow_tokenizer_classrGrHregister) config_classrfast_tokenizer_classr existing_slow existing_fastr8r8r<r4s2    zAutoTokenizer.register)NNF) r@ __module__ __qualname____doc__rv classmethodrrrrr8r8r8r<rsSs Rrs)NFFNNNFrQ)6rrCrkrrc collectionsrtypingrrrrrconfiguration_utilsr dynamic_module_utilsr r tokenization_utilsr tokenization_utils_baser utilsrrrrrencoder_decoderr auto_factoryrconfiguration_autorrrrrtokenization_utils_fastr get_loggerr@rhrstr__annotations__rGrBCONFIG_TO_TYPErPPathLikeboolrrrsr8r8r8r<sJ        *         !"#$%&')/029@FHNOQWXYZ[]ceklmnopqrstuvwxyz{|}~  "#$%'.4568?EFGHIKQRSU\cijklmov|   !"$'()+289:;=AHNOPQSYZ[\]^_`bhikry   n