o h$ @sUdZddlmZmZmZddlmZmZddlm Z e e Z ddiZ dZdZd Zd Zd Zd Zd ZedededededediZeeefed<ddeDZeeefed<GdddeZdS)z Tokenization classes for CANINE.)DictListOptional) AddedTokenPreTrainedTokenizer)loggingznielsr/canine-siiiiiiz[CLS]z[SEP]z[BOS]z[MASK]z[PAD]z [RESERVED]SPECIAL_CODEPOINTScCi|]\}}||qSr ).0 codepointnamer r d/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/canine/tokenization_canine.py ;rSPECIAL_CODEPOINTS_BY_NAMEc sHeZdZdZeZeeeeeeeeee ee ddffdd Z e de fddZd d Zd edeefd d Zdede fddZde defddZddZ d$dee deee dee fddZ d%dee deee dedee ffdd Z d$dee deee dee fddZd$d ed!eefd"d#ZZS)&CanineTokenizera Construct a CANINE tokenizer (i.e. a character splitter). It turns text into a sequence of characters, and then converts each character into its Unicode code point. [`CanineTokenizer`] inherits from [`PreTrainedTokenizer`]. Refer to superclass [`PreTrainedTokenizer`] for usage examples and documentation concerning parameters. Args: model_max_length (`int`, *optional*, defaults to 2048): The maximum sentence length the model accepts. Fr c st|tr t|dddn|}t|trt|dddn|}t|tr(t|dddn|}t|tr6t|dddn|}t|trDt|dddn|}t|trRt|dddn|}i|_tD] \} } | |j| <q[dd|jD|_t|_t |j|_ t j d||||||||d| dS)NF)lstriprstripTcSr r r )r rrr r rris z,CanineTokenizer.__init__..) bos_token eos_token sep_token cls_token pad_token mask_tokenadd_prefix_spacemodel_max_lengthr ) isinstancestrr_special_codepointsr items_special_codepoint_stringsUNICODE_VOCAB_SIZE_unicode_vocab_sizelen_num_special_tokenssuper__init__) selfrrrrrrrrkwargsrr __class__r rr)Ns4    zCanineTokenizer.__init__returncCs|jSN)r%)r*r r r vocab_size|szCanineTokenizer.vocab_sizecCs$ddt|jD}||j|S)NcSsi|]}t||qSr )chr)r ir r rrrz-CanineTokenizer.get_vocab..)ranger0updateadded_tokens_encoder)r*vocabr r r get_vocabs zCanineTokenizer.get_vocabtextcCst|S)z5Tokenize a string (i.e. perform character splitting).)list)r*r8r r r _tokenizeszCanineTokenizer._tokenizetokencCs*zt|WStytd|dw)zaConverts a token (i.e. a Unicode character) in an id (i.e. its integer Unicode code point value).zinvalid token: '')ord TypeError ValueError)r*r;r r r_convert_token_to_ids   z$CanineTokenizer._convert_token_to_idindexcCs:z|tvr t|WSt|WStytd|w)z Converts a Unicode code point (integer) in a token (str). In case it's a special code point, convert to human-readable format. z invalid id: )r r1r>r?)r*rAr r r_convert_id_to_tokens   z$CanineTokenizer._convert_id_to_tokencCs d|S)N)join)r*tokensr r rconvert_tokens_to_strings z(CanineTokenizer.convert_tokens_to_stringN token_ids_0 token_ids_1cCs4|jg}|jg}|||}|dur|||7}|S)a Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A CANINE sequence has the following format: - single sequence: `[CLS] X [SEP]` - pair of sequences: `[CLS] A [SEP] B [SEP]` Args: token_ids_0 (`List[int]`): List of IDs to which the special tokens will be added. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. Returns: `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. N) sep_token_id cls_token_idr*rGrHsepclsresultr r r build_inputs_with_special_tokenss   z0CanineTokenizer.build_inputs_with_special_tokensalready_has_special_tokenscsT|r tj||ddSdgdgt|dg}|dur(|dgt|dg7}|S)a Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` method. Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. T)rGrHrPrN)r(get_special_tokens_maskr&)r*rGrHrPrNr,r rrRsz'CanineTokenizer.get_special_tokens_maskcCsH|jg}|jg}t|||dg}|dur"|t||dg7}|S)a Create a mask from the two sequences passed to be used in a sequence-pair classification task. A CANINE sequence pair mask has the following format: ``` 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 | first sequence | second sequence | ``` If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s). Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. Returns: `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s). rNrQ)rIrJr&rKr r r$create_token_type_ids_from_sequencess z4CanineTokenizer.create_token_type_ids_from_sequencessave_directoryfilename_prefixcCsdS)Nr r )r*rTrUr r rsave_vocabularyszCanineTokenizer.save_vocabularyr/)NF)__name__ __module__ __qualname____doc__&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESmax_model_input_sizesr1CLSSEPPADMASKr)propertyintr0r7r rr:r@rBrFrrOboolrRrSrV __classcell__r r r,rr>s\ .      rN)rZtypingrrrtokenization_utilsrrutilsr get_loggerrWloggerr[r$r_r]r^BOSr`RESERVEDr rbr __annotations__r"rrr r r rs.   "