o h['@sdZddlZddlZddlmZmZmZddlmZm Z er#ddl Z ddl m Z ddl mZeeZdd iZdd d iiZd d iZd dZGdddeZGddde ZdS)z Tokenization classes for CPMAnt.N)ListOptionalTuple)is_jieba_availablerequires_backends)PreTrainedTokenizer)logging vocab_filez vocab.txtzopenbmb/cpm-ant-10bz>https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txticCsft}t|ddd }|}Wdn1swYt|D] \}}|d}|||<q#|S)z*Loads a vocabulary file into a dictionary.rutf-8encodingN ) collections OrderedDictopen readlines enumeraterstrip)r vocabreadertokensindextokenrd/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/cpmant/tokenization_cpmant.py load_vocab-s   rc@seZdZdddZddZdS) WordpieceTokenizercCs||_||_||_dSN)r unk_tokenmax_input_chars_per_word)selfrr"r#rrr__init__9s zWordpieceTokenizer.__init__cCst|}t||jkr|jgSd}g}|t|krXt|}d}||krs,      zWordpieceTokenizer.tokenizeN)rr )__name__ __module__ __qualname__r%r2rrrrr8s  rc sBeZdZdZeZeZeZ ddgZ dZ     d4fdd Z e ddZe ddZe ddZe defddZddZddZfddZdd Zd!eedefd"d#Zd$d%Zd&d'Zd5d)ed*eedeefd+d,Zd5d-eed.eedeefd/d0Z d6d-eed.eeed1e deeffd2d3 Z!Z"S)7CpmAntTokenizera Construct a CPMAnt tokenizer. Based on byte-level Byte-Pair-Encoding. Args: vocab_file (`str`): Path to the vocabulary file. bod_token (`str`, *optional*, defaults to `""`): The beginning of document token. eod_token (`str`, *optional*, defaults to `""`): The end of document token. bos_token (`str`, *optional*, defaults to `""`): The beginning of sequence token. eos_token (`str`, *optional*, defaults to `""`): The end of sequence token. pad_token (`str`, *optional*, defaults to `""`): The token used for padding. unk_token (`str`, *optional*, defaults to `""`): The unknown token. line_token (`str`, *optional*, defaults to `""`): The line token. space_token (`str`, *optional*, defaults to `""`): The space token. input_idsattention_maskFrleftc st|dg||_||_t||_|j| |jd<|j||jd<|j| =|j|=tt|jddd|_dd|jD|_ t |j|d |_ t j d |||||||| | d | dS) Njieba rcS|dSNr'rxrrrz*CpmAntTokenizer.__init__..keycSsi|]\}}||qSrr).0kvrrr sz,CpmAntTokenizer.__init__..)rr") bod_token eod_token bos_token eos_token pad_tokenr" line_token space_token padding_sider)rrOrPrencoderrrsorteditemsdecoderrwordpiece_tokenizersuperr%) r$r rOrPrQrRrSr"rTrUrVkwargs __class__rrr%ws0   zCpmAntTokenizer.__init__cC |j|jSr!)rWrOr$rrr bod_token_id zCpmAntTokenizer.bod_token_idcCr`r!)rWrPrarrr eod_token_idrczCpmAntTokenizer.eod_token_idcCs |jdS)NrrWrarrr newline_id zCpmAntTokenizer.newline_idreturncCs t|jSr!)r)rWrarrr vocab_sizergzCpmAntTokenizer.vocab_sizecCst|jfi|jSr!)dictrWadded_tokens_encoderrarrr get_vocabszCpmAntTokenizer.get_vocabcCs.g}tj|ddD] }||j|q |S)zTokenize a string.F)cut_all)rAcutextendr[r2)r$text output_tokensrFrrr _tokenizeszCpmAntTokenizer._tokenizec s4dd|D}fdd|D}tj|fi|S)zDecode ids into a string.cSsg|]}|dkr|qS)rr)rKirrr sz+CpmAntTokenizer._decode..cs.g|]}|jkr|jkr|jkr|qSr) pad_token_id eos_token_id bos_token_id)rKrFrarrrts()r\_decode)r$ token_idsr]r^rarrxs  zCpmAntTokenizer._decodecCs ||jvSr!rer$rrrrcheck zCpmAntTokenizer.checkrcCs d|S)Nr&)r*)r$rrrrconvert_tokens_to_stringr|z(CpmAntTokenizer.convert_tokens_to_stringcCs|j||j|jS)z0Converts a token (str) in an id using the vocab.)rWgetr"rzrrr_convert_token_to_idsz$CpmAntTokenizer._convert_token_to_idcCs|j||jS)z=Converts an index (integer) in a token (str) using the vocab.)rZr~r")r$rrrr_convert_id_to_tokensz$CpmAntTokenizer._convert_id_to_tokenNsave_directoryfilename_prefixcCs*tj|rtj||r|dndtd}n |r|dnd|}d}d|jvr5|jd|jd<|jd=d|jvrF|jd|jd<|jd=tt|j d d d |_t |d d d.}|j D]\}}||krut d|d|}| |d|d7}qbWd|fS1swY|fS)N-r&r rrBr?rr>cSrCrDrrErrrrGrHz1CpmAntTokenizer.save_vocabulary..rIwr r zSaving vocabulary to z\: vocabulary indices are not consecutive. Please check that the vocabulary is not corrupted!r')ospathisdirr*VOCAB_FILES_NAMESrWrrrXrYrloggerwarningwrite)r$rrr rwriterr token_indexrrrsave_vocabularys6        zCpmAntTokenizer.save_vocabulary token_ids_0 token_ids_1cCs,|dur |jg|S|jg||jg|S)a1 Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. A CPMAnt sequence has the following format: - single sequence: `[BOS] Sequence`. Args: token_ids_0 (`List[int]`): The first tokenized sequence that special tokens will be added. token_ids_1 (`List[int]`): The optional second tokenized sequence that special tokens will be added. Returns: `List[int]`: The model input with special tokens. N)rw)r$rrrrr build_inputs_with_special_tokenss z0CpmAntTokenizer.build_inputs_with_special_tokensalready_has_special_tokenscsZ|r tj||ddS|dur#dgdgt|dgdgt|Sdgdgt|S)a Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` method. Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. T)rrrNr'r)r\get_special_tokens_maskr))r$rrrr^rrrs(z'CpmAntTokenizer.get_special_tokens_mask) r9r:r;r<r=rr>r?r@r!)NF)#r3r4r5__doc__rvocab_files_namesPRETRAINED_VOCAB_FILES_MAPpretrained_vocab_files_map&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESmax_model_input_sizesmodel_input_namesadd_prefix_spacer%propertyrbrdrfintrirlrrrxr{rstrr}rrrrrrboolr __classcell__rrr^rr6XsX*     $ r6)rrrtypingrrrtransformers.utilsrrrAtokenization_utilsrutilsr get_loggerr3rrrrrobjectrr6rrrrs(