o h5@sdZddlZddlmZmZmZddlmZddlm Z ddl m Z e e Zdd iZdd d d iZd d d ZddZGdddeZdS)zTokenization classes for ESM.N)ListOptionalUnion)PreTrainedTokenizer) AddedToken)logging vocab_file vocab.txtzGhttps://huggingface.co/facebook/esm2_t6_8M_UR50D/resolve/main/vocab.txtzIhttps://huggingface.co/facebook/esm2_t12_35M_UR50D/resolve/main/vocab.txt)zfacebook/esm2_t6_8M_UR50Dzfacebook/esm2_t12_35M_UR50DicCsHt|d}|}dd|DWdS1swYdS)NrcSsg|]}|qS)strip).0lr r ^/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/esm/tokenization_esm.py ,sz#load_vocab_file..)openread splitlines)r flinesr r rload_vocab_file)s   $rc s8eZdZdZeZeZeZ ddgZ     d-fd d Z d e d e fd dZde d e fddZddZd.ddZddZde d e fddZd e d e fddZ d/dee deee d ee fd d!Z d0dedeed"ed ee fd#d$Zd%d&Zed e fd'd(Zd.d)eee eefd*ed e ffd+d, ZZ S)1 EsmTokenizerz& Constructs an ESM tokenizer. input_idsattention_maskc sft||_tt|j|_ddt|jD|_tjd|||||d||j|_| |jdS)NcSi|]\}}||qSr r )rindtokr r r Ez)EsmTokenizer.__init__..) unk_token cls_token pad_token mask_token eos_tokenr ) r all_tokensdict enumerate _id_to_token _token_to_idsuper__init__unique_no_split_tokens _update_trie)selfr r%r&r'r(r)kwargs __class__r rr09s  zEsmTokenizer.__init__indexreturncC|j||jSNr-getr%r3r7r r r_convert_id_to_tokenUz!EsmTokenizer._convert_id_to_tokentokencC|j||j|jSr:r.r<r%r3r@r r r_convert_token_to_idXz!EsmTokenizer._convert_token_to_idcKs|Sr:)split)r3textr4r r r _tokenize[szEsmTokenizer._tokenizeFcCs t|jSr:)lenr-)r3with_added_tokensr r rget_vocab_size^s zEsmTokenizer.get_vocab_sizecCsddt|jDS)NcSr r r )rir@r r rr#br$z*EsmTokenizer.get_vocab..)r,r*r3r r r get_vocabaszEsmTokenizer.get_vocabcCrAr:rBrCr r r token_to_iddrEzEsmTokenizer.token_to_idcCr9r:r;r=r r r id_to_tokengr?zEsmTokenizer.id_to_tokenN token_ids_0 token_ids_1cCs\|jg}|jg}|dur|jdur||S|||S|jdur$td|||||S)Nz=Cannot tokenize multiple sequences when EOS token is not set!) cls_token_id eos_token_id ValueError)r3rQrRclssepr r r build_inputs_with_special_tokensjs   z-EsmTokenizer.build_inputs_with_special_tokensalready_has_special_tokenscsd|r|dur tdfdd|DSdgdgt|dg}|dur0|dgt|dg7}|S)a Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods. Args: token_ids_0 (`List[int]`): List of ids of the first sequence. token_ids_1 (`List[int]`, *optional*): List of ids of the second sequence. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. NzYou should not supply a second sequence if the provided sequence of ids is already formatted with special tokens for the model.csg|] }|jvr dndqS)r)all_special_ids)rr@rMr rrsz8EsmTokenizer.get_special_tokens_mask..rZr)rUrI)r3rQrRrYmaskr rMrget_special_tokens_maskxsz$EsmTokenizer.get_special_tokens_maskcCsdtj||r |dndd}t|d}|d|jWd|fS1s*wY|fS)N-r w )ospathjoinrwriter*)r3save_directoryfilename_prefixr rr r rsave_vocabularys  zEsmTokenizer.save_vocabularycCs |jddS)NF)rJ)rKrMr r r vocab_sizes zEsmTokenizer.vocab_size new_tokensspecial_tokenscstj|ddS)NT)rk)r/ _add_tokens)r3rjrkr5r rrlr?zEsmTokenizer._add_tokens)rrrrr)Fr:)NF)!__name__ __module__ __qualname____doc__VOCAB_FILES_NAMESvocab_files_namesPRETRAINED_VOCAB_FILES_MAPpretrained_vocab_files_map&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESmax_model_input_sizesmodel_input_namesr0intstrr>rDrHrKrNrOrPrrrXboolr]rhpropertyrirrrl __classcell__r r r5rr/sR    4r)rprbtypingrrrtokenization_utilsrtokenization_utils_baserutilsr get_loggerrmloggerrqrsrurrr r r rs$