o h$@sdZddlZddlZddlZddlmZmZmZmZm Z m Z ddl m Z ddl mZmZer3ddlZeeZddiZdd d iiZd d iZd d ZGddde ZdS)zTokenization class for VITS.N)AnyDictListOptionalTupleUnion)PreTrainedTokenizer)is_phonemizer_availablelogging vocab_filez vocab.jsonzfacebook/mms-tts-engzChttps://huggingface.co/facebook/mms-tts-eng/resolve/main/vocab.jsonicCs td}||}|du}|S)Nz [^\x00-\x7F])recompilesearch) input_stringnon_roman_patternmatch has_non_romanr`/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/vits/tokenization_vits.pyhas_non_roman_characters/s  rc seZdZdZeZeZeZ ddgZ       d& d'fd d Z e d d Z ddZddZddZ d(dededeed eeeeefffddZded eefddZdeed efddZddZd d!Zd)d"ed#eed eeedffd$d%ZZS)* VitsTokenizera Construct a VITS tokenizer. Also supports MMS-TTS. This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to this superclass for more information regarding those methods. Args: vocab_file (`str`): Path to the vocabulary file. language (`str`, *optional*): Language identifier. add_blank (`bool`, *optional*, defaults to `True`): Whether to insert token id 0 in between the other tokens. normalize (`bool`, *optional*, defaults to `True`): Whether to normalize the input text by removing all casing and punctuation. phonemize (`bool`, *optional*, defaults to `True`): Whether to convert the input text into phonemes. is_uroman (`bool`, *optional*, defaults to `False`): Whether the `uroman` Romanizer needs to be applied to the input text prior to tokenizing. input_idsattention_maskNTFreturnc st|dd} t| |_Wdn1swYdd|jD|_||_||_||_||_ ||_ t j d|||||||d| dS)Nutf-8encodingcSsi|]\}}||qSrr).0kvrrr csz*VitsTokenizer.__init__..) pad_token unk_tokenlanguage add_blank normalize phonemize is_uromanr) openjsonloadencoderitemsdecoderr&r'r(r)r*super__init__) selfr r$r%r&r'r(r)r*kwargs vocab_handle __class__rrr2Ts(  zVitsTokenizer.__init__cCs t|jSN)lenr.r3rrr vocab_sizevs zVitsTokenizer.vocab_sizecs(fddtjD}|j|S)Ncsi|]}||qSr)convert_ids_to_tokens)r ir:rrr#{sz+VitsTokenizer.get_vocab..)ranger;updateadded_tokens_encoder)r3vocabrr:r get_vocabzs zVitsTokenizer.get_vocabcCst|jt|j}d}d}|t|krMd}|D]}|||t||kr8||7}|t|7}d}nq|sG|||7}|d7}|t|ks|S)zfLowercase the input string, respecting any special token ids that may be part or entirely upper-cased.rFT)listr.keysr@r9lower)r3rall_vocabulary filtered_textr= found_matchwordrrrnormalize_texts"    zVitsTokenizer.normalize_textcCs|jdkr |dd}|S)z4Special treatment of characters in certain languagesronuțuţ)r&replace)r3textrrr_preprocess_chars  zVitsTokenizer._preprocess_charrOis_split_into_wordsr(c s|dur|nj}|r|}|}t|r!jr!tdjrAts+t dt j|dddddd}t dd |}||fS|rSd ttfd d |}||fS) a Performs any necessary transformations before tokenization. This method should pop the arguments from kwargs and return the remaining `kwargs` as well. We test the `kwargs` at the end of the encoding process to be sure all the arguments have been used. Args: text (`str`): The text to prepare. is_split_into_words (`bool`, *optional*, defaults to `False`): Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace) which it will tokenize. normalize (`bool`, *optional*, defaults to `None`): Whether or not to apply punctuation and casing normalization to the text inputs. Typically, VITS is trained on lower-cased and un-punctuated text. Hence, normalization is used to ensure that the input text consists only of lower-case characters. kwargs (`Dict[str, Any]`, *optional*): Keyword arguments to use for the tokenization. Returns: `Tuple[str, Dict[str, Any]]`: The prepared text and the unused kwargs. NzText to the tokenizer contains non-Roman characters. Ensure the `uroman` Romanizer is applied to the text prior to passing it to the tokenizer. See `https://github.com/isi-nlp/uroman` for details.zEPlease install the `phonemizer` Python package to use this tokenizer.zen-usespeakT)r&backendstrippreserve_punctuation with_stressz\s+ rCcs |jvSr8)r.)charr:rrs z8VitsTokenizer.prepare_for_tokenization..)r(rLrPrr*loggerwarningr)r ImportError phonemizerr subjoinrEfilterrT)r3rOrQr(r4rIrr:rprepare_for_tokenizations0   z&VitsTokenizer.prepare_for_tokenizationcCs@t|}|jr|dgt|dd}||ddd<|}|S)z]Tokenize a string by inserting the `` token at the boundary between adjacent characters.rrDN)rEr'_convert_id_to_tokenr9)r3rOtokens interspersedrrr _tokenizes zVitsTokenizer._tokenizerdcCs*|jrt|dkr|ddd}d|S)NrDrbrC)r'r9r_)r3rdrrrconvert_tokens_to_strings z&VitsTokenizer.convert_tokens_to_stringcCs|j||j|jS)z0Converts a token (str) in an id using the vocab.)r.getr%)r3tokenrrr_convert_token_to_idsz"VitsTokenizer._convert_token_to_idcCs |j|S)z=Converts an index (integer) in a token (str) using the vocab.)r0rh)r3indexrrrrcs z"VitsTokenizer._convert_id_to_tokensave_directoryfilename_prefixc Cstj|std|ddStj||r|dndtd}t|ddd}|t j |j d d d d d Wd|fS1sEwY|fS)NzVocabulary path (z) should be a directory-rCr wrrrbTF)indent sort_keys ensure_ascii ) ospathisdirrZerrorr_VOCAB_FILES_NAMESr+writer,dumpsr.)r3rlrmr frrrsave_vocabularys   zVitsTokenizer.save_vocabulary)rrNTTTF)rN)FNr8) __name__ __module__ __qualname____doc__rxvocab_files_namesPRETRAINED_VOCAB_FILES_MAPpretrained_vocab_files_map&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESmax_model_input_sizesmodel_input_namesr2propertyr;rBrLrPstrboolrrrrrarrfrgrjrcrr| __classcell__rrr6rr9sH "  < 0r)rr,rtr typingrrrrrrtokenization_utilsr utilsr r r] get_loggerr}rZrxrrrrrrrrs&