o
    h                    @   s  d Z ddlZddlZddlmZ ddlmZmZmZ ddl	Z
ddlZddlZddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZmZmZmZm Z m!Z! ddl"m#Z# e $e%Z&dZ'dZ(dZ)g dZ*dZ+dZ,ddgZ-eG dd deZ.		dRdee/e/f de0de/deej1 de/de
j2fdd Z3G d!d" d"ej4Z5G d#d$ d$ej4Z6G d%d& d&ej4Z7G d'd( d(ej4Z8G d)d* d*ej4Z9G d+d, d,ej4Z:G d-d. d.e:Z;G d/d0 d0ej4Z<G d1d2 d2ej4Z=G d3d4 d4ej4Z>G d5d6 d6ej4Z?G d7d8 d8ej4Z@G d9d: d:ej4ZAG d;d< d<ej4ZBG d=d> d>ej4ZCG d?d@ d@ej4ZDG dAdB dBeZEdCZFdDZGedEeFG dFdG dGeEZHedHeFG dIdJ dJeEZIedKeFdLG dMdN dNeEZJedOeFG dPdQ dQeEZKdS )Sz PyTorch UniSpeech model.    N)	dataclass)OptionalTupleUnion)nn)CrossEntropyLoss   )ACT2FN)is_deepspeed_zero3_enabled)BaseModelOutputCausalLMOutputSequenceClassifierOutputWav2Vec2BaseModelOutput)PreTrainedModel)ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )UniSpeechConfig   r   z/patrickvonplaten/unispeech-large-1500h-cv-timit)r   i$  i   zW'mister quilter is the apposl of the midle classes and weare glad to welcom his gosepl'gQ+1@z"microsoft/unispeech-large-1500h-cvz0microsoft/unispeech-large-multi-lingual-1500h-cvc                   @   sz   e Zd ZU dZdZeej ed< dZ	ejed< dZ
ejed< dZejed< dZeeej  ed< dZeeej  ed< dS )	UniSpeechForPreTrainingOutputaL  
    Output type of [`UniSpeechForPreTrainingOutput`], with potential hidden states and attentions.

    Args:
        loss (*optional*, returned when model is in train mode, `torch.FloatTensor` of shape `(1,)`):
            Total loss as the sum of the contrastive loss (L_m) and the diversity loss (L_d) as stated in the [official
            paper](https://arxiv.org/pdf/2006.11477.pdf) . (classification) loss.
        projected_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
            Hidden-states of the model projected to *config.proj_codevector_dim* that can be used to predict the masked
            projected quantized states.
        projected_quantized_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
            Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
            target vectors for contrastive loss.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nlossprojected_statesprojected_quantized_statescodevector_perplexityhidden_states
attentions)__name__
__module____qualname____doc__r   r   torchFloatTensor__annotations__r   r   r   r   r   r    r'   r'   f/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/unispeech/modeling_unispeech.pyr   B   s   
 r   shape	mask_probmask_lengthattention_mask	min_masksreturnc                    s  | \}dk rt dkrt d d dtjd   fdd}|dur:|d	  n
fd
dt|D }tj	|ft
d}g }	|}
|
dkrZ|S |D ];}||}tjjt|d  |dd}t|dkr}d }n|d }t|tj|
| tjd| g}|	| q\t|	}	t|	dddddf ||
f}	|	||
 }	tddddf }t|||
f||
 }|	| }	|	 d krd |	|	d k< t||	dd	 |S )af  
    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
    ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
    CPU as part of the preprocessing during training.

    Args:
        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
               the first element is the batch size and the second element is the length of the axis to span.
        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                    independently generated mask spans of length `mask_length` is computed by
                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                    actual percentage will be smaller.
        mask_length: size of the mask
        min_masks: minimum number of masked spans
        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                        each batch dimension.
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `c                    sX   t |     }t|}| kr }| d  |k r*t| d  d}|S )z;Given input length, compute how many spans should be maskedr   r   )intmax)input_lengthnum_masked_spanepsilonr+   r*   r-   sequence_lengthr'   r(   compute_num_masked_span   s   
z6_compute_mask_indices.<locals>.compute_num_masked_spanNc                    s   g | ]} qS r'   r'   .0_)r6   r'   r(   
<listcomp>   s    z)_compute_mask_indices.<locals>.<listcomp>dtyper   F)replace)
ValueErrornprandomranditemsumdetachtolistrangezerosboolchoicearangelenconcatenateonesint32appendarraybroadcast_toreshaper1   put_along_axis)r)   r*   r+   r,   r-   
batch_sizer7   input_lengthsspec_aug_maskspec_aug_mask_idxsmax_num_masked_spanr2   r3   spec_aug_mask_idxdummy_mask_idxoffsetsr'   r4   r(   _compute_mask_indicesg   s\   

r^   c                       &   e Zd Zd fdd	Zdd Z  ZS )UniSpeechNoLayerNormConvLayerr   c                    sj   t    |dkr|j|d  nd| _|j| | _tj| j| j|j| |j| |j	d| _
t|j | _d S )Nr   r   kernel_sizestridebias)super__init__conv_dimin_conv_dimout_conv_dimr   Conv1dconv_kernelconv_stride	conv_biasconvr	   feat_extract_activation
activationselfconfiglayer_id	__class__r'   r(   rf      s   
z&UniSpeechNoLayerNormConvLayer.__init__c                 C   s   |  |}| |}|S N)rn   rp   rr   r   r'   r'   r(   forward   s   

z%UniSpeechNoLayerNormConvLayer.forwardr   r    r!   r"   rf   ry   __classcell__r'   r'   ru   r(   r`      s    r`   c                       r_   )UniSpeechLayerNormConvLayerr   c                    s|   t    |dkr|j|d  nd| _|j| | _tj| j| j|j| |j| |j	d| _
tj| jdd| _t|j | _d S )Nr   r   ra   T)elementwise_affine)re   rf   rg   rh   ri   r   rj   rk   rl   rm   rn   	LayerNorm
layer_normr	   ro   rp   rq   ru   r'   r(   rf      s   
z$UniSpeechLayerNormConvLayer.__init__c                 C   s:   |  |}|dd}| |}|dd}| |}|S )Nr8   )rn   	transposer   rp   rx   r'   r'   r(   ry     s   


z#UniSpeechLayerNormConvLayer.forwardrz   r{   r'   r'   ru   r(   r}      s    r}   c                       r_   )UniSpeechGroupNormConvLayerr   c                    s   t    |dkr|j|d  nd| _|j| | _tj| j| j|j| |j| |j	d| _
t|j | _tj| j| jdd| _d S )Nr   r   ra   T)
num_groupsnum_channelsaffine)re   rf   rg   rh   ri   r   rj   rk   rl   rm   rn   r	   ro   rp   	GroupNormr   rq   ru   r'   r(   rf     s   
z$UniSpeechGroupNormConvLayer.__init__c                 C   s"   |  |}| |}| |}|S rw   )rn   r   rp   rx   r'   r'   r(   ry   "  s   


z#UniSpeechGroupNormConvLayer.forwardrz   r{   r'   r'   ru   r(   r     s    r   c                       $   e Zd Z fddZdd Z  ZS ) UniSpeechPositionalConvEmbeddingc                    s   t    tj|j|j|j|jd |jd| _tjj	}t
tjjdr'tjjj	}t rddd l}|jj| jjdd || jddd| _W d    n1 sLw   Y  |j| | jj |j| | jj n	|| jddd| _t|j| _t|j | _d S )Nr   )rb   paddinggroupsweight_normr   )modifier_rankweight)namedim)re   rf   r   rj   hidden_sizenum_conv_pos_embeddingsnum_conv_pos_embedding_groupsrn   utilsr   hasattrparametrizationsr
   	deepspeedzeroGatheredParametersr   register_external_parameterweight_vweight_gUniSpeechSamePadLayerr   r	   ro   rp   )rr   rs   r   r   ru   r'   r(   rf   +  s*   

z)UniSpeechPositionalConvEmbedding.__init__c                 C   s:   | dd}| |}| |}| |}| dd}|S Nr   r   )r   rn   r   rp   rx   r'   r'   r(   ry   F  s   


z(UniSpeechPositionalConvEmbedding.forwardr{   r'   r'   ru   r(   r   *  s    r   c                       r   )r   c                    s*   t    |d dkrd| _d S d| _d S )Nr   r   r   )re   rf   num_pad_remove)rr   r   ru   r'   r(   rf   S  s   
 zUniSpeechSamePadLayer.__init__c                 C   s,   | j dkr|d d d d d | j  f }|S Nr   )r   rx   r'   r'   r(   ry   W  s   
zUniSpeechSamePadLayer.forwardr{   r'   r'   ru   r(   r   R  s    r   c                       s0   e Zd ZdZ fddZdd Zdd Z  ZS )UniSpeechFeatureEncoderz.Construct the features from raw audio waveformc                    s   t     jdkr t ddg fddt jd D  }n jdkr2 fddt jD }n	td	 j d
t|| _	d| _
d| _d S )Ngroupr   rt   c                    s   g | ]
}t  |d  dqS )r   r   )r`   r:   irs   r'   r(   r<   e  s    z4UniSpeechFeatureEncoder.__init__.<locals>.<listcomp>r   layerc                    s   g | ]}t  |d qS )r   )r}   r   r   r'   r(   r<   j  s    z`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)re   rf   feat_extract_normr   rH   num_feat_extract_layersr@   r   
ModuleListconv_layersgradient_checkpointing_requires_grad)rr   rs   r   ru   r   r(   rf   a  s   





z UniSpeechFeatureEncoder.__init__c                 C   s   |   D ]}d|_qd| _d S NF)
parametersrequires_gradr   rr   paramr'   r'   r(   _freeze_parametersu  s   
z*UniSpeechFeatureEncoder._freeze_parametersc                 C   s\   |d d d f }| j r| jrd|_| jD ]}| j r'| jr'| jr'| |j|}q||}q|S )NT)r   trainingr   r   r   _gradient_checkpointing_func__call__)rr   input_valuesr   
conv_layerr'   r'   r(   ry   z  s   

zUniSpeechFeatureEncoder.forward)r    r!   r"   r#   rf   r   ry   r|   r'   r'   ru   r(   r   ^  s
    r   c                       s   e Zd Z fddZ  ZS )UniSpeechFeatureExtractorc                    s8   t  | td| jj d| jjd j dt d S )NzThe class `zD` has been depreciated and will be removed in Transformers v5. Use `r   z
` instead.)re   rf   warningswarnrv   r    	__bases__FutureWarningrr   rs   ru   r'   r(   rf     s   z"UniSpeechFeatureExtractor.__init__)r    r!   r"   rf   r|   r'   r'   ru   r(   r     s    r   c                       r   )UniSpeechFeatureProjectionc                    sJ   t    tj|jd |jd| _t|jd |j| _	t
|j| _d S )Nr8   eps)re   rf   r   r   rg   layer_norm_epsr   Linearr   
projectionDropoutfeat_proj_dropoutdropoutr   ru   r'   r(   rf     s   
z#UniSpeechFeatureProjection.__init__c                 C   s&   |  |}| |}| |}||fS rw   )r   r   r   )rr   r   norm_hidden_statesr'   r'   r(   ry     s   


z"UniSpeechFeatureProjection.forwardr{   r'   r'   ru   r(   r     s    r   c                       s   e Zd ZdZ					ddededed	ed
ededee f fddZ	de
jdedefddZ					dde
jdee
j deee
j  dee
j dee
j dedee
jee
j eee
j  f fddZ  ZS )UniSpeechAttentionz=Multi-headed attention from 'Attention Is All You Need' paper        FTN	embed_dim	num_headsr   
is_decoderrd   	is_causalrs   c                    s   t    || _|| _|| _|| | _|| _| j| | jkr*td| j d| d| jd | _|| _	|| _
tj|||d| _tj|||d| _tj|||d| _tj|||d| _d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      )rd   )re   rf   r   r   r   head_dimrs   r@   scalingr   r   r   r   k_projv_projq_projout_proj)rr   r   r   r   r   rd   r   rs   ru   r'   r(   rf     s&   



zUniSpeechAttention.__init__tensorseq_lenbszc                 C   s    | ||| j| jdd S r   )viewr   r   r   
contiguous)rr   r   r   r   r'   r'   r(   _shape  s    zUniSpeechAttention._shaper   key_value_statespast_key_valuer,   layer_head_maskoutput_attentionsr.   c                 C   sr  |du}|  \}}	}
| || j }|r.|dur.|d jd |jd kr.|d }|d }nZ|rE| | |d|}| | |d|}nC|durt| | |d|}| | |d|}tj|d |gdd}tj|d |gdd}n| | |d|}| | |d|}| j	r||f}|| j
 d| jf}| ||	|j| }|j| }|j| }| d}t||dd}|  || j
 |	|fkrtd|| j
 |	|f d|   |dur|  |d|	|fkrtd	|d|	|f d|   ||| j
|	|| }||| j
 |	|}tjj|dd}|durL|  | j
fkr1td
| j
f d|   |dddd||| j
|	| }||| j
 |	|}|rc||| j
|	|}||| j
 |	|}nd}tjj|| j| jd}t||}|  || j
 |	| jfkrtd|| j
 |	| jf d|   ||| j
|	| j}|dd}|||	| j}| |}|||fS )z#Input shape: Batch x Time x ChannelNr   r   r   r8   r   z$Attention weights should be of size z	, but is z!Attention mask should be of size z/Head mask for a single layer should be of size )pr   z `attn_output` should be of size )sizer   r   r)   r   r   r   r$   catr   r   r   r   rT   bmmr   r@   r   
functionalsoftmaxr   r   r   r   )rr   r   r   r   r,   r   r   is_cross_attentionr   tgt_lenr;   query_states
key_statesvalue_states
proj_shapesrc_lenattn_weightsattn_weights_reshaped
attn_probsattn_outputr'   r'   r(   ry     s   





"

zUniSpeechAttention.forward)r   FTFN)NNNNF)r    r!   r"   r#   r0   floatrJ   r   r   rf   r$   Tensorr   r   ry   r|   r'   r'   ru   r(   r     sV    r   c                       r   )UniSpeechFeedForwardc                    sp   t    t|j| _t|j|j| _	t
|jtr"t|j | _n|j| _t|j|j| _t|j| _d S rw   )re   rf   r   r   activation_dropoutintermediate_dropoutr   r   intermediate_sizeintermediate_dense
isinstance
hidden_actstrr	   intermediate_act_fnoutput_densehidden_dropoutoutput_dropoutr   ru   r'   r(   rf   I  s   
zUniSpeechFeedForward.__init__c                 C   s6   |  |}| |}| |}| |}| |}|S rw   )r   r   r   r   r   rx   r'   r'   r(   ry   V  s   




zUniSpeechFeedForward.forwardr{   r'   r'   ru   r(   r   H  s    r   c                       s&   e Zd Z fddZdddZ  ZS )UniSpeechEncoderLayerc                    sf   t    t|j|j|jdd| _t|j	| _
tj|j|jd| _t|| _tj|j|jd| _d S )NFr   r   r   r   r   )re   rf   r   r   num_attention_headsattention_dropout	attentionr   r   r   r   r   r   r   r   feed_forwardfinal_layer_normr   ru   r'   r(   rf   b  s   

zUniSpeechEncoderLayer.__init__NFc                 C   sf   |}| j |||d\}}}| |}|| }| |}|| | }| |}|f}|r1||f7 }|S Nr,   r   )r  r   r   r  r  rr   r   r,   r   attn_residualr   r;   outputsr'   r'   r(   ry   o  s   



zUniSpeechEncoderLayer.forwardr   r{   r'   r'   ru   r(   r   a  s    r   c                       s,   e Zd Z fddZdejfddZ  ZS )UniSpeechAttnAdapterLayerc                    sZ   t    |j| _|j| _t| j| _t	| j| j| _
t | _t	| j| j| _dS )z
        Implements adapter modules directly with 3D tensor weight as parameters and without using ModuleList to speed
        up training throughput.
        N)re   rf   adapter_attn_dim	input_dimr   
hidden_dimr   r   normr   linear_1ReLUact_fnlinear_2r   ru   r'   r(   rf     s   

z"UniSpeechAttnAdapterLayer.__init__r   c                 C   s,   |  |}| |}| |}| |}|S rw   )r  r  r  r  rx   r'   r'   r(   ry     s
   



z!UniSpeechAttnAdapterLayer.forward)r    r!   r"   rf   r$   r%   ry   r|   r'   r'   ru   r(   r    s    r  c                       s@   e Zd Z fddZ		d
dejdeej defdd	Z  Z	S )$UniSpeechEncoderLayerStableLayerNormc                    s   t    t|j|j|jdd| _t|j	| _
tj|j|jd| _t|| _tj|j|jd| _t|dd d ur@t|| _d S d | _d S )NFr   r   r  )re   rf   r   r   r  r  r  r   r   r   r   r   r   r   r   r  r  getattrr  adapter_layerr   ru   r'   r(   rf     s   


z-UniSpeechEncoderLayerStableLayerNorm.__init__NFr   r,   r   c                 C   sz   |}|  |}| j|||d\}}}| |}|| }|| | | }| jd ur1|| | }|f}|r;||f7 }|S r  )r   r  r   r  r  r  r  r'   r'   r(   ry     s   



z,UniSpeechEncoderLayerStableLayerNorm.forwardr   )
r    r!   r"   rf   r$   r   r   rJ   ry   r|   r'   r'   ru   r(   r    s    r  c                       sL   e Zd Z fddZ				ddejdeej ded	ed
ef
ddZ	  Z
S )UniSpeechEncoderc                    f   t     | _t | _tj j jd| _	t
 j| _t fddt jD | _d| _d S )Nr   c                       g | ]}t  qS r'   )r   r9   r   r'   r(   r<         z-UniSpeechEncoder.__init__.<locals>.<listcomp>Fre   rf   rs   r   pos_conv_embedr   r   r   r   r   r   r   r   r   rH   num_hidden_layerslayersr   r   ru   r   r(   rf     s   

 
zUniSpeechEncoder.__init__NFTr   r,   r   output_hidden_statesreturn_dictc                 C   s  |rdnd }|r
dnd }|d urO| ddd|jd }d|| < d|d d d d d d f j|jd }|t|jj }||jd d|jd |jd }| 	|}	||	 }| 
|}| |}t }
| jD ]G}|rq||f }tg }| jr|| jjk rdnd	}|r|
r| jr| jr| |j|||}n||||d
}|d }|rd}|r||d f }qh|r||f }|stdd |||fD S t|||dS )Nr'   r8   r   r   r         ?r=   TFr  NNc                 s       | ]	}|d ur|V  qd S rw   r'   r:   vr'   r'   r(   	<genexpr>      z+UniSpeechEncoder.forward.<locals>.<genexpr>last_hidden_stater   r   )	unsqueezerepeatr)   tor>   r$   finfominexpandr  r   r   r
   r  rC   r   rs   	layerdropr   r   r   tupler   rr   r   r,   r   r  r   all_hidden_statesall_self_attentionsexpand_attention_maskposition_embeddingsdeepspeed_zero3_is_enabledr   dropout_probabilityskip_the_layerlayer_outputsr'   r'   r(   ry     s\   
&






zUniSpeechEncoder.forwardNFFT)r    r!   r"   rf   r$   r   r   r   rJ   ry   r|   r'   r'   ru   r(   r    s"    r  c                       s.   e Zd Z fddZ				dddZ  ZS )	UniSpeechEncoderStableLayerNormc                    r  )Nr   c                    r  r'   )r  r9   r   r'   r(   r<   &  r  z<UniSpeechEncoderStableLayerNorm.__init__.<locals>.<listcomp>Fr  r   ru   r   r(   rf     s   


z(UniSpeechEncoderStableLayerNorm.__init__NFTc                 C   s  |rdnd }|r
dnd }|d urO| ddd|jd }d|| < d|d d d d d d f j|jd }|t|jj }||jd d|jd |jd }| 	|}	||	 }| 
|}t }
| jD ]G}|rl||f }tg }| jr||| jjk r|dnd	}|r|
r| jr| jr| |j|||}n||||d
}|d }|rd}|r||d f }qc| |}|r||f }|stdd |||fD S t|||dS )Nr'   r8   r   r   r   r!  r=   TFr  r"  c                 s   r#  rw   r'   r$  r'   r'   r(   r&  k  r'  z:UniSpeechEncoderStableLayerNorm.forward.<locals>.<genexpr>r(  )r*  r+  r)   r,  r>   r$   r-  r.  r/  r  r   r
   r  rC   r   rs   r0  r   r   r   r   r1  r   r2  r'   r'   r(   ry   *  s\   
&






z'UniSpeechEncoderStableLayerNorm.forwardr;  r{   r'   r'   ru   r(   r<    s    r<  c                       s4   e Zd ZdZ fddZedd Zdd Z  ZS )UniSpeechGumbelVectorQuantizerz
    Vector quantization using gumbel softmax. See [CATEGORICAL REPARAMETERIZATION WITH
    GUMBEL-SOFTMAX](https://arxiv.org/pdf/1611.01144.pdf) for more information.
    c                    s   t    |j| _|j| _|j| j dkr"td|j d| j dt	t
d| j| j |j| j | _t|jd | j| j | _d| _d S )Nr   z`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenationr   r8   r   )re   rf   num_codevector_groupsr   num_codevectors_per_groupnum_varscodevector_dimr@   r   	Parameterr$   r%   codevectorsr   rg   weight_projtemperaturer   ru   r'   r(   rf   y  s   


z'UniSpeechGumbelVectorQuantizer.__init__c                 C   s8   | j dd}ttj|t|d  dd  }|S )Nr   r   gHz>r8   )meanr$   exprE   log)probsmarginal_probs
perplexityr'   r'   r(   _compute_perplexity  s   (z2UniSpeechGumbelVectorQuantizer._compute_perplexityc                 C   s  |j \}}}| |}||| | j d}| jr?tjj| | j	dd
|}tj||| | jd dd}| |}n$|jdd}|j|j  d|ddd}||| | jd}| |}||| d}|d| j }	|	|| | j| jd}
|
d||d}
|
|fS )Nr8   T)tauhardr   r   r!  r   )r)   rD  r   r   r   r   r   gumbel_softmaxr   rE  type_asr$   r   rL  argmax	new_zerosscatter_r*  rC  r@  rE   )rr   r   rV   r6   r   codevector_probscodevector_soft_distrK  codevector_idxcodevectors_per_grouprC  r'   r'   r(   ry     s0   

z&UniSpeechGumbelVectorQuantizer.forward)	r    r!   r"   r#   rf   staticmethodrL  ry   r|   r'   r'   ru   r(   r=  s  s    
r=  c                   @   sT   e Zd ZdZeZdZdZdZdd Z	de
ejef fdd	Zd
edejfddZdS )UniSpeechPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    	unispeechr   Tc              	   C   s  t |tr|jjjjddd |jjj  tj	
|j dS t |trItj	j|jjddtd|jjd |jj   d tj	|jjd dS t |trqtd|jj }tj	j
|jj| |d tj	j
|jj| |d dS t |tjr|jjjd| jjd |jdur|jj  dS dS t |tjtjfr|jj  |jjd dS t |tjrtj	|j |jdurt|j|j|jd   }tj	j
|j| |d dS dS dS )	zInitialize the weightsr   r   )rF  stdr   r   )abNr!  )r   r=  rD  r   datanormal_rd   zero_r   inituniform_rC  r   rn   mathsqrtrb   in_channels	constant_r   r   in_featuresr   rs   initializer_ranger   r   fill_rj   kaiming_normal_r   )rr   modulekr'   r'   r(   _init_weights  s<   

 


z&UniSpeechPreTrainedModel._init_weightsrW   c                 C   s4   dd }t | jj| jjD ]
\}}||||}q|S )zH
        Computes the output length of the convolutional layers
        c                 S   s   t j| | |ddd S )Nfloor)rounding_moder   )r$   div)r2   rb   rc   r'   r'   r(   _conv_out_length  s   zSUniSpeechPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length)ziprs   rk   rl   )rr   rW   rq  rb   rc   r'   r'   r(    _get_feat_extract_output_lengths  s   z9UniSpeechPreTrainedModel._get_feat_extract_output_lengthsfeature_vector_lengthr,   c                 C   s   |j ddd d df }| |tj}|jd }tj||f|j|jd}d|tj	|jd |jd|d f< |
dg d
dg }|S )Nr8   r   r   )r>   devicer   )ru  )cumsumrs  r,  r$   longr)   rI   r>   ru  rL   fliprJ   )rr   rt  r,   non_padded_lengthsoutput_lengthsrV   r'   r'   r(   "_get_feature_vector_attention_mask  s   
"z;UniSpeechPreTrainedModel._get_feature_vector_attention_maskN)r    r!   r"   r#   r   config_classbase_model_prefixmain_input_namesupports_gradient_checkpointingrm  r   r$   
LongTensorr0   rs  r{  r'   r'   r'   r(   rY    s    !rY  a  
    UniSpeech was proposed in [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled
    Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei,
    Michael Zeng, Xuedong Huang.

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving etc.).

    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`UniSpeechConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aI  
    Args:
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install
            soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and
            conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details.
        attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
            1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            <Tip warning={true}>

            `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask ==
            True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should
            **not** be passed to avoid degraded performance when doing batched inference. For such models
            `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these
            models also yield slightly different results depending on whether `input_values` is padded or not.

            </Tip>

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zcThe bare UniSpeech Model transformer outputting raw hidden-states without any specific head on top.c                       s   e Zd Zdef fddZ		ddejdeej deej fdd	Z	e
eeeeed
ed					ddeej deej deej dee dee dee deeef fddZ  ZS )UniSpeechModelrs   c                    sz   t  | || _t|| _t|| _|jdks|jdkr)t	
t|j | _|jr2t|| _nt|| _|   d S )Nr   )re   rf   rs   r   feature_extractorr   feature_projectionmask_time_probmask_feature_probr   rB  r$   r%   r   rb  masked_spec_embeddo_stable_layer_normr<  encoderr  	post_initr   ru   r'   r(   rf   A  s   


zUniSpeechModel.__init__Nr   mask_time_indicesr,   c                 C   s  t | jdds	|S | \}}}|dur| j|j||< n-| jjdkrK| jrKt||f| jj| jj	|| jj
d}tj||jtjd}| j|j||< | jjdkr| jrt||f| jj| jj| jjd}tj||jtjd}|dddf d|d}d||< |S )	z
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://arxiv.org/abs/1904.08779).
        apply_spec_augmentTNr   )r*   r+   r,   r-   )ru  r>   )r*   r+   r-   r8   )r  rs   r   r  r,  r>   r  r   r^   mask_time_lengthmask_time_min_masksr$   r   ru  rJ   r  mask_feature_lengthmask_feature_min_masksr/  )rr   r   r  r,   rV   r6   r   mask_feature_indicesr'   r'   r(   _mask_hidden_statesS  s4   z"UniSpeechModel._mask_hidden_statesaudio)
checkpointoutput_typer|  modalityexpected_outputr   r   r  r   r.   c           
      C   s   |d ur|n| j j}|d ur|n| j j}|d ur|n| j j}| |}|dd}|d ur6| |jd |}| |\}}| j	|||d}| j
|||||d}	|	d }|s_||f|	dd   S t|||	j|	jdS )Nr   r   )r  r,   r,   r   r  r   r   )r)  extract_featuresr   r   )rs   r   r  use_return_dictr  r   r{  r)   r  r  r  r   r   r   )
rr   r   r,   r  r   r  r   r  r   encoder_outputsr'   r'   r(   ry     s8   
zUniSpeechModel.forwardr"  NNNNN)r    r!   r"   r   rf   r$   r%   r   r  r  r   UNISPEECH_INPUTS_DOCSTRINGr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr   rJ   r   r   ry   r|   r'   r'   ru   r(   r  <  sN    
.

r  zPUniSpeech Model with a vector-quantization module and ctc loss for pre-training.c                       s   e Zd Zdef fddZdefddZdd Zd	d
 Ze		dde
jde
jde
jdefddZeeeeed				ddee
j dee
j dee dee dee deeef fddZ  ZS )UniSpeechForPreTrainingrs   c                    s~   t  | t|| _t|j| _t|| _	t
|j|j| _t
|j|j| _t
|j|j| _t|j| _|   d S rw   )re   rf   r  rZ  r   r   feat_quantizer_dropoutdropout_featuresr=  	quantizerr   rA  proj_codevector_dim	project_qr   project_hidnum_ctc_classesctc_projfinal_dropoutr   r  r   ru   r'   r(   rf     s   

z UniSpeechForPreTraining.__init__rE  c                 C   s   || j _dS )zb
        Set the Gumbel softmax temperature to a given value. Only necessary for training
        N)r  rE  )rr   rE  r'   r'   r(   set_gumbel_temperature  s   z.UniSpeechForPreTraining.set_gumbel_temperaturec                 C      t dt |   dS z
        Calling this function will disable the gradient computation for the feature encoder so that its parameters will
        not be updated during training.
        The method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5. Please use the equivalent `freeze_feature_encoder` method instead.Nr   r   r   freeze_feature_encoderrr   r'   r'   r(   freeze_feature_extractor  
   z0UniSpeechForPreTraining.freeze_feature_extractorc                 C      | j j  dS 
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        NrZ  r  r   r  r'   r'   r(   r       z.UniSpeechForPreTraining.freeze_feature_encoderr   target_featuresnegative_featurespredicted_featuresc                 C   s@   t j| |gdd} t j| |  dd}|| }|| }|S )z
        Compute logits for contrastive loss based using cosine similarity as the distance measure between
        `[positive_feature, negative_features]` and `[predicted_features]`. Additionally, temperature can be applied.
        r   r   r8   )r$   r   cosine_similarityr   rP  )r  r  r  rE  logitsr'   r'   r(   compute_contrastive_logits  s
   
z2UniSpeechForPreTraining.compute_contrastive_logits)r  r|  Nr   r,   r   r  r   r.   c                 C   s>  |dur|n| j j}| j|||||d}|d }| |d }| |\}	}
| |	}	| |	}	t|	d|	d
| j j}|dd}t| |j}|dd}|d}||d|	| d }| |}| |}d}|s|dur|||	|
f|dd  S ||	|
f|dd  S t|||	|
|j|jdS )	a  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        sampled_negative_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_negatives)`, *optional*):
            Indices indicating which quantized target vectors are used as negative sampled vectors in contrastive loss.
            Required input for pre-training.

        Returns:

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, UniSpeechForPreTraining

        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> model = UniSpeechForPreTraining.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> # TODO: Add full pretraining example
        ```Nr  r   r   r8   r   r   )r   r   r   r   r   r   )rs   r  rZ  r  r  r  r  r$   emptyr   ri  replace_probr   	bernoullirJ   r,  ru  r*  masked_fillr   r  r   r   r   )rr   r   r,   r   r  r   r
  transformer_featuresr  quantized_featuresr   prob_replace_matrixsampled_replace_matrixr  r   r'   r'   r(   ry     sL   





zUniSpeechForPreTraining.forward)r   )NNNN)r    r!   r"   r   rf   r0   r  r  r  rX  r$   r%   r  r   r  r   r   r  r   r   rJ   r   r   ry   r|   r'   r'   ru   r(   r    sF    

r  zgUniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).a/  
        target_lang (`str`, *optional*):
            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
            adapter.<lang>.bin. Only relevant when using an instance of [`UniSpeechForCTC`] with adapters. Uses 'eng'
            by default.
    c                       s   e Zd Zddee f fddZdd Zdd Zd	d
 Zdd Z	e
eeeeeeed					ddeej deej dee dee dee deej deeef fddZ  ZS )UniSpeechForCTCNtarget_langc                    s~   t  | t|| _t|j| _|| _|j	d u r#t
d| j dt|dr.|jr.|jn|j}t||j	| _|   d S )NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `UniSpeechForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.add_adapter)re   rf   r  rZ  r   r   r  r   r  
vocab_sizer@   rv   r   r  output_hidden_sizer   r   lm_headr  )rr   rs   r  r  ru   r'   r(   rf   W  s   

zUniSpeechForCTC.__init__c                 C   sv   | j }|durt| jdddu rtd| d|du r,t| jdddur,td dS |dur9| j|dd dS dS )a'  
        This method overwrites [`~PreTrainedModel.tie_weights`] so that adapter weights can be correctly loaded when
        passing `target_lang=...` to `from_pretrained(...)`.

        This method is **not** supposed to be called by the user and is prone to be changed in the future.
        Nr  zCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.T)
force_load)r  r  rs   r@   loggerinfoload_adapter)rr   r  r'   r'   r(   tie_weightsn  s   zUniSpeechForCTC.tie_weightsc                 C   r  )r  r  Nr  r  r'   r'   r(   r    r  z(UniSpeechForCTC.freeze_feature_extractorc                 C   r  r  r  r  r'   r'   r(   r    r  z&UniSpeechForCTC.freeze_feature_encoderc                 C      | j  D ]}d|_qdS z
        Calling this function will disable the gradient computation for the base model so that its parameters will not
        be updated during training. Only the classification head will be updated.
        FNrZ  r   r   r   r'   r'   r(   freeze_base_model     z!UniSpeechForCTC.freeze_base_model)r  r  r|  r  expected_lossr   r,   r   r  r   labelsr.   c              
   C   st  |dur|n| j j}| j|||||d}|d }| |}| |}	d}
|dur| | j jkr9td| j j |dur?|ntj	|tj
d}| |dtj
}|dk}|d}||}tjj|	dtjddd}tjjjd	d
 tjj||||| j j| j j| j jd}
W d   n1 sw   Y  |s|	f|td  }|
dur|
f| S |S t|
|	|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size - 1]`.
        Nr  r   z$Label values must be <= vocab_size: r=   r8   )r   r>   r   F)enabled)blank	reductionzero_infinityr   r  r   r   )rs   r  rZ  r   r  r1   r  r@   r$   	ones_likerw  rs  rE   r,  masked_selectr   r   log_softmaxfloat32r   backendscudnnflagsctc_losspad_token_idctc_loss_reductionctc_zero_infinity_HIDDEN_STATES_START_POSITIONr   r   r   )rr   r   r,   r   r  r   r  r
  r   r  r   rW   labels_masktarget_lengthsflattened_targets	log_probsoutputr'   r'   r(   ry     sN   



zUniSpeechForCTC.forwardrw   r  )r    r!   r"   r   r   rf   r  r  r  r  r   r  r   r  r   r  _CTC_EXPECTED_OUTPUT_CTC_EXPECTED_LOSSr$   r   rJ   r   r   ry   r|   r'   r'   ru   r(   r  K  sD    

r  z
    UniSpeech Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                       s   e Zd Z fddZdd Zdd Zdd Zeee	e
eed	d
					ddeej deej dee dee dee deej deeef fddZ  ZS )"UniSpeechForSequenceClassificationc                    s   t  | t|dr|jrtdt|| _|jd }|jr*t	
t|| | _t	|j|j| _t	|j|j| _|   d S )Nr  z`Sequence classification does not support the use of UniSpeech adapters (config.add_adapter=True)r   )re   rf   r   r  r@   r  rZ  r  use_weighted_layer_sumr   rB  r$   rO   layer_weightsr   r   classifier_proj_size	projector
num_labels
classifierr  )rr   rs   
num_layersru   r'   r(   rf     s   

z+UniSpeechForSequenceClassification.__init__c                 C   r  r  r  r  r'   r'   r(   r    r  z;UniSpeechForSequenceClassification.freeze_feature_extractorc                 C   r  r  r  r  r'   r'   r(   r    r  z9UniSpeechForSequenceClassification.freeze_feature_encoderc                 C   r  r  r  r   r'   r'   r(   r    r  z4UniSpeechForSequenceClassification.freeze_base_modelr  )r  r  r|  r  Nr   r,   r   r  r   r  r.   c                 C   s`  |dur|n| j j}| j jrdn|}| j|||||d}| j jrB|t }tj|dd}tjj	| j
dd}	||	ddd jdd}n|d }| |}|du rV|jdd}
n| |jd |}d|| < |jdd|jdddd }
| |
}d}|durt }||d| j j|d}|s|f|td  }|dur|f| S |S t|||j|jd	S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NTr  r   r   r8   r   r   r  )rs   r  r  rZ  r  r$   stackr   r   r   r  r   rE   r  rF  r{  r)   r  r   r  r   r   r   )rr   r   r,   r   r  r   r  r
  r   norm_weightspooled_outputpadding_maskr  r   loss_fctr  r'   r'   r(   ry   %  sF   

 
z*UniSpeechForSequenceClassification.forwardr  )r    r!   r"   rf   r  r  r  r   r  r   r  r   r  r   r$   r   rJ   r   r   ry   r|   r'   r'   ru   r(   r    s@    

r  r   )Lr#   rc  r   dataclassesr   typingr   r   r   numpyrA   r$   torch.utils.checkpointr   torch.nnr   activationsr	   integrations.deepspeedr
   modeling_outputsr   r   r   r   modeling_utilsr   r   r   r   r   r   r   r   configuration_unispeechr   
get_loggerr    r  r  r  r  r  r  r  'UNISPEECH_PRETRAINED_MODEL_ARCHIVE_LISTr   r0   r   r  ndarrayr^   Moduler`   r}   r   r   r   r   r   r   r   r   r   r  r  r  r<  r=  rY  UNISPEECH_START_DOCSTRINGr  r  r  r  r  r'   r'   r'   r(   <module>   s    
(

x(/  #.RUFK%y  