o
    h                     @   sF  d Z ddlZddlmZmZmZmZ ddlZddlm	  m
Z ddlZddlm	Z	 ddlmZmZmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( e# rddl)m*Z*m+Z+ ddl,m-Z-m.Z.m/Z/ e%0e1Z2dZ3dZ4ddgZ5dd Z6G dd de	j7Z8G dd de8Z9G dd de8Z:dd Z;d8ddZ<G d d! d!e	j7Z=G d"d# d#e	j7Z>G d$d% d%e>Z?e>e?d&Z@G d'd( d(e	j7ZAd)ZBe!d*eBG d+d, d,eZCd-ZDe!d*eBG d.d/ d/eCZEG d0d1 d1eCZFe!d2eBG d3d4 d4eCZGe!d5eBG d6d7 d7eCZHdS )9z PyTorch Phi model.    N)ListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)CacheDynamicCache)!_prepare_4d_causal_attention_mask)BaseModelOutputWithPastCausalLMOutputWithPast SequenceClassifierOutputWithPastTokenClassifierOutput)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardis_flash_attn_2_available#is_flash_attn_greater_or_equal_2_10loggingreplace_return_docstrings   )	PhiConfig)flash_attn_funcflash_attn_varlen_func)index_first_axis	pad_inputunpad_inputzsusnato/phi-1_devr   zsusnato/phi-1_5_devc                 C   sX   | j dtjd}tj|  dd }|  }ttj	|dtjjdd}|||fS )N)dimdtypeF)as_tupler   )r   r   )
sumtorchint32nonzeroflattenmaxitemFpadcumsum)attention_maskseqlens_in_batchindicesmax_seqlen_in_batch
cu_seqlens r5   Z/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/phi/modeling_phi.py_get_unpad_dataD   s   r7   c                       s0   e Zd Zd
 fdd	Zdd Zddd	Z  ZS )PhiRotaryEmbedding   '  Nc                    sr   t    || _|| _|| _d| jtd| jd || j   }| j	d|dd | j
|| jjt d d S )N      ?r      inv_freqF
persistentseq_lendevicer$   )super__init__r#   max_position_embeddingsbaser'   arangefloattoregister_buffer_set_cos_sin_cacher=   rB   get_default_dtype)selfr#   rE   rF   rB   r=   	__class__r5   r6   rD   R   s   
*
zPhiRotaryEmbedding.__init__c                 C   st   || _ tj| j || jjd}t|| j}tj||fdd}| jd| 	|dd | jd|
 	|dd d S NrB   r$   r"   r#   
cos_cachedFr>   
sin_cached)max_seq_len_cachedr'   rG   r=   r$   outercatrJ   cosrI   sinrM   rA   rB   r$   tfreqsembr5   r5   r6   rK   `   s   z%PhiRotaryEmbedding._set_cos_sin_cachec                 C   sN   || j kr| j||j|jd | jd | j|jd| jd | j|jdfS )Nr@   )r$   )rU   rK   rB   r$   rS   rI   rT   )rM   xrA   r5   r5   r6   forwardj   s
   
zPhiRotaryEmbedding.forward)r9   r:   NN)__name__
__module____qualname__rD   rK   r_   __classcell__r5   r5   rN   r6   r8   Q   s    
r8   c                       *   e Zd ZdZd
 fdd	Zdd	 Z  ZS )PhiLinearScalingRotaryEmbeddingzYPhiRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendevr9   r:   Nr;   c                       || _ t |||| d S r`   scaling_factorrC   rD   rM   r#   rE   rF   rB   ri   rN   r5   r6   rD   y      z(PhiLinearScalingRotaryEmbedding.__init__c                 C   s~   || _ tj| j || jjd}|| j }t|| j}tj||fdd}| jd|	 
|dd | jd| 
|dd d S rP   )rU   r'   rG   r=   r$   ri   rV   rW   rJ   rX   rI   rY   rZ   r5   r5   r6   rK   }   s   
z2PhiLinearScalingRotaryEmbedding._set_cos_sin_cacher9   r:   Nr;   ra   rb   rc   __doc__rD   rK   rd   r5   r5   rN   r6   rf   v       rf   c                       re   )#PhiDynamicNTKScalingRotaryEmbeddingzkPhiRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozillar9   r:   Nr;   c                    rg   r`   rh   rj   rN   r5   r6   rD      rk   z,PhiDynamicNTKScalingRotaryEmbedding.__init__c           	      C   s   || _ || jkr<| j| j| | j | jd  | j| jd    }d|td| jd || j   }| j	d|dd tj| j || j
jd}t|| j
}tj||fd	d
}| j	d| |dd | j	d| |dd d S )Nr   r<   r;   r   r=   Fr>   rQ   r"   rR   rS   rT   )rU   rE   rF   ri   r#   r'   rG   rH   rI   rJ   r=   r$   rV   rW   rX   rY   )	rM   rA   rB   r$   rF   r=   r[   r\   r]   r5   r5   r6   rK      s   
(z6PhiDynamicNTKScalingRotaryEmbedding._set_cos_sin_cacherl   rm   r5   r5   rN   r6   rp      ro   rp   c                 C   sH   | dd| j d d f }| d| j d d df }tj| |fddS )z*Rotates half the hidden dims of the input..Nr"   r<   rR   )shaper'   rW   )r^   x1x2r5   r5   r6   rotate_half   s   rt   c                 C   sL   ||  |}||  |}| | t| |  }|| t||  }||fS )an  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`):
            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
            used to pass offsetted position ids when working with a KV-cache.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezert   )qkrX   rY   position_idsunsqueeze_dimq_embedk_embedr5   r5   r6   apply_rotary_pos_emb   s
   r|   c                       s2   e Zd Z fddZdejdejfddZ  ZS )PhiMLPc                    sD   t    || _t|j | _t|j|j	| _
t|j	|j| _d S r`   )rC   rD   configr   
hidden_actactivation_fnr   Linearhidden_sizeintermediate_sizefc1fc2rM   r~   rN   r5   r6   rD      s
   
zPhiMLP.__init__hidden_statesreturnc                 C   s"   |  |}| |}| |}|S r`   )r   r   r   )rM   r   r5   r5   r6   r_      s   


zPhiMLP.forward)ra   rb   rc   rD   r'   Tensorr_   rd   r5   r5   rN   r6   r}      s    r}   c                       s   e Zd ZdZddedee f fddZdd Zd	e	j
d
ee	j
e	j
e	j
f fddZ					dde	j
dee	j
 dee	j dee deded
ee	j
ee	j
 eee	j
  f fddZ  ZS )PhiAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNr~   	layer_idxc                    s2  t    || _|| _|d u rtd| jj d |j| _|j	| _
| j| j
 | _|j| _|j| _|j| _d| _| j| j
 | jkrNtd| j d| j
 dtj| jd| j dd| _tj| j
| j | jdd| _|j| _| jrtj|j| j
 |jdd	| _tj|j| j
 |jdd	| _t|j| _|   d S )
NzInstantiating z without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).r
   bias)epselementwise_affine)rC   rD   r~   r   loggerwarning_oncerO   ra   r   num_attention_heads	num_headshead_dimrE   
rope_thetapartial_rotary_factor	is_causal
ValueErrorr   r   query_key_valuedenseqk_layernorm	LayerNormlayer_norm_epsq_layernormk_layernormDropoutattention_dropout
_init_roperM   r~   r   rN   r5   r6   rD      s@   

zPhiAttention.__init__c                 C   s   | j jd u rtt| j| j | j| jd| _d S | j jd }| j jd }|dkr;t	t| j| j | j|| jd| _d S |dkrRt
t| j| j | j|| jd| _d S td| )N)rE   rF   typefactorlinear)rE   ri   rF   dynamiczUnknown RoPE scaling type )r~   rope_scalingr8   intr   r   rE   r   
rotary_embrf   rp   r   )rM   scaling_typeri   r5   r5   r6   r     s.   zPhiAttention._init_rope	fused_qkvr   c                 C   sV   |j \}}}|||| jd| j}|ddddf |ddddf |ddddf fS )a  
        Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory
        storage as `fused_qkv`

        Args:
            fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim]

        Returns:
            query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim]
            value: [batch_size, seq_length, num_heads, head_dim]
        r
   .r   Nr   r<   )rq   viewr   r   )rM   r   
batch_size
seq_lengththree_times_hidden_sizer5   r5   r6   _split_heads  s   4zPhiAttention._split_headsFr   r0   rx   past_key_valueoutput_attentions	use_cachec                 C   s  |  \}}}	| |}
| |
\}}}| jr!| |}| |}|dd}|dd}|dd}|jd }|d urT| jd u rKt	d| j
j d|||| j7 }| j||d\}}|dd | jjf |d| jjd f }}|dd | jjf |d| jjd f }}t|||||\}}tj||fdd	}tj||fdd	}|d ur||| jjd
}|||| j|\}}t||ddt| j }|  || j||fkrt	d|| j||f d|   |d ur
|  |d||fkrt	d|d||f d|   || }tjj|tjdd|j}| |}t||}|  || j|| jfkrBt	d|| j|| jf d|   |dd }| ||| j!}| "|}|s\d }|||fS )Nr   r<   zFThe cache structure has changed since version v4.36. If you are using zv for auto-regressive decoding with k/v caching, please make sure to initialize the attention class with a layer index.rA   .r"   rR   rY   rX   partial_rotation_sizer
   z$Attention weights should be of size z	, but is z!Attention mask should be of size )r$   r#   z `attn_output` should be of size )#sizer   r   r   r   r   	transposerq   r   r   rO   ra   get_usable_lengthr   r#   r|   r'   rW   updatematmulmathsqrtr   r   r   
functionalsoftmaxfloat32rI   r$   r   
contiguousreshaper   r   )rM   r   r0   rx   r   r   r   bszq_len_r   query_states
key_statesvalue_states
kv_seq_lenrX   rY   	query_rot
query_passkey_rotkey_passcache_kwargsattn_weightsattn_outputr5   r5   r6   r_   /  sr   	




 



zPhiAttention.forwardr`   NNNFF)ra   rb   rc   rn   r   r   r   rD   r   r'   r   r   r   
LongTensorr   boolr_   rd   r5   r5   rN   r6   r      s2    &$r   c                       s   e Zd ZdZ fddZ					ddejdeej deej d	ee	 d
e
de
deejeej eeej  f fddZ	dddZdd Z  ZS )PhiFlashAttention2aB  
    Phi flash attention module. This module inherits from `PhiAttention` as the weights of the module stays untouched.
    The only required change would be on the forward pass where it needs to correctly call the public API of flash
    attention and deal with padding tokens in case the input contains any of them.
    c                    s    t  j|i | t  | _d S r`   )rC   rD   r   _flash_attn_uses_top_left_mask)rM   argskwargsrN   r5   r6   rD     s   zPhiFlashAttention2.__init__NFr   r0   rx   r   r   r   r   c              	   C   st  d}|  \}}}	| |}
| |
\}}}| jr#| |}| |}|dd}|dd}|dd}|jd }|d urG|||| j	7 }| j
||d\}}|dd | j
jf |d| j
jd f }}|dd | j
jf |d| j
jd f }}t|||||\}}tj||fdd}tj||fdd}|d ur||| j
jd	}|||| j	|\}}|jd }|dd||| j| j}|dd||| j| j}|dd||| j| j}| jr| jjnd
}|jtjkrt| jdr| jj}n| jjj}td| d ||}||}||}| j||||||dd}| ||| j| j }| !|}|s5d }|||fS )NFr   r<   r   r   .r"   rR   r           _pre_quantization_dtypezThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .r;   )dropoutsoftmax_scale)"r   r   r   r   r   r   r   rq   r   r   r   r#   r|   r'   rW   r   r   r   r   trainingr~   r   r$   r   hasattrr   q_projweightr   r   rI   _flash_attention_forwardr   r   )rM   r   r0   rx   r   r   r   r   r   r   r   r   r   r   r   rX   rY   r   r   r   r   r   tgt_lenattn_dropouttarget_dtyper   r   r5   r5   r6   r_     sf   











zPhiFlashAttention2.forwardr   c                 C   s   | j s| j}n| jo|dk}|durE|jd }	| |||||\}}}}
}}|\}}|\}}t||||||||||d
}t||
|	|}|S t||||||d}|S )a  
        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
        first unpad the input, then computes the attention scores and pad the final attention scores.

        Args:
            query_states (`torch.Tensor`):
                Input query states to be passed to Flash Attention API
            key_states (`torch.Tensor`):
                Input key states to be passed to Flash Attention API
            value_states (`torch.Tensor`):
                Input value states to be passed to Flash Attention API
            attention_mask (`torch.Tensor`):
                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
                position of padding tokens and 1 for the position of non-padding tokens.
            dropout (`int`, *optional*):
                Attention dropout
            softmax_scale (`float`, *optional*):
                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
        r   Nr   )cu_seqlens_qcu_seqlens_kmax_seqlen_qmax_seqlen_k	dropout_pr   causal)r   r   )r   r   rq   _upad_inputr   r    r   )rM   r   r   r   r0   query_lengthr   r   r   r   	indices_qcu_seq_lensmax_seq_lensr   r   max_seqlen_in_batch_qmax_seqlen_in_batch_kattn_output_unpadr   r5   r5   r6   r     s8   

z+PhiFlashAttention2._flash_attention_forwardc                 C   s   t |\}}}|j\}	}
}}t||	|
 |||}t||	|
 |||}||
kr>t||	|
 | j||}|}|}|}n2|dkr\d}tj|	d tj|jd}|d d }|	d}n|d d | d f }t
||\}}}}||||||f||ffS )Nr   r$   rB   r"   )r7   rq   r   r   r   r'   rG   r(   rB   squeezer!   )rM   query_layer	key_layervalue_layerr0   r   	indices_kr   r   r   r   num_key_value_headsr   r   r   r   r5   r5   r6   r   @  s>   zPhiFlashAttention2._upad_inputr   )r   N)ra   rb   rc   rn   rD   r'   r   r   r   r   r   r   r_   r   r   rd   r5   r5   rN   r6   r     s4    
f
<r   )eagerflash_attention_2c                       s   e Zd Zdedef fddZ					ddejdeej d	eej	 d
ee
 dee
 deeej  deejeeejejf  f fddZ  ZS )PhiDecoderLayerr~   r   c                    sN   t    t|j ||d| _t|| _tj|j	|j
d| _t|j| _d S )N)r   r   )rC   rD   PHI_ATTENTION_CLASSES_attn_implementation	self_attnr}   mlpr   r   r   r   input_layernormr   resid_pdropresid_dropoutr   rN   r5   r6   rD   n  s
   

zPhiDecoderLayer.__init__NFr   r0   rx   r   r   r   r   c                 C   sv   |}|  |}| j||||||d\}}	}
| |}| | |}|| | }|f}|r2||	f7 }|r9||
f7 }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`):
                input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
                Indices of positions of each input sequence tokens in the position embeddings. Selected in the range
                `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
        )r   r0   rx   r   r   r   )r  r  r  r  )rM   r   r0   rx   r   r   r   residualattn_outputsself_attn_weightspresent_key_valuefeed_forward_hidden_statesoutputsr5   r5   r6   r_   u  s&   



zPhiDecoderLayer.forward)NNFFN)ra   rb   rc   r   r   rD   r'   r   r   r   r   r   FloatTensorr_   rd   r5   r5   rN   r6   r   m  s,    
r   aH  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`PhiConfig`]):
            Model configuration class with all the parameters of the model. Initializing with a config file does not
            load the weights associated with the model, only the configuration. Check out the
            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
zQThe bare Phi Model outputting raw hidden-states without any specific head on top.c                   @   s,   e Zd ZeZdZdZdZdZdZ	dd Z
dS )PhiPreTrainedModelmodelTpast_key_valuesc                 C   s   | j j}t|tjr"|jjjd|d |jd ur |jj	  d S d S t|tj
rA|jjjd|d |jd urC|jj|j 	  d S d S d S )Nr   )meanstd)r~   initializer_range
isinstancer   r   r   datanormal_r   zero_	Embeddingpadding_idx)rM   moduler  r5   r5   r6   _init_weights  s   

z PhiPreTrainedModel._init_weightsN)ra   rb   rc   r   config_classbase_model_prefixsupports_gradient_checkpointing_skip_keys_device_placement_supports_flash_attn_2_supports_cache_classr  r5   r5   r5   r6   r    s    r  aL  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
            information on the default strategy.

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.n_positions - 1]`.

            [What are position IDs?](../glossary#position-ids)
        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
            Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

            Two formats are allowed:
            - a [`~cache_utils.Cache`] instance;
            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
            cache format.

            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
            legacy cache format will be returned.

            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
            of shape `(batch_size, sequence_length)`.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                       s   e Zd ZdZdef fddZdd Zdd Zee																			dd
e
jdee
j dee
j deee
j  dee
j dee dee dee dee deeef fddZ  ZS )PhiModelz
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`PhiDecoderLayer`]

    Args:
        config: PhiConfig
    r~   c                    s   t     j| _ j| _t j j| j| _t	 j
| _t fddt jD | _tj j jd| _ jdk| _d| _|   d S )Nc                    s   g | ]}t  |qS r5   )r   ).0r   r~   r5   r6   
<listcomp>/  s    z%PhiModel.__init__.<locals>.<listcomp>r   r   F)rC   rD   pad_token_idr  
vocab_sizer   r  r   embed_tokensr   
embd_pdropembed_dropout
ModuleListrangenum_hidden_layerslayersr   r   final_layernormr  _use_flash_attention_2gradient_checkpointing	post_initr   rN   r$  r6   rD   '  s   zPhiModel.__init__c                 C      | j S r`   r(  rM   r5   r5   r6   get_input_embeddings8     zPhiModel.get_input_embeddingsc                 C   
   || _ d S r`   r4  rM   valuer5   r5   r6   set_input_embeddings;     
zPhiModel.set_input_embeddingsN	input_idsr0   rx   r  inputs_embedsr   r   output_hidden_statesreturn_dictr   c
              	   C   sx  |d ur|n| j j}|d ur|n| j j}|d ur|n| j j}|	d ur$|	n| j j}	|d ur4|d ur4td|d ur>|j\}
}n|d urI|j\}
}}ntdd}|rct|t }|r^t	
|}||}|d u r|d urn|jn|j}tj||| tj|d}|d}|d u r| |}| |}| jr|d urd|v r|nd }n	t||
|f||}|}| jr| jr|rtd d}|rdnd }|rdnd }d }| jD ]=}|r||f7 }| jr| jr| |j|||||}n
|||||||d}|d }|r||rd	nd
 }|r||d
 f7 }q| |}|r||f7 }d }|r$|r"| n|}|	s4tdd ||||fD S t||||dS )NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timezEYou have to specify either decoder_input_ids or decoder_inputs_embedsr   r   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr5   )r0   rx   r   r   r   r<   r   c                 s   s    | ]	}|d ur|V  qd S r`   r5   )r#  vr5   r5   r6   	<genexpr>  s    z#PhiModel.forward.<locals>.<genexpr>)last_hidden_stater  r   
attentions) r~   r   r?  r   use_return_dictr   rq   r  r   r   from_legacy_cacher   rB   r'   rG   longru   r(  r*  r0  r   r1  r   r   r   r.  _gradient_checkpointing_func__call__r/  to_legacy_cachetupler   )rM   r=  r0   rx   r  r>  r   r   r?  r@  r   r   r   past_key_values_lengthuse_legacy_cacherB   r   all_hidden_statesall_self_attnsnext_decoder_cachedecoder_layerlayer_outputs
next_cacher5   r5   r6   r_   >  s   






		

zPhiModel.forward	NNNNNNNNN)ra   rb   rc   rn   r   rD   r6  r;  r   PHI_INPUTS_DOCSTRINGr'   r   r   r   r   r  r   r   r   r   r_   rd   r5   r5   rN   r6   r"    sH    	

r"  c                       s  e Zd ZdgZ fddZdd Zdd Zdd	 Zd
d Zdd Z	dd Z
eeeeed										d#dejdeej deej deeej  deej deej dee dee dee dee deeef fddZ	d$dd Zed!d" Z  ZS )%PhiForCausalLMzlm_head.weightc                    s@   t  | t|| _|j| _tj|j|jdd| _| 	  d S )NTr   )
rC   rD   r"  r  r'  r   r   r   lm_headr2  r   rN   r5   r6   rD     s
   
zPhiForCausalLM.__init__c                 C      | j jS r`   r  r(  r5  r5   r5   r6   r6       z#PhiForCausalLM.get_input_embeddingsc                 C      || j _d S r`   rY  r9  r5   r5   r6   r;       z#PhiForCausalLM.set_input_embeddingsc                 C   r3  r`   rW  r5  r5   r5   r6   get_output_embeddings  r7  z$PhiForCausalLM.get_output_embeddingsc                 C   r8  r`   r]  )rM   new_embeddingsr5   r5   r6   set_output_embeddings  r<  z$PhiForCausalLM.set_output_embeddingsc                 C   r8  r`   r  )rM   decoderr5   r5   r6   set_decoder  r<  zPhiForCausalLM.set_decoderc                 C   r3  r`   ra  r5  r5   r5   r6   get_decoder  r7  zPhiForCausalLM.get_decoder)output_typer  Nr=  r0   rx   r  r>  labelsr   r   r?  r@  r   c                 C   s&  |dur|n| j j}|	dur|	n| j j}	|
dur|
n| j j}
| j||||||||	|
d	}|d }| |}| }d}|durq|dddddf  }|dddf  }t }|	d| j j
}|	d}||j}|||}|
s|f|dd  }|dur|f| S |S t|||j|j|jdS )a  
        Args:
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, PhiForCausalLM

        >>> model = PhiForCausalLM.from_pretrained("susnato/phi-1_5_dev")
        >>> tokenizer = AutoTokenizer.from_pretrained("susnato/phi-1_5_dev")

        >>> prompt = "This is an example script ."
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        'This is an example script .py file that uses the `os` module to create a new directory and write some text to it.\n\n``'
        ```N)	r=  r0   rx   r  r>  r   r   r?  r@  r   .r"   r   losslogitsr  r   rD  )r~   r   r?  rE  r  rW  rH   r   r   r   r'  rI   rB   r   r  r   rD  )rM   r=  r0   rx   r  r>  rf  r   r   r?  r@  r  r   ri  rh  shift_logitsshift_labelsloss_fctoutputr5   r5   r6   r_     sJ   )


zPhiForCausalLM.forwardc                 K   sz  |d urnt |tr| }|j}| }n|d d jd  }}d }|d urA|jd |jd krA|d d |jd |  d f }n||jd k rR|d d |d f }|d urn|d urn||jd  |krn|d d | d f }|dd }	|d ur|	d u r| dd }	|		|dkd |r|	d d |jd  d f }	|d ur|d u rd|i}
nd|i}
|

|	||d|d	 |
S )
Nr   r<   r   rx   r"   r>  r=  r   )rx   r  r   r0   )r  r   get_seq_lengthseen_tokensget_max_lengthrq   getrG  r/   masked_fill_r   )rM   r=  r  r0   r>  r   cache_lengthpast_lengthmax_cache_lengthrx   model_inputsr5   r5   r6   prepare_inputs_for_generation:  s@   

"
z,PhiForCausalLM.prepare_inputs_for_generationc                    s.   d}| D ]}|t  fdd|D f7 }q|S )Nr5   c                 3   s$    | ]}| d  |jV  qdS )r   N)index_selectrI   rB   )r#  
past_statebeam_idxr5   r6   rB  x  s   " z0PhiForCausalLM._reorder_cache.<locals>.<genexpr>)rK  )r  r{  reordered_past
layer_pastr5   rz  r6   _reorder_cacher  s   zPhiForCausalLM._reorder_cache
NNNNNNNNNN)NNN)ra   rb   rc   _tied_weights_keysrD   r6  r;  r^  r`  rc  rd  r   rU  r   r   _CONFIG_FOR_DOCr'   r   r   r   r   r  r   r   r   r_   rw  staticmethodr~  rd   r5   r5   rN   r6   rV    s`    

	

Y
8rV  a  
    The PhiModel with a sequence classification head on top (linear layer).

    [`PhiForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                       s   e Zd Z fddZdd Zdd Zee										ddej	d	e
ej d
e
ej	 de
eej  de
ej de
ej	 de
e de
e de
e de
e deeef fddZ  ZS )PhiForSequenceClassificationc                    s@   t  | |j| _t|| _tj|j| jdd| _| 	  d S )NFr   )
rC   rD   
num_labelsr"  r  r   r   r   scorer2  r   rN   r5   r6   rD     s
   
z%PhiForSequenceClassification.__init__c                 C   rX  r`   rY  r5  r5   r5   r6   r6    rZ  z1PhiForSequenceClassification.get_input_embeddingsc                 C   r[  r`   rY  r9  r5   r5   r6   r;    r\  z1PhiForSequenceClassification.set_input_embeddingsNr=  r0   rx   r  r>  rf  r   r   r?  r@  r   c                 C   s  |
dur|
n| j j}
| j||||||||	|
d	}|d }| |}|dur+|jd }n|jd }| j jdu r>|dkr>td| j jdu rGd}n|dur_t|| j j	 
dd |j}nd}|tj||jd|f }d}|dur||j}| j jdu r| jdkrd| j _n| jdkr|jtjks|jtj	krd	| j _nd
| j _| j jdkrt }| jdkr|| | }n+|||}n%| j jd	krt }||d| j|d}n| j jd
krt }|||}|
s|f|dd  }|dur|f| S |S t|||j|j|jdS )  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r0   rx   r  r>  r   r   r?  r@  r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r"   )rB   
regressionsingle_label_classificationmulti_label_classificationrg  )r~   rE  r  r  rq   r&  r   r'   eqr   argmaxrI   rB   rG   problem_typer  r$   rG  r	   r   r   r   r   r   r  r   rD  )rM   r=  r0   rx   r  r>  rf  r   r   r?  r@  model_outputsr   ri  r   sequence_lengthspooled_logitsrh  rl  rm  r5   r5   r6   r_     sr   



"


z$PhiForSequenceClassification.forwardr  )ra   rb   rc   rD   r6  r;  r   rU  r'   r   r   r   r   r  r   r   r   r   r_   rd   r5   r5   rN   r6   r  }  sL    		

r  z
    PhiModel with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                       s   e Zd Zdef fddZeeeee	e
d									ddeej deeeejejf df  d	eej d
eej deej dee dee dee dee deeej e	f fddZ  ZS )PhiForTokenClassificationr~   c                    s   t  | |j| _t|| _t|dr|jd ur|j}nt|dr+|jd ur+|j}nd}t	|| _
t|j|j| _|   d S )Nclassifier_dropouthidden_dropoutg?)rC   rD   r  r"  r  r   r  r  r   r   r   r   r   
classifierr2  )rM   r~   r  rN   r5   r6   rD     s   
z"PhiForTokenClassification.__init__)
checkpointre  r  Nr=  r  .r0   r>  rf  r   r   r?  r@  r   c
              
   K   s   |	dur|	n| j j}	| j||||||||	d}|d }| |}| |}d}|durJ||j}|j\}}t }||	|| | j
|	|| }|	s`|f|dd  }|dur^|f| S |S t|||j|jdS )r  N)r  r0   r>  r   r   r?  r@  r   r<   )rh  ri  r   rD  )r~   rE  r  r   r  rI   rB   rq   r   r   r  r   r   rD  )rM   r=  r  r0   r>  rf  r   r   r?  r@  deprecated_argumentsr  r   ri  rh  r   r   rl  rm  r5   r5   r6   r_     s>   


z!PhiForTokenClassification.forwardrT  )ra   rb   rc   r   rD   r   rU  r   _CHECKPOINT_FOR_DOCr   r  r   r'   r   r   r   r   r   r_   rd   r5   r5   rN   r6   r    sL    		
r  )r   )Irn   r   typingr   r   r   r   r'   torch.nn.functionalr   r   r-   torch.utils.checkpointtorch.nnr   r   r	   activationsr   cache_utilsr   r   modeling_attn_mask_utilsr   modeling_outputsr   r   r   r   modeling_utilsr   utilsr   r   r   r   r   r   r   configuration_phir   
flash_attnr   r   flash_attn.bert_paddingr   r    r!   
get_loggerra   r   r  r  !PHI_PRETRAINED_MODEL_ARCHIVE_LISTr7   Moduler8   rf   rp   rt   r|   r}   r   r   r   r   PHI_START_DOCSTRINGr  rU  r"  rV  r  r  r5   r5   r5   r6   <module>   s   $	
%
 6 Z?F  Ck