o h@sdZddlZddlmZmZmZmZddlZddlm m Z ddl Zddlm Z ddl mZddlmZddlmZmZdd lmZdd lmZmZmZmZd d lmZeeZd Z dZ!d gZ"Gddde j#Z$Gddde j#Z%Gddde j#Z&Gddde j#Z'Gddde j#Z(Gddde j#Z)Gddde j#Z*Gddde j#Z+Gdd d e j#Z,Gd!d"d"e j#Z-Gd#d$d$e j#Z.Gd%d&d&eZ/d'Z0d(Z1ed)e0Gd*d+d+e/Z2ed,e0Gd-d.d.e/Z3dS)/z PyTorch CPMAntN)ListOptionalTupleUnion)nn)CrossEntropyLoss)ACT2FN)BaseModelOutputWithPastCausalLMOutputWithPast)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging) CpmAntConfigzopenbmb/cpm-ant-10brcs6eZdZdZdeffdd ZdejfddZZ S)CpmAntLayerNormzv We use Root Mean Square (RMS) Layer Normalization, please see https://arxiv.org/abs/1910.07467 for details." configcs2t|j|_|j|_tt|j|_ dSN) super__init__eps hidden_sizedim_normr Parametertorchemptyweightselfr __class__`/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/cpmant/modeling_cpmant.pyr2s zCpmAntLayerNorm.__init__ hidden_statescCs^|d|jkr td|j}|tjdjddd}|t ||j ||j }|S)f Args: hidden_states (`torch.Tensor` of shape `(batch, seq_len, dim_in)`) z'hidden_states.size(-1) != self.dim_normT)dimkeepdim) sizerAssertionErrordtypetorfloat32powmeanrsqrtrr)r r% old_dtypevariancer#r#r$forward9s  zCpmAntLayerNorm.forward) __name__ __module__ __qualname____doc__rrrTensorr5 __classcell__r#r#r!r$r-srcspeZdZdeffdd Z   ddejdejdejd ejd ee d ee ejejfd ee fd dZ Z S)CpmAntAttentionrcst|j|_|j|_|j|_tj|j|j|jdd|_ tj|j|j|jdd|_ tj|j|j|jdd|_ tj|j|j|jdd|_ t jjdd|_|jdurat jj|jd|_dSd|_dS)NFbiasr'r))p)rrr dim_modelnum_attention_heads num_headsdim_headrLinear project_q project_k project_v attention_outrSoftmaxsoftmax dropout_pDropoutdropoutrr!r#r$rGs   zCpmAntAttention.__init__FNhidden_q hidden_kvattention_mask position_biasoutput_attentionspast_key_values use_cachec Cs|d}|d} |d} ||} ||} ||} | || |j|jdddd} | || |j|jdddd} | || |j|jdddd} |durmtj |d| gdd} tj |d| gdd} | d} t | | ddt |j}||}t|||d| | td ktjtd |j|jd }||}t|||d| | td ktjd|j|jd }|r|}nd}|jdur||}t || }|||j| |jdddd}||| |j|j}||}d}|r| | f}|||fS) a Args: hidden_q (`torch.Tensor`): Input of transformer block(self-attention block). It can be the raw embedding of a batch of sequences. hidden_kv (`torch.Tensor` of shape `(batch, len_k, dim_model)`)): Tensor *key_value* and *query* of shape `(batch, len_k, dim_model)` attention_mask (`torch.Tensor` of shape `(batch, len_seq, len_seq)`): Avoid invalid areas to participate in the calculation of self-attention. position_bias (`torch.Tensor` of shape `(batch, len_seq, len_seq)`): Provide positional information to self-attention block. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. past_key_values (`Tuple[torch.Tensor, torch.Tensor]`, *optional*): Cached past key and value projection states. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). rrr(rNr?r'Fz-inf)devicer-)r+rFrGrHviewrCrDpermutercatmatmul transposemathsqrt masked_filltensor scalar_tensorfloatrWr-rKrN contiguousrI)r rOrPrQrRrSrTrU batch_sizelen_qlen_kquerykeyvaluescore attn_weightsr#r#r$r5ZsN                  zCpmAntAttention.forward)FNN) r6r7r8rrrr: BoolTensorrboolrr5r;r#r#r!r$r<Fs(r<cpeZdZdeffdd Z    ddejdejdeejd eed ee ejejfd eef d d Z Z S)CpmAntSelfAttentionBlockrcBtt||_t||_|jrtj |j|_ dSd|_ dSr) rrrlayernorm_before_attentionr<self_attentionrLrrrMrNrr!r#r$r    z!CpmAntSelfAttentionBlock.__init__NFr%rQrRrSrTrUc CsP||}||||||||}|\}}} |jdur||}||}||| fS)a Args: hidden_states (`torch.Tensor` of shape `(batch, len_seq, dim_model)`): Input of transformer block(self-attention block). It can be the raw embedding of a batch of sequences. attention_mask (`torch.Tensor` of shape `(batch, len_seq, len_seq)`): Avoid invalid areas to participate in the calculation of self-attention. position_bias (`torch.Tensor` of shape `(batch, len_seq, len_seq)`): Provide positional information to self-attention block. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. past_key_values (`Tuple(torch.FloatTensor)`, *optional*): Cached past key and value projection states. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). N)rqrrrN) r r%rQrRrSrTrUoutputsrkcurrent_key_valuer#r#r$r5s     z CpmAntSelfAttentionBlock.forwardNFNN r6r7r8rrrr:rrmrr5r;r#r#r!r$ros& roc2eZdZdeffdd ZdejfddZZS)CpmAntDenseGatedACTrcsFttj|j|jdd|_tj|j|jdd|_tj |_ dSNFr=) rrrrErdim_ffw_0w_1rGELUactrr!r#r$rs zCpmAntDenseGatedACT.__init__r%cCs&|||}||}||}|S)zTransform an input tensor from one feature space to another via a nonlinear operation Args: hidden_states (`torch.Tensor` of shape `(batch, seq_len, dim_in)`) )rr|r})r r% gate_scorer#r#r$r5s zCpmAntDenseGatedACT.forward r6r7r8rrrr:r5r;r#r#r!r$rysrycrx)CpmAntFeedForwardrcsPtt||_|jdurtj|j|_nd|_tj |j |j dd|_ dSrz) rrryw_inrLrrrMrNrEr{rw_outrr!r#r$rs   zCpmAntFeedForward.__init__r%cCs,||}|jdur||}||}|S)r&N)rrNrr r%r#r#r$r5s    zCpmAntFeedForward.forwardrr#r#r!r$rs rcrx)CpmAntFFNBlockrcrpr) rrrlayernorm_before_ffnrffnrLrrrMrNrr!r#r$rrszCpmAntFFNBlock.__init__r%cCs4||}||}|jdur||}||}|S)z Args: hidden_states (`torch.Tensor` of shape `(batch, len_seq, dim_model)`): Hidden states before feed forward layer. N)rrrN)r r% ln_outputsrtr#r#r$r5s   zCpmAntFFNBlock.forwardrr#r#r!r$r s  rcrn)CpmAntTransformerBlockrcs"tt||_t||_dSr)rrroself_attrrrr!r#r$r)s  zCpmAntTransformerBlock.__init__NFr%rQrRrSrTrUc Cs4|j||||||d}|\}}}||}|||fS)a Args: hidden_states (`torch.Tensor`): Input to the layer of shape `(batch, seq_len, dim_model)` attention_mask (`torch.Tensor`): Avoid invalid areas to participate in the calculation of shape `(batch, seq_len, seq_len)` position_bias (`torch.Tensor`): Provides position information to attention mechanism of shape `(num_heads, seq_len, seq_len)` output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. past_key_values (`Tuple[torch.Tensor, torch.Tensor])`, *optional*): Cached past key and value projection states use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). )rQrRrSrTrU)rr) r r%rQrRrSrTrUrkrur#r#r$r5.s  zCpmAntTransformerBlock.forwardrvrwr#r#r!r$r(s& rcsteZdZdeffdd Z    ddejdejdejdeed eed ee ejejfd eefd d Z Z S) CpmAntEncoderrcs@tj|_tfddt|jD|_t|_ dS)Ncsg|]}tqSr#)r).0ithrr#r$ [sz*CpmAntEncoder.__init__..) rrnum_hidden_layers num_layersr ModuleListrangelayersroutput_layernormrr!rr$rXs  zCpmAntEncoder.__init__Nr%rQrRrSoutput_hidden_statesrTrUc Cs|rdnd}|r dnd} |rdnd} t|jD]0\} } |r"||f7}| |||||r-|| nd|d} | \}}}|r>| |f7} |durG| |f} q||}|rT||f7}|| || fS)a% Args: hidden_states (`torch.Tensor`): Input to the layer of shape `(batch, seq_len, dim_model)` attention_mask (`torch.Tensor`): Avoid invalid areas to participate in the calculation of shape `(batch, seq_len, seq_len)` position_bias (`torch.Tensor`): Provides position information to attention mechanism of shape `(num_heads, seq_len, seq_len)` output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. past_key_values (`Tuple[torch.Tensor, torch.Tensor])`, *optional*): Cached past key and value projection states use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). r#N)rSrTrU) enumeraterr)r r%rQrRrSrrTrUall_hidden_statesall_self_attnscurrent_key_valuesilayer layer_outputsrkrur#r#r$r5_s0          zCpmAntEncoder.forward)NNNNrwr#r#r!r$rWs* rcs2eZdZfddZdejdejfddZZS)CpmAntIntermediatecsDtt|j|j|_t|jt rt |j|_ dS|j|_ dSr) rrrrErintermediate_sizedense isinstance hidden_actstrr intermediate_act_fnrr!r#r$rs   zCpmAntIntermediate.__init__r%returncCs||}||}|Sr)rrrr#r#r$r5s  zCpmAntIntermediate.forwardr6r7r8rrr:r5r;r#r#r!r$rs rcsVeZdZdeffdd Zdejdejdejdejfdd Zd d ZdddZ Z S)CpmAntSegmentPositionEmbeddingrcsRt|j|_|j|_|j|_|j|_ t t |j|j|j|j|_dSr)rrrBrCposition_bias_num_buckets num_bucketsposition_bias_max_distance max_distance segment_types num_segmentsrrrrrelative_attention_biasrr!r#r$rs  z'CpmAntSegmentPositionEmbedding.__init__key_pos query_pos key_segment query_segmentc Cst|d}|d}|d}|d|dkr/td|dd|dd||dks=||dkrKtd|d|dd||dkr`td|d|dd||d|}|||d}||d|}|||d}|||}||j}|jtj |tj |j d dddftj |tj |j d dddf|j|j d } t ||k| dddddf|}Wdn1swYt||j} | dd dd } | S) Nrrz>key_pos.size(0) should be equal to query_pos.size(0), but got z and !z7keylen should be equal to key_segment.size(1), but got z;querylen should be equal to query_segment.size(1), but got r'r-rW)rrrr()rno_gradr+r,szierX!_segment_relative_position_bucketr_position_bucketarangeint32rWrwhereF embeddingrrYrc) r rrrrbatchkeylenquerylenrelative_position_bucketabsolute_position_bucketembedsr#r#r$r5sL      (z&CpmAntSegmentPositionEmbedding.forwardcCs||j|Sr)r)r rrr#r#r$rsz@CpmAntSegmentPositionEmbedding._segment_relative_position_bucket cCsd}|d}|dktj|}t|}|d}||k}|t||t||||tj}t|t||d}|t ||tj|7}|S)Nrr(r) r.rrabslogrbr]min full_liker)r relative_positionrrrelative_buckets max_exactis_smallrelative_postion_if_larger#r#r$rs(  z/CpmAntSegmentPositionEmbedding._position_bucket)rr) r6r7r8rrrr:r5rrr;r#r#r!r$rs 4rcs8eZdZfddZdejdejdejfddZZS) CpmAntOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dS)N)r) rrrrErrr LayerNormlayer_norm_epsrMhidden_dropout_probrNrr!r#r$rs zCpmAntOutput.__init__r% input_tensorrcCs&||}||}|||}|Sr)rrNr)r r%rr#r#r$r5 s  zCpmAntOutput.forwardrr#r#r!r$rs $rc@s eZdZdZeZdZddZdS)CpmAntPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. cpmantcCst|tjr |jjjd|jjd|jdur|jj dSdSt|tj rC|jjjd|jjd|j durA|jj|j  dSdSt|tj rX|jj |jj ddSt|trf|jj ddSt|trx|jjjd|jjddSdS)zInitialize the weightsg)r1stdNg?)rrrErdatanormal_rinit_stdr>zero_ Embedding padding_idxrfill_rrr)r moduler#r#r$ _init_weightss$        z#CpmAntPreTrainedModel._init_weightsN)r6r7r8r9r config_classbase_model_prefixrr#r#r#r$rs  raB This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters config ([`~CpmAntConfig`]): Model configuration class with all the parameters of the Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`torch.Tensor` of shape `(batch_size, seq_len)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`CPMAntTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zTThe bare CPMAnt Model outputting raw hidden-states without any specific head on top.cseZdZdeffdd ZddZddZdd Zee e e e e d  dd eejd eedeedeeeejdeedeedeeeje ffddZZS) CpmAntModelrcslt|t||_t|j|j|_t|j |j |j |j|_ t ||_|j |_ |j |_ |dSr)rrrencoderrrrrsegment_embedding vocab_size prompt_types prompt_lengthinput_embeddingrrR post_initrr!r#r$rWs    zCpmAntModel.__init__cC|jSrrr r#r#r$get_input_embeddingsdz CpmAntModel.get_input_embeddingscK ||_dSrr)r embeddingskwargsr#r#r$set_input_embeddingsg z CpmAntModel.set_input_embeddingsc Cs>|d}|d}|j}tj||dtj||dddk}|dddddf|dddddf|d||@B} | |dddddf|dddddfk@} tjtt||j ddd|ddddf |d|dddfk} tj tj ||j |d | fdd} | ||d| |d|@| @} | S)Nrr)rWr'r?)r+rWrrrX logical_notr`listrrrepeatrZonesrm) r input_idsspancontextlengthrseqlenrWdirectional_mask_2drQmask_1dr#r#r$_prepare_attention_maskjs  $&08$ z#CpmAntModel._prepare_attention_mask checkpoint output_typerNrrSrrTrU return_dictrc KsV|dur|n|jj}|dur|n|jj}|dur|n|jj}|dur$|n|jj}|jtjkr4|tj}|j|j }} t |dkddj|| d} | dk dj|| d} tj tj |jd|j|jd|j|| d|dd|fdd}|\} } tj tj| |j|| d| fdd} tj| | fd|| d}tj | || d| d}tj| | fd|| d}|durd}tdg|jj}|}||}|| }||}n |ddd}|| }|||ddddddf}||||| }|||| | }|dd|dddf}|dddd|dddf}|dd|dddf}||||||||\}}}}|dkr|dd|jdddf}|dursd }|D]}||dddd|jd|jdff7}qX|}|durd }|D]}||dd|jdddff7}q||}|std d ||||fDSt||||d S) Nrr(rr'rrr?rVr#css|] }|dur|VqdSrr#)rvr#r#r$ sz&CpmAntModel.forward..)last_hidden_staterTr% attentions)rrSruse_return_dictrUr-rrr.rWrsumrZrrrrr+zerosfulltuplerrrcrrrrRr )r rrSrrTrUrrr-rWsegmentrr seq_lengthrpositionr past_lengthr%segment_statesrQrRpresent_key_valuesrall_attentionsnew_attentions attentionnew_hidden_states hidden_stater#r#r$r5|s    "    $   . & zCpmAntModel.forward)NNNNNN)r6r7r8rrrrrrCPMANT_INPUTS_DOCSTRINGr _CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrrr:rmrrr5r;r#r#r!r$rRs@  rzy The CPMAnt Model with a language modeling head on top (linear layer with weights tied to the input embeddings). cseZdZdgZdeffdd Zeeee e e d        dde e jde eee je jfd e ed e ed e ed e e jd e ede e jdeee ffddZddZddZddZddZddZddZZS)CpmAntForCausalLMzlm_head.weightrcsDt|t||_tj|j|j|j|j dd|_ | dSrz) rrrrrrErrrrlm_headrrr!r#r$rs   zCpmAntForCausalLM.__init__rNrrTrUrSrlabelsrrQrc Ks|dur|n|jj}|||||||} |r| jn| d} || } d} |dur:t}|| d| d|d} |sP| f| dd}| durN| f|S|St| | | j | j | j dS)u; Args: input_ids (`torch.Tensor` of shape `(batch_size, seq_len)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`CPMAntTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): CPMAnt will process attention mask automatically, this parameter is a dummy parameter for text-generation pipeline. Example: Text Generation with CpmAntForCausalLM. ```python >>> from transformers import CPMAntTokenizer, CpmAntForCausalLM >>> texts = "今天天气不错，" >>> model = CpmAntForCausalLM.from_pretrained("openbmb/cpm-ant-10b") >>> tokenizer = CPMAntTokenizer.from_pretrained("openbmb/cpm-ant-10b") >>> input_ids = tokenizer(texts, return_tensors="pt") >>> outputs = model.generate(**input_ids) >>> output_texts = tokenizer.batch_decode(outputs) >>> print(output_texts) ['今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的'] ``` Nrr'r)losslogitsrTr%r) rr rrrrrXr+r rTr%r)r rrTrUrSrrrrQr model_outputr%r r loss_funcoutputr#r#r$r5s(=  zCpmAntForCausalLM.forwardcCs|jjSrrrrr#r#r$rMsz&CpmAntForCausalLM.get_input_embeddingscCs ||j_dSrr$)r rr#r#r$rPs z&CpmAntForCausalLM.set_input_embeddingscCrrrrr#r#r$get_output_embeddingsSrz'CpmAntForCausalLM.get_output_embeddingscCrrr%)r new_embeddingsr#r#r$set_output_embeddingsVrz'CpmAntForCausalLM.set_output_embeddingscKs8|}d|vrtdd|d<||d|dddS)NrQrrUrT)rrUrT)intrr get)r rrr#r#r$prepare_inputs_for_generationYs z/CpmAntForCausalLM.prepare_inputs_for_generationcCs<dd|D}|D]}|d||d<|d||d<q |S)NcSs g|] }|dur t|n|qSr)r)reachr#r#r$rfs z4CpmAntForCausalLM._reorder_cache..rrr#)r rTbeam_idxkey_value_layerr#r#r$_reorder_cachees z CpmAntForCausalLM._reorder_cache)NNNNNNNN)r6r7r8_tied_weights_keysrrrrr rr rrrr:rrrmrr5rrr&r(r+r/r;r#r#r!r$rsT   Q r)4r9r]typingrrrrrtorch.nn.functionalr functionalrtorch.utils.checkpointtorch.nnr activationsr modeling_outputsr r modeling_utilsr utilsr rrrconfiguration_cpmantr get_loggerr6loggerrr$CPMANT_PRETRAINED_MODEL_ARCHIVE_LISTModulerr<roryrrrrrrrrCPMANT_START_DOCSTRINGrrrr#r#r#r$sV      h1/B]