o h{@s|dZddlmZmZmZddlZddlZddlmZddlm Z ddl m Z ddl m Z mZdd lmZdd lmZmZmZmZd d lmZeeZd ZdZgdZdededejfddZ dejdejfddZ!dejdejdejdejfddZ"Gdddej#Z$Gdd d ej#Z%Gd!d"d"ej#Z&Gd#d$d$eZ'd%Z(d&Z)ed'e(Gd(d)d)e'Z*ed*e(Gd+d,d,e'Z+dS)-z PyTorch CodeGen model.)OptionalTupleUnionN)nn)CrossEntropyLoss)ACT2FN)BaseModelOutputWithPastCausalLMOutputWithPast)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging) CodeGenConfigSalesforce/codegen-2B-monor) zSalesforce/codegen-350M-nlzSalesforce/codegen-350M-multizSalesforce/codegen-350M-monozSalesforce/codegen-2B-nlzSalesforce/codegen-2B-multirzSalesforce/codegen-6B-nlzSalesforce/codegen-6B-multizSalesforce/codegen-6B-monozSalesforce/codegen-16B-nlzSalesforce/codegen-16B-multizSalesforce/codegen-16B-mononum_posdimreturncCsVddtd|d|}tdtj|tjd|}tjt|t|fddS) N?i'rz i , j -> i jdtyperr)torcharangeeinsumfloatcatsincos)rrinv_freq sinusoid_inpr$b/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/codegen/modeling_codegen.pycreate_sinusoidal_positions7sr&xcCsb|dddddddddf}|dddddddddf}tj| |fdd}|dS)Nrrr)rstackflatten)r'x1x2r$r$r%rotate_every_two>s"" r.tensorr r!cCs`t|dddddddfdd}t|dddddddfdd}||t||S)Nrr)rrepeat_interleaver.)r/r r!r$r$r%apply_rotary_pos_embFs&&r1cseZdZfddZddZddZ  ddd Z     dd eej d ee ej d eej deej deej dee dee dee ej e ej fee ej e ej e ej dffffddZZS)CodeGenAttentioncs t|j}|jdttj||ftjddd||ddt |j |_ t |j |_|j|_|j|_|j|j|_|j|j|jkrTtd|jd|jdttj|jtjdt|_t j|j|jd dd |_t j|j|jdd |_|j|_|jp|j}t|||_dS) N causal_maskrrF) persistentzEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).r)bias) super__init__max_position_embeddingsregister_bufferrtrilonesboolviewrDropout attn_pdrop attn_dropout resid_pdrop resid_dropout hidden_size embed_dimnum_attention_headshead_dim ValueErrorsqrtr/float32toget_default_dtype scale_attnLinearqkv_projout_proj rotary_dimr&embed_positions)selfconfig max_positions pos_embd_dim __class__r$r%r7Ms4  $ zCodeGenAttention.__init__cCsJ||jdd|||f}||jddd|jdd}|S)Nr(r))r()reshapeshape)rRr'n_headdim_headmp_numreshapedr$r$r% _split_headsls &zCodeGenAttention._split_headscCst|jdkr|ddddd}nt|jdkr%|dddd}n tdt|j|dd ||f}||S) zM Merges attn_head_size dim and num_attn_heads dim into n_ctx rrrrz3Input tensor rank should be one of [4, 5], but is: Nr))lenrYpermute contiguousrGsizer=)rRr/rEattn_head_size new_shaper$r$r% _merge_headsqs zCodeGenAttention._merge_headsNc Cs|d|d}}|jdddd|||d|f}|tj}|tj}t||dd} | |j} t| j j } tj | | j d| j } t || | } |dur[| |} tjdd| } | |j } || } |durv| |} t| |} | | fS)Nr)r(rr)rdr3rJrrImatmul transposerLfinforminr/devicewhererSoftmaxr@) rRquerykeyvalueattention_mask head_mask query_length key_lengthr3 attn_weights mask_value attn_outputr$r$r%_attn~s$ &      zCodeGenAttention._attnF hidden_states layer_pastrr position_idsrs use_cacheoutput_attentionsr.cCs||}d} ||jdd| df} |j|j| } tj| | dd\} } }|j| |j|j| d} |j||j|j| d}|j| |j|j| d} | dddd} |j }|j |j krc| |j }||_ ||}tj||jdddd\}}|j dur|ddddddd|j f}|dddddd|j df}| ddddddd|j f}| dddddd|j df}t |||}t |||}tj||gdd}tj||gdd} n t |||}t | ||} |dddd}| dddd} |dur|d}|d}tj||fd d}tj|| fd d} |d ur'| |j| f}nd}|| || ||\}}|||j|j}||}||}||f}|rS||f7}|S) Nr`r(r)r\rrrrr)T)rNrXrYrFrErsplitr^rbrQrlrJrPr1rrryrgrOrB)rRrzr{rrr|rsr}r~qkvr\ qkv_split local_dimrorqrprQsincosr r!k_rotk_passq_rotq_passpast_key past_valuepresentrxrvoutputsr$r$r%forwardsX    """"         zCodeGenAttention.forward)NNNNNNFF)__name__ __module__ __qualname__r7r^rgryrr FloatTensorrTensor LongTensorr<rr __classcell__r$r$rVr%r2LsD  ,  "r2cs6eZdZfddZdeejdejfddZZS) CodeGenMLPcsJt|j}t|||_t|||_t|j|_ t |j |_ dSN) r6r7n_embdrrMfc_infc_outractivation_functionactr>rAdropout)rRintermediate_sizerSrDrVr$r%r7s  zCodeGenMLP.__init__rzrcCs,||}||}||}||}|Sr)rrrr)rRrzr$r$r%rs    zCodeGenMLP.forward) rrrr7rrrrrr$r$rVr%rs " rcseZdZfddZ      ddeejdeeejdeejdeej d eejd ee d ee d e eejeeejeejd ffffddZ Z S) CodeGenBlockcsRt|jdur |jnd|j}tj|j|jd|_t||_ t |||_ dS)Nr`eps) r6r7n_innerrr LayerNormlayer_norm_epsilonln_1r2attnrmlp)rRrS inner_dimrVr$r%r7 s  zCodeGenBlock.__init__NFrzr{rrr|rsr}r~r.c Csx|}||}|j|||||||d} | d} | dd} ||} | | |}|r1|f| } | S|f| dd} | S)Nrzr{rrr|rsr}r~rr)rrr) rRrzr{rrr|rsr}r~residual attn_outputsrxrfeed_forward_hidden_statesr$r$r%rs(     zCodeGenBlock.forwardr)rrrr7rrrrrrr<rrrr$r$rVr%r s2   ( rcs>eZdZdZeZdZdZdgZdZ fddZ dd Z Z S) CodeGenPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. transformerTrpast_key_valuescstj|i|dSr)r6r7)rRinputskwargsrVr$r%r7CszCodeGenPreTrainedModel.__init__cCst|tjfr!|jjjd|jjd|jdur|jj dSdSt|tj rD|jjjd|jjd|j durB|jj|j  dSdSt|tj rY|jj |jj ddSdS)zInitialize the weights.g)meanstdNr) isinstancerrMweightdatanormal_rSinitializer_ranger5zero_ Embedding padding_idxrfill_)rRmoduler$r$r% _init_weightsFs     z$CodeGenPreTrainedModel._init_weights) rrr__doc__r config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placementr7rrr$r$rVr%r7s raJ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`CodeGenConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a: Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoProcenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zaThe bare CodeGen Model transformer outputting raw hidden-states without any specific head on top.cseZdZfddZddZddZeede e e e d dd e ejd e eeejd e ejd e ejde ejde ejde ejde ede ede ede edeee ffddZZS) CodeGenModelcstj|_j|_tj|j|_tj |_ t fddt j D|_tj|jjd|_tjjj|_d|_|dS)Ncsg|]}tqSr$)r).0_rSr$r% sz)CodeGenModel.__init__..rF)r6r7rrD vocab_sizerrwter> embd_pdropdrop ModuleListrangen_layerhrrln_frkrPn_ctxrEgradient_checkpointing post_initrRrSrVrr%r7s   zCodeGenModel.__init__cC|jSrrrRr$r$r%get_input_embeddingsz!CodeGenModel.get_input_embeddingscC ||_dSrrrRnew_embeddingsr$r$r%set_input_embeddings z!CodeGenModel.set_input_embeddingsbatch_size, sequence_length checkpoint output_typerN input_idsrrrtoken_type_idsr|rs inputs_embedsr}r~output_hidden_states return_dictrc  Csn| dur| n|jj} | dur| n|jj} |dur|n|jj}| dur$| n|jj} |dur4|dur4td|durP||||} |d| d}|j d} n|durb|dd} |j d} ntd|durm|j n|j }|dur||d| d}|durd}t dgt |j }n |ddd}|durtj|| d|tj|d}|d}|dur| dkrtd|| d}|ddddddf}|j|jd}d |t|jj}|||jj}|dur||}|}|dur||}||}||}| |df}|jr|jr|rtd d }|r!d nd}| r(d nd}| r/d nd}tt|j |D]S\}\}}| rG||f}|jr_|jr_| |j!|d|||||| }n ||||||||| d }|d}|dur|||df}| r|||rdndf}q9|"|}||}| r||f}| st dd||||fDSt#||||dS)NzDYou cannot specify both input_ids and inputs_embeds at the same timer(rz5You have to specify either input_ids or inputs_embedsr))rrlz$batch_size has to be defined and > 0rrzh`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting `use_cache=False`...Fr$rTrrcss|] }|dur|VqdSrr$)rvr$r$r% Asz'CodeGenModel.forward..)last_hidden_staterrz attentions)$rSr~rr}use_return_dictrG%warn_if_padding_and_no_attention_maskrdr=rYrltuplerarrrlong unsqueezerJrrjrk get_head_maskrrrrtraininglogger warning_once enumeratezip_gradient_checkpointing_func__call__rr )rRrrrrrr|rsrr}r~rr input_shape batch_sizerl past_lengthrztoken_type_embeds output_shapepresentsall_self_attentionsall_hidden_statesiblockr{rr$r$r%rs              zCodeGenModel.forward) NNNNNNNNNNN)rrrr7rrrCODEGEN_INPUTS_DOCSTRINGformatr _CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrrrrrrr<rrrr$r$rVr%rs\       rzM The CodeGen Model transformer with a language modeling head on top. cs0eZdZdgZfddZddZddZdd d Zee d e e e ed             d d eejdeeeejdeejdeejdeejdeejdeejdeejdeedeedeedeedeee ffddZedeeejdejdeeejfddZZS)!CodeGenForCausalLMzlm_head.weightcs4t|t||_t|j|j|_| dSr) r6r7rrrrMrrlm_headrrrVr$r%r7Ts   zCodeGenForCausalLM.__init__cCrrr rr$r$r%get_output_embeddings\rz(CodeGenForCausalLM.get_output_embeddingscCrrr rr$r$r%set_output_embeddings_rz(CodeGenForCausalLM.set_output_embeddingsNc Ks|dd}|r>|ddjd}|jd|kr|}n|jdd}|dd|df}|dur>|dd|jd df}|dd}|dd}|durs|durs|dd}||dkd|rs|dd|jd df}|||d|||d S) Nrrrrrrr|r(r})rrr}r|rrr)getrYrcumsum masked_fill_) rRrrrrrremove_prefix_lengthrrr|r$r$r%prepare_inputs_for_generationbs.   z0CodeGenForCausalLM.prepare_inputs_for_generationrrrrrrrr|rsrlabelsr}r~rrrc  Cs| dur| n|jj} |j|||||||| | | | d } | d}||tj}d}|durb||j}|dddddf}|dddf}t }|| d| d| d}||j }| sx|f| dd}|durv|f|S|St ||| j| j| jdS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` N) rrrrr|rsrr}r~rrr.r(r)losslogitsrrzr)rSrrr rJrrIrlrcrr=rdrr rrzr)rRrrrrrr|rsrrr}r~rrtransformer_outputsrz lm_logitsr shift_logits shift_labelsloss_fctoutputr$r$r%rsD   zCodeGenForCausalLM.forwardbeam_idxcstfdd|DS)a This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct beam_idx at every generation step. c3s&|]}tfdd|DVqdS)c3s$|] }|d|jVqdS)rN) index_selectrJrl)r past_staterr$r%rs"z>CodeGenForCausalLM._reorder_cache...Nr)rr{rr$r%rs  z4CodeGenForCausalLM._reorder_cache..r)rrr$rr%_reorder_caches z!CodeGenForCausalLM._reorder_cacher) NNNNNNNNNNNN)rrr_tied_weights_keysr7r r rrrrr rr rrrrrrrr<rr staticmethodr rr$r$rVr%rKsv   $      D  r),rtypingrrrrtorch.utils.checkpointrtorch.nnr activationsrmodeling_outputsr r modeling_utilsr utilsr r rrconfiguration_codegenr get_loggerrrrr%CODEGEN_PRETRAINED_MODEL_ARCHIVE_LISTintrr&r.r1Moduler2rrrCODEGEN_START_DOCSTRINGrrrr$r$r$r%sH      "-+  24