o
    h$                     @   s  d Z ddlZddlmZmZmZ ddlZddlZddlZddlm	Z	 ddl
mZmZmZ ddlmZ ddlmZmZmZmZ dd	lmZ dd
lmZmZmZmZmZ ddlmZmZ ddl m!Z! e"e#Z$dZ%dZ&dZ'dgZ(de)de)dej*fddZ+ej,j-dd Z.dej*dej*fddZ/dej*dej*dej*dej*fddZ0G d d! d!e	j1Z2G d"d# d#e	j1Z3G d$d% d%e	j1Z4G d&d' d'eZ5d(Z6d)Z7d*Z8d+Z9ed,e6G d-d. d.e5Z:ed/e6G d0d1 d1e5Z;ed2e6G d3d4 d4e5Z<ed5e6G d6d7 d7e5Z=dS )8z PyTorch GPT-J model.    N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )ACT2FN)BaseModelOutputWithPastCausalLMOutputWithPastQuestionAnsweringModelOutput SequenceClassifierOutputWithPast)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardis_torch_fx_proxylogging)assert_device_mapget_device_map   )
GPTJConfigz$hf-internal-testing/tiny-random-gptjzEleutherAI/gpt-j-6Br   num_posdimreturnc                 C   sV   ddt d|d|   }t dt j| t jd| }t jt |t |fddS )	N      ?i'  r      zi , j -> i jdtyper   r   )torcharangeeinsumfloatcatsincos)r   r   inv_freqsinusoid_inp r*   \/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/gptj/modeling_gptj.pycreate_sinusoidal_positions:   s   r,   c                 C   s   |  |j|jd ddS Nr   r   )todevicerepeatshape)embed_positionsposition_idsr*   r*   r+   get_embed_positions@   s   r4   xc                 C   sb   | d d d d d d d d df }| d d d d d d dd df }t j| |fdd} | dS )Nr   r   r    )r!   stackflatten)r5   x1x2r*   r*   r+   rotate_every_twoE   s   ""
r<   tensorr&   r'   c                 C   s`   t |d d d d d d d f dd}t |d d d d d d d f dd}| | t| |  S )Nr   r	   )r!   repeat_interleaver<   )r=   r&   r'   r*   r*   r+   apply_rotary_pos_embL   s   &&r?   c                       s   e Zd Z fddZdd Zdd Z		ddd	Zd
d Z						ddej	de
eej  de
ej	 de
ej de
ej	 de
e de
e deeejeej f e
eejeej eejdf f  f fddZ  ZS )GPTJAttentionc                    s^  t    |j}| jdttj||ftjddd||dd | jdt	ddd t
|j| _t
|j| _|j| _|j| _| j| j | _| j| j | jkr_td| j d	| j d
ttj	| jtjdt | _t
j| j| jdd| _t
j| j| jdd| _t
j| j| jdd| _t
j| j| jdd| _|j| _| jp| j}t ||| _!d S )Nbiasr   r   F)
persistentmasked_biasg    ezEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).rA   )"super__init__max_position_embeddingsregister_bufferr!   trilonesboolviewr=   r   Dropout
attn_pdropattn_dropoutresid_pdropresid_dropouthidden_size	embed_dimnum_attention_headshead_dim
ValueErrorsqrtfloat32r.   get_default_dtype
scale_attnLineark_projv_projq_projout_proj
rotary_dimr,   r2   )selfconfigmax_positionspos_embd_dim	__class__r*   r+   rF   S   s:   

$zGPTJAttention.__init__c                 C   s|   |  dd ||f }||}|r|S t|jdkr%|dddddS t|jdkr4|ddddS td	t|j )
zO
        Splits hidden dim into attn_head_size and num_attention_heads
        Nr6      r   r   r	   r      3Input tensor rank should be one of [4, 5], but is: )sizerL   lenr1   permuterV   )ra   r=   rT   attn_head_sizerotary	new_shaper*   r*   r+   _split_headsu   s   
zGPTJAttention._split_headsc                 C   s   t |jdkr|ddddd }nt |jdkr%|dddd }n
tdt |j | dd	 || f }||S )
zR
        Merges attn_head_size dim and num_attn_heads dim into hidden dim
        rg   r   r   r	   r   rh   ri   Nr7   )rk   r1   rl   
contiguousrV   rj   rL   )ra   r=   rT   rm   ro   r*   r*   r+   _merge_heads   s   
zGPTJAttention._merge_headsNc                 C   s   | d| d}}| jd d d d || |d |f }|tj}|tj}t||dd}	t|	jj	}
tj
|
|	jd|	j}
t||	|
}	|	| j }	|d ur[|	| }	tjj|	dd}	|	|j}	| |	}	|d urv|	| }	t|	|}||	fS )Nr7   r6   r   r    )rj   rA   r.   r!   rX   matmul	transposefinfor   minr=   r/   whererZ   r   
functionalsoftmaxrO   )ra   querykeyvalueattention_mask	head_maskquery_length
key_lengthcausal_maskattn_weights
mask_valueattn_outputr*   r*   r+   _attn   s$   	&

zGPTJAttention._attnc                 C   s8   | j }|j|jkr||j}|| _ ||jd ddS r-   )r2   r/   r.   r0   r1   )ra   r3   r2   r*   r*   r+   _get_embed_positions   s
   z"GPTJAttention._get_embed_positionsFhidden_states
layer_pastr}   r3   r~   	use_cacheoutput_attentionsr   .c                 C   s  |  |}| |}	| |}
| || j| jd}| |	| j| jd}	| |
| j| jd}
t|s6tj	 r=t
| j|}n| |}|ddd|jd }t|d|}tj||jd d dd\}}| jd ur|	d d d d d d d | jf }|	d d d d d d | jd f }|d d d d d d d | jf }|d d d d d d | jd f }t|||}t|||}tj||gdd}	tj||gdd}nt|	||}	t|||}|	dddd}	|dddd}|d ur|d }|d }tj||	fd	d}	tj||
fd	d}
|du r|	|j|
f}nd }| ||	|
||\}}| || j| j}| |}| |}||f}|rB||f7 }|S )
NTFr6   r   r   r    r   r	   r7   )r^   r\   r]   rp   rT   rU   r   r!   jit
is_tracingr4   r2   r   	unsqueezer0   r1   gathersplitr`   r?   r%   rl   r.   r   r   rr   r_   rQ   )ra   r   r   r}   r3   r~   r   r   rz   r{   r|   r2   repeated_position_idssincosr&   r'   k_rotk_passq_rotq_passpast_key
past_valuepresentr   r   outputsr*   r*   r+   forward   sR   




""""




zGPTJAttention.forwardNNNNNNFF)__name__
__module____qualname__rF   rp   rr   r   r   r!   FloatTensorr   r   Tensor
LongTensorrK   r   r   __classcell__r*   r*   re   r+   r@   R   sF    "
*
	"r@   c                       s6   e Zd Z fddZdeej dejfddZ  ZS )GPTJMLPc                    sJ   t    |j}t||| _t||| _t|j | _	t
|j| _d S N)rE   rF   n_embdr   r[   fc_infc_outr
   activation_functionactrM   rP   dropout)ra   intermediate_sizerb   rS   re   r*   r+   rF     s   
zGPTJMLP.__init__r   r   c                 C   s,   |  |}| |}| |}| |}|S r   )r   r   r   r   )ra   r   r*   r*   r+   r     s
   



zGPTJMLP.forward)	r   r   r   rF   r   r!   r   r   r   r*   r*   re   r+   r     s    "
r   c                       s   e Zd Z fddZ						ddeej deeej  deej deej	 d	eej d
ee
 dee
 deeej eeejeejdf f  f fddZ  ZS )	GPTJBlockc                    sR   t    |jd ur|jnd|j }tj|j|jd| _t|| _	t
||| _d S )Nrh   eps)rE   rF   n_innerr   r   	LayerNormlayer_norm_epsilonln_1r@   attnr   mlp)ra   rb   	inner_dimre   r*   r+   rF   $  s
   

zGPTJBlock.__init__NFr   r   r}   r3   r~   r   r   r   .c              	   C   sx   |}|  |}| j|||||||d}	|	d }
|	dd  }| |}|
| | }|r1|f| }|S |f|dd   }|S )Nr   r   r}   r3   r~   r   r   r   r   )r   r   r   )ra   r   r   r}   r3   r~   r   r   residualattn_outputsr   r   feed_forward_hidden_statesr*   r*   r+   r   +  s(   

	

zGPTJBlock.forwardr   )r   r   r   rF   r   r!   r   r   r   r   rK   r   r   r   r*   r*   re   r+   r   #  s2    
(	r   c                       sB   e Zd ZdZeZdZdZdZdgZ	dZ
 fddZdd	 Z  ZS )
GPTJPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    transformerTr   past_key_valuesc                    s   t  j|i | d S r   )rE   rF   )ra   inputskwargsre   r*   r+   rF   [  s   zGPTJPreTrainedModel.__init__c                 C   s   t |tjfr!|jjjd| jjd |jdur|jj	  dS dS t |tj
rD|jjjd| jjd |jdurB|jj|j 	  dS dS t |tjrY|jj	  |jjd dS dS )zInitialize the weights.g        )meanstdNr   )
isinstancer   r[   weightdatanormal_rb   initializer_rangerA   zero_	Embeddingpadding_idxr   fill_)ra   moduler*   r*   r+   _init_weights^  s   

z!GPTJPreTrainedModel._init_weights)r   r   r   __doc__r   config_classbase_model_prefixis_parallelizablesupports_gradient_checkpointing_no_split_modules_skip_keys_device_placementrF   r   r   r*   r*   re   r+   r   N  s    r   aG  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`GPTJConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a9
  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.n_positions - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
a  
    This is an experimental feature and is a subject to change at a moment's notice. Uses a device map to distribute
    attention modules of the model across several devices. If no device map is given, it will evenly distribute blocks
    across all devices.

    Args:
        device_map (`Dict[int, list]`, optional, defaults to None):
            A dictionary that maps attention modules to devices. Note that the embedding module and LMHead are always
            automatically mapped to the first device (for esoteric reasons). That means that the first device should
            have fewer attention modules mapped to it than other devices. For reference, the GPT-J models have the
            following number of attention modules:

                - gpt-j-6B: 28

    Example:

    ```python
    # Here is an example of a device map on a machine with 4 GPUs using gpt-j-6B, which has a total of 28 attention modules:
    model = GPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B")
    device_map = {
        0: [0, 1, 2, 3, 4, 5, 6],
        1: [7, 8, 9, 10, 11, 12, 13],
        2: [14, 15, 16, 17, 18, 19, 20],
        3: [21, 22, 23, 24, 25, 26, 27],
    }
    model.parallelize(device_map)
    ```
a6  
    Moves the model to CPU from a model parallel state.

    Example:

    ```python
    # On a 4 GPU machine with gpt-j-6B:
    model = GPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B")
    device_map = {
        0: [0, 1, 2, 3, 4, 5, 6],
        1: [7, 8, 9, 10, 11, 12, 13],
        2: [14, 15, 16, 17, 18, 19, 20],
        3: [21, 22, 23, 24, 25, 26, 27],
    }
    model.parallelize(device_map)  # Splits the model across several devices
    model.deparallelize()  # Put the model back on cpu and cleans memory by calling torch.cuda.empty_cache()
    ```
z_The bare GPT-J Model transformer outputting raw hidden-states without any specific head on top.c                       s  e Zd Z fddZeedddZeedd Zdd	 Z	d
d Z
eedeeeeed											ddeej deeeej   deej deej deej deej deej dee dee dee dee deeef fddZ  ZS )	GPTJModelc                    s   t     j| _ j| _t j| j| _t j	| _
t fddt jD | _tj| j jd| _d| _d | _d| _|   d S )Nc                    s   g | ]}t  qS r*   )r   ).0_rb   r*   r+   
<listcomp>  s    z&GPTJModel.__init__.<locals>.<listcomp>r   F)rE   rF   r   rS   
vocab_sizer   r   wterM   
embd_pdropdrop
ModuleListrangen_layerhr   r   ln_fmodel_parallel
device_mapgradient_checkpointing	post_initra   rb   re   r   r+   rF     s    zGPTJModel.__init__Nc                 C   s   t dt |d u rtt| jttj	 n|| _
t| j
t| j d| _d| j
 v r/dn
dtt| j
  | _dtt| j
  | _| j| j| _| j
 D ]\}}|D ]}dt| }| j| || j|< qZqT| j| j| _d S )Na6  `GPTJModel.parallelize` is deprecated and will be removed in v5 of Transformers, you should load your model with `device_map='balanced'` in the call to `from_pretrained`. You can also provide your own `device_map` but it needs to be a dictionary module_name to device, so for instance {'h.0': 0, 'h.1': 1, ...}Tcpucuda:)warningswarnFutureWarningr   rk   r   r   r!   cudadevice_countr   r   r   keysstrrv   first_devicemaxlast_devicer   r.   itemsr   )ra   r   kvblockcuda_devicer*   r*   r+   parallelize  s"   $	*zGPTJModel.parallelizec                 C   sx   t dt d| _d | _d| _d| _| jd| _t	t
| jD ]}| j| d| j|< q | jd| _tj  d S )N\Like `parallelize`, `deparallelize` is deprecated and will be removed in v5 of Transformers.Fr   )r   r   r   r   r   r   r   r   r.   r   rk   r   r   r!   r   empty_cache)ra   indexr*   r*   r+   deparallelize  s   zGPTJModel.deparallelizec                 C      | j S r   r   ra   r*   r*   r+   get_input_embeddings     zGPTJModel.get_input_embeddingsc                 C   
   || _ d S r   r   ra   new_embeddingsr*   r*   r+   set_input_embeddings      
zGPTJModel.set_input_embeddingsbatch_size, sequence_length
checkpointoutput_typer   real_checkpoint	input_idsr   r}   token_type_idsr3   r~   inputs_embedsr   r   output_hidden_statesreturn_dictr   c                    s6  |	d ur|	n| j j}	|
d ur|
n| j j}
|d ur|n| j j}|d ur$|n| j j}|d ur4|d ur4td|d urP| || | }|d|d }|j	d }n|d urb| d d }|j	d }ntd|d urm|j
n|j
}|d ur||d|d }|d u rd}td gt| j }n	|d d d}|d u rtj||d | tj|d}|d}|d ur|dkrtd||d}|d d d d d d f }|j| jd}d	| t| jj }| || j j}|d u r| |}| |d ur| |} |  |   d
|dd    df }| jr"| jr"|r"td d}|r'dnd }|	r.dnd }|
r5dnd }tt| j|D ]\}\}}| j rxtj!" j
 |d ur`t fdd|D }|d urk| j
}t#|tj$rx| j
}|
r| f }| jr| jr| %|j& d |||| ||	}n| ||||| ||	d}|d  |du r||d f }|	r|||rdnd f }| j r| j'( D ]!\}}||d krdt)| | j*kr dt)|d   q͐q?| +   | |
r| f }|stdd  |||fD S t, |||dS )NzDYou cannot specify both input_ids and inputs_embeds at the same timer6   r   z5You have to specify either input_ids or inputs_embedsr7   )r   r/   z$batch_size has to be defined and > 0r   r   )r6   r   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr*   c                 3   s    | ]	}|  jV  qd S r   )r.   r/   r   
past_stater   r*   r+   	<genexpr>      z$GPTJModel.forward.<locals>.<genexpr>r   Tr   r   c                 s   s    | ]	}|d ur|V  qd S r   r*   )r   r   r*   r*   r+   r    r  )last_hidden_stater   r   
attentions)-rb   r   r  r   use_return_dictrV   %warn_if_padding_and_no_attention_maskrj   rL   r1   r/   tuplerk   r   r!   r"   longr   r.   r   ru   rv   get_head_maskr   r   r   r   trainingloggerwarning_once	enumeratezipr   r   
set_devicer   r   _gradient_checkpointing_func__call__r   r   r   r   r   r   )ra   r  r   r}   r  r3   r~   r  r   r   r  r  input_shape
batch_sizer/   past_lengthtoken_type_embedsoutput_shapepresentsall_self_attentionsall_hidden_statesir   r   r   r   r   r*   r  r+   r   #  s   








"


zGPTJModel.forwardr   NNNNNNNNNNN)r   r   r   rF   r   PARALLELIZE_DOCSTRINGr   DEPARALLELIZE_DOCSTRINGr   r   r  r   GPTJ_INPUTS_DOCSTRINGformatr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOC_REAL_CHECKPOINT_FOR_DOCr   r!   r   r   r   r   rK   r   r   r   r*   r*   re   r+   r     sf    
	

r   zK
    The GPT-J Model transformer with a language modeling head on top.
    c                       sT  e Zd ZdgZ fddZeed#ddZeedd Z	d	d
 Z
dd Zd$ddZeedeeeeed												d%deej deeeej   deej deej deej deej deej deej dee dee dee dee deeef fddZedeeej  d ejdeeej  fd!d"Z  Z S )&GPTJForCausalLMzlm_head.weightc                    s@   t  | t|| _t|j|j| _d| _	d | _
|   d S NF)rE   rF   r   r   r   r[   r   r   lm_headr   r   r   r   re   r*   r+   rF     s   
zGPTJForCausalLM.__init__Nc                 C   st   t dt |d u rtt| jjttj	
 n|| _t| jt| jj | j| j | j| jj| _d| _d S )NaT  `GPTJForCausalLM.parallelize` is deprecated and will be removed in v5 of Transformers, you should load your model with `device_map='balanced'` in the call to `from_pretrained`. You can also provide your own `device_map` but it needs to be a dictionary module_name to device, so for instance {'transformer.h.0': 0, 'transformer.h.1': 1, ...}T)r   r   r   r   rk   r   r   r   r!   r   r   r   r   r   r8  r.   r   r   )ra   r   r*   r*   r+   r     s   	
zGPTJForCausalLM.parallelizec                 C   sF   t dt | j  | jd| _| jd| _d| _tj	
  d S )Nr   r   F)r   r   r   r   r   r.   r8  r   r!   r   r   r   r*   r*   r+   r     s   
zGPTJForCausalLM.deparallelizec                 C   r   r   r8  r   r*   r*   r+   get_output_embeddings  r  z%GPTJForCausalLM.get_output_embeddingsc                 C   r  r   r9  r  r*   r*   r+   set_output_embeddings  r  z%GPTJForCausalLM.set_output_embeddingsc                 K   s(  | dd }|r>|d d jd }|jd |kr|}n|jd d }|d d |d f }|d ur>|d d |jd  d f }| dd }| dd }	|d urs|	d u rs| dd }	|	|dkd |rs|	d d |jd  d f }	|d ur|d u rd|i}
nd	|i}
|
|| d
|	||d |
S )Nr  r   r   r   r}   r3   r6   r  r  r   )r   r   r3   r}   r  )getr1   r  cumsummasked_fill_update)ra   r  r   r  r   r  r'  remove_prefix_lengthr}   r3   model_inputsr*   r*   r+   prepare_inputs_for_generation
  s8   

z-GPTJForCausalLM.prepare_inputs_for_generationr  r  r  r   r}   r  r3   r~   r  labelsr   r   r  r  r   c                 C   s.  |dur|n| j j}| j||||||||	|
||d}|d }| jr1tj| jj || j	j
j}| 	|tj}d}|duru||j}|dddddf  }|dddf  }t }||d|d|d}||j}|s|f|dd  }|dur|f| S |S t|||j|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N
r   r}   r  r3   r~   r  r   r   r  r  r   .r6   r   losslogitsr   r   r  )rb   r  r   r   r!   r   r"  r   r.   r8  r   r/   rX   rq   r   rL   rj   r   r   r   r   r  )ra   r  r   r}   r  r3   r~   r  rC  r   r   r  r  transformer_outputsr   	lm_logitsrF  shift_logitsshift_labelsloss_fctoutputr*   r*   r+   r   7  sJ   zGPTJForCausalLM.forwardbeam_idxc                    s   t  fdd| D S )a  
        This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
        [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
        beam_idx at every generation step.
        c                 3   s&    | ]}t  fd d|D V  qdS )c                 3   s$    | ]}| d  |jV  qdS )r   N)index_selectr.   r/   r  rN  r*   r+   r    s   " z;GPTJForCausalLM._reorder_cache.<locals>.<genexpr>.<genexpr>Nr  )r   r   rP  r*   r+   r    s
    
z1GPTJForCausalLM._reorder_cache.<locals>.<genexpr>rQ  )r   rN  r*   rP  r+   _reorder_cache  s   	zGPTJForCausalLM._reorder_cacher   r   NNNNNNNNNNNN)!r   r   r   _tied_weights_keysrF   r   r/  r   r0  r   r:  r;  rB  r   r1  r2  r   r3  r   r4  r5  r   r!   r   r   r   r   rK   r   r   staticmethodrR  r   r*   r*   re   r+   r6    s    

-	

Ir6  a  
    The GPT-J Model transformer with a sequence classification head on top (linear layer).

    [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT, GPT-2, GPT-Neo) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                       s   e Zd Z fddZeededee	e
d												ddeej deeeej   d	eej d
eej deej deej deej deej dee dee dee dee deeef fddZ  ZS )GPTJForSequenceClassificationc                    sL   t  | |j| _t|| _tj|j| jdd| _d| _	d | _
|   d S )NFrD   )rE   rF   
num_labelsr   r   r   r[   r   scorer   r   r   r   re   r*   r+   rF     s   
z&GPTJForSequenceClassification.__init__r  z4ydshieh/tiny-random-gptj-for-sequence-classificationr  Nr  r   r}   r  r3   r~   r  rC  r   r   r  r  r   c                 C   s,  |dur|n| j j}| j||||||||	|
||d}|d }| |}|dur-|jd }n|jd }| j jdu r@|dkr@td| j jdu rId}n$|durat|| j j	 
dd |j}nd}t| jj d |tj||jd|f }d}|dur||j}| j jdu r| jdkrd	| j _n| jdkr|jtjks|jtj	krd
| j _nd| j _| j jd	krt }| jdkr|| | }n+|||}n%| j jd
krt }||d| j|d}n| j jdkrt }|||}|s
|f|dd  }|dur|f| S |S t|||j|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NrD  r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.r6   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`)r/   
regressionsingle_label_classificationmulti_label_classificationrE  )rb   r  r   rX  r1   pad_token_idrV   r!   eqintargmaxr.   r/   r  warningrf   r   r"   problem_typerW  r   r  r   squeezer   rL   r   r   r   r   r  )ra   r  r   r}   r  r3   r~   r  rC  r   r   r  r  rH  r   rG  r&  sequence_lengthspooled_logitsrF  rL  rM  r*   r*   r+   r     s|   



"


z%GPTJForSequenceClassification.forwardrS  )r   r   r   rF   r   r1  r2  r   r   r4  r5  r   r!   r   r   r   r   rK   r   r   r   r*   r*   re   r+   rV    s`    	

rV  z
    The GPT-J Model transformer with a span classification head on top for extractive question-answering tasks like
    SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                       s   e Zd Z fddZeedeee	e
ed											ddeej deej deej d	eej d
eej deej deej deej dee dee dee deee	f fddZ  ZS )GPTJForQuestionAnsweringc                    sH   t  | |j| _t|| _t|j|j| _d| _	d | _
|   d S r7  )rE   rF   rW  r   r   r   r[   rR   
qa_outputsr   r   r   r   re   r*   r+   rF   $  s   
z!GPTJForQuestionAnswering.__init__r  r  Nr  r}   r  r3   r~   r  start_positionsend_positionsr   r  r  r   c                 C   sX  |dur|n| j j}| j|||||||	|
|d	}|d }| |}|jddd\}}|d }|d }d}|dur|durt| dkrS|d	|j
}t| dkrd|d	|j
}|d}|d|}|d|}t|d}|||}|||}|| d }|s||f|dd  }|dur|f| S |S t||||j|jd	S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        N)r}   r  r3   r~   r  r   r  r  r   r   r6   r    )ignore_indexr   )rF  start_logits
end_logitsr   r  )rb   r  r   rf  r   rb  rq   rk   rj   r.   r/   clampr   r   r   r  )ra   r  r}   r  r3   r~   r  rg  rh  r   r  r  r   sequence_outputrG  rj  rk  
total_lossignored_indexrL  
start_lossend_lossrM  r*   r*   r+   r   1  sP   




z GPTJForQuestionAnswering.forwardr.  )r   r   r   rF   r   r1  r2  r   r3  r   r4  r5  r   r!   r   r   rK   r   r   r   r   r*   r*   re   r+   re    sZ    	

re  )>r   r   typingr   r   r   r!   torch.fxtorch.utils.checkpointr   torch.nnr   r   r   activationsr
   modeling_outputsr   r   r   r   modeling_utilsr   utilsr   r   r   r   r   utils.model_parallel_utilsr   r   configuration_gptjr   
get_loggerr   r  r3  r5  r4  "GPTJ_PRETRAINED_MODEL_ARCHIVE_LISTr^  r   r,   fxwrapr4   r<   r?   Moduler@   r   r   r   GPTJ_START_DOCSTRINGr1  r/  r0  r   r6  rV  re  r*   r*   r*   r+   <module>   sp   

" ?+!1 q Aw