o h.@s<dZddlZddlmZmZmZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZmZmZmZmZdd lmZdd lmZdd lmZmZmZmZmZmZd d l m!Z!erkddl"Z"ddl#m$Z$e%e&Z'dZ(dZ)ddgZ*Gdddej+Z,Gdddej+Z-Gdddej+Z.Gdddej+Z/Gdddej+Z0Gdddej+Z1Gdddej+Z2d=d#d$Z3Gd%d&d&ej+Z4Gd'd(d(eZ5d>d)d*Z6Gd+d,d,ej+Z7d-Z8d.Z9Gd/d0d0ej+Z:ed1e8Gd2d3d3e5Z;ed4e8Gd5d6d6e5ZdS)?z PyTorch LayoutLMv2 model.N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)BaseModelOutputBaseModelOutputWithPoolingQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward)add_start_docstrings%add_start_docstrings_to_model_forwardis_detectron2_availableloggingreplace_return_docstringsrequires_backends)LayoutLMv2Config)META_ARCH_REGISTRYz!microsoft/layoutlmv2-base-uncasedrz"microsoft/layoutlmv2-large-uncasedcs(eZdZdZfddZddZZS)LayoutLMv2EmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cstt|tj|j|j|jd|_t|j |j|_ t|j |j |_ t|j |j |_t|j |j|_t|j |j|_t|j|j|_tj|j|jd|_t|j|_|jdt|j ddddS)N) padding_idxeps position_ids)rF persistent)superr__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsmax_2d_position_embeddingscoordinate_sizex_position_embeddingsy_position_embeddings shape_sizeh_position_embeddingsw_position_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandselfconfig __class__h/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/layoutlmv2/modeling_layoutlmv2.pyr$Bs zLayoutLMv2Embeddings.__init__c Csz:||dddddf}||dddddf}||dddddf}||dddddf}WntyK}ztd|d}~ww||dddddf|dddddf}||dddddf|dddddf}tj||||||gdd} | S)Nrrr z;The `bbox` coordinate values should be within 0-1000 range.r dim)r.r/ IndexErrorr1r2r;cat) r?bboxleft_position_embeddingsupper_position_embeddingsright_position_embeddingslower_position_embeddingser1r2spatial_position_embeddingsrCrCrD!_calc_spatial_position_embeddingsTs,  22 z6LayoutLMv2Embeddings._calc_spatial_position_embeddings)__name__ __module__ __qualname____doc__r$rQ __classcell__rCrCrArDr?s rcs@eZdZfddZddZddZ     d d d ZZS) LayoutLMv2SelfAttentioncst|j|jdkrt|dstd|jd|jd|j|_|j|_t|j|j|_|j|j|_ |j |_ |j |_ |jrht j |jd|j dd|_t td d |j |_t td d |j |_nt |j|j |_t |j|j |_t |j|j |_t |j|_dS) Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r Fbiasr)r#r$r'num_attention_headshasattr ValueErrorfast_qkvintattention_head_size all_head_sizehas_relative_attention_biashas_spatial_attention_biasrLinear qkv_linear Parameterr;zerosq_biasv_biasquerykeyvaluer7attention_probs_dropout_probr9r>rArCrDr$os*  z LayoutLMv2SelfAttention.__init__cCs6|dd|j|jf}|j|}|ddddS)Nr rrErr )sizer\raviewpermute)r?x new_x_shaperCrCrDtranspose_for_scoress z,LayoutLMv2SelfAttention.transpose_for_scorescCs|jrB||}tj|ddd\}}}||jkr'||j}||j}n*d|dd}||jj|}||jj|}n||}| |}| |}|||fS)Nr r rFrr)r ) r_rfr;chunk ndimensionrirjrprkrlrm)r? hidden_statesqkvqkv_szrCrCrD compute_qkvs       z#LayoutLMv2SelfAttention.compute_qkvNFcCs ||\}}} ||} ||} || } | t|j} t| | dd} |jr0| |7} |j r7| |7} | | tj t| jj} tjj| dtjd| }||}|durb||}t|| }|dddd}|dd|jf}|j|}|r||f}|S|f}|S)Nr )rGdtyperrErr )r~rtmathsqrtrar;matmul transposercrdfloat masked_fill_toboolfinforminr functionalsoftmaxfloat32type_asr9rq contiguousrorbrp)r?rxattention_mask head_maskoutput_attentionsrel_pos rel_2d_posrzr{r| query_layer key_layer value_layerattention_scoresattention_probs context_layernew_context_layer_shapeoutputsrCrCrDforwards2       zLayoutLMv2SelfAttention.forwardNNFNN)rRrSrTr$rtr~rrVrCrCrArDrWns rWcs0eZdZfddZ     dddZZS)LayoutLMv2Attentioncs"tt||_t||_dSN)r#r$rWr?LayoutLMv2SelfOutputoutputr>rArCrDr$s  zLayoutLMv2Attention.__init__NFc Cs<|j||||||d}||d|}|f|dd} | S)Nrrrr)r?r) r?rxrrrrr self_outputsattention_outputrrCrCrDrs zLayoutLMv2Attention.forwardrrRrSrTr$rrVrCrCrArDrs rc$eZdZfddZddZZS)rcsBtt|j|j|_tj|j|jd|_t|j |_ dSNr) r#r$rrer'denser5r6r7r8r9r>rArCrDr$ zLayoutLMv2SelfOutput.__init__cC&||}||}|||}|Srrr9r5r?rx input_tensorrCrCrDr  zLayoutLMv2SelfOutput.forwardrrCrCrArDrs rcs2eZdZfddZdejdejfddZZS)LayoutLMv2IntermediatecsDtt|j|j|_t|jt rt |j|_ dS|j|_ dSr) r#r$rrer'intermediate_sizer isinstance hidden_actstrr intermediate_act_fnr>rArCrDr$s   zLayoutLMv2Intermediate.__init__rxreturncCs||}||}|Sr)rr)r?rxrCrCrDrs  zLayoutLMv2Intermediate.forwardrRrSrTr$r;TensorrrVrCrCrArDrs rcs8eZdZfddZdejdejdejfddZZS)LayoutLMv2OutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSr) r#r$rrerr'rr5r6r7r8r9r>rArCrDr$rzLayoutLMv2Output.__init__rxrrcCrrrrrCrCrDr rzLayoutLMv2Output.forwardrrCrCrArDrs $rcs8eZdZfddZ     d ddZddZZS) LayoutLMv2Layercs:t|j|_d|_t||_t||_t||_ dS)Nr) r#r$chunk_size_feed_forward seq_len_dimr attentionr intermediaterrr>rArCrDr$s   zLayoutLMv2Layer.__init__NFc CsL|j||||||d}|d}|dd} t|j|j|j|} | f| } | S)N)rrrrr)rrfeed_forward_chunkrr) r?rxrrrrrself_attention_outputsrr layer_outputrCrCrDrs   zLayoutLMv2Layer.forwardcCs||}|||}|Sr)rr)r?rintermediate_outputrrCrCrDr9s  z"LayoutLMv2Layer.feed_forward_chunkr)rRrSrTr$rrrVrCrCrArDrs   rT c Csd}|r|d}||dk|7}t|}n t| t|}|d}||k}|t||t||||tj}t |t ||d}|t |||7}|S)a Adapted from Mesh Tensorflow: https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 Translate relative position to a bucket number for relative attention. The relative position is defined as memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for small absolute relative_position and larger buckets for larger absolute relative_positions. All relative positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket. This should allow for more graceful generalization to longer sequences than the model has been trained on. Args: relative_position: an int32 Tensor bidirectional: a boolean - whether the attention is bidirectional num_buckets: an integer max_distance: an integer Returns: a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets) rrEr) longr;absmax zeros_likelogrrrr full_likewhere) relative_position bidirectional num_buckets max_distanceretn max_exactis_small val_if_largerCrCrDrelative_position_bucket?s &rcsDeZdZfddZddZddZ      d d d ZZS) LayoutLMv2Encodercst|_tfddtjD|_j|_j |_ |jr6j |_ j |_ tj |j j dd|_|j rWj|_j|_tj |jj dd|_tj |jj dd|_d|_dS)Ncsg|]}tqSrC)r).0_r@rCrD osz.LayoutLMv2Encoder.__init__..FrZ)r#r$r@r ModuleListrangenum_hidden_layerslayerrcrd rel_pos_bins max_rel_posrer\ rel_pos_biasmax_rel_2d_posrel_2d_pos_binsrel_pos_x_biasrel_pos_y_biasgradient_checkpointingr>rArrDr$ls   zLayoutLMv2Encoder.__init__cCsN|d|d}t||j|jd}|jj|dddd}|}|S)Nrr rrrr rrE) unsqueezerrrrweighttrqr)r?r rel_pos_matrrCrCrD!_calculate_1d_position_embeddingssz3LayoutLMv2Encoder._calculate_1d_position_embeddingsc Cs|dddddf}|dddddf}|d|d}|d|d}t||j|jd}t||j|jd}|jj|dddd}|jj|dddd}| }| }||}|S)Nrr rr rrrE) rrrrrrrrqrr) r?rJposition_coord_xposition_coord_yrel_pos_x_2d_matrel_pos_y_2d_mat rel_pos_x rel_pos_yrrCrCrD!_calculate_2d_position_embeddingss(z3LayoutLMv2Encoder._calculate_2d_position_embeddingsNFTc  Cs|rdnd} |r dnd} |jr||nd} |jr||nd} t|jD]@\} }|r0| |f} |dur8|| nd}|jrN|jrN|j|j ||||| | d}n |||||| | d}|d}|re| |df} q%|rm| |f} |s{t dd|| | fDSt || | dS)NrCrrrcss|] }|dur|VqdSrrC)rr|rCrCrD sz,LayoutLMv2Encoder.forward..)last_hidden_staterx attentions) rcrrdr enumeraterrtraining_gradient_checkpointing_func__call__tupler )r?rxrrroutput_hidden_states return_dictrJrall_hidden_statesall_self_attentionsrri layer_modulelayer_head_mask layer_outputsrCrCrDrsX       zLayoutLMv2Encoder.forward)NNFFTNN)rRrSrTr$rrrrVrCrCrArDrks  rc@s$eZdZdZeZeZdZddZ dS)LayoutLMv2PreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. layoutlmv2cCst|tjr |jjjd|jjd|jdur|jj dSdSt|tj rC|jjjd|jjd|j durA|jj|j  dSdSt|tj rX|jj |jj ddSdS)zInitialize the weightsg)meanstdN?)rrrerdatanormal_r@initializer_ranger[zero_r%rr5fill_)r?modulerCrCrD _init_weightss      z'LayoutLMv2PreTrainedModel._init_weightsN) rRrSrTrUr config_class(LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LISTpretrained_model_archive_mapbase_model_prefixrrCrCrCrDrs  rcCst|tjjjjrtjj||S|}t|tj j rKtjj|j |j dd|d}tj |j|_tj |j|_|j|_|j|_tjdtj|jjd|_|D] \}}||t||qO~|S)NT) num_featuresraffinetrack_running_stats process_grouprrdevice)rr;rmodules batchnorm _BatchNorm SyncBatchNormconvert_sync_batchnorm detectron2layersFrozenBatchNorm2drrrgrr[ running_mean running_vartensorrrnum_batches_trackednamed_children add_modulemy_convert_sync_batchnorm)rr module_outputnamechildrCrCrDr(s(r(cs,eZdZfddZddZddZZS)LayoutLMv2VisualBackbonec st||_|jjj}t||j}t|j t j j j s"J|j |_ t |jjjt |jjjks6Jt |jjj}|jdt|jjj|dddd|jdt|jjj|ddddd|_trtdd}|j |jj}ttt|d ||jd tt|d||jdf|_n t |jdd |_t |jd kr|j!|j |jj"|j |jj"|jd ksJdS) N pixel_meanrFr! pixel_stdp2z0using `AvgPool2d` instead of `AdaptiveAvgPool2d`)r0rrE)#r#r$get_detectron2_configcfgMODELMETA_ARCHITECTURErgetrbackbonermodelingFPNlen PIXEL_MEAN PIXEL_STDr:r;rrpout_feature_key$are_deterministic_algorithms_enabledloggerwarning output_shapestrider AvgPool2drceilimage_feature_pool_shapepoolAdaptiveAvgPool2dappendchannels)r?r@ meta_archmodel num_channels input_shapebackbone_striderArCrDr$s<       $z!LayoutLMv2VisualBackbone.__init__cCsVt|r|n|j|j|j}||}||j}||jdd dd }|S)NrE) start_dimr) r; is_tensorr$r-r.r6r<rEflattenrr)r?images images_inputfeaturesrCrCrDr<s   z LayoutLMv2VisualBackbone.forwardcstjrtjrtjdkstdtj}tjtj}|dks.tdfddt |Dfddt |D}|}t |j ||d|_ dS) Nr z/Make sure torch.distributed is set up properly.rzGMake sure the number of processes can be divided by the number of nodescs&g|]}tt||dqSru)listrrr) node_sizerCrDrQs&zCLayoutLMv2VisualBackbone.synchronize_batch_norm..csg|] }tjj|dqS))ranks)r; distributed new_grouprU)node_global_ranksrCrDrRs)r) r;rX is_availableis_initializedget_rank RuntimeErrorcuda device_countget_world_sizerr(r6)r? self_rank world_sizesync_bn_groups node_rankrC)rZrVrDsynchronize_batch_normCs"      z/LayoutLMv2VisualBackbone.synchronize_batch_norm)rRrSrTr$rrfrVrCrCrArDr,s #r,aM This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`LayoutLMv2Config`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`torch.LongTensor` of shape `{0}`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`): Batch of document images. attention_mask (`torch.FloatTensor` of shape `{0}`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `{0}`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `{0}`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. cr)LayoutLMv2Poolercs*tt|j|j|_t|_dSr)r#r$rrer'rTanh activationr>rArCrDr$s zLayoutLMv2Pooler.__init__cCs(|dddf}||}||}|S)Nr)rri)r?rxfirst_token_tensor pooled_outputrCrCrDrs  zLayoutLMv2Pooler.forwardrrCrCrArDrgs rgzdThe bare LayoutLMv2 Model transformer outputting raw hidden-states without any specific head on top.cseZdZfddZddZddZd dd Zd d Zd d Zd!ddZ e e de eed           d"deejdeejdeejdeejdeejdeejdeejdeejdeedeedeedeeeffddZZS)#LayoutLMv2Modelcst|dt|||_|j|_t||_t||_t |j d|j |_ |jr8t t d|j jd|_t j|j |jd|_t |j|_t||_t||_|dS)Nrr rrr)rr#r$r@has_visual_segment_embeddingr embeddingsr,visualrrerDr' visual_projrgr%rvisual_segment_embeddingr5r6visual_LayerNormr7r8visual_dropoutrencoderrgpooler post_initr>rArCrDr$s       zLayoutLMv2Model.__init__cCs|jjSrrnr)r?rCrCrDget_input_embeddingssz$LayoutLMv2Model.get_input_embeddingscCs ||j_dSrrw)r?rmrCrCrDset_input_embeddingss z$LayoutLMv2Model.set_input_embeddingsNc Cs|dur |}n|dd}|d}|dur+tj|tj|jd}|d|}|dur4t|}|dur>|j |}|j |}|j |} |j |} ||| | } |j | } |j| } | S)Nr rrr)ror;r<rrr expand_asrrnr)r+rQr4r5r9) r? input_idsrJrtoken_type_ids inputs_embedsrL seq_lengthr+rPr4rnrCrCrD_calc_text_embeddingss$        z%LayoutLMv2Model._calc_text_embeddingscCs\|||}|j|}|j|}|||}|jr"||j7}||}||}|Sr) rprornr+rQrmrqrrrs)r?imagerJrvisual_embeddingsr+rPrnrCrCrD_calc_img_embeddingss      z$LayoutLMv2Model._calc_img_embeddingscCs tjtjdd|ddd||jd|jjddd}tjtjdd|jjddd||jd|jjddd}tj|dd|dd|dd|dddd|dd|dd|dd|ddddgdd d| d}||ddd}|S) Nrir)rrfloor) rounding_moder rF) r;divr<rr@rDstackrepeatrrpro)r?rDrJr final_shape visual_bbox_x visual_bbox_y visual_bboxrCrCrD_calc_visual_bboxsD     z!LayoutLMv2Model._calc_visual_bboxcCsH|dur |dur td|dur|S|dur |ddStd)NDYou cannot specify both input_ids and inputs_embeds at the same timer 5You have to specify either input_ids or inputs_embeds)r^ro)r?r|r~rCrCrD_get_input_shapesz LayoutLMv2Model._get_input_shapez(batch_size, sequence_length) output_typerr|rJrrr}rrr~rrrrc  Cs| dur| n|jj} | dur| n|jj} | dur| n|jj} |||} |dur+|jn|j} t| }|jjd|jjd|d<t |}t|||}|d|d7<t |}| |jj|| |}tj ||gdd}|durztj | | d}tj || d}tj ||gdd}|durtj | tj| d}|dur| d}|jjddd|f}|| }tjd|dtj| d| dd}tj ||gdd}|durtj tt| dgtj| d}|j|||||d}|j|||d }tj ||gdd}|dd }|j|jd }d |t|jj}|durX|dkr:|ddd d }||jjd d d d }n|d krL|dd d }|jt|jd }ndg|jj}|j |||||| | | d}|d}|!|}| s||f|ddSt"|||j#|j$dS)au Return: Examples: ```python >>> from transformers import AutoProcessor, LayoutLMv2Model, set_seed >>> from PIL import Image >>> import torch >>> from datasets import load_dataset >>> set_seed(88) >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased") >>> model = LayoutLMv2Model.from_pretrained("microsoft/layoutlmv2-base-uncased") >>> dataset = load_dataset("hf-internal-testing/fixtures_docvqa") >>> image_path = dataset["test"][0]["file"] >>> image = Image.open(image_path).convert("RGB") >>> encoding = processor(image, return_tensors="pt") >>> outputs = model(**encoding) >>> last_hidden_states = outputs.last_hidden_state >>> last_hidden_states.shape torch.Size([1, 342, 768]) ``` NrrrF)rr)r|rJr}rr~rrJrrE)rrr )rJrrrrr)r pooler_outputrxr)%r@rruse_return_dictrrrTrDr;SizerrIonesrhrrnrr=r<rrrrrrrrrrGrnext parametersrtrur rxr)r?r|rJrrr}rrr~rrrrLr visual_shaperr final_bboxvisual_attention_maskfinal_attention_maskrvisual_position_idsfinal_position_idstext_layout_emb visual_emb final_embextended_attention_maskencoder_outputssequence_outputrkrCrCrDr"s.       zLayoutLMv2Model.forwardr)NN) NNNNNNNNNNN)rRrSrTr$ryrzrrrrrLAYOUTLMV2_INPUTS_DOCSTRINGformatrr _CONFIG_FOR_DOCrr; LongTensor FloatTensorrrrr rrVrCrCrArDrls\   %      rlax LayoutLMv2 Model with a sequence classification head on top (a linear layer on top of the concatenation of the final hidden state of the [CLS] token, average-pooled initial visual embeddings and average-pooled final visual embeddings, e.g. for document image classification tasks such as the [RVL-CDIP](https://www.cs.cmu.edu/~aharley/rvl-cdip/) dataset. ceZdZfddZddZeedee e d            dde e j d e e j d e e jd e e jd e e j d e e j de e jde e jde e j de ede ede edeee ffddZZS)#LayoutLMv2ForSequenceClassificationcsNt||j|_t||_t|j|_t |j d|j|_ | dS)Nr r#r$ num_labelsrlrrr7r8r9rer' classifierrvr>rArCrDr$s   z,LayoutLMv2ForSequenceClassification.__init__cC |jjjSrrrnr)rxrCrCrDry z8LayoutLMv2ForSequenceClassification.get_input_embeddingsbatch_size, sequence_lengthrNr|rJrrr}rrr~labelsrrrrc  Cs0| dur| n|jj} |dur|durtd|dur%||||} n|dur2|dd} ntd|dur=|jn|j}t| }|jjd|jjd|d<t |}t| }|d|d7<t |}|j |jj|||}tj d|dtj |d| dd}|j j|||d}|j ||||||||| | | d }|dur|} n|dd} | d}|dddd|f|ddd|df}}|dddddf}|jdd }|jdd }tj|||gdd }||}||}d}| durv|jjdur/|jdkrd |j_n|jdkr+| jtj ks&| jtjkr+d |j_nd |j_|jjd krOt}|jdkrI||| }n-||| }n'|jjd krgt}||d|j| d}n|jjd krvt}||| }| s|f|dd}|dur|f|S|St|||j|jdS)a5 labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Returns: Example: ```python >>> from transformers import AutoProcessor, LayoutLMv2ForSequenceClassification, set_seed >>> from PIL import Image >>> import torch >>> from datasets import load_dataset >>> set_seed(88) >>> dataset = load_dataset("rvl_cdip", split="train", streaming=True) >>> data = next(iter(dataset)) >>> image = data["image"].convert("RGB") >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased") >>> model = LayoutLMv2ForSequenceClassification.from_pretrained( ... "microsoft/layoutlmv2-base-uncased", num_labels=dataset.info.features["label"].num_classes ... ) >>> encoding = processor(image, return_tensors="pt") >>> sequence_label = torch.tensor([data["label"]]) >>> outputs = model(**encoding, labels=sequence_label) >>> loss, logits = outputs.loss, outputs.logits >>> predicted_idx = logits.argmax(dim=-1).item() >>> predicted_answer = dataset.info.features["label"].names[4] >>> predicted_idx, predicted_answer (4, 'advertisement') ``` Nrr rrrrr r|rJrrr}rrr~rrrrF regressionsingle_label_classificationmulti_label_classificationrElosslogitsrxr) r@rr^%warn_if_padding_and_no_attention_maskrorrTrDr;rrrr<rrrrrIr9r problem_typerrr`rsqueezerrprrrxr)r?r|rJrrr}rrr~rrrrrLrrrrrinitial_image_embeddingsrrrfinal_image_embeddingscls_final_outputpooled_initial_image_embeddingspooled_final_image_embeddingsrrloss_fctrrCrCrDrs9      2        (    z+LayoutLMv2ForSequenceClassification.forward NNNNNNNNNNNN)rRrSrTr$ryrrrrrrrr;rrrrrrrVrCrCrArDrX         ra LayoutLMv2 Model with a token classification head on top (a linear layer on top of the text part of the hidden states) e.g. for sequence labeling (information extraction) tasks such as [FUNSD](https://guillaumejaume.github.io/FUNSD/), [SROIE](https://rrc.cvc.uab.es/?ch=13), [CORD](https://github.com/clovaai/cord) and [Kleister-NDA](https://github.com/applicaai/kleister-nda). cr) LayoutLMv2ForTokenClassificationcsJt||j|_t||_t|j|_t |j |j|_ | dSrrr>rArCrDr$ps   z)LayoutLMv2ForTokenClassification.__init__cCrrrrxrCrCrDryzrz5LayoutLMv2ForTokenClassification.get_input_embeddingsrrNr|rJrrr}rrr~rrrrrc  Cs| dur| n|jj} |j||||||||| | | d } |dur#|}n|dd}|d}| dddd|f}||}||}d}| dur[t}||d|j| d}| sq|f| dd}|duro|f|S|St ||| j | j dS)a2 labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Returns: Example: ```python >>> from transformers import AutoProcessor, LayoutLMv2ForTokenClassification, set_seed >>> from PIL import Image >>> from datasets import load_dataset >>> set_seed(88) >>> datasets = load_dataset("nielsr/funsd", split="test") >>> labels = datasets.features["ner_tags"].feature.names >>> id2label = {v: k for v, k in enumerate(labels)} >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased", revision="no_ocr") >>> model = LayoutLMv2ForTokenClassification.from_pretrained( ... "microsoft/layoutlmv2-base-uncased", num_labels=len(labels) ... ) >>> data = datasets[0] >>> image = Image.open(data["image_path"]).convert("RGB") >>> words = data["words"] >>> boxes = data["bboxes"] # make sure to normalize your bounding boxes >>> word_labels = data["ner_tags"] >>> encoding = processor( ... image, ... words, ... boxes=boxes, ... word_labels=word_labels, ... padding="max_length", ... truncation=True, ... return_tensors="pt", ... ) >>> outputs = model(**encoding) >>> logits, loss = outputs.logits, outputs.loss >>> predicted_token_class_ids = logits.argmax(-1) >>> predicted_tokens_classes = [id2label[t.item()] for t in predicted_token_class_ids[0]] >>> predicted_tokens_classes[:5] ['B-ANSWER', 'B-HEADER', 'B-HEADER', 'B-HEADER', 'B-HEADER'] ``` Nrr rrrEr) r@rrror9rrrprrrxr)r?r|rJrrr}rrr~rrrrrrLrrrrrrrCrCrDr}sDB   z(LayoutLMv2ForTokenClassification.forwardr)rRrSrTr$ryrrrrrrrr;rrrrrrrVrCrCrArDrfrra LayoutLMv2 Model with a span classification head on top for extractive question-answering tasks such as [DocVQA](https://rrc.cvc.uab.es/?ch=17) (a linear layer on top of the text part of the hidden-states output to compute `span start logits` and `span end logits`). c!seZdZdfdd ZddZeedee e d             dd e e j d e e j d e e jd e e jd e e j de e j de e jde e jde e j de e j de ede ede edeee ffddZZS)LayoutLMv2ForQuestionAnsweringTcsBt||j|_||_t||_t|j|j|_ | dSr) r#r$rrmrlrrrer' qa_outputsrv)r?r@rmrArCrDr$s   z'LayoutLMv2ForQuestionAnswering.__init__cCrrrrxrCrCrDryrz3LayoutLMv2ForQuestionAnswering.get_input_embeddingsrrNr|rJrrr}rrr~start_positions end_positionsrrrrc Cs| dur| n|jj} |j||||||||| | | d }|dur#|}n|dd}|d}|dddd|f}||}|jddd\}}|d}|d}d}| dur| durt| dkrn| d} t| dkr{| d} |d}| d|} | d|} t |d}||| }||| }||d}| s||f|dd}|dur|f|S|St ||||j |j d S) am start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. Returns: Example: In this example below, we give the LayoutLMv2 model an image (of texts) and ask it a question. It will give us a prediction of what it thinks the answer is (the span of the answer within the texts parsed from the image). ```python >>> from transformers import AutoProcessor, LayoutLMv2ForQuestionAnswering, set_seed >>> import torch >>> from PIL import Image >>> from datasets import load_dataset >>> set_seed(88) >>> processor = AutoProcessor.from_pretrained("microsoft/layoutlmv2-base-uncased") >>> model = LayoutLMv2ForQuestionAnswering.from_pretrained("microsoft/layoutlmv2-base-uncased") >>> dataset = load_dataset("hf-internal-testing/fixtures_docvqa") >>> image_path = dataset["test"][0]["file"] >>> image = Image.open(image_path).convert("RGB") >>> question = "When is coffee break?" >>> encoding = processor(image, question, return_tensors="pt") >>> outputs = model(**encoding) >>> predicted_start_idx = outputs.start_logits.argmax(-1).item() >>> predicted_end_idx = outputs.end_logits.argmax(-1).item() >>> predicted_start_idx, predicted_end_idx (154, 287) >>> predicted_answer_tokens = encoding.input_ids.squeeze()[predicted_start_idx : predicted_end_idx + 1] >>> predicted_answer = processor.tokenizer.decode(predicted_answer_tokens) >>> predicted_answer # results are not very good without further fine-tuning 'council mem - bers conducted by trrf treasurer philip g. kuehn to get answers which the public ... ``` ```python >>> target_start_index = torch.tensor([7]) >>> target_end_index = torch.tensor([14]) >>> outputs = model(**encoding, start_positions=target_start_index, end_positions=target_end_index) >>> predicted_answer_span_start = outputs.start_logits.argmax(-1).item() >>> predicted_answer_span_end = outputs.end_logits.argmax(-1).item() >>> predicted_answer_span_start, predicted_answer_span_end (154, 287) ``` Nrr rrrF) ignore_indexrE)r start_logits end_logitsrxr)r@rrrorsplitrrr9clamprr rxr)r?r|rJrrr}rrr~rrrrrrrLrrrrr total_loss ignored_indexr start_lossend_lossrrCrCrDrs\J           z&LayoutLMv2ForQuestionAnswering.forward)T) NNNNNNNNNNNNN)rRrSrTr$ryrrrrr rrr;rrrrrrrVrCrCrArDrs^         r)Trrr)?rUrtypingrrrr;torch.utils.checkpointrtorch.nnrrr activationsr modeling_outputsr r r rrmodeling_utilsr pytorch_utilsrutilsrrrrrrconfiguration_layoutlmv2rrdetectron2.modelingr get_loggerrRr>_CHECKPOINT_FOR_DOCrrModulerrWrrrrrrrrr(r,LAYOUTLMV2_START_DOCSTRINGrrgrlrrrrCrCrCrDsv        /] +,z B <} . {