o hޜ @sdZddlZddlmZmZmZmZmZddlZddl Zddlm Z ddl m Z m Z mZddlmZddlmZmZmZmZdd lmZdd lmZmZdd lmZmZmZmZm Z d d l!m"Z"e#e$Z%dZ&dZ'gdZ(dZ)dZ*gdZ+dEde,de,dee,de,fddZ-Gddde j.Z/Gddde j.Z0Gddde j.Z1Gd d!d!e j.Z2Gd"d#d#e j.Z3Gd$d%d%e j.Z4Gd&d'd'e j.Z5Gd(d)d)e j.Z6Gd*d+d+e j.Z7Gd,d-d-e j.Z8Gd.d/d/e j.Z9Gd0d1d1e j.Z:Gd2d3d3eZ;d4Zed9e<Gd:d;d;e;Z?Gdd?d?e j.ZAGd@dAdAe j.ZBedBe<GdCdDdDe;ZCdS)Fz PyTorch MobileViT model.N)DictOptionalSetTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)BaseModelOutputWithNoAttention(BaseModelOutputWithPoolingAndNoAttention$ImageClassifierOutputWithNoAttentionSemanticSegmenterOutput)PreTrainedModel) find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)MobileViTConfigrapple/mobilevit-small)rirztabby, tabby cat)rzapple/mobilevit-x-smallzapple/mobilevit-xx-smallzapple/deeplabv3-mobilevit-smallz!apple/deeplabv3-mobilevit-x-smallz"apple/deeplabv3-mobilevit-xx-smallrvaluedivisor min_valuereturncCsF|dur|}t|t||d||}|d|kr||7}t|S)a Ensure that all layers have a channel count that is divisible by `divisor`. This function is taken from the original TensorFlow repo. It can be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py Ng?)maxint)rrr new_valuer%f/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/models/mobilevit/modeling_mobilevit.pymake_divisibleIs  r'csveZdZ      ddededededed ed ed ed ed eeefddffdd Zde j de j fddZ Z S)MobileViTConvLayerrFTconfig in_channels out_channels kernel_sizestridegroupsbiasdilationuse_normalizationuse_activationr Nc stt|dd|} ||dkr td|d|d||dkr1td|d|dtj||||| |||dd |_| rNtj|d d d d d |_nd|_| rst | t r_t | |_ dSt |j t rmt |j |_ dS|j |_ dSd|_ dS)Nrr!rzInput channels (z) are not divisible by z groups.zOutput channels (zeros) r*r+r,r-paddingr0r.r/ padding_modegh㈵>g?T) num_featuresepsmomentumaffinetrack_running_stats)super__init__r# ValueErrorrConv2d convolution BatchNorm2d normalization isinstancestrr activation hidden_act) selfr)r*r+r,r-r.r/r0r1r2r4 __class__r%r&r<YsB         zMobileViTConvLayer.__init__featurescCs6||}|jdur||}|jdur||}|SN)r?rArD)rFrIr%r%r&forwards     zMobileViTConvLayer.forward)rrFrTT) __name__ __module__ __qualname__rr#boolrrCr<torchTensorrK __classcell__r%r%rGr&r(Xs>    6r(c sTeZdZdZ ddedededededd f fd d Zd ejdejfd dZ Z S)MobileViTInvertedResidualzQ Inverted residual block (MobileNetv2): https://arxiv.org/abs/1801.04381 rr)r*r+r-r0r Nc stttt||jd}|dvrtd|d|dko$||k|_t|||dd|_ t|||d|||d|_ t|||dd d |_ dS) Nr)rr!zInvalid stride .rr*r+r,r )r*r+r,r-r.r0Fr*r+r,r2) r;r<r'r#round expand_ratior= use_residualr( expand_1x1conv_3x3 reduce_1x1)rFr)r*r+r-r0expanded_channelsrGr%r&r<s0   z"MobileViTInvertedResidual.__init__rIcCs4|}||}||}||}|jr||S|SrJ)rZr[r\rY)rFrIresidualr%r%r&rKs    z!MobileViTInvertedResidual.forwardr) rLrMrN__doc__rr#r<rPrQrKrRr%r%rGr&rSs"!rSc sPeZdZ ddedededededdf fd d Zd ejdejfd d ZZ S)MobileViTMobileNetLayerrr)r*r+r- num_stagesr NcsRtt|_t|D]}t||||dkr|ndd}|j||}qdS)Nrr)r*r+r-)r;r<r ModuleListlayerrangerSappend)rFr)r*r+r-rbirdrGr%r&r<s    z MobileViTMobileNetLayer.__init__rIcC|jD]}||}q|SrJrd)rFrI layer_moduler%r%r&rK  zMobileViTMobileNetLayer.forward)rr rLrMrNrr#r<rPrQrKrRr%r%rGr&ras racsVeZdZdededdffdd Zdejdejfdd Zd ejdejfd d Z Z S) MobileViTSelfAttentionr) hidden_sizer Ncst||jdkrtd|fd|jd|j|_t||j|_|j|j|_tj||j|j d|_ tj||j|j d|_ tj||j|j d|_ t |j|_dS)NrzThe hidden size z4 is not a multiple of the number of attention heads rT)r/)r;r<num_attention_headsr=r#attention_head_size all_head_sizerLinearqkv_biasquerykeyrDropoutattention_probs_dropout_probdropoutrFr)rnrGr%r&r<s  zMobileViTSelfAttention.__init__xcCs6|dd|j|jf}|j|}|ddddS)Nrr!rr )sizerorpviewpermute)rFrz new_x_shaper%r%r&transpose_for_scoress z+MobileViTSelfAttention.transpose_for_scores hidden_statesc Cs||}|||}|||}||}t||dd}|t|j }t j j |dd}| |}t||}|dddd}|dd|jf} |j| }|S)Nr{dimrr!rr )rtrrurrPmatmul transposemathsqrtrpr functionalsoftmaxrxr~ contiguousr|rqr}) rFrmixed_query_layer key_layer value_layer query_layerattention_scoresattention_probs context_layernew_context_layer_shaper%r%r&rKs     zMobileViTSelfAttention.forward) rLrMrNrr#r<rPrQrrKrRr%r%rGr&rmsrmcs@eZdZdededdffdd Zdejdejfdd ZZ S) MobileViTSelfOutputr)rnr Ncs*tt|||_t|j|_dSrJr;r<rrrdenservhidden_dropout_probrxryrGr%r&r< zMobileViTSelfOutput.__init__rcC||}||}|SrJrrxrFrr%r%r&rKrkzMobileViTSelfOutput.forwardrlr%r%rGr&rsrcsVeZdZdededdffdd Zdeeddfdd Zd ej dej fd d Z Z S) MobileViTAttentionr)rnr Ncs.tt|||_t|||_t|_dSrJ)r;r<rm attentionroutputset pruned_headsryrGr%r&r<s    zMobileViTAttention.__init__headscCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrrr)lenrrrorprrrtrurrrrqunion)rFrindexr%r%r& prune_heads%s zMobileViTAttention.prune_headsrcCs||}||}|SrJ)rr)rFr self_outputsattention_outputr%r%r&rK7rkzMobileViTAttention.forward) rLrMrNrr#r<rrrPrQrKrRr%r%rGr&rsrcDeZdZdedededdffdd Zdejdejfd d ZZ S) MobileViTIntermediater)rnintermediate_sizer Ncs@tt|||_t|jtrt|j|_ dS|j|_ dSrJ) r;r<rrrrrBrErCr intermediate_act_fnrFr)rnrrGr%r&r<>s   zMobileViTIntermediate.__init__rcCrrJ)rrrr%r%r&rKFrkzMobileViTIntermediate.forwardrlr%r%rGr&r=rcsJeZdZdedededdffdd Zdejd ejdejfd d ZZ S) MobileViTOutputr)rnrr Ncs*tt|||_t|j|_dSrJrrrGr%r&r<MrzMobileViTOutput.__init__r input_tensorcCs ||}||}||}|SrJr)rFrrr%r%r&rKRs  zMobileViTOutput.forwardrlr%r%rGr&rLs$rcr) MobileViTTransformerLayerr)rnrr NcsZtt|||_t||||_t||||_tj ||j d|_ tj ||j d|_ dS)Nr7) r;r<rrr intermediaterrr LayerNormlayer_norm_epslayernorm_beforelayernorm_afterrrGr%r&r<Zs  z"MobileViTTransformerLayer.__init__rcCs<|||}||}||}||}|||}|SrJ)rrrrr)rFrr layer_outputr%r%r&rKbs    z!MobileViTTransformerLayer.forwardrlr%r%rGr&rYrrcr) MobileViTTransformerr)rnrbr NcsJtt|_t|D]}t||t||jd}|j |qdS)N)rnr) r;r<rrcrdrerr# mlp_ratiorf)rFr)rnrb_transformer_layerrGr%r&r<ms    zMobileViTTransformer.__init__rcCrhrJri)rFrrjr%r%r&rKyrkzMobileViTTransformer.forwardrlr%r%rGr&rls rcseZdZdZ ddedededededed ed d ffd d Zdejd e eje ffddZ dejde d ejfddZ dejd ejfddZ ZS)MobileViTLayerz; MobileViT block: https://arxiv.org/abs/2110.02178 rr)r*r+r-rnrbr0r Ncst|j|_|j|_|dkr,t||||dkr|nd|dkr$|dndd|_|}nd|_t||||jd|_ t|||dddd|_ t |||d|_ t j||jd|_t|||dd|_t|d|||jd|_dS) Nr!r)r*r+r-r0rUF)r*r+r,r1r2)rnrbr)r;r< patch_size patch_width patch_heightrSdownsampling_layerr(conv_kernel_sizeconv_kxkconv_1x1r transformerrrr layernormconv_projectionfusion)rFr)r*r+r-rnrbr0rGr%r&r<sN   zMobileViTLayer.__init__rIcCs|j|j}}t||}|j\}}}}tt|||} tt|||} d} | |ks4| |krBtjj|| | fddd}d} | |} | |} | | }| ||| || |}| dd}| ||||}| dd}| |||d}||f||| || | d }||fS) NFbilinearr|mode align_cornersTrr!r r{) orig_size batch_sizechannels interpolate num_patchesnum_patches_widthnum_patches_height) rrr#shaperceilrrrreshaper)rFrIrr patch_arearr orig_height orig_width new_height new_widthrnum_patch_widthnum_patch_heightrpatches info_dictr%r%r& unfoldings<     zMobileViTLayer.unfoldingrrc Cs|j|j}}t||}|d}|d}|d}|d} |d} ||||d} | dd} | ||| | ||} | dd } | ||| || |} |d r_tjj | |d d d d} | S)Nrrrrrr{rr r!rrrFr) rrr#rr}rrrrr) rFrrrrrrrrrrrIr%r%r&foldings*    zMobileViTLayer.foldingcCs||jr||}|}||}||}||\}}||}||}|||}||}|t j ||fdd}|SNrr) rrrrrrrrrrPcat)rFrIr^rrr%r%r&rKs       zMobileViTLayer.forwardr_)rLrMrNr`rr#r<rPrQrrrrrKrRr%r%rGr&rs.  :+rc sPeZdZdeddffdd Z  d dejd ed edee e ffd d Z Z S)MobileViTEncoderr)r Nc sXt||_t|_d|_d}}|jdkrd}d}n|jdkr%d}d}t||j d|j dddd}|j |t||j d|j ddd d}|j |t ||j d|j d d|j ddd }|j ||rp|d9}t ||j d |j d d|j dd |d }|j ||r|d9}t ||j d |j d d|j dd |d } |j | dS)NFrTrr)r*r+r-rbr!r )r*r+r-rnrb)r*r+r-rnrbr0) r;r<r)rrcrdgradient_checkpointing output_strideraneck_hidden_sizesrfr hidden_sizes) rFr)dilate_layer_4dilate_layer_5r0layer_1layer_2layer_3layer_4layer_5rGr%r&r<!sx         zMobileViTEncoder.__init__FTroutput_hidden_states return_dictcCsx|rdnd}t|jD]\}}|jr|jr||j|}n||}|r(||f}q |s6tdd||fDSt||dS)Nr%css|] }|dur|VqdSrJr%).0vr%r%r& sz+MobileViTEncoder.forward..)last_hidden_stater) enumeraterdrtraining_gradient_checkpointing_func__call__tupler )rFrrrall_hidden_statesrgrjr%r%r&rKks    zMobileViTEncoder.forward)FT) rLrMrNrr<rPrQrOrrr rKrRr%r%rGr&r sM rc@sBeZdZdZeZdZdZdZde e j e j e j fddfdd ZdS) MobileViTPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. mobilevit pixel_valuesTmoduler NcCstt|tjtjfr#|jjjd|jjd|j dur!|j j dSdSt|tj r8|j j |jj ddSdS)zInitialize the weightsg)meanstdNg?) rBrrrr>weightdatanormal_r)initializer_ranger/zero_rfill_)rFrr%r%r& _init_weightss   z&MobileViTPreTrainedModel._init_weights)rLrMrNr`r config_classbase_model_prefixmain_input_namesupports_gradient_checkpointingrrrrr>rr r%r%r%r&rs&raK This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`MobileViTConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. aF Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`MobileViTImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zWThe bare MobileViT model outputting raw hidden-states without any specific head on top.c seZdZddedeffdd ZddZeee e e e de d  dd eejd eed eedeee ffddZZS)MobileViTModelTr) expand_outputcsnt|||_||_t||j|jdddd|_t||_ |jr1t||jd|jddd|_ | dS) Nrr r!)r*r+r,r-rrrU) r;r<r)rr( num_channelsr conv_stemrencoder conv_1x1_exp post_init)rFr)rrGr%r&r<s&   zMobileViTModel.__init__cCsF|D]\}}|jj|}t|tr |jjD]}|j|qqdS)zPrunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrdrBrrrr)rFheads_to_prune layer_indexrmobilevit_layerrr%r%r& _prune_headss   zMobileViTModel._prune_headsvision) checkpoint output_typer modalityexpected_outputNrrrr c Cs|dur|n|jj}|dur|n|jj}|durtd||}|j|||d}|jr>||d}tj |ddgdd}n|d}d}|sY|durN||fn|f}||ddSt |||j d S) Nz You have to specify pixel_valuesrrrrr{F)rkeepdimr)r pooler_outputr) r)ruse_return_dictr=rrrrrPrrr) rFrrrembedding_outputencoder_outputsr pooled_outputrr%r%r&rKs0 zMobileViTModel.forward)T)NNN)rLrMrNrrOr<rrMOBILEVIT_INPUTS_DOCSTRINGr_CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPErrPrQrrrKrRr%r%rGr&rs.   rz MobileViT model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. cseZdZdeddffdd Zeeeee e e d    d de e jde ed e e jd e edeee ff d d ZZS)MobileViTForImageClassificationr)r Ncsdt||j|_t||_tj|jdd|_|jdkr't |j d|jnt |_ | dS)NT)inplacerr{)r;r< num_labelsrrrrvclassifier_dropout_probrxrrrIdentity classifierrrFr)rGr%r&r<s  $ z(MobileViTForImageClassification.__init__)rrr r rrlabelsrc Csh|dur|n|jj}|j|||d}|r|jn|d}|||}d}|dur|jjdurS|jdkr9d|j_n|jdkrO|jt j ksJ|jt j krOd|j_nd|j_|jjdkrqt } |jdkrk| | | }n+| ||}n%|jjdkrt} | |d|j|d}n|jjdkrt} | ||}|s|f|dd} |dur|f| S| St|||jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss). If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nr!r regressionsingle_label_classificationmulti_label_classificationr{r!)losslogitsr)r)r$rr#r1rx problem_typer.dtyperPlongr#r squeezer r}rrr) rFrrr3routputsr'r8r7loss_fctrr%r%r&rK's>   "       z'MobileViTForImageClassification.forwardNNNN)rLrMrNrr<rr(r_IMAGE_CLASS_CHECKPOINTrr*_IMAGE_CLASS_EXPECTED_OUTPUTrrPrQrOrrrKrRr%r%rGr&r,s0 r,cr) MobileViTASPPPoolingr)r*r+r Nc s4ttjdd|_t|||ddddd|_dS)Nr) output_sizeTrelu)r*r+r,r-r1r2)r;r<rAdaptiveAvgPool2d global_poolr(r)rFr)r*r+rGr%r&r<fs  zMobileViTASPPPooling.__init__rIcCs:|jdd}||}||}tjj||ddd}|S)NrrFr)rrFrrrr)rFrI spatial_sizer%r%r&rKus   zMobileViTASPPPooling.forwardrlr%r%rGr&rBesrBc@eZdZdZdeddffdd Zdejdejfdd ZZ S) MobileViTASPPzs ASPP module defined in DeepLab papers: https://arxiv.org/abs/1606.00915, https://arxiv.org/abs/1706.05587 r)r Ncstjdjtjdkrtdt|_ t ddd}|j ||j fddjDt }|j |t d ddd|_tjjd |_dS) Nrr z"Expected 3 values for atrous_ratesrrDrVc s g|] }td|ddqS)r rD)r*r+r,r0r2)r()rrater)r*r+r%r& s z*MobileViTASPP.__init__..r)p)r;r<raspp_out_channelsr atrous_ratesr=rrcconvsr(rfextendrBprojectrvaspp_dropout_probrx)rFr) in_projection pool_layerrGrKr&r<s2       zMobileViTASPP.__init__rIcCsDg}|jD] }|||qtj|dd}||}||}|Sr)rPrfrPrrRrx)rFrIpyramidconvpooled_featuresr%r%r&rKs   zMobileViTASPP.forward rLrMrNr`rr<rPrQrKrRr%r%rGr&rI}s+rIcrH) MobileViTDeepLabV3zB DeepLabv3 architecture: https://arxiv.org/abs/1706.05587 r)r Nc sBtt||_t|j|_t||j |j ddddd|_ dS)NrFT)r*r+r,r1r2r/) r;r<rIasppr Dropout2dr/rxr(rNr.r1r2rGr%r&r<s   zMobileViTDeepLabV3.__init__rcCs&||d}||}||}|S)Nr{)r[rxr1)rFrrIr%r%r&rKs  zMobileViTDeepLabV3.forwardrYr%r%rGr&rZsrZzX MobileViT model with a semantic segmentation head on top, e.g. for Pascal VOC. cs~eZdZdeddffdd Zeeeee d    d de e j de e j d e e d e e deeeff d d ZZS) MobileViTForSemanticSegmentationr)r Ncs8t||j|_t|dd|_t||_|dS)NF)r)r;r<r.rrrZsegmentation_headrr2rGr%r&r<s   z)MobileViTForSemanticSegmentation.__init__)rr rr3rrc Cs|dur|n|jj}|dur|n|jj}|j|d|d}|r!|jn|d}||}d}|durU|jjdkr:tdtj j ||j ddddd } t |jj d } | | |}|sw|rc|f|dd} n |f|d d} |duru|f| S| St|||r|jdd Sddd S) a labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). Returns: Examples: ```python >>> import requests >>> import torch >>> from PIL import Image >>> from transformers import AutoImageProcessor, MobileViTForSemanticSegmentation >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> image_processor = AutoImageProcessor.from_pretrained("apple/deeplabv3-mobilevit-small") >>> model = MobileViTForSemanticSegmentation.from_pretrained("apple/deeplabv3-mobilevit-small") >>> inputs = image_processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs) >>> # logits are of shape (batch_size, num_labels, height, width) >>> logits = outputs.logits ```NTr!rz/The number of labels should be greater than onerrFr) ignore_indexr!)r7r8r attentions)r)rr$rrr^r.r=rrrrr semantic_loss_ignore_indexr) rFrr3rrr=encoder_hidden_statesr8r7upsampled_logitsr>rr%r%r&rKsB'   z(MobileViTForSemanticSegmentation.forwardr?)rLrMrNrr<rr(rrr*rrPrQrOrrrKrRr%r%rGr&r]s&  r])rN)Dr`rtypingrrrrrrPtorch.utils.checkpointrtorch.nnrr r activationsr modeling_outputsr rrrmodeling_utilsr pytorch_utilsrrutilsrrrrrconfiguration_mobilevitr get_loggerrLloggerr*r)r+r@rA'MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LISTr#r'Moduler(rSrarmrrrrrrrrrMOBILEVIT_START_DOCSTRINGr(rr,rBrIrZr]r%r%r%r&sn      @03  "e  WN;