o h@ @sddlmZddlmZmZddlmZddlmZm Z m Z m Z m Z m Z mZmZddlZddlmZddlmmZddlmZddlZddlmZddlmmZddlZddl Z ddl!Z!ddl"m#Z#de$d e e$e$ffd d Z%d e ed ej&de e$e ffddZ'd ej&de e$e fdejj(fddZ)d6dejj(d ejj(fddZ*dej(d ej(fddZ+dej(deej&deej&deej&fddZ,ej-ej.ej/ej0ej1ej2ej3ej4ej5ej6ej4ej7ej8g Z9ej:ej;gZej/d d!iZ?deej&de e$ej(ffd"d#Z@deej&de e$ej(fd$e ej(ej(ffd%d&ZAGd'd(d(ZBd7d+d,ZCd-eBd eDfd.d/ZEGd0d1d1ZFdejGfdejj(d2e e e$e fd3eejGd ejj(fd4d5ZHdS)8N)ArgumentTarget)fuse_conv_bn_eval)TypeDictAnyTupleIterableOptionalListcast) ShapeProp) defaultdict)EnumtargetreturncCs*|dd^}}|r|d|fSd|fS)zp Splits a qualname into parent path and last atom. For example, `foo.bar.baz` -> (`foo.bar`, `baz`) .r)rsplit)rparentnamerX/var/www/html/ai/venv/lib/python3.10/site-packages/torch/fx/experimental/optimization.py _parent_namesrpatternnodemodulescCst|jdkr dS|jd|f}t||D]2\}}t|tjs"dS|jdkr*dSt|jts3dS|j|vr;dSt ||j|urGdSqdS)NrF call_moduleT) lenargszip isinstancefxNodeoprstrtype)rrrnodes expected_type current_noderrrmatches_module_patterns     r+ new_modulecCs<t|jtsJt|j\}}|||j<t||||dSN)r"rr&rsetattr)rrr, parent_namerrrrreplace_node_module,s r0Fmodelc Cstjtjftjtjftjtjfg}|st|}t |}t | }t|j }|D]E}|jD]?}t|||rot|jdjdkrCq0||jdj}||j} | jsTq0t|| } t|jd|| ||jd||q0q+t ||S)z Fuses convolution/BN layers for inference purposes. Will deepcopy your model by default, but can modify the model inplace as well. rr)nnConv1d BatchNorm1dConv2d BatchNorm2dConv3d BatchNorm3dcopydeepcopyr#symbolic_tracedict named_modulesgraphr(r+rr usersrtrack_running_statsrr0replace_all_uses_with erase_node GraphModule) r1inplacepatternsfx_modelr new_graphrrconvbn fused_convrrrfuse2s2             rKcCs*t|}Gdddtjj}||S)z5 Removes all dropout layers from the module. cs>eZdZdedeedfdeeefdeffdd Z Z S)z&remove_dropout..DropoutRemoverrr .kwargsrcs:t|j|tjrt|dksJ|dSt|||S)Nrr)r" submodulesr2Dropoutrsuperr)selfrr rL __class__rrrVsz2remove_dropout..DropoutRemover.call_module) __name__ __module__ __qualname__rrrrr&rr __classcell__rrrQrDropoutRemoverUs6rW)r#r;torch Transformer transform)r1rFrWrrrremove_dropoutOs  r[ orig_moduler(inputsoutputscs|t}i|D] }||j}||<q|D]}||fdd}||<q|fdd|D|t||S)z Given lists of nodes from an existing graph that represent a subgraph, returns a submodule that executes that subgraph. cs|Sr-r)xenvrrhz"extract_subgraph..csg|]}|qSrr).0outputr`rr jz$extract_subgraph..)r#Graph placeholderr node_copyrelintrC)r\r(r]r^rGinputnew_noderrr`rextract_subgraph^s    rncCs t|Sr-) th_mkldnnMkldnnBatchNorm)a_rrrrb{s rbcCsi}|D]9}|jdkr=t|jtsJ||j}t|tvr=tt||tj}t|tj s0Jt |||<t |||q|S)z For each node, if it's a module that can be preconverted into MKLDNN, then we do so and create a mapping to allow us to convert from the MKLDNN version of the module to the original. r) r%r"rr&r' mkldnn_maprXfloatr2Moduler9r:r0)r(r old_modulesr cur_moduler,rrrmodules_to_mkldnns    rxrvcCsJ|D] }|jdkr"t|jtsJ||j}||vr"t||||qdS)za Maps each module that's been changed with `modules_to_mkldnn` back to its original. rN)r%r"rr&r0)r(rrvrrwrrr reset_moduless  ryc@seZdZdejfddZdS) MklSubgraphfx_graphcCs||_g|_g|_g|_dSr-)r{r( start_nodes end_nodes)rPr{rrr__init__s zMklSubgraph.__init__N)rSrTrUr#rhr~rrrrrzsrz rcs*dddtdtffdd }|S)aW This generates a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by running it with the example_inputs. Example usage: heuristic = gen_mkl_autotuner(example_inputs, iters=10) fast_model = optimization.optimize_for_inference(model, heuristic) Nr>rcs|j}dur|jj|jjtdd|Dtttj dd|j D}t |j ||fdd}|fdd}t jj t|fdd}||kS) NcSsg|]}t|jqSr)rXrandnshaperdrrrrrfsz@gen_mkl_autotuner..use_mkl_heuristic..cSsg|]}|jdqS)r)r rrrrrfscs<tD]}|qt}tD]}|}qt|Sr-)rangetime)frrbeginout)iterswarmuprr benchmarks   z?gen_mkl_autotuner..use_mkl_heuristic..benchmarkcsddddDDS)NcSg|]}|qSr)to_denserdirrrrfrgzRgen_mkl_autotuner..use_mkl_heuristic....cSrr) to_mkldnnrrrrrfrgrr sample_inputs submodulerrrbsz>gen_mkl_autotuner..use_mkl_heuristic..csSr-rrrrrrbrc)r|r{ owning_modulervr propagater r r#r$r}rnr(ryr>r<r=)r> input_nodes output_argsrmkl_time no_mkl_timeexample_inputsrFrrvrrruse_mkl_heuristicsz,gen_mkl_autotuner..use_mkl_heuristic)rzbool)rrrrrrrgen_mkl_autotuners rr>cCst|jdkS)z This is a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by checking if there are more than 2 nodes in it )rr()r>rrruse_mkl_lengthsrc@sFeZdZddZdefddZdedefddZd ed efd d Zd S) UnionFindcCsdg||_dg||_dS)Nrrsize)rPnrrrr~s zUnionFind.__init__vcCs||j|<d|j|<dS)Nrr)rPrrrrmake_sets zUnionFind.make_setrcCsB|j|}||kr |S|dusJ|||j|<tt|j|Sr-)rfindr int)rPrparrrrrs  zUnionFind.findrqbcCsf||||}}||kr|S|j||j|kr ||}}||j|<|j||j|7<dSr-)rrr)rPrqrrrrjoins  zUnionFind.joinN)rSrTrUr~rrrrrrrrrs r pass_configtracerc sDdddtid}|duri}|||drt|}|dr#t|}|ddur+|St|dts6td d|dvr@td |dd}|}|t |t |j }t| }Gd d d t}tjD]} |j} | jd kr|| j} t| tvr|j} t| d} | dur| jtjksJd| jtdksJdn| jdkr| jtvr|j} n| jtvr|j} | |jkr"| |jkrtdd| j Dsqk!| t "| j fdd} Wdn1swYt#t$t j%j&| | _ '| (dd| f}| )|| f|_ Wdn 1swYqkt*tj|}|_+jD]B} | jdkrr| jdkrr| j d}t| j,}|D]}|jdkrc|jdkrc|)|-|qKt.| j,dkrr-| q1t.j}t/|fddt0jD]w\}} | jdkr| jdkr|| _12|q| jdkr| jdkrĈ| j ddusJ| j d| _3qfdd| j4D}t.|dkrאqtdd|DrJt5|}|d| _6|ddD] }7|d|qqt8fd d}jD]9} t9| d!r|:| j6j;| t9| d"r1|:| j1j<;| t9| d#rC|:| j3j=;| q |>D](}||sp|j<|j=D]} | j d}| )|-| qVt?|j||qId}jD]} | jdks| jdkr|d7}qwt@AtBCd$|Dt |}|S)%a Performs a set of optimization passes to optimize a model for the purposes of inference. Specifically, the passes that are run are: 1. Conv/BN fusion 2. Dropout removal 3. MKL layout optimizations The third optimization takes a function `use_mkl_heuristic` that's used to determine whether a subgraph should be explicitly run in MKL layout. Note: As FX does not currently handle aliasing, this pass currently assumes nothing aliases. If that isn't true, use at your own risk. T heuristic) conv_bn_fuser[mkldnn_layout_optimizeNrr[rFz+mkldnn_layout_optimize config is not a dictz4Heuristic not found in mkldnn_layout_optimize configc@seZdZdZdZdZdS)z*optimize_for_inference..MklSupportrrN)rSrTrUNOYESUNKNOWNrrrr MklSupportsrrz)this pass is only for torch.float modulescpuz!this pass is only for CPU modules call_functioncss|]}|jdkVqdS)rN)r)rdargrrr 3sz)optimize_for_inference..csd|fS)Nr) call_methodrr{rrrb6sz(optimize_for_inference..rrrrcs0t|dr |jSt|dr|jSdS)Ncolor start_color)hasattrrrrr)ufrr get_colorSs    z)optimize_for_inference..get_colorcs,g|]}t|tjr|dur|qSr-)r"r#r$r)rrrrfns,z*optimize_for_inference..css|]}|duVqdSr-rrrrrrrsrcstSr-)rzrrrrrbyrcrr end_colorzmkldnn conversions: )ErupdaterKr[r"r< RuntimeErrortracer9r:r#rCrootr=rlistr(rr%rr'mkldnn_supportedrnext parametersdtyperXrtdevicemkldnn_supported_unknownranyr inserting_beforemap_argr rrrinserting_after create_noderArxrvr?rBrr enumeraterrrall_input_nodessortedrrrrrappendr|r}valuesrylogging getLoggerrSinfork)r1rrdefault_pass_configr cur_tracerrFrrrsupports_mkldnnrwsample_parameter mkldnn_argsdense_xrvprv_noder?user num_nodescur_idx cur_colors other_color mkldnn_graphsr>prvmkldnn_conversionsresultr)r{rrroptimize_for_inferences                                     r)F)rr)Itorch.fxr# torch.fx.noderrtorch.nn.utils.fusionrtypingrrrrr r r r rXtorch.nnr2torch.nn.functional functionalFtorch.fx.passes.shape_propr r9 collectionsrtorch.utils.mkldnnutilsmkldnnrooperatorrrenumrr&rr$r+rur0rKr[rnr5Linearr6ReLU MaxPool2d AvgPool2dAdaptiveAvgPool2drelu transposesigmoid avg_pool2dadaptive_avg_pool2draddmulr MkldnnConv2d MkldnnLinearrsrxryrzrrrrTracerrrrrrs^  (    $ $.    "2 &