o h6@s>ddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z ddl Z ddl Z ddl mZddlmZddlmZddlmZddlmZdd lmZmZmZdd lmZdd lmZmZm Z ddl!m"m#Z$e%e&Z'd d Z(eddZ)edej*de fddZ+d1ddZ,ddZ-edej*de fddZ.Gdddej/Z0edej*de fddZ1edd Z2d!d"Z3e j4j5Z5e5j6e5j7e5j8e5j9e5j:e5j;e5je5j?e5j@e5jAe5jBe5jCjDe5jCjEe5jFe5jGe5jHe5jIe5jJe5jKe5jLhZMeeMZMed#d$ZNd%e e ejOffd&d'ZPd(d)ZQdaRd*d+ZSd,d-ZTd2d/d0ZUdS)3N)contextmanager)partial)CallableUnion)SymInt)get_decompositions) bind_symbols) aot_function aot_modulemake_boxed_compiler)strip_overloads)default_partition draw_graph#min_cut_rematerialization_partitioncCs6|jjD]}|jtjjjkrtjjj|_q||SN) graphnodestargettorchopsaten_to_copyto recompile)fx_gnoderP/var/www/html/ai/venv/lib/python3.10/site-packages/torch/_functorch/compilers.py _canonicalize!s  rc cs6tjd}z dVWtj|dStj|w)NF)r_C_jit_set_autocast_mode)old_jit_autocast_flagrrr_disable_jit_autocast)s  r#rreturncCs<tt||jjD]#}|jtjjjkr/t |j dkr/t |j dkr/d|j vr/tjjj |_q |jjD]}i}|j D]\}}t|tjrJ|j}|||<q=||_ q4|j|tj|}tj|jtj|}tj|}tdd|Ds||Wd|SWd|S1swY|S)a Compiles the :attr:`fx_g` with Torchscript compiler. .. warning:: This API is experimental and likely to change. Args: fx_g(fx.GraphModule): The input Fx graph module to be compiled. Returns: Torch scripted model. r dtypecss|] }t|tjjVqdSr) isinstancer _subclasses FakeTensor).0trrr _szts_compile..N)r#r rrrrrrrlenargskwargsritemsr&devicetypelintrjitscriptr _jit_pass_remove_mutationfreezeevaloptimize_for_inferenceany)rinpsr new_kwargskvfrrr ts_compile2s>               r?TcCst|jt|||d|S)N) clear_meta)printcoder)r_namer@rrr_draw_graph_compileds rEcCsttt|dS)NrD)r rrErFrrrdraw_graph_compilejs rGcCs|S)z Returns the :attr:`fx_g` Fx graph module as it is. This is a no-op compiler and can be used to check accuracy. .. warning:: This API is experimental and likely to change. rrrCrrrnopps rIcs(eZdZfddZfddZZS)DebugInterpretercs$t|jg|R|_tj|dSr)rmodulesymbol_mappingsuperrun)selfr- __class__rrrN}szDebugInterpreter.runc sfddfddfddfdd}t|}d |jvrlt|jd \}}t|\}}t|t|ksJJt|d t|ttt|||D]\}} t| t j s`qT||| fd d qT|S) NcsBt|ts|St|jjj}t|j dksJ|t |S)Nr) r&rsympyexpandrexprxreplacerLr, free_symbolsint)nir)rOrr subst_symints z/DebugInterpreter.run_node..subst_symintcstfdd|DS)Nc3s|]}|VqdSrr)r)rXrZrrr+szHDebugInterpreter.run_node..subst_symint_tuple..)tuple)nisr[rrsubst_symint_tuplesz5DebugInterpreter.run_node..subst_symint_tuplecsT|dkr(t|jD]}||||kr'||dkr'dSq dS)Nrr FT)numelrangendimstridesize)abidxr[rrcheck_significant_stridess *z.check_significant_stridesc st|sJ|j|jksJ|d|jd|j||ks=J|d|d|d|||}|s\J|d|d|d|dS)Nz:  != z aka )callabler%rcrb)nvrvdesc same_strides)rgr^rrchecks ** 8z(DebugInterpreter.run_node..checkvalrhcsddjS)Nzoutput z where )rLr)irOrrsz+DebugInterpreter.run_node..) rMrun_nodemetapytree tree_flattenr,zipr`r&rTensor) rOnrnrYn_valsn_specr_valsr_specrjrkrP)rgrprOrZr^rrrs     * zDebugInterpreter.run_node)__name__ __module__ __qualname__rNrr __classcell__rrrPrrJ|s rJcCs t|jS)z Returns a (slow) interpreter over the FX graph module that also checks various debugging properties (e.g., that tracing strides matched real strides.) )rJrNrHrrr debug_nops rcCs(t|tj|}tj|}|Sr)r rr3r4r6r7)rrCr>rrrsimple_ts_compiles rcCs t|tSr)r r)r>rrrnnc_jits rcCst|j|Sr)rArBrHrrr print_compiles rfncKsFttttd}||t|tjjrt|fi|St |fi|S)a Wrapper function over :func:`aot_function` and :func:`aot_module` to perform memory efficient fusion. It uses the :func:`min_cut_rematerialization_partition` partitioner to perform efficient recomputation. It uses NVFuser to compile the generated forward and backward graphs. .. warning:: This API is experimental and likely to change. Args: fn (Union[Callable, nn.Module]): A Python function or a ``nn.Module`` that takes one ore more arguments. Must return one or more Tensors. **kwargs: Any other overrides you want to make to the settings Returns: Returns a ``Callable`` or ``nn.Module`` that retains the eager behavior of the original :attr:`fn`, but whose forward and backward graphs have gone through recomputation optimizations, and the graphs have been compiled with nvfuser.  fw_compiler bw_compiler partition_fndecompositions) r?rdefault_decompositionsupdater&rnnModuler r )rr.configrrrmemory_efficient_fusions rcCsH|dtddd|Ddddlm}||t||S)NfooaQ ############################################################## # To minimize FX graph, copy and paste the below and run it # ############################################################## import torch import torch.fx as fx from functorch.compile import minifier, check_nvfuser_subprocess, check_nvfuser_correctness_subprocess inps = cSsg|]}|j|jfqSr)shaper%)r)rprrr sz!debug_compile..a? inps = [torch.ones(shape, dtype=dtype, device='cuda') for (shape, dtype) in inps] from foo import FxModule mod = FxModule().cuda() with torch.jit.fuser("fuser2"): # check_nvfuser_subprocess can be replaced with check_nvfuser_correctness_subprocess minifier(fx.symbolic_trace(mod), inps, check_nvfuser_subprocess) r)FxModule) to_folderrArrcudar?)rr:rrrr debug_compile s    rc Csg}t|dU}t|}g}|D]B}t|dkr"|}|t}n,|\}}}} } | tjtjtj tj tjtj tt hvrFtj dd|| | d}ntj|| | d}||qWd|S1s_wY|S)zZ Return a random input for the given inputs meta generated from _save_fx_default. rbr r)r%r0N)openpickleloadr,randomrandrrWint32int64booluint8floatrandintappend) input_data_pathinputsr> inputs_metarsr1inputrrbr%r0rrr get_inputs-s6      rc sbddlm}fddfddfdd}fd d }fd d }||||||td S)aO The forward, backward, and joint computation graph will be stored in {folder_name}/{current_name}/{current_name}_forward_{graph_index}, {folder_name}/{current_name}/{current_name}_backward_{graph_index}, and {folder_name}/{current_name}/{current_name}_joint_{graph_index} respectively. The input shape of the graphs will be stored in the .input files. These files can be loaded with pickle, and is a list of format (type, shape, stride, dtype, device). In the case of type = int or float, it is just (type,). For joint graph input, it is a nested list [[],[]] where the two inner lists have the same format. If dump_example_input is True, example_inputs will be stored in .pt file. Since each function might produce multiple graphs, the graph_index is used to distinguish difference graphs r)aot_module_simplifiedcsg}t|dkr!t|dtr!||d7}||d7}|S|D](}t|tks1t|tkr:|t|fq#|t||j||j |j fq#|S)Nrr ) r,r&r\r1rWrrrrbr%r0)r- input_metaarg)get_input_metarrr^sz(_save_fx_default..get_input_metacs4t|jjdkrttjd|tdSt|}|j t j j | |}tjd}|sBtd|ddd|dt t|tddd|dtdd|dtddrt |ddd|dtdd|dtddSdS)Nrz!No nodes in graph {%s}_{%s}_{%s}./rCz.inputwbz.pt)r,rrlogloggingWARNING graph_indexcopydeepcopy set_codegenrfxCodeGenrospathexistsmakedirsrrdumprsave) gm_to_saver- type_namegmrisExist) current_namedump_example_input folder_namerrrgraph_saver_helperms@ 22z,_save_fx_default..graph_saver_helpercs||d|S)Nforwardr)rfw_argsrrrgraph_saver_forwards z-_save_fx_default..graph_saver_forwardcs||dtd7a|S)Nbackwardr )r)rbw_argsrrrgraph_saver_backwards z._save_fx_default..graph_saver_backwardcs||dt||S)Njoint)r)r joint_argsrrrgraph_saver_joints  z+_save_fx_default..graph_saver_jointr)functorch.compilerr) rrrrexample_inputsrrrrr)rrrrrr_save_fx_defaultLs   %  rFcCsdatt|||S)as Dump the forward, backward, and joint computation graph. Example Usage: save_fx_func = graph_dumper_aot(current_name, folder_name, dump_example_input = False) optimize_ctx = torchdynamo.optimize( save_fx_func ) with torch.enable_grad(): with optimize_ctx: result = forward_and_backward_pass(model, example_inputs) r)rrr)rrrrrrgraph_dumper_aots r)T)F)Vrrrrr contextlibr functoolsrtypingrrrRrrtorch.fxrtorch.nnr torch._decompr%torch.fx.experimental.symbolic_shapesr aot_autogradr r r compile_utilsr partitionersrrrtorch.utils._pytreeutils_pytreert getLoggerr}rrr# GraphModuler?rErGrI InterpreterrJrrrrrdetach gelu_backwardleaky_relu_backwardsigmoid_backwardthreshold_backwardhardtanh_backwardhardsigmoid_backwardhardswish_backward tanh_backward silu_backward elu_backwardcudnn_batch_normcudnn_batch_norm_backward masked_fillScalarrwelu leaky_reluhardtanh hardswish hardsigmoid conj_physical is_same_sizerrrrrrrrrrrrrs           1 2    '_