o hM@s$ddlmZmZddlmZmZmZmZmZm Z ddl Z ddl m Z ddl Z ddlZddlmmZddlZddlZddlZddlZddlmZddlmZddlmZddlmZmZdd l m!Z!ddl"Z"e!j#Z$d d Z%d d Z&ddZ'ddZ(GdddZ)e)Z*ddZ+ddZ,ddZ-ddZ.ddZ/de j0fdd Z1de j0fd!d"Z2de j0d#ee j0e j0ffd$d%Z3d&d'Z4d(d)Z5d*e j6d#e7fd+d,Z8d-d.Z9e":dd/d0Z;d1d2Zd7d8Z?d9d:Z@ dHde j0d#ee j0e j0ffdS&z6_extract_graph_with_inputs_outputs..c|SNrr8r9rrWz4_extract_graph_with_inputs_outputs..get_attrcr=r>rr?r9rrr@YrAoutputzNode z couldn't be found in envz was invalid, but is output)r5Graphr3namerrop InvalidNodepytree tree_flattenargskwargsany node_copyr(r6 RuntimeErrorr-appendrCeliminate_dead_codelint) joint_graphinputsoutputs new_graphrnew_nodeall_args output_valuesr8rr9r"_extract_graph_with_inputs_outputs7sF            rYcCs(|jdkod|jvot| ot| SNr3tangents)rFr!_is_bwd_seed_offset_is_fwd_seed_offsetrrrr _is_primalls r^cCs|jdko d|jvSrZrFr!rrrr _is_tangenttsr`cC|jdkod|jvpd|jvS)Nr3bwd_seedbwd_base_offsetr_rrrrr\wr\cCra)Nr3fwd_seedfwd_base_offsetr_rrrrr]zrdr] joint_modulecCs<tdd|jjDd}|d|}||d}||fS)NcSsg|] }|jdkr|jqSrC)rFrJr7rrrrr;z,_extract_fwd_bwd_outputs..r)rHrIrr)rgnum_fwd_outputsrT fwd_outputs bwd_outputsrrr_extract_fwd_bwd_outputs~s  rncCst||d\}}ttt|jj}ttt|jj}ttt|jj}ttt|jj} t |j|||||} t |j|||| |} | jD].} | j dkrw| j sw|D]} | j | j krd| | nqU|D]}|j | j krv| |nqgqIt}g}g}|D]} t| }|r|||| q|| qt|j}t|||D],} d| jvrqt| jd|}t|dddD]}||vrq|||q||O}q||||t |j|||||} t |j|||| |} t|| }t|| }||fS)Nrkr3r&cSs|jSr>rE)srrrr@sz*_extract_fwd_bwd_modules..key)rnlistfilterr^rrr`r]r\rYrFusersrEremovesetraddrOr itertoolschainrrsortedclearextendr5 GraphModule)rg saved_valuessaved_sym_nodesrkrlrm primal_inputstangent_inputsfwd_seed_offset_inputsbwd_seed_offset_inputs fwd_graph bwd_graphr saved_value saved_sym saved_symbolssaved_sym_nodes_bindingsaved_sym_nodes_derivedsymbolsymbol_bindings new_symbolsrq fwd_module bwd_modulerrr_extract_fwd_bwd_moduless|               rreturncsxt|r t|||dSttt|jj}ttt|jj}||}t||d\}}t |j||}dd|jDg} g} |jjD]]} | j vrHq@t | rR| | q@d| j vru| jdkru| j} tdd| DsjJ| D]} | | qlq@fdd | jD}d| j vrtd d|Dr|D]} | | qq@| | q@td d | D} td d | D} t|| | |dS)a Partitions the :attr:`joint_module` in a manner that closely resembles the behavior observed in the original ``.forward()`` and ``.backward()`` of the callable, i.e., the resulting forward graph contains those operators that are executed in the original ``.forward()`` callable passed to :func:`aot_function`. The default partitioner collects the operators that are between the forward inputs and the forward outputs. This helps in finding the tensors which have to be stashed for the backward pass. These stashed tensors become the output of the generated forward graph. The remaining operators are then placed in the backward graph. .. warning:: This API is experimental and likely to change. Args: joint_module(fx.GraphModule): The joint forward and backward graph. This is the result of AOT Autograd tracing. Returns: Returns the generated forward and backward Fx graph modules. rocSsh|] }|jdkr|jqSrhrFrErirrr rjz$default_partition.. tensor_metar4css|] }|jtjkVqdSr>)r!operatorgetitemr7userrrr z$default_partition..csg|] }|jvr|qSrrpr7nforward_node_namesrrr;z%default_partition..css|]}t|VqdSr>rrrrrrcSi|]}|dqSr>rr7krrr &z%default_partition..cSrr>rrrrrr'rrrk)r#min_cut_rematerialization_partitionrtrur^rrr]rnrYrErrOrrFrvallkeysr)rg _joint_inputsrkrrrSrlrmforward_only_graphrrrrvrbackward_usagesrrrdefault_partitions>        rcCsd}|D]}||9}q|SNrr)r8rqirrr_prod,s rcCs ||jSr>)itemsize)numeldtyperrr_tensor_nbytes2s rrcCsd|jvrB|jd}t|trt|tjrdSdSt|ttfr)tdd|DSt|tjr9t t | |j St dt|d|jvrX|jd}ttt|j}|j }ndSt ||S) Nr&ri?Bcss0|]}t|tjrtt||jVqdSr>)r(r"Tensorrrrrrrrrr>s.z_size_of..zUnknown metadata type rr)rr(rr"r)rttuplesumrrrrrrNtypermap to_size_hintshape)rr&metadatarrrrr_size_of5s"        rcCs\ddlm}|t}|jD]}|jdkr||jjd7<q tt| dddddS) Nrr r4rcS|dSrrr?rrrr@VrAz_count_ops..Trsreverse) collectionsr intrrFr!r0printr|items)rr cntrrrr _count_opsPs   rcCslg}ttjjD]+}ttjj|}t|tjjsq|D]}t||}tj j |j vr2| |nqq|Sr>) dirr"opsatengetattrr(_opsOpOverloadPacket overloadsr# pointwiserrO)r attr_nameopoverloadpacketoverload op_overloadrrr pointwise_opsYs   rcs|vr|S|jdkrd|<|S|jdkr3|jd}|D]}t|tjjjr0t|q!dSfdd|jD}t |dkrFdg}t |d|<|S)Nr3rrCcs&g|]}t|tjjjrt|qSr)r(r"r5rr6 get_depthr7arg depth_maprrr;|r<zget_depth..r) rFrJr(r"r5rr6rall_input_nodeslenmax)rrrJr arg_depthsrrrrjs"     rcs(fdd|D}t|ddddS)Ncs&i|]}t|tjjjr||qSr)r(r"r5rr6rrrrrr<zsort_depths..cSrrrr?rrrr@rAzsort_depths..Tr)r|r)rJrrrrr sort_depthssrc s&ti|jjD]}|jdkr|j}|j|_||<q i}t|jjD]\}}|||<q(idd|jjDd}t |fddt t t |jj}d}t j}|D]} | jD]} || |kro|| }| }qaq\|duswJt |jj||dD]}|qtj|} | S)a This pass finds the first bwd node in the graph (by looking at users of tangents) and then reorders the graph by walking from this node to all the way to the end of the graph. At each op in this traveral, we insert this op in a new graph and try to bring only the relevant subgraph from the other non-bwd edges relevant for this op. This closely mimics the behavior of autograd engine. Why is this pass required in the first place? This is an artifact of how partitioners work today. The starting point of partitioner is a joint graph, which is fwd and then bwd graph. In the case of checkpointing, we keep portions of fwd graph in their original place in the joint graph, while obtaining a bwd graph. As a result, the resulting bwd graph has copies of recomputed fwd subgraphs followed by the original bwd graph. If we run this naively, this leads to bad memory footprint, because the fwd subgraphs are live for way longer duration than necessary. This pass reorders the operations such that we prioritize the ops for the original bwd graph while only realizing those ops from the fwd graph that are necessary at any given point in the graph. r3cSg|] }|jdkr|qSrhrFrirrrr;rz7reordering_to_mimic_autograd_engine..rcsR|vr|St|jD] \}}||<q|fdd|<|S)Ncr=r>rr?r9rrr@rAzSreordering_to_mimic_autograd_engine..insert_node_in_graph..)rrrM)rr_depthsr:insert_node_in_graphrUrrrs zAreordering_to_mimic_autograd_engine..insert_node_in_graphN)r5rDrrrFr3rEr enumeraterrtrur`mathinfrvr"r) gmrrVorderidx output_noderfirst_node_in_bwd minimum_ordertangentrnew_gmrrr#reordering_to_mimic_autograd_engines<          rc Cst}dd}dd}dd}||}||} ||} t} |jjD]*} t| rMt| jdrMtj j | jj vrM|| j } | | j }| | j }||d| | <q#tj jj}tj jj}|jjD]} | jd krld | j vrl| }nq\g}| D]\} }|d }|d }|j}||?|jd ||jg|jR|jd}|jd tj|dfid}|jd tj|dfid}||||||Wdn1swY|j}||dt|}||}||||jd<Wdn1swY||#|jd |||jg|jR|jd}||||Wdn 1s'wYqsdd|jjDd}|jd}t||}|d||||d}|j ||j||!|!||fS)NcSsFi}|jjD]}|jdkr t|jdr tjj|jjvr |||j <q|S)Nr4r) rrrFr r!r"r#r$rrE)gmod random_nodesrrrr get_rng_opss    z*functionalize_rng_ops..get_rng_opscSsTd|jvrdS|jd}t|ts|f}|D]}t|tjr'|jjdkr'dSqdS)zV Check the example value of the node outputs to find the device type. r&Ncudacpu)rr(rr"rdevicer)r candidates candidaterrr get_devices     z)functionalize_rng_ops..get_devicecSs|dkr tjStS)Nr)r"r get_rng_state)rrrrget_sample_rng_states z3functionalize_rng_ops..get_sample_rng_stater)fwdbwdr3rrrr4)rJrKrrrng_state_output_r&cSrrhrrirrrr;Orz)functionalize_rng_ops..)"rzcountdictrrrr r!r"r#r$rrE_prims rng_primsrun_and_save_rng_staterun_with_rng_staterFrinserting_before create_noderJrKrrreplace_all_uses_with erase_noderOnextr3rrrC recompile) rg fw_module bw_module num_sym_nodesuidrrrjoint_graph_rng_opsfw_graph_rng_opsbw_graph_rng_opsrecomputable_rng_ops_mapr base_nodefw_nodebw_noderun_and_save_rngrbw_tangent_start_nodefw_rng_state_outputs node_pairfw_graphfunctional_fw_nodestate rng_outputbw_graph state_namebw_rng_state_nodefw_output_node fw_outputssym_node_start_idxrTrrrfunctionalize_rng_opss                      r&cCsL|jjD]}t|r#|jD]}t|r"|jd|jdkr"d|jd<q q|S)a If there are two consecutive checkpointed blocks with no operator in between, we would still want to stash the tensor at the boundary of checkpointed blocks. The following pass makes the last output node non-recomputable to allow for that. rr)rrrrvr)rgrrrrrcleanup_recompute_tagsZs   r'inductorc, szddl}Wnty}ztd|d}~ww|j||j}tjr.t|}||_|j} t |t |} r?t |}i|jj D]} | | j <qE fdd} | |\} }}t|dkrit|| dSt|jj D]} | vryd| _qotd| _| jD] }t| j|jd| _qqotjjtjj}gjjjjjjjjj j!j"j#j$j%j&j'j(j)j*j+j,j-j.j/j0j1j2j3j4j5j6j7j8j9j:j;j<j=j>j?j@jAjBjCjDjEjFjGjHjIjJjKjLjMjNjOjPjQjRjSjTjUjVjWjXjYjZj[j\j]t^j_j`jajbjc}j`jajdgd kr|g|j|jejfjcjg|jh|jWjhji|jjjkjljmjnjojpjqjrjsjtjujvjwjxjyjzj{j|j}j~jj7}jnjpjlj|jjjoj}g7|jg7}|7}|t7}|jg7}|d d tD7} dur$t nt| jjjg}jjjjjjjjjjjjjg }|| t|Btrsd d |jj D}|dd D}td|tdfdd fdd}fddfdddtffdd }| | j D]} | jdkrq| |vrˆ j| j ddtjdqt| st| r؈ jd | j dtjd|| r| vr jd | j dtjdd!| jvrd"| jvp d!| jvo t| jd!tj }t| rt| }n |rtj}n|| } j| j d| j d#|d| jD]} j| j d#|j dtjdq0qz | d d\}}Wntyitd$td%|jj w|\} t} fd&d'|DD]\}| fd(d'|Dqzt} |D]\}!}"|!dd)|"dd*ksJ|!dd)}#| |#qd+d,t|jj D tfd-d'| D fd.d/d0}$ttd1d/|$}%ttd2d/|$}$t||$|% d3\}&}'r| rt||&|'t|%\}&}'t|'}'trltd4tWd5d |$Ddd6d |&jj D}(d7d |'jj D})|(|)@}*tt}+|&jj D]} | j |*vrIt| jd8rI|+t| jjd7<q.td9t|*d:t|(d:t|)td;t|+d|&|'fS)?ay Partitions the joint graph such that the backward recomputes the forward. Recomputing helps in trading off memory bandwidth with computation. To create the fwd and bwd graph, we copy the joint graph, manually set the outputs to just original forward or backward outputs. And then we run the resulting graphs through dead code elimintation. .. warning:: This API is experimental and likely to change. Args: joint_module(fx.GraphModule): The joint forward and backward graph. This is the result of AOT Autograd tracing. _joint_inputs: The inputs to the joint graph. This is unused. compiler: This option determines the default set of recomputable ops. Currently, there are two options: ``nvfuser`` and ``inductor``. recomputable_ops: This is an optional set of recomputable ops. If this is not None, then this set of ops will be used instead of the default set of ops. num_fwd_outputs: The number of outputs from the forward graph. Returns: Returns the generated forward and backward Fx graph modules. rNzANeed networkx installed to perform smart recomputation heuristicsc st|jjD] }|jdkrd|jvr||vr'|jD]}|qqttt |jj}ttt |jj}||}t |d\}} dd|Dt |j||}fdd|jDfdd|jjD} || fS) Nr3r[rocss|] }|dur|VqdSr>r)r7orrrrrzNmin_cut_rematerialization_partition..classify_nodes..cs h|] }|jdkr|jqSrhrri name_to_noderrrs zNmin_cut_rematerialization_partition..classify_nodes..cs h|] }|vr|vr|qSrrrirequired_bw_nodesrequired_fw_nodesrrrs)rxrrrFr!ryrvrtrur^r]rnupdaterY) rgrrrrrSrlrmrunclaimed_nodes)r+rkr,rclassify_nodess"     z;min_cut_rematerialization_partition..classify_nodesrogeArr(cSg|]}t|qSr)r)r7mrrrr;sz7min_cut_rematerialization_partition..cSs.h|]}|jdkrt|jdrt|jjqS)r4_overloadpacket)rFr r!strr4rirrrrs  z6min_cut_rematerialization_partition..cSsh|]}t|qSr)r5r7rrrrrz#Ops banned from rematerialization: Fcsl|h}t|dkr4|}|jD]}|vr||sdS|vr-t|vr-||qt|dks dS)NrTF)rpoprvrry)r cur_nodescurr) is_fusibler.view_opsrris_materialized_backwardss    zFmin_cut_rematerialization_partition..is_materialized_backwardscsd|jvr |jddkSr|jdkot|vS|jdkr dSt|vr(dS|jtjkr0dS|jjjjjfvr=dS|rCdSsQdkrQ|j t j krQdSt dd|j D}t|}|d |kS) Nrrr4FTr(css$|] }t|tjrt|VqdSr>)r(r5r6rr6rrrr s"zQmin_cut_rematerialization_partition..ban_recomputation..r')rrFrr!rrlift_fresh_copydefault lift_fresh dist_from_bwrmax_dist_from_bwrrJr)rinput_tensors_size output_size)AGGRESSIVE_RECOMPUTATIONrcompilergraph_has_recomputable_opsr=recomputable_opsunrecomputable_opsrrban_recomputations(      z>min_cut_rematerialization_partition..ban_recomputationcst|vo t|vSr>)r)ab) fusible_opsrrr;$sz7min_cut_rematerialization_partition..is_fusiblecs*jdkrdStfddjD S)Nr3Tc3s|]}|VqdSr>rr)r;rrrr+szOmin_cut_rematerialization_partition..is_materialized..)rFrrvr)r;rris_materialized's z.is_materializedrcs:t|}t|dtt|jdd}|r|S|dS)Ng?dr)rrrminrA)rmem_sz)rNrrget_node_weight-s z.get_node_weightrC_insink)capacitysourcer&r_outz-Failed to compute min-cut on following graph: c3s|] }||fVqdSr>rr)nx_graphrrrerz6min_cut_rematerialization_partition..c3s |] }|vr|fVqdSr>r)r7v) non_reachableurrrfscSsi|]\}}||qSrr)r7rrrrrrosz7min_cut_rematerialization_partition..c3s|]}|VqdSr>rrir*rrrprcr=r>rr?)node_idxrrr@prAz5min_cut_rematerialization_partition..rrcSst|Sr>rrrrrr@rrAcSs t| Sr>rrarrrr@ss rz Theoretical Activations Stored: cSr2r)rr6rrrr;r7cSh|] }|jdkr|jqSr4rrirrrrrjcSrbrcrrirrrrrjr4z# remat/fw/bw: /zCount of Ops Rematerialized: cSrrrr?rrrr@rATr)networkx ImportErrorrNrrPr rcserrr%r'rrErrreversedrArrvrQr"rrprimsrysubdivatan2mulrpow remainderfmod__and____or____xor__ __lshift__ __rshift__eqnegegtleltabs bitwise_notceilfloorfracnegreluroundsilutruncloglog10log1plog2lgammaexpexpm1erferfccosacoscoshsinasinsinhtanatantanhatanhsqrtrsqrt reciprocalsigmoidsoftplus thresholdthreshold_backwardclampwherelerpaddcmulgelu gelu_backwardrmean_grad_sum_to_size sum_to_sizeamaxtotype_asrrsqueeze unsqueezersub_to_copyaliasconvert_element_typeclone full_likevarstdbroadcast_in_dimselectpermute _unsafe_viewviewexpandslicereshapebroadcast_tensors scalar_tensorones new_zerosr>arangetriuvar_meanisinfrLfull as_stridedzerosargmaxmaximumtindexr zeros_likerrxnative_dropout rand_like randn_likemm convolutionconvolution_backwardbmmaddmmupsample_bilinear2d_softmax_softmax_backward_datanative_layer_normnative_layer_norm_backwardnative_batch_normnative_batch_norm_backward_native_batch_norm_legitAOT_PARTITIONER_DEBUGrDiGraphrFadd_edgerrr^r]rr(rrr, minimum_cut Exceptionjoin readwriteedgelistgenerate_edgelistr/rr|rtrurr&rr r r!r5r4r),rgrrFrHrknxer cse_graph full_bw_graphgraph_has_recomputable_rng_opsrr1orig_fw_outputsr-r0rridefault_recomputable_ops random_opscompute_intensive_opsjoint_module_ops ops_ignoredrJrSis_non_tensor_nodeweight cut_value partition reachablecutsetnbrs cut_nodesnode_innode_out node_namerrr rfw_module_nodesbw_module_nodes remat_nodescountsr)rErrFrMrGr;rNr=r+r`r\rkrZrHr.r]rIr<rris         $  8   '       "  "   &rfx_graphTtracedfnamefignamec Cs|rt|j}t||}|jjD]}i|_qtj |\}}|s$d}t d||t ||}| } t| d|d||dS)Nz.svgzWriting FX graph to file: write_.)copydeepcopyrr5rrrospathsplitextrr FxGraphDrawerget_main_dot_graphrlstrip) rrr clear_metarUrbaseextgr8rrr draw_graphs    $rfull_graph.pngcCst||t||Sr>)rr)r joint_inputs file_namerrrdraw_joint_graphs  r)r(N)rT)r)E"torch.fx.experimental.proxy_tensorrr%torch.fx.experimental.symbolic_shapesrrrrrr r"torch.fxr5rrtorch.utils._pytreeutils_pytreerHrr rzsympyrr torch.fx.passesr typingr compile_utilsrrr functoolsdebug_partitionerrrrr%r,r-rGrYr^r`r\r]rrnrrrrr6rrr lru_cacherrrrr&r'rr5rrrrrrst      5] J K '