o
    he9                     @   s(  d dl Z d dlZd dlmZmZmZmZmZmZ d dl	Z	d dl
Z
d dlZd dlmZ d dlZd dlmZmZ d dlZd dlZeeZdd Zdd Ze Zd	d
 Zeeeedee fZ!dZ"dd Z#G dd deZ$dddddZ%dd Z&dd Z'dd Z(dd Z)dZ*dZ+dd Z,dd  Z-d!d" Z.dS )#    N)AnyDictList
NamedTupleOptionalTuple)NamedTemporaryFile)_frames_fmt_block_extrac                    sD   d  fdd}t |  fddtj fdd}|S )NTc                      s   d d S )NF r   )enabledr   M/var/www/html/ai/venv/lib/python3.10/site-packages/torch/utils/viz/_cycles.pydisable   s   z observe_garbage.<locals>.disablec                    s\   sd S | dkrt t j d S | dkr,t dg fdd}t| d S d S )NstartstopFc                     s   d s	dd< nBt  d z7d dkrt  tj tj  td tj	 }t  tj	 }||krDt
d||  W d nd w d urV| i |S d S )Nr   TF
generation   z.CUDA Memory changed during GC, %d bytes freed.)sys
setprofilegccollectgarbageclear	set_debugtorchcudamemory_allocatedloggerwarning)argskwargsbeforeafter)r   infoobserver
orig_traceself_returnr   r   
do_collect"   s(   






z8observe_garbage.<locals>.gc_callback.<locals>.do_collect)r   r   DEBUG_SAVEALLr   
getprofiler   )phaser#   r'   )r   r$   )r#   r%   r&   r   gc_callback   s   z$observe_garbage.<locals>.gc_callbackc                      s   t j  d S N)r   	callbacksremover   )r+   r   r   r.   D      zobserve_garbage.<locals>.remove)atexitregisterr   r-   append)r$   r   r.   r   )r   r+   r$   r   observe_garbage   s   
)r3   c                  C   s   ddd} t |  jd S )Nc                    s    fddS )Nc                      s    S r,   r   r   xr   r   <lambda>Y   s    z+_get_cell_type.<locals>.f.<locals>.<lambda>r   r4   r   r4   r   fX   s   z_get_cell_type.<locals>.fr   r,   )type__closure__)r7   r   r   r   _get_cell_typeW   s   
r:   c                    s  i fddfdd  fdd} fdd}fd	d
}fdd}fdd} fdd}fdd} fdd} fdd}	t |t|t|t|t|tj|tj|t|tj	|t
j|tj|	i}
tjD ]}||
v rx|
|   qm dd ttr d S )a%  
    Return known information about references held by the given object.

    Returns a mapping from referents to lists of descriptions.  Note that there
    may be more than one edge leading to any particular referent; hence the
    need for a list.  Descriptions are currently strings.

    c                    s     t|g |  d S r,   )
setdefaultidr2   )nameobj)
referencesr   r   add_referencei   s   z+annotated_references.<locals>.add_referencec                     s(   | D ]}t |r |t| qd S r,   )hasattrgetattr)attrsattrr@   r>   r   r   	add_attrsl   s
   
z'annotated_references.<locals>.add_attrsc                      s$   z d W d S  t y   Y d S w )Ncell_contents)
ValueErrorr   rF   r   r   add_cell_referencesq   s
   z1annotated_references.<locals>.add_cell_referencesc                
      s    ddddddddd		 d S )
N__defaults__r9   __globals____code____name__
__module____doc____qualname____annotations____kwdefaults__r   r   rI   r   r   add_function_references{   s   z5annotated_references.<locals>.add_function_referencesc                     s(   t D ]\} } d|  d| qd S )N[])	enumerate)positionitemrE   r   r   add_sequence_references   s   z5annotated_references.<locals>.add_sequence_referencesc                     s6     D ]\} } d|   dt|  d| qd S )NkeyrT   rU   )itemsrepr)rZ   valuerE   r   r   add_dict_references   s   
z1annotated_references.<locals>.add_dict_referencesc                     s   D ]}  d|  qd S )Nelementr   )eltrE   r   r   add_set_references   s   z0annotated_references.<locals>.add_set_referencesc                          ddd d S )N__self____func__im_classr   r   rI   r   r   add_bound_method_references   r/   z9annotated_references.<locals>.add_bound_method_referencesc                     sB   t tju rt} t| dkr| d } d| d S d S d S )N   r   __callback__)r8   weakrefrefr   get_referentslen)	referentstargetrE   r   r   add_weakref_references   s   
z4annotated_references.<locals>.add_weakref_referencesc                     sP   j }  dddddd t| tu r$j  D ]\}}d| | qd S d S )Nf_backf_code
f_builtins	f_globalsf_tracef_localszlocal )ru   r8   dictr[   )ru   r=   local)rF   r@   r>   r   r   add_frame_references   s   z2annotated_references.<locals>.add_frame_referencesc                      rb   )N__objclass__rN   __doc__r   r   rI   r   r    add_getset_descriptor_references   r/   z>annotated_references.<locals>.add_getset_descriptor_references__dict__	__class____mro__)tuplelistrv   set	frozensettypesFunctionType	FrameTypeCellType
MethodTyperi   rj   GetSetDescriptorTyper8   r~   
isinstance)r>   rJ   rS   rY   r^   ra   rf   ro   rx   r{   type_based_referencestype_r   )rF   r@   r>   r?   r   annotated_references^   s@   	





r       c                 C   s|  dd }t | trt| S t| jdkrd| j S t | tjr7z| jj}W n t	y1   d}Y nw d| S t | t
rDd||  dS t | trQd	||  d
S t | tr^dt|  dS t | tjrjd| j S t | trud| j S t | tjr|  }|du rdS dt|dS t | tjr| jj}t|tkrd|td  d  }d| d| j S dt| j dt| j S )zy
    Return a string to be used for Graphviz nodes.  The string
    should be short but as informative as possible.

    c                 S   sD   d dd ttd| D }t| dkr | dt| d  }|S )N,c                 s   s0    | ]\}}t |trt|nt|jV  qd S r,   )r   
BASE_TYPESr\   r8   rN   ).0ir5   r   r   r   	<genexpr>   s   . z=object_annotation.<locals>.format_sequence.<locals>.<genexpr>   z, ...)joinziprangerl   )r>   bodyr   r   r   format_sequence   s   z*object_annotation.<locals>.format_sequencefunctionz	function
z<anonymous>zinstancemethod
rT   rU   ()zdict[zmodule
ztype
Nzweakref (dead referent)zweakref to id 0xr5   z...   zframe
:zobject
.)r   r   r\   r8   rN   r   r   rd   __qualname__AttributeErrorr   r   rv   rl   
ModuleTyperi   rj   r<   r   rq   co_filenameFRAME_FILENAME_LIMITf_linenorO   )r>   r   	func_namereferentfilenamer   r   r   object_annotation   sB   





r   c                   @   s>   e Zd ZU eed< ee ed< eed< eeee	f  ed< dS )Nodelabelcontextroot
referrentsN)
rN   rO   r   strrQ   r   boolr   r   intr   r   r   r   r      s
   
 r   r   filterc                   s   d u rt   d u rt fdd| D }dd | D }dd t| D }| D ]F}|t| }|| }t|}	t|D ]0}
t|
}||d }|d u rOq>|| }|	|dg}|| | |D ]
}|j	||f qcq>q)dd t|D }t
 }|r| }||v rq||| || }|| |s~i g }t|D ]\}}||v rt|< || q|D ]}fdd|j	D |j	d d < q|S )	Nc                    s&   g | ]}t t| ||g qS r   )r   r   r   r>   r   r   r   
<listcomp>  s   & z create_graph.<locals>.<listcomp>c                 S   s   g | ]}g qS r   r   r   r   r   r   r     s    c                 S   s   i | ]	\}}t ||qS r   )r<   )r   r   r>   r   r   r   
<dictcomp>      z create_graph.<locals>.<dictcomp>?c                 S   s   g | ]	\}}|j r|qS r   )r   )r   r   nr   r   r   r     r   c                    s$   g | ]\}}| v r| | fqS r   r   )r   r   idx)id_to_filtered_idr   r   r   /  s
    
)cuda_allocation_contextis_cuda_tensorrV   r<   r   r   rk   getr2   r   r   popaddextendrl   )objectsr   r   nodesnode_referrers
id_to_noder>   fidxr7   r?   	referrentridtidxtlabelsr   	to_searchto_keepr   	referrersfilteredr   r   r   )r   r   r   r   create_graph  sZ   



r   c                 C   s
   t | S r,   )jsondumps)r   r   r   r   escape4  s   
r   c                 C   s   t | tjo| jS r,   )r   r   Tensoris_cuda)r>   r   r   r   r   8  s   r   c                     sp   t jj } i  | d D ]#}|d }|d D ]}|d dkr(t|\}}| |< ||d 7 }qq fdd}|S )	Nsegmentsaddressblocksstateactive_allocatedsizec                    s<   t | r|   } |}|d urdt|ddS d S )N
T)full_filename)r   untyped_storagedata_ptrr   r   r	   )r>   addrframesaddr_to_framer   r   object_contextF  s   
z/cuda_allocation_context.<locals>.object_context)r   r   memory	_snapshotr
   )snapshotsegr   blkr   	real_sizer   r   r   r   r   ;  s   r   c              
   C   s   g d}t | D ]\}}|| dt|j d|jrdnd d qt | D ]\}}|jD ]\}}|| d| dt| d	 q.q'|d
 d|S )N)zdigraph GraphName {znode [shape=rect];zrankdir=LR;z [label=z, color=redblackz];z -> z
 [label = rU   z}
r   )rV   r2   r   r   r   r   r   )r   linesr   r   r7   r   jr   r   r   to_dotO  s   ."

r   az  
<!DOCTYPE html>
<html>
<head>
  <style>
    body {
      margin: 0;
      padding: 0;
      overflow: hidden;
    }

    #container {
      display: flex;
      flex-direction: column;
      height: 100vh;
    }

    #main {
      flex: 2;
      overflow: auto;
    }

    #preContainer {
      flex: 1;
      overflow: auto;
    }

    svg {
        overflow: scroll;
    }

    pre {
      margin: 0;
      padding: 10px;
    }
  </style>
</head>
<body>
  <div id="container">
    <div id="main">
    </div>
    <div id="preContainer">
      <pre id="stacktrace">Mouse over tensor objects to see where they were allocated.</pre>
    </div>
  </div>
<script src='https://cdnjs.cloudflare.com/ajax/libs/viz.js/1.8.0/viz-lite.js'></script>
<script>
let dot = $DOT
let image = Viz(dot, {format: 'svg'});
document.getElementById('main').innerHTML = image
$LISTENERS
</script>
</body>
</html>
z
document.getElementById('node{id}').addEventListener('mouseover', function(event) {{
  document.getElementById("stacktrace").textContent = {stack}
}})
c                 C   sz   g }t | D ]#\}}|jd u rqtjt|d t|j d|j d}|| qt| }t	
dt|
dd|S )Nrg   z:
)r<   stackz$DOTz
$LISTENERSr   )rV   r   _listener_templateformatr   r   r   r2   r   	_templatereplacer\   r   )r   	listenersr   r   sdotr   r   r   to_html  s   
(r   c                    s$   t jjjdd  fdd}t|S )Ni )max_entriesc                    s<   | rt dd | D std d S  tt|  d S d S )Nc                 s   s    | ]}t |V  qd S r,   )r   r   r   r   r   r     s    z:observe_tensor_cycles.<locals>.observer.<locals>.<genexpr>z No CUDA Tensors found in garbage)anyr   r#   r   r   )r   callbackr   r   r$     s   
z'observe_tensor_cycles.<locals>.observer)r   r   r   _record_memory_historyr3   )r   r$   r   r   r   observe_tensor_cycles  s   r   c                  C   s   t d dd } t| S )a  
    Reference cycles are freed by the cycle collector rather than being cleaned up
    when the objects in the cycle first become unreachable. If a cycle points to a tensor,
    the CUDA memory for that tensor will not be freed until garbage collection runs.
    Accumulation of CUDA allocations can lead to out of memory errors (OOMs), as well as
    non-deterministic allocation behavior which is harder to debug.

    This function installs a warning that is reports whenever a cycle that is holding CUDA
    memory is observed. The warning produces a html file that visualizes the cycle,
    and links it to the stack frame that allocted the CUDA tensor.
    z2Watching Python reference cycles for CUDA Tensors.c                 S   sL   t dddd}||  td|j W d    d S 1 sw   Y  d S )Nwz.htmlF)suffixdeletezDReference cycle includes a CUDA Tensor see visualization of cycle %s)r   writer   r   r=   )htmlr7   r   r   r   write_and_log  s   
"z)warn_tensor_cycles.<locals>.write_and_log)r   r#   r   )r  r   r   r   warn_tensor_cycles  s   
r  )/r   r   typingr   r   r   r   r   r   r   ri   r   tempfiler   r   torch.cuda._memory_vizr	   r
   r0   logging	getLoggerrN   r   r3   r:   r   r   r   floatcomplexr8   r   bytesr   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   r   <module>   s<     
Im2.7
