o
    hA                     @   s   d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZmZmZ ddlZddlmZ dd	lmZ eeZd
ZG dd deZG dd dZdefddZdee fddZG dd de	ZG dd dZdS )a  
Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs.

Inspired by: github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py

Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md


To launch debugger while developing:

``` [lfs "customtransfer.multipart"]
path = /path/to/transformers/.env/bin/python args = -m debugpy --listen 5678 --wait-for-client
/path/to/transformers/src/transformers/commands/transformers_cli.py lfs-multipart-upload ```    N)ArgumentParser)AbstractContextManager)DictListOptional   )logging   )BaseTransformersCLICommandzlfs-multipart-uploadc                   @   s"   e Zd ZdZedefddZdS )LfsCommandsu  
    Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs. This lets users upload
    large files >5GB 🔥. Spec for LFS custom transfer agent is:
    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md

    This introduces two commands to the CLI:

    1. $ transformers-cli lfs-enable-largefiles

    This should be executed once for each model repo that contains a model file >5GB. It's documented in the error
    message you get if you just try to git push a 5GB file without having enabled it before.

    2. $ transformers-cli lfs-multipart-upload

    This command is called by lfs directly and is not meant to be called by the user.
    parserc                 C   sP   | j ddd}|jdtdd |jdd d	 | j td
d}|jdd d	 d S )Nzlfs-enable-largefileszeDeprecated: use `huggingface-cli` instead. Configure your repository to enable upload of files > 5GB.)helppathz/Local path to repository you want to configure.)typer   c                 S      t | S N)LfsEnableCommandargs r   O/var/www/html/ai/venv/lib/python3.10/site-packages/transformers/commands/lfs.py<lambda>?       z1LfsCommands.register_subcommand.<locals>.<lambda>)funczgDeprecated: use `huggingface-cli` instead. Command will get called by git-lfs, do not call it directly.c                 S   r   r   )LfsUploadCommandr   r   r   r   r   H   r   )
add_parseradd_argumentstrset_defaultsLFS_MULTIPART_UPLOAD_COMMAND)r   enable_parserupload_parserr   r   r   register_subcommand6   s   zLfsCommands.register_subcommandN)__name__
__module____qualname____doc__staticmethodr   r"   r   r   r   r   r   $   s    r   c                   @      e Zd Zdd Zdd ZdS )r   c                 C   
   || _ d S r   r   selfr   r   r   r   __init__L      
zLfsEnableCommand.__init__c                 C   sp   t d tj| jj}tj|std td t	j
d d|d t	j
dt  d|d td d S )	NzcManaging repositories through transformers-cli is deprecated. Please use `huggingface-cli` instead.z)This does not look like a valid git repo.r	   z=git config lfs.customtransfer.multipart.path transformers-cliT)checkcwdz-git config lfs.customtransfer.multipart.args z Local repo set up for largefiles)warningswarnosr   abspathr   isdirprintexit
subprocessrunsplitr   )r+   
local_pathr   r   r   r8   O   s    
zLfsEnableCommand.runNr#   r$   r%   r,   r8   r   r   r   r   r   K       r   msgc                 C   s(   t | d } tj|  tj  dS )z-Write out the message in Line delimited JSON.
N)jsondumpssysstdoutwriteflushr=   r   r   r   	write_msgb   s   rF   returnc                  C   sV   t tj  } d| d| dfv rdS | ddvr)td t	d | S )z$Read Line delimited JSON from stdin.	terminater   eventN)downloaduploadzReceived unexpected messager	   )
r?   loadsrA   stdinreadlinestripgetloggercriticalr6   rE   r   r   r   read_msgi   s   

rS   c                   @   sP   e Zd ZdZdededefddZdd Zd	d
 ZdddZ	dd Z
dd ZdS )	FileSlicezq
    File-like object that only reads a slice of a file

    Inspired by stackoverflow.com/a/29838711/593036
    filepath	seek_from
read_limitc                 C   s   || _ || _|| _d| _d S )Nr   )rU   rV   rW   n_seen)r+   rU   rV   rW   r   r   r   r,      s   
zFileSlice.__init__c                 C   s    t | jd| _| j| j | S )Nrb)openrU   fseekrV   r+   r   r   r   	__enter__   s   zFileSlice.__enter__c                 C   s$   t | j j}t| j|| j S r   )r2   fstatr[   filenost_sizeminrW   rV   )r+   total_lengthr   r   r   __len__   s   zFileSlice.__len__c                 C   sP   | j | jkrdS | j| j  }| j|dk r|nt||}|  j t|7  _ |S )N    r   )rX   rW   r[   readrb   len)r+   nremaining_amountdatar   r   r   rg      s   zFileSlice.readc                 c   s    | j ddV  d S )Ni  @ )ri   )rg   r]   r   r   r   __iter__   s   zFileSlice.__iter__c                 G   s   | j   d S r   )r[   closer*   r   r   r   __exit__   s   zFileSlice.__exit__N)re   )r#   r$   r%   r&   r   intr,   r^   rd   rg   rl   rn   r   r   r   r   rT   x   s    
rT   c                   @   r(   )r   c                 C   r)   r   r   r*   r   r   r   r,      r-   zLfsUploadCommand.__init__c              	   C   sp  t tj  }|ddkr|ddks&tddddi td	 ti  	 t	 }|d u r7td |d }|d }|d d }|d d }t
|d}t| }g }	t|D ]C\}
}t||
| |d.}tj||d}|  |	|jd|
d	 d td||
d	 | |d W d    n1 sw   Y  q^tj|||	dd}|  td|d q+)NrI   init	operationrK   error    zWrong lfs init operation)codemessager	   Tr   oidr   actionhrefheader
chunk_size)rV   rW   )rk   etag)r{   
partNumberprogress)rI   rv   
bytesSoFarbytesSinceLast)rv   parts)r?   complete)rI   rv   )r?   rL   rA   rM   rN   rO   rP   rF   r6   rS   ro   poplistvalues	enumeraterT   requestsputraise_for_statusappendheaderspost)r+   init_msgr=   rv   rU   completion_urlry   rz   presigned_urlsr   ipresigned_urlrk   rr   r   r   r8      sV   



zLfsUploadCommand.runNr;   r   r   r   r   r      r<   r   )r&   r?   r2   r7   rA   r0   argparser   
contextlibr   typingr   r   r   r   utilsr    r
   
get_loggerr#   rQ   r   r   r   rF   rS   rT   r   r   r   r   r   <module>   s(    
'%