o
    +ke                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlmZmZ ddlmZ ddlmZ ddlT ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlm!Z!m"Z" ddl#m$Z$m%Z% ddlm&Z& ddl'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/ e$e0Z#dZ1e*eZ2d\Z3Z4Z5Z6dZ7dZ8dd Z9G dd deZ:G dd de:Z;G d d! d!eZ<G d"d# d#eZ=G d$d% d%eZ>G d&d' d'eZ?i d(d)d*d+d,d-d.d-d/d-d0d1d2d3d4d5d6d5d7d8d9d-d:d-d;d<d=d>d?d-d@dAZ@dBdC ZAG dDdE dEZBG dFdG dGZCdHdI ZDG dJdK dKZEdLdM ZFG dNdO dOZGG dPdQ dQeGZHdRddddRdSdTdUZIdS )V    N)PopenPIPE   )__version__)
Compressor)*)ErrorIntegrityError)
bin_to_hex)get_base_dir)get_limited_unpacker)replace_placeholders)sysinfo)format_file_size)safe_unlink)prepare_subprocess_envignore_sigint)create_loggersetup_logging)msgpack)
Repository)parse_versionformat_version)xxh64)EfficientCollectionQueue   )   i   m   a   rd   皙?c                 C   sP   t | }}|r&t| |}||8 }|s	 |S ||d }t|d  |s|S )z;os.write wrapper so we do not lose data for partial writes.Ng&.>)lenoswritetimesleep)fddataamount	remainingcount r,   /usr/lib/python3.10/site-packages/borg/remote.pyos_write0   s   r.   c                   @      e Zd ZdZdS )ConnectionClosedz Connection closed by remote hostN__name__
__module____qualname____doc__r,   r,   r,   r-   r0   C       r0   c                   @   r/   )ConnectionClosedWithHintz$Connection closed by remote host. {}Nr1   r,   r,   r,   r-   r7   G   r6   r7   c                   @   r/   )PathNotAllowedzRepository path not allowed: {}Nr1   r,   r,   r,   r-   r8   K   r6   r8   c                   @   r/   )InvalidRPCMethodzRPC method {} is not validNr1   r,   r,   r,   r-   r9   O   r6   r9   c                   @   r/   )!UnexpectedRPCDataFormatFromClientz4Borg {}: Got unexpected RPC data format from client.Nr1   r,   r,   r,   r-   r:   S   r6   r:   c                       s    e Zd ZdZ fddZ  ZS )!UnexpectedRPCDataFormatFromServerz.Got unexpected RPC data format from server:
{}c                    sd   z
|  d d }W n ty)   |d d }dd |D }td|d}Y nw t | d S )N   c                 S   s   g | ]}d | qS )z%02Xr,   ).0Zbyter,   r,   r-   
<listcomp>_       z>UnexpectedRPCDataFormatFromServer.__init__.<locals>.<listcomp> 0   )decodeUnicodeDecodeErrortextwrapfilljoinsuper__init__)selfr(   	__class__r,   r-   rH   Z   s   z*UnexpectedRPCDataFormatFromServer.__init__)r2   r3   r4   r5   rH   __classcell__r,   r,   rJ   r-   r;   W   s    r;   check)repair
save_spacecommit)rO   rollbackr,   destroy__len__list)limitmarkerput)idr(   getrX   deletesave_key)keydataload_key
break_lock	negotiate)client_dataopen)pathcreate	lock_waitlock	exclusiveappend_onlyget_free_noncecommit_nonce_reservation)next_unreservedstart_noncec                    s    fdd D S )Nc                    s   i | ]	}|   | qS r,   )rB   )r=   kdr,   r-   
<dictcomp>       zdecode_keys.<locals>.<dictcomp>r,   rn   r,   rn   r-   decode_keys      rr   c                   @   sV   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z			dddZ
dd ZdS )RepositoryServer)rS   rM   rP   r[   rR   rY   rT   scanr`   rb   rW   rQ   r\   r^   r_   ri   rj   inject_exceptionc                 C   s,   d | _ || _|| _|| _|| _td| _d S )N1.0.8)
repositoryrestrict_to_pathsrestrict_to_repositoriesrh   storage_quotar   client_version)rI   ry   rz   rh   r{   r,   r,   r-   rH      s   zRepositoryServer.__init__c                    sn   z fddt t| D W S  ty6   |dkr(t dkr(t| dd }n  fddt |D  Y S w )z5Translate from positional protocol to named protocol.c                       i | ]	\}}| | qS r,   r,   r=   posnameargvr,   r-   rp      rq   z8RepositoryServer.positional_to_named.<locals>.<dictcomp>rb      Nc                    r}   r,   r,   r~   r   r,   r-   rp      rq   )	enumerate	compatMap
IndexErrorr"   )rI   methodr   mappingr,   r   r-   positional_to_named   s   z$RepositoryServer.positional_to_namedc                    s$   t t|j  fddD S )zWRemove unknown named parameters from call, because client did (implicitly) say it's ok.c                    s   i | ]}| v r|| qS r,   r,   r=   r   Zknownkwargsr,   r-   rp      s    z0RepositoryServer.filter_args.<locals>.<dictcomp>)setinspect	signature
parameters)rI   fr   r,   r   r-   filter_args   s   zRepositoryServer.filter_argsc                 C   s  t j }t j }t j }t|d t|d t|d td}	 t|gg g d\}}}|rt|t	}|sU| j
d urI| j
  d S t|dt  d S || |D ]n}	t|	trwd}
|	t }|	t  }t|	t }n,t|	trt|	dkrd}
|	\}}}}| }| ||}n| j
d ur| j
  ttz/|| jvrt|zt| |}W n ty   t| j
|}Y nw |  ||}|di |}W n t!y } z|
rUt"#|j$|}t"j%t &  }d}t|t'r|( g}|j"}t|t)j*t)j+t,frnt-.d/| zt01t|d|j$j2d	|j3d
|d|d|dt4 i}W n& t5yN   t01t|d|j$j2d	dd |j3D d
|d|d|dt4 i}Y nw t|| nOt|t)j*t)j+t,fran3t|t'rv|j"rnt-j6nt-j7}|( }n	t-j6}d|j$j2 }t"8  dt4  }t-9| t-:|| d}t|t01d||j$j2|f W Y d }~q\d }~ww |
rt|t01t|t;|i q\t|t01d|d |f q\|r| j
  d S q&)NFTserver
   z<Borg {}: Got connection close before repository was opened.
r   
   exception_class   exception_args   exception_full   exception_short   exception_trace   sysinfoc                 S   s$   g | ]}t |tttfr|nd qS N)
isinstancestrbytesint)r=   xr,   r,   r-   r>     s    z*RepositoryServer.serve.<locals>.<listcomp>z%s Exception in RPC callz3Remote Exception (see remote log for the traceback)r   r,   )<sysstdinfilenostdoutr#   set_blockingr   selectreadBUFSIZErx   closer.   formatr   encodefeedr   dictMSGIDMSGrB   rr   ARGStupler"   r   r:   rpc_methodsr9   getattrAttributeErrorr   BaseException	tracebackformat_exception_onlyrK   format_exceptionexc_infor   get_messager   DoesNotExistAlreadyExistsr8   loggingdebugrF   r   packbr2   argsr   	TypeErrorERRORDEBUG
format_excerrorlogRESULT)rI   stdin_fd	stdout_fd	stderr_fdunpackerrwesr(   unpacked
dictFormatmsgidr   r   _r   reseZex_shortZex_fullZex_tracemsgZtb_log_leveltbexcr,   r,   r-   serve   s   


















6
zRepositoryServer.servec                 C   s`   |t krt S t|tr)|d | _ttdj}tdd|d t	
d dtiS t| _dtiS )N   client_version T)Zis_servejsonlevelz2Initialized logging system for JSON-based protocolserver_version)RPC_PROTOCOL_VERSIONr   r   r|   r   getLevelName	getLoggerr   r   loggerr   BORG_VERSION)rI   ra   r   r,   r,   r-   r`   /  s   


zRepositoryServer.negotiatec                 C   s   t |tr
t|}|drtjt |dd  }n9|dr.tjt |dd  }n'|dr>tj|dd  }n|drJtj|}n|drU|dd  }tj	|S )	Nz/~/   z~/r   z/~r   ~z/./)
r   r   r#   fsdecode
startswithrc   rF   r   
expanduserrealpath)rI   rc   r,   r,   r-   _resolve_path@  s   






zRepositoryServer._resolve_pathFNTc              
   C   s   t d| | |}t d| tj|d}| jr7| jD ]}	tjtj|	d}
||
r2 nqt	|| j
rU| j
D ]}tjtj|d}||krP nq=t	|| r[| jp\|}t|||||| j||d| _| j  | jjS )NzResolving repository path %rzResolved repository path to %rr   )re   rf   rh   r{   rg   make_parent_dirs)r   r   r   r#   rc   rF   ry   r   r   r8   rz   rh   r   r{   rx   	__enter__rX   )rI   rc   rd   re   rf   rg   rh   r   Zpath_with_sepZrestrict_to_pathZrestrict_to_path_with_sepZrestrict_to_repositoryZrestrict_to_repository_with_sepr,   r,   r-   rb   P  s6   





zRepositoryServer.openc                 C   s   |  }d}d}|dkrt||dkrt||dkr#t||dkr+t||dkr3td|d	kr=t|||d
krEt||dkrOdd  d S d S )Nztest stringztest string2r   r   CheckNeededr	   r8   ZfooObjectNotFoundr9   divider   )	rB   r   r   r   r   r	   r8   r   r9   )rI   kinds1s2r,   r,   r-   rv   s  s(   


z!RepositoryServer.inject_exception)FNTNFF)r2   r3   r4   r   rH   r   r   r   r`   r   rb   rv   r,   r,   r,   r-   rt      s    k
#rt   c                   @   s   e Zd Zdd Zdd ZdS )SleepingBandwidthLimiterc                 C   s2   |rt |t | _t | _| j| _d S d | _d S r   )r   RATELIMIT_PERIOD	ratelimitr%   	monotonicratelimit_lastratelimit_quota)rI   rU   r,   r,   r-   rH     s
   

z!SleepingBandwidthLimiter.__init__c                 C   s   | j rSt }| jt |kr'|  j| j 7  _| jd| j  kr$d| j  | _|| _| jdkrE| jt | }t| |  j| j 7  _t | _t|| jkrS|d | j }t	||}| j rc|  j|8  _|S )Nr   r   )
r   r%   r   r   r   r   r&   r"   r#   r$   )rI   r'   to_sendnowZtosleepwrittenr,   r,   r-   r$     s$   


zSleepingBandwidthLimiter.writeN)r2   r3   r4   rH   r$   r,   r,   r,   r-   r     s    r   c                    s    fdd}|S )aS  Check version requirements and use self.call to do the remote method call.

    <since> specifies the version in which borg introduced this method.
    Calling this method when connected to an older version will fail without transmitting anything to the server.

    Further kwargs can be used to encode version specific restrictions:

    <previously> is the value resulting in the behaviour before introducing the new parameter.
    If a previous hardcoded behaviour is parameterized in a version, this allows calls that use the previously
    hardcoded behaviour to pass through and generates an error if another behaviour is requested by the client.
    E.g. when 'append_only' was introduced in 1.0.7 the previous behaviour was what now is append_only=False.
    Thus @api(..., append_only={'since': parse_version('1.0.7'), 'previously': False}) allows calls
    with append_only=False for all version but rejects calls using append_only=True on versions older than 1.0.7.

    <dontcare> is a flag to set the behaviour if an old version is called the new way.
    If set to True, the method is called without the (not yet supported) parameter (this should be done if that is the
    more desirable behaviour). If False, an exception is generated.
    E.g. before 'threshold' was introduced in 1.2.0a8, a hardcoded threshold of 0.1 was used in commit().
    c                    s   t   fdd}|S )Nc           
   	      s4  t  }|j| g|R i |}i }i }|j D ]-\}}|dkr#q||jv r<|dkr4|j| ||< q|j| ||< q|j|jurG|j||< q| jk rV| 	 j
t D ]4\}}	|	d | jkrfqZd|	v rs|| |	d krsqZ|	ddrzqZ| 	 j
 d| d|| t|	d | j j
|fi |S )	NrI   waitsince
previouslydontcareFr@   =)r   r   bindr   items	argumentsdefaultemptyr   RPCServerOutdatedr2   r   rY   call)
rI   r   r   sigZ
bound_argsnamedextrar   paramZrestriction)r   kwargs_decoratorr   r,   r-   do_rpc  s6   




z&api.<locals>.decorator.<locals>.do_rpc)	functoolswraps)r   r  r  r   )r   r-   	decorator  s    zapi.<locals>.decoratorr,   )r   r  r  r,   r  r-   api  s   #r  c                   @   sn  e Zd Zg ZG dd deZG dd deZdZ		dUdd	Z	d
d Z
dd Zdd Zdd Zedd Zdd Zdd Zdd Zdd ZdVddZeededdd ed!dd d"		dWd#d$Zeeded%d&d d'dXd(d)Zeeded*ddd+ed,d-dd+ed*ddd+d.dYd/d0Zeedd1d2d3 Zeedd1d4d5 Zeedd1d6d7 Zeedd1dZd8d9Zeed:d1dZd;d<Zd=d> Zd[d?d@Z eedd1d\dAdBZ!eedd1d\dCdDZ"eedd1dEdF Z#eedd1dGdH Z$eedd1dIdJ Z%eedd1dKdL Z&eedd1dMdN Z'dOdP Z(d\dQdRZ)dSdT Z*dS )]RemoteRepositoryc                   @   sL   e Zd Zdd Zdd Zedd Zedd Zed	d
 Zedd Z	dS )zRemoteRepository.RPCErrorc                 C   s
   || _ d S r   )r   )rI   r   r,   r,   r-   rH     s   
z"RemoteRepository.RPCError.__init__c                 C   s$   d| j v rd| j d  S | jS )Nr      
)r   rF   rB   exception_classrI   r,   r,   r-   r     s   
z%RemoteRepository.RPCError.get_messagec                 C   s   | j ddS )Nr   T)r   rY   r  r,   r,   r-   r        z#RemoteRepository.RPCError.tracebackc                 C   s   | j d  S )Nr   r   rB   r  r,   r,   r-   r    r  z)RemoteRepository.RPCError.exception_classc                 C   s*   d| j v rd| j d  S |  d S )Nr   r  z4
Remote Exception (see remote log for the traceback))r   rF   rB   r   r  r,   r,   r-   exception_full  s   
z(RemoteRepository.RPCError.exception_fullc                 C   s   d| j v r| j d  S dS )Nr   r   r  r  r,   r,   r-   r     s   
z!RemoteRepository.RPCError.sysinfoN)
r2   r3   r4   rH   r   propertyr   r  r  r   r,   r,   r,   r-   RPCError  s    


r  c                   @   s(   e Zd ZdZedd Zedd ZdS )z"RemoteRepository.RPCServerOutdatedz2Borg server is too old for {}. Required version {}c                 C   
   | j d S Nr   r   r  r,   r,   r-   r   	     
z)RemoteRepository.RPCServerOutdated.methodc                 C   r  )Nr   r  r  r,   r,   r-   required_version  r  z3RemoteRepository.RPCServerOutdated.required_versionN)r2   r3   r4   r5   r  r   r   r,   r,   r,   r-   r    s    
r  FNTc	              
      s  | _ _g _d_d_d_tdt_d_	i _
t _i _i _d _t|r6|jr6|jd nd_|rF|jrF|jd d nd_td_td_d _|_|jdk}	t|	 d}
||	}|	ss|| }t d	| t!|dt"t"t"|
t#d
_jj$% _&jj'% _(jj)% _*t+,j&d t+,j(d t+,j*d j(j*g_-j&j(j*g_.zz/dddt0ii}W n t1y   t2dd w |t3krd_4nt5|t6rd|v rd_4|d _nt7d|  fdd}j4r|  W d S z|  W W d S  j8yG } z#|j9dkr' d}t:j);| td_dt<d< |  W Y d }~W d S d }~ww  t7yT   =   w )Nr   i       i   Zclientrw   Z__testsuite__)systemzSSH command line: %s)bufsizer   r   stderrenv
preexec_fnFr`   ra   r   zIs borg working on the server?s   server_versionTz8Server insisted on using unsupported protocol version %sc                	      s"   j jj d_d S )N)rc   rd   re   rf   rg   rh   r   )rb   locationrc   rX   r,   rh   rd   rg   rf   re   r   rI   r,   r-   do_openK  s   z*RemoteRepository.__init__.<locals>.do_openr   ao  Please note:
If you see a TypeError complaining about the number of positional arguments
given to open(), you can ignore it if it comes from a borg version < 1.0.7.
This TypeError is a cosmetic side effect of the compatibility code borg
clients >= 1.0.7 have to support older borg servers.
This problem will go away as soon as the server has been upgraded to 1.0.7+.
z1.0.6)rc   rd   re   rf   rb   )>r'  Z	_locationpreload_idsr   rx_bytestx_bytesr   r   r   stderr_receivedchunkid_to_msgidsr   ignore_responses	responsesasync_responsesshutdown_timer   Zupload_ratelimitr   Zupload_bufferupload_buffer_size_limitr   r   r   r   p_argshostr   borg_cmdssh_cmdr   r   r   r   r   r   r   r   r   r   r$  r   r#   r   r_fdsx_fdsr  r   r0   r7   r   r   r   r   	Exceptionr  r  r   r$   r   r   )rI   r'  rd   rg   re   rf   rh   r   r   testingr%  r7  versionr)  errr   r,   r(  r-   rH     s   



	
zRemoteRepository.__init__c                 C   s:   t | jrtdt | jf  | jr|   J dd S )Nz2still %d cached responses left in RemoteRepositoryFz&cleanup happened in Repository.__del__)r"   r0  r   r   r4  r   r  r,   r,   r-   __del__k  s   
zRemoteRepository.__del__c                 C   s   d| j j d| j  dS )N<r@   >)rK   r2   r'  Zcanonical_pathr  r,   r,   r-   __repr__r  s   zRemoteRepository.__repr__c                 C      | S r   r,   r  r,   r,   r-   r   u     zRemoteRepository.__enter__c                 C   s   z;|d ur&t  d | _|   W tdt| jt| j| j	 | 
  d S W tdt| jt| j| j	 | 
  d S tdt| jt| j| j	 | 
  w )N   zDRemoteRepository: %s bytes sent, %s bytes received, %d messages sent)r%   r   r2  rQ   r   r   r   r,  r+  r   r   rI   exc_typeZexc_valZexc_tbr,   r,   r-   __exit__x  s"   

zRemoteRepository.__exit__c                 C   s
   t | jS r   )r
   rX   r  r,   r,   r-   id_str  r  zRemoteRepository.id_strc                 C   s*  g }|durmt  }|t jr|d n/|t jr"|d n#|t jr)n|t jr5|d n|t jrA|d nt	d|j
D ]}d|vrRd| }d	|v r]|d
|  qHd|v rm|jrm|d|j  g }|r|tjdddg | | j S |jptjdd}t|}||dg | S )z return a borg serve command lineNz--debugz--infoz--errorz
--criticalz log level missing, fix this code.zborg.debug.rx   z--debug-topic=%sr{   z--storage-quota=%sz-mzborg.archiverr   ZBORG_REMOTE_PATHborg)r   r   isEnabledForr   appendINFOWARNINGr   CRITICAL
ValueErrorZdebug_topicsr{   r   
executableextra_test_argsremote_pathr#   environrY   r   )rI   r   r<  optsZroot_loggertopicZenv_varsrT  r,   r,   r-   r7    s8   
zRemoteRepository.borg_cmdc                 C   sn   | j jp
tjdd}t|}|jr|dt|jg7 }|j	r-|
|j	 d|j  |S |
d|j  |S )zEreturn a ssh command line that can be prefixed to a borg command lineZBORG_RSHZsshz-p@%s)r5  rshr#   rU  rY   shlexsplitportr   userrM  r6  )rI   r'  rZ  r   r,   r,   r-   r8    s   
zRemoteRepository.ssh_cmdc                    s    fddt | D S )Nc                    s   g | ]} | qS r,   r,   r   r   r,   r-   r>     r?   z8RemoteRepository.named_to_positional.<locals>.<listcomp>)r   )rI   r   r   r,   r_  r-   named_to_positional  s   z$RemoteRepository.named_to_positionalc                 K   s&   | j ||gfi |D ]}|  S d S r   	call_many)rI   cmdr   kwrespr,   r,   r-   r    s   zRemoteRepository.callc                 #   sH   |s	|dkr	d S  fdd} fdd} fdd}t |}g }	|r%dn j}
|  |s0|r jrHt  jkrHtd	t|	t j d S |	rzz$ j	
|	d }|	
d d
|v ra|| n|t V  |	sm|smW d S W n	 tyw   Y nw |	sJ|dkr	 z	 j \}}W n ty   |r jrY nY d S w d
|v r|| n|t V  q js|s jrt|	tk r jg}ng }t j| jd\}}}|rtd|D ]}| ju rXt|t}|st   jt|7  _ j|  jD ]^}t|t r|t! }n(t|t"r)t|dkr)|\}}}}|r"t!|d
|i}nt!|t|i}nt#|| jv rQ j$| d
|v rD| j|< q|t d urP| j|< q| j	|< qq| j%u rt|d}|sjt   jt|7  _ j&r j&| }d _&|j'dd}|r|d (ds|
  _&|D ]	}t)|*  qq|rt j|
kr|s jrt|	tk r|r1|r|dksJ d|d d  j+v r|	,||
dd  nP|
d}|dkr|d  j+v r|	,||d  n4  j-d7  _-|	, j-  j.r j/t01t! j-t2|t3|i n j/t01d j-| 4||f  js| jr| j
d}d|i}  j-d7  _- j+5|g , j-  j.rj j/t01t! j-t2dt3|i n j/t01d j-d 4d|f t j|
kr|s jrt|	tk s|  |s0|s0  jt6|	O  _d S )Nr1  c               
      s|    j r<z j j j  }   j| 7  _ j |  W d S  ty; } z|jtj	tj
fvr0 W Y d }~d S d }~ww d S r   )r   r   r$   r   Z
peek_frontr,  Z	pop_frontOSErrorerrnoEAGAINEWOULDBLOCK)r   r   r  r,   r-   send_buffer  s   z/RemoteRepository.call_many.<locals>.send_bufferc                    s&    j |  d} j |  s j | = |S r  )r.  pop)Zchunkidr   r  r,   r-   pop_preload_msgid  s   
z5RemoteRepository.call_many.<locals>.pop_preload_msgidc                    sL  | d   }d| v}| d}|dkrt jj|dkr%t jj|dkr0t jj|dkrB|r:tdt|d   |d	krV|rMt	dt	|d   |d
krh|r`t
dt
|d   |dkrut|d   |dkr|rtd jjt|d    jj|dkr|rtdt|d    | )Nr   r   r   r   r   r	   z(not available)r   AtticRepositoryr8   z	(unknown)ParentPathDoesNotExistr   r9   )rB   rY   r   r   r'  	processedr   r   r	   rm  r8   rn  r   r9   r  )r   r   Z
old_serverr   r  r,   r-   handle_error  s@   


z0RemoteRepository.call_many.<locals>.handle_errorr   zPshutdown_time reached, shutting down with %d waiting_for and %d async_responses.r   Tr   zFD exception occurredr   i   r!  )keepends)   r  rY   z&is_preload is only supported for 'get'rX   )7rT   r3  r2  r%   r   r   r   r"   r1  r0  rk  r   KeyErrorpopitemr/  r   r*  MAX_INFLIGHTr   r   r9  r:  r;  r   r#   r   r   r0   r+  r   r   r   r   r   r   r;   remover   r-  
splitlinesendswithhandle_remote_linerB   r.  rM  r   r   Z	push_backr   r   r   r   r`  
setdefaultr   )rI   rc  callsr   is_preloaded
async_waitrj  rl  rp  Zwaiting_forZmaximum_to_sendr   r   Zw_fdsr   r   r   r'   r(   r   r   r   linesliner   Zchunk_idr,   r  r-   rb    s   )













,
"$"$,vzRemoteRepository.call_manyz1.0.0z1.0.7)r   r   z1.1.9)r   rh   r   c                 C      dS z;actual remoting is done via self.call in the @api decoratorNr,   )rI   rc   rd   re   rf   rg   rh   r   r,   r,   r-   rb         zRemoteRepository.openz1.2.0a4r   )r   max_durationc                 C   r  r  r,   )rI   rN   rO   r  r,   r,   r-   rM     r  zRemoteRepository.checkz1.2.0a0)r   r   r   z1.2.0a8r!   )r   compact	thresholdcleanup_commitsc                 C   r  r  r,   )rI   rO   r  r  r  r,   r,   r-   rP     r  zRemoteRepository.commit)r   c                 C   r  r  r,   r  r,   r,   r-   rQ     r  zRemoteRepository.rollbackc                 C   r  r  r,   r  r,   r,   r-   rR     r  zRemoteRepository.destroyc                 C   r  r  r,   r  r,   r,   r-   rS     r  zRemoteRepository.__len__c                 C   r  r  r,   rI   rU   rV   r,   r,   r-   rT     r  zRemoteRepository.listz1.1.0b3c                 C   r  r  r,   r  r,   r,   r-   ru     r  zRemoteRepository.scanc                 C   s   |  |gD ]}|  S d S r   )get_many)rI   rX   re  r,   r,   r-   rY     s   zRemoteRepository.getc                 c   s&    | j ddd |D |dE d H  d S )NrY   c                 S   s   g | ]}d |iqS rZ   r,   )r=   rX   r,   r,   r-   r>     r?   z-RemoteRepository.get_many.<locals>.<listcomp>)r}  ra  )rI   idsr}  r,   r,   r-   r    s   $zRemoteRepository.get_manyc                 C   r  r  r,   )rI   rX   r(   r   r,   r,   r-   rW     r  zRemoteRepository.putc                 C   r  r  r,   )rI   rX   r   r,   r,   r-   r[     r  zRemoteRepository.deletec                 C   r  r  r,   )rI   r]   r,   r,   r-   r\     r  zRemoteRepository.save_keyc                 C   r  r  r,   r  r,   r,   r-   r^     r  zRemoteRepository.load_keyc                 C   r  r  r,   r  r,   r,   r-   ri     r  zRemoteRepository.get_free_noncec                 C   r  r  r,   )rI   rk   rl   r,   r,   r-   rj     r  z)RemoteRepository.commit_nonce_reservationc                 C   r  r  r,   r  r,   r,   r-   r_     r  zRemoteRepository.break_lockc                 C   s6   | j r| j j  | j j  | j   d | _ d S d S r   )r4  r   r   r   r   r  r,   r,   r-   r     s   

zRemoteRepository.closec                 C   s"   | j dg d|dD ]}|  S d S )Nr1  T)r|  r   r~  ra  )rI   r   re  r,   r,   r-   async_response  s   zRemoteRepository.async_responsec                 C   s   |  j |7  _ d S r   )r*  )rI   r  r,   r,   r-   preload  rs   zRemoteRepository.preload)FFNTFFN)TFT)FNTFFF)FFr   )FTr!   F)NN)FT)+r2   r3   r4   rS  r;  r  r   r  r   rH   r?  rB  r   rH  r  rI  r7  r8  r`  r  rb  r  r   rb   rM   rP   rQ   rR   rS   rT   ru   rY   r  rW   r[   r\   r^   ri   rj   r_   r   r  r  r,   r,   r,   r-   r    s    "
W
.
 =









r  c                 C   s  |  dsJ | drt| }|d dvr"td|d |  dS |d dkrktt|d tj}t	|t
s8J t|d	 }d
|d  |d< tdjr`|| kr`tjt|d  dS ||d|d  dS |d drtd}| tjkrtdjrd|v rd
|d  |d< tjt|d  dS d|v rtjd
|d  d  dS dS dS dS | dr| dd\}}}tt|tj}|drt||  dS |dd\}}t|||  dS tdjrtdd
|    dS tjd
|   dS )zv
    Handle a remote log line.

    This function is remarkably complex because it handles multiple wire formats.
    )r   {type)Zprogress_messageZprogress_percentlog_messagez3Dropped remote log message with unknown type %r: %sNr  	levelnamer   zRemote: messagerK  r   rY  Z	progress_zborg.output.progressr  z$LOG r@   r   zRemote:r   r   )ry  r   r   loadsr   warningr   r   rP  r   r   r   getEffectiveLevelr   r$  r$   dumpsr   rN  r\  rstripstrip)r  r   r   Ztarget_loggerZprogress_loggerr   Zlognamer,   r,   r-   rz    sH   


	

rz  c                   @   sL   e Zd ZdZdddZdd Zdd Zd	d
 Zdd ZdddZ	dd Z
dS )RepositoryNoCacheaR  A not caching Repository wrapper, passes through to repository.

    Just to have same API (including the context manager) as RepositoryCache.

    *transform* is a callable taking two arguments, key and raw repository data.
    The return value is returned from get()/get_many(). By default, the raw
    repository data is returned.
    Nc                 C   s   || _ |pdd | _d S )Nc                 S   s   |S r   r,   )keyr(   r,   r,   r-   <lambda>4  r  z,RepositoryNoCache.__init__.<locals>.<lambda>)rx   	transform)rI   rx   r  r,   r,   r-   rH   2  s   zRepositoryNoCache.__init__c                 C      d S r   r,   r  r,   r,   r-   r   6  rD  zRepositoryNoCache.closec                 C   rC  r   r,   r  r,   r,   r-   r   9  rD  zRepositoryNoCache.__enter__c                 C   s   |    d S r   )r   rF  r,   r,   r-   rH  <  s   zRepositoryNoCache.__exit__c                 C   s   t | j|gddS )NFcache)nextr  rI   r  r,   r,   r-   rY   ?     zRepositoryNoCache.getTc                 c   s0    t || j|D ]\}}| ||V  q
d S r   )ziprx   r  r  )rI   keysr  r  r(   r,   r,   r-   r  B  s   zRepositoryNoCache.get_manyc                 C   r  r   r,   r  r,   r,   r-   log_instrumentationF  rD  z%RepositoryNoCache.log_instrumentationr   r  )r2   r3   r4   r5   rH   r   r   rH  rY   r  r  r,   r,   r,   r-   r  )  s    

r  c                       s\   e Zd ZdZd fdd	Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dddZ  ZS )RepositoryCacheaf  
    A caching Repository wrapper.

    Caches Repository GET operations locally.

    *pack* and *unpack* complement *transform* of the base class.
    *pack* receives the output of *transform* and should return bytes,
    which are stored in the cache. *unpack* receives these bytes and
    should return the initial data (as returned by *transform*).
    Nc                    sv   t  || |pdd | _|pdd | _t | _tjdd| _| 	  d| _
d| _d| _d| _d| _d| _d| _d S )Nc                 S   rC  r   r,   r(   r,   r,   r-   r  X  r  z*RepositoryCache.__init__.<locals>.<lambda>c                 S   rC  r   r,   r  r,   r,   r-   r  Y  r  zborg-cache-)prefixr   g        )rG   rH   packunpackr   r  tempfilemkdtempbasedirquery_size_limitsizehitsmissesslow_missesslow_lat	evictionsenospc)rI   rx   r  r  r  rJ   r,   r-   rH   V  s   
zRepositoryCache.__init__c                 C   s&   t | jj}tt|d d| _d S )Ng      ?l        )shutil
disk_usager  freer   min
size_limit)rI   Zavailable_spacer,   r,   r-   r  f  s   z RepositoryCache.query_size_limitc                 C   s   t j| jt|S r   )r#   rc   rF   r  r
   r  r,   r,   r-   key_filenamej  r  zRepositoryCache.key_filenamec                 C   s   |    td| j }| j|kr>| jrB| j }| |}|  jt|j	8  _t
| |  jd7  _| j|kr@| jsd S d S d S d S )Ng?r   )r  r   r  r  r  rk  r  r#   statst_sizeunlinkr  )rI   Ztarget_sizer  filer,   r,   r-   backoffm  s   


 zRepositoryCache.backoffc           	      C   s  |  ||}|s
|S | |}| |}zt|d}|| W d    n1 s*w   Y  W n6 tyf } z*zt| W n	 tyG   Y nw |jtj	krZ|  j
d7  _
|   n W Y d }~|S d }~ww |  jt|7  _| j| | j| jkr|   |S )Nwbr   )r  r  r  rb   r$   rf  r   FileNotFoundErrorrg  ENOSPCr  r  r  r"   r  addr  )	rI   r  r(   r  transformedZpackedr  r'   Zos_errorr,   r,   r-   	add_entryw  s:   



zRepositoryCache.add_entryc                 C   s>   t dt| jt| jt| j| j| j| j	| j
| j| j
 d S )NzyRepositoryCache: current items %d, size %s / %s, %d hits, %d misses, %d slow misses (+%.1fs), %d evictions, %d ENOSPC hit)r   r   r"   r  r   r  r  r  r  r  r  r  r  r  r,   r,   r-   r    s
   z#RepositoryCache.log_instrumentationc                 C   s"   |    | j  t| j d S r   )r  r  clearr  rmtreer  r  r,   r,   r-   r     s   
zRepositoryCache.closeTc              	   #   s$    fdd|D }t | j|}|D ]u}| jv rF |}t|d}  jd7  _ | V  W d    n1 s@w   Y  q|D ]\}}	||krc 	||	|}
  j
d7  _
|
V   n'qHt } j|}	  jt | 7  _ 	||	|}
  jd7  _|
V  q|D ]}qd S )Nc                    s   g | ]	}| j vr|qS r,   r  )r=   r  r  r,   r-   r>     rq   z,RepositoryCache.get_many.<locals>.<listcomp>rbr   )r  rx   r  r  r  rb   r  r  r   r  r  r%   perf_counterrY   r  r  )rI   r  r  Zunknown_keysZrepository_iteratorr  r  r'   Zkey_r(   r  t0r   r,   r  r-   r    s8   

zRepositoryCache.get_many)NNNr  )r2   r3   r4   r5   rH   r  r  r  r  r  r   r  rL   r,   r,   rJ   r-   r  J  s    
r  F)decrypted_cacher  r  r  force_cachec                   s   |r|s|s|rt d|r-|td td fdd} fdd}fdd	}t| ts4|r;t| |||S t| |S )
a  
    Return a Repository(No)Cache for *repository*.

    If *decrypted_cache* is a key object, then get and get_many will return a tuple
    (csize, plaintext) instead of the actual data in the repository. The cache will
    store decrypted data, which increases CPU efficiency (by avoiding repeatedly decrypting
    and more importantly MAC and ID checking cached objects).
    Internally, objects are compressed with LZ4.
    z:decrypted_cache and pack/unpack/transform are incompatiblez=I8slz4c                    s&   | \}} |} |t|| S r   )compressr  r   )r(   csize	decrypted
compressedcache_struct
compressorr,   r-   r    s   
zcache_if_remote.<locals>.packc                    sP   t | }  | d  j \}}|  jd  }|t|kr!td||fS )Nz)detected corrupted data in metadata cache)
memoryviewr  r  r   r	   
decompress)r(   r  Zchecksumr  r  r,   r-   r    s   zcache_if_remote.<locals>.unpackc                    s   t |} | |}||fS r   )r"   Zdecrypt)Zid_r(   r  r  )r  r,   r-   r    s   z"cache_if_remote.<locals>.transform)rQ  structStructr   r   r  r  r  )rx   r  r  r  r  r  r,   )r  r  r  r-   cache_if_remote  s   


r  )Jrg  r  r   r   r   r#   r   r[  r  r  r   r  rD   r%   r   
subprocessr   r   r   r   r  r   	constantshelpersr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   rx   r   r=  r   r   Zalgorithms.checksumsr   Zhelpers.datastructr   r2   r   r   r   r   r   r   rv  r   r.   r0   r7   r8   r9   r:   r;   r   rr   rt   r   r  r  rz  r  r  r  r,   r,   r,   r-   <module>   s    "	
 y:    J!p