o
    +ke(                    @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d
dlmZmZmZmZ d
dlmZ e Zd
dlm Z  d
dl!m"Z"m#Z# d
dl$m%Z% d
dl&m'Z'm(Z( d
dl)m*Z*m+Z+ d
dl,T d
dl-m.Z/ d
dl0m1Z1m2Z2m3Z3 d
dl4m5Z5 d
dl4m6Z6 d
dl4m7Z7m8Z8m9Z9 d
dl4m:Z:m.Z.m;Z; d
dl<m=Z=m>Z>m?Z?m@Z@ d
dl4mAZAmBZB d
dl4mCZCmDZDmEZEmFZFmGZG d
dl4mHZHmIZImJZJmKZK d
dl4mLZL d
dl4mMZM d
dl4mNZN d
d l4mOZOmPZPmQZQ d
d!l4mRZRmSZSmTZT d
d"l4mUZU d
d#l4mVZV d
d$l4mWZW d
d%l4mXZX d
d&lYmZZZ d
d'l[m\Z\m]Z]m^Z^ d
d(l_m`Z`maZambZb d
d)l<mcZcmdZdmeZemfZfmgZgmhZh d
d*limjZj d
d+lkmlZlmmZm ened,ZoG d-d. d.Zpd/d0 ZqG d1d2 d2erZsG d3d4 d4erZtG d5d6 d6Zueu Zvd7d8 Zwd9d: Zxe	dddd;d<d=d>d?ZyG d@dA dAZzG dBdC dCZ{G dDdE dEe{Z|ddd d dFdGdHZ}G dIdJ dJZ~G dKdL dLZeZdMdNdO dPZdQdR ZG dSdT dTZG dUdV dVZG dWdX dXZdYdZ ZG d[d\ d\ZG d]d^ d^ZG d_d` d`ZdS )a    N)OrderedDict)contextmanager)timezone	timedelta)partial)getuser)BytesIO)groupbyzip_longest)get_terminal_size   )is_win32is_linux
is_freebsd	is_darwin)create_logger)xattr)get_chunkerChunkChunkListEntry)key_factoryUnsupportedPayloadError)
CompressorCompressionSpec)*)IntegrityError)
ChunkIndexChunkIndexEntryCacheSynchronizer)Manifesthardlinkable)ChunkIteratorFileWrappernormalize_chunker_params	open_item)Errorr   set_ec)uid2useruser2uid	gid2group	group2gid)parse_timestampto_localtime)OutputTimestampformat_timedeltaformat_file_sizefile_statusFileSize)safe_encodesafe_decodemake_path_saferemove_surrogates)
StableDict)
bin_to_hex)safe_ns)ellipsis_truncateProgressIndicatorPercent	log_multi)os_openflags_normal	flags_dir)os_stat)msgpack)sig_int)utcnow)LRUCache)PathPrefixPatternFnmatchPattern	IECommand)ItemArchiveItemItemDiff)acl_getacl_set	set_flags	get_flagsswidthhostname)cache_if_remote)
RepositoryLIST_SCAN_LIMITlinkc                   @   s   e Zd ZdddZdddZdd ZdZd	d
 Zdd Zdd Z	dd Z
edd Zedd Zedd Zedd ZdddZdS )
StatisticsFc                 C   sF   || _ || _d | _ | _ | _| _d | _ | _ | _| _	d| _
d S Nr   )output_jsoniecosizecsizeusizenfilesosize_partscsize_partsusize_partsnfiles_partslast_progress)selfrW   rX    rc   0usr/lib/python3.10/site-packages/borg/archive.py__init__:   s
   
zStatistics.__init__c                 C   sp   |s|  j |7  _ |  j|7  _|r|  j|7  _d S d S |  j|7  _|  j|7  _|r6|  j|7  _d S d S N)rY   rZ   r[   r]   r^   r_   )rb   sizerZ   uniquepartrc   rc   rd   updateA   s   zStatistics.updatec                 C   s   t |ts	tdt| j| j}| j|j |_| j|j |_| j|j |_| j|j |_| j	|j	 |_	| j
|j
 |_
| j|j |_| j|j |_|S )Nzcan only add Statistics objects)
isinstancerU   	TypeErrorrW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   )rb   otherstatsrc   rc   rd   __add__M   s   
zStatistics.__add__zO{label:15} {stats.osize_fmt:>20s} {stats.csize_fmt:>20s} {stats.usize_fmt:>20s}c                 C   s   | j j| ddS )NzThis archive:)rn   label)summaryformatrb   rc   rc   rd   __str__]      zStatistics.__str__c                 C   s   dj t| jt| | dS )NzF<{cls} object at {hash:#x} ({self.osize}, {self.csize}, {self.usize})>)clshashrb   )rr   type__name__idrs   rc   rc   rd   __repr__`   s   zStatistics.__repr__c                 C   s4   t | j| jdt | j| jdt | j| jd| jdS )NrX   )Zoriginal_sizeZcompressed_sizeZdeduplicated_sizer\   )r2   rY   rX   rZ   r[   r\   rs   rc   rc   rd   as_dictd   s
   zStatistics.as_dictc                 C   s   | j | j| j| j| j| jdS )Nrg   rZ   r\   
size_partsr^   r`   rY   rZ   r\   r]   r^   r`   rs   rc   rc   rd   as_raw_dictl   s   zStatistics.as_raw_dictc                 K   sF   |  }|d |_ |d |_|d |_|d |_|d |_|d |_|S )Nrg   rZ   r\   r   r^   r`   r   )rv   kwrb   rc   rc   rd   from_raw_dictv   s   





zStatistics.from_raw_dictc                 C      t | j| jdS Nr|   )r0   rY   rX   rs   rc   rc   rd   	osize_fmt      zStatistics.osize_fmtc                 C   r   r   )r0   r[   rX   rs   rc   rc   rd   	usize_fmt   r   zStatistics.usize_fmtc                 C   r   r   )r0   rZ   rX   rs   rc   rc   rd   	csize_fmt   r   zStatistics.csize_fmtNc                 C   s  t  }|d u s|| j |kr|| _| jr<|s'|  }t|r!|jnd|d< ni }|t   d|d t	|}d}n9t
 \}	}
|sod| }|rOt|jnd}|	t| }|dk rcd}|	t| }|dkrn|t||7 }nd	|	 }d
}t|||p|tjdd d S d S )N pathZarchive_progress)timerx   finished
z={0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N        T)endfileflush)r   	monotonicra   rW   r}   r6   r   rj   jsondumpsr   rr   rO   r:   printsysstderr)rb   itemfinalstreamdtnowdatamsgr   columnslinesr   spacerc   rc   rd   show_progress   s<   


zStatistics.show_progressFFF)NFNN)ry   
__module____qualname__re   rj   ro   rq   rt   r{   r}   r   classmethodr   propertyr   r   r   r   rc   rc   rc   rd   rU   8   s$    







rU   c                 C   s   t | pt | pt | S rf   )statS_ISBLKS_ISCHRS_ISFIFO)moderc   rc   rd   
is_special   s   r   c                   @      e Zd ZdZdS )BackupErrorzY
    Exception raised for non-OSError-based exceptions while accessing backup files.
    Nry   r   r   __doc__rc   rc   rc   rd   r          r   c                   @   s    e Zd ZdZdd Zdd ZdS )BackupOSErrora  
    Wrapper for OSError raised while accessing backup files.

    Borg does different kinds of IO, and IO failures have different consequences.
    This wrapper represents failures of input file or extraction IO.
    These are non-critical and are only reported (exit code = 1, warning).

    Any unwrapped IO error is critical and aborts execution (for example repository IO failure).
    c                 C   s(   || _ || _|j| _|j| _|j| _d S rf   )opos_errorerrnostrerrorfilename)rb   r   r   rc   rc   rd   re      s
   zBackupOSError.__init__c                 C   s"   | j r| j  d| j S t| jS )N: )r   r   strrs   rc   rc   rd   rt      s   
zBackupOSError.__str__N)ry   r   r   r   re   rt   rc   rc   rc   rd   r      s    	r   c                   @   s*   e Zd ZdZd	ddZdd Zdd ZdS )
BackupIOr   c                 C   s
   || _ | S rf   )r   )rb   r   rc   rc   rd   __call__   s   zBackupIO.__call__c                 C      d S rf   rc   rs   rc   rc   rd   	__enter__      zBackupIO.__enter__c                 C   s$   |rt |trt| j||d S d S rf   )
issubclassOSErrorr   r   )rb   exc_typeZexc_valZexc_tbrc   rc   rd   __exit__   s   zBackupIO.__exit__N)r   )ry   r   r   r   r   r   r   rc   rc   rc   rd   r      s
    
r   c              	   c   sd    dt _	 t  zt| }W n ty   Y W d    d S w W d    n1 s)w   Y  |V  q)Nread)	backup_ior   nextStopIteration)iteratorr   rc   rc   rd   backup_io_iter   s   r   c                 C   s8   t | jt |jkrtd| j|jkrtd|S )aX  
    this checks for some race conditions between the first filename-based stat()
    we did before dispatching to the (hopefully correct) file type backup handler
    and the (hopefully) fd-based fstat() we did in the handler.

    if there is a problematic difference (e.g. file type changed), we rather
    skip the file than being tricked into a security problem.

    such races should only happen if:
    - we are backing up a live filesystem (no snapshot, not inactive)
    - if files change due to normal fs activity at an unfortunate time
    - if somebody is doing an attack against us
    z1file type changed (race condition), skipping filez2file inode changed (race condition), skipping file)r   S_IFMTst_moder   st_ino)Zst_oldZst_currrc   rc   rd   stat_update_check   s
   r   Fopen)r   	parent_fdnamenoatimer   c              	   c   sv    t | t|||| |d}W d    n1 sw   Y  z|V  W |d ur.t| d S d S |d ur:t| w w )Nr   r   r   flagsr   )r   r=   osclose)r   r   r   r   r   r   fdrc   rc   rd   OsOpen	  s   
r   c                   @   s(   e Zd Zdd Zd	ddZd
ddZdS )DownloadPipelinec                 C   s   || _ || _d S rf   )
repositorykey)rb   r   r   rc   rc   rd   re        
zDownloadPipeline.__init__NFc                 #   s@   fdd}t  }tjdd}|D ]}	||	 dd |D }
|
D ]}d|v r4dd |jD |_q% r@ fd	d|
D }
|r r|r|
D ]=}t|jr|d
}|du rnd|v ra||j |ddrm|	|j
 qH||vr|| \}}|dur|| |	| qHn|
D ]}d|v r||j q|
D ]}|V  qqdS )a  
        Return iterator of items.

        *ids* is a chunk ID list of an item stream. *filter* is a callable
        to decide whether an item will be yielded. *preload* preloads the data chunks of every yielded item.

        Warning: if *preload* is True then all data chunks of every yielded item have to be retrieved,
        otherwise preloaded chunks will accumulate in RemoteRepository and create a memory leak.
        c                    s    j dd | D  d S )Nc                 S      g | ]}|j qS rc   rz   .0crc   rc   rd   
<listcomp>&      zBDownloadPipeline.unpack_many.<locals>._preload.<locals>.<listcomp>)r   preload)chunksrs   rc   rd   _preload%  s   z.DownloadPipeline.unpack_many.<locals>._preloadFZuse_listc                 S   s   g | ]}t |d qS )Zinternal_dict)rH   r   r   rc   rc   rd   r   ,      z0DownloadPipeline.unpack_many.<locals>.<listcomp>r   c                 S   s   g | ]}t | qS rc   r   )r   erc   rc   rd   r   /      c                    s   g | ]} |r|qS rc   rc   r   filterrc   rd   r   2  s    sourceNhardlink_masterT)setrA   Unpacker
fetch_manyfeedr   r"   r   getaddr   )rb   idsr   partial_extractr   hardlink_mastersr   Zmasters_preloadedunpackerr   itemsr   r   r   _rc   r   rb   rd   unpack_many  sN   






zDownloadPipeline.unpack_manyc                 c   s6    t || jj||dD ]\}}| j||V  qd S )Nis_preloaded)zipr   get_manyr   decrypt)rb   r   r  id_r   rc   rc   rd   r   T  s   zDownloadPipeline.fetch_manyNFFNr   )ry   r   r   re   r  r   rc   rc   rc   rd   r     s    
9r   c                   @   s>   e Zd ZdZefddZdd Zdd Zdd	d
Zdd Z	dS )ChunkBufferi   c                 C   s8   t  | _t | _g | _|| _t|| jjdd| _	d S )NFseedsparse)
r   bufferrA   ZPackerpackerr   r   r   
chunk_seedchunker)rb   r   chunker_paramsrc   rc   rd   re   \  s
   
zChunkBuffer.__init__c                 C   s0   | j | j|  |  r|   d S d S rf   )r  writer  packr}   is_fullr   )rb   r   rc   rc   rd   r   c  s   zChunkBuffer.addc                 C   s   t rf   )NotImplementedError)rb   chunkrc   rc   rd   write_chunkh  r   zChunkBuffer.write_chunkFc                 C   s   | j  dkr	d S | j d g }| j| j D ],}|jd }|tkr)t|j}n|t	t
fv r9td |jd  }ntd| || q| j d | j d |sYt|dkr[d nd}|d | D ]}| j| | qc|dkr}| j |d  d S d S )Nr   
allocationrg   z,chunk allocation has unsupported value of %rr   )r  tellseekr  chunkifymetaCH_DATAbytesr   CH_ALLOCCH_HOLEzeros
ValueErrorappendtruncatelenr   r  r  )rb   r   r   r  Zallocr   r   rc   rc   rd   r   k  s(   
zChunkBuffer.flushc                 C   s   | j  | jkS rf   )r  r  BUFFER_SIZErs   rc   rc   rd   r    ru   zChunkBuffer.is_fullNr   )
ry   r   r   r(  ITEMS_CHUNKER_PARAMSre   r   r  r   r  rc   rc   rc   rd   r
  Y  s    
r
  c                       s(   e Zd Zef fdd	Zdd Z  ZS )CacheChunkBufferc                    s   t  || || _|| _d S rf   )superre   cachern   )rb   r,  r   rn   r  	__class__rc   rd   re     s   
zCacheChunkBuffer.__init__c                 C   s8   | j j| j||| jdd\}}}| j jjdd |S NFwait)r,  	add_chunkr   id_hashrn   r   async_response)rb   r  r  r   rc   rc   rd   r    s   $zCacheChunkBuffer.write_chunk)ry   r   r   r)  re   r  __classcell__rc   rc   r-  rd   r*    s    r*  )
uid_forced
gid_forceduid_defaultgid_defaultc                C   s   |d ur|}n|rd nt | j}|d u r| jn|}|dk r|}|d ur)|}||fS |r-d nt| j}|d u r9| jn|}|dk rA|}||fS rV   )r)   useruidr+   groupgid)r   numericr6  r7  r8  r9  r;  r=  rc   rc   rd   get_item_uid_gid  s   r?  c                   @   s\  e Zd ZG dd deZG dd deZG dd deZddd	dddddddeddddddfd
dZdd Z	dd Z
edd Zedd Zedd Zedd Zedd Zdd Zdd Zdd Zd>d d!Zd?d"d#Zd@d%d&Zd'd( ZdAd)d*ZdBd+d,ZdBd-d.Zed/d0 Z		dCd2d3ZdDd4d5Zd6d7 Zd8d9 Z dEd:d;Z!e"dFd<d=Z#dS )GArchivec                   @   r   )zArchive.DoesNotExistzArchive {} does not existNr   rc   rc   rc   rd   DoesNotExist  r   rA  c                   @   r   )zArchive.AlreadyExistszArchive {} already existsNr   rc   rc   rc   rd   AlreadyExists  r   rB  c                   @   r   )z+Archive.IncompatibleFilesystemEncodingErrorzrFailed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable.Nr   rc   rc   rc   rd   #IncompatibleFilesystemEncodingError  r   rC  NF  c                 C   s  t  | _|| _|| _|| _|| _i | _t||d| _	|| _
|| _|| _|| _d | _d| _|| _|| _|	| _|
| _|| _|| _|| _|d u |d u ksNJ d|d u rYt }t }|| _|| _|| _|d u rit }|| _|| _t| j| j| _ || _!| j!rt"| j| j| j	| _#||j$v r| %|d}	 d&||rd| pd| _'| j'|j$vrd S |d	7 }q| jj$(|}|d u r| )|| *|j+ d S )
NrW   rX   FzULogic error: if start is given, start_monotonic must be given as well and vice versa.r   Tz{}.checkpoint{}z.%dr   r   ),r   getcwdcwdr   r   r,  manifest
hard_linksrU   rn   rX   r   r   Zname_in_manifestcommenttam_verifiedcheckpoint_intervalnumeric_idsr   noctimenoflagsnoaclsnoxattrsrC   r   r   r  startstart_monotonicr   consider_part_filesr   pipelinecreater*  items_bufferarchivesrB  rr   checkpoint_namer   rA  loadrz   )rb   r   r   rH  r   r,  rV  rL  rM  r   rN  rO  rP  rQ  progressr  rR  rS  r   rT  log_jsonrX   iinforc   rc   rd   re     s`   



zArchive.__init__c                 C   sN   | j || j|}| j j|dd\}| _}t|d}|jdkr%td|S )NTZforce_tam_not_requiredr   r    Unknown archive metadata version)	r   r  r   r   unpack_and_verify_archiverK  rI   version	Exception)rb   rz   r   archiver   metadatarc   rc   rd   
_load_meta  s   

zArchive._load_metac                 C   sH   || _ | | j | _dd | jjD | j_| jj| _| jdd| _d S )Nc                 S      g | ]}t |qS rc   r4   r   argrc   rc   rd   r     r   z Archive.load.<locals>.<listcomp>rJ  r   )rz   rf  re  cmdliner   r   rJ  )rb   rz   rc   rc   rd   rZ    s
   
zArchive.loadc                 C   s   | j j}t|S )z,Timestamp of archive creation (start) in UTC)re  r   r,   rb   tsrc   rc   rd   rm    s   z
Archive.tsc                 C   s   | j dp	| j j}t|S )z*Timestamp of archive creation (end) in UTCtime_end)re  r   r   r,   rl  rc   rc   rd   ts_end   s   zArchive.ts_endc                 C   s
   t | jS rf   )r8   rz   rs   rc   rc   rd   fpr     
zArchive.fprc                 C      t | j| j S rf   )r/   r   rR  rs   rc   rc   rd   duration  r   zArchive.durationc                 C   rr  rf   )r/   ro  rm  rs   rc   rc   rd   duration_from_meta  r   zArchive.duration_from_metac              	   C   s   | j r| j}| jjtjd}| jjtjd}n| | j}| j	}| j
}| j| jt|t|||  | d| jj| j jt id}| j rMtj|d< |S | jd}|d ur[t|nd}|| jj| jj| jj| jdd|d |S )	NtzinfoZmax_archive_size)r   rz   rR  r   rs  rn   Zlimitscommand_liner  r   rJ  )rw  rP   usernamerJ  r  )rV  rn   rR  replacer   utcr   
calc_statsr,  rm  ro  r   rp  r.   total_secondsr}   r   rz   rZ   MAX_DATA_SIZEr   argvre  r   r$   rj   rk  rP   rx  )rb   rn   rR  r   r^  cprc   rc   rd   r^    s:   

zArchive.infoc                 C   sL   dj | t| jjtjdt| jjtjd| jj| j	 j
t | jj dS )NzRepository: {location}
Archive name: {0.name}
Archive fingerprint: {0.fpr}
Time (start): {start}
Time (end):   {end}
Duration: {0.duration}
Number of files: {0.stats.nfiles}
Utilization of max. archive size: {csize_max:.0%}
ru  )rR  r   Z	csize_maxlocation)rr   r.   rR  ry  r   rz  r   r,  r   rz   rZ   r}  r   Z	_locationZcanonical_pathrs   rc   rc   rd   rt   5  s   	
zArchive.__str__c                 C   s
   d| j  S )NzArchive(%r))r   rs   rc   rc   rd   r{   G  s   
zArchive.__repr__c                 C   s"   | j s	d|v r	dS |r||S dS )Nri   FT)rT  )rb   r   r   rc   rc   rd   item_filterJ  s   zArchive.item_filterc                 #   sN     r|r|r|d usJ j jjj||| fdddD ]}|V  qd S )Nc                    s    |  S rf   )r  r   r  rc   rd   <lambda>V      z$Archive.iter_items.<locals>.<lambda>)r   r   r   r   )rU  r  re  r   )rb   r   r   r   r   r   rc   r  rd   
iter_itemsP  s   
zArchive.iter_itemsTc                 C   s6   |r| j r|d u r| j}|j |dd | j| d S )N皙?r   r   )r   rn   rW  r   )rb   r   r   rn   rc   rc   rd   add_itemY  s
   
zArchive.add_itemc                 C   s.   |  | j | jj| j= | j| j| j d S rf   )saverY  rH  rX  r,  chunk_decrefrz   rn   rs   rc   rc   rd   write_checkpoint`  s   zArchive.write_checkpointc                 C   s  |p| j }|| jjv r| || jjdd tt | j	 d}|d u r-t
 }|| }n|| }|}|| _|| _d||p>d| jjtjtt |t|t| jd
}	|pW| j}|	|j|j|j|j|j|jd |	|poi  t|	}	| jj|	  dd	}
| j!|
| _"z| j#$| j"|
| j W n t%y } zt&|}d
|v rt'd|  d }~ww | j(j)ddd ur	 | j(j)ddd us| j"|	jf| jj|< | j*  | j(j+dd | j#+  d S )NTr   )secondsr   r   )
rb  r   rJ  r   rk  rP   rx  r   rn  r  r~      archivecontextzMore than allowed put dataz#%s - archive too big (issue #1473)!r0  F)compact),r   rH  rX  rB  rW  r   r   r   r   rS  rC   rR  r   r   r   r~  rP   r   strftimeZ
ISO_FORMATr  rn   rj   rY   rZ   r\   r]   r^   r`   rI   r   pack_and_authenticate_metadatar}   r3  rz   r,  r2  r   r   r&   r   r4  r  commit)rb   r   rJ  	timestamprn   additional_metadatars  r   rR  re  r   errerr_msgrc   rc   rd   r  e  sh   




zArchive.savec                 C   st   | j dd u}z|rttjdi |j| j }W |S  ty9   | j||d}|s6| |j| j< Y |S Y |S w )Nr\   )want_uniquerc   )	re  r   KeyErrorrU   r   Z
pre12_metarp  _calc_statsr   )rb   r,  r  have_borg12_metarn   rc   rc   rd   r{    s   zArchive.calc_statsc                    s  | j dd u}|r|sd}n[ fdd}t  t }|| j | jdd}tt| j j	d| dd	}t
| j j	| j| j j	D ]\}	}
|jd
d ||	 | j|	|
}|| qC jd }|  t| jd}||_|s| jr|j|_|j|_|j|_|S |j|j |_|j|j |_|j|j  |_|S | jr| j j!| j j |_| j j| j j" |_| j j | j j |_|S | j j|_| j j"|_| j j|_|S )Nr\   r   c                    s"   j |  } | d|j|j d S )Nr   )r   r   rg   rZ   )rz   entryZarchive_indexr,  rc   rd   r     s   
z Archive._calc_stats.<locals>.add%z%%z4Calculating statistics for archive %s ... %%3.0f%%%%zarchive.calc_statstotalr   msgidr   )increase   r|   )#re  r   r   r   rz   r   ry  r;   r'  r   r  r   r  showr   r  r   Zstats_againstr   finishrU   rX   r[   rT  Znum_files_totalsr\   Zsize_totalsrY   Zcsize_totalsrZ   Znum_files_partsr   r^   r`   rg   )rb   r,  r  r  Zunique_csizer   syncZarch_name_escdpirz   r  r   rn   rc   r  rd   r    sP   
"



zArchive._calc_statsc                 c   s    d}d|v rMt jj|g|jt j|d  R  }||jd |f\}	}
|
rFtrFtd t 	|
| d}W d    n1 s@w   Y  n|	d urM|	|_
|V  |sf|rhtrcd |f||dp`|< d S 	 d S d S d S )NFr   rT   T)r   r   joinr   splitsepr   has_linkr   rT   r   )rb   destr   r   stripped_componentsoriginal_pathr   hardlink_setr   r   Zlink_targetrc   rc   rd   extract_helper  s&   &
zArchive.extract_helperr   c
                 C   s  |pi }d|v }
|s|rdd|v r\d}| j jdd |jD ddD ] }|	r1|	jt|t|jgd |r:tjj	
| |t|7 }q |rItjj	  d	|v r\|j}||kr\td
|||
rbtddS |ph|j}| j}|jdrvtdtj||j}ztj|dd}t|jrt| nt| W n ty   | |t d ty   Y nw dd }|j}t|rtd || W d   n1 sw   Y  |  ||||||}|r	 W d   dS td t!|d}W d   n	1 sw   Y  | dd |jD }| j j|ddD ]?}|	r+|	jt|t|jgd td |rBt"|rB|#t|d n|
| W d   n	1 sRw   Y  qtd! |$  }}|%| |  | j&|||' d W d   n	1 sw   Y  W d   n	1 sw   Y  d	|v r|j}||krtd
|||
rtdW d   dS 1 sw   Y  dS t t|r|| tj(|st)| |r| &|| nt*|r|| |j+}zt,|| W n ty   | |t dw | j&||dd nt-|r]|| |  ||||||%}|rA	 W d   W d   dS t.| | &|| W d   n	1 sWw   Y  ndt/|sit0|r|| |  ||||||)}|r	 W d   W d   dS t1||j|j2 | &|| W d   n	1 sw   Y  ntd|j W d   dS W d   dS W d   dS W d   dS 1 sw   Y  dS )a  
        Extract archive item.

        :param item: the item to extract
        :param restore_attrs: restore file attributes
        :param dry_run: do not write any data
        :param stdout: write extracted data to stdout
        :param sparse: write sparse files (chunk-granularity, independent of the original being sparse)
        :param hardlink_masters: maps paths to (chunks, link_target) for extracting subtrees with hardlinks correctly
        :param stripped_components: stripped leading path components to correct hard link extraction
        :param original_path: 'path' key as stored in archive
        :param pi: ProgressIndicatorPercent (or similar) for file extraction progress (in bytes)
        chunks_healthyr   r   c                 S   r   rc   r   r   rc   rc   rd   r     r   z(Archive.extract_item.<locals>.<listcomp>Tr  )r  r^  rg   z4Size inconsistency detected: size {}, chunks size {}zDFile has damaged (all-zero) chunks. Try running borg check --repair.N)/z../z!Path should be relative and localFfollow_symlinksc                 S   s*   t j| }t j|st | d S d S rf   )r   r   dirnameexistsmakedirs)r   
parent_dirrc   rc   rd   make_parent/  s   z)Archive.extract_item.<locals>.make_parentr  r   wbc                 S   r   rc   r   r   rc   rc   rd   r   ?  r   r  r   Ztruncate_and_attrsr   )symlinkzUnknown archive item type %r)3rU  r   r   r  r'  r6   r   r   stdoutr  r  r   rg   r   rr   rG  
startswithrc  r   r  r   S_ISDIRr   rmdirunlinkUnicodeEncodeErrorrC  getfilesystemencodingr   r   S_ISREGr   r  r   r#  r  r  r&  restore_attrsfilenor  mkdirS_ISLNKr   r  r   mkfifor   r   mknodrdev)rb   r   r  dry_runr  r  r   r  r  r  Zhas_damaged_chunksitem_chunks_sizer   	item_sizer  r   str  r   r  r   r   posr   rc   rc   rd   extract_item  s   













 $ zArchive.extract_itemc                 C   s  dt _t|| jd\}}tsz|rt||| n	tj|||dd W n	 ty,   Y nw |r7t	||j
 nztj||j
dd W n tyT   |sRt||j
 Y nw | jsat||| j|d | jsxtj|pi||di dd}|rxtt |j}d|v r|j}	n|}	d|v r|j}
z|rtj|d	|	|
fd
 ntj|d	|	|
fdd W n	 ty   Y nw z|rtj|d	|	|fd
 ntj|d	|	|fdd W n	 ty   Y nw | jsd|v rzt||j|d W d	S  ty   Y d	S w d	S d	S d	S )zv
        Restore filesystem attributes on *path* (*fd*) from *item*.

        Does not access the repository.
        attrs)r>  Fr  r  xattrsatime	birthtimeN)ns)r  r  bsdflags)r   r   r?  rM  r   r   fchownchownr   fchmodr   chmodr  rP  rL   rQ  r   Zset_allr   r'   ZEXIT_WARNINGmtimer  r  utimerO  rM   r  )rb   r   r   r  r   r;  r=  warningr  r  r  rc   rc   rd   r  y  sp   :zArchive.restore_attrsc                 C   sz   |  | j}t||| | jj| dd}| j|}| j||| j	 ||j
f| jj| j< | j| j| j	 || _d S )Nr  r  )rf  rz   setattrr   r  r}   r3  r,  r2  rn   r   rH  rX  r   r  )rb   r   valuere  r   Znew_idrc   rc   rd   set_meta  s   
zArchive.set_metac                 C   s<   || j jv r| || j}|| _| d| | j j|= d S )Nr   )rH  rX  rB  r   r  )rb   r   oldnamerc   rc   rd   rename  s   
zArchive.renamec              
      s  G dd dt  t dfdd	d fdd	}dztjdd	}jj}tt|d
dd}tt	|j
|D ]X\}\}	}
|rL|| j|	|
}
||
 ||	| z(|D ]#}t|d}d|v rj ord|v }|jD ]\}}}||||d qvq`W q? ttfy   dkr dY q?w |r|  W n tjtjfy   dkr dY nw |j| jjj= ddd ur	 ddd usǈrtd td d S d S )Nc                   @   r   )z(Archive.delete.<locals>.ChunksIndexErrorzUChunk ID {} missing from chunks index, corrupted chunks index - aborting transaction.Nr   rc   rc   rc   rd   ChunksIndexError  r   r  Tc                    s8   zj j| dW S  tjy   dkr d  Y S w )Nr0  r   T)r   r4  rR   ObjectNotFoundr0  )errorexception_ignoredforcedrb   rc   rd   fetch_async_response  s   z,Archive.delete.<locals>.fetch_async_responseFc                    sF   zj j| |d|d W n ty   t| } |w dd d S )NF)r1  ri   r0  )r,  r  r  r8   )rz   rn   ri   cid)r  r  rb   rc   rd   r    s   z$Archive.delete.<locals>.chunk_decrefr   zDecrementing references %3.0f%%zarchive.deleter  r   r   ri   )ri   r   r0  zAforced deletion succeeded, but the deleted archive was corrupted.z2borg check --repair is required to free all space.Tr   )r&   objectrA   r   re  r   r;   r'  	enumerater  r   r  r  r   r  r   rH   rT  r   rl   r$  r  UnpackExceptionrR   r  rz   rH  rX  r   loggerr  )rb   rn   r[  r  r  r   Z	items_idsr  r]  Zitems_idr   r   ri   chunk_idrg   rZ   rc   )r  r  r  r  r  rb   rd   delete  s\   	"




zArchive.deletec                 #   s   fdddd fdd
dd  fd	d
}
fdd}t  }t  }g i t 	fdd	fddD ][\}	}
|	rc|
rc|	j|
jkrc||	|
sb|	j||	|
fV  qF|	r||	jd}|r}||	|s||	j||	|fV  n|	||	j< |
r||
jd}|r|||
s|j|||
fV  qF|
||
j< qF| D ]}|j}t|}
|| ||||fV  q| D ]}|j}t|}
|| ||||fV  qD ]#\}	}
|	sJ |
sJ |	j|
jksJ d|	j||	|
fV  qdS )a%  
        Yields tuples with a path and an ItemDiff instance describing changes/indicating equality.

        :param matcher: PatternMatcher class to restrict results to only matching paths.
        :param can_compare_chunk_ids: Whether --chunker-params are the same for both archives.
        c                    s   d| vpt | j p| j v S Nr   )r"   r   r   r  )r   rc   rd   hardlink_master_seen#     z;Archive.compare_archives_iter.<locals>.hardlink_master_seenc                 S   s   |  ddod| vot| jS Nr   Tr   )r   r"   r   r  rc   rc   rd   is_hardlink_master&  r  z9Archive.compare_archives_iter.<locals>.is_hardlink_masterc                    s&   | s|r| |f | j < d S d S rf   r   item1item2)r   r  rc   rd   update_hardlink_masters)  s   z>Archive.compare_archives_iter.<locals>.update_hardlink_mastersc                 S   s   t | jo| d|v S r  r"   r   r   )r   r   rc   rc   rd   has_hardlink_master-  s   z:Archive.compare_archives_iter.<locals>.has_hardlink_masterc                    sx   | r| j  d } |r|j  d }t| | jdd | dg D jdd |dg D dS )Nr   r   c                 S   r   rc   r   r   rc   rc   rd   r   6  r   zHArchive.compare_archives_iter.<locals>.compare_items.<locals>.<listcomp>r   c                 S   r   rc   r   r   rc   rc   rd   r   7  r   )can_compare_chunk_idscontent_only)r   rJ   rU  r   r   r  )archive1archive2r  r   r   r  rc   rd   compare_items0  s   

z4Archive.compare_archives_iter.<locals>.compare_itemsc                    s4   | | |  p| }|r  | |f |S )zQAdds item tuple to deferred if necessary and returns True, if items were deferred)r%  )r  r  Zdefer)deferredr  r  rc   rd   defer_if_necessary;  s
   
z9Archive.compare_archives_iter.<locals>.defer_if_necessaryc                         | jS rf   matchr   r  matcherrc   rd   r  I  r  z/Archive.compare_archives_iter.<locals>.<lambda>c                    r  rf   r  r  r	  rc   rd   r  J  r  Nz#Deferred items have different paths)r   r
   r  r   popvaluesrH   Zcreate_deleted)r  r  r
  r  r   r  r  Zorphans_archive1Zorphans_archive2r  r  Zmatching_orphanZaddedr   Zdeleted_itemZdeletedrc   )r  r  r  r   r  r  r   r  r  r
  r  rd   compare_archives_iter  sd   	








zArchive.compare_archives_iterrf   r	  )TN)NNNNNr  )TFFFNr   NNFNr   )NFF)$ry   r   r   r&   rA  rB  rC  CHUNKER_PARAMSre   rf  rZ  r   rm  ro  rp  rs  rt  r^  rt   r{   r  r  r  r  r  r{  r  r   r  r  r  r  r  r  staticmethodr  rc   rc   rc   rd   r@    sX    

6	




"


	

9
0

 
I

Fr@  c                   @   s0   e Zd Zdd Zdd Zd
ddZd
dd	ZdS )MetadataCollectorc                C   s.   || _ || _|| _|| _|| _|| _|| _d S rf   )r   rN  rM  rO  rP  rQ  nobirthtime)rb   r   rN  r  rM  rO  rP  rQ  rc   rc   rd   re   q  s   
zMetadataCollector.__init__c                 C   s   t |j|j|jt|jd}| jst|j|d< | js"t|j	|d< | j
s5t|dr5tt|jd |d< | jrBd  |d< |d< |S t|j|d< t|j|d< |S )	N)r   r;  r=  r  r  ctimest_birthtime ʚ;r  r:  r<  )dictr   st_uidst_gidr9   st_mtime_nsr   st_atime_nsrN  st_ctime_nsr  hasattrintr  rM  r(   r*   )rb   r  r  rc   rc   rd   stat_simple_attrsz  s$   	z#MetadataCollector.stat_simple_attrsNc                 C   s   i }| j s&td t|||d}W d    n1 sw   Y  |r&||d< | jsNtd tj|p3|dd}W d    n1 sAw   Y  |rNt||d< | jsptd t|||| j	|d W d    |S 1 skw   Y  |S )	Nzextended stat (flags)r  r  zextended stat (xattrs)Fr  r  zextended stat (ACLs))
rO  r   rN   rQ  r   get_allr7   rP  rK   rM  )rb   r  r   r   r  r   r  rc   rc   rd   stat_ext_attrs  s(   



z MetadataCollector.stat_ext_attrsc                 C   s$   |  |}|| j|||d |S )Nr  )r  rj   r   )rb   r  r   r   r  rc   rc   rd   
stat_attrs  s   
zMetadataCollector.stat_attrsrf   )ry   r   r   re   r  r   r!  rc   rc   rc   rd   r  p  s
    	
r  
   c                 C   r   rf   rc   )r   rc   rc   rd   r        r  )Zdisposec                 C   s   | j d }|tkr| j}||}||fS |ttfv rQ| j d }|ttks'J ttd | }zt||f }W ||fS  t	yP   ||}|t||f< Y ||fS w t
d)Nr  rg   zunexpected allocation type)r  r  r   r"  r!  r'  r#  
memoryviewzero_chunk_idsr  r$  )r  r3  r  r   r  rg   rc   rc   rd   cached_hash  s$   

r&  c                   @   s0   e Zd Zdd Zdd ZdddZdd	d
ZdS )ChunksProcessorc                C   s2   || _ || _|| _|| _|| _t | _|| _d S rf   )	r   r,  r  r  rL  r   r   last_checkpoint
rechunkify)rb   r   r,  r  r  rL  r)  rc   rc   rd   re     s   

zChunksProcessor.__init__c                 C   st   t | d}t|j}|j|d  |_|jddd | jd| 7  _||_|d7 }| j|dd |   ||fS )Nr   T)memorizefrom_chunksz.borg_part_%dr   F)r   )	rH   r}   r'  r   get_sizer   ri   r  r  )rb   r   
from_chunknumberlengthrc   rc   rd   write_part_file  s   
zChunksProcessor.write_part_fileFc                 C   sv   t ot  }|s|s| jr7t | j | jkr7|rtd | |||\}}t | _|r7t 	  td ||fS )Nz5checkpoint requested: starting checkpoint creation...z3checkpoint requested: finished checkpoint creation!)
rB   Zaction_triggeredrL  r   r   r(  r  r^  r0  Zaction_completed)rb   r   r-  part_numberr  Zsig_int_triggeredrc   rc   rd   maybe_checkpoint  s   


z ChunksProcessor.maybe_checkpointNc           
         s   |s
 fdd}g |_ jrd|v r|`d}d}|D ]}	|j ||	 |r/j|dd j|||dd	\}}q|dkrm|j |d  rQj|||d
d	\}}|j D ]}	 j|	j|	jd
d qT j	|d 7  _	d S d S )Nc                    s8   t | jj\}} j||dd}jjjdd |S r/  )r&  r   r3  r2  r,  r   r4  )r  r  r   chunk_entryr,  rb   rn   rc   rd   chunk_processor  s   z<ChunksProcessor.process_file_chunks.<locals>.chunk_processorr  r   r   r  r  F)r  T)rg   ri   )
r   r)  r  r%  r   r2  chunk_increfrz   rg   r`   )
rb   r   r,  rn   r   Z
chunk_iterr5  r-  r1  r  rc   r4  rd   process_file_chunks  s&   
z#ChunksProcessor.process_file_chunksr   rf   )ry   r   r   re   r0  r2  r7  rc   rc   rc   rd   r'    s
    
r'  c                   @   sf   e Zd ZddddZedddZdd	 Zd
d Zdd Zdd Z	dd Z
dd ZedddZdS )FilesystemObjectProcessorsNfile_status_printerc                C   sh   || _ || _|| _|| _|| _|| _|pdd | _i | _t|	|
d| _	t
 | _t||j|d| _d S )Nc                  W   r   rf   rc   argsrc   rc   rd   r    r#  z5FilesystemObjectProcessors.__init__.<locals>.<lambda>rE  r  )metadata_collectorr,  r   r  r7  r   print_file_statusrI  rU   rn   r   rF  rG  r   r  r  )rb   r=  r,  r   r  r7  r  r   r  r\  rX   r:  rc   rc   rd   re     s   
z#FilesystemObjectProcessors.__init__Tc           
      c   s    t |}t|d}d}|o|jdk}|r+| j|j|jf}	|	d ur)|	|_d}nd}||||fV  | j|| j	d |rG|| j|j|jf< d S d S )Nr  Fr   hTrn   )
r5   rH   st_nlinkrI  r   r   st_devr   r  rn   )
rb   r   r  statusr"   Z	safe_pathr   r   
hardlinkedr   rc   rc   rd   create_helper&  s    
z(FilesystemObjectProcessors.create_helperc                C   sZ   | j ||ddd\}}}}|| jj|||d |W  d    S 1 s&w   Y  d S )NdFr!   r  )rE  rj   r=  r!  )rb   r   r   r  r   rC  rD  r   rc   rc   rd   process_dir_with_fd:  s   $z.FilesystemObjectProcessors.process_dir_with_fdc          
   
   C   s   | j ||ddd\\}}}}t|||tddd=}	|	d ur9td t|t|	}W d    n1 s4w   Y  || jj	|||	d |W  d    W  d    S 1 sXw   Y  W d    d S 1 shw   Y  d S )	NrF  Fr!   TZdir_open)r   r   r   r   r   r   fstatr  )
rE  r   r?   r   r   r   rH  rj   r=  r!  
rb   r   r   r   r  r   rC  rD  r   r   rc   rc   rd   process_dir?  s   

"z&FilesystemObjectProcessors.process_dirc          
   
   C   s   |  ||d^\}}}}t|||tdd@}	td t|t|	}W d    n1 s-w   Y  |r6|r9||_|| j	j
|||	d |W  d    W  d    S 1 sXw   Y  W d    d S 1 shw   Y  d S )NfTr   rH  r  )rE  r   r>   r   r   r   rH  r   rj   r=  r!  rI  rc   rc   rd   process_fifoJ  s   
"z'FilesystemObjectProcessors.process_fifoc          
   
   C   s   |  |||B\}}}}	td t|t|||dd}W d    n1 s&w   Y  |j|_|r3|	r6||_|| j	|| |W  d    S 1 sLw   Y  d S )Nr   F)r   r   r   r  )
rE  r   r   r@   st_rdevr  r   rj   r=  r!  )
rb   r   r   r   r  Zdev_typer   rC  rD  r   rc   rc   rd   process_devT  s   
$z&FilesystemObjectProcessors.process_devc             	   C   s   | j ||dddB\}}}}|d ur|d ur|n|}	td tj|	|d}
W d    n1 s0w   Y  |
|_|| j|| |W  d    S 1 sNw   Y  d S )NsFr!   readlink)dir_fd)rE  r   r   rP  r   rj   r=  r!  )rb   r   r   r   r  r   rC  rD  r   fnamer   rc   rc   rd   process_symlink_  s   
$z*FilesystemObjectProcessors.process_symlinkc                C   s   d}|  || d }t|}|d u rtd| t|}	|	d u r&td| tt d }
t||d@ dB |||	||
|
|
d	}| ||| j| j	t
| j| |jdd	 | j jd
7  _d|_| j|| jd |S )Nr]  zno such user: %szno such group: %sr  i  i   )	r   r   r;  r:  r=  r<  r  r  r  Tr*  r   Fr@  )r>  r)   r&   r+   r  r   rH   r7  rn   r   r   r  r  r,  r\   r   r  )rb   r   r,  r   r   r:  r<  rC  r;  r=  tr   rc   rc   rd   process_pipek  s.   
"z'FilesystemObjectProcessors.process_pipe)r   c                   s   ||d J\}}}	}
t||||dd*}td t|t|}W d    n1 s/w   Y  |j| t	|j
}|rNtjt|jB |_|	rS|
r|sottjj|}j|} |||\}}nd  }}d\}}d }|d ur|D ]} |sd} nq fdd|D }d}n|rdnd	}|| d }|	|_|d ur||_nftd
 | jjtjd | W d    n1 sw   Y  t rd}n"td t|}W d    n1 sw   Y  | o|j!|j!k}|rd}|s|s "|||dd |jD  j j#d7  _#|jj$|||d |j%dd |W  d    W  d    S 1 sEw   Y  W d    d S 1 sVw   Y  d S )NTr   rH  r  Mc                    s   g | ]	}  |jqS rc   )r6  rn   )r   r  r,  rb   rc   rd   r         z;FilesystemObjectProcessors.process_file.<locals>.<listcomp>UAr   FZfstat2Cc                 S   r   rc   r   r   rc   rc   rd   r     r   r   r  rT  )&rE  r   r   r   r   rH  rj   r=  r  r   r   r   S_IFREGS_IMODEr   r3   r   r  rG  r   r3  Zfile_known_and_unchangedZ
seen_chunkr>  r   r   r7  rn   r   r   r  r  r   r  Zmemorize_filer\   r   r,  )rb   r   r   r   r  r,  r   r   rC  rD  r   r   Zis_special_fileZhashed_pathZ	path_hashZknownr   r   r  Zchanged_while_backupZst2rc   rX  rd   process_file  sj   




&
$z'FilesystemObjectProcessors.process_fileNT)ry   r   r   re   r   rE  rG  rJ  rL  rN  rS  rV  r>   r_  rc   rc   rc   rd   r8    s    
r8  c                   @   sP   e Zd ZddddZedddZdd Zd	d
 Zdd Zdd Z	dd Z
dS )TarfileObjectProcessorsNr9  c       	   
      C   sR   || _ || _|| _|| _|| _|	pdd | _t||d| _t||j	dd| _
d S )Nc                  W   r   rf   rc   r;  rc   rc   rd   r    r#  z2TarfileObjectProcessors.__init__.<locals>.<lambda>rE  Fr  )r,  r   r  r7  r   r>  rU   rn   r   r  r  )
rb   r,  r   r  r7  r  r   r\  rX   r:  rc   rc   rd   re     s   z TarfileObjectProcessors.__init__c                 c   s`    t t|j|j|B |j|j|jpd |jpd tt	|j
d d}||fV  | j|| jd d S )Nr  )r   r   r;  r=  r:  r<  r  r@  )rH   r5   r   r   r;  r=  unamegnamer9   r  r  r  rn   rb   tarinforC  rx   r   rc   rc   rd   rE    s   
z%TarfileObjectProcessors.create_helperc                C   :   |  |||\}}|W  d    S 1 sw   Y  d S rf   rE  rd  rc   rc   rd   rJ       $z#TarfileObjectProcessors.process_dirc                C   rf  rf   rg  rd  rc   rc   rd   rL    rh  z$TarfileObjectProcessors.process_fifoc                C   sL   |  |||\}}t|j|j|_|W  d    S 1 sw   Y  d S rf   )rE  r   makedevdevmajordevminorr  rd  rc   rc   rd   rN    s   $z#TarfileObjectProcessors.process_devc                C   sB   |  |||\}}|j|_|W  d    S 1 sw   Y  d S rf   )rE  linknamer   rd  rc   rc   rd   process_link  s   $z$TarfileObjectProcessors.process_linkc                C   s   |  |||:\}}| ||j d }||}| || j| j| jt| j	
| |jdd | j jd7  _|W  d    S 1 sDw   Y  d S )NTrT  r   )rE  r>  r   extractfiler7  r,  rn   r   r   r  r  r,  r\   )rb   re  rC  rx   tarr   r   rc   rc   rd   r_    s   
$z$TarfileObjectProcessors.process_fileNN)ry   r   r   re   r   rE  rJ  rL  rN  rm  r_  rc   rc   rc   rd   ra    s    ra  c                    s   t | }|dkr
dS | d d@ dkrd}n| d dkrd}ndS ||kr&dS | | d@ d	kr/n	| | d
v r6ndS | |d  t fdd|D S )z1check if the data <d> looks like a msgpacked dictr   F      r      r        )         Nc                 3   s    | ]}  |V  qd S rf   )r  )r   patternZkey_serializedrc   rd   	<genexpr>  s    z'valid_msgpacked_dict.<locals>.<genexpr>)r'  any)rF  Zkeys_serializedZd_lenZoffsrc   rz  rd   valid_msgpacked_dict  s"   r}  c                       s@   e Zd ZdZ fddZdd Zdd Zdd	 Zd
d Z  Z	S )RobustUnpackerzCA restartable/robust version of the streaming msgpack unpacker
    c                    s>   t    dd |D | _|| _g | _d| _tjtd| _	d S )Nc                 S      g | ]	}t | qS rc   rA   Zpackbencoder   r   rc   rc   rd   r     rY  z+RobustUnpacker.__init__.<locals>.<listcomp>Fobject_hook)
r+  re   	item_keys	validator_buffered_data_resyncrA   r   r7   	_unpacker)rb   r  r  r-  rc   rd   re     s   
zRobustUnpacker.__init__c                 C   s   g | _ d| _d S r`  )r  r  rs   rc   rc   rd   resync$  r   zRobustUnpacker.resyncc                 C   s&   | j r| j| d S | j| d S rf   )r  r  r%  r  r   )rb   r   rc   rc   rd   r   (  s   zRobustUnpacker.feedc                 C   s   | S rf   rc   rs   rc   rc   rd   __iter__.  r   zRobustUnpacker.__iter__c              	   C   s   | j rUd| j}| j rS|stt|| js|dd  }q	tjtd| _	| j	
| zt| j	}W n tjtfy=   Y nw | |rHd| _ |S |dd  }| j sd S d S t| j	S )N    r   r  F)r  r  r  r   r}  r  rA   r   r7   r  r   r   r  r  )rb   r   r   rc   rc   rd   __next__1  s*   

zRobustUnpacker.__next__)
ry   r   r   r   re   r  r   r  r  r5  rc   rc   r-  rd   r~    s    r~  c                   @   s^   e Zd Zdd Z		dddZd	d
 Zdd Zdd Zdd ZdddZ	dd Z
dddZdS )ArchiveCheckerc                 C   s   d| _ t | _d S NF)error_foundr   possibly_supersededrs   rc   rc   rd   re   M  s   zArchiveChecker.__init__FNr   r   c
              
   C   sL  t d |du ot|||f | _|| _|| _|   | js&t d dS | 	|| _
|r2|   tj| jvrFt d d| _|  | _n7ztj|tjjf| j
d\| _}
W n% ty| } zt d| d| _| jtj= |  | _W Y d}~nd}~ww | j|||||d	 |   | j|	d
 | jrt d nt d | jp| j S )a  Perform a set of checks on 'repository'

        :param repair: enable repair mode, write updated or corrected data into repository
        :param archive: only check this archive
        :param first/last/sort_by: only check this number of first/last archives ordered by sort_by
        :param glob: only check archives matching this glob
        :param verify_data: integrity verification of data referenced by archives
        :param save_space: Repository.commit(save_space)
        z%Starting archive consistency check...NzJRepository contains no apparent data at all, cannot continue check/repair.FzRepository manifest not found!T)r   z$Repository manifest is corrupted: %s)rd  firstlastsort_byglob)
save_spacez3Archive consistency check complete, problems found.z6Archive consistency check complete, no problems found.)r  r^  r|  	check_allrepairr   init_chunksr   r  make_keyr   verify_datar    MANIFEST_IDr  rebuild_manifestrH  rZ  Z	OperationZCHECKIntegrityErrorBaserebuild_refcountsorphan_chunks_checkr  )rb   r   r  rd  r  r  r  r  r  r  r   excrc   rc   rd   checkQ  s>   


"

zArchiveChecker.checkc                 C   s`   t t| jd d| _d}	 | jjt|d}|sdS |d }tdddd}|D ]}|| j|< q'q)	z8Fetch a list of all object keys from repository
        g?)ZusableNTlimitmarkerr  r   refcountrg   rZ   )r   r'  r   r   listrS   r   )rb   r  resultZ
init_entryr  rc   rc   rd   r  |  s   zArchiveChecker.init_chunksc              	   C   s|   d}| j  D ]$\}}|d7 }|dkr n||}zt||W   S  ty+   Y qw |dkr6d}t|d| }t|)Nr   r   i  z*make_key: repository has no chunks at all!z4make_key: failed to create the key (tried %d chunks))r   	iteritemsr   r   r   r   )rb   r   attemptZchunkidr   cdatar   rc   rc   rd   r    s    
zArchiveChecker.make_keyc              
   C   s  t d t| j}d}d}g }t|dddd}d }	 | jjd|d	}|s&n|t|7 }|d
 }| j|}tt	|}	|	r|
  |	d
}
zt|}W n= tjtfy } z.d| _|d7 }t dt|
| t|trr||
 |	rtt	|	}| j|}W Y d }~n?d }~ww |
tjkrd n|
}z	| j|| W n' ty } zd| _|d7 }t dt|
| ||
 W Y d }~nd }~ww |	s>q|  ||krt d t d|| |r>| jr+t d |D ]?}z| j|}|tjkrd n|}| j|| W n ty    | j|= | j| t dt| Y qw t dt| qnt d |D ]}t dt| q2|rDt jnt j}|d|| d S )Nz5Starting cryptographic data integrity verification...r   zVerifying data %6.2f%%{Gz?zcheck.verify_datar  r   stepr  Td   r  r  r   zchunk %s: %szchunk %s, integrity error: %szGRepo/Chunks index object count vs. segment files object count mismatch.z:Repo/Chunks index: %d objects != segment files: %d objectszmFound defect chunks. They will be deleted now, so affected files can get repaired now and maybe healed later.zchunk %s deleted.z0chunk %s not deleted, did not consistently fail.z}Found defect chunks. With --repair, they would get deleted, so affected files could get repaired then and maybe healed later.zchunk %s is defect.z`Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.)r  r^  r'  r   r;   r   scanr  r  reversedr  r  r   rR   r  r  r  r  r8   rk   r%  r    r  r   r  r  r  r  r   r  debug)rb   Zchunks_count_indexZchunks_count_segmentserrorsZdefect_chunksr  r  Z	chunk_idsZchunk_data_iterZchunk_ids_revdr  Zencrypted_datar  Z	_chunk_idintegrity_errorZdefect_chunklogrc   rc   rd   r    s   




 


zArchiveChecker.verify_datac                    s  t dd tD   fdd}td t| j| j}dd tD }tt	| j
dd	d
d}| j
 D ]\}}|  | j|}z	| j||}W n tyd }	 ztd|	 d| _W Y d}	~	q2d}	~	ww t||skq2d|vssd|vrtq2zt|}
W n
 tjy   Y q2w ||
rz| jj|dd\}
}}W n+ ty } z|
dddd}td|| td d| _W Y d}~q2d}~ww t|
d}
|
j}td| ||jv rd}	 d||f }||jvrn|d7 }qtd|| |}||
jf|j|< q2|  td |S )zRebuild the manifest object if it is missing

        Iterates through all objects in the repository looking for archive metadata blocks.
        c                 s       | ]}|  V  qd S rf   r  r   r   rc   rc   rd   r{        z2ArchiveChecker.rebuild_manifest.<locals>.<genexpr>c                    s    t | tsdS t| } |S r  )rk   r  r   issubsetobjkeysZrequired_archive_keysrc   rd   valid_archive  s   

z6ArchiveChecker.rebuild_manifest.<locals>.valid_archivez9Rebuilding missing manifest, this might take some time...c                 S   r  rc   r  r  rc   rc   rd   r     rY  z3ArchiveChecker.rebuild_manifest.<locals>.<listcomp>zRebuilding manifest %6.2f%%r  zcheck.rebuild_manifestr  zSkipping corrupted chunk: %sTNs   cmdlines	   versionFr_  s   names	   <unknown>asciiry  3Archive TAM authentication issue for archive %s: %szMThis archive will *not* be added to the rebuilt manifest! It will be deleted.r   zFound archive %sr   z%s.%dz(Duplicate archive name %s, storing as %szManifest rebuild complete.)	frozensetZREQUIRED_ARCHIVE_KEYSr  r^  r    r   r   ZARCHIVE_KEYSr;   r'  r   r  r  r   r  r  r  r  r}  rA   Zunpackbr  ra  r   decoderI   r   rX  r  r   r  )rb   r  rH  Zarchive_keys_serializedr  r  r   r  r   r  rd  verifiedr  r   r]  new_namerc   r  rd   r    sr   






zArchiveChecker.rebuild_manifestc                    s  j tjd fdd fdd}d+fdd	  fdd	}fd
d}|du r||d}t|||frsjjj||||d}	|rP|	sPt	
d| |rat|	|k rat	
d|t|	 |rrt|	|k rrt	
d|t|	 n&jjj|d}	nz	jj| g}	W n ty   t	d| d_Y dS w t|	}
t|
dddd}tj1t|	D ]\}}|| t	d|j d|d  d|
 d |j}|j vrt	dt| d_jj|j= q| j|}z	j||}W n% ty } zt	dt|| d_jj|j= W Y d}~qd}~ww zjj|dd \}}}W n) tyU } zt	d!|j| t	d" d_jj|j= W Y d}~qd}~ww t|d#}|jdkretd$d%d& |j D |_ t!j}||_"||D ]}d'|v r||j| |#| qz|j$dd( |j%D ]}| q|j |_%jj&|' d)|d*}j(|}j)|} |t|t|| ||j*fjj|j< q|+  W d   dS 1 sw   Y  dS ),zRebuild object reference counts by walking the metadata

        Missing and/or incorrect data is repaired when detected
        Nc                    s0    j | tdddjdkr j|  d S d S rV   )r   r   r   r  r  r   )r  rs   rc   rd   mark_as_possibly_supersededE  s   zEArchiveChecker.rebuild_refcounts.<locals>.mark_as_possibly_supersededc                    s2   j | }j | } |t| t|| |S rf   )r   r3  encryptr'  )r  r  r  )add_referencerb   rc   rd   add_callbackI  s   z6ArchiveChecker.rebuild_refcounts.<locals>.add_callbackc                    sb   z	 j |  W d S  ty0   |d usJ td||d j | <  jr- j| | Y d S Y d S w )Nr   r  )r   Zincrefr  r   r  r   put)r  rg   rZ   r  rs   rc   rd   r  O  s   z7ArchiveChecker.rebuild_refcounts.<locals>.add_referencec                    sn  fdd}d}g }d}d|v }|j }|r|jn|}|r6t|t|kr6t|  d|j d |`d}|}t||D ]\}	}
|
\}}}|j vr|	|
krstd| |j||| t	| d	 _
}||\}}}} |||| nptd
| |j||| t	| |	\}}}|j v r ||| nNtd| |j||| t	| d	 _
}||\}}}} |||| n(|	|
krƈ ||| ntd| |j||| t	|  ||| |	d  ||||g ||7 }q;|r|s|j |_|r||krt|  d|j d |`||_ d|v r3|j}|jdd	d}||kr5td| |j|| dS dS dS )a  Verifies that all file chunks are present.

            Missing file chunks will be replaced with new chunks of the same length containing all zeros.
            If a previously missing file chunk re-appears, the replacement chunk is replaced by the correct one.
            c                    s@   t d t| d}t| jj\}} j|}t|}|| ||fS )N)r  rg   )r   r!  r&  r   r3  r  r'  )rg   r  r  r   r  rZ   rs   rc   rd   replacement_chunk^  s
   zWArchiveChecker.rebuild_refcounts.<locals>.verify_file_chunks.<locals>.replacement_chunkr   Fr  r   z*: Invalid chunks_healthy metadata removed!z^{}: {}: New missing file chunk detected (Byte {}-{}, Chunk {}). Replacing with all-zero chunk.Tz|{}: {}: Previously missing file chunk is still missing (Byte {}-{}, Chunk {}). It has an all-zero replacement chunk already.zm{}: {}: Missing all-zero replacement chunk detected (Byte {}-{}, Chunk {}). Generating new replacement chunk.zE{}: {}: Healed previously missing file chunk! (Byte {}-{}, Chunk {}).z,: Completely healed previously damaged file!rg   )
compressedr+  z<{}: {}: size inconsistency detected: size {}, chunks size {}N)r   r  r'  r  r  r   r  r  rr   r8   r  r^  r%  rg   r,  )archive_namer   r  offsetZ
chunk_listZchunks_replacedZhas_chunks_healthyZchunks_currentr  Zchunk_currentZchunk_healthyr  rg   rZ   r  r  r  )r  r  rb   rc   rd   verify_file_chunksX  s   










z<ArchiveChecker.rebuild_refcounts.<locals>.verify_file_chunksc                 3   s   t dd jjD t dd tD tdd jj}d  fdd}fd	d
}dd fdd}d}t| j|D ]\}}t|}|d r]|D ]}|d|| |d7 }qOqA|dkre|  t	|
|D ]m\}}	z*j||	}
||
 |D ]}||\}}|rt|dV  q|d| || qW n: ty } z|t||| W Y d}~n%d}~w tjy   |d|| |  Y n ty   |d||  w |d7 }qmqAdS )zIterates through all archive items

            Missing item chunks will be skipped and the msgpack stream will be restarted
            c                 s   r  rf   r  r  rc   rc   rd   r{    r  zLArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.<genexpr>c                 s   r  rf   r  r  rc   rc   rd   r{    r  c                 S   s   t | tod| v S )Ns   path)rk   r7   r  rc   rc   rd   r    r   zKArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.<lambda>r   c                    s"    d t | jvkr d7   S )N   r   )r  r   )r  )_staterb   rc   rd   missing_chunk_detector  s   zYArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.missing_chunk_detectorc                    s,   t |}| d||f 7 } d _t|  d S )Nz [chunk: %06d_%s]T)r8   r  r  r  )r   r  Zchunk_nor  rs   rc   rd   report  s   zIArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.reportc                 S   s   d dd | D S )Nz, c                 s   s.    | ]}t |tr|jd dnt|V  qdS )ry  )r  N)rk   r   r  r   )r   krc   rc   rd   r{    s   , zdArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.list_keys_safe.<locals>.<genexpr>)r  )r  rc   rc   rd   list_keys_safe  s   zQArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.list_keys_safec                    sb   t | tsdS | dd  t| }|s dd|  fS | s/dd|   fS dS )N)Fznot a dictionarys   aclFzmissing required keys: zinvalid keys: )Tr   )rk   r7   r  r   r  r  )r  r  required_item_keysrc   rd   
valid_item  s   


zMArchiveChecker.rebuild_refcounts.<locals>.robust_iterator.<locals>.valid_itemr  zitem metadata chunk missingr   r   zDDid not get expected metadata dict when unpacking item metadata (%s)NzCUnpacker crashed while unpacking item metadata, trying to resync...z5Exception while decrypting or unpacking item metadata)r  rH  r  ZREQUIRED_ITEM_KEYSr~  r	   r   r  r  r  r  r   r  r   rH   r   r   rA   r  rc  )rd  r   r  r  r  r]  stater   r  r  r   r   validreasonr  )r   rb   )r  r  r  r  rd   robust_iterator  sX   


z9ArchiveChecker.rebuild_refcounts.<locals>.robust_iterator,)r  r  r  r  z.--glob-archives %s does not match any archivesz+--first %d archives: only found %d archivesz*--last %d archives: only found %d archives)r  zArchive '%s' not found.TzChecking archives %3.1f%%g?zcheck.rebuild_refcountsr  zAnalyzing archive z (r   r  )z%Archive metadata block %s is missing!z*Archive metadata block %s is corrupted: %sFr_  r  zEThis archive will be *removed* from the manifest! It will be deleted.r   r`  c                 S   rg  rc   rh  ri  rc   rc   rd   r   %  r   z4ArchiveChecker.rebuild_refcounts.<locals>.<listcomp>r   r  r  )r  saltrf   ),r   r  r    r  r  r|  rH  rX  r  r  r  r'  r  r  r  r;   rQ   r   r  r  r^  r   rz   r8   r   r   r  r   ra  rI   rb  rc  rk  r
  r  r   r   r   r  r}   r3  r  rm  r  )rb   rd  r  r  r  r  r  r  r  Zarchive_infosZnum_archivesr  r]  r^  Z
archive_idr  r   r  r  r  rW  r   Zprevious_item_idZnew_archive_idrc   )r  r  r   rb   rd   r  =  s   	LH

$


	



$z ArchiveChecker.rebuild_refcountsc                 C   s   | j rJdd | j D }|| j }|r!tt| d d| _| jrF|rHt	dt|t| jf  |D ]}| j
| q6t	d d S d S d S t	d d S )Nc                 S   s   h | ]\}}|j d kr|qS )r   )r  )r   r  r  rc   rc   rd   	<setcomp>9      z5ArchiveChecker.orphan_chunks_check.<locals>.<setcomp>z orphaned objects found!Tz1Deleting %d orphaned and %d superseded objects...z.Finished deleting orphaned/superseded objects.z<Orphaned objects check skipped (needs all archives checked).)r  r   r  r  r  r  r'  r  r  r^  r   r  )rb   ZunusedZorphanedr  rc   rc   rd   r  7  s   

z"ArchiveChecker.orphan_chunks_checkc                 C   s<   | j rtd | j  td | jjd|d d S d S )NzWriting Manifest.zCommitting repo.F)r  r  )r  r  r^  rH  r  r   r  )rb   r  rc   rc   rd   r  G  s   


zArchiveChecker.finish)FNr   r   r   NFF)Nr   r   r   Nr   )ry   r   r   re   r  r  r  r  r  r  r  r  rc   rc   rc   rd   r  K  s    
+Q
L {r  c                   @   s   e Zd ZG dd deZedd Z				d!dd	Zd"d
dZdd Z	dd Z
dd Zdd Zdd Zd#ddZdd Zd$ddZdd Zdd  ZdS )%ArchiveRecreaterc                   @   s   e Zd ZdddZdS )zArchiveRecreater.InterruptedNc                 C   s   |pi | _ d S rf   )re  )rb   re  rc   rc   rd   re   Q  s   z%ArchiveRecreater.Interrupted.__init__rf   )ry   r   r   re   rc   rc   rc   rd   InterruptedP  s    r  c                 C   s
   |  dS )N	.recreate)endswith)r  rc   rc   rd   is_temporary_archiveT  rq  z%ArchiveRecreater.is_temporary_archiveFNrD  c                 C   s   || _ || _|| _|| _|| _|| _|pg | _|| _|	d u| _| jr(t	
d|	 |	p+t| _|| _|| _|
p8td| _t | _|| _|| _|| _|| _|pOdd | _|rXd | _d S || _d S )NzRechunking archives to %snonec                  W   r   rf   rc   r;  rc   rc   rd   r  t  r#  z+ArchiveRecreater.__init__.<locals>.<lambda>)r   r   rH  r,  r
  exclude_cachesexclude_if_presentkeep_exclude_tagsr)  r  r  r  r  
recompressalways_recompressr   compressionr   seen_chunksr  r  rn   r[  r>  rL  )rb   r   rH  r   r,  r
  r  r  r  r  r  r  r  r  rn   r[  r:  r  rL  rc   rc   rd   re   X  s,   


zArchiveRecreater.__init__c                 C   s   |  |rJ | |}| ||}| js| jr| | | j r2| js2|j	s2|d u r2|d u r2dS | 
|| |d u }| j||||d dS )NF)replace_originalT)r  open_archivecreate_targetr  r  matcher_add_tagged_dirsr
  emptyr  recreate_rechunkifyprocess_itemsr  )rb   r  rJ  target_namerd  targetr  rc   rc   rd   recreatew  s$   

zArchiveRecreater.recreatec           
         s  | j }|    ri nd } fdd}| D ]g}||js:| d|j ||r9|d|dd f||j< q rmt|jrm|d|v rm||j	 \}}}	|	d u rj||_
|d ur^||_d d |jf||j	< |`	n|	|_	| jrx| d|j q| ||| q| jr|jjdd	 d S d S )
Nc                    s"    ot | jo| ddod| vS r  r  r  Ztarget_is_subsetrc   rd   item_is_hardlink_master  s   
z?ArchiveRecreater.process_items.<locals>.item_is_hardlink_masterxr   r  r   -T)r   )r
  r  r  r  r   r>  r   r"   r   r   r   r  r  process_itemr[  rn   r   )
rb   rd  r  r
  r   r  r   r   r  Z
new_sourcerc   r  rd   r    s2   
zArchiveRecreater.process_itemsc                 C   sd   t |j}d|v r!| ||j d }| ||| |j jd7  _|j||jd | ||j d S )Nr   r   r@  )r1   r   r>  r   process_chunksrn   r\   r  )rb   rd  r  r   rC  rc   rc   rd   r    s   
zArchiveRecreater.process_itemc           	      C   sr   | j s|js|jD ]\}}}| j||j q	|jS | ||t|j}t| j	|}|
|| j|j| j|| d S rf   )r  r  r   r,  r6  rn   iter_chunksr  r   r5  r7  r[  )	rb   rd  r  r   r  rg   rZ   chunk_iteratorr5  rc   rc   rd   r    s   zArchiveRecreater.process_chunksc                 C   s   t || jj\}}|| jv r| j||jS | j}| jrB| jsB|| jj	v rB| jj
d | j|dd}t|j| jj|jkrBd}| jj|||j|dd}| jjjdd | j|j |S )NF)
decompress)	overwriter1  r0  )r&  r   r3  r  r,  r6  rn   r  r  r   r  r   r   r   detectr   Z
compressorZdecider2  r4  r   rz   )rb   r  r  r  r   r  Z	old_chunkr3  rc   rc   rd   r5    s   
z ArchiveRecreater.chunk_processorc                 c   s^    |j dd |D }|jrt|}|j|E d H  d S |D ]}t|t|tdV  q d S )Nc                 S   s   g | ]\}}}|qS rc   rc   )r   r  r   rc   rc   rd   r     r   z0ArchiveRecreater.iter_chunks.<locals>.<listcomp>)rg   r  )	rU  r   r  r#   r  r  r   r'  r  )rb   rd  r  r   r   r   r  rc   rc   rd   r    s   zArchiveRecreater.iter_chunksTc                 C   s   | j rd S |d u r|jdd}| jr|j}| jd u r0|jj|jdp'|jj|jjtj	d}n|jjtj	d}|j
|| j|d |rR|jt | jd ||j | jro||_t |_ttt|tt|jt| jt d S d S )NrJ  r   rn  )r   rn  rk  recreate_cmdline)rk  r  )rJ  r  r  )r[  )r  re  r   rn   rR  r  r   rk  r   r~  r  r  rU   r[  r  r   rC   r   r<   ZDASHESr   r,  )rb   rd  r  rJ  r  _startr  rc   rc   rd   r    s<   
	zArchiveRecreater.savec           	         s6  fdd}j  g g i }jr2|jdd dD ]}t|jr1d|vr1d|v r1d||j< q|j fd	ddD ]N}jrK|j|v rK|||j< tj	|j\}}|j
v r_||| q<jr|tkrt|jrd|v rr|n||j }t||}|tttkr||| q< tj  tj dS )
zLAdd excludes to the matcher created by exclude_cache and exclude_if_present.c                    sJ    j rt|jdd t| d dd d S t| dd d S )NF)Zrecurse_dirr  )r  r%  rE   r   rF   )dirZtag_item)rb   	tag_filestagged_dirsrc   rd   exclude	  s   z9ArchiveRecreater.matcher_add_tagged_dirs.<locals>.excludec                 S   s   t j| jtkS rf   )r   r   basenameCACHE_TAG_NAMEr  rc   rc   rd   r  	  r   z:ArchiveRecreater.matcher_add_tagged_dirs.<locals>.<lambda>r   r   r   Nc                    s   t j| jtkp | jS rf   )r   r   r
  r  r  r  r	  rc   rd   r  !	  r  )r
  r  r  r   r  r   r   r   r   r  r  r  r%   r   r'  ZCACHE_TAG_CONTENTSr   rG   IncludeZExcludeNoRecurse)	rb   rd  r	  Zcachedir_mastersr   r  Ztag_fileZcontent_itemr   rc   )r
  rb   r  r  rd   r  	  s4   






z(ArchiveRecreater.matcher_add_tagged_dirsc                 C   s   |p|j d }| |}|jd}|durt|nd}|j}| jo%||k|_|jr3t	d|p0d| t
| j| j|j|j| j|jdj|_t|j| jjdd|_|S )	zCreate target archive.r  r  Nz Rechunking archive from %s to %sz	(unknown))r,  r   r  r  rL  r)  Fr  )r   create_target_archivere  r   r$   r  r)  r  r  r  r'  r,  r   r  r  rL  r7  r   r  r  )rb   rd  r  r  Zsrc_cpZdst_cprc   rc   rd   r  /	  s"   
zArchiveRecreater.create_targetc                 C   s,   t | j| j| j|d| j| j| j| jd	}|S )NT)rV  r[  r  r,  rL  )r@  r   r   rH  r[  r  r,  rL  )rb   r   r  rc   rc   rd   r  A	  s
   z&ArchiveRecreater.create_target_archivec                 K   s"   t | j| j| j|fd| ji|S )Nr,  )r@  r   r   rH  r,  )rb   r   kwargsrc   rc   rd   r  G	  s   "zArchiveRecreater.open_archive)FNFNNFFFFFNNrD  rp  r`  rf   )ry   r   r   rc  r  r  r  re   r  r  r  r  r5  r  r  r  r  r  r  rc   rc   rc   rd   r  O  s(    


%
	
'
*r  )r   r   socketr   r   r   collectionsr   
contextlibr   datetimer   r   	functoolsr   Zgetpassr   ior   	itertoolsr	   r
   shutilr   Zplatformflagsr   r   r   r   r  r   r   r   r  r   r   r,  r   Z
crypto.keyr   r   compressr   r   	constantsZcrypto.low_levelr   r  Z	hashindexr   r   r   helpersr    r"   r#   r$   r%   r&   r'   platformr(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   ZlrucacherD   patternsrE   rF   rG   r   rH   rI   rJ   rK   rL   rM   rN   rO   rP   ZremoterQ   r   rR   rS   r  r  rU   r   rc  r   r   r   r   r   r   r   r   r
  r*  r?  r@  r  r%  r&  r'  r8  ra  r}  r~  r  r  rc   rc   rc   rd   <module>   s     
wD1     L=M 862    