o
    +keP                     @   s$  d dl Z d dlZd dlZd dlZd dlZd dlZddlmZmZm	Z	 ddl
mZ ddlmZmZ ddl
mZmZ dd	 Zd
d ZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZdS )     N   )NSIndex
ChunkIndexChunkIndexEntry)	hashindex)IntegrityCheckedFileFileIntegrityError   )BaseTestCaseunopened_tempfilec                 C   s   t d|  dS )Nz%-0.32dascii)bytesx r   <usr/lib/python3.10/site-packages/borg/testsuite/hashindex.pyH   s   r   c                 C   s   t t|  S Nhashlibsha256r   digestr   r   r   r   H2   s   r   c                   @   sD   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dS )HashIndexTestCasec              	   C   s  | }|  t|d tdD ]
}|||t|< q|  t|d tdD ]}|  |t| || q&tdD ]}||d |t|< q9|  t|d tdD ]}|  |t| ||d  qRtdD ]}|t|= qgtddD ]
}t||v s~J qttdD ]
}t||vsJ qtdD ]}| t|jt| q|  t|d t p}|| ~t	|d}|  t
|  | W d    n1 sw   Y  ||}|  t|d tddD ]}|  |t| ||d  q|  |  t|d || ~|  t||d W d    n	1 sw   Y  | }|td|d td|v s9J |td |dksFJ |td|d td|v sYJ |td |dksfJ |td|d|dkswJ |td|d|dksJ ~d S )	Nr   d   r   2   rb*      r	   )assert_equallenranger   assert_raisesKeyError__delitem__r   writeopenr   r   read	hexdigestclear
setdefault)selfclsZ
make_valueZshaidxr   filepathfdr   r   r   _generic_test   s^   


""zHashIndexTestCase._generic_testc                 C      |  tdd d d S )Nc                 S   s   | | fS r   r   r   r   r   r   <lambda>Z   s    z0HashIndexTestCase.test_nsindex.<locals>.<lambda>Z@85f72b036c692c8266e4f51ccf0cff2147204282b5e316ae508d30a448d88fef)r0   r   r+   r   r   r   test_nsindexY      zHashIndexTestCase.test_nsindexc                 C   r1   )Nc                 S   s
   | | | fS r   r   r   r   r   r   r2   ^   s   
 z3HashIndexTestCase.test_chunkindex.<locals>.<lambda>Z@c83fdf33755fc37879285f2ecfc5d1f63b97577494902126b6fb6f3e4d852488)r0   r   r3   r   r   r   test_chunkindex]   r5   z!HashIndexTestCase.test_chunkindexc                 C   s   d}t  `}t }|| tj|}| t|d t|D ]
}||f|t	|< q || |tj|k s:J t|D ]}|t	|= q>| t|d || | |tj| W d    d S 1 shw   Y  d S )N  r   )
r   r   r%   ospathgetsizer   r    r!   r   )r+   nr.   r-   initial_sizer   r   r   r   test_resizea   s    


"zHashIndexTestCase.test_resizec                 C   s   t  }tdD ]
}||f|t|< q| }t|}| t|d | tt	| t|j|d d d}| t|d | ||dd   d S )Nr   1   r   )markerr   )
r   r!   r   	iteritemslistr   r    r"   StopIterationnext)r+   r-   r   iteratorallZsecond_halfr   r   r   test_iteritemsr   s   z HashIndexTestCase.test_iteritemsc                 C   s   t  }d|td< d|td< d|td< t  }d|td< d|td< d	|td
< || |td dks9J |td dksCJ |td dksMJ |td
 d	ksWJ d S )Nr	   r   r   r	   r      rI   r      ,  rL   rK   )   r   r   )   rI   rI   )     rP   rM   )rN   r   r   )   rI   rI   r   r   merger+   idx1idx2r   r   r   test_chunkindex_merge   s   
z'HashIndexTestCase.test_chunkindex_mergec                 C   s   t  }d|td< d|td< d|td< | \}}}}}}|dks%J |dks+J |d	ks1J |d
ks7J |dks=J |dksCJ d S )N)r	   i  r   r	   )r   r7   rI   r   )rK   i  rL   rK   i6  ix  ip  X  rO   )r   r   Z	summarize)r+   r-   sizeZcsizeZunique_sizeZunique_csizeZunique_chunkschunksr   r   r   test_chunkindex_summarize   s   z+HashIndexTestCase.test_chunkindex_summarizeN)
__name__
__module____qualname__r0   r4   r6   r=   rF   rW   r[   r   r   r   r   r      s    <r   c                   @   s   e Zd ZdZdd ZdS )HashIndexExtraTestCasezRThese tests are separate because they should not become part of the selftest.
    c                 C   s  t dtj d }t|}dd t|D }|dd| d  |d| d d  }}t|D ]\}}|||f||< q/t|D ]\}}|||f||< q?|D ]}||= qMt|D ]\}}|| |||fksfJ qW|D ]}||d u stJ qi|D ]}||= qwt| g ksJ d S )Ni  
   c                 S   s   g | ]}t t| qS r   r   ).0kr   r   r   
<listcomp>   s    z=HashIndexExtraTestCase.test_chunk_indexer.<locals>.<listcomp>r   r   rK   )intr   MAX_LOAD_FACTORr!   	enumerategetrA   r@   )r+   Z	key_countindexZall_keyskeysZto_delete_keysikeyr   r   r   test_chunk_indexer   s"   *z)HashIndexExtraTestCase.test_chunk_indexerN)r\   r]   r^   __doc__rl   r   r   r   r   r_      s    r_   c                   @      e Zd Zdd Zdd ZdS )HashIndexSizeTestCasec                 C   s   t  }| dksJ d S )NiF  )r   rY   r+   r-   r   r   r   test_size_on_disk   s   z'HashIndexSizeTestCase.test_size_on_diskc                 C   s~   t  }tdD ]}||d |d f|t|< qt }|| tj|}W d    n1 s0w   Y  | |ks=J d S )Ni  r   rK   )	r   r!   r   r   r%   r8   r9   r:   rY   )r+   r-   rj   r.   rY   r   r   r   test_size_on_disk_accurate   s   
z0HashIndexSizeTestCase.test_size_on_disk_accurateN)r\   r]   r^   rq   rr   r   r   r   r   ro      s    ro   c                   @   t   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd ZdS )HashIndexRefcountingTestCasec                 C   s~   t  }t jd ddf|td< tdD ]}|td^}}|t jks%J qtdD ]}|td^}}|t jks<J q*d S )Nr	   r   rN   )r   	MAX_VALUEr   r!   increfdecref)r+   r-   rj   refcount_r   r   r   test_chunkindex_limit   s   z2HashIndexRefcountingTestCase.test_chunkindex_limitc                 C   s(   dd }|||}||||ksJ |S )Nc                 S   sN   t  }| ddf|td< t  }|ddf|td< || |td ^}}|S )Nr	   r   rR   )Z	refcount1Z	refcount2rU   rV   rx   ry   r   r   r   rS      s   
z2HashIndexRefcountingTestCase._merge.<locals>.merger   )r+   Z	refcountaZ	refcountbrS   resultr   r   r   _merge   s   
z#HashIndexRefcountingTestCase._mergec                 C   s(   t jd }| ||t jd ksJ d S Nr   r	   r   ru   r|   r+   Zhalfr   r   r   test_chunkindex_merge_limit1   s   
z9HashIndexRefcountingTestCase.test_chunkindex_merge_limit1c                 C   s   |  ddtjksJ d S )Nl    ^e i 5w)r|   r   ru   r3   r   r   r   test_chunkindex_merge_limit2   s   z9HashIndexRefcountingTestCase.test_chunkindex_merge_limit2c                 C   s(   t jd }| |d |t jksJ d S r}   r~   r   r   r   r   test_chunkindex_merge_limit3   s   
z9HashIndexRefcountingTestCase.test_chunkindex_merge_limit3c                 C   sF   t jd }| |d |t jksJ | |d |d t jks!J d S r}   r~   r   r   r   r   test_chunkindex_merge_limit4   s   
"z9HashIndexRefcountingTestCase.test_chunkindex_merge_limit4c                 C   sZ   t  }|tdddd |td dksJ |tdddd |td dks+J d S )	Nr	   rN   rO   rQ   rN   rO   rQ   r   rK   )rO   r   rK   )r   addr   r+   rU   r   r   r   test_chunkindex_add   s
   z0HashIndexRefcountingTestCase.test_chunkindex_addc                 C   J   t  }t jddf|td< |td |td ^}}|t jks#J d S NrO   rQ   r	   )r   ru   r   rv   r+   rU   rx   ry   r   r   r   test_incref_limit  
   z.HashIndexRefcountingTestCase.test_incref_limitc                 C   r   r   )r   ru   r   rw   r   r   r   r   test_decref_limit  r   z.HashIndexRefcountingTestCase.test_decref_limitc                 C   sP   t  }d|td< | t |td W d    d S 1 s!w   Y  d S )N)r   r   r   r	   )r   r   r"   AssertionErrorrw   r   r   r   r   test_decref_zero  s
   "z-HashIndexRefcountingTestCase.test_decref_zeroc                 C   sv   t  }|tdddd |td dksJ |td |td dks(J |td |td dks9J d S )Nr	   rN   rO   rQ   r   )rO   rO   rQ   )r   r   r   rv   rw   r   r   r   r   test_incref_decref  s   z/HashIndexRefcountingTestCase.test_incref_decrefc                 C   sN   t  }| t t jd ddf|td< W d    d S 1 s w   Y  d S )Nr	   r   )r   r"   r   ru   r   r   r   r   r   test_setitem_raises"  s   "z0HashIndexRefcountingTestCase.test_setitem_raisesc                 C   s   t  }| t |td W d    n1 sw   Y  | t |td W d    n1 s6w   Y  | t |td  W d    n1 sQw   Y  | t |tdddd W d    d S 1 sqw   Y  d S )Nr	   r   )r   r"   r#   rv   r   rw   OverflowErrorr   rp   r   r   r   test_keyerror'  s   "z*HashIndexRefcountingTestCase.test_keyerrorN)r\   r]   r^   rz   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rt      s    	rt   c                   @   s@   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dS )HashIndexDataTestCases(  eJzt0L0NgmAUhtHLT0LDEI6AuAEhMVYmVnSuYefC7AB3Aj9KNedJbnfyFne6P67P27w0EdG1Eac+Cm1ZybAsy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7LsL9nhc+cqTZ3XlO2Ys++Du5fX+l1/YFmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVn2/+0O2rYccw==c              	   C   s   t  8}tj|d}|| t|d}| | W  d    W  d    S 1 s/w   Y  W d    d S 1 s?w   Y  d S )Nr-   r   )	tempfileTemporaryDirectoryr8   r9   joinr%   r&   _packr'   )r+   r-   tempdirfilefr   r   r   _serialize_hashindex9  s   

"z*HashIndexDataTestCase._serialize_hashindexc              	   C   s~   t  1}tj|d}t|d}|| | W d    n1 s$w   Y  t	|W  d    S 1 s8w   Y  d S )Nr-   wb)
r   r   r8   r9   r   r&   r%   _unpackr   r'   )r+   
bytestringr   r   r   r   r   r   _deserialize_hashindex@  s   
$z,HashIndexDataTestCase._deserialize_hashindexc                 C      t t|S r   )base64	b64encodezlibcompressr+   r   r   r   r   r   G     zHashIndexDataTestCase._packc                 C   r   r   )r   
decompressr   	b64decoder   r   r   r   r   J  r   zHashIndexDataTestCase._unpackc                 C   sR   t  }d|td< d|td< d|td< | |}| || | jks'J d S )Nr	   r   rK   r	   ir   r   r   l   xl r   r   rK   )r   r   r   r   	HASHINDEX)r+   rU   Z
serializedr   r   r   test_identical_creationM  s   
z-HashIndexDataTestCase.test_identical_creationc                 C   s   |  | j}|td dksJ |td dksJ |td dks$J t }d|td< || |td tjdd	fks@J d S )
Nr	   r   r   r   rK   r   )l    rO   rQ   rO   rQ   )r   r   r   r   rS   ru   rT   r   r   r   test_read_known_goodV  s   
 z*HashIndexDataTestCase.test_read_known_goodN)
r\   r]   r^   r   r   r   r   r   r   r   r   r   r   r   r   3  s    	r   c                   @   rn   )HashIndexIntegrityTestCasec                 C   sv   |  | j}tj|d}t|dd}|| W d    n1 s#w   Y  |j}d|v s1J d|v s7J ||fS )Nr-   T)r9   r%   finalZ
HashHeader)r   r   r8   r9   r   r   r%   integrity_data)r+   r   r-   r   r/   r   r   r   r   write_integrity_checked_indexc  s   z8HashIndexIntegrityTestCase.write_integrity_checked_indexc              
   C   s   t  c}| |\}}t|d}|d W d    n1 s!w   Y  | t$ t|d|d}t	| W d    n1 sCw   Y  W d    n1 sRw   Y  W d    d S W d    d S 1 sjw   Y  d S )Nzr+bs   FooF)r9   r%   r   )
r   r   r   r&   r%   r"   r   r   r   r'   )r+   r   r   r   r/   r   r   r   test_integrity_checked_filem  s   
"z6HashIndexIntegrityTestCase.test_integrity_checked_fileN)r\   r]   r^   r   r   r   r   r   r   r   b  s    
r   c                   @   rs   )HashIndexCompactTestCasec                 C   sd   t  }|d ||dd ||dd |ddd |ddd || _d S )Ns   BORG_IDXrM   little    r	      )ioBytesIOr%   to_bytes
index_data)r+   num_entriesnum_bucketsr   r   r   r   rh   x  s   

zHashIndexCompactTestCase.indexc                 C   s   | j d t| j }|S )Nr   )r   seekr   r'   )r+   rh   r   r   r   index_from_data  s   z(HashIndexCompactTestCase.index_from_datac                 C   s   t  }|| | S r   )r   r   r%   getvalue)r+   rh   datar   r   r   index_to_data  s   
z&HashIndexCompactTestCase.index_to_datac                 C   s   |   }|  | |}|S r   )r   compactr   )r+   rh   compact_indexr   r   r   index_from_data_compact_to_data  s   
z8HashIndexCompactTestCase.index_from_data_compact_to_datac                 G   s.   | j | |D ]}| j |dd qd S )NrM   r   )r   r%   r   )r+   rk   valuesvaluer   r   r   write_entry  s   z$HashIndexCompactTestCase.write_entryc                 C      |  |ddd d S )Nl    r   r   r+   rk   r   r   r   write_empty     z$HashIndexCompactTestCase.write_emptyc                 C   r   )Nl    r   r   r   r   r   r   write_deleted  r   z&HashIndexCompactTestCase.write_deletedc                 C   s   | j ddd | tdddd | td | td | tdddd | td	d
dd | td |  }| j ddd | tdddd | tdddd | td	d
dd || j kslJ d S )NrK   rO   r   r   r   r	   r   rN   rQ   rM      	   r`   )rh   r   r   r   r   r   r   r   r+   r   r   r   r   test_simple  s   z$HashIndexCompactTestCase.test_simplec                 C   s   | j ddd | td | tdddd | td | tdddd | td	d
dd | td |  }| j ddd | tdddd | tdddd | td	d
dd || j kslJ d S NrK   rO   r   r	   r   r   rN   rQ   rM   r   r   r`   rh   r   r   r   r   r   r   r   r   r   r   r   test_first_empty  s   z)HashIndexCompactTestCase.test_first_emptyc                 C   s   | j ddd | td | tdddd | td | tdddd | td | td	d
dd |  }| j ddd | tdddd | tdddd | td	d
dd || j kslJ d S r   r   r   r   r   r   test_last_used  s   z'HashIndexCompactTestCase.test_last_usedc                 C   s   | j ddd | td | tdddd | tdddd | td | td | td	d
dd |  }| j ddd | tdddd | tdddd | td	d
dd || j kslJ d S r   r   r   r   r   r   test_too_few_empty_slots  s   z1HashIndexCompactTestCase.test_too_few_empty_slotsc                 C   s   | j ddd | td | td | td | td | td | td |  }| j ddd || j ksEJ d S )	Nr   rO   r   r	   rK   r   rN   rM   )rh   r   r   r   r   r   r   r   r   r   r   
test_empty  s   z#HashIndexCompactTestCase.test_emptyc                 C   s   t  }t  }d|td< d|td< d|td< |  | dks$J || |td dks3J |td dks=J |td dksGJ d S )NrG   r	   rH   r   rJ   rK      )r   r   r   rY   rS   )r+   ZmasterrU   r   r   r   
test_merge  s   
z#HashIndexCompactTestCase.test_mergeN)r\   r]   r^   rh   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   w  s    r   c                   @      e Zd Zdd ZdS )NSIndexTestCasec                 C   s|   t  }| t t jd df|td< W d    n1 sw   Y  td|vs+J t jdf|td< td|v s<J d S )Nr	   r   r   )r   r"   r   ru   r   rp   r   r   r   test_nsindex_segment_limit  s   z*NSIndexTestCase.test_nsindex_segment_limitN)r\   r]   r^   r   r   r   r   r   r         r   c                   @   r   )AllIndexTestCasec                 C   s    t jdk sJ tjdk sJ d S )Ng      ?)r   re   r   r3   r   r   r   test_max_load_factor  s   z%AllIndexTestCase.test_max_load_factorN)r\   r]   r^   r   r   r   r   r   r     r   r   c                   @   r   )IndexCorruptionTestCasec                    s  ddl m fdd t tdD ]}d|f d|< q dks'J tdD ]} d|= q+tdD ]}d	|f d	|< q8 fd
dtddD dd tddD ks]J  fddtdD dd tdD kstJ  fddtdD dd tdD ksJ d S )Nr   packc                    s    d| |dddS )Nz<IIQQQr   r   )r   yr   r   r   HH  s   z1IndexCorruptionTestCase.test_bug_4829.<locals>.HHi  i*  rP   iJ  rX   c                       g | ]
}  d |qS r   rg   ra   r   r   r-   r   r   rc   4      z9IndexCorruptionTestCase.test_bug_4829.<locals>.<listcomp>c                 S      g | ]}d |fqS r   r   r   r   r   r   rc   4      c                    s   g | ]	} d |v qS r   r   r   r   r   r   rc   6  s    c                 S   s   g | ]}d qS )Fr   r   r   r   r   rc   6  s    c                    r   rX   r   r   r   r   r   rc   9  r   c                 S   r   r   r   r   r   r   r   rc   9  r   )structr   r   r!   rY   )r+   r   r   )r   r-   r   r   test_bug_4829  s   2.2z%IndexCorruptionTestCase.test_bug_4829N)r\   r]   r^   r   r   r   r   r   r     r   r   )r   r   r   r8   r   r   r   r   r   r    Zcrypto.file_integrityr   r   r
   r   r   r   r   r_   ro   rt   r   r   r   r   r   r   r   r   r   r   <module>   s0    f/ 
