o
    _`bv                     @   s   d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	m
Z
mZmZ d dlmZmZmZmZ G dd dZG d	d
 d
ZG dd dZG dd deZG dd deZG dd deZG dd deZG dd deZG dd deZdS )    N)BytesIOreadableseekable)IN_MEMORY_UPLOAD_TAG)CompleteMultipartUploadTaskCreateMultipartUploadTaskSubmissionTaskTask)ChunksizeAdjusterDeferredOpenFileget_callbacksget_filtered_dictc                   @   s.   e Zd ZdddZdd Zdd Zdd	 Zd
S )AggregatedProgressCallback   c                 C   s   || _ || _d| _dS )a  Aggregates progress updates for every provided progress callback

        :type callbacks: A list of functions that accepts bytes_transferred
            as a single argument
        :param callbacks: The callbacks to invoke when threshold is reached

        :type threshold: int
        :param threshold: The progress threshold in which to take the
            aggregated progress and invoke the progress callback with that
            aggregated progress total
        r   N)
_callbacks
_threshold_bytes_seen)self	callbacks	threshold r   5usr/lib/python3.10/site-packages/s3transfer/upload.py__init__!   s   
z#AggregatedProgressCallback.__init__c                 C   s*   |  j |7  _ | j | jkr|   d S d S N)r   r   _trigger_callbacks)r   bytes_transferredr   r   r   __call__1   s   z#AggregatedProgressCallback.__call__c                 C   s   | j dkr|   dS dS )z@Flushes out any progress that has not been sent to its callbacksr   N)r   r   r   r   r   r   flush6   s   
z AggregatedProgressCallback.flushc                 C   s"   | j D ]}|| jd qd| _d S )N)r   r   )r   r   )r   callbackr   r   r   r   ;   s   

z-AggregatedProgressCallback._trigger_callbacksN)r   )__name__
__module____qualname__r   r   r   r   r   r   r   r   r       s
    
r   c                   @   sL   e Zd ZdZdd ZdddZddd	Zd
d Zdd Zdd Z	dd Z
dS )InterruptReadera  Wrapper that can interrupt reading using an error

    It uses a transfer coordinator to propagate an error if it notices
    that a read is being made while the file is being read from.

    :type fileobj: file-like obj
    :param fileobj: The file-like object to read from

    :type transfer_coordinator: s3transfer.futures.TransferCoordinator
    :param transfer_coordinator: The transfer coordinator to use if the
        reader needs to be interrupted.
    c                 C   s   || _ || _d S r   )_fileobj_transfer_coordinator)r   fileobjtransfer_coordinatorr   r   r   r   O   s   
zInterruptReader.__init__Nc                 C   s   | j jr| j j| j|S r   )r&   	exceptionr%   read)r   amountr   r   r   r*   S   s   zInterruptReader.readr   c                 C   s   | j || d S r   )r%   seek)r   wherewhencer   r   r   r,   ]   s   zInterruptReader.seekc                 C   s
   | j  S r   )r%   tellr   r   r   r   r/   `   s   
zInterruptReader.tellc                 C   s   | j   d S r   )r%   closer   r   r   r   r0   c      zInterruptReader.closec                 C   s   | S r   r   r   r   r   r   	__enter__f      zInterruptReader.__enter__c                 O   s   |    d S r   )r0   )r   argskwargsr   r   r   __exit__i      zInterruptReader.__exit__r   )r   )r!   r"   r#   __doc__r   r*   r,   r/   r0   r2   r6   r   r   r   r   r$   A   s    


r$   c                   @   sf   e Zd ZdZdddZedd Zdd Zd	d
 Zdd Z	dd Z
dd Zdd Zdd Zdd ZdS )UploadInputManageraJ  Base manager class for handling various types of files for uploads

    This class is typically used for the UploadSubmissionTask class to help
    determine the following:

        * How to determine the size of the file
        * How to determine if a multipart upload is required
        * How to retrieve the body for a PutObject
        * How to retrieve the bodies for a set of UploadParts

    The answers/implementations differ for the various types of file inputs
    that may be accepted. All implementations must subclass and override
    public methods from this class.
    Nc                 C   s   || _ || _|| _d S r   )_osutilr&   _bandwidth_limiterr   osutilr(   bandwidth_limiterr   r   r   r   }   s   
zUploadInputManager.__init__c                 C      t d)a  Determines if the source for the upload is compatible with manager

        :param upload_source: The source for which the upload will pull data
            from.

        :returns: True if the manager can handle the type of source specified
            otherwise returns False.
        zmust implement _is_compatible()NotImplementedErrorclsZupload_sourcer   r   r   is_compatible   s   
z UploadInputManager.is_compatiblec                 C   r?   )a  Whether the body it provides are stored in-memory

        :type operation_name: str
        :param operation_name: The name of the client operation that the body
            is being used for. Valid operation_names are ``put_object`` and
            ``upload_part``.

        :rtype: boolean
        :returns: True if the body returned by the manager will be stored in
            memory. False if the manager will not directly store the body in
            memory.
        z%must implement store_body_in_memory()r@   r   operation_namer   r   r   stores_body_in_memory      z(UploadInputManager.stores_body_in_memoryc                 C   r?   )zProvides the transfer size of an upload

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request
        z&must implement provide_transfer_size()r@   r   transfer_futurer   r   r   provide_transfer_size   s   z(UploadInputManager.provide_transfer_sizec                 C   r?   )a  Determines where a multipart upload is required

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type config: s3transfer.manager.TransferConfig
        :param config: The config associated to the transfer manager

        :rtype: boolean
        :returns: True, if the upload should be multipart based on
            configuration and size. False, otherwise.
        z*must implement requires_multipart_upload()r@   r   rJ   configr   r   r   requires_multipart_upload   rH   z,UploadInputManager.requires_multipart_uploadc                 C   r?   )a  Returns the body to use for PutObject

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type config: s3transfer.manager.TransferConfig
        :param config: The config associated to the transfer manager

        :rtype: s3transfer.utils.ReadFileChunk
        :returns: A ReadFileChunk including all progress callbacks
            associated with the transfer future.
        z$must implement get_put_object_body()r@   rI   r   r   r   get_put_object_body   rH   z&UploadInputManager.get_put_object_bodyc                 C   r?   )a  Yields the part number and body to use for each UploadPart

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type chunksize: int
        :param chunksize: The chunksize to use for this upload.

        :rtype: int, s3transfer.utils.ReadFileChunk
        :returns: Yields the part number and the ReadFileChunk including all
            progress callbacks associated with the transfer future for that
            specific yielded part.
        z)must implement yield_upload_part_bodies()r@   )r   rJ   	chunksizer   r   r   yield_upload_part_bodies   s   z+UploadInputManager.yield_upload_part_bodiesc                 C   s*   t || j}| jr| jj|| jdd}|S )NF)enabled)r$   r&   r;   Zget_bandwith_limited_stream)r   r'   r   r   r   _wrap_fileobj   s   z UploadInputManager._wrap_fileobjc                 C   s   t |d}|rt|gS g S )NZprogress)r   r   )r   rJ   r   r   r   r   _get_progress_callbacks   s   

z*UploadInputManager._get_progress_callbacksc                 C   s   dd |D S )Nc                 S   s   g | ]}|j qS r   )r   ).0r    r   r   r   
<listcomp>   s    z;UploadInputManager._get_close_callbacks.<locals>.<listcomp>r   )r   Zaggregated_progress_callbacksr   r   r   _get_close_callbacks   r1   z'UploadInputManager._get_close_callbacksr   )r!   r"   r#   r8   r   classmethodrD   rG   rK   rN   rO   rQ   rS   rT   rW   r   r   r   r   r9   m   s    

	r9   c                   @   sd   e Zd ZdZedd Zdd Zdd Zdd	 Zd
d Z	dd Z
dd Zdd Zdd Zdd ZdS )UploadFilenameInputManagerzUpload utility for filenamesc                 C   s
   t |tS r   )
isinstancestrrB   r   r   r   rD      s   
z(UploadFilenameInputManager.is_compatiblec                 C      dS )NFr   rE   r   r   r   rG      r3   z0UploadFilenameInputManager.stores_body_in_memoryc                 C   s   |j | j|j jj d S r   )metarK   r:   Zget_file_size	call_argsr'   rI   r   r   r   rK      s   z0UploadFilenameInputManager.provide_transfer_sizec                 C   s   |j j|jkS r   )r]   sizemultipart_thresholdrL   r   r   r   rN      r1   z4UploadFilenameInputManager.requires_multipart_uploadc                 C   sJ   |  |\}}| |}| |}| |}|jj}| jj|||||dS )Nr'   
chunk_sizefull_file_sizer   close_callbacks)&_get_put_object_fileobj_with_full_sizerS   rT   rW   r]   r_   r:   #open_file_chunk_reader_from_fileobj)r   rJ   r'   	full_sizer   rd   r_   r   r   r   rO      s   


z.UploadFilenameInputManager.get_put_object_bodyc                 c   s    |j j}| ||}td|d D ]5}| |}| |}||d  }| j|j jj|||d\}	}
| 	|	}	| j
j|	||
||d}||fV  qd S )N   )
start_byte	part_sizerc   ra   )r]   r_   _get_num_partsrangerT   rW   '_get_upload_part_fileobj_with_full_sizer^   r'   rS   r:   rf   )r   rJ   rP   rc   Z	num_partspart_numberr   rd   ri   r'   rg   Zread_file_chunkr   r   r   rQ     s.   




z3UploadFilenameInputManager.yield_upload_part_bodiesc                 C   s   t ||| jjd}|S )N)Zopen_function)r   r:   open)r   r'   ri   r   r   r   _get_deferred_open_file1  s   
z2UploadFilenameInputManager._get_deferred_open_filec                 C   s"   |j jj}|j j}| |d|fS )Nr   )r]   r^   r'   r_   rp   r   rJ   r'   r_   r   r   r   re   7  s   
zAUploadFilenameInputManager._get_put_object_fileobj_with_full_sizec                 K   s    |d }|d }|  |||fS )Nri   rc   )rp   )r   r'   r5   ri   rg   r   r   r   rm   <  s   zBUploadFilenameInputManager._get_upload_part_fileobj_with_full_sizec                 C   s   t t|jjt| S r   )intmathceilr]   r_   float)r   rJ   rj   r   r   r   rk   A  s   z)UploadFilenameInputManager._get_num_partsN)r!   r"   r#   r8   rX   rD   rG   rK   rN   rO   rQ   rp   re   rm   rk   r   r   r   r   rY      s    
rY   c                   @   s<   e Zd ZdZedd Zdd Zdd Zdd	 Zd
d Z	dS )UploadSeekableInputManagerz&Upload utility for an open file objectc                 C   s   t |ot|S r   r   rB   r   r   r   rD   H  s   z(UploadSeekableInputManager.is_compatiblec                 C   s   |dkrdS dS )N
put_objectFTr   rE   r   r   r   rG   L  s   z0UploadSeekableInputManager.stores_body_in_memoryc                 C   sD   |j jj}| }|dd | }|| |j ||  d S )Nr      )r]   r^   r'   r/   r,   rK   )r   rJ   r'   Zstart_positionZend_positionr   r   r   rK   R  s   

z0UploadSeekableInputManager.provide_transfer_sizec                 K   s   | |d }t|t|fS )Nrj   )r*   r   len)r   r'   r5   datar   r   r   rm   _  s   zBUploadSeekableInputManager._get_upload_part_fileobj_with_full_sizec                 C   s"   |j jj}| |j j }||fS r   )r]   r^   r'   r/   r_   rq   r   r   r   re   n  s   
zAUploadSeekableInputManager._get_put_object_fileobj_with_full_sizeN)
r!   r"   r#   r8   rX   rD   rG   rK   rm   re   r   r   r   r   rv   E  s    
rv   c                       sh   e Zd ZdZd fdd	Zedd Zdd Zd	d
 Zdd Z	dd Z
dd ZdddZdd Z  ZS )UploadNonSeekableInputManagerz7Upload utility for a file-like object that cannot seek.Nc                    s   t  ||| d| _d S )N    )superr   _initial_datar<   	__class__r   r   r   y  s   
z&UploadNonSeekableInputManager.__init__c                 C   s   t |S r   )r   rB   r   r   r   rD   }  s   z+UploadNonSeekableInputManager.is_compatiblec                 C   r\   )NTr   rE   r   r   r   rG     r3   z3UploadNonSeekableInputManager.stores_body_in_memoryc                 C   s   d S r   r   rI   r   r   r   rK     s   z3UploadNonSeekableInputManager.provide_transfer_sizec                 C   sP   |j jd ur|j j|jkS |j jj}|j}| ||d| _t| j|k r&dS dS )NFT)r]   r_   r`   r^   r'   _readr~   ry   )r   rJ   rM   r'   r   r   r   r   rN     s   
z7UploadNonSeekableInputManager.requires_multipart_uploadc                 C   s@   |  |}| |}|jjj}| | j|  ||}d | _|S r   )rT   rW   r]   r^   r'   
_wrap_datar~   r*   )r   rJ   r   rd   r'   bodyr   r   r   rO     s   


z1UploadNonSeekableInputManager.get_put_object_bodyc           	      c   s`    |j jj}d}	 | |}| |}|d7 }| ||}|s!d S | |||}d }||fV  q	)Nr   Trh   )r]   r^   r'   rT   rW   r   r   )	r   rJ   rP   Zfile_objectrn   r   rd   Zpart_contentZpart_objectr   r   r   rQ     s    



z6UploadNonSeekableInputManager.yield_upload_part_bodiesTc                 C   sx   t | jdkr||S |t | jkr&| jd| }|r$| j|d | _|S |t | j }| j|| }|r:d| _|S )a=  
        Reads a specific amount of data from a stream and returns it. If there
        is any data in initial_data, that will be popped out first.

        :type fileobj: A file-like object that implements read
        :param fileobj: The stream to read from.

        :type amount: int
        :param amount: The number of bytes to read from the stream.

        :type truncate: bool
        :param truncate: Whether or not to truncate initial_data after
            reading from it.

        :return: Generator which generates part bodies from the initial data.
        r   Nr|   )ry   r~   r*   )r   r'   r+   truncaterz   Zamount_to_readr   r   r   r     s   
z#UploadNonSeekableInputManager._readc                 C   s,   |  t|}| jj|t|t|||dS )a  
        Wraps data with the interrupt reader and the file chunk reader.

        :type data: bytes
        :param data: The data to wrap.

        :type callbacks: list
        :param callbacks: The callbacks associated with the transfer future.

        :type close_callbacks: list
        :param close_callbacks: The callbacks to be called when closing the
            wrapper for the data.

        :return: Fully wrapped data.
        ra   )rS   r   r:   rf   ry   )r   rz   r   rd   r'   r   r   r   r     s   z(UploadNonSeekableInputManager._wrap_datar   )T)r!   r"   r#   r8   r   rX   rD   rG   rK   rN   rO   rQ   r   r   __classcell__r   r   r   r   r{   v  s    

*r{   c                   @   s\   e Zd ZdZg dZddgZdd Z	ddd	Zd
d Zdd Z	dd Z
dd Zdd ZdS )UploadSubmissionTaskz.Task for submitting tasks to execute an upload)ChecksumAlgorithmZSSECustomerKeyZSSECustomerAlgorithmZSSECustomerKeyMD5RequestPayerExpectedBucketOwnerr   r   c                 C   sD   t ttg}|jjj}|D ]}||r|  S qtd|t	|)ao  Retrieves a class for managing input for an upload based on file type

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The transfer future for the request

        :rtype: class of UploadInputManager
        :returns: The appropriate class to use for managing a specific type of
            input for uploads.
        z&Input {} of type: {} is not supported.)
rY   rv   r{   r]   r^   r'   rD   RuntimeErrorformattype)r   rJ   Zupload_manager_resolver_chainr'   Zupload_manager_clsr   r   r   _get_upload_input_manager_cls  s   

z2UploadSubmissionTask._get_upload_input_manager_clsNc                 C   sf   |  ||| j|}|jjdu r|| |||s'| |||||| dS | |||||| dS )a  
        :param client: The client associated with the transfer manager

        :type config: s3transfer.manager.TransferConfig
        :param config: The transfer config associated with the transfer
            manager

        :type osutil: s3transfer.utils.OSUtil
        :param osutil: The os utility associated to the transfer manager

        :type request_executor: s3transfer.futures.BoundedExecutor
        :param request_executor: The request executor associated with the
            transfer manager

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The transfer future associated with the
            transfer request that tasks are being submitted for
        N)r   r&   r]   r_   rK   rN   _submit_upload_request_submit_multipart_request)r   clientrM   r=   request_executorrJ   r>   upload_input_managerr   r   r   _submit(  s4   
	zUploadSubmissionTask._submitc           	   
   C   sN   |j j}| |d}| jj|t| j||||j|j|j	ddd|d d S )Nrw   )r   r'   bucketkey
extra_argsT)r(   main_kwargsis_finaltag)
r]   r^   _get_upload_task_tagr&   submitPutObjectTaskrO   r   r   r   )	r   r   rM   r=   r   rJ   r   r^   Zput_object_tagr   r   r   r   a  s(   		
z+UploadSubmissionTask._submit_upload_requestc                 C   s   |j j}| j|t| j||j|j|jdd}g }	| |j}
| 	|d}|j j
}t }||j|}|||}|D ]!\}}|	| jj|t| j|||j|j||
dd|id|d q<| |j}| j|t| j||j|j|d||	dd	d
 d S )N)r   r   r   r   )r(   r   upload_part)r   r'   r   r   rn   r   	upload_id)r(   r   pending_main_kwargsr   )r   partsT)r(   r   r   r   )r]   r^   r&   r   r   r   r   r   _extra_upload_part_argsr   r_   r   Zadjust_chunksizeZmultipart_chunksizerQ   appendUploadPartTask_extra_complete_multipart_argsr   )r   r   rM   r=   r   rJ   r   r^   Zcreate_multipart_futureZpart_futuresZextra_part_argsZupload_part_tagr_   ZadjusterrP   Zpart_iteratorrn   r'   Zcomplete_multipart_extra_argsr   r   r   r     sx   		z.UploadSubmissionTask._submit_multipart_requestc                 C      t || jS r   )r   UPLOAD_PART_ARGSr   r   r   r   r   r     s   z,UploadSubmissionTask._extra_upload_part_argsc                 C   r   r   )r   COMPLETE_MULTIPART_ARGSr   r   r   r   r     r7   z3UploadSubmissionTask._extra_complete_multipart_argsc                 C   s   d }| |r	t}|S r   )rG   r   )r   r   rF   r   r   r   r   r     s   
z)UploadSubmissionTask._get_upload_task_tagr   )r!   r"   r#   r8   r   r   r   r   r   r   r   r   r   r   r   r   r   r      s    	!
9#Vr   c                   @      e Zd ZdZdd ZdS )r   z Task to do a nonmultipart uploadc                 C   sB   |}|j d|||d| W d   dS 1 sw   Y  dS )aP  
        :param client: The client to use when calling PutObject
        :param fileobj: The file to upload.
        :param bucket: The name of the bucket to upload to
        :param key: The name of the key to upload to
        :param extra_args: A dictionary of any extra arguments that may be
            used in the upload.
        )BucketKeyBodyNr   )rw   )r   r   r'   r   r   r   r   r   r   r   _main  s   	"zPutObjectTask._mainNr!   r"   r#   r8   r   r   r   r   r   r         r   c                   @   r   )r   z+Task to upload a part in a multipart uploadc              	   C   s   |}|j d|||||d|}	W d   n1 sw   Y  |	d }
|
|d}d|v rB|d  }d| }||	v rB|	| ||< |S )a  
        :param client: The client to use when calling PutObject
        :param fileobj: The file to upload.
        :param bucket: The name of the bucket to upload to
        :param key: The name of the key to upload to
        :param upload_id: The id of the upload
        :param part_number: The number representing the part of the multipart
            upload
        :param extra_args: A dictionary of any extra arguments that may be
            used in the upload.

        :rtype: dict
        :returns: A dictionary representing a part::

            {'Etag': etag_value, 'PartNumber': part_number}

            This value can be appended to a list to be used to complete
            the multipart upload.
        )r   r   ZUploadId
PartNumberr   NETag)r   r   r   ZChecksumr   )r   upper)r   r   r'   r   r   r   rn   r   r   responseZetagZpart_metadataZalgorithm_nameZchecksum_memberr   r   r   r     s&   	

zUploadPartTask._mainNr   r   r   r   r   r     r   r   )rs   ior   Zs3transfer.compatr   r   Zs3transfer.futuresr   Zs3transfer.tasksr   r   r	   r
   Zs3transfer.utilsr   r   r   r   r   r$   r9   rY   rv   r{   r   r   r   r   r   r   r   <module>   s"   !,{]1  j