File: //lib/python3.6/site-packages/awscli/customizations/s3/__pycache__/s3handler.cpython-36.pyc
3
�T_[ � @ s: d dl Z d dlZd dlmZ d dlmZmZmZmZm Z m
Z
d dl m Z d dl
mZ d dl
mZ d dl
mZ d dl
mZ d d l
mZ d d
l
mZ d d l
mZ d d l
mZ d d
l
mZ d dl
mZ d dl
mZ d dl
mZ d dl
mZ d dl
mZ d dl
m Z d dl
m Z d dlm Z d dlm Z d dlm Z d dlm!Z! d dlm"Z" d dlm#Z# d dlm$Z$ d d lm%Z% d d lm&Z& d d lm'Z' d d l(m)Z) e j*e+�Z,G d d!� d!e-�Z.G d"d#� d#e-�Z/G d$d%� d%e-�Z0G d&d'� d'e0�Z1G d(d)� d)e0�Z2G d*d+� d+e0�Z3G d,d-� d-e1�Z4G d.d/� d/e2�Z5G d0d1� d1e0�Z6G d2d3� d3e0�Z7dS )4� N)�TransferManager)�human_readable_size�MAX_UPLOAD_SIZE�find_bucket_key�
relative_path�create_warning�NonSeekableStream)�*create_transfer_config_from_runtime_config)�UploadResultSubscriber)�DownloadResultSubscriber)�CopyResultSubscriber)� UploadStreamResultSubscriber)� DownloadStreamResultSubscriber)�DeleteResultSubscriber)� QueuedResult)�
SuccessResult)�
FailureResult)� DryRunResult)�ResultRecorder)�
ResultPrinter)�OnlyShowErrorsResultPrinter)�NoProgressResultPrinter)�ResultProcessor)�CommandResultRecorder)�RequestParamsMapper)�StdoutBytesWriter)�ProvideSizeSubscriber)�"ProvideUploadContentTypeSubscriber)� ProvideCopyContentTypeSubscriber)�!ProvideLastModifiedTimeSubscriber)�DirectoryCreatorSubscriber)�DeleteSourceFileSubscriber)� DeleteSourceObjectSubscriber)� DeleteCopySourceObjectSubscriber)�get_binary_stdinc @ s( e Zd ZdZdd� Zdd� Zdd� ZdS ) �S3TransferHandlerFactory� c C s || _ || _dS )a Factory for S3TransferHandlers
:type cli_params: dict
:param cli_params: The parameters provide to the CLI command
:type runtime_config: RuntimeConfig
:param runtime_config: The runtime config for the CLI command
being run
N)� _cli_params�_runtime_config)�self�
cli_paramsZruntime_config� r+ � /usr/lib/python3.6/s3handler.py�__init__: s
z!S3TransferHandlerFactory.__init__c C sr t | j�}| j|_| j|_t||�}tjd|j|j � t
� }|g}| j ||� t ||�}t
|||�}t|| j|�S )ag Creates a S3TransferHandler instance
:type client: botocore.client.Client
:param client: The client to power the S3TransferHandler
:type result_queue: queue.Queue
:param result_queue: The result queue to be used to process results
for the S3TransferHandler
:returns: A S3TransferHandler instance
z7Using a multipart threshold of %s and a part size of %s)r r( �MAX_IN_MEMORY_CHUNKSZmax_in_memory_upload_chunksZ max_in_memory_download_chunksr �LOGGER�debugZmultipart_thresholdZmultipart_chunksizer �_add_result_printerr r �S3TransferHandlerr' ) r) Zclient� result_queueZtransfer_config�transfer_manager�result_recorder�result_processor_handlersZresult_processorZcommand_result_recorderr+ r+ r, �__call__G s$
z!S3TransferHandlerFactory.__call__c C sh | j jd�rd S | j jd�r&t|�}n4| j jd�r<t|�}n | j jd�sRt|�}nt|�}|j|� d S )N�quietZonly_show_errors� is_streamZprogress)r' �getr r r �append)r) r5 r6 Zresult_printerr+ r+ r, r1 k s
z,S3TransferHandlerFactory._add_result_printerN)�__name__�
__module__� __qualname__r. r- r7 r1 r+ r+ r+ r, r% 7 s
$r% c @ s e Zd Zdd� Zdd� ZdS )r2 c C sP || _ || _| j | jj|f}t|� t|� t|� t|� t|� t|� t |� g| _
dS )a Backend for performing S3 transfers
:type transfer_manager: s3transfer.manager.TransferManager
:param transfer_manager: Transfer manager to use for transfers
:type cli_params: dict
:param cli_params: The parameters passed to the CLI command in the
form of a dictionary
:type result_command_recorder: ResultCommandRecorder
:param result_command_recorder: The result command recorder to be
used to get the final result of the transfer
N) �_transfer_manager�_result_command_recorderr3 � UploadStreamRequestSubmitter� DownloadStreamRequestSubmitter�UploadRequestSubmitter�DownloadRequestSubmitter�CopyRequestSubmitter�DeleteRequestSubmitter�LocalDeleteRequestSubmitter� _submitters)r) r4 r* Zresult_command_recorderZsubmitter_argsr+ r+ r, r- z s
zS3TransferHandler.__init__c C sz | j �d | j�R d}x:|D ]2}x,| jD ]"}|j|�r&|j|�rF|d7 }P q&W qW | j j|� W dQ R X W dQ R X | j j� S )a� Process iterable of FileInfos for transfer
:type fileinfos: iterable of FileInfos
param fileinfos: Set of FileInfos to submit to underlying transfer
request submitters to make transfer API calls to S3
:rtype: CommandResult
:returns: The result of the command that specifies the number of
failures and warnings encountered.
r � N)r@ r? rH �
can_submit�submitZnotify_total_submissionsZget_command_result)r) Z fileinfosZtotal_submissions�fileinfoZ submitterr+ r+ r, �call� s
zS3TransferHandler.callN)r< r= r> r- rM r+ r+ r+ r, r2 y s %r2 c @ s� e Zd ZdZdZdd� Zdd� Zdd� Zdd � Zd
d � Z d d
� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zdd� Zdd� Zd d � Zd d � ZdS ) � BaseTransferRequestSubmitterNc C s || _ || _|| _dS )a\ Submits transfer requests to the TransferManager
Given a FileInfo object and provided CLI parameters, it will add the
necessary extra arguments and subscribers in making a call to the
TransferManager.
:type transfer_manager: s3transfer.manager.TransferManager
:param transfer_manager: The underlying transfer manager
:type result_queue: queue.Queue
:param result_queue: The result queue to use
:type cli_params: dict
:param cli_params: The associated CLI parameters passed in to the
command as a dictionary.
N)r? �
_result_queuer' )r) r4 r3 r* r+ r+ r, r- � s z%BaseTransferRequestSubmitter.__init__c C s | j |�}|s| j|�S dS )a� Submits a transfer request based on the FileInfo provided
There is no guarantee that the transfer request will be made on
behalf of the fileinfo as a fileinfo may be skipped based on
circumstances in which the transfer is not possible.
:type fileinfo: awscli.customizations.s3.fileinfo.FileInfo
:param fileinfo: The FileInfo to be used to submit a transfer
request to the underlying transfer manager.
:rtype: s3transfer.futures.TransferFuture
:returns: A TransferFuture representing the transfer if it the
transfer was submitted. If it was not submitted nothing
is returned.
N)�_warn_and_signal_if_skip�
_do_submit)r) rL Z should_skipr+ r+ r, rK � s
z#BaseTransferRequestSubmitter.submitc C s t d��dS )a� Checks whether it can submit a particular FileInfo
:type fileinfo: awscli.customizations.s3.fileinfo.FileInfo
:param fileinfo: The FileInfo to check if the transfer request
submitter can handle.
:returns: True if it can use the provided FileInfo to make a transfer
request to the underlying transfer manager. False, otherwise.
z can_submit()N)�NotImplementedError)r) rL r+ r+ r, rJ � s
z'BaseTransferRequestSubmitter.can_submitc C s� i }| j r| j || j� g }| j||� | jr`d| ji}| jjdd�rNd|d< |j| jf |�� | jjd�sz| j|||�S | j|� d S )Nr3 �is_moveF�move�
transfer_typeZdryrun) �REQUEST_MAPPER_METHODr' �_add_additional_subscribers�RESULT_SUBSCRIBER_CLASSrO r: r; �_submit_transfer_request�_submit_dryrun)r) rL �
extra_args� subscribers�
result_kwargsr+ r+ r, rQ � s
z'BaseTransferRequestSubmitter._do_submitc C s@ |j }| jjdd�rd}| j|�\}}| jjt|||d�� d S )NrS FrT )rU �src�dest)�operation_namer' r: �_format_src_destrO �putr )r) rL rU r^ r_ r+ r+ r, rZ s z+BaseTransferRequestSubmitter._submit_dryrunc C s d S )Nr+ )r) r\ rL r+ r+ r, rW s z8BaseTransferRequestSubmitter._add_additional_subscribersc C s t d��d S )Nz_submit_transfer_request())rR )r) rL r[ r\ r+ r+ r, rY s z5BaseTransferRequestSubmitter._submit_transfer_requestc C s" x | j � D ]}||�r
dS q
W d S )NT)�_get_warning_handlers)r) rL Zwarning_handlerr+ r+ r, rP s z5BaseTransferRequestSubmitter._warn_and_signal_if_skipc C s g S )Nr+ )r) r+ r+ r, rc s z2BaseTransferRequestSubmitter._get_warning_handlersc C s | j jd�o| j jd� S )NZguess_mime_typeZ content_type)r' r: )r) r+ r+ r, �_should_inject_content_type$ s z8BaseTransferRequestSubmitter._should_inject_content_typec C sf | j jd�sb|j� sbtjd|j|jf � | j jd�s^td|j d|j|jf �}| jj |� dS dS )NZforce_glacier_transferz@Encountered glacier object s3://%s. Not performing %s on object.Zignore_glacier_warningszs3://z�Object is of storage class GLACIER. Unable to perform %s operations on GLACIER objects. You must restore the object to be able to perform the operation. See aws s3 %s help for additional parameter options to ignore or force these transfers.TF)
r' r: Zis_glacier_compatibler/ r0 r^ r` r rO rb )r) rL �warningr+ r+ r, �
_warn_glacier* s z*BaseTransferRequestSubmitter._warn_glacierc C sD dt jj }t jj|j�j|�}|r@t|jd�}| jj|� dS dS )Nz..z#File references a parent directory.TF) �os�path�sep�normpathZ compare_key�
startswithr rO rb )r) rL Z
parent_prefixZ escapes_cwdre r+ r+ r, �_warn_parent_reference? s
z3BaseTransferRequestSubmitter._warn_parent_referencec C s t d��dS )zAReturns formatted versions of a fileinfos source and destination.ra N)rR )r) rL r+ r+ r, ra L s z-BaseTransferRequestSubmitter._format_src_destc C s t |�S )N)r )r) rh r+ r+ r, �_format_local_pathP s z/BaseTransferRequestSubmitter._format_local_pathc C s |j d�r|S d| S )Nzs3://)rk )r) rh r+ r+ r, �_format_s3_pathS s
z,BaseTransferRequestSubmitter._format_s3_path)r< r= r> rV rX r- rK rJ rQ rZ rW rY rP rc rd rf rl ra rm rn r+ r+ r+ r, rN � s"
rN c @ sN e Zd ZejZeZdd� Zdd� Z dd� Z
dd� Z d d
� Z d d � Z
d
d� ZdS )rC c C s
|j dkS )N�upload)r` )r) rL r+ r+ r, rJ ] s z!UploadRequestSubmitter.can_submitc C sB |j t|j�� | j� r$|j t� � | jjdd�r>|j t� � d S )NrS F)r; r �sizerd r r' r: r! )r) r\ rL r+ r+ r, rW ` s
z2UploadRequestSubmitter._add_additional_subscribersc C s. t |j�\}}| j|�}| jj|||||d�S )N)�fileobj�bucket�keyr[ r\ )r r_ � _get_fileinr? ro )r) rL r[ r\ rr rs Zfileinr+ r+ r, rY g s
z/UploadRequestSubmitter._submit_transfer_requestc C s |j S )N)r^ )r) rL r+ r+ r, rt o s z"UploadRequestSubmitter._get_fileinc C s | j gS )N)�_warn_if_too_large)r) r+ r+ r, rc r s z,UploadRequestSubmitter._get_warning_handlersc C sL t |d�rH|jtkrHt|j�}d|tt�f }t||dd�}| jj|� d S )Nrp z&File %s exceeds s3 upload limit of %s.F)Z skip_file) �getattrrp r r r^ r r rO rb )r) rL Z file_pathZwarning_messagere r+ r+ r, ru u s
z)UploadRequestSubmitter._warn_if_too_largec C s | j |j�}| j|j�}||fS )N)rm r^ rn r_ )r) rL r^ r_ r+ r+ r, ra s z'UploadRequestSubmitter._format_src_destN)r< r= r> r Zmap_put_object_paramsrV r
rX rJ rW rY rt rc ru ra r+ r+ r+ r, rC Y s
rC c @ sF e Zd ZejZeZdd� Zdd� Z dd� Z
dd� Z d d
� Z d d � Z
d
S )rD c C s
|j dkS )N�download)r` )r) rL r+ r+ r, rJ � s z#DownloadRequestSubmitter.can_submitc C sR |j t|j�� |j t� � |j t|j| j�� | jjdd�rN|j t |j
�� d S )NrS F) r; r rp r r Z last_updaterO r' r: r" �
source_client)r) r\ rL r+ r+ r, rW � s z4DownloadRequestSubmitter._add_additional_subscribersc C s. t |j�\}}| j|�}| jj|||||d�S )N)rq rr rs r[ r\ )r r^ � _get_fileoutr? rw )r) rL r[ r\ rr rs Zfileoutr+ r+ r, rY � s
z1DownloadRequestSubmitter._submit_transfer_requestc C s |j S )N)r_ )r) rL r+ r+ r, ry � s z%DownloadRequestSubmitter._get_fileoutc C s | j | jgS )N)rf rl )r) r+ r+ r, rc � s z.DownloadRequestSubmitter._get_warning_handlersc C s | j |j�}| j|j�}||fS )N)rn r^ rm r_ )r) rL r^ r_ r+ r+ r, ra � s z)DownloadRequestSubmitter._format_src_destN)r< r= r> r Zmap_get_object_paramsrV r rX rJ rW rY ry rc ra r+ r+ r+ r, rD � s rD c @ s> e Zd ZejZeZdd� Zdd� Z dd� Z
dd� Z d d
� Z d S ) rE c C s
|j dkS )N�copy)r` )r) rL r+ r+ r, rJ � s z CopyRequestSubmitter.can_submitc C sF |j t|j�� | j� r$|j t� � | jjdd�rB|j t|j�� d S )NrS F) r; r rp rd r r' r: r# rx )r) r\ rL r+ r+ r, rW � s z0CopyRequestSubmitter._add_additional_subscribersc C s@ t |j�\}}t |j�\}}||d�}| jj||||||jd�S )N)ZBucketZKey)rr rs � copy_sourcer[ r\ rx )r r_ r^ r? rz rx ) r) rL r[ r\ rr rs Z
source_bucketZ
source_keyr{ r+ r+ r, rY � s
z-CopyRequestSubmitter._submit_transfer_requestc C s | j gS )N)rf )r) r+ r+ r, rc � s z*CopyRequestSubmitter._get_warning_handlersc C s | j |j�}| j |j�}||fS )N)rn r^ r_ )r) rL r^ r_ r+ r+ r, ra � s z%CopyRequestSubmitter._format_src_destN)
r< r= r> r Zmap_copy_object_paramsrV r rX rJ rW rY rc ra r+ r+ r+ r, rE � s
rE c @ s0 e Zd ZeZdd� Zdd� Zdd� Zdd� Zd S )
rA c C s |j dko| jjd�S )Nro r9 )r` r' r: )r) rL r+ r+ r, rJ � s
z'UploadStreamRequestSubmitter.can_submitc C s, | j jdd �}|d k r(|jtt|��� d S )N�
expected_size)r' r: r; r �int)r) r\ rL r| r+ r+ r, rW � s z8UploadStreamRequestSubmitter._add_additional_subscribersc C s t � }t|�S )N)r$ r )r) rL Z binary_stdinr+ r+ r, rt � s z(UploadStreamRequestSubmitter._get_fileinc C s dS )N�-r+ )r) rh r+ r+ r, rm � s z/UploadStreamRequestSubmitter._format_local_pathN) r< r= r> r
rX rJ rW rt rm r+ r+ r+ r, rA � s
rA c @ s0 e Zd ZeZdd� Zdd� Zdd� Zdd� Zd S )
rB c C s |j dko| jjd�S )Nrw r9 )r` r' r: )r) rL r+ r+ r, rJ � s
z)DownloadStreamRequestSubmitter.can_submitc C s d S )Nr+ )r) r\ rL r+ r+ r, rW � s z:DownloadStreamRequestSubmitter._add_additional_subscribersc C s t � S )N)r )r) rL r+ r+ r, ry � s z+DownloadStreamRequestSubmitter._get_fileoutc C s dS )Nr~ r+ )r) rh r+ r+ r, rm � s z1DownloadStreamRequestSubmitter._format_local_pathN) r< r= r> r rX rJ rW ry rm r+ r+ r+ r, rB � s
rB c @ s. e Zd ZejZeZdd� Zdd� Z dd� Z
dS )rF c C s |j dko|jdkS )N�deleteZs3)r` �src_type)r) rL r+ r+ r, rJ � s
z!DeleteRequestSubmitter.can_submitc C s" t |j�\}}| jj||||d�S )N)rr rs r[ r\ )r r^ r? r )r) rL r[ r\ rr rs r+ r+ r, rY � s z/DeleteRequestSubmitter._submit_transfer_requestc C s | j |j�d fS )N)rn r^ )r) rL r+ r+ r, ra s z'DeleteRequestSubmitter._format_src_destN) r< r= r> r Zmap_delete_object_paramsrV r rX rJ rY ra r+ r+ r+ r, rF � s
rF c @ s, e Zd ZdZdZdd� Zdd� Zdd� ZdS )rG Nc C s |j dko|jdkS )Nr Zlocal)r` r� )r) rL r+ r+ r, rJ
s
z&LocalDeleteRequestSubmitter.can_submitc C s� | j |�\}}d||d�}z~y<| jjtf ddi|��� tj|j� | jjtf |�� W n< tk
r� } z | jjt f d|i|��� W Y d d }~X nX W d dS )Nr )rU r^ r_ Ztotal_transfer_sizer Z exceptionT)
ra rO rb r rg �remover^ r � Exceptionr )r) rL r[ r\ r^ r_ r] �er+ r+ r, rY s
*z4LocalDeleteRequestSubmitter._submit_transfer_requestc C s | j |j�d fS )N)rm r^ )r) rL r+ r+ r, ra - s z,LocalDeleteRequestSubmitter._format_src_dest)r< r= r> rV rX rJ rY ra r+ r+ r+ r, rG s
rG )8Zloggingrg Zs3transfer.managerr Z awscli.customizations.s3.utilsr r r r r r Z'awscli.customizations.s3.transferconfigr Z awscli.customizations.s3.resultsr
r r r
r r r r r r r r r r r r r r r r r r r r! r" r# Z
awscli.compatr$ Z getLoggerr< r/ �objectr% r2 rN rC rD rE rA rB rF rG r+ r+ r+ r, �<module>
sV
B? ",$"