File: //lib/python3.6/site-packages/awscli/customizations/s3/__pycache__/results.cpython-36.opt-1.pyc
3
�T_ h � @ s< d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl m
Z
d dl m Z d dl m
Z
d dlmZmZ d d lmZ d d
lmZ d d lmZ d d lmZ d d
lmZ eje�Zeddddg�Zdefdd�Z e ddg�Z e ddddg�Z e d�Z e ddg�Z e d �Z!ed dg�Z"e d e"d �Z#ed d!d"g�Z$ed#d$g�Z%G d%d&� d&e&�Z'G d'd(� d(e�Z(G d)d*� d*e(�Z)G d+d,� d,e)�Z*G d-d.� d.e(�Z+G d/d0� d0e+�Z,G d1d2� d2e(�Z-G d3d4� d4e(�Z.G d5d6� d6e&�Z/G d7d8� d8e/�Z0G d9d:� d:e/�Z1G d;d<� d<e1�Z2G d=d>� d>e1�Z3G d?d@� d@ej4�Z5G dAdB� dBe&�Z6dS )C� )�divisionN)�
namedtuple)� defaultdict)�CancelledError)�
FatalError)�BaseSubscriber)�queue�ensure_text_type)�
relative_path)�human_readable_size)� uni_print)�
WarningResult)�OnDoneFilteredSubscriber�
BaseResult�
transfer_type�src�destc C s, t |j�}|r||7 }t| t| |�|fi �S )N)�list�_fields�typer )�nameZ extra_fields�base_clsZfields� r � /usr/lib/python3.6/results.py�_create_new_result_cls' s
r � QueuedResult�total_transfer_size�ProgressResult�bytes_transferred� timestamp�
SuccessResult�
FailureResult� exception� DryRunResult� ErrorResult� CtrlCResult)r �
CommandResultZnum_tasks_failedZnum_tasks_warned�FinalTotalSubmissionsResult�total_submissionsc @ s e Zd ZdS )�ShutdownThreadRequestN)�__name__�
__module__� __qualname__r r r r r) K s r) c @ sR e Zd ZdZddd�Zdd� Zdd� Zdd � Zd
d � Zd d
� Z dd� Z
dd� Z dS )�BaseResultSubscriberNc C s& || _ i | _|| _|dkr"| j| _dS )z�Subscriber to send result notifications during transfer process
:param result_queue: The queue to place results to be processed later
on.
N)�
_result_queue�_result_kwargs_cache�_transfer_type�
TRANSFER_TYPE)�self� result_queuer r r r �__init__R s
z BaseResultSubscriber.__init__c K s2 | j |� | j|jj }tf |�}| jj|� d S )N)�_add_to_result_kwargs_cacher/ �meta� transfer_idr r. �put)r2 �future�kwargs�
result_kwargsZ
queued_resultr r r � on_queued^ s
z BaseResultSubscriber.on_queuedc K s6 | j |jj }tf |tj� d�|��}| jj|� d S )N)r r )r/ r6 r7 r �timer. r8 )r2 r9 r r: r; Zprogress_resultr r r � on_progressd s
z BaseResultSubscriber.on_progressc C s | j |�}| jjtf |�� d S )N)�%_on_done_pop_from_result_kwargs_cacher. r8 r )r2 r9 r; r r r � _on_successk s
z BaseResultSubscriber._on_successc C sX | j |�}t|t�r:t}t|t�r&t}| jj||d�� n| jjtf d|i|��� d S )N)r" r" ) r? �
isinstancer r% r r$ r. r8 r! )r2 r9 �er; Zerror_result_clsr r r � _on_failureo s
z BaseResultSubscriber._on_failurec C s4 | j |�\}}| j|||jjd�}|| j|jj< d S )N)r r r r )�
_get_src_destr0 r6 �sizer/ r7 )r2 r9 r r r; r r r r5 y s z0BaseResultSubscriber._add_to_result_kwargs_cachec C s | j j|jj�}|jd� |S )Nr )r/ �popr6 r7 )r2 r9 r; r r r r? � s
z:BaseResultSubscriber._on_done_pop_from_result_kwargs_cachec C s t d��d S )Nz_get_src_dest())�NotImplementedError)r2 r9 r r r rD � s z"BaseResultSubscriber._get_src_dest)N) r* r+ r, r1 r4 r< r> r@ rC r5 r? rD r r r r r- O s
r- c @ s e Zd ZdZdd� Zdd� ZdS )�UploadResultSubscriberZuploadc C s0 |j j}| j|j�}d|j d |j }||fS )Nzs3://�/)r6 � call_args�_get_src�fileobj�bucket�key)r2 r9 rJ r r r r r rD � s z$UploadResultSubscriber._get_src_destc C s t |�S )N)r
)r2 rL r r r rK � s z UploadResultSubscriber._get_srcN)r* r+ r, r1 rD rK r r r r rH � s rH c @ s e Zd Zdd� ZdS )� UploadStreamResultSubscriberc C s dS )N�-r )r2 rL r r r rK � s z%UploadStreamResultSubscriber._get_srcN)r* r+ r, rK r r r r rO � s rO c @ s e Zd ZdZdd� Zdd� ZdS )�DownloadResultSubscriberZdownloadc C s0 |j j}d|j d |j }| j|j�}||fS )Nzs3://rI )r6 rJ rM rN � _get_destrL )r2 r9 rJ r r r r r rD � s z&DownloadResultSubscriber._get_src_destc C s t |�S )N)r
)r2 rL r r r rR � s z"DownloadResultSubscriber._get_destN)r* r+ r, r1 rD rR r r r r rQ � s rQ c @ s e Zd Zdd� ZdS )� DownloadStreamResultSubscriberc C s dS )NrP r )r2 rL r r r rR � s z(DownloadStreamResultSubscriber._get_destN)r* r+ r, rR r r r r rS � s rS c @ s e Zd ZdZdd� ZdS )�CopyResultSubscriber�copyc C sB |j j}|j}d|d d |d }d|j d |j }||fS )Nzs3://ZBucketrI ZKey)r6 rJ � copy_sourcerM rN )r2 r9 rJ rV r r r r r rD � s
z"CopyResultSubscriber._get_src_destN)r* r+ r, r1 rD r r r r rT � s rT c @ s e Zd ZdZdd� ZdS )�DeleteResultSubscriber�deletec C s$ |j j}d|j d |j }|d fS )Nzs3://rI )r6 rJ rM rN )r2 r9 rJ r r r r rD � s z$DeleteResultSubscriber._get_src_destN)r* r+ r, r1 rD r r r r rW � s rW c @ s e Zd ZdZdd� ZdS )�BaseResultHandlerz6Base handler class to be called in the ResultProcessorc C s t d��d S )Nz
__call__())rG )r2 �resultr r r �__call__� s zBaseResultHandler.__call__N)r* r+ r, �__doc__r[ r r r r rY � s rY c @ s� e Zd ZdZdd� Zdd� Zdd� Zdd � Zd
d � Zd d
� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zdd� Zdd� Zd d � Zd S ) �ResultRecorderz@Records and track transfer statistics based on results receievedc C s� d| _ d| _d| _d| _d| _d| _d| _d| _d | _d | _ d| _
t t �| _
i | _t| jt| jt| jt| jt| jt| jt| jt | j i| _ d S )Nr ) r �bytes_failed_to_transfer�files_transferred� files_failed� files_warned�errors�expected_bytes_transferred�expected_files_transferred� final_expected_files_transferred�
start_time�bytes_transfer_speedr �int�_ongoing_progress�_ongoing_total_sizesr �_record_queued_resultr �_record_progress_resultr �_record_success_resultr! �_record_failure_resultr
�_record_warning_resultr$ �_record_error_resultr% r' � _record_final_expected_files�_result_handler_map)r2 r r r r4 � s*
zResultRecorder.__init__c C s | j | jkS )N)re rd )r2 r r r �expected_totals_are_final� s z(ResultRecorder.expected_totals_are_finalc C s | j jt|�| j�|d� dS )z0Record the result of an individual Result object)rZ N)rr �getr � _record_noop)r2 rZ r r r r[ � s zResultRecorder.__call__c C sX t |t�stdt|� ��g }x.|j|j|jgD ]}|d k r0|jt|�� q0W dj |�S )NzdAny result using _get_ongoing_dict_key must subclass from BaseResult. Provided result is of type: %s�:)
rA r �
ValueErrorr r r r �appendr �join)r2 rZ Z key_partsZresult_propertyr r r �_get_ongoing_dict_key� s
z$ResultRecorder._get_ongoing_dict_keyc C s. | j |�}| jj|d�}| jj|d �}||fS )Nr )rz ri rF rj )r2 rZ � ongoing_key�total_progress�total_file_sizer r r � _pop_result_from_ongoing_dicts� s
z-ResultRecorder._pop_result_from_ongoing_dictsc K s d S )Nr )r2 r: r r r ru s zResultRecorder._record_noopc K sN | j d krtj� | _ |j}|| j| j|�< |r<| j|7 _| jd7 _d S )N� )rf r= r rj rz rc rd )r2 rZ r: r r r r rk s
z$ResultRecorder._record_queued_resultc K sZ |j }| j|� | j| j|� |7 < | j |7 _ |j| jkrV| j |j| j | _d S )N)r �(_update_ongoing_transfer_size_if_unknownri rz r rf rg )r2 rZ r: r r r r rl s
z&ResultRecorder._record_progress_resultc C sh | j |�}| j| d krd|j}|jd k rT|| j|< | j| }|| }| j|7 _n| j|j7 _d S )N)rz rj r ri rc r )r2 rZ r{ r Zongoing_progressZunaccounted_bytesr r r r� & s
z7ResultRecorder._update_ongoing_transfer_size_if_unknownc K s | j |� | jd7 _d S )Nr )r~ r_ )r2 rZ r: r r r rm @ s
z%ResultRecorder._record_success_resultc K sL | j |�\}}|d k r,|| }| j|7 _| jd7 _| jd7 _d S )Nr )r~ r^ r` r_ )r2 rZ r: r| r} Z
progress_leftr r r rn D s
z%ResultRecorder._record_failure_resultc K s | j d7 _ d S )Nr )ra )r2 r: r r r ro Q s z%ResultRecorder._record_warning_resultc K s | j d7 _ d S )Nr )rb )r2 r: r r r rp T s z#ResultRecorder._record_error_resultc K s |j | _d S )N)r( re )r2 rZ r: r r r rq W s z+ResultRecorder._record_final_expected_filesN)r* r+ r, r\ r4 rs r[ rz r~ ru rk rl r� rm rn ro rp rq r r r r r] � s
r] c @ s� e Zd ZdZdZdZde Zde ZdZde Z dZ
d Z d
Z d Z
d Zd
Zd8dd�Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Z d)d*� Z d+d,� Z d9d.d/�Z d0d1� Z d2d3� Z!d4d5� Z"d6d7� Z#dS ):�
ResultPrinterz#{remaining_files} file(s) remainingz~{expected_total}z (calculating...)zOCompleted {bytes_completed}/{expected_bytes_completed} ({transfer_speed}) with z)Completed {files_completed} file(s) with z${transfer_type}: {transfer_location}z (dryrun) z7{transfer_type} failed: {transfer_location} {exception}z {message}zfatal error: {exception}zcancelled: ctrl-c receivedz{src} to {dest}z{src}Nc C sv || _ || _| jdkr tj| _|| _| jdkr6tj| _d| _t| jt | j
t | j t
| jt| jt| jt| jt| ji| _dS )a� Prints status of ongoing transfer
:type result_recorder: ResultRecorder
:param result_recorder: The associated result recorder
:type out_file: file-like obj
:param out_file: Location to write progress and success statements.
By default, the location is sys.stdout.
:type error_file: file-like obj
:param error_file: Location to write warnings and errors.
By default, the location is sys.stderr.
Nr )�_result_recorder� _out_file�sys�stdout� _error_file�stderr�_progress_lengthr �_print_progressr �_print_successr! �_print_failurer
�_print_warningr$ � _print_errorr% �
_print_ctrl_cr# �_print_dry_runr' �-_clear_progress_if_no_more_expected_transfersrr )r2 �result_recorderZout_fileZ
error_filer r r r4 z s"
zResultPrinter.__init__c C s | j jt|�| j�|d� dS )z<Print the progress of the ongoing transfer based on a result)rZ N)rr rt r � _print_noop)r2 rZ r r r r[ � s zResultPrinter.__call__c K s d S )Nr )r2 r: r r r r� � s zResultPrinter._print_noopc K s0 | j j|j| j|�d�}| j|�}| j|� d S )N)r �transfer_location)�DRY_RUN_FORMAT�formatr �_get_transfer_location�_adjust_statement_padding�_print_to_out_file)r2 rZ r: � statementr r r r� � s
z ResultPrinter._print_dry_runc K s8 | j j|j| j|�d�}| j|�}| j|� | j� d S )N)r r� )�SUCCESS_FORMATr� r r� r� r� �_redisplay_progress)r2 rZ r: Zsuccess_statementr r r r� � s
z ResultPrinter._print_successc K s< | j j|j| j|�|jd�}| j|�}| j|� | j� d S )N)r r� r" )�FAILURE_FORMATr� r r� r" r� �_print_to_error_filer� )r2 rZ r: Zfailure_statementr r r r� � s
z ResultPrinter._print_failurec K s0 | j j|jd�}| j|�}| j|� | j� d S )N)�message)�WARNING_FORMATr� r� r� r� r� )r2 rZ r: Zwarning_statementr r r r� � s
z ResultPrinter._print_warningc K s | j | jj|jd�� d S )N)r" )�_flush_error_statement� ERROR_FORMATr� r" )r2 rZ r: r r r r� � s zResultPrinter._print_errorc K s | j | j� d S )N)r� �
CTRL_C_MSG)r2 rZ r: r r r r� � s zResultPrinter._print_ctrl_cc C s | j |�}| j|� d S )N)r� r� )r2 Zerror_statementr r r r� � s
z$ResultPrinter._flush_error_statementc C s. |j d kr| jj|jd�S | jj|j|j d�S )N)r )r r )r � SRC_TRANSFER_LOCATION_FORMATr� r �!SRC_DEST_TRANSFER_LOCATION_FORMAT)r2 rZ r r r r� � s
z$ResultPrinter._get_transfer_locationc C s d| _ | j� d S )Nr )r� �_add_progress_if_needed)r2 r r r r� � s z!ResultPrinter._redisplay_progressc C s | j � r| j� d S )N)�_has_remaining_progressr� )r2 r r r r� � s z%ResultPrinter._add_progress_if_neededc K s� | j t| jj| jj ��}| jjdkrrt| jj| jj �}| j t| jj��}t| jj �d }| j
j ||||d�}n| j j | jj|d�}| jj
� s�|| j7 }| j|dd�}t|�d | _| j|� d S )Nr z/s)�bytes_completed�expected_bytes_completed�transfer_speed�remaining_files)Zfiles_completedr� �
)� ending_charr )�_get_expected_total�strr� rd r_ rc r r r^ rg �BYTE_PROGRESS_FORMATr� �FILE_PROGRESS_FORMATrs �_STILL_CALCULATING_TOTALSr� �lenr� r� )r2 r: r� r� r� r� Zprogress_statementr r r r� � s4
z ResultPrinter._print_progressc C s | j j� s| jj|d�S |S )N)�expected_total)r� rs �_ESTIMATED_EXPECTED_TOTALr� )r2 r� r r r r� s
z!ResultPrinter._get_expected_total�
c C s |j | jd�}|| S )N� )�ljustr� )r2 Zprint_statementr� r r r r� s z'ResultPrinter._adjust_statement_paddingc C s& | j j� sdS | j j}| j j}||kS )NT)r� rs r_ rd )r2 �actualZexpectedr r r r� s
z%ResultPrinter._has_remaining_progressc C s t || j� d S )N)r r� )r2 r� r r r r� " s z ResultPrinter._print_to_out_filec C s t || j� d S )N)r r� )r2 r� r r r r� % s z"ResultPrinter._print_to_error_filec K s& | j r"| j� r"t| jd�| j� d S )N� )r� r� r r� r� )r2 r: r r r r� ( s z;ResultPrinter._clear_progress_if_no_more_expected_transfers)NN)r� )$r* r+ r, Z_FILES_REMAININGr� r� r� r� r� r� r� r� r� r� r� r� r4 r[ r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r� [ sD
"
/
r� c @ s e Zd ZdZdd� ZdS )�NoProgressResultPrinterz,A result printer that doesn't print progressc K s d S )Nr )r2 r: r r r r� / s z'NoProgressResultPrinter._print_progressN)r* r+ r, r\ r� r r r r r� - s r� c @ s e Zd ZdZdd� Zdd� ZdS )�OnlyShowErrorsResultPrinterz,A result printer that only prints out errorsc K s d S )Nr )r2 r: r r r r� 5 s z+OnlyShowErrorsResultPrinter._print_progressc K s d S )Nr )r2 rZ r: r r r r� 8 s z*OnlyShowErrorsResultPrinter._print_successN)r* r+ r, r\ r� r� r r r r r� 3 s r� c @ s& e Zd Zddd�Zdd� Zdd� ZdS ) �ResultProcessorNc C s2 t jj| � || _|| _| jdkr(g | _d| _dS )aQ Thread to process results from result queue
This includes recording statistics and printing transfer status
:param result_queue: The result queue to process results from
:param result_handlers: A list of callables that take a result in as
a parameter to process the result for that handler.
NT)� threading�Threadr4 r. �_result_handlers�_result_handlers_enabled)r2 r3 Zresult_handlersr r r r4 = s
zResultProcessor.__init__c C sh xbyF| j jd�}t|t�r&tjd� P | jr6| j|� t|t�rFd| _W q t j
k
r^ Y qX qW d S )NTzSShutdown request received in result processing thread, shutting down result thread.F) r. rt rA r) �LOGGER�debugr� �_process_resultr$ r ZEmpty)r2 rZ r r r �runM s
zResultProcessor.runc C sX xR| j D ]H}y ||� W q tk
rN } ztjd|||dd� W Y d d }~X qX qW d S )Nz.Error processing result %s with handler %s: %sT)�exc_info)r� � Exceptionr� r� )r2 rZ Zresult_handlerrB r r r r� b s z ResultProcessor._process_result)N)r* r+ r, r4 r� r� r r r r r� < s
r� c @ sD e Zd Zdd� Zdd� Zdd� Zdd� Zd d
� Zd d � Zd
d� Z dS )�CommandResultRecorderc C s || _ || _|| _dS )a� Records the result for an entire command
It will fully process all results in a result queue and determine
a CommandResult representing the entire command.
:type result_queue: queue.Queue
:param result_queue: The result queue in which results are placed on
and processed from
:type result_recorder: ResultRecorder
:param result_recorder: The result recorder to track the various
results sent through the result queue
:type result_processor: ResultProcessor
:param result_processor: The result processor to process results
placed on the queue
N)r3 r� �_result_processor)r2 r3 r� Zresult_processorr r r r4 m s z CommandResultRecorder.__init__c C s | j j� d S )N)r� �start)r2 r r r r� � s zCommandResultRecorder.startc C s | j jt� � | jj� d S )N)r3 r8 r) r� ry )r2 r r r �shutdown� s z CommandResultRecorder.shutdownc C s t | jj| jj | jj�S )z�Get the CommandResult representing the result of a command
:rtype: CommandResult
:returns: The CommandResult representing the total result from running
a particular command
)r& r� r` rb ra )r2 r r r �get_command_result� s z(CommandResultRecorder.get_command_resultc C s | j jt|�� d S )N)r3 r8 r' )r2 Ztotalr r r �notify_total_submissions� s z.CommandResultRecorder.notify_total_submissionsc C s | j � | S )N)r� )r2 r r r � __enter__� s z CommandResultRecorder.__enter__c G s> |r2t jd|dd� | jjt|d�� | j� dS | j� d S )Nz-Exception caught during command execution: %sT)r� )r" )r� r� r3 r8 r$ r� )r2 �exc_type� exc_value�argsr r r �__exit__� s
z CommandResultRecorder.__exit__N)
r* r+ r, r4 r� r� r� r� r� r� r r r r r� l s r� )7Z
__future__r Zloggingr� r� r= � collectionsr r Zs3transfer.exceptionsr r Zs3transfer.subscribersr Z
awscli.compatr r Z awscli.customizations.s3.utilsr
r Zawscli.customizations.utilsr r
r Z getLoggerr* r� r r r r r r! r# r$ r% r& r' �objectr) r- rH rO rQ rS rT rW rY r] r� r� r� r� r� r� r r r r �<module>
s`
=
S 0