File: //usr/lib/python3.6/site-packages/s3transfer/__pycache__/bandwidth.cpython-36.pyc
3
\��_= � @ s� d dl Z d dlZG dd� de�ZG dd� de�ZG dd� de�ZG dd � d e�ZG d
d � d e�ZG d d
� d
e�Z G dd� de�Z
G dd� de�Z dS )� Nc s e Zd Z� fdd�Z� ZS )�RequestExceededExceptionc s, || _ || _d||f }tt| �j|� dS )a� Error when requested amount exceeds what is allowed
The request that raised this error should be retried after waiting
the time specified by ``retry_time``.
:type requested_amt: int
:param requested_amt: The originally requested byte amount
:type retry_time: float
:param retry_time: The length in time to wait to retry for the
requested amount
z<Request amount %s exceeded the amount available. Retry in %sN)�
requested_amt�
retry_time�superr �__init__)�selfr r �msg)� __class__� � /usr/lib/python3.6/bandwidth.pyr s
z!RequestExceededException.__init__)�__name__�
__module__� __qualname__r �
__classcell__r
r
)r r r s r c @ s e Zd ZdZdS )� RequestTokenzDA token to pass as an identifier when consuming from the LeakyBucketN)r r
r �__doc__r
r
r
r r ( s r c @ s e Zd Zdd� Zdd� ZdS )� TimeUtilsc C s t j � S )zgGet the current time back
:rtype: float
:returns: The current time in seconds
)�time)r r
r
r r . s zTimeUtils.timec C s
t j|�S )zwSleep for a designated time
:type value: float
:param value: The time to sleep for in seconds
)r �sleep)r �valuer
r
r r 6 s zTimeUtils.sleepN)r r
r r r r
r
r
r r - s r c @ s e Zd Zddd�Zddd�ZdS ) �BandwidthLimiterNc C s || _ || _|dkr t� | _dS )a Limits bandwidth for shared S3 transfers
:type leaky_bucket: LeakyBucket
:param leaky_bucket: The leaky bucket to use limit bandwidth
:type time_utils: TimeUtils
:param time_utils: Time utility to use for interacting with time.
N)�
_leaky_bucket� _time_utilsr )r � leaky_bucket�
time_utilsr
r
r r @ s zBandwidthLimiter.__init__Tc C s" t || j|| j�}|s |j� |S )a� Wraps a fileobj in a bandwidth limited stream wrapper
:type fileobj: file-like obj
:param fileobj: The file-like obj to wrap
:type transfer_coordinator: s3transfer.futures.TransferCoordinator
param transfer_coordinator: The coordinator for the general transfer
that the wrapped stream is a part of
:type enabled: boolean
:param enabled: Whether bandwidth limiting should be enabled to start
)�BandwidthLimitedStreamr r �disable_bandwidth_limiting)r �fileobj�transfer_coordinatorZenabled�streamr
r
r �get_bandwith_limited_streamN s z,BandwidthLimiter.get_bandwith_limited_stream)N)T)r r
r r r r
r
r
r r ? s
r c @ sn e Zd Zd dd�Zdd� Zdd � Zd
d � Zd d
� Zdd� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� ZdS ) r N� i c C sF || _ || _|| _|| _|dkr(t� | _d| _t� | _d| _|| _ dS )a[ Limits bandwidth for reads on a wrapped stream
:type fileobj: file-like object
:param fileobj: The file like object to wrap
:type leaky_bucket: LeakyBucket
:param leaky_bucket: The leaky bucket to use to throttle reads on
the stream
:type transfer_coordinator: s3transfer.futures.TransferCoordinator
param transfer_coordinator: The coordinator for the general transfer
that the wrapped stream is a part of
:type time_utils: TimeUtils
:param time_utils: The time utility to use for interacting with time
NTr )
�_fileobjr �_transfer_coordinatorr r �_bandwidth_limiting_enabledr �_request_token� _bytes_seen�_bytes_threshold)r r r r r Zbytes_thresholdr
r
r r e s z BandwidthLimitedStream.__init__c C s
d| _ dS )z0Enable bandwidth limiting on reads to the streamTN)r$ )r r
r
r �enable_bandwidth_limiting� s z0BandwidthLimitedStream.enable_bandwidth_limitingc C s
d| _ dS )z1Disable bandwidth limiting on reads to the streamFN)r$ )r r
r
r r � s z1BandwidthLimitedStream.disable_bandwidth_limitingc C sL | j s| jj|�S | j|7 _| j| jk r8| jj|�S | j� | jj|�S )zhRead a specified amount
Reads will only be throttled if bandwidth limiting is enabled.
)r$ r" �readr& r' � _consume_through_leaky_bucket)r Zamountr
r
r r) � s zBandwidthLimitedStream.readc C sh xb| j jsZy | jj| j| j� d| _d S tk
rV } z| jj|j � W Y d d }~X qX qW | j j�d S )Nr )
r# Z exceptionr �consumer& r% r r r r )r �er
r
r r* � s
$z4BandwidthLimitedStream._consume_through_leaky_bucketc C s | j � dS )z6Signal that data being read is being transferred to S3N)r( )r r
r
r �signal_transferring� s z*BandwidthLimitedStream.signal_transferringc C s | j � dS )z:Signal that data being read is not being transferred to S3N)r )r r
r
r �signal_not_transferring� s z.BandwidthLimitedStream.signal_not_transferringc C s | j j|� d S )N)r" �seek)r �wherer
r
r r/ � s zBandwidthLimitedStream.seekc C s
| j j� S )N)r" �tell)r r
r
r r1 � s zBandwidthLimitedStream.tellc C s" | j r| jr| j� | jj� d S )N)r$ r& r* r" �close)r r
r
r r2 � s z BandwidthLimitedStream.closec C s | S )Nr
)r r
r
r � __enter__� s z BandwidthLimitedStream.__enter__c O s | j � d S )N)r2 )r �args�kwargsr
r
r �__exit__� s z BandwidthLimitedStream.__exit__� )Nr7 )r r
r r r( r r) r* r- r. r/ r1 r2 r3 r6 r
r
r
r r d s
r c @ s> e Zd Zddd�Zdd� Zdd� Zdd � Zd
d � Zd d
� ZdS )� LeakyBucketNc C sZ t |�| _|| _|dkr t� | _tj� | _|| _|dkr@t� | _|| _ |dkrVt
� | _ dS )a9 A leaky bucket abstraction to limit bandwidth consumption
:type rate: int
:type rate: The maximum rate to allow. This rate is in terms of
bytes per second.
:type time_utils: TimeUtils
:param time_utils: The time utility to use for interacting with time
:type rate_tracker: BandwidthRateTracker
:param rate_tracker: Tracks bandwidth consumption
:type consumption_scheduler: ConsumptionScheduler
:param consumption_scheduler: Schedules consumption retries when
necessary
N) �float� _max_rater r � threadingZLock�_lock�
_rate_tracker�BandwidthRateTracker�_consumption_scheduler�ConsumptionScheduler)r Zmax_rater Z rate_trackerZconsumption_schedulerr
r
r r � s
zLeakyBucket.__init__c C sb | j �R | jj� }| jj|�r,| j|||�S | j||�rH| j|||� n | j||�S W dQ R X dS )ac Consume an a requested amount
:type amt: int
:param amt: The amount of bytes to request to consume
:type request_token: RequestToken
:param request_token: The token associated to the consumption
request that is used to identify the request. So if a
RequestExceededException is raised the token should be used
in subsequent retry consume() request.
:raises RequestExceededException: If the consumption amount would
exceed the maximum allocated bandwidth
:rtype: int
:returns: The amount consumed
N) r< r r r? � is_scheduled�,_release_requested_amt_for_scheduled_request� _projected_to_exceed_max_rate�!_raise_request_exceeded_exception�_release_requested_amt)r �amt�
request_token�time_nowr
r
r r+ � s
zLeakyBucket.consumec C s | j j||�}|| jkS )N)r= �get_projected_rater: )r rF rH Zprojected_rater
r
r rC
s z)LeakyBucket._projected_to_exceed_max_ratec C s | j j|� | j||�S )N)r? � process_scheduled_consumptionrE )r rF rG rH r
r
r rB s z8LeakyBucket._release_requested_amt_for_scheduled_requestc C s. |t | j� }| jj|||�}t||d��d S )N)r r )r9 r: r? �schedule_consumptionr )r rF rG rH Zallocated_timer r
r
r rD s
z-LeakyBucket._raise_request_exceeded_exceptionc C s | j j||� |S )N)r= �record_consumption_rate)r rF rH r
r
r rE s z"LeakyBucket._release_requested_amt)NNN) r r
r r r+ rC rB rD rE r
r
r
r r8 � s
r8 c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
r@ c C s i | _ d| _dS )z*Schedules when to consume a desired amountr N)� _tokens_to_scheduled_consumption� _total_wait)r r
r
r r ! s z ConsumptionScheduler.__init__c C s
|| j kS )z�Indicates if a consumption request has been scheduled
:type token: RequestToken
:param token: The token associated to the consumption
request that is used to identify the request.
)rM )r �tokenr
r
r rA & s z!ConsumptionScheduler.is_scheduledc C s& | j |7 _ | j |d�| j|< | j S )a� Schedules a wait time to be able to consume an amount
:type amt: int
:param amt: The amount of bytes scheduled to be consumed
:type token: RequestToken
:param token: The token associated to the consumption
request that is used to identify the request.
:type time_to_consume: float
:param time_to_consume: The desired time it should take for that
specific request amount to be consumed in regardless of previously
scheduled consumption requests
:rtype: float
:returns: The amount of time to wait for the specific request before
actually consuming the specified amount.
)Z
wait_duration�time_to_consume)rN rM )r rF rO rP r
r
r rK / s z)ConsumptionScheduler.schedule_consumptionc C s& | j j|�}t| j|d d�| _dS )z�Processes a scheduled consumption request that has completed
:type token: RequestToken
:param token: The token associated to the consumption
request that is used to identify the request.
rP r N)rM �pop�maxrN )r rO Zscheduled_retryr
r
r rJ I s z2ConsumptionScheduler.process_scheduled_consumptionN)r r
r r rA rK rJ r
r
r
r r@ s r@ c @ sB e Zd Zddd�Zedd� �Zdd� Zdd � Zd
d � Zd d
� Z dS )r> 皙�����?c C s || _ d| _d| _dS )a� Tracks the rate of bandwidth consumption
:type a: float
:param a: The constant to use in calculating the exponentional moving
average of the bandwidth rate. Specifically it is used in the
following calculation:
current_rate = alpha * new_rate + (1 - alpha) * current_rate
This value of this constant should be between 0 and 1.
N)�_alpha�
_last_time�
_current_rate)r Zalphar
r
r r V s z BandwidthRateTracker.__init__c C s | j dkrdS | jS )zmThe current transfer rate
:rtype: float
:returns: The current tracked transfer rate
Ng )rU rV )r r
r
r � current_ratef s
z!BandwidthRateTracker.current_ratec C s | j dkrdS | j||�S )aZ Get the projected rate using a provided amount and time
:type amt: int
:param amt: The proposed amount to consume
:type time_at_consumption: float
:param time_at_consumption: The proposed time to consume at
:rtype: float
:returns: The consumption rate if that amt and time were consumed
Ng )rU �*_calculate_exponential_moving_average_rate)r rF �time_at_consumptionr
r
r rI q s
z'BandwidthRateTracker.get_projected_ratec C s2 | j dkr|| _ d| _dS | j||�| _|| _ dS )a Record the consumption rate based off amount and time point
:type amt: int
:param amt: The amount that got consumed
:type time_at_consumption: float
:param time_at_consumption: The time at which the amount was consumed
Ng )rU rV rX )r rF rY r
r
r rL � s
z,BandwidthRateTracker.record_consumption_ratec C s" || j }|dkrtd�S || S )Nr �inf)rU r9 )r rF rY Z
time_deltar
r
r �_calculate_rate� s
z$BandwidthRateTracker._calculate_ratec C s& | j ||�}| j| d| j | j S )N� )r[ rT rV )r rF rY Znew_rater
r
r rX � s z?BandwidthRateTracker._calculate_exponential_moving_average_rateN)rS )
r r
r r �propertyrW rI rL r[ rX r
r
r
r r> U s
r> ) r r; � Exceptionr �objectr r r r r8 r@ r> r
r
r
r �<module>
s %jR5