File: //lib/python3.6/site-packages/awscli/customizations/emr/__pycache__/createcluster.cpython-36.pyc
3
�U_�c � @ s� d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm Z d d lm
Z
d d
lm Z d d lm Z d d lm
Z
d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ G dd� de�ZdS )� N)� BasicCommand)�applicationutils)�argumentschema)� constants)�
emrfsutils)�emrutils)�
exceptions)�
hbaseutils)�helptext)�instancegroupsutils)�instancefleetsutils)� steputils)�Command)�
EC2_ROLE_NAME)�
EMR_ROLE_NAME)�jsonc ) @ sV e Zd ZdZejZdejd�dejd�de j
ej d�dej d�dej
d�d d
d ejd �d
d
d d�de jejd�ddejd�dejd�dejd�dejd�dejd�dd
ejd�dejd�deje jd�d d
d ejd �d d
d d�d ejd�d d
d!ej d �d"d
d!d�d#d
d$ej d �d%d
d$d�d&d'ej e j d(�d)ej e j!d�d*ej"e j#d�d+ej$e j%d�d,e j&ej'd�d-ej(d�d.e j)ej*d�d/ej+d�d0ej,d�d1ej-d�d2ej.d�d3e j/ej0d�d4d5ej1d6�d7e j2ej3d�d8e j4ej5d�g&Z6e7j8d9d:�Z9e7j8d9d;�Z:d<d=� Z;d>d?� Z<d@dA� Z=dBdC� Z>dDdE� Z?dFdG� Z@dHdI� ZAdJdK� ZBdLdM� ZCdNdO� ZDdPdQ� ZEdRdS� ZFdTS )U�
CreateClusterzcreate-clusterz
release-label)�name� help_textz ami-versionzinstance-groups)r �schemar z
instance-typezinstance-countzauto-terminate�
store_true�auto_terminate)r �action�
group_namer zno-auto-terminate)r r r zinstance-fleetsr zDevelopment Cluster)r �defaultr zlog-urizlog-encryption-kms-key-idz service-rolezauto-scaling-rolezuse-default-roles)r r r �configurationszec2-attributes)r r r ztermination-protected�termination_protectedzno-termination-protectedzscale-down-behaviorzvisible-to-all-usersZ
visibilityzno-visible-to-all-userszenable-debugging�debugzno-enable-debugging�tags�+)r �nargsr r zbootstrap-actions� applications�emrfs�stepszadditional-infozrestore-from-hbase-backupzsecurity-configurationz
custom-ami-idzebs-root-volume-sizezrepo-upgrade-on-bootzkerberos-attributeszstep-concurrency-levelZinteger)r Z
cli_type_namer zmanaged-scaling-policyzplacement-group-configsZemrzcreate-cluster-synopsis.txtzcreate-cluster-examples.rstc C s� i }|j |d< | j|� d}|jdkr@|jd k r@tjdd|d��|jdkrn|jd k rnd|jkrntjdd|d��|jd k r�|jd k r�tjd d
d ��i }|jd k r�t j
|j�|d < nt j |j|j
|jd
�|d< |jd k �r |j|d< |jd k �r ytj|j�|d< W n tk
�r td��Y nX |jd k�rh|jd k �rhtjd|j�}|d k�r^tj|jd��|j|d< tj|d|j� tj|d|j� |j d k �r�tj|d|j � |jdk�r�t |_|jd k�r�i |_t |jd< tj|d|j� |jd k �r$x4|d D ](}d|j � k�r�|j d k�r�tj!� ��q�W tj|d|j � |j"d k �rPtj|d|j"� |j#d k�rn|j$d k�rnd|_#tj%|j#d |j$d �|d < tj%|j&d |j'd!�|d"< |j(d k�r�|j)d k�r�d|_(tj%|j(d#|j)d$�|d%< tj*|j+�|d&< ||d'< |jd k �r| j,||jd(� tj%|j-d)|j.d*�}|jd k�r8|dk�r8tj/�|dk�r\| j0|d+| j1||�gd,� |j2d k �r�|jd k�r�t3j4| j5|j2|d d-�\} }
} | j0|d.| � | j0|d/|
� | j0|d+| � n(g |d0< x |j2D ]} |d0 j6| � �q�W |j7}
|
d k �r8t8j9|
j:d1�|
j:d2�d3�}tj;t<j=t<j>t<j?|d4�}| j0|d+|g� |j@d k �rT| jA||j@d5� |jBd k �rt| jC||jB|jd6� |jDd k �r�tEjF|jD| j5|jd7�}| j0|d+|d,� |jGd k �r�tj|d8|jG� |jHd k �r�tj|d9|jH� |jId k �r�tj|d:tJ|jI�� |jKd k �rtj|d;|jK� |jLd k �r6tj|d<|jL� |jMd k �rL|jM|d=< |jNd k �rhtj|d>|jN� |jOd k �r�tj|d?|jO� | jP|� tjQ| jRd@|| j5|jS|jT�}| jU|�}tjV| jRd@||� dAS )BN�Namez| Either choose --use-default-roles or use both --service-role <roleName> and --ec2-attributes InstanceProfile=<profileName>.Tz--use-default-rolesz--service-role)�option1�option2�message�InstanceProfilez --ec2-attributes InstanceProfilez--instance-groupsz--instance-fleets)r% r& ZInstanceFleets)�instance_groups�
instance_type�instance_countZInstanceGroupsZ ReleaseLabel�Configurationsz=aws: error: invalid json argument for option --configurationsz\d?\..*)� ami_versionZ
AmiVersionZAdditionalInfoZLogUriZLogEncryptionKmsKeyIdZ ServiceRoleZAutoScalingPolicyZAutoScalingRoleZScaleDownBehaviorFz--no-auto-terminatez--auto-terminateZKeepJobFlowAliveWhenNoStepsz--termination-protectedz--no-termination-protectedZTerminationProtectedz--visible-to-all-usersz--no-visible-to-all-usersZVisibleToAllUsersZTags� Instances)�cluster� parsed_attrsz--enable-debuggingz--no-enable-debuggingZSteps)r/ �key�value)�regionZparsed_applicationsr- ZNewSupportedProducts�BootstrapActionsZ ApplicationsZDirZ
BackupVersion)�dirZbackup_version)�jarr �action_on_failure�args)r/ �parsed_boostrap_actions)r/ �
emrfs_args�
release_label)Zparsed_step_listr3 r; ZSecurityConfigurationZ CustomAmiIdZEbsRootVolumeSizeZRepoUpgradeOnBootZKerberosAttributesZStepConcurrencyLevelZManagedScalingPolicyZPlacementGroupConfigsZ run_job_flowr )Wr �#_validate_release_label_ami_version�use_default_roles� service_roler �MutualExclusiveOptionErrorZec2_attributesr) Zinstance_fleetsr Z"validate_and_build_instance_fleetsr Z"validate_and_build_instance_groupsr* r+ r; r r �loads�
ValueErrorr- �re�matchZInvalidAmiVersionErrorr Z
apply_dictZadditional_infoZlog_uriZlog_encryption_kms_key_idr r �keysZauto_scaling_roleZMissingAutoScalingRoleErrorZscale_down_behaviorZno_auto_terminater Zapply_boolean_optionsr Zno_termination_protectedZvisible_to_all_usersZno_visible_to_all_usersZ
parse_tagsr �_build_ec2_attributesZenable_debuggingZno_enable_debuggingZ LogUriError�_update_cluster_dict�_build_enable_debuggingr! r Zbuild_applicationsr3 �append�restore_from_hbase_backupr Z$build_hbase_restore_from_backup_args�get�
build_stepr ZHBASE_JAR_PATHZHBASE_RESTORE_STEP_NAMEZCANCEL_AND_WAIT�bootstrap_actions�_build_bootstrap_actionsr" �_handle_emrfs_parametersr# r
Zbuild_step_config_listZsecurity_configurationZ
custom_ami_idZebs_root_volume_size�intZrepo_upgrade_on_bootZkerberos_attributesZstep_concurrency_levelZmanaged_scaling_policyZplacement_group_configs� _validate_required_applicationsZcallZ_sessionZ endpoint_urlZ
verify_ssl�_construct_resultZdisplay_response)�self� parsed_args�parsed_globals�paramsZ service_role_validation_messageZinstances_configZis_valid_ami_versionZinstance_groupZdebugging_enabledZapp_listZba_listZ step_listZ applicationZhbase_restore_configr8 Z step_configZ
steps_listZrun_job_flow_responseZconstructed_result� rV �#/usr/lib/python3.6/createcluster.py�_run_main_command� sd
z CreateCluster._run_main_commandc C s> d }d }|d k r$|j d�}|j d�}|d k r6||d�S i S d S )NZ JobFlowId�
ClusterArn)Z ClusterIdrY )rJ )rR Zrun_job_flow_resultZ jobFlowIdZ
clusterArnrV rV rW rQ t s
z CreateCluster._construct_resultc C sx |j � }|d }d|kr.d|kr.tjddd��d|krLd|krLtjddd��d|ks\d|krrd|ksld|krrtj�tj|d|dd � tj|d|d
d � tj|d|d d � d|kr�t� |d < tj|d|d dd � d|kr�t� |d < tj|d|d dd � tj|d
|dd � tj|d|dd � tj|d|dd � tj|d|dd � tj|d|dd � tj|d|dd � tj|d|d� |S )Nr. ZSubnetIdZ SubnetIds)r% r& ZAvailabilityZoneZAvailabilityZonesZKeyNameZ
Ec2KeyName)�
src_params�src_key� dest_params�dest_keyZ Ec2SubnetIdZ Ec2SubnetIdsZ Placementr( Z JobFlowRoleZ EmrManagedMasterSecurityGroupZ EmrManagedSlaveSecurityGroupZServiceAccessSecurityGroupZ AdditionalMasterSecurityGroupsZ AdditionalSlaveSecurityGroups)rU r1 r2 )rD r r? ZSubnetAndAzValidationErrorr � apply_params�dict�apply)rR r/ r0 rD Z instancesrV rV rW rE � sl
z#CreateCluster._build_ec2_attributesc C s� |j d�}|d krg }g }t|�t|� tjkr8td��xz|D ]r}i }|j d�d k rd|j d�|d< n
tj|d< i }tj|d|dd� tj|d|dd� tj|d|d� |j |� q>W || }t|�d kr�||d< |S )
Nr4 zGaws: error: maximum number of bootstrap actions for a cluster exceeded.r$ ZPath)rZ r[ r\ r] ZArgsZScriptBootstrapAction)rU r1 r2 r )
rJ �lenr ZMAX_BOOTSTRAP_ACTION_NUMBERrA ZBOOTSTRAP_ACTION_NAMEr r^ r` rH ) rR r/ r9 Zcluster_ba_listrL ZbaZ ba_configZscript_arg_config�resultrV rV rW rM � s8
z&CreateCluster._build_bootstrap_actionsc C sL |j rtj}tjg}n tj| j�}tjtj| jd�g}tj tj
tj ||d�S )N)Z
relative_pathr3 )r r7 r6 r8 ) r; r ZCOMMAND_RUNNERZDEBUGGING_COMMANDr Zget_script_runnerr3 Z
build_s3_linkZDEBUGGING_PATHrK ZDEBUGGING_NAMEZTERMINATE_CLUSTER)rR rS rT r6 r8 rV rV rW rG � s
z%CreateCluster._build_enable_debuggingc C s> ||j � kr || |7 < n |d k r:t|�dkr:|||< |S )Nr )rD ra )rR r/ r1 r2 rV rV rW rF � s
z"CreateCluster._update_cluster_dictc C sH |j d k r"|jd k r"tjddd��|j d krD|jd krDtjddd��d S )Nz
--ami-versionz--release-label)r% r& )r- r; r r? ZRequiredOptionsError)rR rS rV rV rW r< s
z1CreateCluster._validate_release_label_ami_versionc C sr t g �}|jd k r&t dd� |jD ��}| j||�}|jd k rVtj|krV|jtjj� � t|�dkrnt j
|d��d S )Nc S s g | ]}|d j � �qS )r$ )�lower)�.0ZapprV rV rW �
<listcomp> s zACreateCluster._validate_required_applications.<locals>.<listcomp>r )r! ) �setr! �#_get_missing_applications_for_stepsrI r ZHBASE�add�titlera r ZMissingApplicationsError)rR rS �specified_apps� missing_appsrV rV rW rP s
z-CreateCluster._validate_required_applicationsc C s� t tjtjtjg�}t g �}|jd k r�x\|jD ]R}t|�t|�krDP |jd�}|d k r.|j� }||kr.||kr.|j |d j
� � q.W |S )NZType) rf r ZHIVEZPIGZIMPALAr# ra rJ rc rh ri )rR rj rS Zallowed_app_stepsrk �stepZ step_typerV rV rW rg $ s
z1CreateCluster._get_missing_applications_for_stepsc C s |j rdd� |D �}|S )Nc S s$ g | ] }|j d kr|j dkr|�qS )r> Zinstance_profile)r )rd �xrV rV rW re 8 s
zICreateCluster._filter_configurations_in_special_cases.<locals>.<listcomp>)r= )rR r rS Zparsed_configsrV rV rW �'_filter_configurations_in_special_cases5 s z5CreateCluster._filter_configurations_in_special_casesc C sN |r,| j |� tj|�}| j|d|gd� n tj| j|�}| j|d|d� d S )Nr, )r/ r1 r2 r4 )� validate_no_emrfs_configurationr Zbuild_emrfs_confiurationrF Z build_bootstrap_action_configsr3 )rR r/ r: r; Zemrfs_configurationZemrfs_ba_config_listrV rV rW rN = s
z&CreateCluster._handle_emrfs_parametersc C s< d|kr8x.|d D ]"}|d k r|j d�tjkrtj�qW d S )Nr, ZClassification)rJ r Z
EMRFS_SITEr Z DuplicateEmrFsConfigurationError)rR r/ �configrV rV rW ro M s
z-CreateCluster.validate_no_emrfs_configurationN)G�__name__�
__module__� __qualname__�NAMEr
ZCREATE_CLUSTER_DESCRIPTIONZ DESCRIPTIONZ
RELEASE_LABELZ AMI_VERSIONr ZINSTANCE_GROUPS_SCHEMAZINSTANCE_GROUPSZ
INSTANCE_TYPEZINSTANCE_COUNTZAUTO_TERMINATEZINSTANCE_FLEETS_SCHEMAZINSTANCE_FLEETSZ CLUSTER_NAMEZLOG_URIZLOG_ENCRYPTION_KMS_KEY_IDZ SERVICE_ROLEZAUTOSCALING_ROLEZUSE_DEFAULT_ROLESZCONFIGURATIONSZEC2_ATTRIBUTESZEC2_ATTRIBUTES_SCHEMAZTERMINATION_PROTECTEDZSCALE_DOWN_BEHAVIORZ
VISIBILITYZ DEBUGGINGZTAGSZ TAGS_SCHEMAZBOOTSTRAP_ACTIONSZBOOTSTRAP_ACTIONS_SCHEMAZ APPLICATIONSZAPPLICATIONS_SCHEMAZEMR_FSZ
EMR_FS_SCHEMAZ STEPS_SCHEMAZSTEPSZADDITIONAL_INFOZ HBASE_RESTORE_FROM_BACKUP_SCHEMAZRESTORE_FROM_HBASEZSECURITY_CONFIGZ
CUSTOM_AMI_IDZEBS_ROOT_VOLUME_SIZEZREPO_UPGRADE_ON_BOOTZKERBEROS_ATTRIBUTES_SCHEMAZKERBEROS_ATTRIBUTESZSTEP_CONCURRENCY_LEVELZ MANAGED_SCALING_POLICY_SCHEMAZMANAGED_SCALING_POLICYZ PLACEMENT_GROUP_CONFIGS_SCHEMAZPLACEMENT_GROUP_CONFIGSZ ARG_TABLEr Z FROM_FILEZSYNOPSISZEXAMPLESrX rQ rE rM rG rF r< rP rg rn rN ro rV rV rV rW r " s� m
F%r )rB Z awscli.customizations.commandsr Zawscli.customizations.emrr r r r r r r r
r r r
Z!awscli.customizations.emr.commandr Z#awscli.customizations.emr.constantsr r Zbotocore.compatr r rV rV rV rW �<module> s"