1426 lines
64 KiB
Python
1426 lines
64 KiB
Python
from __future__ import unicode_literals
|
|
import json
|
|
import re
|
|
from datetime import datetime
|
|
from functools import wraps
|
|
|
|
import pytz
|
|
|
|
from urllib.parse import urlparse
|
|
from moto.core.responses import AWSServiceSpec
|
|
from moto.core.responses import BaseResponse
|
|
from moto.core.responses import xml_to_json_response
|
|
from moto.core.utils import tags_from_query_string
|
|
from .exceptions import EmrError
|
|
from .models import emr_backends
|
|
from .utils import steps_from_query_string, Unflattener, ReleaseLabel
|
|
|
|
|
|
def generate_boto3_response(operation):
|
|
"""The decorator to convert an XML response to JSON, if the request is
|
|
determined to be from boto3. Pass the API action as a parameter.
|
|
|
|
"""
|
|
|
|
def _boto3_request(method):
|
|
@wraps(method)
|
|
def f(self, *args, **kwargs):
|
|
rendered = method(self, *args, **kwargs)
|
|
if "json" in self.headers.get("Content-Type", []):
|
|
self.response_headers.update(
|
|
{
|
|
"x-amzn-requestid": "2690d7eb-ed86-11dd-9877-6fad448a8419",
|
|
"date": datetime.now(pytz.utc).strftime(
|
|
"%a, %d %b %Y %H:%M:%S %Z"
|
|
),
|
|
"content-type": "application/x-amz-json-1.1",
|
|
}
|
|
)
|
|
resp = xml_to_json_response(self.aws_service_spec, operation, rendered)
|
|
return "" if resp is None else json.dumps(resp)
|
|
return rendered
|
|
|
|
return f
|
|
|
|
return _boto3_request
|
|
|
|
|
|
class ElasticMapReduceResponse(BaseResponse):
|
|
|
|
# EMR end points are inconsistent in the placement of region name
|
|
# in the URL, so parsing it out needs to be handled differently
|
|
region_regex = [
|
|
re.compile(r"elasticmapreduce\.(.+?)\.amazonaws\.com"),
|
|
re.compile(r"(.+?)\.elasticmapreduce\.amazonaws\.com"),
|
|
]
|
|
|
|
aws_service_spec = AWSServiceSpec("data/emr/2009-03-31/service-2.json")
|
|
|
|
def get_region_from_url(self, request, full_url):
|
|
parsed = urlparse(full_url)
|
|
for regex in self.region_regex:
|
|
match = regex.search(parsed.netloc)
|
|
if match:
|
|
return match.group(1)
|
|
return self.default_region
|
|
|
|
@property
|
|
def backend(self):
|
|
return emr_backends[self.region]
|
|
|
|
@generate_boto3_response("AddInstanceGroups")
|
|
def add_instance_groups(self):
|
|
jobflow_id = self._get_param("JobFlowId")
|
|
instance_groups = self._get_list_prefix("InstanceGroups.member")
|
|
for item in instance_groups:
|
|
item["instance_count"] = int(item["instance_count"])
|
|
# Adding support to EbsConfiguration
|
|
self._parse_ebs_configuration(item)
|
|
# Adding support for auto_scaling_policy
|
|
Unflattener.unflatten_complex_params(item, "auto_scaling_policy")
|
|
instance_groups = self.backend.add_instance_groups(jobflow_id, instance_groups)
|
|
template = self.response_template(ADD_INSTANCE_GROUPS_TEMPLATE)
|
|
return template.render(instance_groups=instance_groups)
|
|
|
|
@generate_boto3_response("AddJobFlowSteps")
|
|
def add_job_flow_steps(self):
|
|
job_flow_id = self._get_param("JobFlowId")
|
|
steps = self.backend.add_job_flow_steps(
|
|
job_flow_id, steps_from_query_string(self._get_list_prefix("Steps.member"))
|
|
)
|
|
template = self.response_template(ADD_JOB_FLOW_STEPS_TEMPLATE)
|
|
return template.render(steps=steps)
|
|
|
|
@generate_boto3_response("AddTags")
|
|
def add_tags(self):
|
|
cluster_id = self._get_param("ResourceId")
|
|
tags = tags_from_query_string(self.querystring, prefix="Tags")
|
|
self.backend.add_tags(cluster_id, tags)
|
|
template = self.response_template(ADD_TAGS_TEMPLATE)
|
|
return template.render()
|
|
|
|
def cancel_steps(self):
|
|
raise NotImplementedError
|
|
|
|
@generate_boto3_response("CreateSecurityConfiguration")
|
|
def create_security_configuration(self):
|
|
name = self._get_param("Name")
|
|
security_configuration = self._get_param("SecurityConfiguration")
|
|
resp = self.backend.create_security_configuration(
|
|
name=name, security_configuration=security_configuration
|
|
)
|
|
template = self.response_template(CREATE_SECURITY_CONFIGURATION_TEMPLATE)
|
|
return template.render(name=name, creation_date_time=resp.creation_date_time)
|
|
|
|
@generate_boto3_response("DescribeSecurityConfiguration")
|
|
def describe_security_configuration(self):
|
|
name = self._get_param("Name")
|
|
security_configuration = self.backend.get_security_configuration(name=name)
|
|
template = self.response_template(DESCRIBE_SECURITY_CONFIGURATION_TEMPLATE)
|
|
return template.render(security_configuration=security_configuration)
|
|
|
|
@generate_boto3_response("DeleteSecurityConfiguration")
|
|
def delete_security_configuration(self):
|
|
name = self._get_param("Name")
|
|
self.backend.delete_security_configuration(name=name)
|
|
template = self.response_template(DELETE_SECURITY_CONFIGURATION_TEMPLATE)
|
|
return template.render()
|
|
|
|
@generate_boto3_response("DescribeCluster")
|
|
def describe_cluster(self):
|
|
cluster_id = self._get_param("ClusterId")
|
|
cluster = self.backend.get_cluster(cluster_id)
|
|
template = self.response_template(DESCRIBE_CLUSTER_TEMPLATE)
|
|
return template.render(cluster=cluster)
|
|
|
|
@generate_boto3_response("DescribeJobFlows")
|
|
def describe_job_flows(self):
|
|
created_after = self._get_param("CreatedAfter")
|
|
created_before = self._get_param("CreatedBefore")
|
|
job_flow_ids = self._get_multi_param("JobFlowIds.member")
|
|
job_flow_states = self._get_multi_param("JobFlowStates.member")
|
|
clusters = self.backend.describe_job_flows(
|
|
job_flow_ids, job_flow_states, created_after, created_before
|
|
)
|
|
template = self.response_template(DESCRIBE_JOB_FLOWS_TEMPLATE)
|
|
return template.render(clusters=clusters)
|
|
|
|
@generate_boto3_response("DescribeStep")
|
|
def describe_step(self):
|
|
cluster_id = self._get_param("ClusterId")
|
|
step_id = self._get_param("StepId")
|
|
step = self.backend.describe_step(cluster_id, step_id)
|
|
template = self.response_template(DESCRIBE_STEP_TEMPLATE)
|
|
return template.render(step=step)
|
|
|
|
@generate_boto3_response("ListBootstrapActions")
|
|
def list_bootstrap_actions(self):
|
|
cluster_id = self._get_param("ClusterId")
|
|
marker = self._get_param("Marker")
|
|
bootstrap_actions, marker = self.backend.list_bootstrap_actions(
|
|
cluster_id, marker
|
|
)
|
|
template = self.response_template(LIST_BOOTSTRAP_ACTIONS_TEMPLATE)
|
|
return template.render(bootstrap_actions=bootstrap_actions, marker=marker)
|
|
|
|
@generate_boto3_response("ListClusters")
|
|
def list_clusters(self):
|
|
cluster_states = self._get_multi_param("ClusterStates.member")
|
|
created_after = self._get_param("CreatedAfter")
|
|
created_before = self._get_param("CreatedBefore")
|
|
marker = self._get_param("Marker")
|
|
clusters, marker = self.backend.list_clusters(
|
|
cluster_states, created_after, created_before, marker
|
|
)
|
|
template = self.response_template(LIST_CLUSTERS_TEMPLATE)
|
|
return template.render(clusters=clusters, marker=marker)
|
|
|
|
@generate_boto3_response("ListInstanceGroups")
|
|
def list_instance_groups(self):
|
|
cluster_id = self._get_param("ClusterId")
|
|
marker = self._get_param("Marker")
|
|
instance_groups, marker = self.backend.list_instance_groups(
|
|
cluster_id, marker=marker
|
|
)
|
|
template = self.response_template(LIST_INSTANCE_GROUPS_TEMPLATE)
|
|
return template.render(instance_groups=instance_groups, marker=marker)
|
|
|
|
@generate_boto3_response("ListInstances")
|
|
def list_instances(self):
|
|
cluster_id = self._get_param("ClusterId")
|
|
marker = self._get_param("Marker")
|
|
instance_group_id = self._get_param("InstanceGroupId")
|
|
instance_group_types = self._get_param("InstanceGroupTypes")
|
|
instances, marker = self.backend.list_instances(
|
|
cluster_id,
|
|
marker=marker,
|
|
instance_group_id=instance_group_id,
|
|
instance_group_types=instance_group_types,
|
|
)
|
|
template = self.response_template(LIST_INSTANCES_TEMPLATE)
|
|
return template.render(instances=instances, marker=marker)
|
|
|
|
@generate_boto3_response("ListSteps")
|
|
def list_steps(self):
|
|
cluster_id = self._get_param("ClusterId")
|
|
marker = self._get_param("Marker")
|
|
step_ids = self._get_multi_param("StepIds.member")
|
|
step_states = self._get_multi_param("StepStates.member")
|
|
steps, marker = self.backend.list_steps(
|
|
cluster_id, marker=marker, step_ids=step_ids, step_states=step_states
|
|
)
|
|
template = self.response_template(LIST_STEPS_TEMPLATE)
|
|
return template.render(steps=steps, marker=marker)
|
|
|
|
@generate_boto3_response("ModifyCluster")
|
|
def modify_cluster(self):
|
|
cluster_id = self._get_param("ClusterId")
|
|
step_concurrency_level = self._get_param("StepConcurrencyLevel")
|
|
cluster = self.backend.modify_cluster(cluster_id, step_concurrency_level)
|
|
template = self.response_template(MODIFY_CLUSTER_TEMPLATE)
|
|
return template.render(cluster=cluster)
|
|
|
|
@generate_boto3_response("ModifyInstanceGroups")
|
|
def modify_instance_groups(self):
|
|
instance_groups = self._get_list_prefix("InstanceGroups.member")
|
|
for item in instance_groups:
|
|
item["instance_count"] = int(item["instance_count"])
|
|
instance_groups = self.backend.modify_instance_groups(instance_groups)
|
|
template = self.response_template(MODIFY_INSTANCE_GROUPS_TEMPLATE)
|
|
return template.render(instance_groups=instance_groups)
|
|
|
|
@generate_boto3_response("RemoveTags")
|
|
def remove_tags(self):
|
|
cluster_id = self._get_param("ResourceId")
|
|
tag_keys = self._get_multi_param("TagKeys.member")
|
|
self.backend.remove_tags(cluster_id, tag_keys)
|
|
template = self.response_template(REMOVE_TAGS_TEMPLATE)
|
|
return template.render()
|
|
|
|
@generate_boto3_response("RunJobFlow")
|
|
def run_job_flow(self):
|
|
instance_attrs = dict(
|
|
master_instance_type=self._get_param("Instances.MasterInstanceType"),
|
|
slave_instance_type=self._get_param("Instances.SlaveInstanceType"),
|
|
instance_count=self._get_int_param("Instances.InstanceCount", 1),
|
|
ec2_key_name=self._get_param("Instances.Ec2KeyName"),
|
|
ec2_subnet_id=self._get_param("Instances.Ec2SubnetId"),
|
|
hadoop_version=self._get_param("Instances.HadoopVersion"),
|
|
availability_zone=self._get_param(
|
|
"Instances.Placement.AvailabilityZone", self.backend.region_name + "a"
|
|
),
|
|
keep_job_flow_alive_when_no_steps=self._get_bool_param(
|
|
"Instances.KeepJobFlowAliveWhenNoSteps", False
|
|
),
|
|
termination_protected=self._get_bool_param(
|
|
"Instances.TerminationProtected", False
|
|
),
|
|
emr_managed_master_security_group=self._get_param(
|
|
"Instances.EmrManagedMasterSecurityGroup"
|
|
),
|
|
emr_managed_slave_security_group=self._get_param(
|
|
"Instances.EmrManagedSlaveSecurityGroup"
|
|
),
|
|
service_access_security_group=self._get_param(
|
|
"Instances.ServiceAccessSecurityGroup"
|
|
),
|
|
additional_master_security_groups=self._get_multi_param(
|
|
"Instances.AdditionalMasterSecurityGroups.member."
|
|
),
|
|
additional_slave_security_groups=self._get_multi_param(
|
|
"Instances.AdditionalSlaveSecurityGroups.member."
|
|
),
|
|
)
|
|
|
|
kwargs = dict(
|
|
name=self._get_param("Name"),
|
|
log_uri=self._get_param("LogUri"),
|
|
job_flow_role=self._get_param("JobFlowRole"),
|
|
service_role=self._get_param("ServiceRole"),
|
|
steps=steps_from_query_string(self._get_list_prefix("Steps.member")),
|
|
visible_to_all_users=self._get_bool_param("VisibleToAllUsers", False),
|
|
instance_attrs=instance_attrs,
|
|
)
|
|
|
|
bootstrap_actions = self._get_list_prefix("BootstrapActions.member")
|
|
if bootstrap_actions:
|
|
for ba in bootstrap_actions:
|
|
args = []
|
|
idx = 1
|
|
keyfmt = "script_bootstrap_action._args.member.{0}"
|
|
key = keyfmt.format(idx)
|
|
while key in ba:
|
|
args.append(ba.pop(key))
|
|
idx += 1
|
|
key = keyfmt.format(idx)
|
|
ba["args"] = args
|
|
ba["script_path"] = ba.pop("script_bootstrap_action._path")
|
|
kwargs["bootstrap_actions"] = bootstrap_actions
|
|
|
|
configurations = self._get_list_prefix("Configurations.member")
|
|
if configurations:
|
|
for idx, config in enumerate(configurations, 1):
|
|
for key in list(config.keys()):
|
|
if key.startswith("properties."):
|
|
config.pop(key)
|
|
config["properties"] = {}
|
|
map_items = self._get_map_prefix(
|
|
"Configurations.member.{0}.Properties.entry".format(idx)
|
|
)
|
|
config["properties"] = map_items
|
|
|
|
kwargs["configurations"] = configurations
|
|
|
|
release_label = self._get_param("ReleaseLabel")
|
|
ami_version = self._get_param("AmiVersion")
|
|
if release_label:
|
|
kwargs["release_label"] = release_label
|
|
if ami_version:
|
|
message = (
|
|
"Only one AMI version and release label may be specified. "
|
|
"Provided AMI: {0}, release label: {1}."
|
|
).format(ami_version, release_label)
|
|
raise EmrError(
|
|
error_type="ValidationException",
|
|
message=message,
|
|
template="error_json",
|
|
)
|
|
else:
|
|
if ami_version:
|
|
kwargs["requested_ami_version"] = ami_version
|
|
kwargs["running_ami_version"] = ami_version
|
|
else:
|
|
kwargs["running_ami_version"] = "1.0.0"
|
|
|
|
custom_ami_id = self._get_param("CustomAmiId")
|
|
if custom_ami_id:
|
|
kwargs["custom_ami_id"] = custom_ami_id
|
|
if release_label and (
|
|
ReleaseLabel(release_label) < ReleaseLabel("emr-5.7.0")
|
|
):
|
|
message = "Custom AMI is not allowed"
|
|
raise EmrError(
|
|
error_type="ValidationException",
|
|
message=message,
|
|
template="error_json",
|
|
)
|
|
elif ami_version:
|
|
message = "Custom AMI is not supported in this version of EMR"
|
|
raise EmrError(
|
|
error_type="ValidationException",
|
|
message=message,
|
|
template="error_json",
|
|
)
|
|
|
|
step_concurrency_level = self._get_param("StepConcurrencyLevel")
|
|
if step_concurrency_level:
|
|
kwargs["step_concurrency_level"] = step_concurrency_level
|
|
|
|
security_configuration = self._get_param("SecurityConfiguration")
|
|
if security_configuration:
|
|
kwargs["security_configuration"] = security_configuration
|
|
|
|
kerberos_attributes = {}
|
|
kwargs["kerberos_attributes"] = kerberos_attributes
|
|
|
|
realm = self._get_param("KerberosAttributes.Realm")
|
|
if realm:
|
|
kerberos_attributes["Realm"] = realm
|
|
|
|
kdc_admin_password = self._get_param("KerberosAttributes.KdcAdminPassword")
|
|
if kdc_admin_password:
|
|
kerberos_attributes["KdcAdminPassword"] = kdc_admin_password
|
|
|
|
cross_realm_principal_password = self._get_param(
|
|
"KerberosAttributes.CrossRealmTrustPrincipalPassword"
|
|
)
|
|
if cross_realm_principal_password:
|
|
kerberos_attributes[
|
|
"CrossRealmTrustPrincipalPassword"
|
|
] = cross_realm_principal_password
|
|
|
|
ad_domain_join_user = self._get_param("KerberosAttributes.ADDomainJoinUser")
|
|
if ad_domain_join_user:
|
|
kerberos_attributes["ADDomainJoinUser"] = ad_domain_join_user
|
|
|
|
ad_domain_join_password = self._get_param(
|
|
"KerberosAttributes.ADDomainJoinPassword"
|
|
)
|
|
if ad_domain_join_password:
|
|
kerberos_attributes["ADDomainJoinPassword"] = ad_domain_join_password
|
|
|
|
cluster = self.backend.run_job_flow(**kwargs)
|
|
|
|
applications = self._get_list_prefix("Applications.member")
|
|
if applications:
|
|
self.backend.add_applications(cluster.id, applications)
|
|
else:
|
|
self.backend.add_applications(
|
|
cluster.id, [{"Name": "Hadoop", "Version": "0.18"}]
|
|
)
|
|
|
|
instance_groups = self._get_list_prefix("Instances.InstanceGroups.member")
|
|
if instance_groups:
|
|
for ig in instance_groups:
|
|
ig["instance_count"] = int(ig["instance_count"])
|
|
# Adding support to EbsConfiguration
|
|
self._parse_ebs_configuration(ig)
|
|
# Adding support for auto_scaling_policy
|
|
Unflattener.unflatten_complex_params(ig, "auto_scaling_policy")
|
|
instance_group_result = self.backend.add_instance_groups(
|
|
cluster.id, instance_groups
|
|
)
|
|
for i in range(0, len(instance_group_result)):
|
|
self.backend.add_instances(
|
|
cluster.id, instance_groups[i], instance_group_result[i]
|
|
)
|
|
|
|
tags = self._get_list_prefix("Tags.member")
|
|
if tags:
|
|
self.backend.add_tags(
|
|
cluster.id, dict((d["key"], d["value"]) for d in tags)
|
|
)
|
|
|
|
template = self.response_template(RUN_JOB_FLOW_TEMPLATE)
|
|
return template.render(cluster=cluster)
|
|
|
|
def _has_key_prefix(self, key_prefix, value):
|
|
for key in value: # iter on both keys and values
|
|
if key.startswith(key_prefix):
|
|
return True
|
|
return False
|
|
|
|
def _parse_ebs_configuration(self, instance_group):
|
|
key_ebs_config = "ebs_configuration"
|
|
ebs_configuration = dict()
|
|
# Filter only EBS config keys
|
|
for key in instance_group:
|
|
if key.startswith(key_ebs_config):
|
|
ebs_configuration[key] = instance_group[key]
|
|
|
|
if len(ebs_configuration) > 0:
|
|
# Key that should be extracted
|
|
ebs_optimized = "ebs_optimized"
|
|
ebs_block_device_configs = "ebs_block_device_configs"
|
|
volume_specification = "volume_specification"
|
|
size_in_gb = "size_in_gb"
|
|
volume_type = "volume_type"
|
|
iops = "iops"
|
|
volumes_per_instance = "volumes_per_instance"
|
|
|
|
key_ebs_optimized = "{0}._{1}".format(key_ebs_config, ebs_optimized)
|
|
# EbsOptimized config
|
|
if key_ebs_optimized in ebs_configuration:
|
|
instance_group.pop(key_ebs_optimized)
|
|
ebs_configuration[ebs_optimized] = ebs_configuration.pop(
|
|
key_ebs_optimized
|
|
)
|
|
|
|
# Ebs Blocks
|
|
ebs_blocks = []
|
|
idx = 1
|
|
keyfmt = "{0}._{1}.member.{{}}".format(
|
|
key_ebs_config, ebs_block_device_configs
|
|
)
|
|
key = keyfmt.format(idx)
|
|
while self._has_key_prefix(key, ebs_configuration):
|
|
vlespc_keyfmt = "{0}._{1}._{{}}".format(key, volume_specification)
|
|
vol_size = vlespc_keyfmt.format(size_in_gb)
|
|
vol_iops = vlespc_keyfmt.format(iops)
|
|
vol_type = vlespc_keyfmt.format(volume_type)
|
|
|
|
ebs_block = dict()
|
|
ebs_block[volume_specification] = dict()
|
|
if vol_size in ebs_configuration:
|
|
instance_group.pop(vol_size)
|
|
ebs_block[volume_specification][size_in_gb] = int(
|
|
ebs_configuration.pop(vol_size)
|
|
)
|
|
if vol_iops in ebs_configuration:
|
|
instance_group.pop(vol_iops)
|
|
ebs_block[volume_specification][iops] = ebs_configuration.pop(
|
|
vol_iops
|
|
)
|
|
if vol_type in ebs_configuration:
|
|
instance_group.pop(vol_type)
|
|
ebs_block[volume_specification][
|
|
volume_type
|
|
] = ebs_configuration.pop(vol_type)
|
|
|
|
per_instance = "{0}._{1}".format(key, volumes_per_instance)
|
|
if per_instance in ebs_configuration:
|
|
instance_group.pop(per_instance)
|
|
ebs_block[volumes_per_instance] = int(
|
|
ebs_configuration.pop(per_instance)
|
|
)
|
|
|
|
if len(ebs_block) > 0:
|
|
ebs_blocks.append(ebs_block)
|
|
idx += 1
|
|
key = keyfmt.format(idx)
|
|
|
|
if len(ebs_blocks) > 0:
|
|
ebs_configuration[ebs_block_device_configs] = ebs_blocks
|
|
instance_group[key_ebs_config] = ebs_configuration
|
|
|
|
@generate_boto3_response("SetTerminationProtection")
|
|
def set_termination_protection(self):
|
|
termination_protection = self._get_bool_param("TerminationProtected")
|
|
job_ids = self._get_multi_param("JobFlowIds.member")
|
|
self.backend.set_termination_protection(job_ids, termination_protection)
|
|
template = self.response_template(SET_TERMINATION_PROTECTION_TEMPLATE)
|
|
return template.render()
|
|
|
|
@generate_boto3_response("SetVisibleToAllUsers")
|
|
def set_visible_to_all_users(self):
|
|
visible_to_all_users = self._get_param("VisibleToAllUsers")
|
|
job_ids = self._get_multi_param("JobFlowIds.member")
|
|
self.backend.set_visible_to_all_users(job_ids, visible_to_all_users)
|
|
template = self.response_template(SET_VISIBLE_TO_ALL_USERS_TEMPLATE)
|
|
return template.render()
|
|
|
|
@generate_boto3_response("TerminateJobFlows")
|
|
def terminate_job_flows(self):
|
|
job_ids = self._get_multi_param("JobFlowIds.member.")
|
|
self.backend.terminate_job_flows(job_ids)
|
|
template = self.response_template(TERMINATE_JOB_FLOWS_TEMPLATE)
|
|
return template.render()
|
|
|
|
@generate_boto3_response("PutAutoScalingPolicy")
|
|
def put_auto_scaling_policy(self):
|
|
cluster_id = self._get_param("ClusterId")
|
|
cluster = self.backend.get_cluster(cluster_id)
|
|
instance_group_id = self._get_param("InstanceGroupId")
|
|
auto_scaling_policy = self._get_param("AutoScalingPolicy")
|
|
instance_group = self.backend.put_auto_scaling_policy(
|
|
instance_group_id, auto_scaling_policy
|
|
)
|
|
template = self.response_template(PUT_AUTO_SCALING_POLICY)
|
|
return template.render(
|
|
cluster_id=cluster_id, cluster=cluster, instance_group=instance_group
|
|
)
|
|
|
|
@generate_boto3_response("RemoveAutoScalingPolicy")
|
|
def remove_auto_scaling_policy(self):
|
|
cluster_id = self._get_param("ClusterId")
|
|
instance_group_id = self._get_param("InstanceGroupId")
|
|
instance_group = self.backend.put_auto_scaling_policy(instance_group_id, None)
|
|
template = self.response_template(REMOVE_AUTO_SCALING_POLICY)
|
|
return template.render(cluster_id=cluster_id, instance_group=instance_group)
|
|
|
|
|
|
ADD_INSTANCE_GROUPS_TEMPLATE = """<AddInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<AddInstanceGroupsResult>
|
|
<InstanceGroupIds>
|
|
{% for instance_group in instance_groups %}
|
|
<member>{{ instance_group.id }}</member>
|
|
{% endfor %}
|
|
</InstanceGroupIds>
|
|
</AddInstanceGroupsResult>
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</AddInstanceGroupsResponse>"""
|
|
|
|
ADD_JOB_FLOW_STEPS_TEMPLATE = """<AddJobFlowStepsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<AddJobFlowStepsResult>
|
|
<StepIds>
|
|
{% for step in steps %}
|
|
<member>{{ step.id }}</member>
|
|
{% endfor %}
|
|
</StepIds>
|
|
</AddJobFlowStepsResult>
|
|
<ResponseMetadata>
|
|
<RequestId>df6f4f4a-ed85-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</AddJobFlowStepsResponse>"""
|
|
|
|
ADD_TAGS_TEMPLATE = """<AddTagsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</AddTagsResponse>"""
|
|
|
|
DESCRIBE_CLUSTER_TEMPLATE = """<DescribeClusterResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<DescribeClusterResult>
|
|
<Cluster>
|
|
<Applications>
|
|
{% for application in cluster.applications %}
|
|
<member>
|
|
<Name>{{ application.name }}</Name>
|
|
<Version>{{ application.version }}</Version>
|
|
</member>
|
|
{% endfor %}
|
|
</Applications>
|
|
<AutoTerminate>{{ (not cluster.keep_job_flow_alive_when_no_steps)|lower }}</AutoTerminate>
|
|
<Configurations>
|
|
{% for configuration in cluster.configurations %}
|
|
<member>
|
|
<Classification>{{ configuration['classification'] }}</Classification>
|
|
<Properties>
|
|
{% for key, value in configuration['properties'].items() %}
|
|
<entry>
|
|
<key>{{ key }}</key>
|
|
<value>{{ value }}</value>
|
|
</entry>
|
|
{% endfor %}
|
|
</Properties>
|
|
</member>
|
|
{% endfor %}
|
|
</Configurations>
|
|
{% if cluster.custom_ami_id is not none %}
|
|
<CustomAmiId>{{ cluster.custom_ami_id }}</CustomAmiId>
|
|
{% endif %}
|
|
<Ec2InstanceAttributes>
|
|
<AdditionalMasterSecurityGroups>
|
|
{% for each in cluster.additional_master_security_groups %}
|
|
<member>{{ each }}</member>
|
|
{% endfor %}
|
|
</AdditionalMasterSecurityGroups>
|
|
<AdditionalSlaveSecurityGroups>
|
|
{% for each in cluster.additional_slave_security_groups %}
|
|
<member>{{ each }}</member>
|
|
{% endfor %}
|
|
</AdditionalSlaveSecurityGroups>
|
|
<Ec2AvailabilityZone>{{ cluster.availability_zone }}</Ec2AvailabilityZone>
|
|
<Ec2KeyName>{{ cluster.ec2_key_name }}</Ec2KeyName>
|
|
<Ec2SubnetId>{{ cluster.ec2_subnet_id }}</Ec2SubnetId>
|
|
<IamInstanceProfile>{{ cluster.role }}</IamInstanceProfile>
|
|
<EmrManagedMasterSecurityGroup>{{ cluster.master_security_group }}</EmrManagedMasterSecurityGroup>
|
|
<EmrManagedSlaveSecurityGroup>{{ cluster.slave_security_group }}</EmrManagedSlaveSecurityGroup>
|
|
<ServiceAccessSecurityGroup>{{ cluster.service_access_security_group }}</ServiceAccessSecurityGroup>
|
|
</Ec2InstanceAttributes>
|
|
<Id>{{ cluster.id }}</Id>
|
|
<KerberosAttributes>
|
|
{% if 'Realm' in cluster.kerberos_attributes%}
|
|
<Realm>{{ cluster.kerberos_attributes['Realm'] }}</Realm>
|
|
{% endif %}
|
|
{% if 'KdcAdminPassword' in cluster.kerberos_attributes%}
|
|
<KdcAdminPassword>{{ cluster.kerberos_attributes['KdcAdminPassword'] }}</KdcAdminPassword>
|
|
{% endif %}
|
|
{% if 'CrossRealmTrustPrincipalPassword' in cluster.kerberos_attributes%}
|
|
<CrossRealmTrustPrincipalPassword>{{ cluster.kerberos_attributes['CrossRealmTrustPrincipalPassword'] }}</CrossRealmTrustPrincipalPassword>
|
|
{% endif %}
|
|
{% if 'ADDomainJoinUser' in cluster.kerberos_attributes%}
|
|
<ADDomainJoinUser>{{ cluster.kerberos_attributes['ADDomainJoinUser'] }}</ADDomainJoinUser>
|
|
{% endif %}
|
|
{% if 'ADDomainJoinPassword' in cluster.kerberos_attributes%}
|
|
<ADDomainJoinPassword>{{ cluster.kerberos_attributes['ADDomainJoinPassword'] }}</ADDomainJoinPassword>
|
|
{% endif %}
|
|
</KerberosAttributes>
|
|
<LogUri>{{ cluster.log_uri }}</LogUri>
|
|
<MasterPublicDnsName>ec2-184-0-0-1.us-west-1.compute.amazonaws.com</MasterPublicDnsName>
|
|
<Name>{{ cluster.name }}</Name>
|
|
<NormalizedInstanceHours>{{ cluster.normalized_instance_hours }}</NormalizedInstanceHours>
|
|
{% if cluster.release_label is not none %}
|
|
<ReleaseLabel>{{ cluster.release_label }}</ReleaseLabel>
|
|
{% endif %}
|
|
{% if cluster.requested_ami_version is not none %}
|
|
<RequestedAmiVersion>{{ cluster.requested_ami_version }}</RequestedAmiVersion>
|
|
{% endif %}
|
|
{% if cluster.running_ami_version is not none %}
|
|
<RunningAmiVersion>{{ cluster.running_ami_version }}</RunningAmiVersion>
|
|
{% endif %}
|
|
{% if cluster.security_configuration is not none %}
|
|
<SecurityConfiguration>{{ cluster.security_configuration }}</SecurityConfiguration>
|
|
{% endif %}
|
|
<ServiceRole>{{ cluster.service_role }}</ServiceRole>
|
|
<Status>
|
|
<State>{{ cluster.state }}</State>
|
|
<StateChangeReason>
|
|
{% if cluster.last_state_change_reason is not none %}
|
|
<Message>{{ cluster.last_state_change_reason }}</Message>
|
|
{% endif %}
|
|
<Code>USER_REQUEST</Code>
|
|
</StateChangeReason>
|
|
<Timeline>
|
|
<CreationDateTime>{{ cluster.creation_datetime.isoformat() }}</CreationDateTime>
|
|
{% if cluster.end_datetime is not none %}
|
|
<EndDateTime>{{ cluster.end_datetime.isoformat() }}</EndDateTime>
|
|
{% endif %}
|
|
{% if cluster.ready_datetime is not none %}
|
|
<ReadyDateTime>{{ cluster.ready_datetime.isoformat() }}</ReadyDateTime>
|
|
{% endif %}
|
|
</Timeline>
|
|
</Status>
|
|
<Tags>
|
|
{% for tag_key, tag_value in cluster.tags.items() %}
|
|
<member>
|
|
<Key>{{ tag_key }}</Key>
|
|
<Value>{{ tag_value }}</Value>
|
|
</member>
|
|
{% endfor %}
|
|
</Tags>
|
|
<TerminationProtected>{{ cluster.termination_protected|lower }}</TerminationProtected>
|
|
<VisibleToAllUsers>{{ cluster.visible_to_all_users|lower }}</VisibleToAllUsers>
|
|
<StepConcurrencyLevel>{{ cluster.step_concurrency_level }}</StepConcurrencyLevel>
|
|
<ClusterArn>{{ cluster.arn }}</ClusterArn>
|
|
</Cluster>
|
|
</DescribeClusterResult>
|
|
<ResponseMetadata>
|
|
<RequestId>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</RequestId>
|
|
</ResponseMetadata>
|
|
</DescribeClusterResponse>"""
|
|
|
|
DESCRIBE_JOB_FLOWS_TEMPLATE = """<DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<DescribeJobFlowsResult>
|
|
<JobFlows>
|
|
{% for cluster in clusters %}
|
|
<member>
|
|
{% if cluster.running_ami_version is not none %}
|
|
<AmiVersion>{{ cluster.running_ami_version }}</AmiVersion>
|
|
{% endif %}
|
|
{% if cluster.bootstrap_actions %}
|
|
<BootstrapActions>
|
|
{% for bootstrap_action in cluster.bootstrap_actions %}
|
|
<member>
|
|
<BootstrapActionConfig>
|
|
<Name>{{ bootstrap_action.name }}</Name>
|
|
<ScriptBootstrapAction>
|
|
<Args>
|
|
{% for arg in bootstrap_action.args %}
|
|
<member>{{ arg | escape }}</member>
|
|
{% endfor %}
|
|
</Args>
|
|
<Path>{{ bootstrap_action.script_path | escape }}</Path>
|
|
</ScriptBootstrapAction>
|
|
</BootstrapActionConfig>
|
|
</member>
|
|
{% endfor %}
|
|
</BootstrapActions>
|
|
{% endif %}
|
|
<ExecutionStatusDetail>
|
|
<CreationDateTime>{{ cluster.creation_datetime.isoformat() }}</CreationDateTime>
|
|
{% if cluster.end_datetime is not none %}
|
|
<EndDateTime>{{ cluster.end_datetime.isoformat() }}</EndDateTime>
|
|
{% endif %}
|
|
{% if cluster.last_state_change_reason is not none %}
|
|
<LastStateChangeReason>{{ cluster.last_state_change_reason }}</LastStateChangeReason>
|
|
{% endif %}
|
|
{% if cluster.ready_datetime is not none %}
|
|
<ReadyDateTime>{{ cluster.ready_datetime.isoformat() }}</ReadyDateTime>
|
|
{% endif %}
|
|
{% if cluster.start_datetime is not none %}
|
|
<StartDateTime>{{ cluster.start_datetime.isoformat() }}</StartDateTime>
|
|
{% endif %}
|
|
<State>{{ cluster.state }}</State>
|
|
</ExecutionStatusDetail>
|
|
<Instances>
|
|
{% if cluster.ec2_key_name is not none %}
|
|
<Ec2KeyName>{{ cluster.ec2_key_name }}</Ec2KeyName>
|
|
{% endif %}
|
|
{% if cluster.ec2_subnet_id is not none %}
|
|
<Ec2SubnetId>{{ cluster.ec2_subnet_id }}</Ec2SubnetId>
|
|
{% endif %}
|
|
<HadoopVersion>{{ cluster.hadoop_version }}</HadoopVersion>
|
|
<InstanceCount>{{ cluster.instance_count }}</InstanceCount>
|
|
<InstanceGroups>
|
|
{% for instance_group in cluster.instance_groups %}
|
|
<member>
|
|
{% if instance_group.bid_price is not none %}
|
|
<BidPrice>{{ instance_group.bid_price }}</BidPrice>
|
|
{% endif %}
|
|
<CreationDateTime>{{ instance_group.creation_datetime.isoformat() }}</CreationDateTime>
|
|
{% if instance_group.end_datetime is not none %}
|
|
<EndDateTime>{{ instance_group.end_datetime.isoformat() }}</EndDateTime>
|
|
{% endif %}
|
|
|
|
<InstanceGroupId>{{ instance_group.id }}</InstanceGroupId>
|
|
<InstanceRequestCount>{{ instance_group.num_instances }}</InstanceRequestCount>
|
|
<InstanceRole>{{ instance_group.role }}</InstanceRole>
|
|
<InstanceRunningCount>{{ instance_group.num_instances }}</InstanceRunningCount>
|
|
<InstanceType>{{ instance_group.type }}</InstanceType>
|
|
<LastStateChangeReason/>
|
|
<Market>{{ instance_group.market }}</Market>
|
|
<Name>{{ instance_group.name }}</Name>
|
|
{% if instance_group.ready_datetime is not none %}
|
|
<ReadyDateTime>{{ instance_group.ready_datetime.isoformat() }}</ReadyDateTime>
|
|
{% endif %}
|
|
{% if instance_group.start_datetime is not none %}
|
|
<StartDateTime>{{ instance_group.start_datetime.isoformat() }}</StartDateTime>
|
|
{% endif %}
|
|
<State>{{ instance_group.state }}</State>
|
|
</member>
|
|
{% endfor %}
|
|
</InstanceGroups>
|
|
<KeepJobFlowAliveWhenNoSteps>{{ cluster.keep_job_flow_alive_when_no_steps|lower }}</KeepJobFlowAliveWhenNoSteps>
|
|
<MasterInstanceId>{{ cluster.master_instance_id }}</MasterInstanceId>
|
|
<MasterInstanceType>{{ cluster.master_instance_type }}</MasterInstanceType>
|
|
<MasterPublicDnsName>ec2-184-0-0-1.{{ cluster.region }}.compute.amazonaws.com</MasterPublicDnsName>
|
|
<NormalizedInstanceHours>{{ cluster.normalized_instance_hours }}</NormalizedInstanceHours>
|
|
<Placement>
|
|
<AvailabilityZone>{{ cluster.availability_zone }}</AvailabilityZone>
|
|
</Placement>
|
|
<SlaveInstanceType>{{ cluster.slave_instance_type }}</SlaveInstanceType>
|
|
<TerminationProtected>{{ cluster.termination_protected|lower }}</TerminationProtected>
|
|
</Instances>
|
|
<JobFlowId>{{ cluster.id }}</JobFlowId>
|
|
<JobFlowRole>{{ cluster.role }}</JobFlowRole>
|
|
<LogUri>{{ cluster.log_uri }}</LogUri>
|
|
<Name>{{ cluster.name }}</Name>
|
|
<ServiceRole>{{ cluster.service_role }}</ServiceRole>
|
|
<Steps>
|
|
{% for step in cluster.steps %}
|
|
<member>
|
|
<ExecutionStatusDetail>
|
|
<CreationDateTime>{{ step.creation_datetime.isoformat() }}</CreationDateTime>
|
|
{% if step.end_datetime is not none %}
|
|
<EndDateTime>{{ step.end_datetime.isoformat() }}</EndDateTime>
|
|
{% endif %}
|
|
{% if step.last_state_change_reason is not none %}
|
|
<LastStateChangeReason>{{ step.last_state_change_reason }}</LastStateChangeReason>
|
|
{% endif %}
|
|
{% if step.ready_datetime is not none %}
|
|
<ReadyDateTime>{{ step.ready_datetime.isoformat() }}</ReadyDateTime>
|
|
{% endif %}
|
|
{% if step.start_datetime is not none %}
|
|
<StartDateTime>{{ step.start_datetime.isoformat() }}</StartDateTime>
|
|
{% endif %}
|
|
<State>{{ step.state }}</State>
|
|
</ExecutionStatusDetail>
|
|
<StepConfig>
|
|
<ActionOnFailure>{{ step.action_on_failure }}</ActionOnFailure>
|
|
<HadoopJarStep>
|
|
<Jar>{{ step.jar }}</Jar>
|
|
<MainClass>{{ step.main_class }}</MainClass>
|
|
<Args>
|
|
{% for arg in step.args %}
|
|
<member>{{ arg | escape }}</member>
|
|
{% endfor %}
|
|
</Args>
|
|
<Properties/>
|
|
</HadoopJarStep>
|
|
<Name>{{ step.name | escape }}</Name>
|
|
</StepConfig>
|
|
</member>
|
|
{% endfor %}
|
|
</Steps>
|
|
<SupportedProducts/>
|
|
<VisibleToAllUsers>{{ cluster.visible_to_all_users|lower }}</VisibleToAllUsers>
|
|
</member>
|
|
{% endfor %}
|
|
</JobFlows>
|
|
</DescribeJobFlowsResult>
|
|
<ResponseMetadata>
|
|
<RequestId>9cea3229-ed85-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</DescribeJobFlowsResponse>"""
|
|
|
|
DESCRIBE_STEP_TEMPLATE = """<DescribeStepResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<DescribeStepResult>
|
|
<Step>
|
|
<ActionOnFailure>{{ step.action_on_failure }}</ActionOnFailure>
|
|
<Config>
|
|
<Args>
|
|
{% for arg in step.args %}
|
|
<member>{{ arg | escape }}</member>
|
|
{% endfor %}
|
|
</Args>
|
|
<Jar>{{ step.jar }}</Jar>
|
|
<MainClass/>
|
|
<Properties>
|
|
{% for key, val in step.properties.items() %}
|
|
<member>
|
|
<key>{{ key }}</key>
|
|
<value>{{ val | escape }}</value>
|
|
</member>
|
|
{% endfor %}
|
|
</Properties>
|
|
</Config>
|
|
<Id>{{ step.id }}</Id>
|
|
<Name>{{ step.name | escape }}</Name>
|
|
<Status>
|
|
<FailureDetails>
|
|
<Reason/>
|
|
<Message/>
|
|
<LogFile/>
|
|
</FailureDetails>
|
|
<State>{{ step.state }}</State>
|
|
<StateChangeReason>{{ step.state_change_reason }}</StateChangeReason>
|
|
<Timeline>
|
|
<CreationDateTime>{{ step.creation_datetime.isoformat() }}</CreationDateTime>
|
|
{% if step.end_datetime is not none %}
|
|
<EndDateTime>{{ step.end_datetime.isoformat() }}</EndDateTime>
|
|
{% endif %}
|
|
{% if step.ready_datetime is not none %}
|
|
<StartDateTime>{{ step.start_datetime.isoformat() }}</StartDateTime>
|
|
{% endif %}
|
|
</Timeline>
|
|
</Status>
|
|
</Step>
|
|
</DescribeStepResult>
|
|
<ResponseMetadata>
|
|
<RequestId>df6f4f4a-ed85-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</DescribeStepResponse>"""
|
|
|
|
LIST_BOOTSTRAP_ACTIONS_TEMPLATE = """<ListBootstrapActionsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ListBootstrapActionsResult>
|
|
<BootstrapActions>
|
|
{% for bootstrap_action in bootstrap_actions %}
|
|
<member>
|
|
<Args>
|
|
{% for arg in bootstrap_action.args %}
|
|
<member>{{ arg | escape }}</member>
|
|
{% endfor %}
|
|
</Args>
|
|
<Name>{{ bootstrap_action.name }}</Name>
|
|
<ScriptPath>{{ bootstrap_action.script_path }}</ScriptPath>
|
|
</member>
|
|
{% endfor %}
|
|
</BootstrapActions>
|
|
{% if marker is not none %}
|
|
<Marker>{{ marker }}</Marker>
|
|
{% endif %}
|
|
</ListBootstrapActionsResult>
|
|
<ResponseMetadata>
|
|
<RequestId>df6f4f4a-ed85-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</ListBootstrapActionsResponse>"""
|
|
|
|
LIST_CLUSTERS_TEMPLATE = """<ListClustersResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ListClustersResult>
|
|
<Clusters>
|
|
{% for cluster in clusters %}
|
|
<member>
|
|
<Id>{{ cluster.id }}</Id>
|
|
<Name>{{ cluster.name }}</Name>
|
|
<NormalizedInstanceHours>{{ cluster.normalized_instance_hours }}</NormalizedInstanceHours>
|
|
<Status>
|
|
<State>{{ cluster.state }}</State>
|
|
<StateChangeReason>
|
|
<Code>USER_REQUEST</Code>
|
|
{% if cluster.last_state_change_reason is not none %}
|
|
<Message>{{ cluster.last_state_change_reason }}</Message>
|
|
{% endif %}
|
|
</StateChangeReason>
|
|
<Timeline>
|
|
<CreationDateTime>{{ cluster.creation_datetime.isoformat() }}</CreationDateTime>
|
|
{% if cluster.end_datetime is not none %}
|
|
<EndDateTime>{{ cluster.end_datetime.isoformat() }}</EndDateTime>
|
|
{% endif %}
|
|
{% if cluster.ready_datetime is not none %}
|
|
<ReadyDateTime>{{ cluster.ready_datetime.isoformat() }}</ReadyDateTime>
|
|
{% endif %}
|
|
</Timeline>
|
|
</Status>
|
|
<ClusterArn>{{ cluster.arn }}</ClusterArn>
|
|
</member>
|
|
{% endfor %}
|
|
</Clusters>
|
|
{% if marker is not none %}
|
|
<Marker>{{ marker }}</Marker>
|
|
{% endif %}
|
|
</ListClustersResult>
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8418</RequestId>
|
|
</ResponseMetadata>
|
|
</ListClustersResponse>"""
|
|
|
|
LIST_INSTANCE_GROUPS_TEMPLATE = """<ListInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ListInstanceGroupsResult>
|
|
<InstanceGroups>
|
|
{% for instance_group in instance_groups %}
|
|
<member>
|
|
{% if instance_group.bid_price is not none %}
|
|
<BidPrice>{{ instance_group.bid_price }}</BidPrice>
|
|
{% endif %}
|
|
<Configurations/>
|
|
{% if instance_group.ebs_configuration is not none %}
|
|
<EbsBlockDevices>
|
|
{% for ebs_block_device in instance_group.ebs_configuration.ebs_block_device_configs %}
|
|
{% for i in range(ebs_block_device.volumes_per_instance) %}
|
|
<member>
|
|
<VolumeSpecification>
|
|
<VolumeType>{{ebs_block_device.volume_specification.volume_type}}</VolumeType>
|
|
<Iops>{{ebs_block_device.volume_specification.iops}}</Iops>
|
|
<SizeInGB>{{ebs_block_device.volume_specification.size_in_gb}}</SizeInGB>
|
|
</VolumeSpecification>
|
|
<Device>/dev/sd{{i}}</Device>
|
|
</member>
|
|
{% endfor %}
|
|
{% endfor %}
|
|
</EbsBlockDevices>
|
|
{% endif %}
|
|
{% if instance_group.auto_scaling_policy is not none %}
|
|
<AutoScalingPolicy>
|
|
{% if instance_group.auto_scaling_policy.constraints is not none %}
|
|
<Constraints>
|
|
{% if instance_group.auto_scaling_policy.constraints.min_capacity is not none %}
|
|
<MinCapacity>{{instance_group.auto_scaling_policy.constraints.min_capacity}}</MinCapacity>
|
|
{% endif %}
|
|
{% if instance_group.auto_scaling_policy.constraints.max_capacity is not none %}
|
|
<MaxCapacity>{{instance_group.auto_scaling_policy.constraints.max_capacity}}</MaxCapacity>
|
|
{% endif %}
|
|
</Constraints>
|
|
{% endif %}
|
|
{% if instance_group.auto_scaling_policy.rules is not none %}
|
|
<Rules>
|
|
{% for rule in instance_group.auto_scaling_policy.rules %}
|
|
<member>
|
|
{% if 'name' in rule %}
|
|
<Name>{{rule['name']}}</Name>
|
|
{% endif %}
|
|
{% if 'description' in rule %}
|
|
<Description>{{rule['description']}}</Description>
|
|
{% endif %}
|
|
{% if 'action' in rule %}
|
|
<Action>
|
|
{% if 'market' in rule['action'] %}
|
|
<Market>{{rule['action']['market']}}</Market>
|
|
{% endif %}
|
|
{% if 'simple_scaling_policy_configuration' in rule['action'] %}
|
|
<SimpleScalingPolicyConfiguration>
|
|
{% if 'adjustment_type' in rule['action']['simple_scaling_policy_configuration'] %}
|
|
<AdjustmentType>{{rule['action']['simple_scaling_policy_configuration']['adjustment_type']}}</AdjustmentType>
|
|
{% endif %}
|
|
{% if 'scaling_adjustment' in rule['action']['simple_scaling_policy_configuration'] %}
|
|
<ScalingAdjustment>{{rule['action']['simple_scaling_policy_configuration']['scaling_adjustment']}}</ScalingAdjustment>
|
|
{% endif %}
|
|
{% if 'cool_down' in rule['action']['simple_scaling_policy_configuration'] %}
|
|
<CoolDown>{{rule['action']['simple_scaling_policy_configuration']['cool_down']}}</CoolDown>
|
|
{% endif %}
|
|
</SimpleScalingPolicyConfiguration>
|
|
{% endif %}
|
|
</Action>
|
|
{% endif %}
|
|
{% if 'trigger' in rule %}
|
|
<Trigger>
|
|
{% if 'cloud_watch_alarm_definition' in rule['trigger'] %}
|
|
<CloudWatchAlarmDefinition>
|
|
{% if 'comparison_operator' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<ComparisonOperator>{{rule['trigger']['cloud_watch_alarm_definition']['comparison_operator']}}</ComparisonOperator>
|
|
{% endif %}
|
|
{% if 'evaluation_periods' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<EvaluationPeriods>{{rule['trigger']['cloud_watch_alarm_definition']['evaluation_periods']}}</EvaluationPeriods>
|
|
{% endif %}
|
|
{% if 'metric_name' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<MetricName>{{rule['trigger']['cloud_watch_alarm_definition']['metric_name']}}</MetricName>
|
|
{% endif %}
|
|
{% if 'namespace' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Namespace>{{rule['trigger']['cloud_watch_alarm_definition']['namespace']}}</Namespace>
|
|
{% endif %}
|
|
{% if 'period' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Period>{{rule['trigger']['cloud_watch_alarm_definition']['period']}}</Period>
|
|
{% endif %}
|
|
{% if 'statistic' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Statistic>{{rule['trigger']['cloud_watch_alarm_definition']['statistic']}}</Statistic>
|
|
{% endif %}
|
|
{% if 'threshold' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Threshold>{{rule['trigger']['cloud_watch_alarm_definition']['threshold']}}</Threshold>
|
|
{% endif %}
|
|
{% if 'unit' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Unit>{{rule['trigger']['cloud_watch_alarm_definition']['unit']}}</Unit>
|
|
{% endif %}
|
|
{% if 'dimensions' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Dimensions>
|
|
{% for dimension in rule['trigger']['cloud_watch_alarm_definition']['dimensions'] %}
|
|
<member>
|
|
{% if 'key' in dimension %}
|
|
<Key>{{dimension['key']}}</Key>
|
|
{% endif %}
|
|
{% if 'value' in dimension %}
|
|
<Value>{{dimension['value']}}</Value>
|
|
{% endif %}
|
|
</member>
|
|
{% endfor %}
|
|
</Dimensions>
|
|
{% endif %}
|
|
</CloudWatchAlarmDefinition>
|
|
{% endif %}
|
|
</Trigger>
|
|
{% endif %}
|
|
</member>
|
|
{% endfor %}
|
|
</Rules>
|
|
{% endif %}
|
|
{% if instance_group.auto_scaling_policy.status is not none %}
|
|
<Status>
|
|
{% if 'state' in instance_group.auto_scaling_policy.status %}
|
|
<State>{{instance_group.auto_scaling_policy.status['state']}}</State>
|
|
{% endif %}
|
|
</Status>
|
|
{% endif %}
|
|
</AutoScalingPolicy>
|
|
{% endif %}
|
|
{% if instance_group.ebs_optimized is not none %}
|
|
<EbsOptimized>{{ instance_group.ebs_optimized }}</EbsOptimized>
|
|
{% endif %}
|
|
<Id>{{ instance_group.id }}</Id>
|
|
<InstanceGroupType>{{ instance_group.role }}</InstanceGroupType>
|
|
<InstanceType>{{ instance_group.type }}</InstanceType>
|
|
<Market>{{ instance_group.market }}</Market>
|
|
<Name>{{ instance_group.name }}</Name>
|
|
<RequestedInstanceCount>{{ instance_group.num_instances }}</RequestedInstanceCount>
|
|
<RunningInstanceCount>{{ instance_group.num_instances }}</RunningInstanceCount>
|
|
<Status>
|
|
<State>{{ instance_group.state }}</State>
|
|
<StateChangeReason>
|
|
{% if instance_group.state_change_reason is not none %}
|
|
<Message>{{ instance_group.state_change_reason }}</Message>
|
|
{% endif %}
|
|
<Code>USER_REQUEST</Code>
|
|
</StateChangeReason>
|
|
<Timeline>
|
|
<CreationDateTime>{{ instance_group.creation_datetime.isoformat() }}</CreationDateTime>
|
|
{% if instance_group.end_datetime is not none %}
|
|
<EndDateTime>{{ instance_group.end_datetime.isoformat() }}</EndDateTime>
|
|
{% endif %}
|
|
{% if instance_group.ready_datetime is not none %}
|
|
<ReadyDateTime>{{ instance_group.ready_datetime.isoformat() }}</ReadyDateTime>
|
|
{% endif %}
|
|
</Timeline>
|
|
</Status>
|
|
</member>
|
|
{% endfor %}
|
|
</InstanceGroups>
|
|
{% if marker is not none %}
|
|
<Marker>{{ marker }}</Marker>
|
|
{% endif %}
|
|
</ListInstanceGroupsResult>
|
|
<ResponseMetadata>
|
|
<RequestId>8296d8b8-ed85-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</ListInstanceGroupsResponse>"""
|
|
|
|
LIST_INSTANCES_TEMPLATE = """<ListInstancesResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ListInstancesResult>
|
|
<Instances>
|
|
{% for instance in instances %}
|
|
<member>
|
|
<Id>{{ instance.id }}</Id>
|
|
<Ec2InstanceId>{{ instance.ec2_instance_id }}</Ec2InstanceId>
|
|
<PublicDnsName>{{ instance.details.public_dns }}</PublicDnsName>
|
|
<PublicIpAddress>{{ instance.details.public_ip }}</PublicIpAddress>
|
|
<PrivateDnsName>{{ instance.details.private_dns }}</PrivateDnsName>
|
|
<PrivateIpAddress>{{ instance.details.private_ip }}</PrivateIpAddress>
|
|
<InstanceGroupId>{{ instance.instance_group.id }}</InstanceGroupId>
|
|
<InstanceFleetId>{{ instance.instance_fleet_id }}</InstanceFleetId>
|
|
<Market>{{ instance.instance_group.market }}</Market>
|
|
<InstanceType>{{ instance.details.instance_type }}</InstanceType>
|
|
<EbsVolumes>
|
|
{% for volume in instance.details.block_device_mapping %}
|
|
<member>
|
|
<Device>{{ volume }}</Device>
|
|
<VolumeId>{{ instance.details.block_device_mapping[volume].volume_id }}</VolumeId>
|
|
</member>
|
|
{% endfor %}
|
|
</EbsVolumes>
|
|
<Status>
|
|
<State>{{ instance.instance_group.state }}</State>
|
|
<StateChangeReason>
|
|
{% if instance.state_change_reason is not none %}
|
|
<Message>{{ instance.state_change_reason }}</Message>
|
|
{% endif %}
|
|
</StateChangeReason>
|
|
<Timeline>
|
|
<CreationDateTime>{{ instance.instance_group.creation_datetime.isoformat() }}</CreationDateTime>
|
|
{% if instance.instance_group.end_datetime is not none %}
|
|
<EndDateTime>{{ instance.instance_group.end_datetime.isoformat() }}</EndDateTime>
|
|
{% endif %}
|
|
{% if instance.instance_group.ready_datetime is not none %}
|
|
<ReadyDateTime>{{ instance.instance_group.ready_datetime.isoformat() }}</ReadyDateTime>
|
|
{% endif %}
|
|
</Timeline>
|
|
</Status>
|
|
</member>
|
|
{% endfor %}
|
|
</Instances>
|
|
</ListInstancesResult>
|
|
<ResponseMetadata>
|
|
<RequestId>4248c46c-71c0-4772-b155-0e992dc30027</RequestId>
|
|
</ResponseMetadata>
|
|
</ListInstancesResponse>"""
|
|
|
|
LIST_STEPS_TEMPLATE = """<ListStepsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ListStepsResult>
|
|
<Steps>
|
|
{% for step in steps %}
|
|
<member>
|
|
<ActionOnFailure>{{ step.action_on_failure }}</ActionOnFailure>
|
|
<Config>
|
|
<Args>
|
|
{% for arg in step.args %}
|
|
<member>{{ arg | escape }}</member>
|
|
{% endfor %}
|
|
</Args>
|
|
<Jar>{{ step.jar | escape }}</Jar>
|
|
<MainClass/>
|
|
<Properties>
|
|
{% for key, val in step.properties.items() %}
|
|
<member>
|
|
<key>{{ key }}</key>
|
|
<value>{{ val | escape }}</value>
|
|
</member>
|
|
{% endfor %}
|
|
</Properties>
|
|
</Config>
|
|
<Id>{{ step.id }}</Id>
|
|
<Name>{{ step.name | escape }}</Name>
|
|
<Status>
|
|
<!-- does not exist for botocore 1.4.28
|
|
<FailureDetails>
|
|
<Reason/>
|
|
<Message/>
|
|
<LogFile/>
|
|
</FailureDetails>
|
|
-->
|
|
<State>{{ step.state }}</State>
|
|
<StateChangeReason>{{ step.state_change_reason }}</StateChangeReason>
|
|
<Timeline>
|
|
<CreationDateTime>{{ step.creation_datetime.isoformat() }}</CreationDateTime>
|
|
{% if step.end_datetime is not none %}
|
|
<EndDateTime>{{ step.end_datetime.isoformat() }}</EndDateTime>
|
|
{% endif %}
|
|
{% if step.start_datetime is not none %}
|
|
<StartDateTime>{{ step.start_datetime.isoformat() }}</StartDateTime>
|
|
{% endif %}
|
|
</Timeline>
|
|
</Status>
|
|
</member>
|
|
{% endfor %}
|
|
</Steps>
|
|
{% if marker is not none %}
|
|
<Marker>{{ marker }}</Marker>
|
|
{% endif %}
|
|
</ListStepsResult>
|
|
<ResponseMetadata>
|
|
<RequestId>df6f4f4a-ed85-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</ListStepsResponse>"""
|
|
|
|
MODIFY_CLUSTER_TEMPLATE = """<ModifyClusterResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ModifyClusterResult>
|
|
<StepConcurrencyLevel>{{ cluster.step_concurrency_level }}</StepConcurrencyLevel>
|
|
</ModifyClusterResult>
|
|
<ResponseMetadata>
|
|
<RequestId>0751c837-e78d-4aef-95c9-9c4d29a092ff</RequestId>
|
|
</ResponseMetadata>
|
|
</ModifyClusterResponse>
|
|
"""
|
|
|
|
MODIFY_INSTANCE_GROUPS_TEMPLATE = """<ModifyInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</ModifyInstanceGroupsResponse>"""
|
|
|
|
REMOVE_TAGS_TEMPLATE = """<RemoveTagsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</RemoveTagsResponse>"""
|
|
|
|
RUN_JOB_FLOW_TEMPLATE = """<RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<RunJobFlowResult>
|
|
<JobFlowId>{{ cluster.id }}</JobFlowId>
|
|
<ClusterArn>{{ cluster.arn }}</ClusterArn>
|
|
</RunJobFlowResult>
|
|
<ResponseMetadata>
|
|
<RequestId>8296d8b8-ed85-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</RunJobFlowResponse>"""
|
|
|
|
SET_TERMINATION_PROTECTION_TEMPLATE = """<SetTerminationProtection xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</SetTerminationProtection>"""
|
|
|
|
SET_VISIBLE_TO_ALL_USERS_TEMPLATE = """<SetVisibleToAllUsersResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</SetVisibleToAllUsersResponse>"""
|
|
|
|
TERMINATE_JOB_FLOWS_TEMPLATE = """<TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</TerminateJobFlowsResponse>"""
|
|
|
|
PUT_AUTO_SCALING_POLICY = """<PutAutoScalingPolicyResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<PutAutoScalingPolicyResult>
|
|
<ClusterId>{{cluster_id}}</ClusterId>
|
|
<InstanceGroupId>{{instance_group.id}}</InstanceGroupId>
|
|
{% if instance_group.auto_scaling_policy is not none %}
|
|
<AutoScalingPolicy>
|
|
{% if instance_group.auto_scaling_policy.constraints is not none %}
|
|
<Constraints>
|
|
{% if instance_group.auto_scaling_policy.constraints.min_capacity is not none %}
|
|
<MinCapacity>{{instance_group.auto_scaling_policy.constraints.min_capacity}}</MinCapacity>
|
|
{% endif %}
|
|
{% if instance_group.auto_scaling_policy.constraints.max_capacity is not none %}
|
|
<MaxCapacity>{{instance_group.auto_scaling_policy.constraints.max_capacity}}</MaxCapacity>
|
|
{% endif %}
|
|
</Constraints>
|
|
{% endif %}
|
|
{% if instance_group.auto_scaling_policy.rules is not none %}
|
|
<Rules>
|
|
{% for rule in instance_group.auto_scaling_policy.rules %}
|
|
<member>
|
|
{% if 'name' in rule %}
|
|
<Name>{{rule['name']}}</Name>
|
|
{% endif %}
|
|
{% if 'description' in rule %}
|
|
<Description>{{rule['description']}}</Description>
|
|
{% endif %}
|
|
{% if 'action' in rule %}
|
|
<Action>
|
|
{% if 'market' in rule['action'] %}
|
|
<Market>{{rule['action']['market']}}</Market>
|
|
{% endif %}
|
|
{% if 'simple_scaling_policy_configuration' in rule['action'] %}
|
|
<SimpleScalingPolicyConfiguration>
|
|
{% if 'adjustment_type' in rule['action']['simple_scaling_policy_configuration'] %}
|
|
<AdjustmentType>{{rule['action']['simple_scaling_policy_configuration']['adjustment_type']}}</AdjustmentType>
|
|
{% endif %}
|
|
{% if 'scaling_adjustment' in rule['action']['simple_scaling_policy_configuration'] %}
|
|
<ScalingAdjustment>{{rule['action']['simple_scaling_policy_configuration']['scaling_adjustment']}}</ScalingAdjustment>
|
|
{% endif %}
|
|
{% if 'cool_down' in rule['action']['simple_scaling_policy_configuration'] %}
|
|
<CoolDown>{{rule['action']['simple_scaling_policy_configuration']['cool_down']}}</CoolDown>
|
|
{% endif %}
|
|
</SimpleScalingPolicyConfiguration>
|
|
{% endif %}
|
|
</Action>
|
|
{% endif %}
|
|
{% if 'trigger' in rule %}
|
|
<Trigger>
|
|
{% if 'cloud_watch_alarm_definition' in rule['trigger'] %}
|
|
<CloudWatchAlarmDefinition>
|
|
{% if 'comparison_operator' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<ComparisonOperator>{{rule['trigger']['cloud_watch_alarm_definition']['comparison_operator']}}</ComparisonOperator>
|
|
{% endif %}
|
|
{% if 'evaluation_periods' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<EvaluationPeriods>{{rule['trigger']['cloud_watch_alarm_definition']['evaluation_periods']}}</EvaluationPeriods>
|
|
{% endif %}
|
|
{% if 'metric_name' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<MetricName>{{rule['trigger']['cloud_watch_alarm_definition']['metric_name']}}</MetricName>
|
|
{% endif %}
|
|
{% if 'namespace' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Namespace>{{rule['trigger']['cloud_watch_alarm_definition']['namespace']}}</Namespace>
|
|
{% endif %}
|
|
{% if 'period' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Period>{{rule['trigger']['cloud_watch_alarm_definition']['period']}}</Period>
|
|
{% endif %}
|
|
{% if 'statistic' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Statistic>{{rule['trigger']['cloud_watch_alarm_definition']['statistic']}}</Statistic>
|
|
{% endif %}
|
|
{% if 'threshold' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Threshold>{{rule['trigger']['cloud_watch_alarm_definition']['threshold']}}</Threshold>
|
|
{% endif %}
|
|
{% if 'unit' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Unit>{{rule['trigger']['cloud_watch_alarm_definition']['unit']}}</Unit>
|
|
{% endif %}
|
|
{% if 'dimensions' in rule['trigger']['cloud_watch_alarm_definition'] %}
|
|
<Dimensions>
|
|
{% for dimension in rule['trigger']['cloud_watch_alarm_definition']['dimensions'] %}
|
|
<member>
|
|
{% if 'key' in dimension %}
|
|
<Key>{{dimension['key']}}</Key>
|
|
{% endif %}
|
|
{% if 'value' in dimension %}
|
|
<Value>{{dimension['value']}}</Value>
|
|
{% endif %}
|
|
</member>
|
|
{% endfor %}
|
|
</Dimensions>
|
|
{% endif %}
|
|
</CloudWatchAlarmDefinition>
|
|
{% endif %}
|
|
</Trigger>
|
|
{% endif %}
|
|
</member>
|
|
{% endfor %}
|
|
</Rules>
|
|
{% endif %}
|
|
{% if instance_group.auto_scaling_policy.status is not none %}
|
|
<Status>
|
|
{% if 'state' in instance_group.auto_scaling_policy.status %}
|
|
<State>{{instance_group.auto_scaling_policy.status['state']}}</State>
|
|
{% endif %}
|
|
</Status>
|
|
{% endif %}
|
|
</AutoScalingPolicy>
|
|
{% endif %}
|
|
<ClusterArn>{{ cluster.arn }}</ClusterArn>
|
|
</PutAutoScalingPolicyResult>
|
|
<ResponseMetadata>
|
|
<RequestId>d47379d9-b505-49af-9335-a68950d82535</RequestId>
|
|
</ResponseMetadata>
|
|
</PutAutoScalingPolicyResponse>"""
|
|
|
|
REMOVE_AUTO_SCALING_POLICY = """<RemoveAutoScalingPolicyResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>c04a1042-5340-4c0a-a7b5-7779725ce4f7</RequestId>
|
|
</ResponseMetadata>
|
|
</RemoveAutoScalingPolicyResponse>"""
|
|
|
|
CREATE_SECURITY_CONFIGURATION_TEMPLATE = """<CreateSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<CreateSecurityConfigurationResult>
|
|
<Name>{{name}}</Name>
|
|
<CreationDateTime>{{creation_date_time}}</CreationDateTime>
|
|
</CreateSecurityConfigurationResult>
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</CreateSecurityConfigurationResponse>"""
|
|
|
|
DESCRIBE_SECURITY_CONFIGURATION_TEMPLATE = """<DescribeSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<DescribeSecurityConfigurationResult>
|
|
<Name>{{security_configuration['name']}}</Name>
|
|
<SecurityConfiguration>{{security_configuration['security_configuration']}}</SecurityConfiguration>
|
|
<CreationDateTime>{{security_configuration['creation_date_time']}}</CreationDateTime>
|
|
</DescribeSecurityConfigurationResult>
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</DescribeSecurityConfigurationResponse>"""
|
|
|
|
DELETE_SECURITY_CONFIGURATION_TEMPLATE = """<DeleteSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
|
</ResponseMetadata>
|
|
</DeleteSecurityConfigurationResponse>"""
|