360 lines
14 KiB
Python
360 lines
14 KiB
Python
from __future__ import unicode_literals
|
|
import json
|
|
|
|
from moto.core.responses import BaseResponse
|
|
from .models import emr_backends
|
|
from .utils import tags_from_query_string
|
|
|
|
|
|
class ElasticMapReduceResponse(BaseResponse):
|
|
|
|
@property
|
|
def backend(self):
|
|
return emr_backends[self.region]
|
|
|
|
@property
|
|
def boto3_request(self):
|
|
return 'json' in self.headers.get('Content-Type', [])
|
|
|
|
def add_job_flow_steps(self):
|
|
job_flow_id = self._get_param('JobFlowId')
|
|
steps = self._get_list_prefix('Steps.member')
|
|
|
|
job_flow = self.backend.add_job_flow_steps(job_flow_id, steps)
|
|
template = self.response_template(ADD_JOB_FLOW_STEPS_TEMPLATE)
|
|
return template.render(job_flow=job_flow)
|
|
|
|
def run_job_flow(self):
|
|
flow_name = self._get_param('Name')
|
|
log_uri = self._get_param('LogUri')
|
|
steps = self._get_list_prefix('Steps.member')
|
|
instance_attrs = self._get_dict_param('Instances.')
|
|
job_flow_role = self._get_param('JobFlowRole')
|
|
visible_to_all_users = self._get_param('VisibleToAllUsers')
|
|
|
|
job_flow = self.backend.run_job_flow(
|
|
flow_name, log_uri, job_flow_role,
|
|
visible_to_all_users, steps, instance_attrs
|
|
)
|
|
instance_groups = self._get_list_prefix('Instances.InstanceGroups.member')
|
|
if instance_groups:
|
|
self.backend.add_instance_groups(job_flow.id, instance_groups)
|
|
|
|
if self.boto3_request:
|
|
return json.dumps({
|
|
"JobFlowId": job_flow.id
|
|
})
|
|
|
|
template = self.response_template(RUN_JOB_FLOW_TEMPLATE)
|
|
return template.render(job_flow=job_flow)
|
|
|
|
def describe_job_flows(self):
|
|
job_flow_ids = self._get_multi_param("JobFlowIds.member")
|
|
job_flows = self.backend.describe_job_flows(job_flow_ids)
|
|
template = self.response_template(DESCRIBE_JOB_FLOWS_TEMPLATE)
|
|
return template.render(job_flows=job_flows)
|
|
|
|
def terminate_job_flows(self):
|
|
job_ids = self._get_multi_param('JobFlowIds.member.')
|
|
job_flows = self.backend.terminate_job_flows(job_ids)
|
|
template = self.response_template(TERMINATE_JOB_FLOWS_TEMPLATE)
|
|
return template.render(job_flows=job_flows)
|
|
|
|
def add_instance_groups(self):
|
|
jobflow_id = self._get_param('JobFlowId')
|
|
instance_groups = self._get_list_prefix('InstanceGroups.member')
|
|
instance_groups = self.backend.add_instance_groups(jobflow_id, instance_groups)
|
|
template = self.response_template(ADD_INSTANCE_GROUPS_TEMPLATE)
|
|
return template.render(instance_groups=instance_groups)
|
|
|
|
def modify_instance_groups(self):
|
|
instance_groups = self._get_list_prefix('InstanceGroups.member')
|
|
instance_groups = self.backend.modify_instance_groups(instance_groups)
|
|
template = self.response_template(MODIFY_INSTANCE_GROUPS_TEMPLATE)
|
|
return template.render(instance_groups=instance_groups)
|
|
|
|
def set_visible_to_all_users(self):
|
|
visible_to_all_users = self._get_param('VisibleToAllUsers')
|
|
job_ids = self._get_multi_param('JobFlowIds.member')
|
|
self.backend.set_visible_to_all_users(job_ids, visible_to_all_users)
|
|
template = self.response_template(SET_VISIBLE_TO_ALL_USERS_TEMPLATE)
|
|
return template.render()
|
|
|
|
def set_termination_protection(self):
|
|
termination_protection = self._get_param('TerminationProtected')
|
|
job_ids = self._get_multi_param('JobFlowIds.member')
|
|
self.backend.set_termination_protection(job_ids, termination_protection)
|
|
template = self.response_template(SET_TERMINATION_PROTECTION_TEMPLATE)
|
|
return template.render()
|
|
|
|
def list_clusters(self):
|
|
clusters = self.backend.list_clusters()
|
|
|
|
if self.boto3_request:
|
|
return json.dumps({
|
|
"Clusters": [
|
|
{
|
|
"Id": cluster.id,
|
|
"Name": cluster.name,
|
|
"Status": {
|
|
"State": cluster.state,
|
|
"StatusChangeReason": {},
|
|
"TimeLine": {},
|
|
},
|
|
"NormalizedInstanceHours": cluster.normalized_instance_hours,
|
|
} for cluster in clusters
|
|
],
|
|
"Marker": ""
|
|
})
|
|
|
|
template = self.response_template(LIST_CLUSTERS_TEMPLATE)
|
|
return template.render(clusters=clusters)
|
|
|
|
def describe_cluster(self):
|
|
cluster_id = self._get_param('ClusterId')
|
|
cluster = self.backend.get_cluster(cluster_id)
|
|
template = self.response_template(DESCRIBE_CLUSTER_TEMPLATE)
|
|
return template.render(cluster=cluster)
|
|
|
|
def add_tags(self):
|
|
cluster_id = self._get_param('ResourceId')
|
|
tags = tags_from_query_string(self.querystring)
|
|
self.backend.add_tags(cluster_id, tags)
|
|
template = self.response_template(ADD_TAGS_TEMPLATE)
|
|
return template.render()
|
|
|
|
def remove_tags(self):
|
|
cluster_id = self._get_param('ResourceId')
|
|
tag_keys = self._get_multi_param('TagKeys.member')
|
|
self.backend.remove_tags(cluster_id, tag_keys)
|
|
template = self.response_template(REMOVE_TAGS_TEMPLATE)
|
|
return template.render()
|
|
|
|
|
|
RUN_JOB_FLOW_TEMPLATE = """<RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<RunJobFlowResult>
|
|
<JobFlowId>{{ job_flow.id }}</JobFlowId>
|
|
</RunJobFlowResult>
|
|
<ResponseMetadata>
|
|
<RequestId>
|
|
8296d8b8-ed85-11dd-9877-6fad448a8419
|
|
</RequestId>
|
|
</ResponseMetadata>
|
|
</RunJobFlowResponse>"""
|
|
|
|
DESCRIBE_JOB_FLOWS_TEMPLATE = """<DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<DescribeJobFlowsResult>
|
|
<JobFlows>
|
|
{% for job_flow in job_flows %}
|
|
<member>
|
|
<ExecutionStatusDetail>
|
|
<CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime>
|
|
<StartDateTime>2009-01-28T21:49:16Z</StartDateTime>
|
|
<State>{{ job_flow.state }}</State>
|
|
</ExecutionStatusDetail>
|
|
<Name>{{ job_flow.name }}</Name>
|
|
<JobFlowRole>{{ job_flow.role }}</JobFlowRole>
|
|
<LogUri>{{ job_flow.log_uri }}</LogUri>
|
|
<Steps>
|
|
{% for step in job_flow.steps %}
|
|
<member>
|
|
<ExecutionStatusDetail>
|
|
<CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime>
|
|
<State>{{ step.state }}</State>
|
|
</ExecutionStatusDetail>
|
|
<StepConfig>
|
|
<HadoopJarStep>
|
|
<Jar>{{ step.jar }}</Jar>
|
|
<MainClass>MyMainClass</MainClass>
|
|
<Args>
|
|
{% for arg in step.args %}
|
|
<member>{{ arg }}</member>
|
|
{% endfor %}
|
|
</Args>
|
|
<Properties/>
|
|
</HadoopJarStep>
|
|
<Name>{{ step.name }}</Name>
|
|
<ActionOnFailure>CONTINUE</ActionOnFailure>
|
|
</StepConfig>
|
|
</member>
|
|
{% endfor %}
|
|
</Steps>
|
|
<JobFlowId>{{ job_flow.id }}</JobFlowId>
|
|
<Instances>
|
|
<Placement>
|
|
<AvailabilityZone>us-east-1a</AvailabilityZone>
|
|
</Placement>
|
|
<SlaveInstanceType>{{ job_flow.slave_instance_type }}</SlaveInstanceType>
|
|
<MasterInstanceType>{{ job_flow.master_instance_type }}</MasterInstanceType>
|
|
<Ec2KeyName>{{ job_flow.ec2_key_name }}</Ec2KeyName>
|
|
<NormalizedInstanceHours>{{ job_flow.normalized_instance_hours }}</NormalizedInstanceHours>
|
|
<VisibleToAllUsers>{{ job_flow.visible_to_all_users }}</VisibleToAllUsers>
|
|
<InstanceCount>{{ job_flow.instance_count }}</InstanceCount>
|
|
<KeepJobFlowAliveWhenNoSteps>{{ job_flow.keep_job_flow_alive_when_no_steps }}</KeepJobFlowAliveWhenNoSteps>
|
|
<TerminationProtected>{{ job_flow.termination_protected }}</TerminationProtected>
|
|
<MasterPublicDnsName>ec2-184-0-0-1.us-west-1.compute.amazonaws.com</MasterPublicDnsName>
|
|
<InstanceGroups>
|
|
{% for instance_group in job_flow.instance_groups %}
|
|
<member>
|
|
<InstanceGroupId>{{ instance_group.id }}</InstanceGroupId>
|
|
<InstanceRole>{{ instance_group.role }}</InstanceRole>
|
|
<InstanceRunningCount>{{ instance_group.num_instances }}</InstanceRunningCount>
|
|
<InstanceType>{{ instance_group.type }}</InstanceType>
|
|
<Market>{{ instance_group.market }}</Market>
|
|
<Name>{{ instance_group.name }}</Name>
|
|
<BidPrice>{{ instance_group.bid_price }}</BidPrice>
|
|
</member>
|
|
{% endfor %}
|
|
</InstanceGroups>
|
|
</Instances>
|
|
</member>
|
|
{% endfor %}
|
|
</JobFlows>
|
|
</DescribeJobFlowsResult>
|
|
<ResponseMetadata>
|
|
<RequestId>
|
|
9cea3229-ed85-11dd-9877-6fad448a8419
|
|
</RequestId>
|
|
</ResponseMetadata>
|
|
</DescribeJobFlowsResponse>"""
|
|
|
|
TERMINATE_JOB_FLOWS_TEMPLATE = """<TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>
|
|
2690d7eb-ed86-11dd-9877-6fad448a8419
|
|
</RequestId>
|
|
</ResponseMetadata>
|
|
</TerminateJobFlowsResponse>"""
|
|
|
|
ADD_JOB_FLOW_STEPS_TEMPLATE = """<AddJobFlowStepsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>
|
|
df6f4f4a-ed85-11dd-9877-6fad448a8419
|
|
</RequestId>
|
|
</ResponseMetadata>
|
|
</AddJobFlowStepsResponse>"""
|
|
|
|
LIST_CLUSTERS_TEMPLATE = """<ListClustersResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<Clusters>
|
|
{% for cluster in clusters %}
|
|
<member>
|
|
<Id>{{ cluster.id }}</Id>
|
|
<Name>{{ cluster.name }}</Name>
|
|
<NormalizedInstanceHours>{{ cluster.normalized_instance_hours }}</NormalizedInstanceHours>
|
|
<Status>
|
|
<State>{{ cluster.state }}</State>
|
|
<StateChangeReason>
|
|
<Code></Code>
|
|
<Message></Message>
|
|
</StateChangeReason>
|
|
<Timeline></Timeline>
|
|
</Status>
|
|
</member>
|
|
{% endfor %}
|
|
</Clusters>
|
|
<Marker></Marker>
|
|
<ResponseMetadata>
|
|
<RequestId>
|
|
2690d7eb-ed86-11dd-9877-6fad448a8418
|
|
</RequestId>
|
|
</ResponseMetadata>
|
|
</ListClustersResponse>"""
|
|
|
|
DESCRIBE_CLUSTER_TEMPLATE = """<DescribeClusterResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<DescribeClusterResult>
|
|
<Cluster>
|
|
<Id>{{ cluster.id }}</Id>
|
|
<Tags>
|
|
{% for tag_key, tag_value in cluster.tags.items() %}
|
|
<member>
|
|
<Key>{{ tag_key }}</Key>
|
|
<Value>{{ tag_value }}</Value>
|
|
</member>
|
|
{% endfor %}
|
|
</Tags>
|
|
<Ec2InstanceAttributes>
|
|
<Ec2AvailabilityZone>{{ cluster.availability_zone }}</Ec2AvailabilityZone>
|
|
<Ec2SubnetId>{{ cluster.subnet_id }}</Ec2SubnetId>
|
|
<Ec2KeyName>{{ cluster.ec2_key_name }}</Ec2KeyName>
|
|
</Ec2InstanceAttributes>
|
|
<RunningAmiVersion>{{ cluster.running_ami_version }}</RunningAmiVersion>
|
|
<VisibleToAllUsers>{{ cluster.visible_to_all_users }}</VisibleToAllUsers>
|
|
<Status>
|
|
<StateChangeReason>
|
|
<Message>Terminated by user request</Message>
|
|
<Code>USER_REQUEST</Code>
|
|
</StateChangeReason>
|
|
<State>{{ cluster.state }}</State>
|
|
<Timeline>
|
|
<CreationDateTime>2014-01-24T01:21:21Z</CreationDateTime>
|
|
<ReadyDateTime>2014-01-24T01:25:26Z</ReadyDateTime>
|
|
<EndDateTime>2014-01-24T02:19:46Z</EndDateTime>
|
|
</Timeline>
|
|
</Status>
|
|
<AutoTerminate>{{ cluster.auto_terminate }}</AutoTerminate>
|
|
<Name>{{ cluster.name }}</Name>
|
|
<RequestedAmiVersion>{{ cluster.requested_ami_version }}</RequestedAmiVersion>
|
|
<Applications>
|
|
{% for application in cluster.applications %}
|
|
<member>
|
|
<Name>{{ application.name }}</Name>
|
|
<Version>{{ application.version }}</Version>
|
|
</member>
|
|
{% endfor %}
|
|
</Applications>
|
|
<TerminationProtected>{{ cluster.termination_protection }}</TerminationProtected>
|
|
<MasterPublicDnsName>ec2-184-0-0-1.us-west-1.compute.amazonaws.com</MasterPublicDnsName>
|
|
<NormalizedInstanceHours>{{ cluster.normalized_instance_hours }}</NormalizedInstanceHours>
|
|
<ServiceRole>{{ cluster.service_role }}</ServiceRole>
|
|
</Cluster>
|
|
</DescribeClusterResult>
|
|
<ResponseMetadata>
|
|
<RequestId>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</RequestId>
|
|
</ResponseMetadata>
|
|
</DescribeClusterResponse>"""
|
|
|
|
ADD_INSTANCE_GROUPS_TEMPLATE = """<AddInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<InstanceGroupIds>{% for instance_group in instance_groups %}{{ instance_group.id }}{% if loop.index != loop.length %},{% endif %}{% endfor %}</InstanceGroupIds>
|
|
</AddInstanceGroupsResponse>"""
|
|
|
|
MODIFY_INSTANCE_GROUPS_TEMPLATE = """<ModifyInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>
|
|
2690d7eb-ed86-11dd-9877-6fad448a8419
|
|
</RequestId>
|
|
</ResponseMetadata>
|
|
</ModifyInstanceGroupsResponse>"""
|
|
|
|
SET_VISIBLE_TO_ALL_USERS_TEMPLATE = """<SetVisibleToAllUsersResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>
|
|
2690d7eb-ed86-11dd-9877-6fad448a8419
|
|
</RequestId>
|
|
</ResponseMetadata>
|
|
</SetVisibleToAllUsersResponse>"""
|
|
|
|
|
|
SET_TERMINATION_PROTECTION_TEMPLATE = """<SetTerminationProtection xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>
|
|
2690d7eb-ed86-11dd-9877-6fad448a8419
|
|
</RequestId>
|
|
</ResponseMetadata>
|
|
</SetTerminationProtection>"""
|
|
|
|
ADD_TAGS_TEMPLATE = """<AddTagsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>
|
|
2690d7eb-ed86-11dd-9877-6fad448a8419
|
|
</RequestId>
|
|
</ResponseMetadata>
|
|
</AddTagsResponse>"""
|
|
|
|
REMOVE_TAGS_TEMPLATE = """<RemoveTagsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
|
<ResponseMetadata>
|
|
<RequestId>
|
|
2690d7eb-ed86-11dd-9877-6fad448a8419
|
|
</RequestId>
|
|
</ResponseMetadata>
|
|
</RemoveTagsResponse>"""
|