from jinja2 import Template from moto.core.responses import BaseResponse from moto.core.utils import camelcase_to_underscores from .models import emr_backend class ElasticMapReduceResponse(BaseResponse): def _get_param(self, param_name): return self.querystring.get(param_name, [None])[0] def _get_multi_param(self, param_prefix): return [value[0] for key, value in self.querystring.items() if key.startswith(param_prefix)] def _get_dict_param(self, param_prefix): params = {} for key, value in self.querystring.items(): if key.startswith(param_prefix): params[camelcase_to_underscores(key.replace(param_prefix, ""))] = value[0] return params def _get_list_prefix(self, param_prefix): results = [] param_index = 1 while True: index_prefix = "{0}.{1}.".format(param_prefix, param_index) new_items = {} for key, value in self.querystring.items(): if key.startswith(index_prefix): new_items[camelcase_to_underscores(key.replace(index_prefix, ""))] = value[0] if not new_items: break results.append(new_items) param_index += 1 return results def add_job_flow_steps(self): job_flow_id = self._get_param('JobFlowId') steps = self._get_list_prefix('Steps.member') job_flow = emr_backend.add_job_flow_steps(job_flow_id, steps) template = Template(ADD_JOB_FLOW_STEPS_TEMPLATE) return template.render(job_flow=job_flow) def run_job_flow(self): flow_name = self._get_param('Name') log_uri = self._get_param('LogUri') steps = self._get_list_prefix('Steps.member') instance_attrs = self._get_dict_param('Instances.') job_flow_role = self._get_param('JobFlowRole') visible_to_all_users = self._get_param('VisibleToAllUsers') job_flow = emr_backend.run_job_flow( flow_name, log_uri, job_flow_role, visible_to_all_users, steps, instance_attrs ) template = Template(RUN_JOB_FLOW_TEMPLATE) return template.render(job_flow=job_flow) def describe_job_flows(self): job_flows = emr_backend.describe_job_flows() template = Template(DESCRIBE_JOB_FLOWS_TEMPLATE) return template.render(job_flows=job_flows) def terminate_job_flows(self): job_ids = self._get_multi_param('JobFlowIds.member.') job_flows = emr_backend.terminate_job_flows(job_ids) template = Template(TERMINATE_JOB_FLOWS_TEMPLATE) return template.render(job_flows=job_flows) def add_instance_groups(self): jobflow_id = self._get_param('JobFlowId') instance_groups = self._get_list_prefix('InstanceGroups.member') instance_groups = emr_backend.add_instance_groups(jobflow_id, instance_groups) template = Template(ADD_INSTANCE_GROUPS_TEMPLATE) return template.render(instance_groups=instance_groups) def modify_instance_groups(self): instance_groups = self._get_list_prefix('InstanceGroups.member') instance_groups = emr_backend.modify_instance_groups(instance_groups) template = Template(MODIFY_INSTANCE_GROUPS_TEMPLATE) return template.render(instance_groups=instance_groups) def set_visible_to_all_users(self): visible_to_all_users = self._get_param('VisibleToAllUsers') job_ids = self._get_multi_param('JobFlowIds.member') emr_backend.set_visible_to_all_users(job_ids, visible_to_all_users) template = Template(SET_VISIBLE_TO_ALL_USERS_TEMPLATE) return template.render() RUN_JOB_FLOW_TEMPLATE = """ {{ job_flow.id }} 8296d8b8-ed85-11dd-9877-6fad448a8419 """ DESCRIBE_JOB_FLOWS_TEMPLATE = """ {% for job_flow in job_flows %} 2009-01-28T21:49:16Z 2009-01-28T21:49:16Z {{ job_flow.state }} {{ job_flow.name }} {{ job_flow.role }} {{ job_flow.log_uri }} {% for step in job_flow.steps %} 2009-01-28T21:49:16Z {{ step.state }} {{ step.jar }} MyMainClass {% for arg in step.args %} {{ arg }} {% endfor %} {{ step.name }} CONTINUE {% endfor %} {{ job_flow.id }} us-east-1a {{ job_flow.slave_instance_type }} {{ job_flow.master_instance_type }} {{ job_flow.ec2_key_name }} {{ job_flow.normalized_instance_hours }} {{ job_flow.visible_to_all_users }} {{ job_flow.instance_count }} {{ job_flow.keep_job_flow_alive_when_no_steps }} {{ job_flow.termination_protected }} {% for instance_group in job_flow.instance_groups %} {{ instance_group.id }} {{ instance_group.role }} {{ instance_group.num_instances }} {{ instance_group.type }} {{ instance_group.market }} {{ instance_group.name }} {{ instance_group.bid_price }} {% endfor %} {% endfor %} 9cea3229-ed85-11dd-9877-6fad448a8419 """ TERMINATE_JOB_FLOWS_TEMPLATE = """ 2690d7eb-ed86-11dd-9877-6fad448a8419 """ ADD_JOB_FLOW_STEPS_TEMPLATE = """ df6f4f4a-ed85-11dd-9877-6fad448a8419 """ ADD_INSTANCE_GROUPS_TEMPLATE = """ {% for instance_group in instance_groups %}{{ instance_group.id }}{% if loop.index != loop.length %},{% endif %}{% endfor %} """ MODIFY_INSTANCE_GROUPS_TEMPLATE = """ 2690d7eb-ed86-11dd-9877-6fad448a8419 """ SET_VISIBLE_TO_ALL_USERS_TEMPLATE = """ 2690d7eb-ed86-11dd-9877-6fad448a8419 """