Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

EMR: fix handling of Properties for job flows (#6513)

Browse Source
This commit is contained in:
steffyP 2023-07-12 19:30:58 +02:00 committed by GitHub
parent 6668310884
commit 33c665fc21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 30 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
moto/emr/responses.py
View File

@ -820,7 +820,14 @@ DESCRIBE_JOB_FLOWS_TEMPLATE = """<DescribeJobFlowsResponse xmlns="http://elastic
<member>{{ arg | escape }}</member> <member>{{ arg | escape }}</member>
{% endfor %} {% endfor %}
</Args> </Args>
<Properties/> <Properties>
{% for key, val in step.properties.items() %}
<member>
<Key>{{ key }}</Key>
<Value>{{ val | escape }}</Value>
</member>
{% endfor %}
</Properties>
</HadoopJarStep> </HadoopJarStep>
<Name>{{ step.name | escape }}</Name> <Name>{{ step.name | escape }}</Name>
</StepConfig> </StepConfig>
@ -852,10 +859,10 @@ DESCRIBE_STEP_TEMPLATE = """<DescribeStepResponse xmlns="http://elasticmapreduce
<MainClass/> <MainClass/>
<Properties> <Properties>
{% for key, val in step.properties.items() %} {% for key, val in step.properties.items() %}
<member> <entry>
<key>{{ key }}</key> <key>{{ key }}</key>
<value>{{ val | escape }}</value> <value>{{ val | escape }}</value>
</member> </entry>
{% endfor %} {% endfor %}
</Properties> </Properties>
</Config> </Config>
@ -1180,10 +1187,10 @@ LIST_STEPS_TEMPLATE = """<ListStepsResponse xmlns="http://elasticmapreduce.amazo
<MainClass/> <MainClass/>
<Properties> <Properties>
{% for key, val in step.properties.items() %} {% for key, val in step.properties.items() %}
<member> <entry>
<key>{{ key }}</key> <key>{{ key }}</key>
<value>{{ val | escape }}</value> <value>{{ val | escape }}</value>
</member> </entry>
{% endfor %} {% endfor %}
</Properties> </Properties>
</Config> </Config>

14
moto/emr/utils.py
View File

@ -33,15 +33,23 @@ def steps_from_query_string(
steps = [] steps = []
for step in querystring_dict: for step in querystring_dict:
step["jar"] = step.pop("hadoop_jar_step._jar") step["jar"] = step.pop("hadoop_jar_step._jar")
step["properties"] = dict(
(o["Key"], o["Value"]) for o in step.get("properties", [])
)
step["args"] = [] step["args"] = []
idx = 1 idx = 1
keyfmt = "hadoop_jar_step._args.member.{0}" keyfmt = "hadoop_jar_step._args.member.{0}"
while keyfmt.format(idx) in step: while keyfmt.format(idx) in step:
step["args"].append(step.pop(keyfmt.format(idx))) step["args"].append(step.pop(keyfmt.format(idx)))
idx += 1 idx += 1
idx = 1
keyfmt_prop = "hadoop_jar_step._properties.member.{0}._key"
properties = {}
while keyfmt_prop.format(idx) in step:
key = keyfmt_prop.format(idx)
value = key.replace("_key", "_value")
properties[step.pop(key)] = step.pop(value)
idx += 1
step["properties"] = properties
steps.append(step) steps.append(step)
return steps return steps

7
tests/test_emr/test_emr_boto3.py
View File

@ -944,6 +944,9 @@ def test_steps():
"aggregate", "aggregate",
], ],
"Jar": "command-runner.jar", "Jar": "command-runner.jar",
"Properties": [
{"Key": "mapred.tasktracker.map.tasks.maximum", "Value": "2"}
],
}, },
"Name": "My wordcount example", "Name": "My wordcount example",
}, },
@ -963,6 +966,10 @@ def test_steps():
"aggregate", "aggregate",
], ],
"Jar": "command-runner.jar", "Jar": "command-runner.jar",
"Properties": [
{"Key": "mapred.reduce.tasks", "Value": "0"},
{"Key": "stream.map.output.field.separator", "Value": "."},
],
}, },
"Name": "My wordcount example2", "Name": "My wordcount example2",
}, },