EMR and SWF - add arn to response (#3873)

* emr: add ClusterArn to describe_cluster response

* emr: add ClusterArn to list_clusters response

* emr: add ClusterArn to put_auto_scaling_policy response

* emr: add ClusterArn to run_job_flow response

* emr: rename property "cluster_arn" to simply "arn"

* emr: generalize arn for account_id and region

* swf: add arn to list_domains response

* black reformat source code

* fix double import

* swf: require region on Domain object

Co-authored-by: Kevin Neal <Kevin_Neal@intuit.com>

moto/emr/models.py

@@ -7,7 +7,7 @@ import warnings
 import pytz
 from boto3 import Session
 from dateutil.parser import parse as dtparse
-from moto.core import BaseBackend, BaseModel
+from moto.core import ACCOUNT_ID, BaseBackend, BaseModel
 from moto.emr.exceptions import EmrError, InvalidRequestException
 from .utils import (
     random_instance_group_id,
@@ -272,6 +272,12 @@ class FakeCluster(BaseModel):
         )
         self.kerberos_attributes = kerberos_attributes
 
+    @property
+    def arn(self):
+        return "arn:aws:elasticmapreduce:{0}:{1}:cluster/{2}".format(
+            self.emr_backend.region_name, ACCOUNT_ID, self.id
+        )
+
     @property
     def instance_groups(self):
         return self.emr_backend.get_instance_groups(self.instance_group_ids)

moto/emr/responses.py

@@ -529,13 +529,16 @@ class ElasticMapReduceResponse(BaseResponse):
     @generate_boto3_response("PutAutoScalingPolicy")
     def put_auto_scaling_policy(self):
         cluster_id = self._get_param("ClusterId")
+        cluster = self.backend.get_cluster(cluster_id)
         instance_group_id = self._get_param("InstanceGroupId")
         auto_scaling_policy = self._get_param("AutoScalingPolicy")
         instance_group = self.backend.put_auto_scaling_policy(
             instance_group_id, auto_scaling_policy
         )
         template = self.response_template(PUT_AUTO_SCALING_POLICY)
-        return template.render(cluster_id=cluster_id, instance_group=instance_group)
+        return template.render(
+            cluster_id=cluster_id, cluster=cluster, instance_group=instance_group
+        )
 
     @generate_boto3_response("RemoveAutoScalingPolicy")
     def remove_auto_scaling_policy(self):
@@ -691,6 +694,7 @@ DESCRIBE_CLUSTER_TEMPLATE = """<DescribeClusterResponse xmlns="http://elasticmap
       <TerminationProtected>{{ cluster.termination_protected|lower }}</TerminationProtected>
       <VisibleToAllUsers>{{ cluster.visible_to_all_users|lower }}</VisibleToAllUsers>
       <StepConcurrencyLevel>{{ cluster.step_concurrency_level }}</StepConcurrencyLevel>
+      <ClusterArn>{{ cluster.arn }}</ClusterArn>
     </Cluster>
   </DescribeClusterResult>
   <ResponseMetadata>
@@ -940,6 +944,7 @@ LIST_CLUSTERS_TEMPLATE = """<ListClustersResponse xmlns="http://elasticmapreduce
             {% endif %}
           </Timeline>
         </Status>
+        <ClusterArn>{{ cluster.arn }}</ClusterArn>
       </member>
       {% endfor %}
     </Clusters>
@@ -1249,6 +1254,7 @@ REMOVE_TAGS_TEMPLATE = """<RemoveTagsResponse xmlns="http://elasticmapreduce.ama
 RUN_JOB_FLOW_TEMPLATE = """<RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
   <RunJobFlowResult>
     <JobFlowId>{{ cluster.id }}</JobFlowId>
+    <ClusterArn>{{ cluster.arn }}</ClusterArn>
   </RunJobFlowResult>
   <ResponseMetadata>
     <RequestId>8296d8b8-ed85-11dd-9877-6fad448a8419</RequestId>
@@ -1378,6 +1384,7 @@ PUT_AUTO_SCALING_POLICY = """<PutAutoScalingPolicyResponse xmlns="http://elastic
         {% endif %}
     </AutoScalingPolicy>
     {% endif %}
+    <ClusterArn>{{ cluster.arn }}</ClusterArn>
   </PutAutoScalingPolicyResult>
   <ResponseMetadata>
     <RequestId>d47379d9-b505-49af-9335-a68950d82535</RequestId>

moto/swf/models/__init__.py

@@ -112,7 +112,12 @@ class SWFBackend(BaseBackend):
     ):
         if self._get_domain(name, ignore_empty=True):
             raise SWFDomainAlreadyExistsFault(name)
-        domain = Domain(name, workflow_execution_retention_period_in_days, description)
+        domain = Domain(
+            name,
+            workflow_execution_retention_period_in_days,
+            self.region_name,
+            description,
+        )
         self.domains.append(domain)
 
     def deprecate_domain(self, name):

moto/swf/models/domain.py

@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 from collections import defaultdict
 
-from moto.core import BaseModel
+from moto.core import ACCOUNT_ID, BaseModel
 from ..exceptions import (
     SWFUnknownResourceFault,
     SWFWorkflowExecutionAlreadyStartedFault,
@@ -9,9 +9,10 @@ from ..exceptions import (
 
 
 class Domain(BaseModel):
-    def __init__(self, name, retention, description=None):
+    def __init__(self, name, retention, region_name, description=None):
         self.name = name
         self.retention = retention
+        self.region_name = region_name
         self.description = description
         self.status = "REGISTERED"
         self.types = {"activity": defaultdict(dict), "workflow": defaultdict(dict)}
@@ -31,6 +32,9 @@ class Domain(BaseModel):
         hsh = {"name": self.name, "status": self.status}
         if self.description:
             hsh["description"] = self.description
+        hsh["arn"] = "arn:aws:swf:{0}:{1}:/domain/{2}".format(
+            self.region_name, ACCOUNT_ID, self.name
+        )
         return hsh
 
     def to_full_dict(self):

tests/test_emr/test_emr_boto3.py

@@ -13,6 +13,7 @@ from botocore.exceptions import ClientError
 import pytest
 
 from moto import mock_emr
+from moto.core import ACCOUNT_ID
 
 
 run_job_flow_args = dict(
@@ -76,7 +77,8 @@ input_instance_groups = [
 
 @mock_emr
 def test_describe_cluster():
-    client = boto3.client("emr", region_name="us-east-1")
+    region_name = "us-east-1"
+    client = boto3.client("emr", region_name=region_name)
 
     args = deepcopy(run_job_flow_args)
     args["Applications"] = [{"Name": "Spark", "Version": "2.4.2"}]
@@ -178,6 +180,11 @@ def test_describe_cluster():
 
     cl["TerminationProtected"].should.equal(False)
     cl["VisibleToAllUsers"].should.equal(True)
+    cl["ClusterArn"].should.equal(
+        "arn:aws:elasticmapreduce:{0}:{1}:cluster/{2}".format(
+            region_name, ACCOUNT_ID, cluster_id
+        )
+    )
 
 
 @mock_emr
@@ -384,12 +391,17 @@ def test_list_clusters():
 
 @mock_emr
 def test_run_job_flow():
-    client = boto3.client("emr", region_name="us-east-1")
+    region_name = "us-east-1"
+    client = boto3.client("emr", region_name=region_name)
     args = deepcopy(run_job_flow_args)
-    cluster_id = client.run_job_flow(**args)["JobFlowId"]
+    resp = client.run_job_flow(**args)
-    resp = client.describe_job_flows(JobFlowIds=[cluster_id])["JobFlows"][0]
+    resp["ClusterArn"].startswith(
+        "arn:aws:elasticmapreduce:{0}:{1}:cluster/".format(region_name, ACCOUNT_ID)
+    )
+    job_flow_id = resp["JobFlowId"]
+    resp = client.describe_job_flows(JobFlowIds=[job_flow_id])["JobFlows"][0]
     resp["ExecutionStatusDetail"]["State"].should.equal("WAITING")
-    resp["JobFlowId"].should.equal(cluster_id)
+    resp["JobFlowId"].should.equal(job_flow_id)
     resp["Name"].should.equal(args["Name"])
     resp["Instances"]["MasterInstanceType"].should.equal(
         args["Instances"]["MasterInstanceType"]
@@ -544,8 +556,9 @@ def test_run_job_flow_with_instance_groups_with_autoscaling():
 
 @mock_emr
 def test_put_remove_auto_scaling_policy():
+    region_name = "us-east-1"
     input_groups = dict((g["Name"], g) for g in input_instance_groups)
-    client = boto3.client("emr", region_name="us-east-1")
+    client = boto3.client("emr", region_name=region_name)
     args = deepcopy(run_job_flow_args)
     args["Instances"] = {"InstanceGroups": input_instance_groups}
     cluster_id = client.run_job_flow(**args)["JobFlowId"]
@@ -567,6 +580,11 @@ def test_put_remove_auto_scaling_policy():
     )
     del resp["AutoScalingPolicy"]["Status"]
     resp["AutoScalingPolicy"].should.equal(auto_scaling_policy_with_cluster_id)
+    resp["ClusterArn"].should.equal(
+        "arn:aws:elasticmapreduce:{0}:{1}:cluster/{2}".format(
+            region_name, ACCOUNT_ID, cluster_id
+        )
+    )
 
     core_instance_group = [
         ig

tests/test_swf/models/test_domain.py

@@ -1,9 +1,11 @@
 from collections import namedtuple
 import sure  # noqa
 
+from moto.core import ACCOUNT_ID
 from moto.swf.exceptions import SWFUnknownResourceFault
 from moto.swf.models import Domain
 
+TEST_REGION = "us-east-1"
 # Fake WorkflowExecution for tests purposes
 WorkflowExecution = namedtuple(
     "WorkflowExecution", ["workflow_id", "run_id", "execution_status", "open"]
@@ -11,15 +13,21 @@ WorkflowExecution = namedtuple(
 
 
 def test_domain_short_dict_representation():
-    domain = Domain("foo", "52")
+    domain = Domain("foo", "52", TEST_REGION)
-    domain.to_short_dict().should.equal({"name": "foo", "status": "REGISTERED"})
+    domain.to_short_dict().should.equal(
+        {
+            "name": "foo",
+            "status": "REGISTERED",
+            "arn": "arn:aws:swf:{0}:{1}:/domain/foo".format(TEST_REGION, ACCOUNT_ID),
+        }
+    )
 
     domain.description = "foo bar"
     domain.to_short_dict()["description"].should.equal("foo bar")
 
 
 def test_domain_full_dict_representation():
-    domain = Domain("foo", "52")
+    domain = Domain("foo", "52", TEST_REGION)
 
     domain.to_full_dict()["domainInfo"].should.equal(domain.to_short_dict())
     _config = domain.to_full_dict()["configuration"]
@@ -27,38 +35,38 @@ def test_domain_full_dict_representation():
 
 
 def test_domain_string_representation():
-    domain = Domain("my-domain", "60")
+    domain = Domain("my-domain", "60", TEST_REGION)
     str(domain).should.equal("Domain(name: my-domain, status: REGISTERED)")
 
 
 def test_domain_add_to_activity_task_list():
-    domain = Domain("my-domain", "60")
+    domain = Domain("my-domain", "60", TEST_REGION)
     domain.add_to_activity_task_list("foo", "bar")
     domain.activity_task_lists.should.equal({"foo": ["bar"]})
 
 
 def test_domain_activity_tasks():
-    domain = Domain("my-domain", "60")
+    domain = Domain("my-domain", "60", TEST_REGION)
     domain.add_to_activity_task_list("foo", "bar")
     domain.add_to_activity_task_list("other", "baz")
     sorted(domain.activity_tasks).should.equal(["bar", "baz"])
 
 
 def test_domain_add_to_decision_task_list():
-    domain = Domain("my-domain", "60")
+    domain = Domain("my-domain", "60", TEST_REGION)
     domain.add_to_decision_task_list("foo", "bar")
     domain.decision_task_lists.should.equal({"foo": ["bar"]})
 
 
 def test_domain_decision_tasks():
-    domain = Domain("my-domain", "60")
+    domain = Domain("my-domain", "60", TEST_REGION)
     domain.add_to_decision_task_list("foo", "bar")
     domain.add_to_decision_task_list("other", "baz")
     sorted(domain.decision_tasks).should.equal(["bar", "baz"])
 
 
 def test_domain_get_workflow_execution():
-    domain = Domain("my-domain", "60")
+    domain = Domain("my-domain", "60", TEST_REGION)
 
     wfe1 = WorkflowExecution(
         workflow_id="wf-id-1", run_id="run-id-1", execution_status="OPEN", open=True

tests/test_swf/responses/test_domains.py

@@ -6,6 +6,7 @@ import sure  # noqa
 
 from moto import mock_swf_deprecated
 from moto import mock_swf
+from moto.core import ACCOUNT_ID
 
 
 # RegisterDomain endpoint
@@ -20,6 +21,9 @@ def test_register_domain():
     domain["name"].should.equal("test-domain")
     domain["status"].should.equal("REGISTERED")
     domain["description"].should.equal("A test domain")
+    domain["arn"].should.equal(
+        "arn:aws:swf:us-east-1:{0}:/domain/test-domain".format(ACCOUNT_ID)
+    )
 
 
 @mock_swf_deprecated

tests/test_swf/utils.py

@@ -31,7 +31,7 @@ for key, value in ACTIVITY_TASK_TIMEOUTS.items():
 
 # A test Domain
 def get_basic_domain():
-    return Domain("test-domain", "90")
+    return Domain("test-domain", "90", "us-east-1")
 
 
 # A test WorkflowType