Added support for EMR Security Configurations and Kerberos Attributes. (#3456)

* Added support for EMR Security Configurations and Kerberos Attributes.

* Revised exception-raising test to work with pytest api.

* Added htmlcov to .gitignore; upgrading botocore to 1.18.17, per commit d29475e.

Co-authored-by: Joseph Weitekamp <jweite@amazon.com>

.gitignore

@@ -23,3 +23,4 @@ tests/file.tmp
 .mypy_cache/
 *.tmp
 .venv/
+htmlcov/

moto/emr/exceptions.py

@@ -1,7 +1,14 @@
 from __future__ import unicode_literals
 
-from moto.core.exceptions import RESTError
+from moto.core.exceptions import RESTError, JsonRESTError
 
 
 class EmrError(RESTError):
     code = 400
+
+
+class InvalidRequestException(JsonRESTError):
+    def __init__(self, message, **kwargs):
+        super(InvalidRequestException, self).__init__(
+            "InvalidRequestException", message, **kwargs
+        )

moto/emr/models.py

@@ -6,7 +6,7 @@ import pytz
 from boto3 import Session
 from dateutil.parser import parse as dtparse
 from moto.core import BaseBackend, BaseModel
-from moto.emr.exceptions import EmrError
+from moto.emr.exceptions import EmrError, InvalidRequestException
 from .utils import (
     random_instance_group_id,
     random_cluster_id,
@@ -147,6 +147,8 @@ class FakeCluster(BaseModel):
         running_ami_version=None,
         custom_ami_id=None,
         step_concurrency_level=1,
+        security_configuration=None,
+        kerberos_attributes=None,
     ):
         self.id = cluster_id or random_cluster_id()
         emr_backend.clusters[self.id] = self
@@ -249,6 +251,10 @@ class FakeCluster(BaseModel):
         self.run_bootstrap_actions()
         if self.steps:
             self.steps[0].start()
+        self.security_configuration = (
+            security_configuration  # ToDo: Raise if doesn't already exist.
+        )
+        self.kerberos_attributes = kerberos_attributes
 
     @property
     def instance_groups(self):
@@ -337,12 +343,20 @@ class FakeCluster(BaseModel):
         self.visible_to_all_users = visibility
 
 
+class FakeSecurityConfiguration(BaseModel):
+    def __init__(self, name, security_configuration):
+        self.name = name
+        self.security_configuration = security_configuration
+        self.creation_date_time = datetime.now(pytz.utc)
+
+
 class ElasticMapReduceBackend(BaseBackend):
     def __init__(self, region_name):
         super(ElasticMapReduceBackend, self).__init__()
         self.region_name = region_name
         self.clusters = {}
         self.instance_groups = {}
+        self.security_configurations = {}
 
     def reset(self):
         region_name = self.region_name
@@ -527,6 +541,37 @@ class ElasticMapReduceBackend(BaseBackend):
         instance_group = instance_groups[0]
         instance_group.auto_scaling_policy = None
 
+    def create_security_configuration(self, name, security_configuration):
+        if name in self.security_configurations:
+            raise InvalidRequestException(
+                message="SecurityConfiguration with name '{}' already exists.".format(
+                    name
+                )
+            )
+        security_configuration = FakeSecurityConfiguration(
+            name=name, security_configuration=security_configuration
+        )
+        self.security_configurations[name] = security_configuration
+        return security_configuration
+
+    def get_security_configuration(self, name):
+        if name not in self.security_configurations:
+            raise InvalidRequestException(
+                message="Security configuration with name '{}' does not exist.".format(
+                    name
+                )
+            )
+        return self.security_configurations[name]
+
+    def delete_security_configuration(self, name):
+        if name not in self.security_configurations:
+            raise InvalidRequestException(
+                message="Security configuration with name '{}' does not exist.".format(
+                    name
+                )
+            )
+        del self.security_configurations[name]
+
 
 emr_backends = {}
 for region in Session().get_available_regions("emr"):

moto/emr/responses.py

@@ -102,11 +102,29 @@ class ElasticMapReduceResponse(BaseResponse):
     def cancel_steps(self):
         raise NotImplementedError
 
+    @generate_boto3_response("CreateSecurityConfiguration")
     def create_security_configuration(self):
-        raise NotImplementedError
+        name = self._get_param("Name")
+        security_configuration = self._get_param("SecurityConfiguration")
+        resp = self.backend.create_security_configuration(
+            name=name, security_configuration=security_configuration
+        )
+        template = self.response_template(CREATE_SECURITY_CONFIGURATION_TEMPLATE)
+        return template.render(name=name, creation_date_time=resp.creation_date_time)
 
+    @generate_boto3_response("DescribeSecurityConfiguration")
+    def describe_security_configuration(self):
+        name = self._get_param("Name")
+        security_configuration = self.backend.get_security_configuration(name=name)
+        template = self.response_template(DESCRIBE_SECURITY_CONFIGURATION_TEMPLATE)
+        return template.render(security_configuration=security_configuration)
+
+    @generate_boto3_response("DeleteSecurityConfiguration")
     def delete_security_configuration(self):
-        raise NotImplementedError
+        name = self._get_param("Name")
+        self.backend.delete_security_configuration(name=name)
+        template = self.response_template(DELETE_SECURITY_CONFIGURATION_TEMPLATE)
+        return template.render()
 
     @generate_boto3_response("DescribeCluster")
     def describe_cluster(self):
@@ -190,9 +208,6 @@ class ElasticMapReduceResponse(BaseResponse):
         template = self.response_template(MODIFY_CLUSTER_TEMPLATE)
         return template.render(cluster=cluster)
 
-    def describe_security_configuration(self):
-        raise NotImplementedError
-
     @generate_boto3_response("ModifyInstanceGroups")
     def modify_instance_groups(self):
         instance_groups = self._get_list_prefix("InstanceGroups.member")
@@ -327,6 +342,39 @@ class ElasticMapReduceResponse(BaseResponse):
         if step_concurrency_level:
             kwargs["step_concurrency_level"] = step_concurrency_level
 
+        security_configuration = self._get_param("SecurityConfiguration")
+        if security_configuration:
+            kwargs["security_configuration"] = security_configuration
+
+        kerberos_attributes = {}
+        kwargs["kerberos_attributes"] = kerberos_attributes
+
+        realm = self._get_param("KerberosAttributes.Realm")
+        if realm:
+            kerberos_attributes["Realm"] = realm
+
+        kdc_admin_password = self._get_param("KerberosAttributes.KdcAdminPassword")
+        if kdc_admin_password:
+            kerberos_attributes["KdcAdminPassword"] = kdc_admin_password
+
+        cross_realm_principal_password = self._get_param(
+            "KerberosAttributes.CrossRealmTrustPrincipalPassword"
+        )
+        if cross_realm_principal_password:
+            kerberos_attributes[
+                "CrossRealmTrustPrincipalPassword"
+            ] = cross_realm_principal_password
+
+        ad_domain_join_user = self._get_param("KerberosAttributes.ADDomainJoinUser")
+        if ad_domain_join_user:
+            kerberos_attributes["ADDomainJoinUser"] = ad_domain_join_user
+
+        ad_domain_join_password = self._get_param(
+            "KerberosAttributes.ADDomainJoinPassword"
+        )
+        if ad_domain_join_password:
+            kerberos_attributes["ADDomainJoinPassword"] = ad_domain_join_password
+
         cluster = self.backend.run_job_flow(**kwargs)
 
         applications = self._get_list_prefix("Applications.member")
@@ -560,6 +608,23 @@ DESCRIBE_CLUSTER_TEMPLATE = """<DescribeClusterResponse xmlns="http://elasticmap
         <ServiceAccessSecurityGroup>{{ cluster.service_access_security_group }}</ServiceAccessSecurityGroup>
       </Ec2InstanceAttributes>
       <Id>{{ cluster.id }}</Id>
+      <KerberosAttributes>
+        {% if 'Realm' in cluster.kerberos_attributes%}
+        <Realm>{{ cluster.kerberos_attributes['Realm'] }}</Realm>
+        {% endif %}
+        {% if 'KdcAdminPassword' in cluster.kerberos_attributes%}
+        <KdcAdminPassword>{{ cluster.kerberos_attributes['KdcAdminPassword'] }}</KdcAdminPassword>
+        {% endif %}
+        {% if 'CrossRealmTrustPrincipalPassword' in cluster.kerberos_attributes%}
+        <CrossRealmTrustPrincipalPassword>{{ cluster.kerberos_attributes['CrossRealmTrustPrincipalPassword'] }}</CrossRealmTrustPrincipalPassword>
+        {% endif %}
+        {% if 'ADDomainJoinUser' in cluster.kerberos_attributes%}
+        <ADDomainJoinUser>{{ cluster.kerberos_attributes['ADDomainJoinUser'] }}</ADDomainJoinUser>
+        {% endif %}
+        {% if 'ADDomainJoinPassword' in cluster.kerberos_attributes%}
+        <ADDomainJoinPassword>{{ cluster.kerberos_attributes['ADDomainJoinPassword'] }}</ADDomainJoinPassword>
+        {% endif %}
+      </KerberosAttributes>
       <LogUri>{{ cluster.log_uri }}</LogUri>
       <MasterPublicDnsName>ec2-184-0-0-1.us-west-1.compute.amazonaws.com</MasterPublicDnsName>
       <Name>{{ cluster.name }}</Name>
@@ -573,7 +638,9 @@ DESCRIBE_CLUSTER_TEMPLATE = """<DescribeClusterResponse xmlns="http://elasticmap
       {% if cluster.running_ami_version is not none %}
       <RunningAmiVersion>{{ cluster.running_ami_version }}</RunningAmiVersion>
       {% endif %}
-      <SecurityConfiguration/>
+      {% if cluster.security_configuration is not none %}
+      <SecurityConfiguration>{{ cluster.security_configuration }}</SecurityConfiguration>
+      {% endif %}
       <ServiceRole>{{ cluster.service_role }}</ServiceRole>
       <Status>
         <State>{{ cluster.state }}</State>
@@ -1253,3 +1320,30 @@ REMOVE_AUTO_SCALING_POLICY = """<RemoveAutoScalingPolicyResponse xmlns="http://e
     <RequestId>c04a1042-5340-4c0a-a7b5-7779725ce4f7</RequestId>
   </ResponseMetadata>
 </RemoveAutoScalingPolicyResponse>"""
+
+CREATE_SECURITY_CONFIGURATION_TEMPLATE = """<CreateSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+  <CreateSecurityConfigurationResult>
+    <Name>{{name}}</Name>
+    <CreationDateTime>{{creation_date_time}}</CreationDateTime>
+  </CreateSecurityConfigurationResult>
+  <ResponseMetadata>
+    <RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
+  </ResponseMetadata>
+</CreateSecurityConfigurationResponse>"""
+
+DESCRIBE_SECURITY_CONFIGURATION_TEMPLATE = """<DescribeSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+  <DescribeSecurityConfigurationResult>
+    <Name>{{security_configuration['name']}}</Name>
+    <SecurityConfiguration>{{security_configuration['security_configuration']}}</SecurityConfiguration>
+    <CreationDateTime>{{security_configuration['creation_date_time']}}</CreationDateTime>
+  </DescribeSecurityConfigurationResult>
+  <ResponseMetadata>
+    <RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
+  </ResponseMetadata>
+</DescribeSecurityConfigurationResponse>"""
+
+DELETE_SECURITY_CONFIGURATION_TEMPLATE = """<DeleteSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+  <ResponseMetadata>
+    <RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
+  </ResponseMetadata>
+</DeleteSecurityConfigurationResponse>"""

requirements-dev.txt

@@ -9,7 +9,7 @@ flask
 flask-cors
 boto>=2.45.0
 boto3>=1.4.4
-botocore>=1.15.13
+botocore>=1.18.17
 six>=1.9
 prompt-toolkit==2.0.10 # 3.x is not available with python2
 click==6.7

tests/test_emr/test_emr_boto3.py

@@ -107,7 +107,15 @@ def test_describe_cluster():
     args["Instances"]["EmrManagedSlaveSecurityGroup"] = "slave-security-group"
     args["Instances"]["KeepJobFlowAliveWhenNoSteps"] = False
     args["Instances"]["ServiceAccessSecurityGroup"] = "service-access-security-group"
+    args["KerberosAttributes"] = {
+        "Realm": "MY-REALM.COM",
+        "KdcAdminPassword": "SuperSecretPassword2",
+        "CrossRealmTrustPrincipalPassword": "SuperSecretPassword3",
+        "ADDomainJoinUser": "Bob",
+        "ADDomainJoinPassword": "SuperSecretPassword4",
+    }
     args["Tags"] = [{"Key": "tag1", "Value": "val1"}, {"Key": "tag2", "Value": "val2"}]
+    args["SecurityConfiguration"] = "my-security-configuration"
 
     cluster_id = client.run_job_flow(**args)["JobFlowId"]
 
@@ -145,6 +153,7 @@ def test_describe_cluster():
         args["Instances"]["ServiceAccessSecurityGroup"]
     )
     cl["Id"].should.equal(cluster_id)
+    cl["KerberosAttributes"].should.equal(args["KerberosAttributes"])
     cl["LogUri"].should.equal(args["LogUri"])
     cl["MasterPublicDnsName"].should.be.a(six.string_types)
     cl["Name"].should.equal(args["Name"])
@@ -152,7 +161,8 @@ def test_describe_cluster():
     # cl['ReleaseLabel'].should.equal('emr-5.0.0')
     cl.shouldnt.have.key("RequestedAmiVersion")
     cl["RunningAmiVersion"].should.equal("1.0.0")
-    # cl['SecurityConfiguration'].should.be.a(six.string_types)
+    cl["SecurityConfiguration"].should.be.a(six.string_types)
+    cl["SecurityConfiguration"].should.equal(args["SecurityConfiguration"])
     cl["ServiceRole"].should.equal(args["ServiceRole"])
 
     status = cl["Status"]
@@ -985,3 +995,53 @@ def test_tags():
     client.remove_tags(ResourceId=cluster_id, TagKeys=[t["Key"] for t in input_tags])
     resp = client.describe_cluster(ClusterId=cluster_id)["Cluster"]
     resp["Tags"].should.equal([])
+
+
+@mock_emr
+def test_security_configurations():
+
+    client = boto3.client("emr", region_name="us-east-1")
+
+    security_configuration_name = "MySecurityConfiguration"
+
+    security_configuration = """
+{
+  "EncryptionConfiguration": {
+    "AtRestEncryptionConfiguration": {
+      "S3EncryptionConfiguration": {
+        "EncryptionMode": "SSE-S3"
+      }
+    },
+    "EnableInTransitEncryption": false,
+    "EnableAtRestEncryption": true
+  }
+}
+    """.strip()
+
+    resp = client.create_security_configuration(
+        Name=security_configuration_name, SecurityConfiguration=security_configuration
+    )
+
+    resp["Name"].should.equal(security_configuration_name)
+    resp["CreationDateTime"].should.be.a("datetime.datetime")
+
+    resp = client.describe_security_configuration(Name=security_configuration_name)
+    resp["Name"].should.equal(security_configuration_name)
+    resp["SecurityConfiguration"].should.equal(security_configuration)
+    resp["CreationDateTime"].should.be.a("datetime.datetime")
+
+    client.delete_security_configuration(Name=security_configuration_name)
+
+    with pytest.raises(ClientError) as ex:
+        client.describe_security_configuration(Name=security_configuration_name)
+    ex.value.response["Error"]["Code"].should.equal("InvalidRequestException")
+    ex.value.response["Error"]["Message"].should.match(
+        r"Security configuration with name .* does not exist."
+    )
+
+    with pytest.raises(ClientError) as ex:
+        client.delete_security_configuration(Name=security_configuration_name)
+    ex.value.response["Error"]["Code"].should.equal("InvalidRequestException")
+    ex.value.response["Error"]["Message"].should.match(
+        r"Security configuration with name .* does not exist."
+    )