Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added support for EMR Security Configurations and Kerberos Attributes. (#3456)

Browse Source 
* Added support for EMR Security Configurations and Kerberos Attributes.

* Revised exception-raising test to work with pytest api.

* Added htmlcov to .gitignore; upgrading botocore to 1.18.17, per commit d29475e.

Co-authored-by: Joseph Weitekamp <jweite@amazon.com>
This commit is contained in:
jweite 2020-11-17 05:54:34 -05:00 committed by GitHub
parent f045af7e0a
commit 5fe921c2bc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 217 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -23,3 +23,4 @@ tests/file.tmp
.mypy_cache/
*.tmp
.venv/
htmlcov/

9
moto/emr/exceptions.py
View File

@ -1,7 +1,14 @@
from __future__ import unicode_literals
from moto.core.exceptions import RESTError
from moto.core.exceptions import RESTError, JsonRESTError
class EmrError(RESTError):
code = 400
class InvalidRequestException(JsonRESTError):
def __init__(self, message, **kwargs):
super(InvalidRequestException, self).__init__(
"InvalidRequestException", message, **kwargs
)

47
moto/emr/models.py
View File

@ -6,7 +6,7 @@ import pytz
from boto3 import Session
from dateutil.parser import parse as dtparse
from moto.core import BaseBackend, BaseModel
from moto.emr.exceptions import EmrError
from moto.emr.exceptions import EmrError, InvalidRequestException
from .utils import (
random_instance_group_id,
random_cluster_id,
@ -147,6 +147,8 @@ class FakeCluster(BaseModel):
running_ami_version=None,
custom_ami_id=None,
step_concurrency_level=1,
security_configuration=None,
kerberos_attributes=None,
):
self.id = cluster_id or random_cluster_id()
emr_backend.clusters[self.id] = self
@ -249,6 +251,10 @@ class FakeCluster(BaseModel):
self.run_bootstrap_actions()
if self.steps:
self.steps[0].start()
self.security_configuration = (
security_configuration # ToDo: Raise if doesn't already exist.
)
self.kerberos_attributes = kerberos_attributes
@property
def instance_groups(self):
@ -337,12 +343,20 @@ class FakeCluster(BaseModel):
self.visible_to_all_users = visibility
class FakeSecurityConfiguration(BaseModel):
def __init__(self, name, security_configuration):
self.name = name
self.security_configuration = security_configuration
self.creation_date_time = datetime.now(pytz.utc)
class ElasticMapReduceBackend(BaseBackend):
def __init__(self, region_name):
super(ElasticMapReduceBackend, self).__init__()
self.region_name = region_name
self.clusters = {}
self.instance_groups = {}
self.security_configurations = {}
def reset(self):
region_name = self.region_name
@ -527,6 +541,37 @@ class ElasticMapReduceBackend(BaseBackend):
instance_group = instance_groups[0]
instance_group.auto_scaling_policy = None
def create_security_configuration(self, name, security_configuration):
if name in self.security_configurations:
raise InvalidRequestException(
message="SecurityConfiguration with name '{}' already exists.".format(
name
)
)
security_configuration = FakeSecurityConfiguration(
name=name, security_configuration=security_configuration
)
self.security_configurations[name] = security_configuration
return security_configuration
def get_security_configuration(self, name):
if name not in self.security_configurations:
raise InvalidRequestException(
message="Security configuration with name '{}' does not exist.".format(
name
)
)
return self.security_configurations[name]
def delete_security_configuration(self, name):
if name not in self.security_configurations:
raise InvalidRequestException(
message="Security configuration with name '{}' does not exist.".format(
name
)
)
del self.security_configurations[name]
emr_backends = {}
for region in Session().get_available_regions("emr"):

106
moto/emr/responses.py
View File

@ -102,11 +102,29 @@ class ElasticMapReduceResponse(BaseResponse):
def cancel_steps(self):
raise NotImplementedError
@generate_boto3_response("CreateSecurityConfiguration")
def create_security_configuration(self):
raise NotImplementedError
name = self._get_param("Name")
security_configuration = self._get_param("SecurityConfiguration")
resp = self.backend.create_security_configuration(
name=name, security_configuration=security_configuration
)
template = self.response_template(CREATE_SECURITY_CONFIGURATION_TEMPLATE)
return template.render(name=name, creation_date_time=resp.creation_date_time)
@generate_boto3_response("DescribeSecurityConfiguration")
def describe_security_configuration(self):
name = self._get_param("Name")
security_configuration = self.backend.get_security_configuration(name=name)
template = self.response_template(DESCRIBE_SECURITY_CONFIGURATION_TEMPLATE)
return template.render(security_configuration=security_configuration)
@generate_boto3_response("DeleteSecurityConfiguration")
def delete_security_configuration(self):
raise NotImplementedError
name = self._get_param("Name")
self.backend.delete_security_configuration(name=name)
template = self.response_template(DELETE_SECURITY_CONFIGURATION_TEMPLATE)
return template.render()
@generate_boto3_response("DescribeCluster")
def describe_cluster(self):
@ -190,9 +208,6 @@ class ElasticMapReduceResponse(BaseResponse):
template = self.response_template(MODIFY_CLUSTER_TEMPLATE)
return template.render(cluster=cluster)
def describe_security_configuration(self):
raise NotImplementedError
@generate_boto3_response("ModifyInstanceGroups")
def modify_instance_groups(self):
instance_groups = self._get_list_prefix("InstanceGroups.member")
@ -327,6 +342,39 @@ class ElasticMapReduceResponse(BaseResponse):
if step_concurrency_level:
kwargs["step_concurrency_level"] = step_concurrency_level
security_configuration = self._get_param("SecurityConfiguration")
if security_configuration:
kwargs["security_configuration"] = security_configuration
kerberos_attributes = {}
kwargs["kerberos_attributes"] = kerberos_attributes
realm = self._get_param("KerberosAttributes.Realm")
if realm:
kerberos_attributes["Realm"] = realm
kdc_admin_password = self._get_param("KerberosAttributes.KdcAdminPassword")
if kdc_admin_password:
kerberos_attributes["KdcAdminPassword"] = kdc_admin_password
cross_realm_principal_password = self._get_param(
"KerberosAttributes.CrossRealmTrustPrincipalPassword"
)
if cross_realm_principal_password:
kerberos_attributes[
"CrossRealmTrustPrincipalPassword"
] = cross_realm_principal_password
ad_domain_join_user = self._get_param("KerberosAttributes.ADDomainJoinUser")
if ad_domain_join_user:
kerberos_attributes["ADDomainJoinUser"] = ad_domain_join_user
ad_domain_join_password = self._get_param(
"KerberosAttributes.ADDomainJoinPassword"
)
if ad_domain_join_password:
kerberos_attributes["ADDomainJoinPassword"] = ad_domain_join_password
cluster = self.backend.run_job_flow(**kwargs)
applications = self._get_list_prefix("Applications.member")
@ -560,6 +608,23 @@ DESCRIBE_CLUSTER_TEMPLATE = """<DescribeClusterResponse xmlns="http://elasticmap
<ServiceAccessSecurityGroup>{{ cluster.service_access_security_group }}</ServiceAccessSecurityGroup>
</Ec2InstanceAttributes>
<Id>{{ cluster.id }}</Id>
<KerberosAttributes>
{% if 'Realm' in cluster.kerberos_attributes%}
<Realm>{{ cluster.kerberos_attributes['Realm'] }}</Realm>
{% endif %}
{% if 'KdcAdminPassword' in cluster.kerberos_attributes%}
<KdcAdminPassword>{{ cluster.kerberos_attributes['KdcAdminPassword'] }}</KdcAdminPassword>
{% endif %}
{% if 'CrossRealmTrustPrincipalPassword' in cluster.kerberos_attributes%}
<CrossRealmTrustPrincipalPassword>{{ cluster.kerberos_attributes['CrossRealmTrustPrincipalPassword'] }}</CrossRealmTrustPrincipalPassword>
{% endif %}
{% if 'ADDomainJoinUser' in cluster.kerberos_attributes%}
<ADDomainJoinUser>{{ cluster.kerberos_attributes['ADDomainJoinUser'] }}</ADDomainJoinUser>
{% endif %}
{% if 'ADDomainJoinPassword' in cluster.kerberos_attributes%}
<ADDomainJoinPassword>{{ cluster.kerberos_attributes['ADDomainJoinPassword'] }}</ADDomainJoinPassword>
{% endif %}
</KerberosAttributes>
<LogUri>{{ cluster.log_uri }}</LogUri>
<MasterPublicDnsName>ec2-184-0-0-1.us-west-1.compute.amazonaws.com</MasterPublicDnsName>
<Name>{{ cluster.name }}</Name>
@ -573,7 +638,9 @@ DESCRIBE_CLUSTER_TEMPLATE = """<DescribeClusterResponse xmlns="http://elasticmap
{% if cluster.running_ami_version is not none %}
<RunningAmiVersion>{{ cluster.running_ami_version }}</RunningAmiVersion>
{% endif %}
<SecurityConfiguration/>
{% if cluster.security_configuration is not none %}
<SecurityConfiguration>{{ cluster.security_configuration }}</SecurityConfiguration>
{% endif %}
<ServiceRole>{{ cluster.service_role }}</ServiceRole>
<Status>
<State>{{ cluster.state }}</State>
@ -1253,3 +1320,30 @@ REMOVE_AUTO_SCALING_POLICY = """<RemoveAutoScalingPolicyResponse xmlns="http://e
<RequestId>c04a1042-5340-4c0a-a7b5-7779725ce4f7</RequestId>
</ResponseMetadata>
</RemoveAutoScalingPolicyResponse>"""
CREATE_SECURITY_CONFIGURATION_TEMPLATE = """<CreateSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<CreateSecurityConfigurationResult>
<Name>{{name}}</Name>
<CreationDateTime>{{creation_date_time}}</CreationDateTime>
</CreateSecurityConfigurationResult>
<ResponseMetadata>
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
</ResponseMetadata>
</CreateSecurityConfigurationResponse>"""
DESCRIBE_SECURITY_CONFIGURATION_TEMPLATE = """<DescribeSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<DescribeSecurityConfigurationResult>
<Name>{{security_configuration['name']}}</Name>
<SecurityConfiguration>{{security_configuration['security_configuration']}}</SecurityConfiguration>
<CreationDateTime>{{security_configuration['creation_date_time']}}</CreationDateTime>
</DescribeSecurityConfigurationResult>
<ResponseMetadata>
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
</ResponseMetadata>
</DescribeSecurityConfigurationResponse>"""
DELETE_SECURITY_CONFIGURATION_TEMPLATE = """<DeleteSecurityConfigurationResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<ResponseMetadata>
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
</ResponseMetadata>
</DeleteSecurityConfigurationResponse>"""

2
requirements-dev.txt
View File

@ -9,7 +9,7 @@ flask
flask-cors
boto>=2.45.0
boto3>=1.4.4
botocore>=1.15.13
botocore>=1.18.17
six>=1.9
prompt-toolkit==2.0.10 # 3.x is not available with python2
click==6.7

62
tests/test_emr/test_emr_boto3.py
View File

@ -107,7 +107,15 @@ def test_describe_cluster():
args["Instances"]["EmrManagedSlaveSecurityGroup"] = "slave-security-group"
args["Instances"]["KeepJobFlowAliveWhenNoSteps"] = False
args["Instances"]["ServiceAccessSecurityGroup"] = "service-access-security-group"
args["KerberosAttributes"] = {
"Realm": "MY-REALM.COM",
"KdcAdminPassword": "SuperSecretPassword2",
"CrossRealmTrustPrincipalPassword": "SuperSecretPassword3",
"ADDomainJoinUser": "Bob",
"ADDomainJoinPassword": "SuperSecretPassword4",
}
args["Tags"] = [{"Key": "tag1", "Value": "val1"}, {"Key": "tag2", "Value": "val2"}]
args["SecurityConfiguration"] = "my-security-configuration"
cluster_id = client.run_job_flow(**args)["JobFlowId"]
@ -145,6 +153,7 @@ def test_describe_cluster():
args["Instances"]["ServiceAccessSecurityGroup"]
)
cl["Id"].should.equal(cluster_id)
cl["KerberosAttributes"].should.equal(args["KerberosAttributes"])
cl["LogUri"].should.equal(args["LogUri"])
cl["MasterPublicDnsName"].should.be.a(six.string_types)
cl["Name"].should.equal(args["Name"])
@ -152,7 +161,8 @@ def test_describe_cluster():
# cl['ReleaseLabel'].should.equal('emr-5.0.0')
cl.shouldnt.have.key("RequestedAmiVersion")
cl["RunningAmiVersion"].should.equal("1.0.0")
# cl['SecurityConfiguration'].should.be.a(six.string_types)
cl["SecurityConfiguration"].should.be.a(six.string_types)
cl["SecurityConfiguration"].should.equal(args["SecurityConfiguration"])
cl["ServiceRole"].should.equal(args["ServiceRole"])
status = cl["Status"]
@ -985,3 +995,53 @@ def test_tags():
client.remove_tags(ResourceId=cluster_id, TagKeys=[t["Key"] for t in input_tags])
resp = client.describe_cluster(ClusterId=cluster_id)["Cluster"]
resp["Tags"].should.equal([])
@mock_emr
def test_security_configurations():
client = boto3.client("emr", region_name="us-east-1")
security_configuration_name = "MySecurityConfiguration"
security_configuration = """
{
"EncryptionConfiguration": {
"AtRestEncryptionConfiguration": {
"S3EncryptionConfiguration": {
"EncryptionMode": "SSE-S3"
}
},
"EnableInTransitEncryption": false,
"EnableAtRestEncryption": true
}
}
""".strip()
resp = client.create_security_configuration(
Name=security_configuration_name, SecurityConfiguration=security_configuration
)
resp["Name"].should.equal(security_configuration_name)
resp["CreationDateTime"].should.be.a("datetime.datetime")
resp = client.describe_security_configuration(Name=security_configuration_name)
resp["Name"].should.equal(security_configuration_name)
resp["SecurityConfiguration"].should.equal(security_configuration)
resp["CreationDateTime"].should.be.a("datetime.datetime")
client.delete_security_configuration(Name=security_configuration_name)
with pytest.raises(ClientError) as ex:
client.describe_security_configuration(Name=security_configuration_name)
ex.value.response["Error"]["Code"].should.equal("InvalidRequestException")
ex.value.response["Error"]["Message"].should.match(
r"Security configuration with name .* does not exist."
)
with pytest.raises(ClientError) as ex:
client.delete_security_configuration(Name=security_configuration_name)
ex.value.response["Error"]["Code"].should.equal("InvalidRequestException")
ex.value.response["Error"]["Message"].should.match(
r"Security configuration with name .* does not exist."
)