621 lines
23 KiB
Python
621 lines
23 KiB
Python
|
import json
|
||
|
|
|
||
|
|
import boto3
|
||
|
|
|
||
|
|
from moto import mock_aws
|
||
|
|
from tests import EXAMPLE_AMI_ID
|
||
|
|
|
||
|
|
template = """{
|
||
|
|
"AWSTemplateFormatVersion": "2010-09-09",
|
||
|
|
"Description": "The AWS CloudFormation template for this Serverless application",
|
||
|
|
"Resources": {
|
||
|
|
"Cluster1": {
|
||
|
|
"Type" : "AWS::EMR::Cluster",
|
||
|
|
"Properties" : {
|
||
|
|
"Instances" : {
|
||
|
|
"CoreInstanceGroup": {
|
||
|
|
"InstanceCount": 3,
|
||
|
|
"InstanceType": "m3g",
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"JobFlowRole" : "EMR_EC2_DefaultRole",
|
||
|
|
"Name" : "my cluster",
|
||
|
|
"ServiceRole" : "EMR_DefaultRole",
|
||
|
|
}
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"Outputs": {
|
||
|
|
"ClusterId": {
|
||
|
|
"Description": "Cluster info",
|
||
|
|
"Value": {
|
||
|
|
"Fn::GetAtt": ["Cluster1", "Id"]
|
||
|
|
},
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}"""
|
||
|
|
|
||
|
|
|
||
|
|
@mock_aws
|
||
|
|
def test_create_simple_cluster__using_cloudformation():
|
||
|
|
region = "us-east-1"
|
||
|
|
cf = boto3.client("cloudformation", region_name=region)
|
||
|
|
emr = boto3.client("emr", region_name=region)
|
||
|
|
cf.create_stack(StackName="teststack", TemplateBody=template)
|
||
|
|
|
||
|
|
# Verify resources
|
||
|
|
res = cf.describe_stack_resources(StackName="teststack")["StackResources"][0]
|
||
|
|
cluster_id = res["PhysicalResourceId"]
|
||
|
|
assert res["LogicalResourceId"] == "Cluster1"
|
||
|
|
assert res["ResourceType"] == "AWS::EMR::Cluster"
|
||
|
|
assert cluster_id.startswith("j-")
|
||
|
|
|
||
|
|
# Verify outputs
|
||
|
|
stack = cf.describe_stacks(StackName="teststack")["Stacks"][0]
|
||
|
|
assert {"OutputKey": "ClusterId", "OutputValue": cluster_id} in stack["Outputs"]
|
||
|
|
|
||
|
|
# Verify EMR Cluster
|
||
|
|
cl = emr.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
||
|
|
assert cl["Name"] == "my cluster"
|
||
|
|
assert cl["Ec2InstanceAttributes"]["IamInstanceProfile"] == "EMR_EC2_DefaultRole"
|
||
|
|
assert cl["ServiceRole"] == "EMR_DefaultRole"
|
||
|
|
assert cl["Tags"] == []
|
||
|
|
|
||
|
|
|
||
|
|
template_with_tags = {
|
||
|
|
"Resources": {
|
||
|
|
"Cluster1": {
|
||
|
|
"Type": "AWS::EMR::Cluster",
|
||
|
|
"Properties": {
|
||
|
|
"Instances": {
|
||
|
|
"CoreInstanceGroup": {
|
||
|
|
"InstanceCount": 3,
|
||
|
|
"InstanceType": "m3g",
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"JobFlowRole": "EMR_EC2_DefaultRole",
|
||
|
|
"Name": "my cluster",
|
||
|
|
"ServiceRole": "EMR_DefaultRole",
|
||
|
|
"Tags": [
|
||
|
|
{"Key": "k1", "Value": "v1"},
|
||
|
|
{"Key": "k2", "Value": "v2"},
|
||
|
|
],
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
@mock_aws
|
||
|
|
def test_create_simple_cluster_with_tags():
|
||
|
|
region = "us-east-1"
|
||
|
|
cf = boto3.client("cloudformation", region_name=region)
|
||
|
|
emr = boto3.client("emr", region_name=region)
|
||
|
|
cf.create_stack(StackName="teststack", TemplateBody=json.dumps(template_with_tags))
|
||
|
|
|
||
|
|
# Verify resources
|
||
|
|
res = cf.describe_stack_resources(StackName="teststack")["StackResources"][0]
|
||
|
|
cluster_id = res["PhysicalResourceId"]
|
||
|
|
|
||
|
|
# Verify EMR Cluster
|
||
|
|
cl = emr.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
||
|
|
assert cl["Tags"] == [{"Key": "k1", "Value": "v1"}, {"Key": "k2", "Value": "v2"}]
|
||
|
|
|
||
|
|
|
||
|
|
template_with_custom_ami = {
|
||
|
|
"AWSTemplateFormatVersion": "2010-09-09",
|
||
|
|
"Parameters": {
|
||
|
|
"SubnetId": {"Type": "String"},
|
||
|
|
"InstanceType": {"Type": "String"},
|
||
|
|
"TerminationProtected": {"Type": "String", "Default": "false"},
|
||
|
|
"ElasticMapReducePrincipal": {"Type": "String"},
|
||
|
|
"Ec2Principal": {"Type": "String"},
|
||
|
|
},
|
||
|
|
"Resources": {
|
||
|
|
"cluster": {
|
||
|
|
"Type": "AWS::EMR::Cluster",
|
||
|
|
"Properties": {
|
||
|
|
"CustomAmiId": EXAMPLE_AMI_ID,
|
||
|
|
"Instances": {
|
||
|
|
"MasterInstanceGroup": {
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnMaster",
|
||
|
|
},
|
||
|
|
"CoreInstanceGroup": {
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnCore",
|
||
|
|
},
|
||
|
|
"TaskInstanceGroups": [
|
||
|
|
{
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnTask-1",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnTask-2",
|
||
|
|
},
|
||
|
|
],
|
||
|
|
"TerminationProtected": {"Ref": "TerminationProtected"},
|
||
|
|
"Ec2SubnetId": {"Ref": "SubnetId"},
|
||
|
|
},
|
||
|
|
"Name": "CFNtest",
|
||
|
|
"JobFlowRole": {"Ref": "emrEc2InstanceProfile"},
|
||
|
|
"ServiceRole": {"Ref": "emrRole"},
|
||
|
|
"ReleaseLabel": "release_2024",
|
||
|
|
"VisibleToAllUsers": True,
|
||
|
|
"Tags": [{"Key": "key1", "Value": "value1"}],
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"emrRole": {
|
||
|
|
"Type": "AWS::IAM::Role",
|
||
|
|
"Properties": {
|
||
|
|
"AssumeRolePolicyDocument": {
|
||
|
|
"Version": "2008-10-17",
|
||
|
|
"Statement": [
|
||
|
|
{
|
||
|
|
"Sid": "",
|
||
|
|
"Effect": "Allow",
|
||
|
|
"Principal": {
|
||
|
|
"Service": {"Ref": "ElasticMapReducePrincipal"}
|
||
|
|
},
|
||
|
|
"Action": "sts:AssumeRole",
|
||
|
|
}
|
||
|
|
],
|
||
|
|
},
|
||
|
|
"Path": "/",
|
||
|
|
"ManagedPolicyArns": [
|
||
|
|
"arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole"
|
||
|
|
],
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"emrEc2Role": {
|
||
|
|
"Type": "AWS::IAM::Role",
|
||
|
|
"Properties": {
|
||
|
|
"AssumeRolePolicyDocument": {
|
||
|
|
"Version": "2008-10-17",
|
||
|
|
"Statement": [
|
||
|
|
{
|
||
|
|
"Sid": "",
|
||
|
|
"Effect": "Allow",
|
||
|
|
"Principal": {"Service": {"Ref": "Ec2Principal"}},
|
||
|
|
"Action": "sts:AssumeRole",
|
||
|
|
}
|
||
|
|
],
|
||
|
|
},
|
||
|
|
"Path": "/",
|
||
|
|
"ManagedPolicyArns": [
|
||
|
|
"arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
|
||
|
|
],
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"emrEc2InstanceProfile": {
|
||
|
|
"Type": "AWS::IAM::InstanceProfile",
|
||
|
|
"Properties": {"Path": "/", "Roles": [{"Ref": "emrEc2Role"}]},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
@mock_aws
|
||
|
|
def test_create_cluster_with_custom_ami():
|
||
|
|
ec2 = boto3.resource("ec2", region_name="us-east-1")
|
||
|
|
vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16")
|
||
|
|
subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18")
|
||
|
|
|
||
|
|
params = [
|
||
|
|
{"ParameterKey": "InstanceType", "ParameterValue": "i4s"},
|
||
|
|
{"ParameterKey": "SubnetId", "ParameterValue": subnet.id},
|
||
|
|
{
|
||
|
|
"ParameterKey": "ElasticMapReducePrincipal",
|
||
|
|
"ParameterValue": "emr_principal",
|
||
|
|
},
|
||
|
|
{"ParameterKey": "Ec2Principal", "ParameterValue": "ec2_principal"},
|
||
|
|
]
|
||
|
|
|
||
|
|
region = "us-east-1"
|
||
|
|
cf = boto3.client("cloudformation", region_name=region)
|
||
|
|
emr = boto3.client("emr", region_name=region)
|
||
|
|
cf.create_stack(
|
||
|
|
StackName="teststack",
|
||
|
|
TemplateBody=json.dumps(template_with_custom_ami),
|
||
|
|
Parameters=params,
|
||
|
|
)
|
||
|
|
|
||
|
|
# Verify resources
|
||
|
|
res = cf.describe_stack_resources(StackName="teststack")["StackResources"]
|
||
|
|
cluster = [r for r in res if r["ResourceType"] == "AWS::EMR::Cluster"][0]
|
||
|
|
cluster_id = cluster["PhysicalResourceId"]
|
||
|
|
|
||
|
|
instance_profile = [
|
||
|
|
r for r in res if r["ResourceType"] == "AWS::IAM::InstanceProfile"
|
||
|
|
][0]
|
||
|
|
instance_profile_id = instance_profile["PhysicalResourceId"]
|
||
|
|
|
||
|
|
role = [r for r in res if r["ResourceType"] == "AWS::IAM::Role"][0]
|
||
|
|
role_id = role["PhysicalResourceId"]
|
||
|
|
|
||
|
|
# Verify EMR Cluster
|
||
|
|
cl = emr.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
||
|
|
assert cl["Name"] == "CFNtest"
|
||
|
|
assert cl["Ec2InstanceAttributes"]["Ec2SubnetId"] == subnet.id
|
||
|
|
assert cl["Ec2InstanceAttributes"]["IamInstanceProfile"] == instance_profile_id
|
||
|
|
assert cl["ServiceRole"] == role_id
|
||
|
|
assert cl["ReleaseLabel"] == "release_2024"
|
||
|
|
assert cl["CustomAmiId"] == EXAMPLE_AMI_ID
|
||
|
|
|
||
|
|
|
||
|
|
template_with_root_volume = {
|
||
|
|
"AWSTemplateFormatVersion": "2010-09-09",
|
||
|
|
"Parameters": {
|
||
|
|
"InstanceType": {"Type": "String"},
|
||
|
|
"ReleaseLabel": {"Type": "String"},
|
||
|
|
"SubnetId": {"Type": "String"},
|
||
|
|
"TerminationProtected": {"Type": "String", "Default": "false"},
|
||
|
|
"EbsRootVolumeSize": {"Type": "String"},
|
||
|
|
},
|
||
|
|
"Resources": {
|
||
|
|
"cluster": {
|
||
|
|
"Type": "AWS::EMR::Cluster",
|
||
|
|
"Properties": {
|
||
|
|
"EbsRootVolumeSize": {"Ref": "EbsRootVolumeSize"},
|
||
|
|
"Instances": {
|
||
|
|
"MasterInstanceGroup": {
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnMaster",
|
||
|
|
},
|
||
|
|
"CoreInstanceGroup": {
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnCore",
|
||
|
|
},
|
||
|
|
"TaskInstanceGroups": [
|
||
|
|
{
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnTask-1",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnTask-2",
|
||
|
|
},
|
||
|
|
],
|
||
|
|
"TerminationProtected": {"Ref": "TerminationProtected"},
|
||
|
|
"Ec2SubnetId": {"Ref": "SubnetId"},
|
||
|
|
},
|
||
|
|
"Name": "CFNtest",
|
||
|
|
"JobFlowRole": {"Ref": "emrEc2InstanceProfile"},
|
||
|
|
"ServiceRole": {"Ref": "emrRole"},
|
||
|
|
"ReleaseLabel": {"Ref": "ReleaseLabel"},
|
||
|
|
"VisibleToAllUsers": True,
|
||
|
|
"Tags": [{"Key": "key1", "Value": "value1"}],
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"emrRole": {
|
||
|
|
"Type": "AWS::IAM::Role",
|
||
|
|
"Properties": {
|
||
|
|
"AssumeRolePolicyDocument": {
|
||
|
|
"Version": "2008-10-17",
|
||
|
|
"Statement": [
|
||
|
|
{
|
||
|
|
"Sid": "",
|
||
|
|
"Effect": "Allow",
|
||
|
|
"Principal": {"Service": "elasticmapreduce.amazonaws.com"},
|
||
|
|
"Action": "sts:AssumeRole",
|
||
|
|
}
|
||
|
|
],
|
||
|
|
},
|
||
|
|
"Path": "/",
|
||
|
|
"ManagedPolicyArns": [
|
||
|
|
"arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole"
|
||
|
|
],
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"emrEc2Role": {
|
||
|
|
"Type": "AWS::IAM::Role",
|
||
|
|
"Properties": {
|
||
|
|
"AssumeRolePolicyDocument": {
|
||
|
|
"Version": "2008-10-17",
|
||
|
|
"Statement": [
|
||
|
|
{
|
||
|
|
"Sid": "",
|
||
|
|
"Effect": "Allow",
|
||
|
|
"Principal": {"Service": "ec2.amazonaws.com"},
|
||
|
|
"Action": "sts:AssumeRole",
|
||
|
|
}
|
||
|
|
],
|
||
|
|
},
|
||
|
|
"Path": "/",
|
||
|
|
"ManagedPolicyArns": [
|
||
|
|
"arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
|
||
|
|
],
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"emrEc2InstanceProfile": {
|
||
|
|
"Type": "AWS::IAM::InstanceProfile",
|
||
|
|
"Properties": {"Path": "/", "Roles": [{"Ref": "emrEc2Role"}]},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
@mock_aws
|
||
|
|
def test_create_cluster_with_root_volume():
|
||
|
|
ec2 = boto3.resource("ec2", region_name="us-east-1")
|
||
|
|
vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16")
|
||
|
|
subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18")
|
||
|
|
|
||
|
|
params = [
|
||
|
|
{"ParameterKey": "InstanceType", "ParameterValue": "i4s"},
|
||
|
|
{"ParameterKey": "ReleaseLabel", "ParameterValue": "latest"},
|
||
|
|
{"ParameterKey": "EbsRootVolumeSize", "ParameterValue": "15"},
|
||
|
|
{"ParameterKey": "SubnetId", "ParameterValue": subnet.id},
|
||
|
|
{
|
||
|
|
"ParameterKey": "ElasticMapReducePrincipal",
|
||
|
|
"ParameterValue": "emr_principal",
|
||
|
|
},
|
||
|
|
{"ParameterKey": "Ec2Principal", "ParameterValue": "ec2_principal"},
|
||
|
|
]
|
||
|
|
|
||
|
|
region = "us-east-1"
|
||
|
|
cf = boto3.client("cloudformation", region_name=region)
|
||
|
|
emr = boto3.client("emr", region_name=region)
|
||
|
|
cf.create_stack(
|
||
|
|
StackName="teststack",
|
||
|
|
TemplateBody=json.dumps(template_with_root_volume),
|
||
|
|
Parameters=params,
|
||
|
|
)
|
||
|
|
|
||
|
|
# Verify resources
|
||
|
|
res = cf.describe_stack_resources(StackName="teststack")["StackResources"]
|
||
|
|
cluster = [r for r in res if r["ResourceType"] == "AWS::EMR::Cluster"][0]
|
||
|
|
cluster_id = cluster["PhysicalResourceId"]
|
||
|
|
|
||
|
|
instance_profile = [
|
||
|
|
r for r in res if r["ResourceType"] == "AWS::IAM::InstanceProfile"
|
||
|
|
][0]
|
||
|
|
instance_profile_id = instance_profile["PhysicalResourceId"]
|
||
|
|
|
||
|
|
role = [r for r in res if r["ResourceType"] == "AWS::IAM::Role"][0]
|
||
|
|
role_id = role["PhysicalResourceId"]
|
||
|
|
|
||
|
|
# Verify EMR Cluster
|
||
|
|
cl = emr.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
||
|
|
assert cl["Name"] == "CFNtest"
|
||
|
|
assert cl["Ec2InstanceAttributes"]["Ec2SubnetId"] == subnet.id
|
||
|
|
assert cl["Ec2InstanceAttributes"]["IamInstanceProfile"] == instance_profile_id
|
||
|
|
assert cl["ServiceRole"] == role_id
|
||
|
|
|
||
|
|
|
||
|
|
template_with_kerberos_attrs = {
|
||
|
|
"AWSTemplateFormatVersion": "2010-09-09",
|
||
|
|
"Parameters": {
|
||
|
|
"CrossRealmTrustPrincipalPassword": {"Type": "String"},
|
||
|
|
"KdcAdminPassword": {"Type": "String"},
|
||
|
|
"InstanceType": {"Type": "String"},
|
||
|
|
"ReleaseLabel": {"Type": "String"},
|
||
|
|
"SubnetId": {"Type": "String"},
|
||
|
|
},
|
||
|
|
"Resources": {
|
||
|
|
"cluster": {
|
||
|
|
"Type": "AWS::EMR::Cluster",
|
||
|
|
"Properties": {
|
||
|
|
"Instances": {
|
||
|
|
"MasterInstanceGroup": {
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnMaster",
|
||
|
|
},
|
||
|
|
"CoreInstanceGroup": {
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnCore",
|
||
|
|
},
|
||
|
|
"TaskInstanceGroups": [
|
||
|
|
{
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnTask-1",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"InstanceCount": 1,
|
||
|
|
"InstanceType": {"Ref": "InstanceType"},
|
||
|
|
"Market": "ON_DEMAND",
|
||
|
|
"Name": "cfnTask-2",
|
||
|
|
},
|
||
|
|
],
|
||
|
|
"Ec2SubnetId": {"Ref": "SubnetId"},
|
||
|
|
},
|
||
|
|
"Name": "CFNtest",
|
||
|
|
"JobFlowRole": {"Ref": "emrEc2InstanceProfile"},
|
||
|
|
"KerberosAttributes": {
|
||
|
|
"CrossRealmTrustPrincipalPassword": {
|
||
|
|
"Ref": "CrossRealmTrustPrincipalPassword"
|
||
|
|
},
|
||
|
|
"KdcAdminPassword": {"Ref": "KdcAdminPassword"},
|
||
|
|
"Realm": "EC2.INTERNAL",
|
||
|
|
},
|
||
|
|
"ServiceRole": {"Ref": "emrRole"},
|
||
|
|
"ReleaseLabel": {"Ref": "ReleaseLabel"},
|
||
|
|
"SecurityConfiguration": {"Ref": "securityConfiguration"},
|
||
|
|
"VisibleToAllUsers": True,
|
||
|
|
"Tags": [{"Key": "key1", "Value": "value1"}],
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"key": {
|
||
|
|
"Type": "AWS::KMS::Key",
|
||
|
|
"Properties": {
|
||
|
|
"KeyPolicy": {
|
||
|
|
"Version": "2012-10-17",
|
||
|
|
"Id": "key-default-1",
|
||
|
|
"Statement": [
|
||
|
|
{
|
||
|
|
"Sid": "Enable IAM User Permissions",
|
||
|
|
"Effect": "Allow",
|
||
|
|
"Principal": {"AWS": {"Fn::GetAtt": ["emrEc2Role", "Arn"]}},
|
||
|
|
"Action": "kms:*",
|
||
|
|
"Resource": "*",
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"Sid": "Enable IAM User Permissions",
|
||
|
|
"Effect": "Allow",
|
||
|
|
"Principal": {
|
||
|
|
"AWS": {
|
||
|
|
"Fn::Join": [
|
||
|
|
"",
|
||
|
|
[
|
||
|
|
"arn:aws:iam::",
|
||
|
|
{"Ref": "AWS::AccountId"},
|
||
|
|
":root",
|
||
|
|
],
|
||
|
|
]
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"Action": "kms:*",
|
||
|
|
"Resource": "*",
|
||
|
|
},
|
||
|
|
],
|
||
|
|
}
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"securityConfiguration": {
|
||
|
|
"Type": "AWS::EMR::SecurityConfiguration",
|
||
|
|
"Properties": {
|
||
|
|
"SecurityConfiguration": {
|
||
|
|
"AuthenticationConfiguration": {
|
||
|
|
"KerberosConfiguration": {
|
||
|
|
"Provider": "ClusterDedicatedKdc",
|
||
|
|
"ClusterDedicatedKdcConfiguration": {
|
||
|
|
"TicketLifetimeInHours": 24,
|
||
|
|
"CrossRealmTrustConfiguration": {
|
||
|
|
"Realm": "AD.DOMAIN.COM",
|
||
|
|
"Domain": "ad.domain.com",
|
||
|
|
"AdminServer": "ad.domain.com",
|
||
|
|
"KdcServer": "ad.domain.com",
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"emrRole": {
|
||
|
|
"Type": "AWS::IAM::Role",
|
||
|
|
"Properties": {
|
||
|
|
"AssumeRolePolicyDocument": {
|
||
|
|
"Version": "2008-10-17",
|
||
|
|
"Statement": [
|
||
|
|
{
|
||
|
|
"Sid": "",
|
||
|
|
"Effect": "Allow",
|
||
|
|
"Principal": {"Service": "elasticmapreduce.amazonaws.com"},
|
||
|
|
"Action": "sts:AssumeRole",
|
||
|
|
}
|
||
|
|
],
|
||
|
|
},
|
||
|
|
"Path": "/",
|
||
|
|
"ManagedPolicyArns": [
|
||
|
|
"arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceRole"
|
||
|
|
],
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"emrEc2Role": {
|
||
|
|
"Type": "AWS::IAM::Role",
|
||
|
|
"Properties": {
|
||
|
|
"AssumeRolePolicyDocument": {
|
||
|
|
"Version": "2008-10-17",
|
||
|
|
"Statement": [
|
||
|
|
{
|
||
|
|
"Sid": "",
|
||
|
|
"Effect": "Allow",
|
||
|
|
"Principal": {"Service": "ec2.amazonaws.com"},
|
||
|
|
"Action": "sts:AssumeRole",
|
||
|
|
}
|
||
|
|
],
|
||
|
|
},
|
||
|
|
"Path": "/",
|
||
|
|
"ManagedPolicyArns": [
|
||
|
|
"arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
|
||
|
|
],
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"emrEc2InstanceProfile": {
|
||
|
|
"Type": "AWS::IAM::InstanceProfile",
|
||
|
|
"Properties": {"Path": "/", "Roles": [{"Ref": "emrEc2Role"}]},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
"Outputs": {"keyArn": {"Value": {"Fn::GetAtt": ["key", "Arn"]}}},
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
@mock_aws
|
||
|
|
def test_create_cluster_with_kerberos_attrs():
|
||
|
|
ec2 = boto3.resource("ec2", region_name="us-east-1")
|
||
|
|
vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16")
|
||
|
|
subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/18")
|
||
|
|
|
||
|
|
params = [
|
||
|
|
{"ParameterKey": "CrossRealmTrustPrincipalPassword", "ParameterValue": "p2ss"},
|
||
|
|
{"ParameterKey": "KdcAdminPassword", "ParameterValue": "adminp2ss"},
|
||
|
|
{"ParameterKey": "InstanceType", "ParameterValue": "i4s"},
|
||
|
|
{"ParameterKey": "ReleaseLabel", "ParameterValue": "latest"},
|
||
|
|
{"ParameterKey": "EbsRootVolumeSize", "ParameterValue": "15"},
|
||
|
|
{"ParameterKey": "SubnetId", "ParameterValue": subnet.id},
|
||
|
|
{
|
||
|
|
"ParameterKey": "ElasticMapReducePrincipal",
|
||
|
|
"ParameterValue": "emr_principal",
|
||
|
|
},
|
||
|
|
{"ParameterKey": "Ec2Principal", "ParameterValue": "ec2_principal"},
|
||
|
|
]
|
||
|
|
|
||
|
|
region = "us-east-1"
|
||
|
|
cf = boto3.client("cloudformation", region_name=region)
|
||
|
|
emr = boto3.client("emr", region_name=region)
|
||
|
|
cf.create_stack(
|
||
|
|
StackName="teststack",
|
||
|
|
TemplateBody=json.dumps(template_with_kerberos_attrs),
|
||
|
|
Parameters=params,
|
||
|
|
)
|
||
|
|
|
||
|
|
# Verify resources
|
||
|
|
res = cf.describe_stack_resources(StackName="teststack")["StackResources"]
|
||
|
|
cluster = [r for r in res if r["ResourceType"] == "AWS::EMR::Cluster"][0]
|
||
|
|
cluster_id = cluster["PhysicalResourceId"]
|
||
|
|
|
||
|
|
instance_profile = [
|
||
|
|
r for r in res if r["ResourceType"] == "AWS::IAM::InstanceProfile"
|
||
|
|
][0]
|
||
|
|
instance_profile_id = instance_profile["PhysicalResourceId"]
|
||
|
|
|
||
|
|
role = [r for r in res if r["ResourceType"] == "AWS::IAM::Role"][0]
|
||
|
|
role_id = role["PhysicalResourceId"]
|
||
|
|
|
||
|
|
# Verify EMR Cluster
|
||
|
|
cl = emr.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
||
|
|
assert cl["Name"] == "CFNtest"
|
||
|
|
assert cl["Ec2InstanceAttributes"]["Ec2SubnetId"] == subnet.id
|
||
|
|
assert cl["Ec2InstanceAttributes"]["IamInstanceProfile"] == instance_profile_id
|
||
|
|
assert cl["ServiceRole"] == role_id
|
||
|
|
|
||
|
|
kerberos = cl["KerberosAttributes"]
|
||
|
|
assert kerberos == {
|
||
|
|
"Realm": "EC2.INTERNAL",
|
||
|
|
"KdcAdminPassword": "adminp2ss",
|
||
|
|
"CrossRealmTrustPrincipalPassword": "p2ss",
|
||
|
|
}
|