1202 lines
44 KiB
Python
1202 lines
44 KiB
Python
import json
|
|
import time
|
|
from copy import deepcopy
|
|
from datetime import datetime, timezone
|
|
|
|
import boto3
|
|
import pytest
|
|
from botocore.exceptions import ClientError
|
|
|
|
from moto import mock_emr
|
|
from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
|
|
|
|
run_job_flow_args = dict(
|
|
Instances={
|
|
"InstanceCount": 3,
|
|
"KeepJobFlowAliveWhenNoSteps": True,
|
|
"MasterInstanceType": "c3.medium",
|
|
"Placement": {"AvailabilityZone": "us-east-1a"},
|
|
"SlaveInstanceType": "c3.xlarge",
|
|
},
|
|
JobFlowRole="EMR_EC2_DefaultRole",
|
|
LogUri="s3://mybucket/log",
|
|
Name="cluster",
|
|
ServiceRole="EMR_DefaultRole",
|
|
VisibleToAllUsers=True,
|
|
)
|
|
|
|
|
|
input_instance_groups = [
|
|
{
|
|
"InstanceCount": 1,
|
|
"InstanceRole": "MASTER",
|
|
"InstanceType": "c1.medium",
|
|
"Market": "ON_DEMAND",
|
|
"Name": "master",
|
|
},
|
|
{
|
|
"InstanceCount": 3,
|
|
"InstanceRole": "CORE",
|
|
"InstanceType": "c1.medium",
|
|
"Market": "ON_DEMAND",
|
|
"Name": "core",
|
|
},
|
|
{
|
|
"InstanceCount": 6,
|
|
"InstanceRole": "TASK",
|
|
"InstanceType": "c3.large",
|
|
"Market": "SPOT",
|
|
"Name": "task-1",
|
|
"BidPrice": "0.07",
|
|
},
|
|
{
|
|
"InstanceCount": 10,
|
|
"InstanceRole": "TASK",
|
|
"InstanceType": "c1.xlarge",
|
|
"Market": "SPOT",
|
|
"Name": "task-2",
|
|
"BidPrice": "0.05",
|
|
"EbsConfiguration": {
|
|
"EbsBlockDeviceConfigs": [
|
|
{
|
|
"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 800},
|
|
"VolumesPerInstance": 6,
|
|
},
|
|
],
|
|
"EbsOptimized": True,
|
|
},
|
|
},
|
|
]
|
|
|
|
|
|
@mock_emr
|
|
@pytest.mark.filterwarnings("ignore")
|
|
def test_describe_cluster():
|
|
region_name = "us-east-1"
|
|
client = boto3.client("emr", region_name=region_name)
|
|
|
|
args = deepcopy(run_job_flow_args)
|
|
args["Applications"] = [{"Name": "Spark", "Version": "2.4.2"}]
|
|
args["Configurations"] = [
|
|
{
|
|
"Classification": "yarn-site",
|
|
"Properties": {
|
|
"someproperty": "somevalue",
|
|
"someotherproperty": "someothervalue",
|
|
},
|
|
},
|
|
{
|
|
"Classification": "nested-configs",
|
|
"Properties": {},
|
|
"Configurations": [
|
|
{
|
|
"Classification": "nested-config",
|
|
"Properties": {"nested-property": "nested-value"},
|
|
}
|
|
],
|
|
},
|
|
]
|
|
args["Instances"]["AdditionalMasterSecurityGroups"] = ["additional-master"]
|
|
args["Instances"]["AdditionalSlaveSecurityGroups"] = ["additional-slave"]
|
|
args["Instances"]["Ec2KeyName"] = "mykey"
|
|
args["Instances"]["Ec2SubnetId"] = "subnet-8be41cec"
|
|
args["Instances"]["EmrManagedMasterSecurityGroup"] = "master-security-group"
|
|
args["Instances"]["EmrManagedSlaveSecurityGroup"] = "slave-security-group"
|
|
args["Instances"]["KeepJobFlowAliveWhenNoSteps"] = False
|
|
args["Instances"]["ServiceAccessSecurityGroup"] = "service-access-security-group"
|
|
args["KerberosAttributes"] = {
|
|
"Realm": "MY-REALM.COM",
|
|
"KdcAdminPassword": "SuperSecretPassword2",
|
|
"CrossRealmTrustPrincipalPassword": "SuperSecretPassword3",
|
|
"ADDomainJoinUser": "Bob",
|
|
"ADDomainJoinPassword": "SuperSecretPassword4",
|
|
}
|
|
args["Tags"] = [{"Key": "tag1", "Value": "val1"}, {"Key": "tag2", "Value": "val2"}]
|
|
args["SecurityConfiguration"] = "my-security-configuration"
|
|
args["AutoScalingRole"] = "EMR_AutoScaling_DefaultRole"
|
|
args["AutoTerminationPolicy"] = {"IdleTimeout": 123}
|
|
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
|
|
cl = client.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
|
assert cl["Applications"][0]["Name"] == "Spark"
|
|
assert cl["Applications"][0]["Version"] == "2.4.2"
|
|
assert cl["AutoTerminate"] is True
|
|
|
|
config = cl["Configurations"][0]
|
|
assert config["Classification"] == "yarn-site"
|
|
assert config["Properties"] == args["Configurations"][0]["Properties"]
|
|
|
|
nested_config = cl["Configurations"][1]
|
|
assert nested_config["Classification"] == "nested-configs"
|
|
assert nested_config["Properties"] == args["Configurations"][1]["Properties"]
|
|
|
|
attrs = cl["Ec2InstanceAttributes"]
|
|
assert (
|
|
attrs["AdditionalMasterSecurityGroups"]
|
|
== args["Instances"]["AdditionalMasterSecurityGroups"]
|
|
)
|
|
assert (
|
|
attrs["AdditionalSlaveSecurityGroups"]
|
|
== args["Instances"]["AdditionalSlaveSecurityGroups"]
|
|
)
|
|
assert attrs["Ec2AvailabilityZone"] == "us-east-1a"
|
|
assert attrs["Ec2KeyName"] == args["Instances"]["Ec2KeyName"]
|
|
assert attrs["Ec2SubnetId"] == args["Instances"]["Ec2SubnetId"]
|
|
assert (
|
|
attrs["EmrManagedMasterSecurityGroup"]
|
|
== args["Instances"]["EmrManagedMasterSecurityGroup"]
|
|
)
|
|
assert (
|
|
attrs["EmrManagedSlaveSecurityGroup"]
|
|
== args["Instances"]["EmrManagedSlaveSecurityGroup"]
|
|
)
|
|
assert attrs["IamInstanceProfile"] == args["JobFlowRole"]
|
|
assert (
|
|
attrs["ServiceAccessSecurityGroup"]
|
|
== args["Instances"]["ServiceAccessSecurityGroup"]
|
|
)
|
|
assert cl["Id"] == cluster_id
|
|
assert cl["KerberosAttributes"] == args["KerberosAttributes"]
|
|
assert cl["LogUri"] == args["LogUri"]
|
|
assert isinstance(cl["MasterPublicDnsName"], str)
|
|
assert cl["Name"] == args["Name"]
|
|
assert cl["NormalizedInstanceHours"] == 0
|
|
# assert cl['ReleaseLabel'] == 'emr-5.0.0'
|
|
assert "RequestedAmiVersion" not in cl
|
|
assert cl["RunningAmiVersion"] == "1.0.0"
|
|
assert isinstance(cl["SecurityConfiguration"], str)
|
|
assert cl["SecurityConfiguration"] == args["SecurityConfiguration"]
|
|
assert cl["ServiceRole"] == args["ServiceRole"]
|
|
assert cl["AutoScalingRole"] == args["AutoScalingRole"]
|
|
|
|
status = cl["Status"]
|
|
assert status["State"] == "TERMINATED"
|
|
# cluster['Status']['StateChangeReason']
|
|
assert isinstance(status["Timeline"]["CreationDateTime"], datetime)
|
|
# assert status['Timeline']['EndDateTime'] == datetime(2014, 1, 24, 2, 19, 46, tzinfo=timezone.utc)
|
|
assert isinstance(status["Timeline"]["ReadyDateTime"], datetime)
|
|
|
|
assert {t["Key"]: t["Value"] for t in cl["Tags"]} == {
|
|
t["Key"]: t["Value"] for t in args["Tags"]
|
|
}
|
|
|
|
assert cl["TerminationProtected"] is False
|
|
assert cl["VisibleToAllUsers"] is True
|
|
assert (
|
|
cl["ClusterArn"]
|
|
== f"arn:aws:elasticmapreduce:{region_name}:{ACCOUNT_ID}:cluster/{cluster_id}"
|
|
)
|
|
|
|
|
|
@mock_emr
|
|
def test_describe_cluster_not_found():
|
|
conn = boto3.client("emr", region_name="us-east-1")
|
|
with pytest.raises(ClientError) as e:
|
|
conn.describe_cluster(ClusterId="DummyId")
|
|
|
|
assert e.value.response["Error"]["Code"] == "ResourceNotFoundException"
|
|
|
|
|
|
@mock_emr
|
|
def test_describe_job_flows():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
expected = {}
|
|
|
|
for idx in range(4):
|
|
cluster_name = "cluster" + str(idx)
|
|
args["Name"] = cluster_name
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
expected[cluster_id] = {
|
|
"Id": cluster_id,
|
|
"Name": cluster_name,
|
|
"State": "WAITING",
|
|
}
|
|
|
|
# need sleep since it appears the timestamp is always rounded to
|
|
# the nearest second internally
|
|
time.sleep(1)
|
|
timestamp = datetime.now(timezone.utc)
|
|
time.sleep(1)
|
|
|
|
for idx in range(4, 6):
|
|
cluster_name = "cluster" + str(idx)
|
|
args["Name"] = cluster_name
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
client.terminate_job_flows(JobFlowIds=[cluster_id])
|
|
expected[cluster_id] = {
|
|
"Id": cluster_id,
|
|
"Name": cluster_name,
|
|
"State": "TERMINATED",
|
|
}
|
|
|
|
resp = client.describe_job_flows()
|
|
assert len(resp["JobFlows"]) == 6
|
|
|
|
for cluster_id in expected:
|
|
resp = client.describe_job_flows(JobFlowIds=[cluster_id])
|
|
assert len(resp["JobFlows"]) == 1
|
|
assert resp["JobFlows"][0]["JobFlowId"] == cluster_id
|
|
|
|
resp = client.describe_job_flows(JobFlowStates=["WAITING"])
|
|
assert len(resp["JobFlows"]) == 4
|
|
for x in resp["JobFlows"]:
|
|
assert x["ExecutionStatusDetail"]["State"] == "WAITING"
|
|
|
|
resp = client.describe_job_flows(CreatedBefore=timestamp)
|
|
assert len(resp["JobFlows"]) == 4
|
|
|
|
resp = client.describe_job_flows(CreatedAfter=timestamp)
|
|
assert len(resp["JobFlows"]) == 2
|
|
|
|
|
|
@mock_emr
|
|
@pytest.mark.filterwarnings("ignore")
|
|
def test_describe_job_flow():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
|
|
args = deepcopy(run_job_flow_args)
|
|
args["AmiVersion"] = "3.8.1"
|
|
args["Instances"].update(
|
|
{
|
|
"Ec2KeyName": "ec2keyname",
|
|
"Ec2SubnetId": "subnet-8be41cec",
|
|
"HadoopVersion": "2.4.0",
|
|
}
|
|
)
|
|
args["VisibleToAllUsers"] = True
|
|
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
|
|
jf = client.describe_job_flows(JobFlowIds=[cluster_id])["JobFlows"][0]
|
|
|
|
assert jf["AmiVersion"] == args["AmiVersion"]
|
|
assert "BootstrapActions" not in jf
|
|
esd = jf["ExecutionStatusDetail"]
|
|
assert isinstance(esd["CreationDateTime"], datetime)
|
|
# assert isinstance(esd['EndDateTime'], 'datetime.datetime')
|
|
# assert isinstance(esd['LastStateChangeReason'], str)
|
|
assert isinstance(esd["ReadyDateTime"], datetime)
|
|
assert isinstance(esd["StartDateTime"], datetime)
|
|
assert esd["State"] == "WAITING"
|
|
attrs = jf["Instances"]
|
|
assert attrs["Ec2KeyName"] == args["Instances"]["Ec2KeyName"]
|
|
assert attrs["Ec2SubnetId"] == args["Instances"]["Ec2SubnetId"]
|
|
assert attrs["HadoopVersion"] == args["Instances"]["HadoopVersion"]
|
|
assert attrs["InstanceCount"] == args["Instances"]["InstanceCount"]
|
|
for ig in attrs["InstanceGroups"]:
|
|
# ig['BidPrice']
|
|
assert isinstance(ig["CreationDateTime"], datetime)
|
|
# assert isinstance(ig['EndDateTime'], 'datetime.datetime')
|
|
assert isinstance(ig["InstanceGroupId"], str)
|
|
assert isinstance(ig["InstanceRequestCount"], int)
|
|
assert ig["InstanceRole"] in ["MASTER", "CORE"]
|
|
assert isinstance(ig["InstanceRunningCount"], int)
|
|
assert ig["InstanceType"] in ["c3.medium", "c3.xlarge"]
|
|
# assert isinstance(ig['LastStateChangeReason'], str)
|
|
assert ig["Market"] == "ON_DEMAND"
|
|
assert isinstance(ig["Name"], str)
|
|
assert isinstance(ig["ReadyDateTime"], datetime)
|
|
assert isinstance(ig["StartDateTime"], datetime)
|
|
assert ig["State"] == "RUNNING"
|
|
assert attrs["KeepJobFlowAliveWhenNoSteps"] is True
|
|
# assert isinstance(attrs['MasterInstanceId'], str)
|
|
assert attrs["MasterInstanceType"] == args["Instances"]["MasterInstanceType"]
|
|
assert isinstance(attrs["MasterPublicDnsName"], str)
|
|
assert attrs["NormalizedInstanceHours"] == 0
|
|
assert (
|
|
attrs["Placement"]["AvailabilityZone"]
|
|
== args["Instances"]["Placement"]["AvailabilityZone"]
|
|
)
|
|
assert attrs["SlaveInstanceType"] == args["Instances"]["SlaveInstanceType"]
|
|
assert attrs["TerminationProtected"] is False
|
|
assert jf["JobFlowId"] == cluster_id
|
|
assert jf["JobFlowRole"] == args["JobFlowRole"]
|
|
assert jf["LogUri"] == args["LogUri"]
|
|
assert jf["Name"] == args["Name"]
|
|
assert jf["ServiceRole"] == args["ServiceRole"]
|
|
assert jf["Steps"] == []
|
|
assert jf["SupportedProducts"] == []
|
|
assert jf["VisibleToAllUsers"] is True
|
|
|
|
|
|
@mock_emr
|
|
def test_list_clusters():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
expected = {}
|
|
|
|
for idx in range(40):
|
|
cluster_name = "jobflow" + str(idx)
|
|
args["Name"] = cluster_name
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
expected[cluster_id] = {
|
|
"Id": cluster_id,
|
|
"Name": cluster_name,
|
|
"NormalizedInstanceHours": 0,
|
|
"State": "WAITING",
|
|
}
|
|
|
|
# need sleep since it appears the timestamp is always rounded to
|
|
# the nearest second internally
|
|
time.sleep(1)
|
|
timestamp = datetime.now(timezone.utc)
|
|
time.sleep(1)
|
|
|
|
for idx in range(40, 70):
|
|
cluster_name = "jobflow" + str(idx)
|
|
args["Name"] = cluster_name
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
client.terminate_job_flows(JobFlowIds=[cluster_id])
|
|
expected[cluster_id] = {
|
|
"Id": cluster_id,
|
|
"Name": cluster_name,
|
|
"NormalizedInstanceHours": 0,
|
|
"State": "TERMINATED",
|
|
}
|
|
|
|
args = {}
|
|
while 1:
|
|
resp = client.list_clusters(**args)
|
|
clusters = resp["Clusters"]
|
|
assert len(clusters) <= 50
|
|
for x in clusters:
|
|
y = expected[x["Id"]]
|
|
assert x["Id"] == y["Id"]
|
|
assert x["Name"] == y["Name"]
|
|
assert x["NormalizedInstanceHours"] == y["NormalizedInstanceHours"]
|
|
assert x["Status"]["State"] == y["State"]
|
|
assert isinstance(x["Status"]["Timeline"]["CreationDateTime"], datetime)
|
|
if y["State"] == "TERMINATED":
|
|
assert isinstance(x["Status"]["Timeline"]["EndDateTime"], datetime)
|
|
else:
|
|
assert "EndDateTime" not in x["Status"]["Timeline"]
|
|
assert isinstance(x["Status"]["Timeline"]["ReadyDateTime"], datetime)
|
|
marker = resp.get("Marker")
|
|
if marker is None:
|
|
break
|
|
args = {"Marker": marker}
|
|
|
|
resp = client.list_clusters(ClusterStates=["TERMINATED"])
|
|
assert len(resp["Clusters"]) == 30
|
|
for x in resp["Clusters"]:
|
|
assert x["Status"]["State"] == "TERMINATED"
|
|
|
|
resp = client.list_clusters(CreatedBefore=timestamp)
|
|
assert len(resp["Clusters"]) == 40
|
|
|
|
resp = client.list_clusters(CreatedAfter=timestamp)
|
|
assert len(resp["Clusters"]) == 30
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow():
|
|
region_name = "us-east-1"
|
|
client = boto3.client("emr", region_name=region_name)
|
|
args = deepcopy(run_job_flow_args)
|
|
resp = client.run_job_flow(**args)
|
|
resp["ClusterArn"].startswith(
|
|
f"arn:aws:elasticmapreduce:{region_name}:{ACCOUNT_ID}:cluster/"
|
|
)
|
|
job_flow_id = resp["JobFlowId"]
|
|
resp = client.describe_job_flows(JobFlowIds=[job_flow_id])["JobFlows"][0]
|
|
assert resp["ExecutionStatusDetail"]["State"] == "WAITING"
|
|
assert resp["JobFlowId"] == job_flow_id
|
|
assert resp["Name"] == args["Name"]
|
|
assert (
|
|
resp["Instances"]["MasterInstanceType"]
|
|
== args["Instances"]["MasterInstanceType"]
|
|
)
|
|
assert (
|
|
resp["Instances"]["SlaveInstanceType"] == args["Instances"]["SlaveInstanceType"]
|
|
)
|
|
assert resp["LogUri"] == args["LogUri"]
|
|
assert resp["VisibleToAllUsers"] == args["VisibleToAllUsers"]
|
|
assert resp["Instances"]["NormalizedInstanceHours"] == 0
|
|
assert resp["Steps"] == []
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow_with_invalid_params():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
with pytest.raises(ClientError) as ex:
|
|
# cannot set both AmiVersion and ReleaseLabel
|
|
args = deepcopy(run_job_flow_args)
|
|
args["AmiVersion"] = "2.4"
|
|
args["ReleaseLabel"] = "emr-5.0.0"
|
|
client.run_job_flow(**args)
|
|
assert ex.value.response["Error"]["Code"] == "ValidationException"
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow_in_multiple_regions():
|
|
regions = {}
|
|
for region in ["us-east-1", "eu-west-1"]:
|
|
client = boto3.client("emr", region_name=region)
|
|
args = deepcopy(run_job_flow_args)
|
|
args["Name"] = region
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
regions[region] = {"client": client, "cluster_id": cluster_id}
|
|
|
|
for region in regions.keys():
|
|
client = regions[region]["client"]
|
|
resp = client.describe_cluster(ClusterId=regions[region]["cluster_id"])
|
|
assert resp["Cluster"]["Name"] == region
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow_with_new_params():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
resp = client.run_job_flow(**run_job_flow_args)
|
|
assert "JobFlowId" in resp
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow_with_visible_to_all_users():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
for expected in (True, False):
|
|
args = deepcopy(run_job_flow_args)
|
|
args["VisibleToAllUsers"] = expected
|
|
resp = client.run_job_flow(**args)
|
|
cluster_id = resp["JobFlowId"]
|
|
resp = client.describe_cluster(ClusterId=cluster_id)
|
|
assert resp["Cluster"]["VisibleToAllUsers"] == expected
|
|
|
|
|
|
def _do_assertion_ebs_configuration(x, y):
|
|
total_volumes = 0
|
|
total_size = 0
|
|
for ebs_block in y["EbsConfiguration"]["EbsBlockDeviceConfigs"]:
|
|
total_volumes += ebs_block["VolumesPerInstance"]
|
|
total_size += ebs_block["VolumeSpecification"]["SizeInGB"]
|
|
# Multiply by total volumes
|
|
total_size = total_size * total_volumes
|
|
comp_total_size = 0
|
|
for ebs_block in x["EbsBlockDevices"]:
|
|
comp_total_size += ebs_block["VolumeSpecification"]["SizeInGB"]
|
|
assert len(x["EbsBlockDevices"]) == total_volumes
|
|
assert comp_total_size == comp_total_size
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow_with_instance_groups():
|
|
input_groups = dict((g["Name"], g) for g in input_instance_groups)
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
args["Instances"] = {"InstanceGroups": input_instance_groups}
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
groups = client.list_instance_groups(ClusterId=cluster_id)["InstanceGroups"]
|
|
for x in groups:
|
|
y = input_groups[x["Name"]]
|
|
assert "Id" in x
|
|
assert x["RequestedInstanceCount"] == y["InstanceCount"]
|
|
assert x["InstanceGroupType"] == y["InstanceRole"]
|
|
assert x["InstanceType"] == y["InstanceType"]
|
|
assert x["Market"] == y["Market"]
|
|
if "BidPrice" in y:
|
|
assert x["BidPrice"] == y["BidPrice"]
|
|
|
|
if "EbsConfiguration" in y:
|
|
_do_assertion_ebs_configuration(x, y)
|
|
|
|
|
|
auto_scaling_policy = {
|
|
"Constraints": {"MinCapacity": 2, "MaxCapacity": 10},
|
|
"Rules": [
|
|
{
|
|
"Name": "Default-scale-out",
|
|
"Description": "Replicates the default scale-out rule in the console for YARN memory.",
|
|
"Action": {
|
|
"SimpleScalingPolicyConfiguration": {
|
|
"AdjustmentType": "CHANGE_IN_CAPACITY",
|
|
"ScalingAdjustment": 1,
|
|
"CoolDown": 300,
|
|
}
|
|
},
|
|
"Trigger": {
|
|
"CloudWatchAlarmDefinition": {
|
|
"ComparisonOperator": "LESS_THAN",
|
|
"EvaluationPeriods": 1,
|
|
"MetricName": "YARNMemoryAvailablePercentage",
|
|
"Namespace": "AWS/ElasticMapReduce",
|
|
"Period": 300,
|
|
"Threshold": 15.0,
|
|
"Statistic": "AVERAGE",
|
|
"Unit": "PERCENT",
|
|
"Dimensions": [{"Key": "JobFlowId", "Value": "${emr.clusterId}"}],
|
|
}
|
|
},
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow_with_instance_groups_with_autoscaling():
|
|
input_groups = dict((g["Name"], g) for g in input_instance_groups)
|
|
|
|
input_groups["core"]["AutoScalingPolicy"] = auto_scaling_policy
|
|
input_groups["task-1"]["AutoScalingPolicy"] = auto_scaling_policy
|
|
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
args["Instances"] = {"InstanceGroups": input_instance_groups}
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
groups = client.list_instance_groups(ClusterId=cluster_id)["InstanceGroups"]
|
|
for x in groups:
|
|
y = deepcopy(input_groups[x["Name"]])
|
|
if "AutoScalingPolicy" in y:
|
|
assert x["AutoScalingPolicy"]["Status"]["State"] == "ATTACHED"
|
|
returned_policy = deepcopy(x["AutoScalingPolicy"])
|
|
auto_scaling_policy_with_cluster_id = (
|
|
_patch_cluster_id_placeholder_in_autoscaling_policy(
|
|
y["AutoScalingPolicy"], cluster_id
|
|
)
|
|
)
|
|
del returned_policy["Status"]
|
|
assert returned_policy == auto_scaling_policy_with_cluster_id
|
|
|
|
|
|
@mock_emr
|
|
def test_put_remove_auto_scaling_policy():
|
|
region_name = "us-east-1"
|
|
client = boto3.client("emr", region_name=region_name)
|
|
args = deepcopy(run_job_flow_args)
|
|
args["Instances"] = {"InstanceGroups": input_instance_groups}
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
|
|
core_instance_group = [
|
|
ig
|
|
for ig in client.list_instance_groups(ClusterId=cluster_id)["InstanceGroups"]
|
|
if ig["InstanceGroupType"] == "CORE"
|
|
][0]
|
|
|
|
resp = client.put_auto_scaling_policy(
|
|
ClusterId=cluster_id,
|
|
InstanceGroupId=core_instance_group["Id"],
|
|
AutoScalingPolicy=auto_scaling_policy,
|
|
)
|
|
|
|
auto_scaling_policy_with_cluster_id = (
|
|
_patch_cluster_id_placeholder_in_autoscaling_policy(
|
|
auto_scaling_policy, cluster_id
|
|
)
|
|
)
|
|
del resp["AutoScalingPolicy"]["Status"]
|
|
assert resp["AutoScalingPolicy"] == auto_scaling_policy_with_cluster_id
|
|
assert (
|
|
resp["ClusterArn"]
|
|
== f"arn:aws:elasticmapreduce:{region_name}:{ACCOUNT_ID}:cluster/{cluster_id}"
|
|
)
|
|
|
|
core_instance_group = [
|
|
ig
|
|
for ig in client.list_instance_groups(ClusterId=cluster_id)["InstanceGroups"]
|
|
if ig["InstanceGroupType"] == "CORE"
|
|
][0]
|
|
|
|
assert "AutoScalingPolicy" in core_instance_group
|
|
|
|
client.remove_auto_scaling_policy(
|
|
ClusterId=cluster_id, InstanceGroupId=core_instance_group["Id"]
|
|
)
|
|
|
|
core_instance_group = [
|
|
ig
|
|
for ig in client.list_instance_groups(ClusterId=cluster_id)["InstanceGroups"]
|
|
if ig["InstanceGroupType"] == "CORE"
|
|
][0]
|
|
|
|
assert "AutoScalingPolicy" not in core_instance_group
|
|
|
|
|
|
def _patch_cluster_id_placeholder_in_autoscaling_policy(policy, cluster_id):
|
|
policy_copy = deepcopy(policy)
|
|
for rule in policy_copy["Rules"]:
|
|
for dimension in rule["Trigger"]["CloudWatchAlarmDefinition"]["Dimensions"]:
|
|
dimension["Value"] = cluster_id
|
|
return policy_copy
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow_with_custom_ami():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
|
|
with pytest.raises(ClientError) as ex:
|
|
# CustomAmiId available in Amazon EMR 5.7.0 and later
|
|
args = deepcopy(run_job_flow_args)
|
|
args["CustomAmiId"] = "MyEmrCustomId"
|
|
args["ReleaseLabel"] = "emr-5.6.0"
|
|
client.run_job_flow(**args)
|
|
assert ex.value.response["Error"]["Code"] == "ValidationException"
|
|
assert ex.value.response["Error"]["Message"] == "Custom AMI is not allowed"
|
|
|
|
with pytest.raises(ClientError) as ex:
|
|
args = deepcopy(run_job_flow_args)
|
|
args["CustomAmiId"] = "MyEmrCustomId"
|
|
args["AmiVersion"] = "3.8.1"
|
|
client.run_job_flow(**args)
|
|
err = ex.value.response["Error"]
|
|
assert err["Code"] == "ValidationException"
|
|
assert err["Message"] == "Custom AMI is not supported in this version of EMR"
|
|
|
|
with pytest.raises(ClientError) as ex:
|
|
# AMI version and release label exception raises before CustomAmi exception
|
|
args = deepcopy(run_job_flow_args)
|
|
args["CustomAmiId"] = "MyEmrCustomId"
|
|
args["ReleaseLabel"] = "emr-5.6.0"
|
|
args["AmiVersion"] = "3.8.1"
|
|
client.run_job_flow(**args)
|
|
err = ex.value.response["Error"]
|
|
assert err["Code"] == "ValidationException"
|
|
assert "Only one AMI version and release label may be specified." in err["Message"]
|
|
|
|
args = deepcopy(run_job_flow_args)
|
|
args["CustomAmiId"] = "MyEmrCustomAmi"
|
|
args["ReleaseLabel"] = "emr-5.31.0"
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
resp = client.describe_cluster(ClusterId=cluster_id)
|
|
assert resp["Cluster"]["CustomAmiId"] == "MyEmrCustomAmi"
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow_with_step_concurrency():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
args["StepConcurrencyLevel"] = 2
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
resp = client.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
|
assert resp["Name"] == args["Name"]
|
|
assert resp["Status"]["State"] == "WAITING"
|
|
assert resp["StepConcurrencyLevel"] == 2
|
|
|
|
|
|
@mock_emr
|
|
def test_modify_cluster():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
args["StepConcurrencyLevel"] = 2
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
resp = client.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
|
assert resp["Name"] == args["Name"]
|
|
assert resp["Status"]["State"] == "WAITING"
|
|
assert resp["StepConcurrencyLevel"] == 2
|
|
|
|
resp = client.modify_cluster(ClusterId=cluster_id, StepConcurrencyLevel=4)
|
|
assert resp["StepConcurrencyLevel"] == 4
|
|
|
|
resp = client.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
|
assert resp["StepConcurrencyLevel"] == 4
|
|
|
|
|
|
@mock_emr
|
|
def test_set_termination_protection():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
args["Instances"]["TerminationProtected"] = False
|
|
resp = client.run_job_flow(**args)
|
|
cluster_id = resp["JobFlowId"]
|
|
resp = client.describe_cluster(ClusterId=cluster_id)
|
|
assert resp["Cluster"]["TerminationProtected"] is False
|
|
|
|
for expected in (True, False):
|
|
resp = client.set_termination_protection(
|
|
JobFlowIds=[cluster_id], TerminationProtected=expected
|
|
)
|
|
resp = client.describe_cluster(ClusterId=cluster_id)
|
|
assert resp["Cluster"]["TerminationProtected"] == expected
|
|
|
|
|
|
@mock_emr
|
|
def test_terminate_protected_job_flow_raises_error():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
resp = client.run_job_flow(**run_job_flow_args)
|
|
cluster_id = resp["JobFlowId"]
|
|
client.set_termination_protection(
|
|
JobFlowIds=[cluster_id], TerminationProtected=True
|
|
)
|
|
with pytest.raises(ClientError) as ex:
|
|
client.terminate_job_flows(JobFlowIds=[cluster_id])
|
|
error = ex.value.response["Error"]
|
|
assert error["Code"] == "ValidationException"
|
|
assert (
|
|
error["Message"]
|
|
== "Could not shut down one or more job flows since they are termination protected."
|
|
)
|
|
|
|
|
|
@mock_emr
|
|
def test_set_visible_to_all_users():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
args["VisibleToAllUsers"] = False
|
|
resp = client.run_job_flow(**args)
|
|
cluster_id = resp["JobFlowId"]
|
|
resp = client.describe_cluster(ClusterId=cluster_id)
|
|
assert resp["Cluster"]["VisibleToAllUsers"] is False
|
|
|
|
for expected in (True, False):
|
|
resp = client.set_visible_to_all_users(
|
|
JobFlowIds=[cluster_id], VisibleToAllUsers=expected
|
|
)
|
|
resp = client.describe_cluster(ClusterId=cluster_id)
|
|
assert resp["Cluster"]["VisibleToAllUsers"] == expected
|
|
|
|
|
|
@mock_emr
|
|
def test_terminate_job_flows():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
|
|
resp = client.run_job_flow(**run_job_flow_args)
|
|
cluster_id = resp["JobFlowId"]
|
|
resp = client.describe_cluster(ClusterId=cluster_id)
|
|
assert resp["Cluster"]["Status"]["State"] == "WAITING"
|
|
|
|
resp = client.terminate_job_flows(JobFlowIds=[cluster_id])
|
|
resp = client.describe_cluster(ClusterId=cluster_id)
|
|
assert resp["Cluster"]["Status"]["State"] == "TERMINATED"
|
|
|
|
|
|
# testing multiple end points for each feature
|
|
|
|
|
|
@mock_emr
|
|
def test_bootstrap_actions():
|
|
bootstrap_actions = [
|
|
{
|
|
"Name": "bs1",
|
|
"ScriptBootstrapAction": {
|
|
"Args": ["arg1", "arg2"],
|
|
"Path": "s3://path/to/script",
|
|
},
|
|
},
|
|
{
|
|
"Name": "bs2",
|
|
"ScriptBootstrapAction": {"Args": [], "Path": "s3://path/to/anotherscript"},
|
|
},
|
|
]
|
|
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
args["BootstrapActions"] = bootstrap_actions
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
|
|
cl = client.describe_job_flows(JobFlowIds=[cluster_id])["JobFlows"][0]
|
|
for x, y in zip(cl["BootstrapActions"], bootstrap_actions):
|
|
assert x["BootstrapActionConfig"] == y
|
|
|
|
resp = client.list_bootstrap_actions(ClusterId=cluster_id)
|
|
for x, y in zip(resp["BootstrapActions"], bootstrap_actions):
|
|
assert x["Name"] == y["Name"]
|
|
if "Args" in y["ScriptBootstrapAction"]:
|
|
assert x["Args"] == y["ScriptBootstrapAction"]["Args"]
|
|
assert x["ScriptPath"] == y["ScriptBootstrapAction"]["Path"]
|
|
|
|
|
|
@mock_emr
|
|
def test_instances():
|
|
input_groups = dict((g["Name"], g) for g in input_instance_groups)
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
args["Instances"] = {"InstanceGroups": input_instance_groups}
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
jf = client.describe_job_flows(JobFlowIds=[cluster_id])["JobFlows"][0]
|
|
instances = client.list_instances(ClusterId=cluster_id)["Instances"]
|
|
assert len(instances) == sum(g["InstanceCount"] for g in input_instance_groups)
|
|
for x in instances:
|
|
assert "InstanceGroupId" in x
|
|
instance_group = [
|
|
j
|
|
for j in jf["Instances"]["InstanceGroups"]
|
|
if j["InstanceGroupId"] == x["InstanceGroupId"]
|
|
]
|
|
assert len(instance_group) == 1
|
|
y = input_groups[instance_group[0]["Name"]]
|
|
assert "Id" in x
|
|
assert "Ec2InstanceId" in x
|
|
assert "PublicDnsName" in x
|
|
assert "PublicIpAddress" in x
|
|
assert "PrivateDnsName" in x
|
|
assert "PrivateIpAddress" in x
|
|
assert "InstanceFleetId" in x
|
|
assert x["InstanceType"] == y["InstanceType"]
|
|
assert x["Market"] == y["Market"]
|
|
assert isinstance(x["Status"]["Timeline"]["ReadyDateTime"], datetime)
|
|
assert isinstance(x["Status"]["Timeline"]["CreationDateTime"], datetime)
|
|
assert x["Status"]["State"] == "RUNNING"
|
|
|
|
for x in [["MASTER"], ["CORE"], ["TASK"], ["MASTER", "TASK"]]:
|
|
instances = client.list_instances(ClusterId=cluster_id, InstanceGroupTypes=x)[
|
|
"Instances"
|
|
]
|
|
assert len(instances) == sum(
|
|
g["InstanceCount"] for g in input_instance_groups if g["InstanceRole"] in x
|
|
)
|
|
|
|
|
|
@mock_emr
|
|
def test_instance_groups():
|
|
input_groups = dict((g["Name"], g) for g in input_instance_groups)
|
|
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
for key in ["MasterInstanceType", "SlaveInstanceType", "InstanceCount"]:
|
|
del args["Instances"][key]
|
|
args["Instances"]["InstanceGroups"] = input_instance_groups[:2]
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
|
|
jf = client.describe_job_flows(JobFlowIds=[cluster_id])["JobFlows"][0]
|
|
base_instance_count = jf["Instances"]["InstanceCount"]
|
|
|
|
instance_groups_to_add = deepcopy(input_instance_groups[2:])
|
|
instance_groups_to_add[0]["AutoScalingPolicy"] = auto_scaling_policy
|
|
instance_groups_to_add[1]["AutoScalingPolicy"] = auto_scaling_policy
|
|
client.add_instance_groups(
|
|
JobFlowId=cluster_id, InstanceGroups=instance_groups_to_add
|
|
)
|
|
|
|
jf = client.describe_job_flows(JobFlowIds=[cluster_id])["JobFlows"][0]
|
|
assert jf["Instances"]["InstanceCount"] == sum(
|
|
g["InstanceCount"] for g in input_instance_groups
|
|
)
|
|
for x in jf["Instances"]["InstanceGroups"]:
|
|
y = input_groups[x["Name"]]
|
|
if "BidPrice" in y:
|
|
assert x["BidPrice"] == y["BidPrice"]
|
|
assert isinstance(x["CreationDateTime"], datetime)
|
|
# assert isinstance(x['EndDateTime'], 'datetime.datetime')
|
|
assert "InstanceGroupId" in x
|
|
assert x["InstanceRequestCount"] == y["InstanceCount"]
|
|
assert x["InstanceRole"] == y["InstanceRole"]
|
|
assert x["InstanceRunningCount"] == y["InstanceCount"]
|
|
assert x["InstanceType"] == y["InstanceType"]
|
|
# assert x['LastStateChangeReason'] == y['LastStateChangeReason']
|
|
assert x["Market"] == y["Market"]
|
|
assert x["Name"] == y["Name"]
|
|
assert isinstance(x["ReadyDateTime"], datetime)
|
|
assert isinstance(x["StartDateTime"], datetime)
|
|
assert x["State"] == "RUNNING"
|
|
groups = client.list_instance_groups(ClusterId=cluster_id)["InstanceGroups"]
|
|
for x in groups:
|
|
y = deepcopy(input_groups[x["Name"]])
|
|
if "BidPrice" in y:
|
|
assert x["BidPrice"] == y["BidPrice"]
|
|
if "AutoScalingPolicy" in y:
|
|
assert x["AutoScalingPolicy"]["Status"]["State"] == "ATTACHED"
|
|
returned_policy = dict(x["AutoScalingPolicy"])
|
|
del returned_policy["Status"]
|
|
policy = json.loads(
|
|
json.dumps(y["AutoScalingPolicy"]).replace(
|
|
"${emr.clusterId}", cluster_id
|
|
)
|
|
)
|
|
assert returned_policy == policy
|
|
if "EbsConfiguration" in y:
|
|
_do_assertion_ebs_configuration(x, y)
|
|
# Configurations
|
|
# EbsBlockDevices
|
|
# EbsOptimized
|
|
assert "Id" in x
|
|
assert x["InstanceGroupType"] == y["InstanceRole"]
|
|
assert x["InstanceType"] == y["InstanceType"]
|
|
assert x["Market"] == y["Market"]
|
|
assert x["Name"] == y["Name"]
|
|
assert x["RequestedInstanceCount"] == y["InstanceCount"]
|
|
assert x["RunningInstanceCount"] == y["InstanceCount"]
|
|
# ShrinkPolicy
|
|
assert x["Status"]["State"] == "RUNNING"
|
|
assert isinstance(x["Status"]["StateChangeReason"]["Code"], str)
|
|
# assert isinstance(x['Status']['StateChangeReason']['Message'], str)
|
|
assert isinstance(x["Status"]["Timeline"]["CreationDateTime"], datetime)
|
|
# assert isinstance(x['Status']['Timeline']['EndDateTime'], 'datetime.datetime')
|
|
assert isinstance(x["Status"]["Timeline"]["ReadyDateTime"], datetime)
|
|
|
|
igs = dict((g["Name"], g) for g in groups)
|
|
client.modify_instance_groups(
|
|
InstanceGroups=[
|
|
{"InstanceGroupId": igs["task-1"]["Id"], "InstanceCount": 2},
|
|
{"InstanceGroupId": igs["task-2"]["Id"], "InstanceCount": 3},
|
|
]
|
|
)
|
|
jf = client.describe_job_flows(JobFlowIds=[cluster_id])["JobFlows"][0]
|
|
assert jf["Instances"]["InstanceCount"] == base_instance_count + 5
|
|
igs = dict((g["Name"], g) for g in jf["Instances"]["InstanceGroups"])
|
|
assert igs["task-1"]["InstanceRunningCount"] == 2
|
|
assert igs["task-2"]["InstanceRunningCount"] == 3
|
|
|
|
|
|
@mock_emr
|
|
def test_steps():
|
|
input_steps = [
|
|
{
|
|
"HadoopJarStep": {
|
|
"Args": [
|
|
"hadoop-streaming",
|
|
"-files",
|
|
"s3://elasticmapreduce/samples/wordcount/wordSplitter.py#wordSplitter.py",
|
|
"-mapper",
|
|
"python wordSplitter.py",
|
|
"-input",
|
|
"s3://elasticmapreduce/samples/wordcount/input",
|
|
"-output",
|
|
"s3://output_bucket/output/wordcount_output",
|
|
"-reducer",
|
|
"aggregate",
|
|
],
|
|
"Jar": "command-runner.jar",
|
|
"Properties": [
|
|
{"Key": "mapred.tasktracker.map.tasks.maximum", "Value": "2"}
|
|
],
|
|
},
|
|
"Name": "My wordcount example",
|
|
},
|
|
{
|
|
"HadoopJarStep": {
|
|
"Args": [
|
|
"hadoop-streaming",
|
|
"-files",
|
|
"s3://elasticmapreduce/samples/wordcount/wordSplitter2.py#wordSplitter2.py",
|
|
"-mapper",
|
|
"python wordSplitter2.py",
|
|
"-input",
|
|
"s3://elasticmapreduce/samples/wordcount/input2",
|
|
"-output",
|
|
"s3://output_bucket/output/wordcount_output2",
|
|
"-reducer",
|
|
"aggregate",
|
|
],
|
|
"Jar": "command-runner.jar",
|
|
"Properties": [
|
|
{"Key": "mapred.reduce.tasks", "Value": "0"},
|
|
{"Key": "stream.map.output.field.separator", "Value": "."},
|
|
],
|
|
},
|
|
"Name": "My wordcount example2",
|
|
},
|
|
]
|
|
|
|
# TODO: implementation and test for cancel_steps
|
|
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
args = deepcopy(run_job_flow_args)
|
|
args["Steps"] = [input_steps[0]]
|
|
cluster_id = client.run_job_flow(**args)["JobFlowId"]
|
|
|
|
jf = client.describe_job_flows(JobFlowIds=[cluster_id])["JobFlows"][0]
|
|
assert len(jf["Steps"]) == 1
|
|
|
|
client.add_job_flow_steps(JobFlowId=cluster_id, Steps=[input_steps[1]])
|
|
|
|
jf = client.describe_job_flows(JobFlowIds=[cluster_id])["JobFlows"][0]
|
|
assert len(jf["Steps"]) == 2
|
|
for idx, (x, y) in enumerate(zip(jf["Steps"], input_steps)):
|
|
assert "CreationDateTime" in x["ExecutionStatusDetail"]
|
|
# assert 'EndDateTime' in x['ExecutionStatusDetail']
|
|
# assert 'LastStateChangeReason' in x['ExecutionStatusDetail']
|
|
# assert 'StartDateTime' in x['ExecutionStatusDetail']
|
|
assert (
|
|
x["ExecutionStatusDetail"]["State"] == "RUNNING" if idx == 0 else "PENDING"
|
|
)
|
|
assert x["StepConfig"]["ActionOnFailure"] == "TERMINATE_CLUSTER"
|
|
assert x["StepConfig"]["HadoopJarStep"]["Args"] == y["HadoopJarStep"]["Args"]
|
|
assert x["StepConfig"]["HadoopJarStep"]["Jar"] == y["HadoopJarStep"]["Jar"]
|
|
if "MainClass" in y["HadoopJarStep"]:
|
|
assert (
|
|
x["StepConfig"]["HadoopJarStep"]["MainClass"]
|
|
== y["HadoopJarStep"]["MainClass"]
|
|
)
|
|
if "Properties" in y["HadoopJarStep"]:
|
|
assert (
|
|
x["StepConfig"]["HadoopJarStep"]["Properties"]
|
|
== y["HadoopJarStep"]["Properties"]
|
|
)
|
|
assert x["StepConfig"]["Name"] == y["Name"]
|
|
|
|
expected = dict((s["Name"], s) for s in input_steps)
|
|
|
|
steps = client.list_steps(ClusterId=cluster_id)["Steps"]
|
|
assert len(steps) == 2
|
|
# Steps should be returned in reverse order.
|
|
assert (
|
|
sorted(
|
|
steps,
|
|
key=lambda o: o["Status"]["Timeline"]["CreationDateTime"],
|
|
reverse=True,
|
|
)
|
|
== steps
|
|
)
|
|
for x in steps:
|
|
y = expected[x["Name"]]
|
|
assert x["ActionOnFailure"] == "TERMINATE_CLUSTER"
|
|
assert x["Config"]["Args"] == y["HadoopJarStep"]["Args"]
|
|
assert x["Config"]["Jar"] == y["HadoopJarStep"]["Jar"]
|
|
# assert x['Config']['MainClass'] == y['HadoopJarStep']['MainClass']
|
|
# Properties
|
|
assert isinstance(x["Id"], str)
|
|
assert x["Name"] == y["Name"]
|
|
assert x["Status"]["State"] in ["RUNNING", "PENDING"]
|
|
# StateChangeReason
|
|
assert isinstance(x["Status"]["Timeline"]["CreationDateTime"], datetime)
|
|
# assert isinstance(x['Status']['Timeline']['EndDateTime'], 'datetime.datetime')
|
|
# Only the first step will have started - we don't know anything about when it finishes, so the second step never starts
|
|
if x["Name"] == "My wordcount example":
|
|
assert isinstance(x["Status"]["Timeline"]["StartDateTime"], datetime)
|
|
|
|
x = client.describe_step(ClusterId=cluster_id, StepId=x["Id"])["Step"]
|
|
assert x["ActionOnFailure"] == "TERMINATE_CLUSTER"
|
|
assert x["Config"]["Args"] == y["HadoopJarStep"]["Args"]
|
|
assert x["Config"]["Jar"] == y["HadoopJarStep"]["Jar"]
|
|
# assert x['Config']['MainClass'] == y['HadoopJarStep']['MainClass']
|
|
# Properties
|
|
assert isinstance(x["Id"], str)
|
|
assert x["Name"] == y["Name"]
|
|
assert x["Status"]["State"] in ["RUNNING", "PENDING"]
|
|
# StateChangeReason
|
|
assert isinstance(x["Status"]["Timeline"]["CreationDateTime"], datetime)
|
|
# assert isinstance(x['Status']['Timeline']['EndDateTime'], 'datetime.datetime')
|
|
# assert isinstance(x['Status']['Timeline']['StartDateTime'], 'datetime.datetime')
|
|
|
|
step_id = steps[-1]["Id"] # Last step is first created step.
|
|
steps = client.list_steps(ClusterId=cluster_id, StepIds=[step_id])["Steps"]
|
|
assert len(steps) == 1
|
|
assert steps[0]["Id"] == step_id
|
|
|
|
steps = client.list_steps(ClusterId=cluster_id, StepStates=["RUNNING"])["Steps"]
|
|
assert len(steps) == 1
|
|
assert steps[0]["Id"] == step_id
|
|
|
|
|
|
@mock_emr
|
|
def test_tags():
|
|
input_tags = [
|
|
{"Key": "newkey1", "Value": "newval1"},
|
|
{"Key": "newkey2", "Value": "newval2"},
|
|
]
|
|
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
cluster_id = client.run_job_flow(**run_job_flow_args)["JobFlowId"]
|
|
|
|
client.add_tags(ResourceId=cluster_id, Tags=input_tags)
|
|
resp = client.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
|
assert len(resp["Tags"]) == 2
|
|
assert {t["Key"]: t["Value"] for t in resp["Tags"]} == {
|
|
t["Key"]: t["Value"] for t in input_tags
|
|
}
|
|
|
|
client.remove_tags(ResourceId=cluster_id, TagKeys=[t["Key"] for t in input_tags])
|
|
resp = client.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
|
assert resp["Tags"] == []
|
|
|
|
|
|
@mock_emr
|
|
def test_security_configurations():
|
|
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
|
|
security_configuration_name = "MySecurityConfiguration"
|
|
|
|
security_configuration = """
|
|
{
|
|
"EncryptionConfiguration": {
|
|
"AtRestEncryptionConfiguration": {
|
|
"S3EncryptionConfiguration": {
|
|
"EncryptionMode": "SSE-S3"
|
|
}
|
|
},
|
|
"EnableInTransitEncryption": false,
|
|
"EnableAtRestEncryption": true
|
|
}
|
|
}
|
|
""".strip()
|
|
|
|
resp = client.create_security_configuration(
|
|
Name=security_configuration_name, SecurityConfiguration=security_configuration
|
|
)
|
|
|
|
assert resp["Name"] == security_configuration_name
|
|
assert isinstance(resp["CreationDateTime"], datetime)
|
|
|
|
resp = client.describe_security_configuration(Name=security_configuration_name)
|
|
assert resp["Name"] == security_configuration_name
|
|
assert resp["SecurityConfiguration"] == security_configuration
|
|
assert isinstance(resp["CreationDateTime"], datetime)
|
|
|
|
client.delete_security_configuration(Name=security_configuration_name)
|
|
|
|
with pytest.raises(ClientError) as ex:
|
|
client.describe_security_configuration(Name=security_configuration_name)
|
|
err = ex.value.response["Error"]
|
|
assert err["Code"] == "InvalidRequestException"
|
|
assert (
|
|
err["Message"]
|
|
== "Security configuration with name 'MySecurityConfiguration' does not exist."
|
|
)
|
|
|
|
with pytest.raises(ClientError) as ex:
|
|
client.delete_security_configuration(Name=security_configuration_name)
|
|
err = ex.value.response["Error"]
|
|
assert err["Code"] == "InvalidRequestException"
|
|
assert (
|
|
err["Message"]
|
|
== "Security configuration with name 'MySecurityConfiguration' does not exist."
|
|
)
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow_with_invalid_number_of_master_nodes_raises_error():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
params = dict(
|
|
Name="test-cluster",
|
|
Instances={
|
|
"InstanceGroups": [
|
|
{
|
|
"InstanceCount": 2,
|
|
"InstanceRole": "MASTER",
|
|
"InstanceType": "c1.medium",
|
|
"Market": "ON_DEMAND",
|
|
"Name": "master",
|
|
}
|
|
]
|
|
},
|
|
)
|
|
with pytest.raises(ClientError) as ex:
|
|
client.run_job_flow(**params)
|
|
error = ex.value.response["Error"]
|
|
assert error["Code"] == "ValidationException"
|
|
assert (
|
|
error["Message"]
|
|
== "Master instance group must have exactly 3 instances for HA clusters."
|
|
)
|
|
|
|
|
|
@mock_emr
|
|
def test_run_job_flow_with_multiple_master_nodes():
|
|
client = boto3.client("emr", region_name="us-east-1")
|
|
params = dict(
|
|
Name="test-cluster",
|
|
Instances={
|
|
"InstanceGroups": [
|
|
{
|
|
"InstanceCount": 3,
|
|
"InstanceRole": "MASTER",
|
|
"InstanceType": "c1.medium",
|
|
"Market": "ON_DEMAND",
|
|
"Name": "master",
|
|
}
|
|
],
|
|
"KeepJobFlowAliveWhenNoSteps": False,
|
|
"TerminationProtected": False,
|
|
},
|
|
)
|
|
cluster_id = client.run_job_flow(**params)["JobFlowId"]
|
|
cluster = client.describe_cluster(ClusterId=cluster_id)["Cluster"]
|
|
assert cluster["AutoTerminate"] is False
|
|
assert cluster["TerminationProtected"] is True
|
|
groups = client.list_instance_groups(ClusterId=cluster_id)["InstanceGroups"]
|
|
master_instance_group = next(
|
|
group for group in groups if group["InstanceGroupType"] == "MASTER"
|
|
)
|
|
assert master_instance_group["RequestedInstanceCount"] == 3
|
|
assert master_instance_group["RunningInstanceCount"] == 3
|