diff --git a/tests/test_batch/__init__.py b/tests/test_batch/__init__.py index 08a1c1568..52713db50 100644 --- a/tests/test_batch/__init__.py +++ b/tests/test_batch/__init__.py @@ -1 +1,43 @@ -# This file is intentionally left blank. +import boto3 + + +DEFAULT_REGION = "eu-central-1" + + +def _get_clients(): + return ( + boto3.client("ec2", region_name=DEFAULT_REGION), + boto3.client("iam", region_name=DEFAULT_REGION), + boto3.client("ecs", region_name=DEFAULT_REGION), + boto3.client("logs", region_name=DEFAULT_REGION), + boto3.client("batch", region_name=DEFAULT_REGION), + ) + + +def _setup(ec2_client, iam_client): + """ + Do prerequisite setup + :return: VPC ID, Subnet ID, Security group ID, IAM Role ARN + :rtype: tuple + """ + resp = ec2_client.create_vpc(CidrBlock="172.30.0.0/24") + vpc_id = resp["Vpc"]["VpcId"] + resp = ec2_client.create_subnet( + AvailabilityZone="eu-central-1a", CidrBlock="172.30.0.0/25", VpcId=vpc_id + ) + subnet_id = resp["Subnet"]["SubnetId"] + resp = ec2_client.create_security_group( + Description="test_sg_desc", GroupName="test_sg", VpcId=vpc_id + ) + sg_id = resp["GroupId"] + + resp = iam_client.create_role( + RoleName="TestRole", AssumeRolePolicyDocument="some_policy" + ) + iam_arn = resp["Role"]["Arn"] + iam_client.create_instance_profile(InstanceProfileName="TestRole") + iam_client.add_role_to_instance_profile( + InstanceProfileName="TestRole", RoleName="TestRole" + ) + + return vpc_id, subnet_id, sg_id, iam_arn diff --git a/tests/test_batch/test_batch_compute_envs.py b/tests/test_batch/test_batch_compute_envs.py new file mode 100644 index 000000000..2e577b89f --- /dev/null +++ b/tests/test_batch/test_batch_compute_envs.py @@ -0,0 +1,203 @@ +from __future__ import unicode_literals + +from . import _get_clients, _setup +import sure # noqa +from moto import mock_batch, mock_iam, mock_ec2, mock_ecs, mock_logs + + +# Yes, yes it talks to all the things +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_create_managed_compute_environment(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + resp = batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="MANAGED", + state="ENABLED", + computeResources={ + "type": "EC2", + "minvCpus": 5, + "maxvCpus": 10, + "desiredvCpus": 5, + "instanceTypes": ["t2.small", "t2.medium"], + "imageId": "some_image_id", + "subnets": [subnet_id], + "securityGroupIds": [sg_id], + "ec2KeyPair": "string", + "instanceRole": iam_arn.replace("role", "instance-profile"), + "tags": {"string": "string"}, + "bidPercentage": 123, + "spotIamFleetRole": "string", + }, + serviceRole=iam_arn, + ) + resp.should.contain("computeEnvironmentArn") + resp["computeEnvironmentName"].should.equal(compute_name) + + # Given a t2.medium is 2 vcpu and t2.small is 1, therefore 2 mediums and 1 small should be created + resp = ec2_client.describe_instances() + resp.should.contain("Reservations") + len(resp["Reservations"]).should.equal(3) + + # Should have created 1 ECS cluster + resp = ecs_client.list_clusters() + resp.should.contain("clusterArns") + len(resp["clusterArns"]).should.equal(1) + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_create_unmanaged_compute_environment(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + resp = batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="UNMANAGED", + state="ENABLED", + serviceRole=iam_arn, + ) + resp.should.contain("computeEnvironmentArn") + resp["computeEnvironmentName"].should.equal(compute_name) + + # Its unmanaged so no instances should be created + resp = ec2_client.describe_instances() + resp.should.contain("Reservations") + len(resp["Reservations"]).should.equal(0) + + # Should have created 1 ECS cluster + resp = ecs_client.list_clusters() + resp.should.contain("clusterArns") + len(resp["clusterArns"]).should.equal(1) + + +# TODO create 1000s of tests to test complex option combinations of create environment + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_describe_compute_environment(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="UNMANAGED", + state="ENABLED", + serviceRole=iam_arn, + ) + + resp = batch_client.describe_compute_environments() + len(resp["computeEnvironments"]).should.equal(1) + resp["computeEnvironments"][0]["computeEnvironmentName"].should.equal(compute_name) + + # Test filtering + resp = batch_client.describe_compute_environments(computeEnvironments=["test1"]) + len(resp["computeEnvironments"]).should.equal(0) + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_delete_unmanaged_compute_environment(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="UNMANAGED", + state="ENABLED", + serviceRole=iam_arn, + ) + + batch_client.delete_compute_environment(computeEnvironment=compute_name) + + resp = batch_client.describe_compute_environments() + len(resp["computeEnvironments"]).should.equal(0) + + resp = ecs_client.list_clusters() + len(resp.get("clusterArns", [])).should.equal(0) + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_delete_managed_compute_environment(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="MANAGED", + state="ENABLED", + computeResources={ + "type": "EC2", + "minvCpus": 5, + "maxvCpus": 10, + "desiredvCpus": 5, + "instanceTypes": ["t2.small", "t2.medium"], + "imageId": "some_image_id", + "subnets": [subnet_id], + "securityGroupIds": [sg_id], + "ec2KeyPair": "string", + "instanceRole": iam_arn.replace("role", "instance-profile"), + "tags": {"string": "string"}, + "bidPercentage": 123, + "spotIamFleetRole": "string", + }, + serviceRole=iam_arn, + ) + + batch_client.delete_compute_environment(computeEnvironment=compute_name) + + resp = batch_client.describe_compute_environments() + len(resp["computeEnvironments"]).should.equal(0) + + resp = ec2_client.describe_instances() + resp.should.contain("Reservations") + len(resp["Reservations"]).should.equal(3) + for reservation in resp["Reservations"]: + reservation["Instances"][0]["State"]["Name"].should.equal("terminated") + + resp = ecs_client.list_clusters() + len(resp.get("clusterArns", [])).should.equal(0) + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_update_unmanaged_compute_environment_state(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="UNMANAGED", + state="ENABLED", + serviceRole=iam_arn, + ) + + batch_client.update_compute_environment( + computeEnvironment=compute_name, state="DISABLED" + ) + + resp = batch_client.describe_compute_environments() + len(resp["computeEnvironments"]).should.equal(1) + resp["computeEnvironments"][0]["state"].should.equal("DISABLED") diff --git a/tests/test_batch/test_batch_job_queue.py b/tests/test_batch/test_batch_job_queue.py new file mode 100644 index 000000000..41612baf8 --- /dev/null +++ b/tests/test_batch/test_batch_job_queue.py @@ -0,0 +1,225 @@ +from . import _get_clients, _setup + +from botocore.exceptions import ClientError +import pytest +import sure # noqa +from moto import mock_batch, mock_iam, mock_ec2, mock_ecs + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_create_job_queue(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + resp = batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="UNMANAGED", + state="ENABLED", + serviceRole=iam_arn, + ) + arn = resp["computeEnvironmentArn"] + + resp = batch_client.create_job_queue( + jobQueueName="test_job_queue", + state="ENABLED", + priority=123, + computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], + ) + resp.should.contain("jobQueueArn") + resp.should.contain("jobQueueName") + queue_arn = resp["jobQueueArn"] + + resp = batch_client.describe_job_queues() + resp.should.have.key("jobQueues").being.length_of(1) + resp["jobQueues"][0]["jobQueueArn"].should.equal(queue_arn) + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_describe_job_queue_unknown_value(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + + resp = batch_client.describe_job_queues(jobQueues=["test_invalid_queue"]) + resp.should.have.key("jobQueues").being.length_of(0) + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_create_job_queue_twice(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + resp = batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="UNMANAGED", + state="ENABLED", + serviceRole=iam_arn, + ) + compute_env_arn = resp["computeEnvironmentArn"] + + batch_client.create_job_queue( + jobQueueName="test_job_queue", + state="ENABLED", + priority=123, + computeEnvironmentOrder=[{"order": 123, "computeEnvironment": compute_env_arn}], + ) + with pytest.raises(ClientError) as ex: + batch_client.create_job_queue( + jobQueueName="test_job_queue", + state="ENABLED", + priority=123, + computeEnvironmentOrder=[ + {"order": 123, "computeEnvironment": compute_env_arn} + ], + ) + + err = ex.value.response["Error"] + err["Code"].should.equal("ClientException") + err["Message"].should.equal("Job queue test_job_queue already exists") + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_create_job_queue_incorrect_state(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + + with pytest.raises(ClientError) as ex: + batch_client.create_job_queue( + jobQueueName="test_job_queue2", + state="JUNK", + priority=123, + computeEnvironmentOrder=[], + ) + err = ex.value.response["Error"] + err["Code"].should.equal("ClientException") + err["Message"].should.equal("state JUNK must be one of ENABLED | DISABLED") + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_create_job_queue_without_compute_environment(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + + with pytest.raises(ClientError) as ex: + batch_client.create_job_queue( + jobQueueName="test_job_queue3", + state="ENABLED", + priority=123, + computeEnvironmentOrder=[], + ) + err = ex.value.response["Error"] + err["Code"].should.equal("ClientException") + err["Message"].should.equal("At least 1 compute environment must be provided") + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_job_queue_bad_arn(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + resp = batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="UNMANAGED", + state="ENABLED", + serviceRole=iam_arn, + ) + arn = resp["computeEnvironmentArn"] + + with pytest.raises(ClientError) as ex: + batch_client.create_job_queue( + jobQueueName="test_job_queue", + state="ENABLED", + priority=123, + computeEnvironmentOrder=[ + {"order": 123, "computeEnvironment": arn + "LALALA"} + ], + ) + err = ex.value.response["Error"] + err["Code"].should.equal("ClientException") + err["Message"].should.equal("computeEnvironmentOrder is malformed") + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_update_job_queue(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + resp = batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="UNMANAGED", + state="ENABLED", + serviceRole=iam_arn, + ) + arn = resp["computeEnvironmentArn"] + + resp = batch_client.create_job_queue( + jobQueueName="test_job_queue", + state="ENABLED", + priority=123, + computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], + ) + queue_arn = resp["jobQueueArn"] + + batch_client.update_job_queue(jobQueue=queue_arn, priority=5) + + resp = batch_client.describe_job_queues() + resp.should.have.key("jobQueues").being.length_of(1) + resp["jobQueues"][0]["priority"].should.equal(5) + + batch_client.update_job_queue(jobQueue="test_job_queue", priority=5) + + resp = batch_client.describe_job_queues() + resp.should.have.key("jobQueues").being.length_of(1) + resp["jobQueues"][0]["priority"].should.equal(5) + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_delete_job_queue(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = "test_compute_env" + resp = batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type="UNMANAGED", + state="ENABLED", + serviceRole=iam_arn, + ) + arn = resp["computeEnvironmentArn"] + + resp = batch_client.create_job_queue( + jobQueueName="test_job_queue", + state="ENABLED", + priority=123, + computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], + ) + queue_arn = resp["jobQueueArn"] + + batch_client.delete_job_queue(jobQueue=queue_arn) + + resp = batch_client.describe_job_queues() + resp.should.have.key("jobQueues").being.length_of(0) diff --git a/tests/test_batch/test_batch.py b/tests/test_batch/test_batch_jobs.py similarity index 51% rename from tests/test_batch/test_batch.py rename to tests/test_batch/test_batch_jobs.py index 04554383d..09b3dd329 100644 --- a/tests/test_batch/test_batch.py +++ b/tests/test_batch/test_batch_jobs.py @@ -1,606 +1,10 @@ -from __future__ import unicode_literals +from . import _get_clients, _setup -import time import datetime -import boto3 -from botocore.exceptions import ClientError import sure # noqa from moto import mock_batch, mock_iam, mock_ec2, mock_ecs, mock_logs import pytest - -DEFAULT_REGION = "eu-central-1" - - -def _get_clients(): - return ( - boto3.client("ec2", region_name=DEFAULT_REGION), - boto3.client("iam", region_name=DEFAULT_REGION), - boto3.client("ecs", region_name=DEFAULT_REGION), - boto3.client("logs", region_name=DEFAULT_REGION), - boto3.client("batch", region_name=DEFAULT_REGION), - ) - - -def _setup(ec2_client, iam_client): - """ - Do prerequisite setup - :return: VPC ID, Subnet ID, Security group ID, IAM Role ARN - :rtype: tuple - """ - resp = ec2_client.create_vpc(CidrBlock="172.30.0.0/24") - vpc_id = resp["Vpc"]["VpcId"] - resp = ec2_client.create_subnet( - AvailabilityZone="eu-central-1a", CidrBlock="172.30.0.0/25", VpcId=vpc_id - ) - subnet_id = resp["Subnet"]["SubnetId"] - resp = ec2_client.create_security_group( - Description="test_sg_desc", GroupName="test_sg", VpcId=vpc_id - ) - sg_id = resp["GroupId"] - - resp = iam_client.create_role( - RoleName="TestRole", AssumeRolePolicyDocument="some_policy" - ) - iam_arn = resp["Role"]["Arn"] - iam_client.create_instance_profile(InstanceProfileName="TestRole") - iam_client.add_role_to_instance_profile( - InstanceProfileName="TestRole", RoleName="TestRole" - ) - - return vpc_id, subnet_id, sg_id, iam_arn - - -# Yes, yes it talks to all the things -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_create_managed_compute_environment(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - compute_name = "test_compute_env" - resp = batch_client.create_compute_environment( - computeEnvironmentName=compute_name, - type="MANAGED", - state="ENABLED", - computeResources={ - "type": "EC2", - "minvCpus": 5, - "maxvCpus": 10, - "desiredvCpus": 5, - "instanceTypes": ["t2.small", "t2.medium"], - "imageId": "some_image_id", - "subnets": [subnet_id], - "securityGroupIds": [sg_id], - "ec2KeyPair": "string", - "instanceRole": iam_arn.replace("role", "instance-profile"), - "tags": {"string": "string"}, - "bidPercentage": 123, - "spotIamFleetRole": "string", - }, - serviceRole=iam_arn, - ) - resp.should.contain("computeEnvironmentArn") - resp["computeEnvironmentName"].should.equal(compute_name) - - # Given a t2.medium is 2 vcpu and t2.small is 1, therefore 2 mediums and 1 small should be created - resp = ec2_client.describe_instances() - resp.should.contain("Reservations") - len(resp["Reservations"]).should.equal(3) - - # Should have created 1 ECS cluster - resp = ecs_client.list_clusters() - resp.should.contain("clusterArns") - len(resp["clusterArns"]).should.equal(1) - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_create_unmanaged_compute_environment(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - compute_name = "test_compute_env" - resp = batch_client.create_compute_environment( - computeEnvironmentName=compute_name, - type="UNMANAGED", - state="ENABLED", - serviceRole=iam_arn, - ) - resp.should.contain("computeEnvironmentArn") - resp["computeEnvironmentName"].should.equal(compute_name) - - # Its unmanaged so no instances should be created - resp = ec2_client.describe_instances() - resp.should.contain("Reservations") - len(resp["Reservations"]).should.equal(0) - - # Should have created 1 ECS cluster - resp = ecs_client.list_clusters() - resp.should.contain("clusterArns") - len(resp["clusterArns"]).should.equal(1) - - -# TODO create 1000s of tests to test complex option combinations of create environment - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_describe_compute_environment(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - compute_name = "test_compute_env" - batch_client.create_compute_environment( - computeEnvironmentName=compute_name, - type="UNMANAGED", - state="ENABLED", - serviceRole=iam_arn, - ) - - resp = batch_client.describe_compute_environments() - len(resp["computeEnvironments"]).should.equal(1) - resp["computeEnvironments"][0]["computeEnvironmentName"].should.equal(compute_name) - - # Test filtering - resp = batch_client.describe_compute_environments(computeEnvironments=["test1"]) - len(resp["computeEnvironments"]).should.equal(0) - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_delete_unmanaged_compute_environment(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - compute_name = "test_compute_env" - batch_client.create_compute_environment( - computeEnvironmentName=compute_name, - type="UNMANAGED", - state="ENABLED", - serviceRole=iam_arn, - ) - - batch_client.delete_compute_environment(computeEnvironment=compute_name) - - resp = batch_client.describe_compute_environments() - len(resp["computeEnvironments"]).should.equal(0) - - resp = ecs_client.list_clusters() - len(resp.get("clusterArns", [])).should.equal(0) - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_delete_managed_compute_environment(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - compute_name = "test_compute_env" - batch_client.create_compute_environment( - computeEnvironmentName=compute_name, - type="MANAGED", - state="ENABLED", - computeResources={ - "type": "EC2", - "minvCpus": 5, - "maxvCpus": 10, - "desiredvCpus": 5, - "instanceTypes": ["t2.small", "t2.medium"], - "imageId": "some_image_id", - "subnets": [subnet_id], - "securityGroupIds": [sg_id], - "ec2KeyPair": "string", - "instanceRole": iam_arn.replace("role", "instance-profile"), - "tags": {"string": "string"}, - "bidPercentage": 123, - "spotIamFleetRole": "string", - }, - serviceRole=iam_arn, - ) - - batch_client.delete_compute_environment(computeEnvironment=compute_name) - - resp = batch_client.describe_compute_environments() - len(resp["computeEnvironments"]).should.equal(0) - - resp = ec2_client.describe_instances() - resp.should.contain("Reservations") - len(resp["Reservations"]).should.equal(3) - for reservation in resp["Reservations"]: - reservation["Instances"][0]["State"]["Name"].should.equal("terminated") - - resp = ecs_client.list_clusters() - len(resp.get("clusterArns", [])).should.equal(0) - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_update_unmanaged_compute_environment_state(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - compute_name = "test_compute_env" - batch_client.create_compute_environment( - computeEnvironmentName=compute_name, - type="UNMANAGED", - state="ENABLED", - serviceRole=iam_arn, - ) - - batch_client.update_compute_environment( - computeEnvironment=compute_name, state="DISABLED" - ) - - resp = batch_client.describe_compute_environments() - len(resp["computeEnvironments"]).should.equal(1) - resp["computeEnvironments"][0]["state"].should.equal("DISABLED") - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_create_job_queue(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - compute_name = "test_compute_env" - resp = batch_client.create_compute_environment( - computeEnvironmentName=compute_name, - type="UNMANAGED", - state="ENABLED", - serviceRole=iam_arn, - ) - arn = resp["computeEnvironmentArn"] - - resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", - state="ENABLED", - priority=123, - computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], - ) - resp.should.contain("jobQueueArn") - resp.should.contain("jobQueueName") - queue_arn = resp["jobQueueArn"] - - resp = batch_client.describe_job_queues() - resp.should.contain("jobQueues") - len(resp["jobQueues"]).should.equal(1) - resp["jobQueues"][0]["jobQueueArn"].should.equal(queue_arn) - - resp = batch_client.describe_job_queues(jobQueues=["test_invalid_queue"]) - resp.should.contain("jobQueues") - len(resp["jobQueues"]).should.equal(0) - - # Create job queue which already exists - try: - resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", - state="ENABLED", - priority=123, - computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], - ) - - except ClientError as err: - err.response["Error"]["Code"].should.equal("ClientException") - - # Create job queue with incorrect state - try: - resp = batch_client.create_job_queue( - jobQueueName="test_job_queue2", - state="JUNK", - priority=123, - computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], - ) - - except ClientError as err: - err.response["Error"]["Code"].should.equal("ClientException") - - # Create job queue with no compute env - try: - resp = batch_client.create_job_queue( - jobQueueName="test_job_queue3", - state="JUNK", - priority=123, - computeEnvironmentOrder=[], - ) - - except ClientError as err: - err.response["Error"]["Code"].should.equal("ClientException") - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_job_queue_bad_arn(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - compute_name = "test_compute_env" - resp = batch_client.create_compute_environment( - computeEnvironmentName=compute_name, - type="UNMANAGED", - state="ENABLED", - serviceRole=iam_arn, - ) - arn = resp["computeEnvironmentArn"] - - try: - batch_client.create_job_queue( - jobQueueName="test_job_queue", - state="ENABLED", - priority=123, - computeEnvironmentOrder=[ - {"order": 123, "computeEnvironment": arn + "LALALA"} - ], - ) - except ClientError as err: - err.response["Error"]["Code"].should.equal("ClientException") - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_update_job_queue(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - compute_name = "test_compute_env" - resp = batch_client.create_compute_environment( - computeEnvironmentName=compute_name, - type="UNMANAGED", - state="ENABLED", - serviceRole=iam_arn, - ) - arn = resp["computeEnvironmentArn"] - - resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", - state="ENABLED", - priority=123, - computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], - ) - queue_arn = resp["jobQueueArn"] - - batch_client.update_job_queue(jobQueue=queue_arn, priority=5) - - resp = batch_client.describe_job_queues() - resp.should.contain("jobQueues") - len(resp["jobQueues"]).should.equal(1) - resp["jobQueues"][0]["priority"].should.equal(5) - - batch_client.update_job_queue(jobQueue="test_job_queue", priority=5) - - resp = batch_client.describe_job_queues() - resp.should.contain("jobQueues") - len(resp["jobQueues"]).should.equal(1) - resp["jobQueues"][0]["priority"].should.equal(5) - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_update_job_queue(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - compute_name = "test_compute_env" - resp = batch_client.create_compute_environment( - computeEnvironmentName=compute_name, - type="UNMANAGED", - state="ENABLED", - serviceRole=iam_arn, - ) - arn = resp["computeEnvironmentArn"] - - resp = batch_client.create_job_queue( - jobQueueName="test_job_queue", - state="ENABLED", - priority=123, - computeEnvironmentOrder=[{"order": 123, "computeEnvironment": arn}], - ) - queue_arn = resp["jobQueueArn"] - - batch_client.delete_job_queue(jobQueue=queue_arn) - - resp = batch_client.describe_job_queues() - resp.should.contain("jobQueues") - len(resp["jobQueues"]).should.equal(0) - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_register_task_definition(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - resp = batch_client.register_job_definition( - jobDefinitionName="sleep10", - type="container", - containerProperties={ - "image": "busybox", - "vcpus": 1, - "memory": 128, - "command": ["sleep", "10"], - }, - ) - - resp.should.contain("jobDefinitionArn") - resp.should.contain("jobDefinitionName") - resp.should.contain("revision") - - assert resp["jobDefinitionArn"].endswith( - "{0}:{1}".format(resp["jobDefinitionName"], resp["revision"]) - ) - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_reregister_task_definition(): - # Reregistering task with the same name bumps the revision number - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - resp1 = batch_client.register_job_definition( - jobDefinitionName="sleep10", - type="container", - containerProperties={ - "image": "busybox", - "vcpus": 1, - "memory": 128, - "command": ["sleep", "10"], - }, - ) - - resp1.should.contain("jobDefinitionArn") - resp1.should.contain("jobDefinitionName") - resp1.should.contain("revision") - - assert resp1["jobDefinitionArn"].endswith( - "{0}:{1}".format(resp1["jobDefinitionName"], resp1["revision"]) - ) - resp1["revision"].should.equal(1) - - resp2 = batch_client.register_job_definition( - jobDefinitionName="sleep10", - type="container", - containerProperties={ - "image": "busybox", - "vcpus": 1, - "memory": 68, - "command": ["sleep", "10"], - }, - ) - resp2["revision"].should.equal(2) - - resp2["jobDefinitionArn"].should_not.equal(resp1["jobDefinitionArn"]) - - resp3 = batch_client.register_job_definition( - jobDefinitionName="sleep10", - type="container", - containerProperties={ - "image": "busybox", - "vcpus": 1, - "memory": 42, - "command": ["sleep", "10"], - }, - ) - resp3["revision"].should.equal(3) - - resp3["jobDefinitionArn"].should_not.equal(resp1["jobDefinitionArn"]) - resp3["jobDefinitionArn"].should_not.equal(resp2["jobDefinitionArn"]) - - resp4 = batch_client.register_job_definition( - jobDefinitionName="sleep10", - type="container", - containerProperties={ - "image": "busybox", - "vcpus": 1, - "memory": 41, - "command": ["sleep", "10"], - }, - ) - resp4["revision"].should.equal(4) - - resp4["jobDefinitionArn"].should_not.equal(resp1["jobDefinitionArn"]) - resp4["jobDefinitionArn"].should_not.equal(resp2["jobDefinitionArn"]) - resp4["jobDefinitionArn"].should_not.equal(resp3["jobDefinitionArn"]) - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_delete_task_definition(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - resp = batch_client.register_job_definition( - jobDefinitionName="sleep10", - type="container", - containerProperties={ - "image": "busybox", - "vcpus": 1, - "memory": 128, - "command": ["sleep", "10"], - }, - ) - - batch_client.deregister_job_definition(jobDefinition=resp["jobDefinitionArn"]) - - resp = batch_client.describe_job_definitions() - len(resp["jobDefinitions"]).should.equal(0) - - -@mock_ec2 -@mock_ecs -@mock_iam -@mock_batch -def test_describe_task_definition(): - ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() - vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) - - batch_client.register_job_definition( - jobDefinitionName="sleep10", - type="container", - containerProperties={ - "image": "busybox", - "vcpus": 1, - "memory": 128, - "command": ["sleep", "10"], - }, - ) - batch_client.register_job_definition( - jobDefinitionName="sleep10", - type="container", - containerProperties={ - "image": "busybox", - "vcpus": 1, - "memory": 64, - "command": ["sleep", "10"], - }, - ) - batch_client.register_job_definition( - jobDefinitionName="test1", - type="container", - containerProperties={ - "image": "busybox", - "vcpus": 1, - "memory": 64, - "command": ["sleep", "10"], - }, - ) - - resp = batch_client.describe_job_definitions(jobDefinitionName="sleep10") - len(resp["jobDefinitions"]).should.equal(2) - - resp = batch_client.describe_job_definitions() - len(resp["jobDefinitions"]).should.equal(3) - - resp = batch_client.describe_job_definitions(jobDefinitions=["sleep10", "test1"]) - len(resp["jobDefinitions"]).should.equal(3) - - for job_definition in resp["jobDefinitions"]: - job_definition["status"].should.equal("ACTIVE") +import time @mock_logs @@ -730,7 +134,7 @@ def test_submit_job(): resp = logs_client.describe_log_streams( logGroupName="/aws/batch/job", logStreamNamePrefix="sayhellotomylittlefriend" ) - len(resp["logStreams"]).should.equal(1) + resp["logStreams"].should.have.length_of(1) ls_name = resp["logStreams"][0]["logStreamName"] resp = logs_client.get_log_events( @@ -787,20 +191,17 @@ def test_list_jobs(): ) job_id2 = resp["jobId"] - resp_finished_jobs = batch_client.list_jobs( - jobQueue=queue_arn, jobStatus="SUCCEEDED" - ) + batch_client.list_jobs(jobQueue=queue_arn, jobStatus="SUCCEEDED")[ + "jobSummaryList" + ].should.have.length_of(0) # Wait only as long as it takes to run the jobs for job_id in [job_id1, job_id2]: _wait_for_job_status(batch_client, job_id, "SUCCEEDED") - resp_finished_jobs2 = batch_client.list_jobs( - jobQueue=queue_arn, jobStatus="SUCCEEDED" - ) - - len(resp_finished_jobs["jobSummaryList"]).should.equal(0) - len(resp_finished_jobs2["jobSummaryList"]).should.equal(2) + batch_client.list_jobs(jobQueue=queue_arn, jobStatus="SUCCEEDED")[ + "jobSummaryList" + ].should.have.length_of(2) @mock_logs @@ -860,7 +261,7 @@ def test_terminate_job(): resp = logs_client.describe_log_streams( logGroupName="/aws/batch/job", logStreamNamePrefix="echo-sleep-echo" ) - len(resp["logStreams"]).should.equal(1) + resp["logStreams"].should.have.length_of(1) ls_name = resp["logStreams"][0]["logStreamName"] resp = logs_client.get_log_events( diff --git a/tests/test_batch/test_batch_task_definition.py b/tests/test_batch/test_batch_task_definition.py new file mode 100644 index 000000000..07c2c4690 --- /dev/null +++ b/tests/test_batch/test_batch_task_definition.py @@ -0,0 +1,116 @@ +from . import _get_clients, _setup +import random +import sure # noqa +from moto import mock_batch, mock_iam, mock_ec2, mock_ecs + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_register_task_definition(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + _setup(ec2_client, iam_client) + + resp = register_job_def(batch_client) + + resp.should.contain("jobDefinitionArn") + resp.should.contain("jobDefinitionName") + resp.should.contain("revision") + + assert resp["jobDefinitionArn"].endswith( + "{0}:{1}".format(resp["jobDefinitionName"], resp["revision"]) + ) + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_reregister_task_definition(): + # Reregistering task with the same name bumps the revision number + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + _setup(ec2_client, iam_client) + + resp1 = register_job_def(batch_client) + + resp1.should.contain("jobDefinitionArn") + resp1.should.contain("jobDefinitionName") + resp1.should.contain("revision") + + assert resp1["jobDefinitionArn"].endswith( + "{0}:{1}".format(resp1["jobDefinitionName"], resp1["revision"]) + ) + resp1["revision"].should.equal(1) + + resp2 = register_job_def(batch_client) + resp2["revision"].should.equal(2) + + resp2["jobDefinitionArn"].should_not.equal(resp1["jobDefinitionArn"]) + + resp3 = register_job_def(batch_client) + resp3["revision"].should.equal(3) + + resp3["jobDefinitionArn"].should_not.equal(resp1["jobDefinitionArn"]) + resp3["jobDefinitionArn"].should_not.equal(resp2["jobDefinitionArn"]) + + resp4 = register_job_def(batch_client) + resp4["revision"].should.equal(4) + + resp4["jobDefinitionArn"].should_not.equal(resp1["jobDefinitionArn"]) + resp4["jobDefinitionArn"].should_not.equal(resp2["jobDefinitionArn"]) + resp4["jobDefinitionArn"].should_not.equal(resp3["jobDefinitionArn"]) + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_delete_task_definition(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + _setup(ec2_client, iam_client) + + resp = register_job_def(batch_client) + + batch_client.deregister_job_definition(jobDefinition=resp["jobDefinitionArn"]) + + resp = batch_client.describe_job_definitions() + len(resp["jobDefinitions"]).should.equal(0) + + +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_describe_task_definition(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + _setup(ec2_client, iam_client) + + register_job_def(batch_client, definition_name="sleep10") + register_job_def(batch_client, definition_name="sleep10") + register_job_def(batch_client, definition_name="test1") + + resp = batch_client.describe_job_definitions(jobDefinitionName="sleep10") + len(resp["jobDefinitions"]).should.equal(2) + + resp = batch_client.describe_job_definitions() + len(resp["jobDefinitions"]).should.equal(3) + + resp = batch_client.describe_job_definitions(jobDefinitions=["sleep10", "test1"]) + len(resp["jobDefinitions"]).should.equal(3) + + for job_definition in resp["jobDefinitions"]: + job_definition["status"].should.equal("ACTIVE") + + +def register_job_def(batch_client, definition_name="sleep10"): + return batch_client.register_job_definition( + jobDefinitionName=definition_name, + type="container", + containerProperties={ + "image": "busybox", + "vcpus": 1, + "memory": random.randint(4, 128), + "command": ["sleep", "10"], + }, + )