From 80b64f9b3ff57515db1fc07329bf8e5f519597aa Mon Sep 17 00:00:00 2001 From: Zach Brookler <39153813+zbrookle@users.noreply.github.com> Date: Mon, 18 May 2020 04:47:18 -0400 Subject: [PATCH] Cloud formation "depends_on" #2845 Add depends on and update name type mapping (#2994) * ENH: Add unit test for cloudformation DependsOn * ENH: Add implementation of retrieving list of resources that account for dependencies * ENH: Update the name mappings so that they are consistent with the latest cloudformation names * ENH: Add launch configuration to type names * ENH: Create subnet for test and test creation with dependencies * CLN: Code reformatting * CLN: Remove print statements * BUG: Fix error resulting in possible infinite loop * CLN: Remove commented out fixture decorator * BUG: Remove subnet creation * CLN: Remove main and ec2 dependencies * BUG: Add back in instance profile name type * CLN: Remove print * BUG: Fix broken unit test * CLN: Code reformatting * CLN: Remove main * ENH: Add autoscaling group name to type names * ENH: Add unit test for string only dependency and add assertions to unit tests * ENH: Add unit test for chained depends_on in cloudformation stack * BUG: Remove f strings for python 2.7 compatibility * BUG: List needs to be sorted for python2.7 * CLN: Fix code formatting --- moto/cloudformation/parsing.py | 61 +++++++- .../test_cloudformation_depends_on.py | 143 ++++++++++++++++++ .../test_cloudformation_stack_integration.py | 8 +- 3 files changed, 204 insertions(+), 8 deletions(-) create mode 100644 tests/test_cloudformation/test_cloudformation_depends_on.py diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index a32ff6736..d59b21b82 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -98,20 +98,46 @@ MODEL_MAP = { "AWS::Events::Rule": events_models.Rule, } +UNDOCUMENTED_NAME_TYPE_MAP = { + "AWS::AutoScaling::AutoScalingGroup": "AutoScalingGroupName", + "AWS::AutoScaling::LaunchConfiguration": "LaunchConfigurationName", + "AWS::IAM::InstanceProfile": "InstanceProfileName", +} + # http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-name.html NAME_TYPE_MAP = { - "AWS::CloudWatch::Alarm": "Alarm", + "AWS::ApiGateway::ApiKey": "Name", + "AWS::ApiGateway::Model": "Name", + "AWS::CloudWatch::Alarm": "AlarmName", "AWS::DynamoDB::Table": "TableName", - "AWS::ElastiCache::CacheCluster": "ClusterName", "AWS::ElasticBeanstalk::Application": "ApplicationName", "AWS::ElasticBeanstalk::Environment": "EnvironmentName", + "AWS::CodeDeploy::Application": "ApplicationName", + "AWS::CodeDeploy::DeploymentConfig": "DeploymentConfigName", + "AWS::CodeDeploy::DeploymentGroup": "DeploymentGroupName", + "AWS::Config::ConfigRule": "ConfigRuleName", + "AWS::Config::DeliveryChannel": "Name", + "AWS::Config::ConfigurationRecorder": "Name", "AWS::ElasticLoadBalancing::LoadBalancer": "LoadBalancerName", + "AWS::ElasticLoadBalancingV2::LoadBalancer": "Name", "AWS::ElasticLoadBalancingV2::TargetGroup": "Name", + "AWS::EC2::SecurityGroup": "GroupName", + "AWS::ElastiCache::CacheCluster": "ClusterName", + "AWS::ECR::Repository": "RepositoryName", + "AWS::ECS::Cluster": "ClusterName", + "AWS::Elasticsearch::Domain": "DomainName", + "AWS::Events::Rule": "Name", + "AWS::IAM::Group": "GroupName", + "AWS::IAM::ManagedPolicy": "ManagedPolicyName", + "AWS::IAM::Role": "RoleName", + "AWS::IAM::User": "UserName", + "AWS::Lambda::Function": "FunctionName", "AWS::RDS::DBInstance": "DBInstanceIdentifier", "AWS::S3::Bucket": "BucketName", "AWS::SNS::Topic": "TopicName", "AWS::SQS::Queue": "QueueName", } +NAME_TYPE_MAP.update(UNDOCUMENTED_NAME_TYPE_MAP) # Just ignore these models types for now NULL_MODELS = [ @@ -455,6 +481,7 @@ class ResourceMap(collections_abc.Mapping): return self._parsed_resources[resource_logical_id] else: resource_json = self._resource_json_map.get(resource_logical_id) + if not resource_json: raise KeyError(resource_logical_id) new_resource = parse_and_create_resource( @@ -470,6 +497,34 @@ class ResourceMap(collections_abc.Mapping): def __len__(self): return len(self._resource_json_map) + def __get_resources_in_dependency_order(self): + resource_map = copy.deepcopy(self._resource_json_map) + resources_in_dependency_order = [] + + def recursively_get_dependencies(resource): + resource_info = resource_map[resource] + + if "DependsOn" not in resource_info: + resources_in_dependency_order.append(resource) + del resource_map[resource] + return + + dependencies = resource_info["DependsOn"] + if isinstance(dependencies, str): # Dependencies may be a string or list + dependencies = [dependencies] + + for dependency in dependencies: + if dependency in resource_map: + recursively_get_dependencies(dependency) + + resources_in_dependency_order.append(resource) + del resource_map[resource] + + while resource_map: + recursively_get_dependencies(list(resource_map.keys())[0]) + + return resources_in_dependency_order + @property def resources(self): return self._resource_json_map.keys() @@ -547,7 +602,7 @@ class ResourceMap(collections_abc.Mapping): "aws:cloudformation:stack-id": self.get("AWS::StackId"), } ) - for resource in self.resources: + for resource in self.__get_resources_in_dependency_order(): if isinstance(self[resource], ec2_models.TaggedEC2Resource): self.tags["aws:cloudformation:logical-id"] = resource ec2_models.ec2_backends[self._region_name].create_tags( diff --git a/tests/test_cloudformation/test_cloudformation_depends_on.py b/tests/test_cloudformation/test_cloudformation_depends_on.py new file mode 100644 index 000000000..1b47b4064 --- /dev/null +++ b/tests/test_cloudformation/test_cloudformation_depends_on.py @@ -0,0 +1,143 @@ +import boto3 +from moto import mock_cloudformation, mock_ecs, mock_autoscaling, mock_s3 +import json + +depends_on_template_list = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "ECSCluster": { + "Type": "AWS::ECS::Cluster", + "Properties": {"ClusterName": "test-cluster"}, + }, + "AutoScalingGroup": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "test-scaling-group", + "DesiredCapacity": 1, + "MinSize": 1, + "MaxSize": 50, + "LaunchConfigurationName": "test-launch-config", + "AvailabilityZones": ["us-east-1a"], + }, + "DependsOn": ["ECSCluster", "LaunchConfig"], + }, + "LaunchConfig": { + "Type": "AWS::AutoScaling::LaunchConfiguration", + "Properties": {"LaunchConfigurationName": "test-launch-config",}, + }, + }, +} + +depends_on_template_string = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "AutoScalingGroup": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "test-scaling-group", + "DesiredCapacity": 1, + "MinSize": 1, + "MaxSize": 50, + "LaunchConfigurationName": "test-launch-config", + "AvailabilityZones": ["us-east-1a"], + }, + "DependsOn": "LaunchConfig", + }, + "LaunchConfig": { + "Type": "AWS::AutoScaling::LaunchConfiguration", + "Properties": {"LaunchConfigurationName": "test-launch-config",}, + }, + }, +} + + +def make_chained_depends_on_template(): + depends_on_template_linked_dependencies = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "Bucket1": { + "Type": "AWS::S3::Bucket", + "Properties": {"BucketName": "test-bucket-0-us-east-1"}, + }, + }, + } + + for i in range(1, 10): + depends_on_template_linked_dependencies["Resources"]["Bucket" + str(i)] = { + "Type": "AWS::S3::Bucket", + "Properties": {"BucketName": "test-bucket-" + str(i) + "-us-east-1"}, + "DependsOn": ["Bucket" + str(i - 1)], + } + + return json.dumps(depends_on_template_linked_dependencies) + + +depends_on_template_list_json = json.dumps(depends_on_template_list) +depends_on_template_string_json = json.dumps(depends_on_template_string) + + +@mock_cloudformation +@mock_autoscaling +@mock_ecs +def test_create_stack_with_depends_on(): + boto3.client("cloudformation", region_name="us-east-1").create_stack( + StackName="depends_on_test", TemplateBody=depends_on_template_list_json + ) + + autoscaling = boto3.client("autoscaling", region_name="us-east-1") + autoscaling_group = autoscaling.describe_auto_scaling_groups()["AutoScalingGroups"][ + 0 + ] + assert autoscaling_group["AutoScalingGroupName"] == "test-scaling-group" + assert autoscaling_group["DesiredCapacity"] == 1 + assert autoscaling_group["MinSize"] == 1 + assert autoscaling_group["MaxSize"] == 50 + assert autoscaling_group["AvailabilityZones"] == ["us-east-1a"] + + launch_configuration = autoscaling.describe_launch_configurations()[ + "LaunchConfigurations" + ][0] + assert launch_configuration["LaunchConfigurationName"] == "test-launch-config" + + ecs = boto3.client("ecs", region_name="us-east-1") + cluster_arn = ecs.list_clusters()["clusterArns"][0] + assert cluster_arn == "arn:aws:ecs:us-east-1:012345678910:cluster/test-cluster" + + +@mock_cloudformation +@mock_autoscaling +def test_create_stack_with_depends_on_string(): + boto3.client("cloudformation", region_name="us-east-1").create_stack( + StackName="depends_on_string_test", TemplateBody=depends_on_template_string_json + ) + + autoscaling = boto3.client("autoscaling", region_name="us-east-1") + autoscaling_group = autoscaling.describe_auto_scaling_groups()["AutoScalingGroups"][ + 0 + ] + assert autoscaling_group["AutoScalingGroupName"] == "test-scaling-group" + assert autoscaling_group["DesiredCapacity"] == 1 + assert autoscaling_group["MinSize"] == 1 + assert autoscaling_group["MaxSize"] == 50 + assert autoscaling_group["AvailabilityZones"] == ["us-east-1a"] + + launch_configuration = autoscaling.describe_launch_configurations()[ + "LaunchConfigurations" + ][0] + assert launch_configuration["LaunchConfigurationName"] == "test-launch-config" + + +@mock_cloudformation +@mock_s3 +def test_create_chained_depends_on_stack(): + boto3.client("cloudformation", region_name="us-east-1").create_stack( + StackName="linked_depends_on_test", + TemplateBody=make_chained_depends_on_template(), + ) + + s3 = boto3.client("s3", region_name="us-east-1") + bucket_response = s3.list_buckets()["Buckets"] + + assert sorted([bucket["Name"] for bucket in bucket_response]) == [ + "test-bucket-" + str(i) + "-us-east-1" for i in range(1, 10) + ] diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index 3abbab02d..27bac5e57 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -49,7 +49,7 @@ from moto import ( from moto.core import ACCOUNT_ID from moto.dynamodb2.models import Table -from .fixtures import ( +from tests.test_cloudformation.fixtures import ( ec2_classic_eip, fn_join, rds_mysql_with_db_parameter_group, @@ -940,12 +940,10 @@ def test_iam_roles(): role_name_to_id = {} for role_result in role_results: role = iam_conn.get_role(role_result.role_name) - if "my-role" not in role.role_name: + # Role name is not specified, so randomly generated - can't check exact name + if "with-path" in role.role_name: role_name_to_id["with-path"] = role.role_id role.path.should.equal("my-path") - len(role.role_name).should.equal( - 5 - ) # Role name is not specified, so randomly generated - can't check exact name else: role_name_to_id["no-path"] = role.role_id role.role_name.should.equal("my-role-no-path-name")