From 166f4893baded0ae34da73010005be372d37f4cc Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 11 Jan 2015 16:15:08 -0500 Subject: [PATCH] Add cloudformation for RDS. --- moto/cloudformation/parsing.py | 39 ++++-- moto/ec2/models.py | 1 + moto/rds/models.py | 127 +++++++++++++++++- .../test_cloudformation_stack_integration.py | 78 ++++++++++- .../test_cloudformation/test_stack_parsing.py | 11 +- 5 files changed, 238 insertions(+), 18 deletions(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 8650b4f1b..75aade56f 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -1,11 +1,13 @@ from __future__ import unicode_literals import collections +import functools import logging from moto.autoscaling import models as autoscaling_models from moto.ec2 import models as ec2_models from moto.elb import models as elb_models from moto.iam import models as iam_models +from moto.rds import models as rds_models from moto.sqs import models as sqs_models from .utils import random_suffix from .exceptions import MissingParameterError, UnformattedGetAttTemplateException @@ -31,6 +33,9 @@ MODEL_MAP = { "AWS::ElasticLoadBalancing::LoadBalancer": elb_models.FakeLoadBalancer, "AWS::IAM::InstanceProfile": iam_models.InstanceProfile, "AWS::IAM::Role": iam_models.Role, + "AWS::RDS::DBInstance": rds_models.Database, + "AWS::RDS::DBSecurityGroup": rds_models.SecurityGroup, + "AWS::RDS::DBSubnetGroup": rds_models.SubnetGroup, "AWS::SQS::Queue": sqs_models.Queue, } @@ -57,6 +62,15 @@ NULL_MODELS = [ logger = logging.getLogger("moto") +class LazyDict(dict): + def __getitem__(self, key): + val = dict.__getitem__(self, key) + if callable(val): + val = val() + self[key] = val + return val + + def clean_json(resource_json, resources_map): """ Cleanup the a resource dict. For now, this just means replacing any Ref node @@ -99,15 +113,15 @@ def clean_json(resource_json, resources_map): if 'Fn::If' in resource_json: condition_name, true_value, false_value = resource_json['Fn::If'] if resources_map[condition_name]: - return true_value + return clean_json(true_value, resources_map) else: - return false_value + return clean_json(false_value, resources_map) if 'Fn::Join' in resource_json: join_list = [] for val in resource_json['Fn::Join'][1]: cleaned_val = clean_json(val, resources_map) - join_list.append(cleaned_val if cleaned_val else '{0}'.format(val)) + join_list.append('{0}'.format(cleaned_val) if cleaned_val else '{0}'.format(val)) return resource_json['Fn::Join'][0].join(join_list) cleaned_json = {} @@ -168,7 +182,7 @@ def parse_resource(logical_id, resource_json, resources_map, region_name): return resource -def parse_condition(condition, resources_map): +def parse_condition(condition, resources_map, condition_map): if isinstance(condition, bool): return condition @@ -178,22 +192,22 @@ def parse_condition(condition, resources_map): for value in list(condition.values())[0]: # Check if we are referencing another Condition if 'Condition' in value: - condition_values.append(resources_map[value['Condition']]) + condition_values.append(condition_map[value['Condition']]) else: condition_values.append(clean_json(value, resources_map)) if condition_operator == "Fn::Equals": return condition_values[0] == condition_values[1] elif condition_operator == "Fn::Not": - return not parse_condition(condition_values[0], resources_map) + return not parse_condition(condition_values[0], resources_map, condition_map) elif condition_operator == "Fn::And": return all([ - parse_condition(condition_value, resources_map) + parse_condition(condition_value, resources_map, condition_map) for condition_value in condition_values]) elif condition_operator == "Fn::Or": return any([ - parse_condition(condition_value, resources_map) + parse_condition(condition_value, resources_map, condition_map) for condition_value in condition_values]) @@ -227,6 +241,7 @@ class ResourceMap(collections.Mapping): "AWS::Region": self._region_name, "AWS::StackId": stack_id, "AWS::StackName": stack_name, + "AWS::NoValue": None, } def __getitem__(self, key): @@ -273,9 +288,13 @@ class ResourceMap(collections.Mapping): def load_conditions(self): conditions = self._template.get('Conditions', {}) - + lazy_condition_map = LazyDict() for condition_name, condition in conditions.items(): - self._parsed_resources[condition_name] = parse_condition(condition, self._parsed_resources) + lazy_condition_map[condition_name] = functools.partial(parse_condition, + condition, self._parsed_resources, lazy_condition_map) + + for condition_name in lazy_condition_map: + self._parsed_resources[condition_name] = lazy_condition_map[condition_name] def create(self): self.load_mapping() diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 1a5f0a21e..e9246a6bf 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -996,6 +996,7 @@ class SecurityGroup(object): self.egress_rules = [] self.enis = {} self.vpc_id = vpc_id + self.owner_id = "123456789012" @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): diff --git a/moto/rds/models.py b/moto/rds/models.py index 60c009010..2c91a755a 100644 --- a/moto/rds/models.py +++ b/moto/rds/models.py @@ -5,7 +5,10 @@ import copy import boto.rds from jinja2 import Template +from moto.cloudformation.exceptions import UnformattedGetAttTemplateException from moto.core import BaseBackend +from moto.core.utils import get_random_hex +from moto.ec2.models import ec2_backends from .exceptions import DBInstanceNotFoundError, DBSecurityGroupNotFoundError, DBSubnetGroupNotFoundError @@ -19,11 +22,15 @@ class Database(object): self.region = kwargs.get('region') self.engine = kwargs.get("engine") self.engine_version = kwargs.get("engine_version") + if self.engine_version is None: + self.engine_version = "5.6.21" self.iops = kwargs.get("iops") self.storage_type = kwargs.get("storage_type") self.master_username = kwargs.get('master_username') self.master_password = kwargs.get('master_password') self.auto_minor_version_upgrade = kwargs.get('auto_minor_version_upgrade') + if self.auto_minor_version_upgrade is None: + self.auto_minor_version_upgrade = True self.allocated_storage = kwargs.get('allocated_storage') self.db_instance_identifier = kwargs.get('db_instance_identifier') self.source_db_identifier = kwargs.get("source_db_identifier") @@ -32,6 +39,8 @@ class Database(object): self.db_instance_identifier = kwargs.get('db_instance_identifier') self.db_name = kwargs.get("db_name") self.publicly_accessible = kwargs.get("publicly_accessible") + if self.publicly_accessible is None: + self.publicly_accessible = True self.backup_retention_period = kwargs.get("backup_retention_period") if self.backup_retention_period is None: @@ -73,6 +82,58 @@ class Database(object): if value is not None: setattr(self, key, value) + def get_cfn_attribute(self, attribute_name): + if attribute_name == 'Endpoint.Address': + return self.address + elif attribute_name == 'Endpoint.Port': + return self.port + raise UnformattedGetAttTemplateException() + + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + properties = cloudformation_json['Properties'] + + db_instance_identifier = properties.get('DBInstanceIdentifier') + if not db_instance_identifier: + db_instance_identifier = resource_name.lower() + get_random_hex(12) + db_security_groups = properties.get('DBSecurityGroups') + if not db_security_groups: + db_security_groups = [] + security_groups = [group.group_name for group in db_security_groups] + db_subnet_group = properties.get("DBSubnetGroupName") + db_subnet_group_name = db_subnet_group.subnet_name if db_subnet_group else None + db_kwargs = { + "auto_minor_version_upgrade": properties.get('AutoMinorVersionUpgrade'), + "allocated_storage": properties.get('AllocatedStorage'), + "availability_zone": properties.get("AvailabilityZone"), + "backup_retention_period": properties.get("BackupRetentionPeriod"), + "db_instance_class": properties.get('DBInstanceClass'), + "db_instance_identifier": db_instance_identifier, + "db_name": properties.get("DBName"), + "db_subnet_group_name": db_subnet_group_name, + "engine": properties.get("Engine"), + "engine_version": properties.get("EngineVersion"), + "iops": properties.get("Iops"), + "master_password": properties.get('MasterUserPassword'), + "master_username": properties.get('MasterUsername'), + "multi_az": properties.get("MultiAZ"), + "port": properties.get('Port', 3306), + "publicly_accessible": properties.get("PubliclyAccessible"), + "region": region_name, + "security_groups": security_groups, + "storage_type": properties.get("StorageType"), + } + + rds_backend = rds_backends[region_name] + source_db_identifier = properties.get("SourceDBInstanceIdentifier") + if source_db_identifier: + # Replica + db_kwargs["source_db_identifier"] = source_db_identifier.db_instance_identifier + database = rds_backend.create_database_replica(db_kwargs) + else: + database = rds_backend.create_database(db_kwargs) + return database + def to_xml(self): template = Template(""" {{ database.backup_retention_period }} @@ -115,6 +176,7 @@ class Database(object): {% endfor %} + {% if database.db_subnet_group %} {{ database.db_subnet_group.subnet_name }} {{ database.db_subnet_group.description }} @@ -133,6 +195,7 @@ class Database(object): {{ database.db_subnet_group.vpc_id }} + {% endif %} {{ database.publicly_accessible }} {{ database.auto_minor_version_upgrade }} {{ database.allocated_storage }} @@ -150,12 +213,23 @@ class SecurityGroup(object): def __init__(self, group_name, description): self.group_name = group_name self.description = description - self.ip_ranges = [] self.status = "authorized" + self.ip_ranges = [] + self.ec2_security_groups = [] def to_xml(self): template = Template(""" - + + {% for security_group in security_group.ec2_security_groups %} + + {{ security_group.id }} + {{ security_group.name }} + {{ security_group.owner_id }} + authorized + + {% endfor %} + + {{ security_group.description }} {% for ip_range in security_group.ip_ranges %} @@ -170,9 +244,36 @@ class SecurityGroup(object): """) return template.render(security_group=self) - def authorize(self, cidr_ip): + def authorize_cidr(self, cidr_ip): self.ip_ranges.append(cidr_ip) + def authorize_security_group(self, security_group): + self.ec2_security_groups.append(security_group) + + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + properties = cloudformation_json['Properties'] + group_name = resource_name.lower() + get_random_hex(12) + description = properties['GroupDescription'] + security_group_ingress = properties['DBSecurityGroupIngress'] + + ec2_backend = ec2_backends[region_name] + rds_backend = rds_backends[region_name] + security_group = rds_backend.create_security_group( + group_name, + description, + ) + for ingress_type, ingress_value in security_group_ingress.items(): + if ingress_type == "CIDRIP": + security_group.authorize_cidr(ingress_value) + elif ingress_type == "EC2SecurityGroupName": + subnet = ec2_backend.get_security_group_from_name(ingress_value) + security_group.authorize_security_group(subnet) + elif ingress_type == "EC2SecurityGroupId": + subnet = ec2_backend.get_security_group_from_id(ingress_value) + security_group.authorize_security_group(subnet) + return security_group + class SubnetGroup(object): def __init__(self, subnet_name, description, subnets): @@ -204,6 +305,24 @@ class SubnetGroup(object): """) return template.render(subnet_group=self) + @classmethod + def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): + properties = cloudformation_json['Properties'] + + subnet_name = resource_name.lower() + get_random_hex(12) + description = properties['DBSubnetGroupDescription'] + subnet_ids = properties['SubnetIds'] + + ec2_backend = ec2_backends[region_name] + subnets = [ec2_backend.get_subnet(subnet_id) for subnet_id in subnet_ids] + rds_backend = rds_backends[region_name] + subnet_group = rds_backend.create_subnet_group( + subnet_name, + description, + subnets, + ) + return subnet_group + class RDSBackend(BaseBackend): @@ -273,7 +392,7 @@ class RDSBackend(BaseBackend): def authorize_security_group(self, security_group_name, cidr_ip): security_group = self.describe_security_groups(security_group_name)[0] - security_group.authorize(cidr_ip) + security_group.authorize_cidr(cidr_ip) return security_group def create_subnet_group(self, subnet_name, description, subnets): diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index b88be5e70..60353f205 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -18,15 +18,17 @@ from moto import ( mock_ec2, mock_elb, mock_iam, + mock_rds, mock_sqs, ) from .fixtures import ( - single_instance_with_ebs_volume, - vpc_single_instance_in_subnet, ec2_classic_eip, + fn_join, + rds_mysql_with_read_replica, + single_instance_with_ebs_volume, vpc_eip, - fn_join + vpc_single_instance_in_subnet, ) @@ -350,6 +352,76 @@ def test_vpc_single_instance_in_subnet(): eip_resource.physical_resource_id.should.equal(eip.allocation_id) +@mock_cloudformation() +@mock_ec2() +@mock_rds() +def test_rds_mysql_with_read_replica(): + ec2_conn = boto.ec2.connect_to_region("us-west-1") + ec2_conn.create_security_group('application', 'Our Application Group') + + template_json = json.dumps(rds_mysql_with_read_replica.template) + conn = boto.cloudformation.connect_to_region("us-west-1") + conn.create_stack( + "test_stack", + template_body=template_json, + parameters=[ + ("DBInstanceIdentifier", "master_db"), + ("DBName", "my_db"), + ("DBUser", "my_user"), + ("DBPassword", "my_password"), + ("DBAllocatedStorage", "20"), + ("DBInstanceClass", "db.m1.medium"), + ("EC2SecurityGroup", "application"), + ("MultiAZ", "true"), + ], + ) + + rds_conn = boto.rds.connect_to_region("us-west-1") + + primary = rds_conn.get_all_dbinstances("master_db")[0] + primary.master_username.should.equal("my_user") + primary.allocated_storage.should.equal(20) + primary.instance_class.should.equal("db.m1.medium") + primary.multi_az.should.equal(True) + list(primary.read_replica_dbinstance_identifiers).should.have.length_of(1) + replica_id = primary.read_replica_dbinstance_identifiers[0] + + replica = rds_conn.get_all_dbinstances(replica_id)[0] + replica.instance_class.should.equal("db.m1.medium") + + security_group_name = primary.security_groups[0].name + security_group = rds_conn.get_all_dbsecurity_groups(security_group_name)[0] + security_group.ec2_groups[0].name.should.equal("application") + + +@mock_cloudformation() +@mock_ec2() +@mock_rds() +def test_rds_mysql_with_read_replica_in_vpc(): + template_json = json.dumps(rds_mysql_with_read_replica.template) + conn = boto.cloudformation.connect_to_region("eu-central-1") + conn.create_stack( + "test_stack", + template_body=template_json, + parameters=[ + ("DBInstanceIdentifier", "master_db"), + ("DBName", "my_db"), + ("DBUser", "my_user"), + ("DBPassword", "my_password"), + ("DBAllocatedStorage", "20"), + ("DBInstanceClass", "db.m1.medium"), + ("MultiAZ", "true"), + ], + ) + + rds_conn = boto.rds.connect_to_region("eu-central-1") + primary = rds_conn.get_all_dbinstances("master_db")[0] + + subnet_group_name = primary.subnet_group.name + subnet_group = rds_conn.get_all_db_subnet_groups(subnet_group_name)[0] + subnet_group.description.should.equal("my db subnet group") + + @mock_autoscaling() @mock_iam() @mock_cloudformation() diff --git a/tests/test_cloudformation/test_stack_parsing.py b/tests/test_cloudformation/test_stack_parsing.py index 13d602ebb..29d8282fd 100644 --- a/tests/test_cloudformation/test_stack_parsing.py +++ b/tests/test_cloudformation/test_stack_parsing.py @@ -151,11 +151,13 @@ def test_parse_equals_condition(): parse_condition( condition={"Fn::Equals": [{"Ref": "EnvType"}, "prod"]}, resources_map={"EnvType": "prod"}, + condition_map={}, ).should.equal(True) parse_condition( condition={"Fn::Equals": [{"Ref": "EnvType"}, "prod"]}, resources_map={"EnvType": "staging"}, + condition_map={}, ).should.equal(False) @@ -167,6 +169,7 @@ def test_parse_not_condition(): }] }, resources_map={"EnvType": "prod"}, + condition_map={}, ).should.equal(False) parse_condition( @@ -176,6 +179,7 @@ def test_parse_not_condition(): }] }, resources_map={"EnvType": "staging"}, + condition_map={}, ).should.equal(True) @@ -188,6 +192,7 @@ def test_parse_and_condition(): ] }, resources_map={"EnvType": "prod"}, + condition_map={}, ).should.equal(False) parse_condition( @@ -198,6 +203,7 @@ def test_parse_and_condition(): ] }, resources_map={"EnvType": "prod"}, + condition_map={}, ).should.equal(True) @@ -210,6 +216,7 @@ def test_parse_or_condition(): ] }, resources_map={"EnvType": "prod"}, + condition_map={}, ).should.equal(True) parse_condition( @@ -220,11 +227,13 @@ def test_parse_or_condition(): ] }, resources_map={"EnvType": "prod"}, + condition_map={}, ).should.equal(False) def test_reference_other_conditions(): parse_condition( condition={"Fn::Not": [{"Condition": "OtherCondition"}]}, - resources_map={"OtherCondition": True}, + resources_map={}, + condition_map={"OtherCondition": True}, ).should.equal(False)