From 63f01039c3f321c6f726c620eba6ec66e98f55ec Mon Sep 17 00:00:00 2001 From: Jack Danger Canty Date: Tue, 20 Jun 2017 13:51:25 -0700 Subject: [PATCH 1/5] Implementing RDS Snapshots --- moto/rds2/exceptions.py | 8 ++++ moto/rds2/models.py | 81 +++++++++++++++++++++++++++++++++++- moto/rds2/responses.py | 59 +++++++++++++++++++++++++- tests/test_rds2/test_rds2.py | 75 +++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 2 deletions(-) diff --git a/moto/rds2/exceptions.py b/moto/rds2/exceptions.py index 29e92941d..057a13ba2 100644 --- a/moto/rds2/exceptions.py +++ b/moto/rds2/exceptions.py @@ -28,6 +28,14 @@ class DBInstanceNotFoundError(RDSClientError): "Database {0} not found.".format(database_identifier)) +class DBSnapshotNotFoundError(RDSClientError): + + def __init__(self): + super(DBSnapshotNotFoundError, self).__init__( + 'DBSnapshotNotFound', + "DBSnapshotIdentifier does not refer to an existing DB snapshot.") + + class DBSecurityGroupNotFoundError(RDSClientError): def __init__(self, security_group_name): diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 4036cdcd1..ae97ba1f2 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import copy +import datetime from collections import defaultdict import boto.rds2 @@ -10,9 +11,11 @@ from moto.cloudformation.exceptions import UnformattedGetAttTemplateException from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import get_random_hex +from moto.core.utils import iso_8601_datetime_with_milliseconds from moto.ec2.models import ec2_backends from .exceptions import (RDSClientError, DBInstanceNotFoundError, + DBSnapshotNotFoundError, DBSecurityGroupNotFoundError, DBSubnetGroupNotFoundError, DBParameterGroupNotFoundError) @@ -205,7 +208,7 @@ class Database(BaseModel): {% endif %} {% if database.iops %} {{ database.iops }} - io1 + standard {% else %} {{ database.storage_type }} {% endif %} @@ -399,6 +402,53 @@ class Database(BaseModel): backend.delete_database(self.db_instance_identifier) +class Snapshot(BaseModel): + def __init__(self, database, snapshot_id, tags): + self.database = database + self.snapshot_id = snapshot_id + self.tags = tags + self.created_at = iso_8601_datetime_with_milliseconds(datetime.datetime.now()) + + @property + def snapshot_arn(self): + return "arn:aws:rds:{0}:1234567890:snapshot:{1}".format(self.database.region, self.snapshot_id) + + def to_xml(self): + template = Template(""" + {{ snapshot.snapshot_id }} + {{ database.db_instance_identifier }} + {{ snapshot.created_at }} + {{ database.engine }} + {{ database.allocated_storage }} + available + {{ database.port }} + {{ database.availability_zone }} + {{ database.db_subnet_group.vpc_id }} + {{ snapshot.created_at }} + {{ database.master_username }} + {{ database.engine_version }} + general-public-license + manual + {% if database.iops %} + {{ database.iops }} + io1 + {% else %} + {{ database.storage_type }} + {% endif %} + {{ database.option_group_name }} + {{ 100 }} + {{ database.region }} + + + {{ database.storage_encrypted }} + {{ database.kms_key_id }} + {{ snapshot.snapshot_arn }} + + false + """) + return template.render(snapshot=self, database=self.database) + + class SecurityGroup(BaseModel): def __init__(self, group_name, description, tags): @@ -607,6 +657,7 @@ class RDS2Backend(BaseBackend): self.arn_regex = re_compile( r'^arn:aws:rds:.*:[0-9]*:(db|es|og|pg|ri|secgrp|snapshot|subgrp):.*$') self.databases = OrderedDict() + self.snapshots = OrderedDict() self.db_parameter_groups = {} self.option_groups = {} self.security_groups = {} @@ -624,6 +675,20 @@ class RDS2Backend(BaseBackend): self.databases[database_id] = database return database + def create_snapshot(self, db_instance_identifier, db_snapshot_identifier, tags): + database = self.databases.get(db_instance_identifier) + if not database: + raise DBInstanceNotFoundError(db_instance_identifier) + snapshot = Snapshot(database, db_snapshot_identifier, tags) + self.snapshots[db_snapshot_identifier] = snapshot + return snapshot + + def delete_snapshot(self, db_snapshot_identifier): + if db_snapshot_identifier not in self.snapshots: + raise DBSnapshotNotFoundError() + + return self.snapshots.pop(db_snapshot_identifier) + def create_database_replica(self, db_kwargs): database_id = db_kwargs['db_instance_identifier'] source_database_id = db_kwargs['source_db_identifier'] @@ -646,6 +711,20 @@ class RDS2Backend(BaseBackend): raise DBInstanceNotFoundError(db_instance_identifier) return self.databases.values() + def describe_snapshots(self, db_instance_identifier, db_snapshot_identifier): + if db_instance_identifier: + for snapshot in self.snapshots.values(): + if snapshot.database.db_instance_identifier == db_instance_identifier: + return [snapshot] + raise DBSnapshotNotFoundError() + + if db_snapshot_identifier: + if db_snapshot_identifier in self.snapshots: + return [self.snapshots[db_snapshot_identifier]] + raise DBSnapshotNotFoundError() + + return self.snapshots.values() + def modify_database(self, db_instance_identifier, db_kwargs): database = self.describe_databases(db_instance_identifier)[0] database.update(db_kwargs) diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index f8f33f2b9..cdadd3424 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -39,7 +39,7 @@ class RDS2Response(BaseResponse): "region": self.region, "security_groups": self._get_multi_param('DBSecurityGroups.DBSecurityGroupName'), "storage_encrypted": self._get_param("StorageEncrypted"), - "storage_type": self._get_param("StorageType"), + "storage_type": self._get_param("StorageType", 'standard'), # VpcSecurityGroupIds.member.N "tags": list(), } @@ -150,6 +150,27 @@ class RDS2Response(BaseResponse): template = self.response_template(REBOOT_DATABASE_TEMPLATE) return template.render(database=database) + def create_db_snapshot(self): + db_instance_identifier = self._get_param('DBInstanceIdentifier') + db_snapshot_identifier = self._get_param('DBSnapshotIdentifier') + tags = self._get_param('Tags', []) + snapshot = self.backend.create_snapshot(db_instance_identifier, db_snapshot_identifier, tags) + template = self.response_template(CREATE_SNAPSHOT_TEMPLATE) + return template.render(snapshot=snapshot) + + def describe_db_snapshots(self): + db_instance_identifier = self._get_param('DBInstanceIdentifier') + db_snapshot_identifier = self._get_param('DBSnapshotIdentifier') + snapshots = self.backend.describe_snapshots(db_instance_identifier, db_snapshot_identifier) + template = self.response_template(DESCRIBE_SNAPSHOTS_TEMPLATE) + return template.render(snapshots=snapshots) + + def delete_db_snapshot(self): + db_snapshot_identifier = self._get_param('DBSnapshotIdentifier') + snapshot = self.backend.delete_snapshot(db_snapshot_identifier) + template = self.response_template(DELETE_SNAPSHOT_TEMPLATE) + return template.render(snapshot=snapshot) + def list_tags_for_resource(self): arn = self._get_param('ResourceName') template = self.response_template(LIST_TAGS_FOR_RESOURCE_TEMPLATE) @@ -397,6 +418,42 @@ DELETE_DATABASE_TEMPLATE = """ + + {{ snapshot.to_xml() }} + + + 523e3218-afc7-11c3-90f5-f90431260ab4 + + +""" + +DESCRIBE_SNAPSHOTS_TEMPLATE = """ + + + {%- for snapshot in snapshots -%} + {{ snapshot.to_xml() }} + {%- endfor -%} + + {% if marker %} + {{ marker }} + {% endif %} + + + 523e3218-afc7-11c3-90f5-f90431260ab4 + +""" + +DELETE_SNAPSHOT_TEMPLATE = """ + + {{ snapshot.to_xml() }} + + + 523e3218-afc7-11c3-90f5-f90431260ab4 + + +""" + CREATE_SECURITY_GROUP_TEMPLATE = """ {{ security_group.to_xml() }} diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 148b00aa1..7a801257c 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -168,6 +168,81 @@ def test_delete_non_existant_database(): DBInstanceIdentifier="not-a-db").should.throw(ClientError) +@mock_rds2 +def test_create_db_snapshots(): + conn = boto3.client('rds', region_name='us-west-2') + conn.create_db_snapshot.when.called_with( + DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-1').should.throw(ClientError) + + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"]) + + snapshot = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='g-1').get('DBSnapshot') + + snapshot.get('Engine').should.equal('postgres') + snapshot.get('DBInstanceIdentifier').should.equal('db-primary-1') + snapshot.get('DBSnapshotIdentifier').should.equal('g-1') + + +@mock_rds2 +def test_describe_db_snapshots(): + conn = boto3.client('rds', region_name='us-west-2') + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"]) + conn.describe_db_snapshots.when.called_with( + DBInstanceIdentifier="db-primary-1").should.throw(ClientError) + + created = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-1').get('DBSnapshot') + + created.get('Engine').should.equal('postgres') + + by_database_id = conn.describe_db_snapshots(DBInstanceIdentifier='db-primary-1').get('DBSnapshots') + by_snapshot_id = conn.describe_db_snapshots(DBSnapshotIdentifier='snapshot-1').get('DBSnapshots') + by_snapshot_id.should.equal(by_database_id) + + snapshot = by_snapshot_id[0] + snapshot.should.equal(created) + snapshot.get('Engine').should.equal('postgres') + + +@mock_rds2 +def test_delete_db_snapshot(): + conn = boto3.client('rds', region_name='us-west-2') + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"]) + conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-1') + + conn.describe_db_snapshots(DBSnapshotIdentifier='snapshot-1').get('DBSnapshots')[0] + conn.delete_db_snapshot(DBSnapshotIdentifier='snapshot-1') + conn.describe_db_snapshots.when.called_with( + DBSnapshotIdentifier='snapshot-1').should.throw(ClientError) + + @mock_rds2 def test_create_option_group(): conn = boto3.client('rds', region_name='us-west-2') From ccb4ffde7c4f5bede92a4601f2832316a3335d56 Mon Sep 17 00:00:00 2001 From: Jack Danger Canty Date: Tue, 20 Jun 2017 13:53:22 -0700 Subject: [PATCH 2/5] Supporting io1 type --- moto/rds2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index ae97ba1f2..2d9a66401 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -208,7 +208,7 @@ class Database(BaseModel): {% endif %} {% if database.iops %} {{ database.iops }} - standard + io1 {% else %} {{ database.storage_type }} {% endif %} From fb2efb1c6dca45b6781fa4e2d977735aeedc2d9a Mon Sep 17 00:00:00 2001 From: Jack Danger Canty Date: Tue, 20 Jun 2017 14:00:56 -0700 Subject: [PATCH 3/5] Implementing snapshots on rds delete --- moto/rds/responses.py | 3 ++- moto/rds2/models.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/moto/rds/responses.py b/moto/rds/responses.py index 0895a8bf2..cdcbe3603 100644 --- a/moto/rds/responses.py +++ b/moto/rds/responses.py @@ -114,7 +114,8 @@ class RDSResponse(BaseResponse): def delete_db_instance(self): db_instance_identifier = self._get_param('DBInstanceIdentifier') - database = self.backend.delete_database(db_instance_identifier) + db_snapshot_name = self._get_param('FinalDBSnapshotIdentifier') + database = self.backend.delete_database(db_instance_identifier, db_snapshot_name) template = self.response_template(DELETE_DATABASE_TEMPLATE) return template.render(database=database) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 2d9a66401..549f6e247 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -403,10 +403,10 @@ class Database(BaseModel): class Snapshot(BaseModel): - def __init__(self, database, snapshot_id, tags): + def __init__(self, database, snapshot_id, tags=None): self.database = database self.snapshot_id = snapshot_id - self.tags = tags + self.tags = tags or [] self.created_at = iso_8601_datetime_with_milliseconds(datetime.datetime.now()) @property @@ -746,13 +746,15 @@ class RDS2Backend(BaseBackend): return backend.describe_databases(db_name)[0] - def delete_database(self, db_instance_identifier): + def delete_database(self, db_instance_identifier, db_snapshot_name): if db_instance_identifier in self.databases: database = self.databases.pop(db_instance_identifier) if database.is_replica: primary = self.find_db_from_id(database.source_db_identifier) primary.remove_replica(database) database.status = 'deleting' + if db_snapshot_name: + self.snapshots[db_snapshot_name] = Snapshot(database, db_snapshot_name) return database else: raise DBInstanceNotFoundError(db_instance_identifier) From 8df7169915cf4e894133c7640ea8758e2eddda6a Mon Sep 17 00:00:00 2001 From: Jack Danger Canty Date: Tue, 20 Jun 2017 14:01:28 -0700 Subject: [PATCH 4/5] Snapshots are optional --- moto/rds2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 549f6e247..86f7bae9a 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -746,7 +746,7 @@ class RDS2Backend(BaseBackend): return backend.describe_databases(db_name)[0] - def delete_database(self, db_instance_identifier, db_snapshot_name): + def delete_database(self, db_instance_identifier, db_snapshot_name=None): if db_instance_identifier in self.databases: database = self.databases.pop(db_instance_identifier) if database.is_replica: From e57798cb96966b83e42fb9ab3a9cc74f32879084 Mon Sep 17 00:00:00 2001 From: Jack Danger Canty Date: Tue, 20 Jun 2017 14:46:13 -0700 Subject: [PATCH 5/5] Implementing snapshots on rds instance deletion --- moto/rds/responses.py | 3 +-- moto/rds2/responses.py | 3 ++- tests/test_rds2/test_rds2.py | 14 ++++++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/moto/rds/responses.py b/moto/rds/responses.py index cdcbe3603..0895a8bf2 100644 --- a/moto/rds/responses.py +++ b/moto/rds/responses.py @@ -114,8 +114,7 @@ class RDSResponse(BaseResponse): def delete_db_instance(self): db_instance_identifier = self._get_param('DBInstanceIdentifier') - db_snapshot_name = self._get_param('FinalDBSnapshotIdentifier') - database = self.backend.delete_database(db_instance_identifier, db_snapshot_name) + database = self.backend.delete_database(db_instance_identifier) template = self.response_template(DELETE_DATABASE_TEMPLATE) return template.render(database=database) diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index cdadd3424..b26f2e347 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -140,7 +140,8 @@ class RDS2Response(BaseResponse): def delete_db_instance(self): db_instance_identifier = self._get_param('DBInstanceIdentifier') - database = self.backend.delete_database(db_instance_identifier) + db_snapshot_name = self._get_param('FinalDBSnapshotIdentifier') + database = self.backend.delete_database(db_instance_identifier, db_snapshot_name) template = self.response_template(DELETE_DATABASE_TEMPLATE) return template.render(database=database) diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 7a801257c..f869dc1ce 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -145,10 +145,10 @@ def test_delete_database(): conn = boto3.client('rds', region_name='us-west-2') instances = conn.describe_db_instances() list(instances['DBInstances']).should.have.length_of(0) - conn.create_db_instance(DBInstanceIdentifier='db-master-1', + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', AllocatedStorage=10, - DBInstanceClass='postgres', - Engine='db.m1.small', + Engine='postgres', + DBInstanceClass='db.m1.small', MasterUsername='root', MasterUserPassword='hunter2', Port=1234, @@ -156,10 +156,16 @@ def test_delete_database(): instances = conn.describe_db_instances() list(instances['DBInstances']).should.have.length_of(1) - conn.delete_db_instance(DBInstanceIdentifier="db-master-1") + conn.delete_db_instance(DBInstanceIdentifier="db-primary-1", + FinalDBSnapshotIdentifier='primary-1-snapshot') + instances = conn.describe_db_instances() list(instances['DBInstances']).should.have.length_of(0) + # Saved the snapshot + snapshots = conn.describe_db_snapshots(DBInstanceIdentifier="db-primary-1").get('DBSnapshots') + snapshots[0].get('Engine').should.equal('postgres') + @mock_rds2 def test_delete_non_existant_database():