diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index f67cdca6a..c037f2bc7 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3127,18 +3127,18 @@ ## dynamodb
-44% implemented +54% implemented - [ ] batch_execute_statement - [X] batch_get_item - [X] batch_write_item -- [ ] create_backup +- [X] create_backup - [ ] create_global_table - [X] create_table -- [ ] delete_backup +- [X] delete_backup - [X] delete_item - [X] delete_table -- [ ] describe_backup +- [X] describe_backup - [X] describe_continuous_backups - [ ] describe_contributor_insights - [ ] describe_endpoints @@ -3156,7 +3156,7 @@ - [ ] execute_transaction - [ ] export_table_to_point_in_time - [X] get_item -- [ ] list_backups +- [X] list_backups - [ ] list_contributor_insights - [ ] list_exports - [ ] list_global_tables @@ -3164,7 +3164,7 @@ - [X] list_tags_of_resource - [X] put_item - [X] query -- [ ] restore_table_from_backup +- [X] restore_table_from_backup - [ ] restore_table_to_point_in_time - [X] scan - [X] tag_resource diff --git a/moto/dynamodb2/models/__init__.py b/moto/dynamodb2/models/__init__.py index d83a526b9..06bd3e857 100644 --- a/moto/dynamodb2/models/__init__.py +++ b/moto/dynamodb2/models/__init__.py @@ -9,8 +9,9 @@ import uuid from boto3 import Session from moto.compat import OrderedDict +from moto.core import ACCOUNT_ID from moto.core import BaseBackend, BaseModel, CloudFormationModel -from moto.core.utils import unix_time +from moto.core.utils import unix_time, unix_time_millis from moto.core.exceptions import JsonRESTError from moto.dynamodb2.comparisons import get_filter_expression from moto.dynamodb2.comparisons import get_expected @@ -969,10 +970,112 @@ class Table(CloudFormationModel): dynamodb_backends[region_name].delete_table(self.name) +class RestoredTable(Table): + def __init__(self, name, backup): + params = self._parse_params_from_backup(backup) + super(RestoredTable, self).__init__(name, **params) + self.indexes = copy.deepcopy(backup.table.indexes) + self.global_indexes = copy.deepcopy(backup.table.global_indexes) + self.items = copy.deepcopy(backup.table.items) + # Restore Attrs + self.source_backup_arn = backup.arn + self.source_table_arn = backup.table.table_arn + self.restore_date_time = self.created_at + + @staticmethod + def _parse_params_from_backup(backup): + params = { + "schema": copy.deepcopy(backup.table.schema), + "attr": copy.deepcopy(backup.table.attr), + "throughput": copy.deepcopy(backup.table.throughput), + } + return params + + def describe(self, base_key="TableDescription"): + result = super(RestoredTable, self).describe(base_key=base_key) + result[base_key]["RestoreSummary"] = { + "SourceBackupArn": self.source_backup_arn, + "SourceTableArn": self.source_table_arn, + "RestoreDateTime": unix_time(self.restore_date_time), + "RestoreInProgress": False, + } + return result + + +class Backup(object): + def __init__( + self, backend, name, table, status=None, type_=None, + ): + self.backend = backend + self.name = name + self.table = copy.deepcopy(table) + self.status = status or "AVAILABLE" + self.type = type_ or "USER" + self.creation_date_time = datetime.datetime.utcnow() + self.identifier = self._make_identifier() + + def _make_identifier(self): + timestamp = int(unix_time_millis(self.creation_date_time)) + timestamp_padded = str("0" + str(timestamp))[-16:16] + guid = str(uuid.uuid4()) + guid_shortened = guid[:8] + return "{}-{}".format(timestamp_padded, guid_shortened) + + @property + def arn(self): + return "arn:aws:dynamodb:{region}:{account}:table/{table_name}/backup/{identifier}".format( + region=self.backend.region_name, + account=ACCOUNT_ID, + table_name=self.table.name, + identifier=self.identifier, + ) + + @property + def details(self): + details = { + "BackupArn": self.arn, + "BackupName": self.name, + "BackupSizeBytes": 123, + "BackupStatus": self.status, + "BackupType": self.type, + "BackupCreationDateTime": unix_time(self.creation_date_time), + } + return details + + @property + def summary(self): + summary = { + "TableName": self.table.name, + # 'TableId': 'string', + "TableArn": self.table.table_arn, + "BackupArn": self.arn, + "BackupName": self.name, + "BackupCreationDateTime": unix_time(self.creation_date_time), + # 'BackupExpiryDateTime': datetime(2015, 1, 1), + "BackupStatus": self.status, + "BackupType": self.type, + "BackupSizeBytes": 123, + } + return summary + + @property + def description(self): + source_table_details = self.table.describe()["TableDescription"] + source_table_details["TableCreationDateTime"] = source_table_details[ + "CreationDateTime" + ] + description = { + "BackupDetails": self.details, + "SourceTableDetails": source_table_details, + } + return description + + class DynamoDBBackend(BaseBackend): def __init__(self, region_name=None): self.region_name = region_name self.tables = OrderedDict() + self.backups = OrderedDict() def reset(self): region_name = self.region_name @@ -1505,6 +1608,48 @@ class DynamoDBBackend(BaseBackend): return table.continuous_backups + def get_backup(self, backup_arn): + return self.backups.get(backup_arn) + + def list_backups(self, table_name): + backups = list(self.backups.values()) + if table_name is not None: + backups = [backup for backup in backups if backup.table.name == table_name] + return backups + + def create_backup(self, table_name, backup_name): + table = self.get_table(table_name) + if table is None: + raise KeyError() + backup = Backup(self, backup_name, table) + self.backups[backup.arn] = backup + return backup + + def delete_backup(self, backup_arn): + backup = self.get_backup(backup_arn) + if backup is None: + raise KeyError() + backup_deleted = self.backups.pop(backup_arn) + backup_deleted.status = "DELETED" + return backup_deleted + + def describe_backup(self, backup_arn): + backup = self.get_backup(backup_arn) + if backup is None: + raise KeyError() + return backup + + def restore_table_from_backup(self, target_table_name, backup_arn): + backup = self.get_backup(backup_arn) + if backup is None: + raise KeyError() + existing_table = self.get_table(target_table_name) + if existing_table is not None: + raise ValueError() + new_table = RestoredTable(target_table_name, backup) + self.tables[target_table_name] = new_table + return new_table + ###################### # LIST of methods where the logic completely resides in responses.py # Duplicated here so that the implementation coverage script is aware diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 63857eae5..e147141eb 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -986,3 +986,60 @@ class DynamoHandler(BaseResponse): ) return json.dumps({"ContinuousBackupsDescription": response}) + + def list_backups(self): + body = self.body + table_name = body.get("TableName") + backups = self.dynamodb_backend.list_backups(table_name) + response = {"BackupSummaries": [backup.summary for backup in backups]} + return dynamo_json_dump(response) + + def create_backup(self): + body = self.body + table_name = body.get("TableName") + backup_name = body.get("BackupName") + try: + backup = self.dynamodb_backend.create_backup(table_name, backup_name) + response = {"BackupDetails": backup.details} + return dynamo_json_dump(response) + except KeyError: + er = "com.amazonaws.dynamodb.v20111205#TableNotFoundException" + return self.error(er, "Table not found: %s" % table_name) + + def delete_backup(self): + body = self.body + backup_arn = body.get("BackupArn") + try: + backup = self.dynamodb_backend.delete_backup(backup_arn) + response = {"BackupDescription": backup.description} + return dynamo_json_dump(response) + except KeyError: + er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException" + return self.error(er, "Backup not found: %s" % backup_arn) + + def describe_backup(self): + body = self.body + backup_arn = body.get("BackupArn") + try: + backup = self.dynamodb_backend.describe_backup(backup_arn) + response = {"BackupDescription": backup.description} + return dynamo_json_dump(response) + except KeyError: + er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException" + return self.error(er, "Backup not found: %s" % backup_arn) + + def restore_table_from_backup(self): + body = self.body + target_table_name = body.get("TargetTableName") + backup_arn = body.get("BackupArn") + try: + restored_table = self.dynamodb_backend.restore_table_from_backup( + target_table_name, backup_arn + ) + return dynamo_json_dump(restored_table.describe()) + except KeyError: + er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException" + return self.error(er, "Backup not found: %s" % backup_arn) + except ValueError: + er = "com.amazonaws.dynamodb.v20111205#TableAlreadyExistsException" + return self.error(er, "Table already exists: %s" % target_table_name) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 122b32c15..9fad91b7c 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals, print_function +import uuid from datetime import datetime from decimal import Decimal @@ -6008,3 +6009,303 @@ def test_get_item_for_non_existent_table_raises_error(): client.get_item(TableName="non-existent", Key={"site-id": {"S": "foo"}}) ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") ex.value.response["Error"]["Message"].should.equal("Requested resource not found") + + +@mock_dynamodb2 +def test_create_backup_for_non_existent_table_raises_error(): + client = boto3.client("dynamodb", "us-east-1") + with pytest.raises(ClientError) as ex: + client.create_backup(TableName="non-existent", BackupName="backup") + error = ex.value.response["Error"] + error["Code"].should.equal("TableNotFoundException") + error["Message"].should.equal("Table not found: non-existent") + + +@mock_dynamodb2 +def test_create_backup(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + backup_name = "backup-test-table" + resp = client.create_backup(TableName=table_name, BackupName=backup_name) + details = resp.get("BackupDetails") + details.should.have.key("BackupArn").should.contain(table_name) + details.should.have.key("BackupName").should.equal(backup_name) + details.should.have.key("BackupSizeBytes").should.be.a(int) + details.should.have.key("BackupStatus") + details.should.have.key("BackupType").should.equal("USER") + details.should.have.key("BackupCreationDateTime").should.be.a(datetime) + + +@mock_dynamodb2 +def test_create_multiple_backups_with_same_name(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + backup_name = "backup-test-table" + backup_arns = [] + for i in range(4): + backup = client.create_backup(TableName=table_name, BackupName=backup_name).get( + "BackupDetails" + ) + backup["BackupName"].should.equal(backup_name) + backup_arns.should_not.contain(backup["BackupArn"]) + backup_arns.append(backup["BackupArn"]) + + +@mock_dynamodb2 +def test_describe_backup_for_non_existent_backup_raises_error(): + client = boto3.client("dynamodb", "us-east-1") + non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9" + with pytest.raises(ClientError) as ex: + client.describe_backup(BackupArn=non_existent_arn) + error = ex.value.response["Error"] + error["Code"].should.equal("BackupNotFoundException") + error["Message"].should.equal("Backup not found: {}".format(non_existent_arn)) + + +@mock_dynamodb2 +def test_describe_backup(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + table = client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ).get("TableDescription") + backup_name = "backup-test-table" + backup_arn = ( + client.create_backup(TableName=table_name, BackupName=backup_name) + .get("BackupDetails") + .get("BackupArn") + ) + resp = client.describe_backup(BackupArn=backup_arn) + description = resp.get("BackupDescription") + details = description.get("BackupDetails") + details.should.have.key("BackupArn").should.contain(table_name) + details.should.have.key("BackupName").should.equal(backup_name) + details.should.have.key("BackupSizeBytes").should.be.a(int) + details.should.have.key("BackupStatus") + details.should.have.key("BackupType").should.equal("USER") + details.should.have.key("BackupCreationDateTime").should.be.a(datetime) + source = description.get("SourceTableDetails") + source.should.have.key("TableName").should.equal(table_name) + source.should.have.key("TableArn").should.equal(table["TableArn"]) + source.should.have.key("TableSizeBytes").should.be.a(int) + source.should.have.key("KeySchema").should.equal(table["KeySchema"]) + source.should.have.key("TableCreationDateTime").should.equal( + table["CreationDateTime"] + ) + source.should.have.key("ProvisionedThroughput").should.be.a(dict) + source.should.have.key("ItemCount").should.equal(table["ItemCount"]) + + +@mock_dynamodb2 +def test_list_backups_for_non_existent_table(): + client = boto3.client("dynamodb", "us-east-1") + resp = client.list_backups(TableName="non-existent") + resp["BackupSummaries"].should.have.length_of(0) + + +@mock_dynamodb2 +def test_list_backups(): + client = boto3.client("dynamodb", "us-east-1") + table_names = ["test-table-1", "test-table-2"] + backup_names = ["backup-1", "backup-2"] + for table_name in table_names: + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + for backup_name in backup_names: + client.create_backup(TableName=table_name, BackupName=backup_name) + resp = client.list_backups(BackupType="USER") + resp["BackupSummaries"].should.have.length_of(4) + for table_name in table_names: + resp = client.list_backups(TableName=table_name) + resp["BackupSummaries"].should.have.length_of(2) + for summary in resp["BackupSummaries"]: + summary.should.have.key("TableName").should.equal(table_name) + summary.should.have.key("TableArn").should.contain(table_name) + summary.should.have.key("BackupName").should.be.within(backup_names) + summary.should.have.key("BackupArn") + summary.should.have.key("BackupCreationDateTime").should.be.a(datetime) + summary.should.have.key("BackupStatus") + summary.should.have.key("BackupType").should.be.within(["USER", "SYSTEM"]) + summary.should.have.key("BackupSizeBytes").should.be.a(int) + + +@mock_dynamodb2 +def test_restore_table_from_non_existent_backup_raises_error(): + client = boto3.client("dynamodb", "us-east-1") + non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9" + with pytest.raises(ClientError) as ex: + client.restore_table_from_backup( + TargetTableName="from-backup", BackupArn=non_existent_arn + ) + error = ex.value.response["Error"] + error["Code"].should.equal("BackupNotFoundException") + error["Message"].should.equal("Backup not found: {}".format(non_existent_arn)) + + +@mock_dynamodb2 +def test_restore_table_from_backup_raises_error_when_table_already_exists(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + resp = client.create_backup(TableName=table_name, BackupName="backup") + backup = resp.get("BackupDetails") + with pytest.raises(ClientError) as ex: + client.restore_table_from_backup( + TargetTableName=table_name, BackupArn=backup["BackupArn"] + ) + error = ex.value.response["Error"] + error["Code"].should.equal("TableAlreadyExistsException") + error["Message"].should.equal("Table already exists: {}".format(table_name)) + + +@mock_dynamodb2 +def test_restore_table_from_backup(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table" + resp = client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + table = resp.get("TableDescription") + for i in range(5): + client.put_item(TableName=table_name, Item={"id": {"S": "item %d" % i}}) + + backup_arn = ( + client.create_backup(TableName=table_name, BackupName="backup") + .get("BackupDetails") + .get("BackupArn") + ) + + restored_table_name = "restored-from-backup" + restored = client.restore_table_from_backup( + TargetTableName=restored_table_name, BackupArn=backup_arn + ).get("TableDescription") + restored.should.have.key("AttributeDefinitions").should.equal( + table["AttributeDefinitions"] + ) + restored.should.have.key("TableName").should.equal(restored_table_name) + restored.should.have.key("KeySchema").should.equal(table["KeySchema"]) + restored.should.have.key("TableStatus") + restored.should.have.key("ItemCount").should.equal(5) + restored.should.have.key("TableArn").should.contain(restored_table_name) + restored.should.have.key("RestoreSummary").should.be.a(dict) + summary = restored.get("RestoreSummary") + summary.should.have.key("SourceBackupArn").should.equal(backup_arn) + summary.should.have.key("SourceTableArn").should.equal(table["TableArn"]) + summary.should.have.key("RestoreDateTime").should.be.a(datetime) + summary.should.have.key("RestoreInProgress").should.equal(False) + + +@mock_dynamodb2 +def test_delete_non_existent_backup_raises_error(): + client = boto3.client("dynamodb", "us-east-1") + non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9" + with pytest.raises(ClientError) as ex: + client.delete_backup(BackupArn=non_existent_arn) + error = ex.value.response["Error"] + error["Code"].should.equal("BackupNotFoundException") + error["Message"].should.equal("Backup not found: {}".format(non_existent_arn)) + + +@mock_dynamodb2 +def test_delete_backup(): + client = boto3.client("dynamodb", "us-east-1") + table_name = "test-table-1" + backup_names = ["backup-1", "backup-2"] + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + for backup_name in backup_names: + client.create_backup(TableName=table_name, BackupName=backup_name) + resp = client.list_backups(TableName=table_name, BackupType="USER") + resp["BackupSummaries"].should.have.length_of(2) + backup_to_delete = resp["BackupSummaries"][0]["BackupArn"] + backup_deleted = client.delete_backup(BackupArn=backup_to_delete).get( + "BackupDescription" + ) + backup_deleted.should.have.key("SourceTableDetails") + backup_deleted.should.have.key("BackupDetails") + details = backup_deleted["BackupDetails"] + details.should.have.key("BackupArn").should.equal(backup_to_delete) + details.should.have.key("BackupName").should.be.within(backup_names) + details.should.have.key("BackupStatus").should.equal("DELETED") + resp = client.list_backups(TableName=table_name, BackupType="USER") + resp["BackupSummaries"].should.have.length_of(1) + + +@mock_dynamodb2 +def test_source_and_restored_table_items_are_not_linked(): + client = boto3.client("dynamodb", "us-east-1") + + def add_guids_to_table(table, num_items): + guids = [] + for i in range(num_items): + guid = str(uuid.uuid4()) + client.put_item(TableName=table, Item={"id": {"S": guid}}) + guids.append(guid) + return guids + + source_table_name = "source-table" + client.create_table( + TableName=source_table_name, + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + guids_original = add_guids_to_table(source_table_name, 5) + + backup_arn = ( + client.create_backup(TableName=source_table_name, BackupName="backup") + .get("BackupDetails") + .get("BackupArn") + ) + guids_added_after_backup = add_guids_to_table(source_table_name, 5) + + restored_table_name = "restored-from-backup" + client.restore_table_from_backup( + TargetTableName=restored_table_name, BackupArn=backup_arn + ) + guids_added_after_restore = add_guids_to_table(restored_table_name, 5) + + source_table_items = client.scan(TableName=source_table_name) + source_table_items.should.have.key("Count").should.equal(10) + source_table_guids = [x["id"]["S"] for x in source_table_items["Items"]] + set(source_table_guids).should.equal( + set(guids_original) | set(guids_added_after_backup) + ) + + restored_table_items = client.scan(TableName=restored_table_name) + restored_table_items.should.have.key("Count").should.equal(10) + restored_table_guids = [x["id"]["S"] for x in restored_table_items["Items"]] + set(restored_table_guids).should.equal( + set(guids_original) | set(guids_added_after_restore) + )