Add support for DynamoDB Backup/Restore (#3995)

* Add support for DynamoDB Backup/Restore

Basic support for the following endpoints has been implemented with full test coverage:
- create_backup
- delete_backup
- describe_backup
- list_backups
- restore_table_from_backup

Behavior and error messages verified against a real AWS backend.

* Refactor test based on PR feedback

IMPLEMENTATION_COVERAGE.md

@@ -3127,18 +3127,18 @@
 
 ## dynamodb
 <details>
-<summary>44% implemented</summary>
+<summary>54% implemented</summary>
 
 - [ ] batch_execute_statement
 - [X] batch_get_item
 - [X] batch_write_item
-- [ ] create_backup
+- [X] create_backup
 - [ ] create_global_table
 - [X] create_table
-- [ ] delete_backup
+- [X] delete_backup
 - [X] delete_item
 - [X] delete_table
-- [ ] describe_backup
+- [X] describe_backup
 - [X] describe_continuous_backups
 - [ ] describe_contributor_insights
 - [ ] describe_endpoints
@@ -3156,7 +3156,7 @@
 - [ ] execute_transaction
 - [ ] export_table_to_point_in_time
 - [X] get_item
-- [ ] list_backups
+- [X] list_backups
 - [ ] list_contributor_insights
 - [ ] list_exports
 - [ ] list_global_tables
@@ -3164,7 +3164,7 @@
 - [X] list_tags_of_resource
 - [X] put_item
 - [X] query
-- [ ] restore_table_from_backup
+- [X] restore_table_from_backup
 - [ ] restore_table_to_point_in_time
 - [X] scan
 - [X] tag_resource

moto/dynamodb2/models/__init__.py

@@ -9,8 +9,9 @@ import uuid
 
 from boto3 import Session
 from moto.compat import OrderedDict
+from moto.core import ACCOUNT_ID
 from moto.core import BaseBackend, BaseModel, CloudFormationModel
-from moto.core.utils import unix_time
+from moto.core.utils import unix_time, unix_time_millis
 from moto.core.exceptions import JsonRESTError
 from moto.dynamodb2.comparisons import get_filter_expression
 from moto.dynamodb2.comparisons import get_expected
@@ -969,10 +970,112 @@ class Table(CloudFormationModel):
         dynamodb_backends[region_name].delete_table(self.name)
 
 
+class RestoredTable(Table):
+    def __init__(self, name, backup):
+        params = self._parse_params_from_backup(backup)
+        super(RestoredTable, self).__init__(name, **params)
+        self.indexes = copy.deepcopy(backup.table.indexes)
+        self.global_indexes = copy.deepcopy(backup.table.global_indexes)
+        self.items = copy.deepcopy(backup.table.items)
+        # Restore Attrs
+        self.source_backup_arn = backup.arn
+        self.source_table_arn = backup.table.table_arn
+        self.restore_date_time = self.created_at
+
+    @staticmethod
+    def _parse_params_from_backup(backup):
+        params = {
+            "schema": copy.deepcopy(backup.table.schema),
+            "attr": copy.deepcopy(backup.table.attr),
+            "throughput": copy.deepcopy(backup.table.throughput),
+        }
+        return params
+
+    def describe(self, base_key="TableDescription"):
+        result = super(RestoredTable, self).describe(base_key=base_key)
+        result[base_key]["RestoreSummary"] = {
+            "SourceBackupArn": self.source_backup_arn,
+            "SourceTableArn": self.source_table_arn,
+            "RestoreDateTime": unix_time(self.restore_date_time),
+            "RestoreInProgress": False,
+        }
+        return result
+
+
+class Backup(object):
+    def __init__(
+        self, backend, name, table, status=None, type_=None,
+    ):
+        self.backend = backend
+        self.name = name
+        self.table = copy.deepcopy(table)
+        self.status = status or "AVAILABLE"
+        self.type = type_ or "USER"
+        self.creation_date_time = datetime.datetime.utcnow()
+        self.identifier = self._make_identifier()
+
+    def _make_identifier(self):
+        timestamp = int(unix_time_millis(self.creation_date_time))
+        timestamp_padded = str("0" + str(timestamp))[-16:16]
+        guid = str(uuid.uuid4())
+        guid_shortened = guid[:8]
+        return "{}-{}".format(timestamp_padded, guid_shortened)
+
+    @property
+    def arn(self):
+        return "arn:aws:dynamodb:{region}:{account}:table/{table_name}/backup/{identifier}".format(
+            region=self.backend.region_name,
+            account=ACCOUNT_ID,
+            table_name=self.table.name,
+            identifier=self.identifier,
+        )
+
+    @property
+    def details(self):
+        details = {
+            "BackupArn": self.arn,
+            "BackupName": self.name,
+            "BackupSizeBytes": 123,
+            "BackupStatus": self.status,
+            "BackupType": self.type,
+            "BackupCreationDateTime": unix_time(self.creation_date_time),
+        }
+        return details
+
+    @property
+    def summary(self):
+        summary = {
+            "TableName": self.table.name,
+            # 'TableId': 'string',
+            "TableArn": self.table.table_arn,
+            "BackupArn": self.arn,
+            "BackupName": self.name,
+            "BackupCreationDateTime": unix_time(self.creation_date_time),
+            # 'BackupExpiryDateTime': datetime(2015, 1, 1),
+            "BackupStatus": self.status,
+            "BackupType": self.type,
+            "BackupSizeBytes": 123,
+        }
+        return summary
+
+    @property
+    def description(self):
+        source_table_details = self.table.describe()["TableDescription"]
+        source_table_details["TableCreationDateTime"] = source_table_details[
+            "CreationDateTime"
+        ]
+        description = {
+            "BackupDetails": self.details,
+            "SourceTableDetails": source_table_details,
+        }
+        return description
+
+
 class DynamoDBBackend(BaseBackend):
     def __init__(self, region_name=None):
         self.region_name = region_name
         self.tables = OrderedDict()
+        self.backups = OrderedDict()
 
     def reset(self):
         region_name = self.region_name
@@ -1505,6 +1608,48 @@ class DynamoDBBackend(BaseBackend):
 
         return table.continuous_backups
 
+    def get_backup(self, backup_arn):
+        return self.backups.get(backup_arn)
+
+    def list_backups(self, table_name):
+        backups = list(self.backups.values())
+        if table_name is not None:
+            backups = [backup for backup in backups if backup.table.name == table_name]
+        return backups
+
+    def create_backup(self, table_name, backup_name):
+        table = self.get_table(table_name)
+        if table is None:
+            raise KeyError()
+        backup = Backup(self, backup_name, table)
+        self.backups[backup.arn] = backup
+        return backup
+
+    def delete_backup(self, backup_arn):
+        backup = self.get_backup(backup_arn)
+        if backup is None:
+            raise KeyError()
+        backup_deleted = self.backups.pop(backup_arn)
+        backup_deleted.status = "DELETED"
+        return backup_deleted
+
+    def describe_backup(self, backup_arn):
+        backup = self.get_backup(backup_arn)
+        if backup is None:
+            raise KeyError()
+        return backup
+
+    def restore_table_from_backup(self, target_table_name, backup_arn):
+        backup = self.get_backup(backup_arn)
+        if backup is None:
+            raise KeyError()
+        existing_table = self.get_table(target_table_name)
+        if existing_table is not None:
+            raise ValueError()
+        new_table = RestoredTable(target_table_name, backup)
+        self.tables[target_table_name] = new_table
+        return new_table
+
     ######################
     # LIST of methods where the logic completely resides in responses.py
     # Duplicated here so that the implementation coverage script is aware

moto/dynamodb2/responses.py

@@ -986,3 +986,60 @@ class DynamoHandler(BaseResponse):
         )
 
         return json.dumps({"ContinuousBackupsDescription": response})
+
+    def list_backups(self):
+        body = self.body
+        table_name = body.get("TableName")
+        backups = self.dynamodb_backend.list_backups(table_name)
+        response = {"BackupSummaries": [backup.summary for backup in backups]}
+        return dynamo_json_dump(response)
+
+    def create_backup(self):
+        body = self.body
+        table_name = body.get("TableName")
+        backup_name = body.get("BackupName")
+        try:
+            backup = self.dynamodb_backend.create_backup(table_name, backup_name)
+            response = {"BackupDetails": backup.details}
+            return dynamo_json_dump(response)
+        except KeyError:
+            er = "com.amazonaws.dynamodb.v20111205#TableNotFoundException"
+            return self.error(er, "Table not found: %s" % table_name)
+
+    def delete_backup(self):
+        body = self.body
+        backup_arn = body.get("BackupArn")
+        try:
+            backup = self.dynamodb_backend.delete_backup(backup_arn)
+            response = {"BackupDescription": backup.description}
+            return dynamo_json_dump(response)
+        except KeyError:
+            er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException"
+            return self.error(er, "Backup not found: %s" % backup_arn)
+
+    def describe_backup(self):
+        body = self.body
+        backup_arn = body.get("BackupArn")
+        try:
+            backup = self.dynamodb_backend.describe_backup(backup_arn)
+            response = {"BackupDescription": backup.description}
+            return dynamo_json_dump(response)
+        except KeyError:
+            er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException"
+            return self.error(er, "Backup not found: %s" % backup_arn)
+
+    def restore_table_from_backup(self):
+        body = self.body
+        target_table_name = body.get("TargetTableName")
+        backup_arn = body.get("BackupArn")
+        try:
+            restored_table = self.dynamodb_backend.restore_table_from_backup(
+                target_table_name, backup_arn
+            )
+            return dynamo_json_dump(restored_table.describe())
+        except KeyError:
+            er = "com.amazonaws.dynamodb.v20111205#BackupNotFoundException"
+            return self.error(er, "Backup not found: %s" % backup_arn)
+        except ValueError:
+            er = "com.amazonaws.dynamodb.v20111205#TableAlreadyExistsException"
+            return self.error(er, "Table already exists: %s" % target_table_name)

tests/test_dynamodb2/test_dynamodb.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals, print_function
 
+import uuid
 from datetime import datetime
 from decimal import Decimal
 
@@ -6008,3 +6009,303 @@ def test_get_item_for_non_existent_table_raises_error():
         client.get_item(TableName="non-existent", Key={"site-id": {"S": "foo"}})
     ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException")
     ex.value.response["Error"]["Message"].should.equal("Requested resource not found")
+
+
+@mock_dynamodb2
+def test_create_backup_for_non_existent_table_raises_error():
+    client = boto3.client("dynamodb", "us-east-1")
+    with pytest.raises(ClientError) as ex:
+        client.create_backup(TableName="non-existent", BackupName="backup")
+    error = ex.value.response["Error"]
+    error["Code"].should.equal("TableNotFoundException")
+    error["Message"].should.equal("Table not found: non-existent")
+
+
+@mock_dynamodb2
+def test_create_backup():
+    client = boto3.client("dynamodb", "us-east-1")
+    table_name = "test-table"
+    client.create_table(
+        TableName=table_name,
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    backup_name = "backup-test-table"
+    resp = client.create_backup(TableName=table_name, BackupName=backup_name)
+    details = resp.get("BackupDetails")
+    details.should.have.key("BackupArn").should.contain(table_name)
+    details.should.have.key("BackupName").should.equal(backup_name)
+    details.should.have.key("BackupSizeBytes").should.be.a(int)
+    details.should.have.key("BackupStatus")
+    details.should.have.key("BackupType").should.equal("USER")
+    details.should.have.key("BackupCreationDateTime").should.be.a(datetime)
+
+
+@mock_dynamodb2
+def test_create_multiple_backups_with_same_name():
+    client = boto3.client("dynamodb", "us-east-1")
+    table_name = "test-table"
+    client.create_table(
+        TableName=table_name,
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    backup_name = "backup-test-table"
+    backup_arns = []
+    for i in range(4):
+        backup = client.create_backup(TableName=table_name, BackupName=backup_name).get(
+            "BackupDetails"
+        )
+        backup["BackupName"].should.equal(backup_name)
+        backup_arns.should_not.contain(backup["BackupArn"])
+        backup_arns.append(backup["BackupArn"])
+
+
+@mock_dynamodb2
+def test_describe_backup_for_non_existent_backup_raises_error():
+    client = boto3.client("dynamodb", "us-east-1")
+    non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9"
+    with pytest.raises(ClientError) as ex:
+        client.describe_backup(BackupArn=non_existent_arn)
+    error = ex.value.response["Error"]
+    error["Code"].should.equal("BackupNotFoundException")
+    error["Message"].should.equal("Backup not found: {}".format(non_existent_arn))
+
+
+@mock_dynamodb2
+def test_describe_backup():
+    client = boto3.client("dynamodb", "us-east-1")
+    table_name = "test-table"
+    table = client.create_table(
+        TableName=table_name,
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    ).get("TableDescription")
+    backup_name = "backup-test-table"
+    backup_arn = (
+        client.create_backup(TableName=table_name, BackupName=backup_name)
+        .get("BackupDetails")
+        .get("BackupArn")
+    )
+    resp = client.describe_backup(BackupArn=backup_arn)
+    description = resp.get("BackupDescription")
+    details = description.get("BackupDetails")
+    details.should.have.key("BackupArn").should.contain(table_name)
+    details.should.have.key("BackupName").should.equal(backup_name)
+    details.should.have.key("BackupSizeBytes").should.be.a(int)
+    details.should.have.key("BackupStatus")
+    details.should.have.key("BackupType").should.equal("USER")
+    details.should.have.key("BackupCreationDateTime").should.be.a(datetime)
+    source = description.get("SourceTableDetails")
+    source.should.have.key("TableName").should.equal(table_name)
+    source.should.have.key("TableArn").should.equal(table["TableArn"])
+    source.should.have.key("TableSizeBytes").should.be.a(int)
+    source.should.have.key("KeySchema").should.equal(table["KeySchema"])
+    source.should.have.key("TableCreationDateTime").should.equal(
+        table["CreationDateTime"]
+    )
+    source.should.have.key("ProvisionedThroughput").should.be.a(dict)
+    source.should.have.key("ItemCount").should.equal(table["ItemCount"])
+
+
+@mock_dynamodb2
+def test_list_backups_for_non_existent_table():
+    client = boto3.client("dynamodb", "us-east-1")
+    resp = client.list_backups(TableName="non-existent")
+    resp["BackupSummaries"].should.have.length_of(0)
+
+
+@mock_dynamodb2
+def test_list_backups():
+    client = boto3.client("dynamodb", "us-east-1")
+    table_names = ["test-table-1", "test-table-2"]
+    backup_names = ["backup-1", "backup-2"]
+    for table_name in table_names:
+        client.create_table(
+            TableName=table_name,
+            KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+            AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
+            ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+        )
+        for backup_name in backup_names:
+            client.create_backup(TableName=table_name, BackupName=backup_name)
+    resp = client.list_backups(BackupType="USER")
+    resp["BackupSummaries"].should.have.length_of(4)
+    for table_name in table_names:
+        resp = client.list_backups(TableName=table_name)
+        resp["BackupSummaries"].should.have.length_of(2)
+        for summary in resp["BackupSummaries"]:
+            summary.should.have.key("TableName").should.equal(table_name)
+            summary.should.have.key("TableArn").should.contain(table_name)
+            summary.should.have.key("BackupName").should.be.within(backup_names)
+            summary.should.have.key("BackupArn")
+            summary.should.have.key("BackupCreationDateTime").should.be.a(datetime)
+            summary.should.have.key("BackupStatus")
+            summary.should.have.key("BackupType").should.be.within(["USER", "SYSTEM"])
+            summary.should.have.key("BackupSizeBytes").should.be.a(int)
+
+
+@mock_dynamodb2
+def test_restore_table_from_non_existent_backup_raises_error():
+    client = boto3.client("dynamodb", "us-east-1")
+    non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9"
+    with pytest.raises(ClientError) as ex:
+        client.restore_table_from_backup(
+            TargetTableName="from-backup", BackupArn=non_existent_arn
+        )
+    error = ex.value.response["Error"]
+    error["Code"].should.equal("BackupNotFoundException")
+    error["Message"].should.equal("Backup not found: {}".format(non_existent_arn))
+
+
+@mock_dynamodb2
+def test_restore_table_from_backup_raises_error_when_table_already_exists():
+    client = boto3.client("dynamodb", "us-east-1")
+    table_name = "test-table"
+    client.create_table(
+        TableName=table_name,
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    resp = client.create_backup(TableName=table_name, BackupName="backup")
+    backup = resp.get("BackupDetails")
+    with pytest.raises(ClientError) as ex:
+        client.restore_table_from_backup(
+            TargetTableName=table_name, BackupArn=backup["BackupArn"]
+        )
+    error = ex.value.response["Error"]
+    error["Code"].should.equal("TableAlreadyExistsException")
+    error["Message"].should.equal("Table already exists: {}".format(table_name))
+
+
+@mock_dynamodb2
+def test_restore_table_from_backup():
+    client = boto3.client("dynamodb", "us-east-1")
+    table_name = "test-table"
+    resp = client.create_table(
+        TableName=table_name,
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    table = resp.get("TableDescription")
+    for i in range(5):
+        client.put_item(TableName=table_name, Item={"id": {"S": "item %d" % i}})
+
+    backup_arn = (
+        client.create_backup(TableName=table_name, BackupName="backup")
+        .get("BackupDetails")
+        .get("BackupArn")
+    )
+
+    restored_table_name = "restored-from-backup"
+    restored = client.restore_table_from_backup(
+        TargetTableName=restored_table_name, BackupArn=backup_arn
+    ).get("TableDescription")
+    restored.should.have.key("AttributeDefinitions").should.equal(
+        table["AttributeDefinitions"]
+    )
+    restored.should.have.key("TableName").should.equal(restored_table_name)
+    restored.should.have.key("KeySchema").should.equal(table["KeySchema"])
+    restored.should.have.key("TableStatus")
+    restored.should.have.key("ItemCount").should.equal(5)
+    restored.should.have.key("TableArn").should.contain(restored_table_name)
+    restored.should.have.key("RestoreSummary").should.be.a(dict)
+    summary = restored.get("RestoreSummary")
+    summary.should.have.key("SourceBackupArn").should.equal(backup_arn)
+    summary.should.have.key("SourceTableArn").should.equal(table["TableArn"])
+    summary.should.have.key("RestoreDateTime").should.be.a(datetime)
+    summary.should.have.key("RestoreInProgress").should.equal(False)
+
+
+@mock_dynamodb2
+def test_delete_non_existent_backup_raises_error():
+    client = boto3.client("dynamodb", "us-east-1")
+    non_existent_arn = "arn:aws:dynamodb:us-east-1:123456789012:table/table-name/backup/01623095754481-2cfcd6f9"
+    with pytest.raises(ClientError) as ex:
+        client.delete_backup(BackupArn=non_existent_arn)
+    error = ex.value.response["Error"]
+    error["Code"].should.equal("BackupNotFoundException")
+    error["Message"].should.equal("Backup not found: {}".format(non_existent_arn))
+
+
+@mock_dynamodb2
+def test_delete_backup():
+    client = boto3.client("dynamodb", "us-east-1")
+    table_name = "test-table-1"
+    backup_names = ["backup-1", "backup-2"]
+    client.create_table(
+        TableName=table_name,
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    for backup_name in backup_names:
+        client.create_backup(TableName=table_name, BackupName=backup_name)
+    resp = client.list_backups(TableName=table_name, BackupType="USER")
+    resp["BackupSummaries"].should.have.length_of(2)
+    backup_to_delete = resp["BackupSummaries"][0]["BackupArn"]
+    backup_deleted = client.delete_backup(BackupArn=backup_to_delete).get(
+        "BackupDescription"
+    )
+    backup_deleted.should.have.key("SourceTableDetails")
+    backup_deleted.should.have.key("BackupDetails")
+    details = backup_deleted["BackupDetails"]
+    details.should.have.key("BackupArn").should.equal(backup_to_delete)
+    details.should.have.key("BackupName").should.be.within(backup_names)
+    details.should.have.key("BackupStatus").should.equal("DELETED")
+    resp = client.list_backups(TableName=table_name, BackupType="USER")
+    resp["BackupSummaries"].should.have.length_of(1)
+
+
+@mock_dynamodb2
+def test_source_and_restored_table_items_are_not_linked():
+    client = boto3.client("dynamodb", "us-east-1")
+
+    def add_guids_to_table(table, num_items):
+        guids = []
+        for i in range(num_items):
+            guid = str(uuid.uuid4())
+            client.put_item(TableName=table, Item={"id": {"S": guid}})
+            guids.append(guid)
+        return guids
+
+    source_table_name = "source-table"
+    client.create_table(
+        TableName=source_table_name,
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    guids_original = add_guids_to_table(source_table_name, 5)
+
+    backup_arn = (
+        client.create_backup(TableName=source_table_name, BackupName="backup")
+        .get("BackupDetails")
+        .get("BackupArn")
+    )
+    guids_added_after_backup = add_guids_to_table(source_table_name, 5)
+
+    restored_table_name = "restored-from-backup"
+    client.restore_table_from_backup(
+        TargetTableName=restored_table_name, BackupArn=backup_arn
+    )
+    guids_added_after_restore = add_guids_to_table(restored_table_name, 5)
+
+    source_table_items = client.scan(TableName=source_table_name)
+    source_table_items.should.have.key("Count").should.equal(10)
+    source_table_guids = [x["id"]["S"] for x in source_table_items["Items"]]
+    set(source_table_guids).should.equal(
+        set(guids_original) | set(guids_added_after_backup)
+    )
+
+    restored_table_items = client.scan(TableName=restored_table_name)
+    restored_table_items.should.have.key("Count").should.equal(10)
+    restored_table_guids = [x["id"]["S"] for x in restored_table_items["Items"]]
+    set(restored_table_guids).should.equal(
+        set(guids_original) | set(guids_added_after_restore)
+    )