Completed DynamoDBv2 endpoints

IMPLEMENTATION_COVERAGE.md

@@ -203,26 +203,26 @@
 - [ ] set_load_balancer_policies_for_backend_server
 - [X] set_load_balancer_policies_of_listener
 
-## dynamodb - 36% implemented
+## dynamodb - 100% implemented (is dynamodbv2)
-- [ ] batch_get_item
+- [X] batch_get_item
-- [ ] batch_write_item
+- [X] batch_write_item
 - [X] create_table
 - [X] delete_item
 - [X] delete_table
-- [ ] describe_limits
+- [X] describe_limits
-- [ ] describe_table
+- [X] describe_table
-- [ ] describe_time_to_live
+- [X] describe_time_to_live
 - [X] get_item
-- [ ] list_tables
+- [X] list_tables
-- [ ] list_tags_of_resource
+- [X] list_tags_of_resource
 - [X] put_item
 - [X] query
 - [X] scan
-- [ ] tag_resource
+- [X] tag_resource
-- [ ] untag_resource
+- [X] untag_resource
-- [ ] update_item
+- [X] update_item
-- [ ] update_table
+- [X] update_table
-- [ ] update_time_to_live
+- [X] update_time_to_live
 
 ## cloudhsmv2 - 0% implemented
 - [ ] create_cluster
@@ -1853,31 +1853,31 @@
 - [ ] refresh_trusted_advisor_check
 - [ ] resolve_case
 
-## lambda - 0% implemented
+## lambda - 32% implemented
 - [ ] add_permission
 - [ ] create_alias
 - [ ] create_event_source_mapping
-- [ ] create_function
+- [X] create_function
 - [ ] delete_alias
 - [ ] delete_event_source_mapping
-- [ ] delete_function
+- [X] delete_function
 - [ ] get_account_settings
 - [ ] get_alias
 - [ ] get_event_source_mapping
-- [ ] get_function
+- [X] get_function
 - [ ] get_function_configuration
-- [ ] get_policy
+- [X] get_policy
-- [ ] invoke
+- [X] invoke
 - [ ] invoke_async
 - [ ] list_aliases
 - [ ] list_event_source_mappings
-- [ ] list_functions
+- [X] list_functions
-- [ ] list_tags
+- [X] list_tags
 - [ ] list_versions_by_function
 - [ ] publish_version
 - [ ] remove_permission
-- [ ] tag_resource
+- [X] tag_resource
-- [ ] untag_resource
+- [X] untag_resource
 - [ ] update_alias
 - [ ] update_event_source_mapping
 - [ ] update_function_code

README.md

@@ -73,7 +73,7 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L
 | Data Pipeline         | @mock_datapipeline| basic endpoints done             |
 |------------------------------------------------------------------------------|
 | DynamoDB              | @mock_dynamodb   | core endpoints done               |
-| DynamoDB2             | @mock_dynamodb2  | core endpoints + partial indexes  |
+| DynamoDB2             | @mock_dynamodb2  | all endpoints + partial indexes   |
 |------------------------------------------------------------------------------|
 | EC2                   | @mock_ec2        | core endpoints done               |
 |     - AMI             |                  | core endpoints done               |

moto/dynamodb2/__init__.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
-from .models import dynamodb_backend2
+from .models import dynamodb_backends as dynamodb_backends2
+from ..core.models import base_decorator, deprecated_base_decorator
 
-dynamodb_backends2 = {"global": dynamodb_backend2}
+dynamodb_backend2 = dynamodb_backends2['us-east-1']
-mock_dynamodb2 = dynamodb_backend2.decorator
+mock_dynamodb2 = base_decorator(dynamodb_backends2)
-mock_dynamodb2_deprecated = dynamodb_backend2.deprecated_decorator
+mock_dynamodb2_deprecated = deprecated_base_decorator(dynamodb_backends2)

moto/dynamodb2/models.py

@@ -5,9 +5,11 @@ import decimal
 import json
 import re
 
+import boto3
 from moto.compat import OrderedDict
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import unix_time
+from moto.core.exceptions import JsonRESTError
 from .comparisons import get_comparison_func, get_filter_expression, Op
 
 
@@ -271,6 +273,10 @@ class Table(BaseModel):
         self.items = defaultdict(dict)
         self.table_arn = self._generate_arn(table_name)
         self.tags = []
+        self.ttl = {
+            'TimeToLiveStatus': 'DISABLED'  # One of 'ENABLING'|'DISABLING'|'ENABLED'|'DISABLED',
+            # 'AttributeName': 'string'  # Can contain this
+        }
 
     def _generate_arn(self, name):
         return 'arn:aws:dynamodb:us-east-1:123456789011:table/' + name
@@ -577,9 +583,16 @@ class Table(BaseModel):
 
 class DynamoDBBackend(BaseBackend):
 
-    def __init__(self):
+    def __init__(self, region_name=None):
+        self.region_name = region_name
         self.tables = OrderedDict()
 
+    def reset(self):
+        region_name = self.region_name
+
+        self.__dict__ = {}
+        self.__init__(region_name)
+
     def create_table(self, name, **params):
         if name in self.tables:
             return None
@@ -595,6 +608,11 @@ class DynamoDBBackend(BaseBackend):
             if self.tables[table].table_arn == table_arn:
                 self.tables[table].tags.extend(tags)
 
+    def untag_resource(self, table_arn, tag_keys):
+        for table in self.tables:
+            if self.tables[table].table_arn == table_arn:
+                self.tables[table].tags = [tag for tag in self.tables[table].tags if tag['Key'] not in tag_keys]
+
     def list_tags_of_resource(self, table_arn):
         required_table = None
         for table in self.tables:
@@ -796,5 +814,28 @@ class DynamoDBBackend(BaseBackend):
         hash_key, range_key = self.get_keys_value(table, keys)
         return table.delete_item(hash_key, range_key)
 
+    def update_ttl(self, table_name, ttl_spec):
+        table = self.tables.get(table_name)
+        if table is None:
+            raise JsonRESTError('ResourceNotFound', 'Table not found')
 
-dynamodb_backend2 = DynamoDBBackend()
+        if 'Enabled' not in ttl_spec or 'AttributeName' not in ttl_spec:
+            raise JsonRESTError('InvalidParameterValue',
+                                'TimeToLiveSpecification does not contain Enabled and AttributeName')
+
+        if ttl_spec['Enabled']:
+            table.ttl['TimeToLiveStatus'] = 'ENABLED'
+        else:
+            table.ttl['TimeToLiveStatus'] = 'DISABLED'
+        table.ttl['AttributeName'] = ttl_spec['AttributeName']
+
+    def describe_ttl(self, table_name):
+        table = self.tables.get(table_name)
+        if table is None:
+            raise JsonRESTError('ResourceNotFound', 'Table not found')
+
+        return table.ttl
+
+
+available_regions = boto3.session.Session().get_available_regions("dynamodb")
+dynamodb_backends = {region: DynamoDBBackend(region_name=region) for region in available_regions}

moto/dynamodb2/responses.py

@@ -5,7 +5,7 @@ import re
 
 from moto.core.responses import BaseResponse
 from moto.core.utils import camelcase_to_underscores, amzn_request_id
-from .models import dynamodb_backend2, dynamo_json_dump
+from .models import dynamodb_backends, dynamo_json_dump
 
 
 class DynamoHandler(BaseResponse):
@@ -24,6 +24,14 @@ class DynamoHandler(BaseResponse):
     def error(self, type_, message, status=400):
         return status, self.response_headers, dynamo_json_dump({'__type': type_, 'message': message})
 
+    @property
+    def dynamodb_backend(self):
+        """
+        :return: DynamoDB2 Backend
+        :rtype: moto.dynamodb2.models.DynamoDBBackend
+        """
+        return dynamodb_backends[self.region]
+
     @amzn_request_id
     def call_action(self):
         self.body = json.loads(self.body or '{}')
@@ -46,10 +54,10 @@ class DynamoHandler(BaseResponse):
         limit = body.get('Limit', 100)
         if body.get("ExclusiveStartTableName"):
             last = body.get("ExclusiveStartTableName")
-            start = list(dynamodb_backend2.tables.keys()).index(last) + 1
+            start = list(self.dynamodb_backend.tables.keys()).index(last) + 1
         else:
             start = 0
-        all_tables = list(dynamodb_backend2.tables.keys())
+        all_tables = list(self.dynamodb_backend.tables.keys())
         if limit:
             tables = all_tables[start:start + limit]
         else:
@@ -74,12 +82,12 @@ class DynamoHandler(BaseResponse):
         global_indexes = body.get("GlobalSecondaryIndexes", [])
         local_secondary_indexes = body.get("LocalSecondaryIndexes", [])
 
-        table = dynamodb_backend2.create_table(table_name,
+        table = self.dynamodb_backend.create_table(table_name,
-                                               schema=key_schema,
+                                                   schema=key_schema,
-                                               throughput=throughput,
+                                                   throughput=throughput,
-                                               attr=attr,
+                                                   attr=attr,
-                                               global_indexes=global_indexes,
+                                                   global_indexes=global_indexes,
-                                               indexes=local_secondary_indexes)
+                                                   indexes=local_secondary_indexes)
         if table is not None:
             return dynamo_json_dump(table.describe())
         else:
@@ -88,7 +96,7 @@ class DynamoHandler(BaseResponse):
 
     def delete_table(self):
         name = self.body['TableName']
-        table = dynamodb_backend2.delete_table(name)
+        table = self.dynamodb_backend.delete_table(name)
         if table is not None:
             return dynamo_json_dump(table.describe())
         else:
@@ -96,15 +104,21 @@ class DynamoHandler(BaseResponse):
             return self.error(er, 'Requested resource not found')
 
     def tag_resource(self):
-        tags = self.body['Tags']
         table_arn = self.body['ResourceArn']
-        dynamodb_backend2.tag_resource(table_arn, tags)
+        tags = self.body['Tags']
-        return json.dumps({})
+        self.dynamodb_backend.tag_resource(table_arn, tags)
+        return ''
+
+    def untag_resource(self):
+        table_arn = self.body['ResourceArn']
+        tags = self.body['TagKeys']
+        self.dynamodb_backend.untag_resource(table_arn, tags)
+        return ''
 
     def list_tags_of_resource(self):
         try:
             table_arn = self.body['ResourceArn']
-            all_tags = dynamodb_backend2.list_tags_of_resource(table_arn)
+            all_tags = self.dynamodb_backend.list_tags_of_resource(table_arn)
             all_tag_keys = [tag['Key'] for tag in all_tags]
             marker = self.body.get('NextToken')
             if marker:
@@ -127,17 +141,17 @@ class DynamoHandler(BaseResponse):
     def update_table(self):
         name = self.body['TableName']
         if 'GlobalSecondaryIndexUpdates' in self.body:
-            table = dynamodb_backend2.update_table_global_indexes(
+            table = self.dynamodb_backend.update_table_global_indexes(
                 name, self.body['GlobalSecondaryIndexUpdates'])
         if 'ProvisionedThroughput' in self.body:
             throughput = self.body["ProvisionedThroughput"]
-            table = dynamodb_backend2.update_table_throughput(name, throughput)
+            table = self.dynamodb_backend.update_table_throughput(name, throughput)
         return dynamo_json_dump(table.describe())
 
     def describe_table(self):
         name = self.body['TableName']
         try:
-            table = dynamodb_backend2.tables[name]
+            table = self.dynamodb_backend.tables[name]
         except KeyError:
             er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
             return self.error(er, 'Requested resource not found')
@@ -188,8 +202,7 @@ class DynamoHandler(BaseResponse):
                         expected[not_exists_m.group(1)] = {'Exists': False}
 
         try:
-            result = dynamodb_backend2.put_item(
+            result = self.dynamodb_backend.put_item(name, item, expected, overwrite)
-                name, item, expected, overwrite)
         except ValueError:
             er = 'com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException'
             return self.error(er, 'A condition specified in the operation could not be evaluated.')
@@ -214,10 +227,10 @@ class DynamoHandler(BaseResponse):
                 request = list(table_request.values())[0]
                 if request_type == 'PutRequest':
                     item = request['Item']
-                    dynamodb_backend2.put_item(table_name, item)
+                    self.dynamodb_backend.put_item(table_name, item)
                 elif request_type == 'DeleteRequest':
                     keys = request['Key']
-                    item = dynamodb_backend2.delete_item(table_name, keys)
+                    item = self.dynamodb_backend.delete_item(table_name, keys)
 
         response = {
             "ConsumedCapacity": [
@@ -237,7 +250,7 @@ class DynamoHandler(BaseResponse):
         name = self.body['TableName']
         key = self.body['Key']
         try:
-            item = dynamodb_backend2.get_item(name, key)
+            item = self.dynamodb_backend.get_item(name, key)
         except ValueError:
             er = 'com.amazon.coral.validate#ValidationException'
             return self.error(er, 'Validation Exception')
@@ -268,7 +281,7 @@ class DynamoHandler(BaseResponse):
             attributes_to_get = table_request.get('AttributesToGet')
             results["Responses"][table_name] = []
             for key in keys:
-                item = dynamodb_backend2.get_item(table_name, key)
+                item = self.dynamodb_backend.get_item(table_name, key)
                 if item:
                     item_describe = item.describe_attrs(attributes_to_get)
                     results["Responses"][table_name].append(
@@ -297,7 +310,7 @@ class DynamoHandler(BaseResponse):
         if key_condition_expression:
             value_alias_map = self.body['ExpressionAttributeValues']
 
-            table = dynamodb_backend2.get_table(name)
+            table = self.dynamodb_backend.get_table(name)
 
             # If table does not exist
             if table is None:
@@ -365,7 +378,7 @@ class DynamoHandler(BaseResponse):
             key_conditions = self.body.get('KeyConditions')
             query_filters = self.body.get("QueryFilter")
             if key_conditions:
-                hash_key_name, range_key_name = dynamodb_backend2.get_table_keys_name(
+                hash_key_name, range_key_name = self.dynamodb_backend.get_table_keys_name(
                     name, key_conditions.keys())
                 for key, value in key_conditions.items():
                     if key not in (hash_key_name, range_key_name):
@@ -398,9 +411,10 @@ class DynamoHandler(BaseResponse):
         exclusive_start_key = self.body.get('ExclusiveStartKey')
         limit = self.body.get("Limit")
         scan_index_forward = self.body.get("ScanIndexForward")
-        items, scanned_count, last_evaluated_key = dynamodb_backend2.query(
+        items, scanned_count, last_evaluated_key = self.dynamodb_backend.query(
             name, hash_key, range_comparison, range_values, limit,
-            exclusive_start_key, scan_index_forward, projection_expression, index_name=index_name, **filter_kwargs)
+            exclusive_start_key, scan_index_forward, projection_expression, index_name=index_name, **filter_kwargs
+        )
         if items is None:
             er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
             return self.error(er, 'Requested resource not found')
@@ -442,12 +456,12 @@ class DynamoHandler(BaseResponse):
         limit = self.body.get("Limit")
 
         try:
-            items, scanned_count, last_evaluated_key = dynamodb_backend2.scan(name, filters,
+            items, scanned_count, last_evaluated_key = self.dynamodb_backend.scan(name, filters,
-                                                                              limit,
+                                                                                  limit,
-                                                                              exclusive_start_key,
+                                                                                  exclusive_start_key,
-                                                                              filter_expression,
+                                                                                  filter_expression,
-                                                                              expression_attribute_names,
+                                                                                  expression_attribute_names,
-                                                                              expression_attribute_values)
+                                                                                  expression_attribute_values)
         except ValueError as err:
             er = 'com.amazonaws.dynamodb.v20111205#ValidationError'
             return self.error(er, 'Bad Filter Expression: {0}'.format(err))
@@ -478,12 +492,12 @@ class DynamoHandler(BaseResponse):
         name = self.body['TableName']
         keys = self.body['Key']
         return_values = self.body.get('ReturnValues', '')
-        table = dynamodb_backend2.get_table(name)
+        table = self.dynamodb_backend.get_table(name)
         if not table:
             er = 'com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException'
             return self.error(er, 'A condition specified in the operation could not be evaluated.')
 
-        item = dynamodb_backend2.delete_item(name, keys)
+        item = self.dynamodb_backend.delete_item(name, keys)
         if item and return_values == 'ALL_OLD':
             item_dict = item.to_json()
         else:
@@ -500,7 +514,7 @@ class DynamoHandler(BaseResponse):
             'ExpressionAttributeNames', {})
         expression_attribute_values = self.body.get(
             'ExpressionAttributeValues', {})
-        existing_item = dynamodb_backend2.get_item(name, key)
+        existing_item = self.dynamodb_backend.get_item(name, key)
 
         if 'Expected' in self.body:
             expected = self.body['Expected']
@@ -536,9 +550,10 @@ class DynamoHandler(BaseResponse):
                 '\s*([=\+-])\s*', '\\1', update_expression)
 
         try:
-            item = dynamodb_backend2.update_item(
+            item = self.dynamodb_backend.update_item(
-                name, key, update_expression, attribute_updates, expression_attribute_names, expression_attribute_values,
+                name, key, update_expression, attribute_updates, expression_attribute_names,
-                expected)
+                expression_attribute_values, expected
+            )
         except ValueError:
             er = 'com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException'
             return self.error(er, 'A condition specified in the operation could not be evaluated.')
@@ -555,3 +570,27 @@ class DynamoHandler(BaseResponse):
             item_dict['Attributes'] = {}
 
         return dynamo_json_dump(item_dict)
+
+    def describe_limits(self):
+        return json.dumps({
+            'AccountMaxReadCapacityUnits': 20000,
+            'TableMaxWriteCapacityUnits': 10000,
+            'AccountMaxWriteCapacityUnits': 20000,
+            'TableMaxReadCapacityUnits': 10000
+        })
+
+    def update_time_to_live(self):
+        name = self.body['TableName']
+        ttl_spec = self.body['TimeToLiveSpecification']
+
+        self.dynamodb_backend.update_ttl(name, ttl_spec)
+
+        return json.dumps({'TimeToLiveSpecification': ttl_spec})
+
+    def describe_time_to_live(self):
+        name = self.body['TableName']
+
+        ttl_spec = self.dynamodb_backend.describe_ttl(name)
+
+        return json.dumps({'TimeToLiveDescription': ttl_spec})
+

tests/test_dynamodb2/test_dynamodb.py

@@ -88,12 +88,22 @@ def test_list_table_tags():
                       ProvisionedThroughput={'ReadCapacityUnits':5,'WriteCapacityUnits':5})
     table_description = conn.describe_table(TableName=name)
     arn = table_description['Table']['TableArn']
-    tags = [{'Key':'TestTag', 'Value': 'TestValue'}]
+
-    conn.tag_resource(ResourceArn=arn,
+    # Tag table
-                      Tags=tags)
+    tags = [{'Key': 'TestTag', 'Value': 'TestValue'}, {'Key': 'TestTag2', 'Value': 'TestValue2'}]
+    conn.tag_resource(ResourceArn=arn, Tags=tags)
+
+    # Check tags
     resp = conn.list_tags_of_resource(ResourceArn=arn)
     assert resp["Tags"] == tags
 
+    # Remove 1 tag
+    conn.untag_resource(ResourceArn=arn, TagKeys=['TestTag'])
+
+    # Check tags
+    resp = conn.list_tags_of_resource(ResourceArn=arn)
+    assert resp["Tags"] == [{'Key': 'TestTag2', 'Value': 'TestValue2'}]
+
 
 @requires_boto_gte("2.9")
 @mock_dynamodb2
@@ -868,3 +878,49 @@ def test_delete_item():
 
     response = table.scan()
     assert response['Count'] == 0
+
+
+@mock_dynamodb2
+def test_describe_limits():
+    client = boto3.client('dynamodb', region_name='eu-central-1')
+    resp = client.describe_limits()
+
+    resp['AccountMaxReadCapacityUnits'].should.equal(20000)
+    resp['AccountMaxWriteCapacityUnits'].should.equal(20000)
+    resp['TableMaxWriteCapacityUnits'].should.equal(10000)
+    resp['TableMaxReadCapacityUnits'].should.equal(10000)
+
+
+@mock_dynamodb2
+def test_set_ttl():
+    client = boto3.client('dynamodb', region_name='us-east-1')
+
+    # Create the DynamoDB table.
+    client.create_table(
+        TableName='test1',
+        AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'S'}],
+        KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}],
+        ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123}
+    )
+
+    client.update_time_to_live(
+        TableName='test1',
+        TimeToLiveSpecification={
+            'Enabled': True,
+            'AttributeName': 'expire'
+        }
+    )
+
+    resp = client.describe_time_to_live(TableName='test1')
+    resp['TimeToLiveDescription']['TimeToLiveStatus'].should.equal('ENABLED')
+    resp['TimeToLiveDescription']['AttributeName'].should.equal('expire')
+
+    client.update_time_to_live(
+        TableName='test1',
+        TimeToLiveSpecification={
+            'Enabled': False,
+            'AttributeName': 'expire'
+        }
+    )
+
+    resp['TimeToLiveDescription']['TimeToLiveStatus'].should.equal('DISABLED')