From ab767416fed71ac729f66ce2d9a7b869e2d3911a Mon Sep 17 00:00:00 2001 From: Terry Cain Date: Sun, 29 Oct 2017 16:06:09 +0000 Subject: [PATCH] Completed DynamoDBv2 endpoints --- IMPLEMENTATION_COVERAGE.md | 46 +++++----- README.md | 2 +- moto/dynamodb2/__init__.py | 9 +- moto/dynamodb2/models.py | 45 +++++++++- moto/dynamodb2/responses.py | 117 +++++++++++++++++--------- tests/test_dynamodb2/test_dynamodb.py | 62 +++++++++++++- 6 files changed, 209 insertions(+), 72 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 2725a0570..0377b27ed 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -203,26 +203,26 @@ - [ ] set_load_balancer_policies_for_backend_server - [X] set_load_balancer_policies_of_listener -## dynamodb - 36% implemented -- [ ] batch_get_item -- [ ] batch_write_item +## dynamodb - 100% implemented (is dynamodbv2) +- [X] batch_get_item +- [X] batch_write_item - [X] create_table - [X] delete_item - [X] delete_table -- [ ] describe_limits -- [ ] describe_table -- [ ] describe_time_to_live +- [X] describe_limits +- [X] describe_table +- [X] describe_time_to_live - [X] get_item -- [ ] list_tables -- [ ] list_tags_of_resource +- [X] list_tables +- [X] list_tags_of_resource - [X] put_item - [X] query - [X] scan -- [ ] tag_resource -- [ ] untag_resource -- [ ] update_item -- [ ] update_table -- [ ] update_time_to_live +- [X] tag_resource +- [X] untag_resource +- [X] update_item +- [X] update_table +- [X] update_time_to_live ## cloudhsmv2 - 0% implemented - [ ] create_cluster @@ -1853,31 +1853,31 @@ - [ ] refresh_trusted_advisor_check - [ ] resolve_case -## lambda - 0% implemented +## lambda - 32% implemented - [ ] add_permission - [ ] create_alias - [ ] create_event_source_mapping -- [ ] create_function +- [X] create_function - [ ] delete_alias - [ ] delete_event_source_mapping -- [ ] delete_function +- [X] delete_function - [ ] get_account_settings - [ ] get_alias - [ ] get_event_source_mapping -- [ ] get_function +- [X] get_function - [ ] get_function_configuration -- [ ] get_policy -- [ ] invoke +- [X] get_policy +- [X] invoke - [ ] invoke_async - [ ] list_aliases - [ ] list_event_source_mappings -- [ ] list_functions -- [ ] list_tags +- [X] list_functions +- [X] list_tags - [ ] list_versions_by_function - [ ] publish_version - [ ] remove_permission -- [ ] tag_resource -- [ ] untag_resource +- [X] tag_resource +- [X] untag_resource - [ ] update_alias - [ ] update_event_source_mapping - [ ] update_function_code diff --git a/README.md b/README.md index 402b443b1..b2ca5a807 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L | Data Pipeline | @mock_datapipeline| basic endpoints done | |------------------------------------------------------------------------------| | DynamoDB | @mock_dynamodb | core endpoints done | -| DynamoDB2 | @mock_dynamodb2 | core endpoints + partial indexes | +| DynamoDB2 | @mock_dynamodb2 | all endpoints + partial indexes | |------------------------------------------------------------------------------| | EC2 | @mock_ec2 | core endpoints done | | - AMI | | core endpoints done | diff --git a/moto/dynamodb2/__init__.py b/moto/dynamodb2/__init__.py index ad3f042d2..a56a83b35 100644 --- a/moto/dynamodb2/__init__.py +++ b/moto/dynamodb2/__init__.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals -from .models import dynamodb_backend2 +from .models import dynamodb_backends as dynamodb_backends2 +from ..core.models import base_decorator, deprecated_base_decorator -dynamodb_backends2 = {"global": dynamodb_backend2} -mock_dynamodb2 = dynamodb_backend2.decorator -mock_dynamodb2_deprecated = dynamodb_backend2.deprecated_decorator +dynamodb_backend2 = dynamodb_backends2['us-east-1'] +mock_dynamodb2 = base_decorator(dynamodb_backends2) +mock_dynamodb2_deprecated = deprecated_base_decorator(dynamodb_backends2) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index bec72d327..855728ec1 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -5,9 +5,11 @@ import decimal import json import re +import boto3 from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time +from moto.core.exceptions import JsonRESTError from .comparisons import get_comparison_func, get_filter_expression, Op @@ -271,6 +273,10 @@ class Table(BaseModel): self.items = defaultdict(dict) self.table_arn = self._generate_arn(table_name) self.tags = [] + self.ttl = { + 'TimeToLiveStatus': 'DISABLED' # One of 'ENABLING'|'DISABLING'|'ENABLED'|'DISABLED', + # 'AttributeName': 'string' # Can contain this + } def _generate_arn(self, name): return 'arn:aws:dynamodb:us-east-1:123456789011:table/' + name @@ -577,9 +583,16 @@ class Table(BaseModel): class DynamoDBBackend(BaseBackend): - def __init__(self): + def __init__(self, region_name=None): + self.region_name = region_name self.tables = OrderedDict() + def reset(self): + region_name = self.region_name + + self.__dict__ = {} + self.__init__(region_name) + def create_table(self, name, **params): if name in self.tables: return None @@ -595,6 +608,11 @@ class DynamoDBBackend(BaseBackend): if self.tables[table].table_arn == table_arn: self.tables[table].tags.extend(tags) + def untag_resource(self, table_arn, tag_keys): + for table in self.tables: + if self.tables[table].table_arn == table_arn: + self.tables[table].tags = [tag for tag in self.tables[table].tags if tag['Key'] not in tag_keys] + def list_tags_of_resource(self, table_arn): required_table = None for table in self.tables: @@ -796,5 +814,28 @@ class DynamoDBBackend(BaseBackend): hash_key, range_key = self.get_keys_value(table, keys) return table.delete_item(hash_key, range_key) + def update_ttl(self, table_name, ttl_spec): + table = self.tables.get(table_name) + if table is None: + raise JsonRESTError('ResourceNotFound', 'Table not found') -dynamodb_backend2 = DynamoDBBackend() + if 'Enabled' not in ttl_spec or 'AttributeName' not in ttl_spec: + raise JsonRESTError('InvalidParameterValue', + 'TimeToLiveSpecification does not contain Enabled and AttributeName') + + if ttl_spec['Enabled']: + table.ttl['TimeToLiveStatus'] = 'ENABLED' + else: + table.ttl['TimeToLiveStatus'] = 'DISABLED' + table.ttl['AttributeName'] = ttl_spec['AttributeName'] + + def describe_ttl(self, table_name): + table = self.tables.get(table_name) + if table is None: + raise JsonRESTError('ResourceNotFound', 'Table not found') + + return table.ttl + + +available_regions = boto3.session.Session().get_available_regions("dynamodb") +dynamodb_backends = {region: DynamoDBBackend(region_name=region) for region in available_regions} diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 218cfc21d..1e422aaf5 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -5,7 +5,7 @@ import re from moto.core.responses import BaseResponse from moto.core.utils import camelcase_to_underscores, amzn_request_id -from .models import dynamodb_backend2, dynamo_json_dump +from .models import dynamodb_backends, dynamo_json_dump class DynamoHandler(BaseResponse): @@ -24,6 +24,14 @@ class DynamoHandler(BaseResponse): def error(self, type_, message, status=400): return status, self.response_headers, dynamo_json_dump({'__type': type_, 'message': message}) + @property + def dynamodb_backend(self): + """ + :return: DynamoDB2 Backend + :rtype: moto.dynamodb2.models.DynamoDBBackend + """ + return dynamodb_backends[self.region] + @amzn_request_id def call_action(self): self.body = json.loads(self.body or '{}') @@ -46,10 +54,10 @@ class DynamoHandler(BaseResponse): limit = body.get('Limit', 100) if body.get("ExclusiveStartTableName"): last = body.get("ExclusiveStartTableName") - start = list(dynamodb_backend2.tables.keys()).index(last) + 1 + start = list(self.dynamodb_backend.tables.keys()).index(last) + 1 else: start = 0 - all_tables = list(dynamodb_backend2.tables.keys()) + all_tables = list(self.dynamodb_backend.tables.keys()) if limit: tables = all_tables[start:start + limit] else: @@ -74,12 +82,12 @@ class DynamoHandler(BaseResponse): global_indexes = body.get("GlobalSecondaryIndexes", []) local_secondary_indexes = body.get("LocalSecondaryIndexes", []) - table = dynamodb_backend2.create_table(table_name, - schema=key_schema, - throughput=throughput, - attr=attr, - global_indexes=global_indexes, - indexes=local_secondary_indexes) + table = self.dynamodb_backend.create_table(table_name, + schema=key_schema, + throughput=throughput, + attr=attr, + global_indexes=global_indexes, + indexes=local_secondary_indexes) if table is not None: return dynamo_json_dump(table.describe()) else: @@ -88,7 +96,7 @@ class DynamoHandler(BaseResponse): def delete_table(self): name = self.body['TableName'] - table = dynamodb_backend2.delete_table(name) + table = self.dynamodb_backend.delete_table(name) if table is not None: return dynamo_json_dump(table.describe()) else: @@ -96,15 +104,21 @@ class DynamoHandler(BaseResponse): return self.error(er, 'Requested resource not found') def tag_resource(self): - tags = self.body['Tags'] table_arn = self.body['ResourceArn'] - dynamodb_backend2.tag_resource(table_arn, tags) - return json.dumps({}) + tags = self.body['Tags'] + self.dynamodb_backend.tag_resource(table_arn, tags) + return '' + + def untag_resource(self): + table_arn = self.body['ResourceArn'] + tags = self.body['TagKeys'] + self.dynamodb_backend.untag_resource(table_arn, tags) + return '' def list_tags_of_resource(self): try: table_arn = self.body['ResourceArn'] - all_tags = dynamodb_backend2.list_tags_of_resource(table_arn) + all_tags = self.dynamodb_backend.list_tags_of_resource(table_arn) all_tag_keys = [tag['Key'] for tag in all_tags] marker = self.body.get('NextToken') if marker: @@ -127,17 +141,17 @@ class DynamoHandler(BaseResponse): def update_table(self): name = self.body['TableName'] if 'GlobalSecondaryIndexUpdates' in self.body: - table = dynamodb_backend2.update_table_global_indexes( + table = self.dynamodb_backend.update_table_global_indexes( name, self.body['GlobalSecondaryIndexUpdates']) if 'ProvisionedThroughput' in self.body: throughput = self.body["ProvisionedThroughput"] - table = dynamodb_backend2.update_table_throughput(name, throughput) + table = self.dynamodb_backend.update_table_throughput(name, throughput) return dynamo_json_dump(table.describe()) def describe_table(self): name = self.body['TableName'] try: - table = dynamodb_backend2.tables[name] + table = self.dynamodb_backend.tables[name] except KeyError: er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException' return self.error(er, 'Requested resource not found') @@ -188,8 +202,7 @@ class DynamoHandler(BaseResponse): expected[not_exists_m.group(1)] = {'Exists': False} try: - result = dynamodb_backend2.put_item( - name, item, expected, overwrite) + result = self.dynamodb_backend.put_item(name, item, expected, overwrite) except ValueError: er = 'com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException' return self.error(er, 'A condition specified in the operation could not be evaluated.') @@ -214,10 +227,10 @@ class DynamoHandler(BaseResponse): request = list(table_request.values())[0] if request_type == 'PutRequest': item = request['Item'] - dynamodb_backend2.put_item(table_name, item) + self.dynamodb_backend.put_item(table_name, item) elif request_type == 'DeleteRequest': keys = request['Key'] - item = dynamodb_backend2.delete_item(table_name, keys) + item = self.dynamodb_backend.delete_item(table_name, keys) response = { "ConsumedCapacity": [ @@ -237,7 +250,7 @@ class DynamoHandler(BaseResponse): name = self.body['TableName'] key = self.body['Key'] try: - item = dynamodb_backend2.get_item(name, key) + item = self.dynamodb_backend.get_item(name, key) except ValueError: er = 'com.amazon.coral.validate#ValidationException' return self.error(er, 'Validation Exception') @@ -268,7 +281,7 @@ class DynamoHandler(BaseResponse): attributes_to_get = table_request.get('AttributesToGet') results["Responses"][table_name] = [] for key in keys: - item = dynamodb_backend2.get_item(table_name, key) + item = self.dynamodb_backend.get_item(table_name, key) if item: item_describe = item.describe_attrs(attributes_to_get) results["Responses"][table_name].append( @@ -297,7 +310,7 @@ class DynamoHandler(BaseResponse): if key_condition_expression: value_alias_map = self.body['ExpressionAttributeValues'] - table = dynamodb_backend2.get_table(name) + table = self.dynamodb_backend.get_table(name) # If table does not exist if table is None: @@ -365,7 +378,7 @@ class DynamoHandler(BaseResponse): key_conditions = self.body.get('KeyConditions') query_filters = self.body.get("QueryFilter") if key_conditions: - hash_key_name, range_key_name = dynamodb_backend2.get_table_keys_name( + hash_key_name, range_key_name = self.dynamodb_backend.get_table_keys_name( name, key_conditions.keys()) for key, value in key_conditions.items(): if key not in (hash_key_name, range_key_name): @@ -398,9 +411,10 @@ class DynamoHandler(BaseResponse): exclusive_start_key = self.body.get('ExclusiveStartKey') limit = self.body.get("Limit") scan_index_forward = self.body.get("ScanIndexForward") - items, scanned_count, last_evaluated_key = dynamodb_backend2.query( + items, scanned_count, last_evaluated_key = self.dynamodb_backend.query( name, hash_key, range_comparison, range_values, limit, - exclusive_start_key, scan_index_forward, projection_expression, index_name=index_name, **filter_kwargs) + exclusive_start_key, scan_index_forward, projection_expression, index_name=index_name, **filter_kwargs + ) if items is None: er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException' return self.error(er, 'Requested resource not found') @@ -442,12 +456,12 @@ class DynamoHandler(BaseResponse): limit = self.body.get("Limit") try: - items, scanned_count, last_evaluated_key = dynamodb_backend2.scan(name, filters, - limit, - exclusive_start_key, - filter_expression, - expression_attribute_names, - expression_attribute_values) + items, scanned_count, last_evaluated_key = self.dynamodb_backend.scan(name, filters, + limit, + exclusive_start_key, + filter_expression, + expression_attribute_names, + expression_attribute_values) except ValueError as err: er = 'com.amazonaws.dynamodb.v20111205#ValidationError' return self.error(er, 'Bad Filter Expression: {0}'.format(err)) @@ -478,12 +492,12 @@ class DynamoHandler(BaseResponse): name = self.body['TableName'] keys = self.body['Key'] return_values = self.body.get('ReturnValues', '') - table = dynamodb_backend2.get_table(name) + table = self.dynamodb_backend.get_table(name) if not table: er = 'com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException' return self.error(er, 'A condition specified in the operation could not be evaluated.') - item = dynamodb_backend2.delete_item(name, keys) + item = self.dynamodb_backend.delete_item(name, keys) if item and return_values == 'ALL_OLD': item_dict = item.to_json() else: @@ -500,7 +514,7 @@ class DynamoHandler(BaseResponse): 'ExpressionAttributeNames', {}) expression_attribute_values = self.body.get( 'ExpressionAttributeValues', {}) - existing_item = dynamodb_backend2.get_item(name, key) + existing_item = self.dynamodb_backend.get_item(name, key) if 'Expected' in self.body: expected = self.body['Expected'] @@ -536,9 +550,10 @@ class DynamoHandler(BaseResponse): '\s*([=\+-])\s*', '\\1', update_expression) try: - item = dynamodb_backend2.update_item( - name, key, update_expression, attribute_updates, expression_attribute_names, expression_attribute_values, - expected) + item = self.dynamodb_backend.update_item( + name, key, update_expression, attribute_updates, expression_attribute_names, + expression_attribute_values, expected + ) except ValueError: er = 'com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException' return self.error(er, 'A condition specified in the operation could not be evaluated.') @@ -555,3 +570,27 @@ class DynamoHandler(BaseResponse): item_dict['Attributes'] = {} return dynamo_json_dump(item_dict) + + def describe_limits(self): + return json.dumps({ + 'AccountMaxReadCapacityUnits': 20000, + 'TableMaxWriteCapacityUnits': 10000, + 'AccountMaxWriteCapacityUnits': 20000, + 'TableMaxReadCapacityUnits': 10000 + }) + + def update_time_to_live(self): + name = self.body['TableName'] + ttl_spec = self.body['TimeToLiveSpecification'] + + self.dynamodb_backend.update_ttl(name, ttl_spec) + + return json.dumps({'TimeToLiveSpecification': ttl_spec}) + + def describe_time_to_live(self): + name = self.body['TableName'] + + ttl_spec = self.dynamodb_backend.describe_ttl(name) + + return json.dumps({'TimeToLiveDescription': ttl_spec}) + diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 5df03f8d8..282a89335 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -88,12 +88,22 @@ def test_list_table_tags(): ProvisionedThroughput={'ReadCapacityUnits':5,'WriteCapacityUnits':5}) table_description = conn.describe_table(TableName=name) arn = table_description['Table']['TableArn'] - tags = [{'Key':'TestTag', 'Value': 'TestValue'}] - conn.tag_resource(ResourceArn=arn, - Tags=tags) + + # Tag table + tags = [{'Key': 'TestTag', 'Value': 'TestValue'}, {'Key': 'TestTag2', 'Value': 'TestValue2'}] + conn.tag_resource(ResourceArn=arn, Tags=tags) + + # Check tags resp = conn.list_tags_of_resource(ResourceArn=arn) assert resp["Tags"] == tags + # Remove 1 tag + conn.untag_resource(ResourceArn=arn, TagKeys=['TestTag']) + + # Check tags + resp = conn.list_tags_of_resource(ResourceArn=arn) + assert resp["Tags"] == [{'Key': 'TestTag2', 'Value': 'TestValue2'}] + @requires_boto_gte("2.9") @mock_dynamodb2 @@ -868,3 +878,49 @@ def test_delete_item(): response = table.scan() assert response['Count'] == 0 + + +@mock_dynamodb2 +def test_describe_limits(): + client = boto3.client('dynamodb', region_name='eu-central-1') + resp = client.describe_limits() + + resp['AccountMaxReadCapacityUnits'].should.equal(20000) + resp['AccountMaxWriteCapacityUnits'].should.equal(20000) + resp['TableMaxWriteCapacityUnits'].should.equal(10000) + resp['TableMaxReadCapacityUnits'].should.equal(10000) + + +@mock_dynamodb2 +def test_set_ttl(): + client = boto3.client('dynamodb', region_name='us-east-1') + + # Create the DynamoDB table. + client.create_table( + TableName='test1', + AttributeDefinitions=[{'AttributeName': 'client', 'AttributeType': 'S'}, {'AttributeName': 'app', 'AttributeType': 'S'}], + KeySchema=[{'AttributeName': 'client', 'KeyType': 'HASH'}, {'AttributeName': 'app', 'KeyType': 'RANGE'}], + ProvisionedThroughput={'ReadCapacityUnits': 123, 'WriteCapacityUnits': 123} + ) + + client.update_time_to_live( + TableName='test1', + TimeToLiveSpecification={ + 'Enabled': True, + 'AttributeName': 'expire' + } + ) + + resp = client.describe_time_to_live(TableName='test1') + resp['TimeToLiveDescription']['TimeToLiveStatus'].should.equal('ENABLED') + resp['TimeToLiveDescription']['AttributeName'].should.equal('expire') + + client.update_time_to_live( + TableName='test1', + TimeToLiveSpecification={ + 'Enabled': False, + 'AttributeName': 'expire' + } + ) + + resp['TimeToLiveDescription']['TimeToLiveStatus'].should.equal('DISABLED')