From 6affc7a4ec9cfa323f45b184d42429b309a7e074 Mon Sep 17 00:00:00 2001 From: Brian Rower Date: Tue, 12 Sep 2017 21:42:29 -0700 Subject: [PATCH 1/5] Add basic support for the add operation in an update operation Add basic delete functionality Improve testing coverage and make behave more like actual dynamo on errors Lint fix --- moto/dynamodb2/models.py | 60 ++++++++- moto/dynamodb2/responses.py | 3 + .../test_dynamodb_table_with_range_key.py | 126 ++++++++++++++++++ 3 files changed, 188 insertions(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index bdc5743c2..5747f260d 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -57,7 +57,7 @@ class DynamoType(object): @property def cast_value(self): - if self.type == 'N': + if self.is_number(): try: return int(self.value) except ValueError: @@ -76,6 +76,15 @@ class DynamoType(object): comparison_func = get_comparison_func(range_comparison) return comparison_func(self.cast_value, *range_values) + def is_number(self): + return self.type == 'N' + + def is_set(self): + return self.type == 'SS' or self.type == 'NS' or self.type == 'BS' + + def same_type(self, other): + return self.type == other.type + class Item(BaseModel): @@ -140,6 +149,55 @@ class Item(BaseModel): self.attrs[key] = DynamoType(expression_attribute_values[value]) else: self.attrs[key] = DynamoType({"S": value}) + elif action == 'ADD': + key, value = value.split(" ", maxsplit=1) + key = key.strip() + value_str = value.strip() + if value_str in expression_attribute_values: + dyn_value = DynamoType(expression_attribute_values[value]) + else: + raise TypeError + + # Handle adding numbers - value gets added to existing value, + # or added to 0 if it doesn't exist yet + if dyn_value.is_number(): + existing = self.attrs.get(key, DynamoType({"N": '0'})) + if not existing.same_type(dyn_value): + raise TypeError() + self.attrs[key] = DynamoType({"N": str( + decimal.Decimal(existing.value) + + decimal.Decimal(dyn_value.value) + )}) + + # Handle adding sets - value is added to the set, or set is + # created with only this value if it doesn't exist yet + # New value must be of same set type as previous value + elif dyn_value.is_set(): + existing = self.attrs.get(key, DynamoType({dyn_value.type: {}})) + if not existing.same_type(dyn_value): + raise TypeError() + new_set = set(existing.value).union(dyn_value.value) + self.attrs[key] = DynamoType({existing.type: list(new_set)}) + else: # Number and Sets are the only supported types for ADD + raise TypeError + + elif action == 'DELETE': + key, value = value.split(" ", maxsplit=1) + key = key.strip() + value_str = value.strip() + if value_str in expression_attribute_values: + dyn_value = DynamoType(expression_attribute_values[value]) + else: + raise TypeError + + if not dyn_value.is_set(): + raise TypeError + existing = self.attrs.get(key, None) + if existing: + if not existing.same_type(dyn_value): + raise TypeError + new_set = set(existing.value).difference(dyn_value.value) + self.attrs[key] = DynamoType({existing.type: list(new_set)}) else: raise NotImplementedError('{} update action not yet supported'.format(action)) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 586a1db7b..c3cb4ef72 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -497,6 +497,9 @@ class DynamoHandler(BaseResponse): except ValueError: er = 'com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException' return self.error(er) + except TypeError: + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er) item_dict = item.to_json() item_dict['ConsumedCapacityUnits'] = 0.5 diff --git a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py index 93dc5b3ef..1ab25dc7a 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py @@ -5,6 +5,7 @@ from decimal import Decimal import boto import boto3 from boto3.dynamodb.conditions import Key +from botocore.exceptions import ClientError import sure # noqa from freezegun import freeze_time from moto import mock_dynamodb2, mock_dynamodb2_deprecated @@ -1190,6 +1191,14 @@ def _create_table_with_range_key(): 'AttributeName': 'subject', 'AttributeType': 'S' }, + { + 'AttributeName': 'username', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'created', + 'AttributeType': 'N' + } ], ProvisionedThroughput={ 'ReadCapacityUnits': 5, @@ -1392,6 +1401,123 @@ def test_update_item_with_expression(): 'subject': '123', }) +@mock_dynamodb2 +def test_update_item_add_with_expression(): + table = _create_table_with_range_key() + + item_key = {'forum_name': 'the-key', 'subject': '123'} + current_item = { + 'forum_name': 'the-key', + 'subject': '123', + 'str_set': {'item1', 'item2', 'item3'}, + 'num_set': {1, 2, 3}, + 'num_val': 6 + } + + # Put an entry in the DB to play with + table.put_item(Item=current_item) + + # Update item to add a string value to a string set + table.update_item( + Key=item_key, + UpdateExpression='ADD str_set :v', + ExpressionAttributeValues={ + ':v': {'item4'} + } + ) + current_item['str_set'] = current_item['str_set'].union({'item4'}) + dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + + # Update item to add a num value to a num set + table.update_item( + Key=item_key, + UpdateExpression='ADD num_set :v', + ExpressionAttributeValues={ + ':v': {6} + } + ) + current_item['num_set'] = current_item['num_set'].union({6}) + dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + + # Update item to add a value to a number value + table.update_item( + Key=item_key, + UpdateExpression='ADD num_val :v', + ExpressionAttributeValues={ + ':v': 20 + } + ) + current_item['num_val'] = current_item['num_val'] + 20 + dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + + # Attempt to add a number value to a string set, should raise Client Error + table.update_item.when.called_with( + Key=item_key, + UpdateExpression='ADD str_set :v', + ExpressionAttributeValues={ + ':v': 20 + } + ).should.have.raised(ClientError) + dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + + # Attempt to add a number set to the string set, should raise a ClientError + table.update_item.when.called_with( + Key=item_key, + UpdateExpression='ADD str_set :v', + ExpressionAttributeValues={ + ':v': { 20 } + } + ).should.have.raised(ClientError) + dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + +@mock_dynamodb2 +def test_update_item_delete_with_expression(): + table = _create_table_with_range_key() + + item_key = {'forum_name': 'the-key', 'subject': '123'} + current_item = { + 'forum_name': 'the-key', + 'subject': '123', + 'str_set': {'item1', 'item2', 'item3'}, + 'num_set': {1, 2, 3}, + 'num_val': 6 + } + + # Put an entry in the DB to play with + table.put_item(Item=current_item) + + # Update item to delete a string value from a string set + table.update_item( + Key=item_key, + UpdateExpression='DELETE str_set :v', + ExpressionAttributeValues={ + ':v': {'item2'} + } + ) + current_item['str_set'] = current_item['str_set'].difference({'item2'}) + dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + + # Update item to delete a num value from a num set + table.update_item( + Key=item_key, + UpdateExpression='DELETE num_set :v', + ExpressionAttributeValues={ + ':v': {2} + } + ) + current_item['num_set'] = current_item['num_set'].difference({2}) + dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + + # Try to delete on a number, this should fail + table.update_item.when.called_with( + Key=item_key, + UpdateExpression='DELETE num_val :v', + ExpressionAttributeValues={ + ':v': 20 + } + ).should.have.raised(ClientError) + dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + @mock_dynamodb2 def test_boto3_query_gsi_range_comparison(): From c1c592609abfb3699e7692facf30d9ca7a9103dc Mon Sep 17 00:00:00 2001 From: Brian Rower Date: Tue, 12 Sep 2017 23:30:15 -0700 Subject: [PATCH 2/5] Fix python 2.7 issue --- moto/dynamodb2/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 5747f260d..5915d6eea 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -150,7 +150,7 @@ class Item(BaseModel): else: self.attrs[key] = DynamoType({"S": value}) elif action == 'ADD': - key, value = value.split(" ", maxsplit=1) + key, value = value.split(" ", 1) key = key.strip() value_str = value.strip() if value_str in expression_attribute_values: @@ -182,7 +182,7 @@ class Item(BaseModel): raise TypeError elif action == 'DELETE': - key, value = value.split(" ", maxsplit=1) + key, value = value.split(" ", 1) key = key.strip() value_str = value.strip() if value_str in expression_attribute_values: From 0097ab4c67972ae169cc0613a6c7aae72fbacfea Mon Sep 17 00:00:00 2001 From: Brian Rower Date: Tue, 12 Sep 2017 23:41:12 -0700 Subject: [PATCH 3/5] Improve code coverage --- .../test_dynamodb_table_with_range_key.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py index 1ab25dc7a..a9ab298b7 100644 --- a/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py +++ b/tests/test_dynamodb2/test_dynamodb_table_with_range_key.py @@ -1470,6 +1470,22 @@ def test_update_item_add_with_expression(): ).should.have.raised(ClientError) dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + # Attempt to update with a bad expression + table.update_item.when.called_with( + Key=item_key, + UpdateExpression='ADD str_set bad_value' + ).should.have.raised(ClientError) + + # Attempt to add a string value instead of a string set + table.update_item.when.called_with( + Key=item_key, + UpdateExpression='ADD str_set :v', + ExpressionAttributeValues={ + ':v': 'new_string' + } + ).should.have.raised(ClientError) + + @mock_dynamodb2 def test_update_item_delete_with_expression(): table = _create_table_with_range_key() @@ -1518,6 +1534,22 @@ def test_update_item_delete_with_expression(): ).should.have.raised(ClientError) dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + # Try to delete a string set from a number set + table.update_item.when.called_with( + Key=item_key, + UpdateExpression='DELETE num_set :v', + ExpressionAttributeValues={ + ':v': {'del_str'} + } + ).should.have.raised(ClientError) + dict(table.get_item(Key=item_key)['Item']).should.equal(current_item) + + # Attempt to update with a bad expression + table.update_item.when.called_with( + Key=item_key, + UpdateExpression='DELETE num_val badvalue' + ).should.have.raised(ClientError) + @mock_dynamodb2 def test_boto3_query_gsi_range_comparison(): From 98f95dd4a7c656a258f0e16afed95b69aafefad2 Mon Sep 17 00:00:00 2001 From: aohara-admin Date: Wed, 13 Sep 2017 15:00:39 -0400 Subject: [PATCH 4/5] add Lambda tag_resource, untag_resource, and list_tags methods --- moto/awslambda/models.py | 26 +++++++++ moto/awslambda/responses.py | 57 +++++++++++++++++++ moto/awslambda/urls.py | 1 + tests/test_awslambda/test_lambda.py | 86 +++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 13d4726ac..523393ba3 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -81,6 +81,8 @@ class LambdaFunction(BaseModel): self.function_arn = 'arn:aws:lambda:123456789012:function:{0}'.format( self.function_name) + self.tags = dict() + @property def vpc_config(self): config = self._vpc_config.copy() @@ -278,6 +280,9 @@ class LambdaBackend(BaseBackend): def has_function(self, function_name): return function_name in self._functions + def has_function_arn(self, function_arn): + return self.get_function_by_arn(function_arn) is not None + def create_function(self, spec): fn = LambdaFunction(spec) self._functions[fn.function_name] = fn @@ -286,12 +291,33 @@ class LambdaBackend(BaseBackend): def get_function(self, function_name): return self._functions[function_name] + def get_function_by_arn(self, function_arn): + for function in self._functions.values(): + if function.function_arn == function_arn: + return function + return None + def delete_function(self, function_name): del self._functions[function_name] def list_functions(self): return self._functions.values() + def list_tags(self, resource): + return self.get_function_by_arn(resource).tags + + def tag_resource(self, resource, tags): + self.get_function_by_arn(resource).tags.update(tags) + + def untag_resource(self, resource, tagKeys): + function = self.get_function_by_arn(resource) + for key in tagKeys: + try: + del function.tags[key] + except KeyError: + pass + # Don't care + def do_validate_s3(): return os.environ.get('VALIDATE_LAMBDA_S3', '') in ['', '1', 'true'] diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index d145f4760..b7a6921c5 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -3,6 +3,12 @@ from __future__ import unicode_literals import json import re +try: + from urllib import unquote + from urlparse import urlparse, parse_qs +except: + from urllib.parse import unquote, urlparse, parse_qs + from moto.core.responses import BaseResponse @@ -33,6 +39,17 @@ class LambdaResponse(BaseResponse): else: raise ValueError("Cannot handle request") + def tag(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + if request.method == 'GET': + return self._list_tags(request, full_url) + elif request.method == 'POST': + return self._tag_resource(request, full_url) + elif request.method == 'DELETE': + return self._untag_resource(request, full_url) + else: + raise ValueError("Cannot handle {0} request".format(request.method)) + def _invoke(self, request, full_url): response_headers = {} lambda_backend = self.get_lambda_backend(full_url) @@ -102,3 +119,43 @@ class LambdaResponse(BaseResponse): return region.group(1) else: return self.default_region + + def _list_tags(self, request, full_url): + lambda_backend = self.get_lambda_backend(full_url) + + path = request.path if hasattr(request, 'path') else request.path_url + function_arn = unquote(path.split('/')[-1]) + + if lambda_backend.has_function_arn(function_arn): + function = lambda_backend.get_function_by_arn(function_arn) + return 200, {}, json.dumps(dict(Tags=function.tags)) + else: + return 404, {}, "{}" + + def _tag_resource(self, request, full_url): + lambda_backend = self.get_lambda_backend(full_url) + + path = request.path if hasattr(request, 'path') else request.path_url + function_arn = unquote(path.split('/')[-1]) + + spec = json.loads(self.body) + + if lambda_backend.has_function_arn(function_arn): + lambda_backend.tag_resource(function_arn, spec['Tags']) + return 200, {}, "{}" + else: + return 404, {}, "{}" + + def _untag_resource(self, request, full_url): + lambda_backend = self.get_lambda_backend(full_url) + + path = request.path if hasattr(request, 'path') else request.path_url + function_arn = unquote(path.split('/')[-1].split('?')[0]) + + tag_keys = parse_qs(urlparse(full_url).query)['tagKeys'] + + if lambda_backend.has_function_arn(function_arn): + lambda_backend.untag_resource(function_arn, tag_keys) + return 204, {}, "{}" + else: + return 404, {}, "{}" diff --git a/moto/awslambda/urls.py b/moto/awslambda/urls.py index c63135766..c0917be57 100644 --- a/moto/awslambda/urls.py +++ b/moto/awslambda/urls.py @@ -11,4 +11,5 @@ url_paths = { '{0}/(?P[^/]+)/functions/?$': response.root, '{0}/(?P[^/]+)/functions/(?P[\w_-]+)/?$': response.function, '{0}/(?P[^/]+)/functions/(?P[\w_-]+)/invocations/?$': response.invoke, + '{0}/(?P[^/]+)/tags/(?P.+)': response.tag } diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 007516f56..4ec504365 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -469,3 +469,89 @@ def test_invoke_lambda_error(): assert 'FunctionError' in result assert result['FunctionError'] == 'Handled' + +@mock_lambda +@mock_s3 +def test_tags(): + """ + test list_tags -> tag_resource -> list_tags -> tag_resource -> list_tags -> untag_resource -> list_tags integration + """ + s3_conn = boto3.client('s3', 'us-west-2') + s3_conn.create_bucket(Bucket='test-bucket') + + zip_content = get_test_zip_file2() + s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content) + conn = boto3.client('lambda', 'us-west-2') + + function = conn.create_function( + FunctionName='testFunction', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.handler', + Code={ + 'S3Bucket': 'test-bucket', + 'S3Key': 'test.zip', + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + + # List tags when there are none + conn.list_tags( + Resource=function['FunctionArn'] + )['Tags'].should.equal(dict()) + + # List tags when there is one + conn.tag_resource( + Resource=function['FunctionArn'], + Tags=dict(spam='eggs') + )['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + conn.list_tags( + Resource=function['FunctionArn'] + )['Tags'].should.equal(dict(spam='eggs')) + + # List tags when another has been added + conn.tag_resource( + Resource=function['FunctionArn'], + Tags=dict(foo='bar') + )['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + conn.list_tags( + Resource=function['FunctionArn'] + )['Tags'].should.equal(dict(spam='eggs', foo='bar')) + + # Untag resource + conn.untag_resource( + Resource=function['FunctionArn'], + TagKeys=['spam', 'trolls'] + )['ResponseMetadata']['HTTPStatusCode'].should.equal(204) + conn.list_tags( + Resource=function['FunctionArn'] + )['Tags'].should.equal(dict(foo='bar')) + + # Untag a tag that does not exist (no error and no change) + conn.untag_resource( + Resource=function['FunctionArn'], + TagKeys=['spam'] + )['ResponseMetadata']['HTTPStatusCode'].should.equal(204) + +@mock_lambda +def test_tags_not_found(): + """ + Test list_tags and tag_resource when the lambda with the given arn does not exist + """ + conn = boto3.client('lambda', 'us-west-2') + conn.list_tags.when.called_with( + Resource='arn:aws:lambda:123456789012:function:not-found' + ).should.throw(botocore.client.ClientError) + + conn.tag_resource.when.called_with( + Resource='arn:aws:lambda:123456789012:function:not-found', + Tags=dict(spam='eggs') + ).should.throw(botocore.client.ClientError) + + conn.untag_resource.when.called_with( + Resource='arn:aws:lambda:123456789012:function:not-found', + TagKeys=['spam'] + ).should.throw(botocore.client.ClientError) From b2423f44acc2fde2bd6c781c05af2fae9095808f Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 13 Sep 2017 15:25:16 -0700 Subject: [PATCH 5/5] bumping to version 1.1.6 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b28ff0e12..7d1c745dd 100755 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ extras_require = { setup( name='moto', - version='1.1.5', + version='1.1.6', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec',