From e50ce7287dedabdb477e726d2289ff0e17e97189 Mon Sep 17 00:00:00 2001 From: IVIURRAY Date: Wed, 26 Jun 2019 21:54:48 +0100 Subject: [PATCH 1/5] ProjectionExpression works with table.scan() --- moto/dynamodb2/models.py | 14 ++++- moto/dynamodb2/responses.py | 5 +- tests/test_dynamodb2/test_dynamodb.py | 84 +++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 6bcde41b2..f04bb830f 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -724,7 +724,7 @@ class Table(BaseModel): if idx_col_set.issubset(set(hash_set.attrs)): yield hash_set - def scan(self, filters, limit, exclusive_start_key, filter_expression=None, index_name=None): + def scan(self, filters, limit, exclusive_start_key, filter_expression=None, index_name=None, projection_expression=None): results = [] scanned_count = 0 all_indexes = self.all_indexes() @@ -763,6 +763,14 @@ class Table(BaseModel): if passes_all_conditions: results.append(item) + if projection_expression: + expressions = [x.strip() for x in projection_expression.split(',')] + results = copy.deepcopy(results) + for result in results: + for attr in list(result.attrs): + if attr not in expressions: + result.attrs.pop(attr) + results, last_evaluated_key = self._trim_results(results, limit, exclusive_start_key, index_name) return results, scanned_count, last_evaluated_key @@ -962,7 +970,7 @@ class DynamoDBBackend(BaseBackend): return table.query(hash_key, range_comparison, range_values, limit, exclusive_start_key, scan_index_forward, projection_expression, index_name, filter_expression, **filter_kwargs) - def scan(self, table_name, filters, limit, exclusive_start_key, filter_expression, expr_names, expr_values, index_name): + def scan(self, table_name, filters, limit, exclusive_start_key, filter_expression, expr_names, expr_values, index_name, projection_expression): table = self.tables.get(table_name) if not table: return None, None, None @@ -977,7 +985,7 @@ class DynamoDBBackend(BaseBackend): else: filter_expression = Op(None, None) # Will always eval to true - return table.scan(scan_filters, limit, exclusive_start_key, filter_expression, index_name) + return table.scan(scan_filters, limit, exclusive_start_key, filter_expression, index_name, projection_expression) def update_item(self, table_name, key, update_expression, attribute_updates, expression_attribute_names, expression_attribute_values, expected=None): diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 7eb565747..01f2df088 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -558,7 +558,7 @@ class DynamoHandler(BaseResponse): filter_expression = self.body.get('FilterExpression') expression_attribute_values = self.body.get('ExpressionAttributeValues', {}) expression_attribute_names = self.body.get('ExpressionAttributeNames', {}) - + projection_expression = self.body.get('ProjectionExpression') exclusive_start_key = self.body.get('ExclusiveStartKey') limit = self.body.get("Limit") index_name = self.body.get('IndexName') @@ -570,7 +570,8 @@ class DynamoHandler(BaseResponse): filter_expression, expression_attribute_names, expression_attribute_values, - index_name) + index_name, + projection_expression) except InvalidIndexNameError as err: er = 'com.amazonaws.dynamodb.v20111205#ValidationException' return self.error(er, str(err)) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 77846de04..67610ad72 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -452,6 +452,90 @@ def test_basic_projection_expressions(): assert 'body' in results['Items'][1] assert 'forum_name' in results['Items'][1] +@mock_dynamodb2 +def test_basic_projection_expressions_using_scan(): + dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName='users', + KeySchema=[ + { + 'AttributeName': 'forum_name', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'subject', + 'KeyType': 'RANGE' + }, + ], + AttributeDefinitions=[ + { + 'AttributeName': 'forum_name', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'subject', + 'AttributeType': 'S' + }, + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 5, + 'WriteCapacityUnits': 5 + } + ) + table = dynamodb.Table('users') + + table.put_item(Item={ + 'forum_name': 'the-key', + 'subject': '123', + 'body': 'some test message' + }) + + table.put_item(Item={ + 'forum_name': 'not-the-key', + 'subject': '123', + 'body': 'some other test message' + }) + # Test a query returning all items + results = table.scan( + FilterExpression=Key('forum_name').eq( + 'the-key'), + ProjectionExpression='body, subject' + ) + + assert 'body' in results['Items'][0] + assert results['Items'][0]['body'] == 'some test message' + assert 'subject' in results['Items'][0] + + table.put_item(Item={ + 'forum_name': 'the-key', + 'subject': '1234', + 'body': 'yet another test message' + }) + + results = table.scan( + FilterExpression=Key('forum_name').eq( + 'the-key'), + ProjectionExpression='body' + ) + + assert 'body' in results['Items'][0] + assert 'subject' not in results['Items'][0] + assert results['Items'][0]['body'] == 'some test message' + assert 'body' in results['Items'][1] + assert 'subject' not in results['Items'][1] + assert results['Items'][1]['body'] == 'yet another test message' + + # The projection expression should not remove data from storage + results = table.query( + KeyConditionExpression=Key('forum_name').eq( + 'the-key'), + ) + assert 'subject' in results['Items'][0] + assert 'body' in results['Items'][1] + assert 'forum_name' in results['Items'][1] + @mock_dynamodb2 def test_basic_projection_expressions_with_attr_expression_names(): From b60097fab2603420ed8dc3b793b77d1f83284f32 Mon Sep 17 00:00:00 2001 From: IVIURRAY Date: Thu, 27 Jun 2019 19:08:32 +0100 Subject: [PATCH 2/5] improve test case --- tests/test_dynamodb2/test_dynamodb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 67610ad72..532ee4a62 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -522,10 +522,10 @@ def test_basic_projection_expressions_using_scan(): assert 'body' in results['Items'][0] assert 'subject' not in results['Items'][0] - assert results['Items'][0]['body'] == 'some test message' + assert 'forum_name' not in results['Items'][0] assert 'body' in results['Items'][1] assert 'subject' not in results['Items'][1] - assert results['Items'][1]['body'] == 'yet another test message' + assert 'forum_name' not in results['Items'][1] # The projection expression should not remove data from storage results = table.query( @@ -536,6 +536,7 @@ def test_basic_projection_expressions_using_scan(): assert 'body' in results['Items'][1] assert 'forum_name' in results['Items'][1] +test_basic_projection_expressions_using_scan() @mock_dynamodb2 def test_basic_projection_expressions_with_attr_expression_names(): From 949637a14cf407d91bddef94b07187a7583f782e Mon Sep 17 00:00:00 2001 From: IVIURRAY Date: Thu, 27 Jun 2019 19:12:53 +0100 Subject: [PATCH 3/5] remove debug --- tests/test_dynamodb2/test_dynamodb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 532ee4a62..e0d44fb82 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -536,7 +536,6 @@ def test_basic_projection_expressions_using_scan(): assert 'body' in results['Items'][1] assert 'forum_name' in results['Items'][1] -test_basic_projection_expressions_using_scan() @mock_dynamodb2 def test_basic_projection_expressions_with_attr_expression_names(): From 4ce0e6bbcb51763cb281977fd3be7c937da23df5 Mon Sep 17 00:00:00 2001 From: IVIURRAY Date: Thu, 27 Jun 2019 19:37:46 +0100 Subject: [PATCH 4/5] add extra test for ProjectionExpression using scan with ExpressionAttributes --- moto/dynamodb2/models.py | 3 + moto/dynamodb2/responses.py | 2 +- tests/test_dynamodb2/test_dynamodb.py | 80 ++++++++++++++++++++++++++- 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index f04bb830f..fb34e19cc 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -985,6 +985,9 @@ class DynamoDBBackend(BaseBackend): else: filter_expression = Op(None, None) # Will always eval to true + projection_expression = ','.join([expr_names[attr] if attr in expr_names else attr + for attr in projection_expression.replace(' ', '').split(',')]) + return table.scan(scan_filters, limit, exclusive_start_key, filter_expression, index_name, projection_expression) def update_item(self, table_name, key, update_expression, attribute_updates, expression_attribute_names, diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 01f2df088..943340438 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -558,7 +558,7 @@ class DynamoHandler(BaseResponse): filter_expression = self.body.get('FilterExpression') expression_attribute_values = self.body.get('ExpressionAttributeValues', {}) expression_attribute_names = self.body.get('ExpressionAttributeNames', {}) - projection_expression = self.body.get('ProjectionExpression') + projection_expression = self.body.get('ProjectionExpression', '') exclusive_start_key = self.body.get('ExclusiveStartKey') limit = self.body.get("Limit") index_name = self.body.get('IndexName') diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index e0d44fb82..faa467aab 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -497,7 +497,7 @@ def test_basic_projection_expressions_using_scan(): 'subject': '123', 'body': 'some other test message' }) - # Test a query returning all items + # Test a scan returning all items results = table.scan( FilterExpression=Key('forum_name').eq( 'the-key'), @@ -603,6 +603,84 @@ def test_basic_projection_expressions_with_attr_expression_names(): assert 'attachment' in results['Items'][0] assert results['Items'][0]['attachment'] == 'something' +@mock_dynamodb2 +def test_basic_projection_expressions_using_scan_with_attr_expression_names(): + dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + + # Create the DynamoDB table. + table = dynamodb.create_table( + TableName='users', + KeySchema=[ + { + 'AttributeName': 'forum_name', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'subject', + 'KeyType': 'RANGE' + }, + ], + AttributeDefinitions=[ + { + 'AttributeName': 'forum_name', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'subject', + 'AttributeType': 'S' + }, + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 5, + 'WriteCapacityUnits': 5 + } + ) + table = dynamodb.Table('users') + + table.put_item(Item={ + 'forum_name': 'the-key', + 'subject': '123', + 'body': 'some test message', + 'attachment': 'something' + }) + + table.put_item(Item={ + 'forum_name': 'not-the-key', + 'subject': '123', + 'body': 'some other test message', + 'attachment': 'something' + }) + # Test a scan returning all items + + results = table.scan( + FilterExpression=Key('forum_name').eq( + 'the-key'), + ProjectionExpression='#rl, #rt, subject', + ExpressionAttributeNames={ + '#rl': 'body', + '#rt': 'attachment' + }, + ) + + assert 'body' in results['Items'][0] + assert 'attachment' in results['Items'][0] + assert 'subject' in results['Items'][0] + assert 'form_name' not in results['Items'][0] + + # Test without a FilterExpression + results = table.scan( + ProjectionExpression='#rl, #rt, subject', + ExpressionAttributeNames={ + '#rl': 'body', + '#rt': 'attachment' + }, + ) + + assert 'body' in results['Items'][0] + assert 'attachment' in results['Items'][0] + assert 'subject' in results['Items'][0] + assert 'form_name' not in results['Items'][0] + @mock_dynamodb2 def test_put_item_returns_consumed_capacity(): From cd666758f609e1067cf182880e7c4dd7525e4732 Mon Sep 17 00:00:00 2001 From: IVIURRAY Date: Thu, 27 Jun 2019 20:13:36 +0100 Subject: [PATCH 5/5] one liner --- moto/dynamodb2/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index fb34e19cc..bfbb654b4 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -985,8 +985,7 @@ class DynamoDBBackend(BaseBackend): else: filter_expression = Op(None, None) # Will always eval to true - projection_expression = ','.join([expr_names[attr] if attr in expr_names else attr - for attr in projection_expression.replace(' ', '').split(',')]) + projection_expression = ','.join([expr_names.get(attr, attr) for attr in projection_expression.replace(' ', '').split(',')]) return table.scan(scan_filters, limit, exclusive_start_key, filter_expression, index_name, projection_expression)