diff --git a/moto/dynamodb2/models/__init__.py b/moto/dynamodb2/models/__init__.py index 6c03482c7..2e9284f3c 100644 --- a/moto/dynamodb2/models/__init__.py +++ b/moto/dynamodb2/models/__init__.py @@ -834,21 +834,23 @@ class Table(CloudFormationModel): scanned_count = len(list(self.all_items())) - if filter_expression is not None: - results = [item for item in results if filter_expression.expr(item)] - results = copy.deepcopy(results) if index_name: index = self.get_index(index_name) for result in results: index.project(result) - if projection_expression: - for result in results: - result.filter(projection_expression) results, last_evaluated_key = self._trim_results( results, limit, exclusive_start_key, scanned_index=index_name ) + + if filter_expression is not None: + results = [item for item in results if filter_expression.expr(item)] + + if projection_expression: + for result in results: + result.filter(projection_expression) + return results, scanned_count, last_evaluated_key def all_items(self): @@ -927,20 +929,21 @@ class Table(CloudFormationModel): passes_all_conditions = False break - if filter_expression is not None: - passes_all_conditions &= filter_expression.expr(item) - if passes_all_conditions: results.append(item) + results, last_evaluated_key = self._trim_results( + results, limit, exclusive_start_key, scanned_index=index_name + ) + + if filter_expression is not None: + results = [item for item in results if filter_expression.expr(item)] + if projection_expression: results = copy.deepcopy(results) for result in results: result.filter(projection_expression) - results, last_evaluated_key = self._trim_results( - results, limit, exclusive_start_key, scanned_index=index_name - ) return results, scanned_count, last_evaluated_key def _trim_results(self, results, limit, exclusive_start_key, scanned_index=None): diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index e0631241c..594fdf3ad 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -5749,3 +5749,151 @@ def test_gsi_lastevaluatedkey(): last_evaluated_key = response["LastEvaluatedKey"] last_evaluated_key.should.have.length_of(2) last_evaluated_key.should.equal({"main_key": "testkey1", "index_key": "indexkey"}) + + +@mock_dynamodb2 +def test_filter_expression_execution_order(): + # As mentioned here: https://github.com/spulec/moto/issues/3909 + # and documented here: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.html#Query.FilterExpression + # the filter expression should be evaluated after the query. + # The same applies to scan operations: + # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Scan.html#Scan.FilterExpression + + # If we set limit=1 and apply a filter expression whixh excludes the first result + # then we should get no items in response. + + conn = boto3.resource("dynamodb", region_name="us-west-2") + name = "test-filter-expression-table" + table = conn.Table(name) + + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "hash_key", "KeyType": "HASH"}, + {"AttributeName": "range_key", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "hash_key", "AttributeType": "S"}, + {"AttributeName": "range_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + table.put_item( + Item={"hash_key": "keyvalue", "range_key": "A", "filtered_attribute": "Y"}, + ) + table.put_item( + Item={"hash_key": "keyvalue", "range_key": "B", "filtered_attribute": "Z"}, + ) + + # test query + + query_response_1 = table.query( + Limit=1, + KeyConditionExpression=Key("hash_key").eq("keyvalue"), + FilterExpression=Attr("filtered_attribute").eq("Z"), + ) + + query_items_1 = query_response_1["Items"] + query_items_1.should.have.length_of(0) + + query_last_evaluated_key = query_response_1["LastEvaluatedKey"] + query_last_evaluated_key.should.have.length_of(2) + query_last_evaluated_key.should.equal({"hash_key": "keyvalue", "range_key": "A"}) + + query_response_2 = table.query( + Limit=1, + KeyConditionExpression=Key("hash_key").eq("keyvalue"), + FilterExpression=Attr("filtered_attribute").eq("Z"), + ExclusiveStartKey=query_last_evaluated_key, + ) + + query_items_2 = query_response_2["Items"] + query_items_2.should.have.length_of(1) + query_items_2[0].should.equal( + {"hash_key": "keyvalue", "filtered_attribute": "Z", "range_key": "B"} + ) + + # test scan + + scan_response_1 = table.scan( + Limit=1, FilterExpression=Attr("filtered_attribute").eq("Z"), + ) + + scan_items_1 = scan_response_1["Items"] + scan_items_1.should.have.length_of(0) + + scan_last_evaluated_key = scan_response_1["LastEvaluatedKey"] + scan_last_evaluated_key.should.have.length_of(2) + scan_last_evaluated_key.should.equal({"hash_key": "keyvalue", "range_key": "A"}) + + scan_response_2 = table.scan( + Limit=1, + FilterExpression=Attr("filtered_attribute").eq("Z"), + ExclusiveStartKey=query_last_evaluated_key, + ) + + scan_items_2 = scan_response_2["Items"] + scan_items_2.should.have.length_of(1) + scan_items_2[0].should.equal( + {"hash_key": "keyvalue", "filtered_attribute": "Z", "range_key": "B"} + ) + + +@mock_dynamodb2 +def test_projection_expression_execution_order(): + # projection expression needs to be applied after calculation of + # LastEvaluatedKey as it is possible for LastEvaluatedKey to + # include attributes which are not projected. + + conn = boto3.resource("dynamodb", region_name="us-west-2") + name = "test-projection-expression-with-gsi" + table = conn.Table(name) + + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "hash_key", "KeyType": "HASH"}, + {"AttributeName": "range_key", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "hash_key", "AttributeType": "S"}, + {"AttributeName": "range_key", "AttributeType": "S"}, + {"AttributeName": "index_key", "AttributeType": "S"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "test_index", + "KeySchema": [{"AttributeName": "index_key", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL",}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1, + }, + } + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + table.put_item(Item={"hash_key": "keyvalue", "range_key": "A", "index_key": "Z"},) + table.put_item(Item={"hash_key": "keyvalue", "range_key": "B", "index_key": "Z"},) + + # test query + + # if projection expression is applied before LastEvaluatedKey is computed + # then this raises an exception. + table.query( + Limit=1, + IndexName="test_index", + KeyConditionExpression=Key("index_key").eq("Z"), + ProjectionExpression="#a", + ExpressionAttributeNames={"#a": "hashKey"}, + ) + # if projection expression is applied before LastEvaluatedKey is computed + # then this raises an exception. + table.scan( + Limit=1, + IndexName="test_index", + ProjectionExpression="#a", + ExpressionAttributeNames={"#a": "hashKey"}, + )