dynamodb: filter expression should applied after query/scan. fixes spulec/moto#3909 (#4805)

moto/dynamodb2/models/__init__.py

@@ -834,21 +834,23 @@ class Table(CloudFormationModel):
 
         scanned_count = len(list(self.all_items()))
 
-        if filter_expression is not None:
-            results = [item for item in results if filter_expression.expr(item)]
-
         results = copy.deepcopy(results)
         if index_name:
             index = self.get_index(index_name)
             for result in results:
                 index.project(result)
-        if projection_expression:
-            for result in results:
-                result.filter(projection_expression)
 
         results, last_evaluated_key = self._trim_results(
             results, limit, exclusive_start_key, scanned_index=index_name
         )
+
+        if filter_expression is not None:
+            results = [item for item in results if filter_expression.expr(item)]
+
+        if projection_expression:
+            for result in results:
+                result.filter(projection_expression)
+
         return results, scanned_count, last_evaluated_key
 
     def all_items(self):
@@ -927,20 +929,21 @@ class Table(CloudFormationModel):
                     passes_all_conditions = False
                     break
 
-            if filter_expression is not None:
-                passes_all_conditions &= filter_expression.expr(item)
-
             if passes_all_conditions:
                 results.append(item)
 
+        results, last_evaluated_key = self._trim_results(
+            results, limit, exclusive_start_key, scanned_index=index_name
+        )
+
+        if filter_expression is not None:
+            results = [item for item in results if filter_expression.expr(item)]
+
         if projection_expression:
             results = copy.deepcopy(results)
             for result in results:
                 result.filter(projection_expression)
 
-        results, last_evaluated_key = self._trim_results(
-            results, limit, exclusive_start_key, scanned_index=index_name
-        )
         return results, scanned_count, last_evaluated_key
 
     def _trim_results(self, results, limit, exclusive_start_key, scanned_index=None):

tests/test_dynamodb2/test_dynamodb.py

@@ -5749,3 +5749,151 @@ def test_gsi_lastevaluatedkey():
     last_evaluated_key = response["LastEvaluatedKey"]
     last_evaluated_key.should.have.length_of(2)
     last_evaluated_key.should.equal({"main_key": "testkey1", "index_key": "indexkey"})
+
+
+@mock_dynamodb2
+def test_filter_expression_execution_order():
+    # As mentioned here: https://github.com/spulec/moto/issues/3909
+    # and documented here: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.html#Query.FilterExpression
+    # the filter expression should be evaluated after the query.
+    # The same applies to scan operations:
+    # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Scan.html#Scan.FilterExpression
+
+    # If we set limit=1 and apply a filter expression whixh excludes the first result
+    # then we should get no items in response.
+
+    conn = boto3.resource("dynamodb", region_name="us-west-2")
+    name = "test-filter-expression-table"
+    table = conn.Table(name)
+
+    conn.create_table(
+        TableName=name,
+        KeySchema=[
+            {"AttributeName": "hash_key", "KeyType": "HASH"},
+            {"AttributeName": "range_key", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "hash_key", "AttributeType": "S"},
+            {"AttributeName": "range_key", "AttributeType": "S"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+
+    table.put_item(
+        Item={"hash_key": "keyvalue", "range_key": "A", "filtered_attribute": "Y"},
+    )
+    table.put_item(
+        Item={"hash_key": "keyvalue", "range_key": "B", "filtered_attribute": "Z"},
+    )
+
+    # test query
+
+    query_response_1 = table.query(
+        Limit=1,
+        KeyConditionExpression=Key("hash_key").eq("keyvalue"),
+        FilterExpression=Attr("filtered_attribute").eq("Z"),
+    )
+
+    query_items_1 = query_response_1["Items"]
+    query_items_1.should.have.length_of(0)
+
+    query_last_evaluated_key = query_response_1["LastEvaluatedKey"]
+    query_last_evaluated_key.should.have.length_of(2)
+    query_last_evaluated_key.should.equal({"hash_key": "keyvalue", "range_key": "A"})
+
+    query_response_2 = table.query(
+        Limit=1,
+        KeyConditionExpression=Key("hash_key").eq("keyvalue"),
+        FilterExpression=Attr("filtered_attribute").eq("Z"),
+        ExclusiveStartKey=query_last_evaluated_key,
+    )
+
+    query_items_2 = query_response_2["Items"]
+    query_items_2.should.have.length_of(1)
+    query_items_2[0].should.equal(
+        {"hash_key": "keyvalue", "filtered_attribute": "Z", "range_key": "B"}
+    )
+
+    # test scan
+
+    scan_response_1 = table.scan(
+        Limit=1, FilterExpression=Attr("filtered_attribute").eq("Z"),
+    )
+
+    scan_items_1 = scan_response_1["Items"]
+    scan_items_1.should.have.length_of(0)
+
+    scan_last_evaluated_key = scan_response_1["LastEvaluatedKey"]
+    scan_last_evaluated_key.should.have.length_of(2)
+    scan_last_evaluated_key.should.equal({"hash_key": "keyvalue", "range_key": "A"})
+
+    scan_response_2 = table.scan(
+        Limit=1,
+        FilterExpression=Attr("filtered_attribute").eq("Z"),
+        ExclusiveStartKey=query_last_evaluated_key,
+    )
+
+    scan_items_2 = scan_response_2["Items"]
+    scan_items_2.should.have.length_of(1)
+    scan_items_2[0].should.equal(
+        {"hash_key": "keyvalue", "filtered_attribute": "Z", "range_key": "B"}
+    )
+
+
+@mock_dynamodb2
+def test_projection_expression_execution_order():
+    # projection expression needs to be applied after calculation of
+    # LastEvaluatedKey as it is possible for LastEvaluatedKey to
+    # include attributes which are not projected.
+
+    conn = boto3.resource("dynamodb", region_name="us-west-2")
+    name = "test-projection-expression-with-gsi"
+    table = conn.Table(name)
+
+    conn.create_table(
+        TableName=name,
+        KeySchema=[
+            {"AttributeName": "hash_key", "KeyType": "HASH"},
+            {"AttributeName": "range_key", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "hash_key", "AttributeType": "S"},
+            {"AttributeName": "range_key", "AttributeType": "S"},
+            {"AttributeName": "index_key", "AttributeType": "S"},
+        ],
+        GlobalSecondaryIndexes=[
+            {
+                "IndexName": "test_index",
+                "KeySchema": [{"AttributeName": "index_key", "KeyType": "HASH"}],
+                "Projection": {"ProjectionType": "ALL",},
+                "ProvisionedThroughput": {
+                    "ReadCapacityUnits": 1,
+                    "WriteCapacityUnits": 1,
+                },
+            }
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+
+    table.put_item(Item={"hash_key": "keyvalue", "range_key": "A", "index_key": "Z"},)
+    table.put_item(Item={"hash_key": "keyvalue", "range_key": "B", "index_key": "Z"},)
+
+    # test query
+
+    # if projection expression is applied before LastEvaluatedKey is computed
+    # then this raises an exception.
+    table.query(
+        Limit=1,
+        IndexName="test_index",
+        KeyConditionExpression=Key("index_key").eq("Z"),
+        ProjectionExpression="#a",
+        ExpressionAttributeNames={"#a": "hashKey"},
+    )
+    # if projection expression is applied before LastEvaluatedKey is computed
+    # then this raises an exception.
+    table.scan(
+        Limit=1,
+        IndexName="test_index",
+        ProjectionExpression="#a",
+        ExpressionAttributeNames={"#a": "hashKey"},
+    )