From 599446fee27e2ec044731cc3d832d57f30895c2f Mon Sep 17 00:00:00 2001 From: Filips Nastins <47500046+filipsnastins@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:34:31 +0100 Subject: [PATCH] DynamoDB: raise validation error on consistent read on GSI (#7450) --- moto/dynamodb/models/__init__.py | 4 + moto/dynamodb/models/table.py | 24 ++++- moto/dynamodb/responses.py | 5 ++ .../exceptions/test_dynamodb_exceptions.py | 87 +++++++++++++++++++ .../test_dynamodb_table_with_range_key.py | 76 +++++++++++++++- .../test_dynamodb_table_without_range_key.py | 4 + 6 files changed, 195 insertions(+), 5 deletions(-) diff --git a/moto/dynamodb/models/__init__.py b/moto/dynamodb/models/__init__.py index 0627a238e..60a075471 100644 --- a/moto/dynamodb/models/__init__.py +++ b/moto/dynamodb/models/__init__.py @@ -318,6 +318,7 @@ class DynamoDBBackend(BaseBackend): scan_index_forward: bool, projection_expressions: Optional[List[List[str]]], index_name: Optional[str] = None, + consistent_read: bool = False, expr_names: Optional[Dict[str, str]] = None, expr_values: Optional[Dict[str, Dict[str, str]]] = None, filter_expression: Optional[str] = None, @@ -341,6 +342,7 @@ class DynamoDBBackend(BaseBackend): scan_index_forward, projection_expressions, index_name, + consistent_read, filter_expression_op, **filter_kwargs, ) @@ -355,6 +357,7 @@ class DynamoDBBackend(BaseBackend): expr_names: Dict[str, Any], expr_values: Dict[str, Any], index_name: str, + consistent_read: bool, projection_expression: Optional[List[List[str]]], ) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]: table = self.get_table(table_name) @@ -374,6 +377,7 @@ class DynamoDBBackend(BaseBackend): exclusive_start_key, filter_expression_op, index_name, + consistent_read, projection_expression, ) diff --git a/moto/dynamodb/models/table.py b/moto/dynamodb/models/table.py index 99a47e8eb..aa7596694 100644 --- a/moto/dynamodb/models/table.py +++ b/moto/dynamodb/models/table.py @@ -653,6 +653,7 @@ class Table(CloudFormationModel): scan_index_forward: bool, projection_expressions: Optional[List[List[str]]], index_name: Optional[str] = None, + consistent_read: bool = False, filter_expression: Any = None, **filter_kwargs: Any, ) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]: @@ -668,6 +669,12 @@ class Table(CloudFormationModel): ) index = indexes_by_name[index_name] + + if consistent_read and index in self.global_indexes: + raise MockValidationException( + "Consistent reads are not supported on global secondary indexes" + ) + try: index_hash_key = [ key for key in index.schema if key["KeyType"] == "HASH" @@ -715,9 +722,11 @@ class Table(CloudFormationModel): return float(x.value) if x.type == "N" else x.value possible_results.sort( - key=lambda item: conv(item.attrs[index_range_key["AttributeName"]]) # type: ignore - if item.attrs.get(index_range_key["AttributeName"]) - else None + key=lambda item: ( # type: ignore + conv(item.attrs[index_range_key["AttributeName"]]) # type: ignore + if item.attrs.get(index_range_key["AttributeName"]) + else None + ) ) else: possible_results.sort(key=lambda item: item.range_key) # type: ignore @@ -834,6 +843,7 @@ class Table(CloudFormationModel): exclusive_start_key: Dict[str, Any], filter_expression: Any = None, index_name: Optional[str] = None, + consistent_read: bool = False, projection_expression: Optional[List[List[str]]] = None, ) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]: results: List[Item] = [] @@ -841,7 +851,13 @@ class Table(CloudFormationModel): scanned_count = 0 if index_name: - self.get_index(index_name, error_if_not=True) + index = self.get_index(index_name, error_if_not=True) + + if consistent_read and index in self.global_indexes: + raise MockValidationException( + "Consistent reads are not supported on global secondary indexes" + ) + items = self.has_idx_items(index_name) else: items = self.all_items() diff --git a/moto/dynamodb/responses.py b/moto/dynamodb/responses.py index 9f03efa2c..b6b02df38 100644 --- a/moto/dynamodb/responses.py +++ b/moto/dynamodb/responses.py @@ -731,6 +731,8 @@ class DynamoHandler(BaseResponse): exclusive_start_key = self.body.get("ExclusiveStartKey") limit = self.body.get("Limit") scan_index_forward = self.body.get("ScanIndexForward") + consistent_read = self.body.get("ConsistentRead", False) + items, scanned_count, last_evaluated_key = self.dynamodb_backend.query( name, hash_key, @@ -741,6 +743,7 @@ class DynamoHandler(BaseResponse): scan_index_forward, projection_expressions, index_name=index_name, + consistent_read=consistent_read, expr_names=expression_attribute_names, expr_values=expression_attribute_values, filter_expression=filter_expression, @@ -801,6 +804,7 @@ class DynamoHandler(BaseResponse): exclusive_start_key = self.body.get("ExclusiveStartKey") limit = self.body.get("Limit") index_name = self.body.get("IndexName") + consistent_read = self.body.get("ConsistentRead", False) projection_expressions = self._adjust_projection_expression( projection_expression, expression_attribute_names @@ -816,6 +820,7 @@ class DynamoHandler(BaseResponse): expression_attribute_names, expression_attribute_values, index_name, + consistent_read, projection_expressions, ) except ValueError as err: diff --git a/tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py b/tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py index 486d3dbf1..a29f15e09 100644 --- a/tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py +++ b/tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py @@ -1273,3 +1273,90 @@ def test_too_many_key_schema_attributes(): err = exc.value.response["Error"] assert err["Code"] == "ValidationException" assert err["Message"] == expected_err + + +@mock_aws +def test_cannot_query_gsi_with_consistent_read(): + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "gsi_hash_key", "AttributeType": "S"}, + {"AttributeName": "gsi_range_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + GlobalSecondaryIndexes=[ + { + "IndexName": "test_gsi", + "KeySchema": [ + {"AttributeName": "gsi_hash_key", "KeyType": "HASH"}, + {"AttributeName": "gsi_range_key", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1, + }, + } + ], + ) + + with pytest.raises(ClientError) as exc: + dynamodb.query( + TableName="test", + IndexName="test_gsi", + KeyConditionExpression="gsi_hash_key = :gsi_hash_key and gsi_range_key = :gsi_range_key", + ExpressionAttributeValues={ + ":gsi_hash_key": {"S": "key1"}, + ":gsi_range_key": {"S": "range1"}, + }, + ConsistentRead=True, + ) + + assert exc.value.response["Error"] == { + "Code": "ValidationException", + "Message": "Consistent reads are not supported on global secondary indexes", + } + + +@mock_aws +def test_cannot_scan_gsi_with_consistent_read(): + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + dynamodb.create_table( + TableName="test", + KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}], + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "gsi_hash_key", "AttributeType": "S"}, + {"AttributeName": "gsi_range_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + GlobalSecondaryIndexes=[ + { + "IndexName": "test_gsi", + "KeySchema": [ + {"AttributeName": "gsi_hash_key", "KeyType": "HASH"}, + {"AttributeName": "gsi_range_key", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, + "WriteCapacityUnits": 1, + }, + } + ], + ) + + with pytest.raises(ClientError) as exc: + dynamodb.scan( + TableName="test", + IndexName="test_gsi", + ConsistentRead=True, + ) + + assert exc.value.response["Error"] == { + "Code": "ValidationException", + "Message": "Consistent reads are not supported on global secondary indexes", + } diff --git a/tests/test_dynamodb/test_dynamodb_table_with_range_key.py b/tests/test_dynamodb/test_dynamodb_table_with_range_key.py index aafad7f1c..16fb11bf5 100644 --- a/tests/test_dynamodb/test_dynamodb_table_with_range_key.py +++ b/tests/test_dynamodb/test_dynamodb_table_with_range_key.py @@ -812,7 +812,6 @@ def test_boto3_query_gsi_range_comparison(): # And reverse order of hash + range key results = table.query( KeyConditionExpression=Key("created").gt(1) & Key("username").eq("johndoe"), - ConsistentRead=True, IndexName="TestGSI", ) assert results["Count"] == 2 @@ -1096,6 +1095,76 @@ def test_query_pagination(): assert subjects == set(range(10)) +@mock_aws +def test_query_by_local_secondary_index(): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + + table = dynamodb.create_table( + TableName="test", + KeySchema=[ + {"AttributeName": "id", "KeyType": "HASH"}, + {"AttributeName": "range_key", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + {"AttributeName": "range_key", "AttributeType": "S"}, + {"AttributeName": "lsi_range_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1}, + LocalSecondaryIndexes=[ + { + "IndexName": "test_lsi", + "KeySchema": [ + {"AttributeName": "id", "KeyType": "HASH"}, + {"AttributeName": "lsi_range_key", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) + + table.put_item( + Item={ + "id": "1", + "range_key": "1", + "col1": "val1", + "lsi_range_key": "1", + }, + ) + + table.put_item( + Item={ + "id": "1", + "range_key": "2", + "col1": "val2", + "lsi_range_key": "2", + }, + ) + + table.put_item( + Item={"id": "3", "range_key": "1", "col1": "val3"}, + ) + + res = table.query( + KeyConditionExpression=Key("id").eq("1") & Key("lsi_range_key").eq("1"), + IndexName="test_lsi", + ) + assert res["Count"] == 1 + assert res["Items"] == [ + {"id": "1", "range_key": "1", "col1": "val1", "lsi_range_key": "1"} + ] + + res = table.query( + KeyConditionExpression=Key("id").eq("1") & Key("lsi_range_key").eq("2"), + IndexName="test_lsi", + ConsistentRead=True, + ) + assert res["Count"] == 1 + assert res["Items"] == [ + {"id": "1", "range_key": "2", "col1": "val2", "lsi_range_key": "2"} + ] + + @mock_aws def test_scan_by_index(): dynamodb = boto3.client("dynamodb", region_name="us-east-1") @@ -1206,6 +1275,11 @@ def test_scan_by_index(): assert res["ScannedCount"] == 2 assert len(res["Items"]) == 2 + res = dynamodb.scan(TableName="test", IndexName="test_lsi", ConsistentRead=True) + assert res["Count"] == 2 + assert res["ScannedCount"] == 2 + assert len(res["Items"]) == 2 + res = dynamodb.scan(TableName="test", IndexName="test_lsi", Limit=1) assert res["Count"] == 1 assert res["ScannedCount"] == 1 diff --git a/tests/test_dynamodb/test_dynamodb_table_without_range_key.py b/tests/test_dynamodb/test_dynamodb_table_without_range_key.py index 774ba3f58..ac8afc9dc 100644 --- a/tests/test_dynamodb/test_dynamodb_table_without_range_key.py +++ b/tests/test_dynamodb/test_dynamodb_table_without_range_key.py @@ -579,6 +579,10 @@ def test_scan_by_index(): assert res["Count"] == 3 assert len(res["Items"]) == 3 + res = dynamodb.scan(TableName="test", ConsistentRead=True) + assert res["Count"] == 3 + assert len(res["Items"]) == 3 + res = dynamodb.scan(TableName="test", IndexName="test_gsi") assert res["Count"] == 2 assert len(res["Items"]) == 2