Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

DynamoDB: raise validation error on consistent read on GSI (#7450)

Browse Source
This commit is contained in:
Filips Nastins 2024-03-11 12:34:31 +01:00 committed by GitHub
parent 3ef0f94fd5
commit 599446fee2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 195 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
moto/dynamodb/models/__init__.py
View File

@ -318,6 +318,7 @@ class DynamoDBBackend(BaseBackend):
scan_index_forward: bool,
projection_expressions: Optional[List[List[str]]],
index_name: Optional[str] = None,
consistent_read: bool = False,
expr_names: Optional[Dict[str, str]] = None,
expr_values: Optional[Dict[str, Dict[str, str]]] = None,
filter_expression: Optional[str] = None,
@ -341,6 +342,7 @@ class DynamoDBBackend(BaseBackend):
scan_index_forward,
projection_expressions,
index_name,
consistent_read,
filter_expression_op,
**filter_kwargs,
)
@ -355,6 +357,7 @@ class DynamoDBBackend(BaseBackend):
expr_names: Dict[str, Any],
expr_values: Dict[str, Any],
index_name: str,
consistent_read: bool,
projection_expression: Optional[List[List[str]]],
) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]:
table = self.get_table(table_name)
@ -374,6 +377,7 @@ class DynamoDBBackend(BaseBackend):
exclusive_start_key,
filter_expression_op,
index_name,
consistent_read,
projection_expression,
)

24
moto/dynamodb/models/table.py
View File

@ -653,6 +653,7 @@ class Table(CloudFormationModel):
scan_index_forward: bool,
projection_expressions: Optional[List[List[str]]],
index_name: Optional[str] = None,
consistent_read: bool = False,
filter_expression: Any = None,
**filter_kwargs: Any,
) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]:
@ -668,6 +669,12 @@ class Table(CloudFormationModel):
)
index = indexes_by_name[index_name]
if consistent_read and index in self.global_indexes:
raise MockValidationException(
"Consistent reads are not supported on global secondary indexes"
)
try:
index_hash_key = [
key for key in index.schema if key["KeyType"] == "HASH"
@ -715,9 +722,11 @@ class Table(CloudFormationModel):
return float(x.value) if x.type == "N" else x.value
possible_results.sort(
key=lambda item: conv(item.attrs[index_range_key["AttributeName"]]) # type: ignore
if item.attrs.get(index_range_key["AttributeName"])
else None
key=lambda item: ( # type: ignore
conv(item.attrs[index_range_key["AttributeName"]]) # type: ignore
if item.attrs.get(index_range_key["AttributeName"])
else None
)
)
else:
possible_results.sort(key=lambda item: item.range_key) # type: ignore
@ -834,6 +843,7 @@ class Table(CloudFormationModel):
exclusive_start_key: Dict[str, Any],
filter_expression: Any = None,
index_name: Optional[str] = None,
consistent_read: bool = False,
projection_expression: Optional[List[List[str]]] = None,
) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]:
results: List[Item] = []
@ -841,7 +851,13 @@ class Table(CloudFormationModel):
scanned_count = 0
if index_name:
self.get_index(index_name, error_if_not=True)
index = self.get_index(index_name, error_if_not=True)
if consistent_read and index in self.global_indexes:
raise MockValidationException(
"Consistent reads are not supported on global secondary indexes"
)
items = self.has_idx_items(index_name)
else:
items = self.all_items()

5
moto/dynamodb/responses.py
View File

@ -731,6 +731,8 @@ class DynamoHandler(BaseResponse):
exclusive_start_key = self.body.get("ExclusiveStartKey")
limit = self.body.get("Limit")
scan_index_forward = self.body.get("ScanIndexForward")
consistent_read = self.body.get("ConsistentRead", False)
items, scanned_count, last_evaluated_key = self.dynamodb_backend.query(
name,
hash_key,
@ -741,6 +743,7 @@ class DynamoHandler(BaseResponse):
scan_index_forward,
projection_expressions,
index_name=index_name,
consistent_read=consistent_read,
expr_names=expression_attribute_names,
expr_values=expression_attribute_values,
filter_expression=filter_expression,
@ -801,6 +804,7 @@ class DynamoHandler(BaseResponse):
exclusive_start_key = self.body.get("ExclusiveStartKey")
limit = self.body.get("Limit")
index_name = self.body.get("IndexName")
consistent_read = self.body.get("ConsistentRead", False)
projection_expressions = self._adjust_projection_expression(
projection_expression, expression_attribute_names
@ -816,6 +820,7 @@ class DynamoHandler(BaseResponse):
expression_attribute_names,
expression_attribute_values,
index_name,
consistent_read,
projection_expressions,
)
except ValueError as err:

87
tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py
View File

@ -1273,3 +1273,90 @@ def test_too_many_key_schema_attributes():
err = exc.value.response["Error"]
assert err["Code"] == "ValidationException"
assert err["Message"] == expected_err
@mock_aws
def test_cannot_query_gsi_with_consistent_read():
dynamodb = boto3.client("dynamodb", region_name="us-east-1")
dynamodb.create_table(
TableName="test",
KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
AttributeDefinitions=[
{"AttributeName": "id", "AttributeType": "S"},
{"AttributeName": "gsi_hash_key", "AttributeType": "S"},
{"AttributeName": "gsi_range_key", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
GlobalSecondaryIndexes=[
{
"IndexName": "test_gsi",
"KeySchema": [
{"AttributeName": "gsi_hash_key", "KeyType": "HASH"},
{"AttributeName": "gsi_range_key", "KeyType": "RANGE"},
],
"Projection": {"ProjectionType": "ALL"},
"ProvisionedThroughput": {
"ReadCapacityUnits": 1,
"WriteCapacityUnits": 1,
},
}
],
)
with pytest.raises(ClientError) as exc:
dynamodb.query(
TableName="test",
IndexName="test_gsi",
KeyConditionExpression="gsi_hash_key = :gsi_hash_key and gsi_range_key = :gsi_range_key",
ExpressionAttributeValues={
":gsi_hash_key": {"S": "key1"},
":gsi_range_key": {"S": "range1"},
},
ConsistentRead=True,
)
assert exc.value.response["Error"] == {
"Code": "ValidationException",
"Message": "Consistent reads are not supported on global secondary indexes",
}
@mock_aws
def test_cannot_scan_gsi_with_consistent_read():
dynamodb = boto3.client("dynamodb", region_name="us-east-1")
dynamodb.create_table(
TableName="test",
KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
AttributeDefinitions=[
{"AttributeName": "id", "AttributeType": "S"},
{"AttributeName": "gsi_hash_key", "AttributeType": "S"},
{"AttributeName": "gsi_range_key", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
GlobalSecondaryIndexes=[
{
"IndexName": "test_gsi",
"KeySchema": [
{"AttributeName": "gsi_hash_key", "KeyType": "HASH"},
{"AttributeName": "gsi_range_key", "KeyType": "RANGE"},
],
"Projection": {"ProjectionType": "ALL"},
"ProvisionedThroughput": {
"ReadCapacityUnits": 1,
"WriteCapacityUnits": 1,
},
}
],
)
with pytest.raises(ClientError) as exc:
dynamodb.scan(
TableName="test",
IndexName="test_gsi",
ConsistentRead=True,
)
assert exc.value.response["Error"] == {
"Code": "ValidationException",
"Message": "Consistent reads are not supported on global secondary indexes",
}

76
tests/test_dynamodb/test_dynamodb_table_with_range_key.py
View File

@ -812,7 +812,6 @@ def test_boto3_query_gsi_range_comparison():
# And reverse order of hash + range key
results = table.query(
KeyConditionExpression=Key("created").gt(1) & Key("username").eq("johndoe"),
ConsistentRead=True,
IndexName="TestGSI",
)
assert results["Count"] == 2
@ -1096,6 +1095,76 @@ def test_query_pagination():
assert subjects == set(range(10))
@mock_aws
def test_query_by_local_secondary_index():
dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
table = dynamodb.create_table(
TableName="test",
KeySchema=[
{"AttributeName": "id", "KeyType": "HASH"},
{"AttributeName": "range_key", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "id", "AttributeType": "S"},
{"AttributeName": "range_key", "AttributeType": "S"},
{"AttributeName": "lsi_range_key", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
LocalSecondaryIndexes=[
{
"IndexName": "test_lsi",
"KeySchema": [
{"AttributeName": "id", "KeyType": "HASH"},
{"AttributeName": "lsi_range_key", "KeyType": "RANGE"},
],
"Projection": {"ProjectionType": "ALL"},
}
],
)
table.put_item(
Item={
"id": "1",
"range_key": "1",
"col1": "val1",
"lsi_range_key": "1",
},
)
table.put_item(
Item={
"id": "1",
"range_key": "2",
"col1": "val2",
"lsi_range_key": "2",
},
)
table.put_item(
Item={"id": "3", "range_key": "1", "col1": "val3"},
)
res = table.query(
KeyConditionExpression=Key("id").eq("1") & Key("lsi_range_key").eq("1"),
IndexName="test_lsi",
)
assert res["Count"] == 1
assert res["Items"] == [
{"id": "1", "range_key": "1", "col1": "val1", "lsi_range_key": "1"}
]
res = table.query(
KeyConditionExpression=Key("id").eq("1") & Key("lsi_range_key").eq("2"),
IndexName="test_lsi",
ConsistentRead=True,
)
assert res["Count"] == 1
assert res["Items"] == [
{"id": "1", "range_key": "2", "col1": "val2", "lsi_range_key": "2"}
]
@mock_aws
def test_scan_by_index():
dynamodb = boto3.client("dynamodb", region_name="us-east-1")
@ -1206,6 +1275,11 @@ def test_scan_by_index():
assert res["ScannedCount"] == 2
assert len(res["Items"]) == 2
res = dynamodb.scan(TableName="test", IndexName="test_lsi", ConsistentRead=True)
assert res["Count"] == 2
assert res["ScannedCount"] == 2
assert len(res["Items"]) == 2
res = dynamodb.scan(TableName="test", IndexName="test_lsi", Limit=1)
assert res["Count"] == 1
assert res["ScannedCount"] == 1

4
tests/test_dynamodb/test_dynamodb_table_without_range_key.py
View File

@ -579,6 +579,10 @@ def test_scan_by_index():
assert res["Count"] == 3
assert len(res["Items"]) == 3
res = dynamodb.scan(TableName="test", ConsistentRead=True)
assert res["Count"] == 3
assert len(res["Items"]) == 3
res = dynamodb.scan(TableName="test", IndexName="test_gsi")
assert res["Count"] == 2
assert len(res["Items"]) == 2