Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

DynamoDB: Improve validation (#6986)

Browse Source
This commit is contained in:
Bert Blommers 2023-11-04 09:37:32 -01:00 committed by GitHub
parent 87f816f24f
commit 9136030ecf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 222 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
moto/dynamodb/comparisons.py
View File

@ -10,7 +10,7 @@ from moto.dynamodb.parsing.reserved_keywords import ReservedKeywords
def get_filter_expression(
expr: Optional[str],
names: Optional[Dict[str, str]],
values: Optional[Dict[str, str]],
values: Optional[Dict[str, Dict[str, str]]],
) -> Union["Op", "Func"]:
"""
Parse a filter expression into an Op.
@ -145,7 +145,7 @@ class ConditionExpressionParser:
self,
condition_expression: Optional[str],
expression_attribute_names: Optional[Dict[str, str]],
expression_attribute_values: Optional[Dict[str, str]],
expression_attribute_values: Optional[Dict[str, Dict[str, str]]],
):
self.condition_expression = condition_expression
self.expression_attribute_names = expression_attribute_names
@ -423,7 +423,7 @@ class ConditionExpressionParser:
children=[],
)
def _lookup_expression_attribute_value(self, name: str) -> str:
def _lookup_expression_attribute_value(self, name: str) -> Dict[str, str]:
return self.expression_attribute_values[name] # type: ignore[index]
def _lookup_expression_attribute_name(self, name: str) -> str:

6
moto/dynamodb/exceptions.py
View File

@ -368,3 +368,9 @@ class TransactWriteSingleOpException(MockValidationException):
class SerializationException(DynamodbException):
def __init__(self, msg: str):
super().__init__(error_type="SerializationException", message=msg)
class UnknownKeyType(MockValidationException):
def __init__(self, key_type: str, position: str):
msg = f"1 validation error detected: Value '{key_type}' at '{position}' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]"
super().__init__(msg)

2
moto/dynamodb/models/__init__.py
View File

@ -317,7 +317,7 @@ class DynamoDBBackend(BaseBackend):
projection_expressions: Optional[List[List[str]]],
index_name: Optional[str] = None,
expr_names: Optional[Dict[str, str]] = None,
expr_values: Optional[Dict[str, str]] = None,
expr_values: Optional[Dict[str, Dict[str, str]]] = None,
filter_expression: Optional[str] = None,
**filter_kwargs: Any,
) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]:

6
moto/dynamodb/parsing/key_condition_expression.py
View File

@ -1,5 +1,5 @@
from enum import Enum
from typing import Any, List, Dict, Tuple, Optional
from typing import Any, List, Dict, Tuple, Optional, Union
from moto.dynamodb.exceptions import MockValidationException
from moto.utilities.tokenizer import GenericTokenizer
@ -19,7 +19,7 @@ def get_key(schema: List[Dict[str, str]], key_type: str) -> Optional[str]:
def parse_expression(
key_condition_expression: str,
expression_attribute_values: Dict[str, str],
expression_attribute_values: Dict[str, Dict[str, str]],
expression_attribute_names: Dict[str, str],
schema: List[Dict[str, str]],
) -> Tuple[Dict[str, Any], Optional[str], List[Dict[str, Any]]]:
@ -35,7 +35,7 @@ def parse_expression(
current_stage: Optional[EXPRESSION_STAGES] = None
current_phrase = ""
key_name = comparison = ""
key_values = []
key_values: List[Union[Dict[str, str], str]] = []
results: List[Tuple[str, str, Any]] = []
tokenizer = GenericTokenizer(key_condition_expression)
for crnt_char in tokenizer:

43
moto/dynamodb/responses.py
View File

@ -13,6 +13,7 @@ from moto.dynamodb.parsing.reserved_keywords import ReservedKeywords
from .exceptions import (
MockValidationException,
ResourceNotFoundException,
UnknownKeyType,
)
from moto.dynamodb.models import dynamodb_backends, Table, DynamoDBBackend
from moto.dynamodb.models.utilities import dynamo_json_dump
@ -242,21 +243,42 @@ class DynamoHandler(BaseResponse):
sse_spec = body.get("SSESpecification")
# getting the schema
key_schema = body["KeySchema"]
for idx, _key in enumerate(key_schema, start=1):
key_type = _key["KeyType"]
if key_type not in ["HASH", "RANGE"]:
raise UnknownKeyType(
key_type=key_type, position=f"keySchema.{idx}.member.keyType"
)
# getting attribute definition
attr = body["AttributeDefinitions"]
# getting the indexes
# getting/validating the indexes
global_indexes = body.get("GlobalSecondaryIndexes")
if global_indexes == []:
raise MockValidationException(
"One or more parameter values were invalid: List of GlobalSecondaryIndexes is empty"
)
global_indexes = global_indexes or []
for idx, g_idx in enumerate(global_indexes, start=1):
for idx2, _key in enumerate(g_idx["KeySchema"], start=1):
key_type = _key["KeyType"]
if key_type not in ["HASH", "RANGE"]:
position = f"globalSecondaryIndexes.{idx}.member.keySchema.{idx2}.member.keyType"
raise UnknownKeyType(key_type=key_type, position=position)
local_secondary_indexes = body.get("LocalSecondaryIndexes")
if local_secondary_indexes == []:
raise MockValidationException(
"One or more parameter values were invalid: List of LocalSecondaryIndexes is empty"
)
local_secondary_indexes = local_secondary_indexes or []
for idx, g_idx in enumerate(local_secondary_indexes, start=1):
for idx2, _key in enumerate(g_idx["KeySchema"], start=1):
key_type = _key["KeyType"]
if key_type not in ["HASH", "RANGE"]:
position = f"localSecondaryIndexes.{idx}.member.keySchema.{idx2}.member.keyType"
raise UnknownKeyType(key_type=key_type, position=position)
# Verify AttributeDefinitions list all
expected_attrs = []
expected_attrs.extend([key["AttributeName"] for key in key_schema])
@ -462,7 +484,7 @@ class DynamoHandler(BaseResponse):
# expression
condition_expression = self.body.get("ConditionExpression")
expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
expression_attribute_values = self._get_expr_attr_values()
if condition_expression:
overwrite = False
@ -650,7 +672,7 @@ class DynamoHandler(BaseResponse):
projection_expression = self._get_projection_expression()
expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
filter_expression = self._get_filter_expression()
expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
expression_attribute_values = self._get_expr_attr_values()
projection_expressions = self._adjust_projection_expression(
projection_expression, expression_attribute_names
@ -776,7 +798,7 @@ class DynamoHandler(BaseResponse):
filters[attribute_name] = (comparison_operator, comparison_values)
filter_expression = self._get_filter_expression()
expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
expression_attribute_values = self._get_expr_attr_values()
expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
projection_expression = self._get_projection_expression()
exclusive_start_key = self.body.get("ExclusiveStartKey")
@ -824,7 +846,7 @@ class DynamoHandler(BaseResponse):
# expression
condition_expression = self.body.get("ConditionExpression")
expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
expression_attribute_values = self._get_expr_attr_values()
item = self.dynamodb_backend.delete_item(
name,
@ -879,7 +901,7 @@ class DynamoHandler(BaseResponse):
# expression
condition_expression = self.body.get("ConditionExpression")
expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
expression_attribute_values = self._get_expr_attr_values()
item = self.dynamodb_backend.update_item(
name,
@ -920,6 +942,15 @@ class DynamoHandler(BaseResponse):
)
return dynamo_json_dump(item_dict)
def _get_expr_attr_values(self) -> Dict[str, Dict[str, str]]:
values = self.body.get("ExpressionAttributeValues", {})
for key in values.keys():
if not key.startswith(":"):
raise MockValidationException(
f'ExpressionAttributeValues contains invalid key: Syntax error; key: "{key}"'
)
return values
def _build_updated_new_attributes(self, original: Any, changed: Any) -> Any:
if type(changed) != type(original):
return changed

72
tests/test_dynamodb/__init__.py
View File

@ -5,7 +5,7 @@ from moto import mock_dynamodb
from uuid import uuid4
def dynamodb_aws_verified(func):
def dynamodb_aws_verified(create_table: bool = True):
"""
Function that is verified to work against AWS.
Can be run against AWS at any time by setting:
@ -19,39 +19,47 @@ def dynamodb_aws_verified(func):
- Delete the table
"""
@wraps(func)
def pagination_wrapper():
client = boto3.client("dynamodb", region_name="us-east-1")
table_name = "t" + str(uuid4())[0:6]
def inner(func):
@wraps(func)
def pagination_wrapper():
client = boto3.client("dynamodb", region_name="us-east-1")
table_name = "t" + str(uuid4())[0:6]
allow_aws_request = (
os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true"
)
allow_aws_request = (
os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true"
)
if allow_aws_request:
print(f"Test {func} will create DynamoDB Table {table_name}")
resp = create_table_and_test(table_name, client)
else:
with mock_dynamodb():
resp = create_table_and_test(table_name, client)
return resp
if allow_aws_request:
if create_table:
print(f"Test {func} will create DynamoDB Table {table_name}")
return create_table_and_test(table_name, client)
else:
return func()
else:
with mock_dynamodb():
if create_table:
return create_table_and_test(table_name, client)
else:
return func()
def create_table_and_test(table_name, client):
client.create_table(
TableName=table_name,
KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 5},
Tags=[{"Key": "environment", "Value": "moto_tests"}],
)
waiter = client.get_waiter("table_exists")
waiter.wait(TableName=table_name)
try:
resp = func(table_name)
finally:
### CLEANUP ###
client.delete_table(TableName=table_name)
def create_table_and_test(table_name, client):
client.create_table(
TableName=table_name,
KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 5},
Tags=[{"Key": "environment", "Value": "moto_tests"}],
)
waiter = client.get_waiter("table_exists")
waiter.wait(TableName=table_name)
try:
resp = func(table_name)
finally:
### CLEANUP ###
client.delete_table(TableName=table_name)
return resp
return resp
return pagination_wrapper
return pagination_wrapper
return inner

35
tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py
View File

@ -1117,7 +1117,7 @@ def test_query_with_missing_expression_attribute():
@pytest.mark.aws_verified
@dynamodb_aws_verified
@dynamodb_aws_verified()
def test_update_item_returns_old_item(table_name=None):
dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
table = dynamodb.Table(table_name)
@ -1164,3 +1164,36 @@ def test_update_item_returns_old_item(table_name=None):
"lock": {"M": {"acquired_at": {"N": "123"}}},
"pk": {"S": "mark"},
}
@pytest.mark.aws_verified
@dynamodb_aws_verified()
def test_scan_with_missing_value(table_name=None):
dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
table = dynamodb.Table(table_name)
with pytest.raises(ClientError) as exc:
table.scan(
FilterExpression="attr = loc",
# Missing ':'
ExpressionAttributeValues={"loc": "sth"},
)
err = exc.value.response["Error"]
assert err["Code"] == "ValidationException"
assert (
err["Message"]
== 'ExpressionAttributeValues contains invalid key: Syntax error; key: "loc"'
)
with pytest.raises(ClientError) as exc:
table.query(
KeyConditionExpression="attr = loc",
# Missing ':'
ExpressionAttributeValues={"loc": "sth"},
)
err = exc.value.response["Error"]
assert err["Code"] == "ValidationException"
assert (
err["Message"]
== 'ExpressionAttributeValues contains invalid key: Syntax error; key: "loc"'
)

4
tests/test_dynamodb/test_dynamodb.py
View File

@ -3686,7 +3686,7 @@ def test_transact_write_items_put():
@pytest.mark.aws_verified
@dynamodb_aws_verified
@dynamodb_aws_verified()
def test_transact_write_items_put_conditional_expressions(table_name=None):
dynamodb = boto3.client("dynamodb", region_name="us-east-1")
@ -3731,7 +3731,7 @@ def test_transact_write_items_put_conditional_expressions(table_name=None):
@pytest.mark.aws_verified
@dynamodb_aws_verified
@dynamodb_aws_verified()
def test_transact_write_items_failure__return_item(table_name=None):
dynamodb = boto3.client("dynamodb", region_name="us-east-1")
dynamodb.put_item(TableName=table_name, Item={"pk": {"S": "foo2"}})

121
tests/test_dynamodb/test_dynamodb_create_table.py
View File

@ -3,9 +3,10 @@ from botocore.exceptions import ClientError
from datetime import datetime
import pytest
from moto import mock_dynamodb, settings
from moto import mock_dynamodb
from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
from moto.dynamodb.models import dynamodb_backends
from . import dynamodb_aws_verified
@mock_dynamodb
@ -399,36 +400,94 @@ def test_create_table_with_ssespecification__custom_kms_key():
assert actual["SSEDescription"]["KMSMasterKeyArn"] == "custom-kms-key"
@mock_dynamodb
@pytest.mark.aws_verified
@dynamodb_aws_verified(create_table=False)
def test_create_table__specify_non_key_column():
client = boto3.client("dynamodb", "us-east-2")
client.create_table(
TableName="tab",
KeySchema=[
{"AttributeName": "PK", "KeyType": "HASH"},
{"AttributeName": "SomeColumn", "KeyType": "N"},
],
BillingMode="PAY_PER_REQUEST",
AttributeDefinitions=[
{"AttributeName": "PK", "AttributeType": "S"},
{"AttributeName": "SomeColumn", "AttributeType": "N"},
],
dynamodb = boto3.client("dynamodb", region_name="us-east-1")
with pytest.raises(ClientError) as exc:
dynamodb.create_table(
TableName="unknown-key-type",
KeySchema=[
{"AttributeName": "pk", "KeyType": "HASH"},
{"AttributeName": "sk", "KeyType": "SORT"},
],
AttributeDefinitions=[
{"AttributeName": "pk", "AttributeType": "S"},
{"AttributeName": "sk", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
err = exc.value.response["Error"]
assert err["Code"] == "ValidationException"
assert (
err["Message"]
== "1 validation error detected: Value 'SORT' at 'keySchema.2.member.keyType' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]"
)
actual = client.describe_table(TableName="tab")["Table"]
assert actual["KeySchema"] == [
{"AttributeName": "PK", "KeyType": "HASH"},
{"AttributeName": "SomeColumn", "KeyType": "N"},
]
# Verify we get the same message for Global Secondary Indexes
with pytest.raises(ClientError) as exc:
dynamodb.create_table(
TableName="unknown-key-type",
KeySchema=[
{"AttributeName": "pk", "KeyType": "HASH"},
{"AttributeName": "sk", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "pk", "AttributeType": "S"},
{"AttributeName": "sk", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
GlobalSecondaryIndexes=[
{
"IndexName": "TestGSI",
# Note that the attributes are not declared, which is also invalid
# But AWS trips over the KeyType=SORT first
"KeySchema": [
{"AttributeName": "n/a", "KeyType": "HASH"},
{"AttributeName": "sth", "KeyType": "SORT"},
],
"Projection": {"ProjectionType": "ALL"},
"ProvisionedThroughput": {
"ReadCapacityUnits": 5,
"WriteCapacityUnits": 5,
},
}
],
)
err = exc.value.response["Error"]
assert err["Code"] == "ValidationException"
assert (
err["Message"]
== "1 validation error detected: Value 'SORT' at 'globalSecondaryIndexes.1.member.keySchema.2.member.keyType' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]"
)
if not settings.TEST_SERVER_MODE:
ddb = dynamodb_backends[ACCOUNT_ID]["us-east-2"]
assert {"AttributeName": "PK", "AttributeType": "S"} in ddb.tables["tab"].attr
assert {"AttributeName": "SomeColumn", "AttributeType": "N"} in ddb.tables[
"tab"
].attr
# It should recognize PK is the Hash Key
assert ddb.tables["tab"].hash_key_attr == "PK"
# It should recognize that SomeColumn is not a Range Key
assert ddb.tables["tab"].has_range_key is False
assert ddb.tables["tab"].range_key_names == []
# Verify we get the same message for Local Secondary Indexes
with pytest.raises(ClientError) as exc:
dynamodb.create_table(
TableName="unknown-key-type",
KeySchema=[
{"AttributeName": "pk", "KeyType": "HASH"},
{"AttributeName": "sk", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "pk", "AttributeType": "S"},
{"AttributeName": "sk", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
LocalSecondaryIndexes=[
{
"IndexName": "test_lsi",
"KeySchema": [
{"AttributeName": "pk", "KeyType": "HASH"},
{"AttributeName": "lsi_range_key", "KeyType": "SORT"},
],
"Projection": {"ProjectionType": "ALL"},
}
],
)
err = exc.value.response["Error"]
assert err["Code"] == "ValidationException"
assert (
err["Message"]
== "1 validation error detected: Value 'SORT' at 'localSecondaryIndexes.1.member.keySchema.2.member.keyType' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]"
)

12
tests/test_dynamodb/test_dynamodb_statements.py
View File

@ -25,7 +25,7 @@ def create_items(table_name):
@pytest.mark.aws_verified
@dynamodb_aws_verified
@dynamodb_aws_verified()
def test_execute_statement_select_star(table_name=None):
client = boto3.client("dynamodb", "us-east-1")
create_items(table_name)
@ -35,7 +35,7 @@ def test_execute_statement_select_star(table_name=None):
@pytest.mark.aws_verified
@dynamodb_aws_verified
@dynamodb_aws_verified()
def test_execute_statement_select_attr(table_name=None):
client = boto3.client("dynamodb", "us-east-1")
create_items(table_name)
@ -47,7 +47,7 @@ def test_execute_statement_select_attr(table_name=None):
@pytest.mark.aws_verified
@dynamodb_aws_verified
@dynamodb_aws_verified()
def test_execute_statement_with_quoted_table(table_name=None):
client = boto3.client("dynamodb", "us-east-1")
create_items(table_name)
@ -57,7 +57,7 @@ def test_execute_statement_with_quoted_table(table_name=None):
@pytest.mark.aws_verified
@dynamodb_aws_verified
@dynamodb_aws_verified()
def test_execute_statement_with_parameter(table_name=None):
client = boto3.client("dynamodb", "us-east-1")
create_items(table_name)
@ -77,7 +77,7 @@ def test_execute_statement_with_parameter(table_name=None):
@pytest.mark.aws_verified
@dynamodb_aws_verified
@dynamodb_aws_verified()
def test_execute_statement_with_no_results(table_name=None):
client = boto3.client("dynamodb", "us-east-1")
create_items(table_name)
@ -201,7 +201,7 @@ class TestBatchExecuteStatement(TestCase):
@pytest.mark.aws_verified
@dynamodb_aws_verified
@dynamodb_aws_verified()
def test_execute_statement_with_all_clauses(table_name=None):
dynamodb_client = boto3.client("dynamodb", "us-east-1")