diff --git a/moto/dynamodb/comparisons.py b/moto/dynamodb/comparisons.py index 26b74a6ed..3f330dc84 100644 --- a/moto/dynamodb/comparisons.py +++ b/moto/dynamodb/comparisons.py @@ -10,7 +10,7 @@ from moto.dynamodb.parsing.reserved_keywords import ReservedKeywords def get_filter_expression( expr: Optional[str], names: Optional[Dict[str, str]], - values: Optional[Dict[str, str]], + values: Optional[Dict[str, Dict[str, str]]], ) -> Union["Op", "Func"]: """ Parse a filter expression into an Op. @@ -145,7 +145,7 @@ class ConditionExpressionParser: self, condition_expression: Optional[str], expression_attribute_names: Optional[Dict[str, str]], - expression_attribute_values: Optional[Dict[str, str]], + expression_attribute_values: Optional[Dict[str, Dict[str, str]]], ): self.condition_expression = condition_expression self.expression_attribute_names = expression_attribute_names @@ -423,7 +423,7 @@ class ConditionExpressionParser: children=[], ) - def _lookup_expression_attribute_value(self, name: str) -> str: + def _lookup_expression_attribute_value(self, name: str) -> Dict[str, str]: return self.expression_attribute_values[name] # type: ignore[index] def _lookup_expression_attribute_name(self, name: str) -> str: diff --git a/moto/dynamodb/exceptions.py b/moto/dynamodb/exceptions.py index fa8de1535..57823cfee 100644 --- a/moto/dynamodb/exceptions.py +++ b/moto/dynamodb/exceptions.py @@ -368,3 +368,9 @@ class TransactWriteSingleOpException(MockValidationException): class SerializationException(DynamodbException): def __init__(self, msg: str): super().__init__(error_type="SerializationException", message=msg) + + +class UnknownKeyType(MockValidationException): + def __init__(self, key_type: str, position: str): + msg = f"1 validation error detected: Value '{key_type}' at '{position}' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]" + super().__init__(msg) diff --git a/moto/dynamodb/models/__init__.py b/moto/dynamodb/models/__init__.py index cf0318e3a..fe12880f0 100644 --- a/moto/dynamodb/models/__init__.py +++ b/moto/dynamodb/models/__init__.py @@ -317,7 +317,7 @@ class DynamoDBBackend(BaseBackend): projection_expressions: Optional[List[List[str]]], index_name: Optional[str] = None, expr_names: Optional[Dict[str, str]] = None, - expr_values: Optional[Dict[str, str]] = None, + expr_values: Optional[Dict[str, Dict[str, str]]] = None, filter_expression: Optional[str] = None, **filter_kwargs: Any, ) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]: diff --git a/moto/dynamodb/parsing/key_condition_expression.py b/moto/dynamodb/parsing/key_condition_expression.py index 0b30c78dc..7638f0e24 100644 --- a/moto/dynamodb/parsing/key_condition_expression.py +++ b/moto/dynamodb/parsing/key_condition_expression.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, List, Dict, Tuple, Optional +from typing import Any, List, Dict, Tuple, Optional, Union from moto.dynamodb.exceptions import MockValidationException from moto.utilities.tokenizer import GenericTokenizer @@ -19,7 +19,7 @@ def get_key(schema: List[Dict[str, str]], key_type: str) -> Optional[str]: def parse_expression( key_condition_expression: str, - expression_attribute_values: Dict[str, str], + expression_attribute_values: Dict[str, Dict[str, str]], expression_attribute_names: Dict[str, str], schema: List[Dict[str, str]], ) -> Tuple[Dict[str, Any], Optional[str], List[Dict[str, Any]]]: @@ -35,7 +35,7 @@ def parse_expression( current_stage: Optional[EXPRESSION_STAGES] = None current_phrase = "" key_name = comparison = "" - key_values = [] + key_values: List[Union[Dict[str, str], str]] = [] results: List[Tuple[str, str, Any]] = [] tokenizer = GenericTokenizer(key_condition_expression) for crnt_char in tokenizer: diff --git a/moto/dynamodb/responses.py b/moto/dynamodb/responses.py index bcbad5883..a751aad8c 100644 --- a/moto/dynamodb/responses.py +++ b/moto/dynamodb/responses.py @@ -13,6 +13,7 @@ from moto.dynamodb.parsing.reserved_keywords import ReservedKeywords from .exceptions import ( MockValidationException, ResourceNotFoundException, + UnknownKeyType, ) from moto.dynamodb.models import dynamodb_backends, Table, DynamoDBBackend from moto.dynamodb.models.utilities import dynamo_json_dump @@ -242,21 +243,42 @@ class DynamoHandler(BaseResponse): sse_spec = body.get("SSESpecification") # getting the schema key_schema = body["KeySchema"] + for idx, _key in enumerate(key_schema, start=1): + key_type = _key["KeyType"] + if key_type not in ["HASH", "RANGE"]: + raise UnknownKeyType( + key_type=key_type, position=f"keySchema.{idx}.member.keyType" + ) # getting attribute definition attr = body["AttributeDefinitions"] - # getting the indexes + + # getting/validating the indexes global_indexes = body.get("GlobalSecondaryIndexes") if global_indexes == []: raise MockValidationException( "One or more parameter values were invalid: List of GlobalSecondaryIndexes is empty" ) global_indexes = global_indexes or [] + for idx, g_idx in enumerate(global_indexes, start=1): + for idx2, _key in enumerate(g_idx["KeySchema"], start=1): + key_type = _key["KeyType"] + if key_type not in ["HASH", "RANGE"]: + position = f"globalSecondaryIndexes.{idx}.member.keySchema.{idx2}.member.keyType" + raise UnknownKeyType(key_type=key_type, position=position) + local_secondary_indexes = body.get("LocalSecondaryIndexes") if local_secondary_indexes == []: raise MockValidationException( "One or more parameter values were invalid: List of LocalSecondaryIndexes is empty" ) local_secondary_indexes = local_secondary_indexes or [] + for idx, g_idx in enumerate(local_secondary_indexes, start=1): + for idx2, _key in enumerate(g_idx["KeySchema"], start=1): + key_type = _key["KeyType"] + if key_type not in ["HASH", "RANGE"]: + position = f"localSecondaryIndexes.{idx}.member.keySchema.{idx2}.member.keyType" + raise UnknownKeyType(key_type=key_type, position=position) + # Verify AttributeDefinitions list all expected_attrs = [] expected_attrs.extend([key["AttributeName"] for key in key_schema]) @@ -462,7 +484,7 @@ class DynamoHandler(BaseResponse): # expression condition_expression = self.body.get("ConditionExpression") expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) - expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) + expression_attribute_values = self._get_expr_attr_values() if condition_expression: overwrite = False @@ -650,7 +672,7 @@ class DynamoHandler(BaseResponse): projection_expression = self._get_projection_expression() expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) filter_expression = self._get_filter_expression() - expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) + expression_attribute_values = self._get_expr_attr_values() projection_expressions = self._adjust_projection_expression( projection_expression, expression_attribute_names @@ -776,7 +798,7 @@ class DynamoHandler(BaseResponse): filters[attribute_name] = (comparison_operator, comparison_values) filter_expression = self._get_filter_expression() - expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) + expression_attribute_values = self._get_expr_attr_values() expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) projection_expression = self._get_projection_expression() exclusive_start_key = self.body.get("ExclusiveStartKey") @@ -824,7 +846,7 @@ class DynamoHandler(BaseResponse): # expression condition_expression = self.body.get("ConditionExpression") expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) - expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) + expression_attribute_values = self._get_expr_attr_values() item = self.dynamodb_backend.delete_item( name, @@ -879,7 +901,7 @@ class DynamoHandler(BaseResponse): # expression condition_expression = self.body.get("ConditionExpression") expression_attribute_names = self.body.get("ExpressionAttributeNames", {}) - expression_attribute_values = self.body.get("ExpressionAttributeValues", {}) + expression_attribute_values = self._get_expr_attr_values() item = self.dynamodb_backend.update_item( name, @@ -920,6 +942,15 @@ class DynamoHandler(BaseResponse): ) return dynamo_json_dump(item_dict) + def _get_expr_attr_values(self) -> Dict[str, Dict[str, str]]: + values = self.body.get("ExpressionAttributeValues", {}) + for key in values.keys(): + if not key.startswith(":"): + raise MockValidationException( + f'ExpressionAttributeValues contains invalid key: Syntax error; key: "{key}"' + ) + return values + def _build_updated_new_attributes(self, original: Any, changed: Any) -> Any: if type(changed) != type(original): return changed diff --git a/tests/test_dynamodb/__init__.py b/tests/test_dynamodb/__init__.py index 311c0bbd6..fe46e5127 100644 --- a/tests/test_dynamodb/__init__.py +++ b/tests/test_dynamodb/__init__.py @@ -5,7 +5,7 @@ from moto import mock_dynamodb from uuid import uuid4 -def dynamodb_aws_verified(func): +def dynamodb_aws_verified(create_table: bool = True): """ Function that is verified to work against AWS. Can be run against AWS at any time by setting: @@ -19,39 +19,47 @@ def dynamodb_aws_verified(func): - Delete the table """ - @wraps(func) - def pagination_wrapper(): - client = boto3.client("dynamodb", region_name="us-east-1") - table_name = "t" + str(uuid4())[0:6] + def inner(func): + @wraps(func) + def pagination_wrapper(): + client = boto3.client("dynamodb", region_name="us-east-1") + table_name = "t" + str(uuid4())[0:6] - allow_aws_request = ( - os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true" - ) + allow_aws_request = ( + os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true" + ) - if allow_aws_request: - print(f"Test {func} will create DynamoDB Table {table_name}") - resp = create_table_and_test(table_name, client) - else: - with mock_dynamodb(): - resp = create_table_and_test(table_name, client) - return resp + if allow_aws_request: + if create_table: + print(f"Test {func} will create DynamoDB Table {table_name}") + return create_table_and_test(table_name, client) + else: + return func() + else: + with mock_dynamodb(): + if create_table: + return create_table_and_test(table_name, client) + else: + return func() - def create_table_and_test(table_name, client): - client.create_table( - TableName=table_name, - KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 5}, - Tags=[{"Key": "environment", "Value": "moto_tests"}], - ) - waiter = client.get_waiter("table_exists") - waiter.wait(TableName=table_name) - try: - resp = func(table_name) - finally: - ### CLEANUP ### - client.delete_table(TableName=table_name) + def create_table_and_test(table_name, client): + client.create_table( + TableName=table_name, + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 5}, + Tags=[{"Key": "environment", "Value": "moto_tests"}], + ) + waiter = client.get_waiter("table_exists") + waiter.wait(TableName=table_name) + try: + resp = func(table_name) + finally: + ### CLEANUP ### + client.delete_table(TableName=table_name) - return resp + return resp - return pagination_wrapper + return pagination_wrapper + + return inner diff --git a/tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py b/tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py index 9cd1ec6c0..e3a788c3c 100644 --- a/tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py +++ b/tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py @@ -1117,7 +1117,7 @@ def test_query_with_missing_expression_attribute(): @pytest.mark.aws_verified -@dynamodb_aws_verified +@dynamodb_aws_verified() def test_update_item_returns_old_item(table_name=None): dynamodb = boto3.resource("dynamodb", region_name="us-east-1") table = dynamodb.Table(table_name) @@ -1164,3 +1164,36 @@ def test_update_item_returns_old_item(table_name=None): "lock": {"M": {"acquired_at": {"N": "123"}}}, "pk": {"S": "mark"}, } + + +@pytest.mark.aws_verified +@dynamodb_aws_verified() +def test_scan_with_missing_value(table_name=None): + dynamodb = boto3.resource("dynamodb", region_name="us-east-1") + table = dynamodb.Table(table_name) + + with pytest.raises(ClientError) as exc: + table.scan( + FilterExpression="attr = loc", + # Missing ':' + ExpressionAttributeValues={"loc": "sth"}, + ) + err = exc.value.response["Error"] + assert err["Code"] == "ValidationException" + assert ( + err["Message"] + == 'ExpressionAttributeValues contains invalid key: Syntax error; key: "loc"' + ) + + with pytest.raises(ClientError) as exc: + table.query( + KeyConditionExpression="attr = loc", + # Missing ':' + ExpressionAttributeValues={"loc": "sth"}, + ) + err = exc.value.response["Error"] + assert err["Code"] == "ValidationException" + assert ( + err["Message"] + == 'ExpressionAttributeValues contains invalid key: Syntax error; key: "loc"' + ) diff --git a/tests/test_dynamodb/test_dynamodb.py b/tests/test_dynamodb/test_dynamodb.py index 15f45e915..679ae20db 100644 --- a/tests/test_dynamodb/test_dynamodb.py +++ b/tests/test_dynamodb/test_dynamodb.py @@ -3686,7 +3686,7 @@ def test_transact_write_items_put(): @pytest.mark.aws_verified -@dynamodb_aws_verified +@dynamodb_aws_verified() def test_transact_write_items_put_conditional_expressions(table_name=None): dynamodb = boto3.client("dynamodb", region_name="us-east-1") @@ -3731,7 +3731,7 @@ def test_transact_write_items_put_conditional_expressions(table_name=None): @pytest.mark.aws_verified -@dynamodb_aws_verified +@dynamodb_aws_verified() def test_transact_write_items_failure__return_item(table_name=None): dynamodb = boto3.client("dynamodb", region_name="us-east-1") dynamodb.put_item(TableName=table_name, Item={"pk": {"S": "foo2"}}) diff --git a/tests/test_dynamodb/test_dynamodb_create_table.py b/tests/test_dynamodb/test_dynamodb_create_table.py index 1cb8eb16c..c659ca22b 100644 --- a/tests/test_dynamodb/test_dynamodb_create_table.py +++ b/tests/test_dynamodb/test_dynamodb_create_table.py @@ -3,9 +3,10 @@ from botocore.exceptions import ClientError from datetime import datetime import pytest -from moto import mock_dynamodb, settings +from moto import mock_dynamodb from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID -from moto.dynamodb.models import dynamodb_backends + +from . import dynamodb_aws_verified @mock_dynamodb @@ -399,36 +400,94 @@ def test_create_table_with_ssespecification__custom_kms_key(): assert actual["SSEDescription"]["KMSMasterKeyArn"] == "custom-kms-key" -@mock_dynamodb +@pytest.mark.aws_verified +@dynamodb_aws_verified(create_table=False) def test_create_table__specify_non_key_column(): - client = boto3.client("dynamodb", "us-east-2") - client.create_table( - TableName="tab", - KeySchema=[ - {"AttributeName": "PK", "KeyType": "HASH"}, - {"AttributeName": "SomeColumn", "KeyType": "N"}, - ], - BillingMode="PAY_PER_REQUEST", - AttributeDefinitions=[ - {"AttributeName": "PK", "AttributeType": "S"}, - {"AttributeName": "SomeColumn", "AttributeType": "N"}, - ], + dynamodb = boto3.client("dynamodb", region_name="us-east-1") + with pytest.raises(ClientError) as exc: + dynamodb.create_table( + TableName="unknown-key-type", + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "SORT"}, + ], + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + err = exc.value.response["Error"] + assert err["Code"] == "ValidationException" + assert ( + err["Message"] + == "1 validation error detected: Value 'SORT' at 'keySchema.2.member.keyType' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]" ) - actual = client.describe_table(TableName="tab")["Table"] - assert actual["KeySchema"] == [ - {"AttributeName": "PK", "KeyType": "HASH"}, - {"AttributeName": "SomeColumn", "KeyType": "N"}, - ] + # Verify we get the same message for Global Secondary Indexes + with pytest.raises(ClientError) as exc: + dynamodb.create_table( + TableName="unknown-key-type", + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + GlobalSecondaryIndexes=[ + { + "IndexName": "TestGSI", + # Note that the attributes are not declared, which is also invalid + # But AWS trips over the KeyType=SORT first + "KeySchema": [ + {"AttributeName": "n/a", "KeyType": "HASH"}, + {"AttributeName": "sth", "KeyType": "SORT"}, + ], + "Projection": {"ProjectionType": "ALL"}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5, + }, + } + ], + ) + err = exc.value.response["Error"] + assert err["Code"] == "ValidationException" + assert ( + err["Message"] + == "1 validation error detected: Value 'SORT' at 'globalSecondaryIndexes.1.member.keySchema.2.member.keyType' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]" + ) - if not settings.TEST_SERVER_MODE: - ddb = dynamodb_backends[ACCOUNT_ID]["us-east-2"] - assert {"AttributeName": "PK", "AttributeType": "S"} in ddb.tables["tab"].attr - assert {"AttributeName": "SomeColumn", "AttributeType": "N"} in ddb.tables[ - "tab" - ].attr - # It should recognize PK is the Hash Key - assert ddb.tables["tab"].hash_key_attr == "PK" - # It should recognize that SomeColumn is not a Range Key - assert ddb.tables["tab"].has_range_key is False - assert ddb.tables["tab"].range_key_names == [] + # Verify we get the same message for Local Secondary Indexes + with pytest.raises(ClientError) as exc: + dynamodb.create_table( + TableName="unknown-key-type", + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + LocalSecondaryIndexes=[ + { + "IndexName": "test_lsi", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "lsi_range_key", "KeyType": "SORT"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) + err = exc.value.response["Error"] + assert err["Code"] == "ValidationException" + assert ( + err["Message"] + == "1 validation error detected: Value 'SORT' at 'localSecondaryIndexes.1.member.keySchema.2.member.keyType' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]" + ) diff --git a/tests/test_dynamodb/test_dynamodb_statements.py b/tests/test_dynamodb/test_dynamodb_statements.py index 285c4f61c..a5cda8e0c 100644 --- a/tests/test_dynamodb/test_dynamodb_statements.py +++ b/tests/test_dynamodb/test_dynamodb_statements.py @@ -25,7 +25,7 @@ def create_items(table_name): @pytest.mark.aws_verified -@dynamodb_aws_verified +@dynamodb_aws_verified() def test_execute_statement_select_star(table_name=None): client = boto3.client("dynamodb", "us-east-1") create_items(table_name) @@ -35,7 +35,7 @@ def test_execute_statement_select_star(table_name=None): @pytest.mark.aws_verified -@dynamodb_aws_verified +@dynamodb_aws_verified() def test_execute_statement_select_attr(table_name=None): client = boto3.client("dynamodb", "us-east-1") create_items(table_name) @@ -47,7 +47,7 @@ def test_execute_statement_select_attr(table_name=None): @pytest.mark.aws_verified -@dynamodb_aws_verified +@dynamodb_aws_verified() def test_execute_statement_with_quoted_table(table_name=None): client = boto3.client("dynamodb", "us-east-1") create_items(table_name) @@ -57,7 +57,7 @@ def test_execute_statement_with_quoted_table(table_name=None): @pytest.mark.aws_verified -@dynamodb_aws_verified +@dynamodb_aws_verified() def test_execute_statement_with_parameter(table_name=None): client = boto3.client("dynamodb", "us-east-1") create_items(table_name) @@ -77,7 +77,7 @@ def test_execute_statement_with_parameter(table_name=None): @pytest.mark.aws_verified -@dynamodb_aws_verified +@dynamodb_aws_verified() def test_execute_statement_with_no_results(table_name=None): client = boto3.client("dynamodb", "us-east-1") create_items(table_name) @@ -201,7 +201,7 @@ class TestBatchExecuteStatement(TestCase): @pytest.mark.aws_verified -@dynamodb_aws_verified +@dynamodb_aws_verified() def test_execute_statement_with_all_clauses(table_name=None): dynamodb_client = boto3.client("dynamodb", "us-east-1")