DynamoDB: Improve validation (#6986)

moto/dynamodb/comparisons.py

@@ -10,7 +10,7 @@ from moto.dynamodb.parsing.reserved_keywords import ReservedKeywords
 def get_filter_expression(
     expr: Optional[str],
     names: Optional[Dict[str, str]],
-    values: Optional[Dict[str, str]],
+    values: Optional[Dict[str, Dict[str, str]]],
 ) -> Union["Op", "Func"]:
     """
     Parse a filter expression into an Op.
@@ -145,7 +145,7 @@ class ConditionExpressionParser:
         self,
         condition_expression: Optional[str],
         expression_attribute_names: Optional[Dict[str, str]],
-        expression_attribute_values: Optional[Dict[str, str]],
+        expression_attribute_values: Optional[Dict[str, Dict[str, str]]],
     ):
         self.condition_expression = condition_expression
         self.expression_attribute_names = expression_attribute_names
@@ -423,7 +423,7 @@ class ConditionExpressionParser:
                 children=[],
             )
 
-    def _lookup_expression_attribute_value(self, name: str) -> str:
+    def _lookup_expression_attribute_value(self, name: str) -> Dict[str, str]:
         return self.expression_attribute_values[name]  # type: ignore[index]
 
     def _lookup_expression_attribute_name(self, name: str) -> str:

moto/dynamodb/exceptions.py

@@ -368,3 +368,9 @@ class TransactWriteSingleOpException(MockValidationException):
 class SerializationException(DynamodbException):
     def __init__(self, msg: str):
         super().__init__(error_type="SerializationException", message=msg)
+
+
+class UnknownKeyType(MockValidationException):
+    def __init__(self, key_type: str, position: str):
+        msg = f"1 validation error detected: Value '{key_type}' at '{position}' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]"
+        super().__init__(msg)

moto/dynamodb/models/__init__.py

@@ -317,7 +317,7 @@ class DynamoDBBackend(BaseBackend):
         projection_expressions: Optional[List[List[str]]],
         index_name: Optional[str] = None,
         expr_names: Optional[Dict[str, str]] = None,
-        expr_values: Optional[Dict[str, str]] = None,
+        expr_values: Optional[Dict[str, Dict[str, str]]] = None,
         filter_expression: Optional[str] = None,
         **filter_kwargs: Any,
     ) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]:

moto/dynamodb/parsing/key_condition_expression.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Any, List, Dict, Tuple, Optional
+from typing import Any, List, Dict, Tuple, Optional, Union
 from moto.dynamodb.exceptions import MockValidationException
 from moto.utilities.tokenizer import GenericTokenizer
 
@@ -19,7 +19,7 @@ def get_key(schema: List[Dict[str, str]], key_type: str) -> Optional[str]:
 
 def parse_expression(
     key_condition_expression: str,
-    expression_attribute_values: Dict[str, str],
+    expression_attribute_values: Dict[str, Dict[str, str]],
     expression_attribute_names: Dict[str, str],
     schema: List[Dict[str, str]],
 ) -> Tuple[Dict[str, Any], Optional[str], List[Dict[str, Any]]]:
@@ -35,7 +35,7 @@ def parse_expression(
     current_stage: Optional[EXPRESSION_STAGES] = None
     current_phrase = ""
     key_name = comparison = ""
-    key_values = []
+    key_values: List[Union[Dict[str, str], str]] = []
     results: List[Tuple[str, str, Any]] = []
     tokenizer = GenericTokenizer(key_condition_expression)
     for crnt_char in tokenizer:

moto/dynamodb/responses.py

@@ -13,6 +13,7 @@ from moto.dynamodb.parsing.reserved_keywords import ReservedKeywords
 from .exceptions import (
     MockValidationException,
     ResourceNotFoundException,
+    UnknownKeyType,
 )
 from moto.dynamodb.models import dynamodb_backends, Table, DynamoDBBackend
 from moto.dynamodb.models.utilities import dynamo_json_dump
@@ -242,21 +243,42 @@ class DynamoHandler(BaseResponse):
         sse_spec = body.get("SSESpecification")
         # getting the schema
         key_schema = body["KeySchema"]
+        for idx, _key in enumerate(key_schema, start=1):
+            key_type = _key["KeyType"]
+            if key_type not in ["HASH", "RANGE"]:
+                raise UnknownKeyType(
+                    key_type=key_type, position=f"keySchema.{idx}.member.keyType"
+                )
         # getting attribute definition
         attr = body["AttributeDefinitions"]
-        # getting the indexes
+
+        # getting/validating the indexes
         global_indexes = body.get("GlobalSecondaryIndexes")
         if global_indexes == []:
             raise MockValidationException(
                 "One or more parameter values were invalid: List of GlobalSecondaryIndexes is empty"
             )
         global_indexes = global_indexes or []
+        for idx, g_idx in enumerate(global_indexes, start=1):
+            for idx2, _key in enumerate(g_idx["KeySchema"], start=1):
+                key_type = _key["KeyType"]
+                if key_type not in ["HASH", "RANGE"]:
+                    position = f"globalSecondaryIndexes.{idx}.member.keySchema.{idx2}.member.keyType"
+                    raise UnknownKeyType(key_type=key_type, position=position)
+
         local_secondary_indexes = body.get("LocalSecondaryIndexes")
         if local_secondary_indexes == []:
             raise MockValidationException(
                 "One or more parameter values were invalid: List of LocalSecondaryIndexes is empty"
             )
         local_secondary_indexes = local_secondary_indexes or []
+        for idx, g_idx in enumerate(local_secondary_indexes, start=1):
+            for idx2, _key in enumerate(g_idx["KeySchema"], start=1):
+                key_type = _key["KeyType"]
+                if key_type not in ["HASH", "RANGE"]:
+                    position = f"localSecondaryIndexes.{idx}.member.keySchema.{idx2}.member.keyType"
+                    raise UnknownKeyType(key_type=key_type, position=position)
+
         # Verify AttributeDefinitions list all
         expected_attrs = []
         expected_attrs.extend([key["AttributeName"] for key in key_schema])
@@ -462,7 +484,7 @@ class DynamoHandler(BaseResponse):
         # expression
         condition_expression = self.body.get("ConditionExpression")
         expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
-        expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
+        expression_attribute_values = self._get_expr_attr_values()
 
         if condition_expression:
             overwrite = False
@@ -650,7 +672,7 @@ class DynamoHandler(BaseResponse):
         projection_expression = self._get_projection_expression()
         expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
         filter_expression = self._get_filter_expression()
-        expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
+        expression_attribute_values = self._get_expr_attr_values()
 
         projection_expressions = self._adjust_projection_expression(
             projection_expression, expression_attribute_names
@@ -776,7 +798,7 @@ class DynamoHandler(BaseResponse):
             filters[attribute_name] = (comparison_operator, comparison_values)
 
         filter_expression = self._get_filter_expression()
-        expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
+        expression_attribute_values = self._get_expr_attr_values()
         expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
         projection_expression = self._get_projection_expression()
         exclusive_start_key = self.body.get("ExclusiveStartKey")
@@ -824,7 +846,7 @@ class DynamoHandler(BaseResponse):
         # expression
         condition_expression = self.body.get("ConditionExpression")
         expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
-        expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
+        expression_attribute_values = self._get_expr_attr_values()
 
         item = self.dynamodb_backend.delete_item(
             name,
@@ -879,7 +901,7 @@ class DynamoHandler(BaseResponse):
         # expression
         condition_expression = self.body.get("ConditionExpression")
         expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
-        expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
+        expression_attribute_values = self._get_expr_attr_values()
 
         item = self.dynamodb_backend.update_item(
             name,
@@ -920,6 +942,15 @@ class DynamoHandler(BaseResponse):
             )
         return dynamo_json_dump(item_dict)
 
+    def _get_expr_attr_values(self) -> Dict[str, Dict[str, str]]:
+        values = self.body.get("ExpressionAttributeValues", {})
+        for key in values.keys():
+            if not key.startswith(":"):
+                raise MockValidationException(
+                    f'ExpressionAttributeValues contains invalid key: Syntax error; key: "{key}"'
+                )
+        return values
+
     def _build_updated_new_attributes(self, original: Any, changed: Any) -> Any:
         if type(changed) != type(original):
             return changed

tests/test_dynamodb/__init__.py

@@ -5,7 +5,7 @@ from moto import mock_dynamodb
 from uuid import uuid4
 
 
-def dynamodb_aws_verified(func):
+def dynamodb_aws_verified(create_table: bool = True):
     """
     Function that is verified to work against AWS.
     Can be run against AWS at any time by setting:
@@ -19,39 +19,47 @@ def dynamodb_aws_verified(func):
       - Delete the table
     """
 
-    @wraps(func)
+    def inner(func):
-    def pagination_wrapper():
+        @wraps(func)
-        client = boto3.client("dynamodb", region_name="us-east-1")
+        def pagination_wrapper():
-        table_name = "t" + str(uuid4())[0:6]
+            client = boto3.client("dynamodb", region_name="us-east-1")
+            table_name = "t" + str(uuid4())[0:6]
 
-        allow_aws_request = (
+            allow_aws_request = (
-            os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true"
+                os.environ.get("MOTO_TEST_ALLOW_AWS_REQUEST", "false").lower() == "true"
-        )
+            )
 
-        if allow_aws_request:
+            if allow_aws_request:
-            print(f"Test {func} will create DynamoDB Table {table_name}")
+                if create_table:
-            resp = create_table_and_test(table_name, client)
+                    print(f"Test {func} will create DynamoDB Table {table_name}")
-        else:
+                    return create_table_and_test(table_name, client)
-            with mock_dynamodb():
+                else:
-                resp = create_table_and_test(table_name, client)
+                    return func()
-        return resp
+            else:
+                with mock_dynamodb():
+                    if create_table:
+                        return create_table_and_test(table_name, client)
+                    else:
+                        return func()
 
-    def create_table_and_test(table_name, client):
+        def create_table_and_test(table_name, client):
-        client.create_table(
+            client.create_table(
-            TableName=table_name,
+                TableName=table_name,
-            KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],
+                KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],
-            AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],
+                AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],
-            ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 5},
+                ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 5},
-            Tags=[{"Key": "environment", "Value": "moto_tests"}],
+                Tags=[{"Key": "environment", "Value": "moto_tests"}],
-        )
+            )
-        waiter = client.get_waiter("table_exists")
+            waiter = client.get_waiter("table_exists")
-        waiter.wait(TableName=table_name)
+            waiter.wait(TableName=table_name)
-        try:
+            try:
-            resp = func(table_name)
+                resp = func(table_name)
-        finally:
+            finally:
-            ### CLEANUP ###
+                ### CLEANUP ###
-            client.delete_table(TableName=table_name)
+                client.delete_table(TableName=table_name)
 
-        return resp
+            return resp
 
-    return pagination_wrapper
+        return pagination_wrapper
+
+    return inner

tests/test_dynamodb/exceptions/test_dynamodb_exceptions.py

@@ -1117,7 +1117,7 @@ def test_query_with_missing_expression_attribute():
 
 
 @pytest.mark.aws_verified
-@dynamodb_aws_verified
+@dynamodb_aws_verified()
 def test_update_item_returns_old_item(table_name=None):
     dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
     table = dynamodb.Table(table_name)
@@ -1164,3 +1164,36 @@ def test_update_item_returns_old_item(table_name=None):
         "lock": {"M": {"acquired_at": {"N": "123"}}},
         "pk": {"S": "mark"},
     }
+
+
+@pytest.mark.aws_verified
+@dynamodb_aws_verified()
+def test_scan_with_missing_value(table_name=None):
+    dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
+    table = dynamodb.Table(table_name)
+
+    with pytest.raises(ClientError) as exc:
+        table.scan(
+            FilterExpression="attr = loc",
+            # Missing ':'
+            ExpressionAttributeValues={"loc": "sth"},
+        )
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ValidationException"
+    assert (
+        err["Message"]
+        == 'ExpressionAttributeValues contains invalid key: Syntax error; key: "loc"'
+    )
+
+    with pytest.raises(ClientError) as exc:
+        table.query(
+            KeyConditionExpression="attr = loc",
+            # Missing ':'
+            ExpressionAttributeValues={"loc": "sth"},
+        )
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ValidationException"
+    assert (
+        err["Message"]
+        == 'ExpressionAttributeValues contains invalid key: Syntax error; key: "loc"'
+    )

tests/test_dynamodb/test_dynamodb.py

@@ -3686,7 +3686,7 @@ def test_transact_write_items_put():
 
 
 @pytest.mark.aws_verified
-@dynamodb_aws_verified
+@dynamodb_aws_verified()
 def test_transact_write_items_put_conditional_expressions(table_name=None):
     dynamodb = boto3.client("dynamodb", region_name="us-east-1")
 
@@ -3731,7 +3731,7 @@ def test_transact_write_items_put_conditional_expressions(table_name=None):
 
 
 @pytest.mark.aws_verified
-@dynamodb_aws_verified
+@dynamodb_aws_verified()
 def test_transact_write_items_failure__return_item(table_name=None):
     dynamodb = boto3.client("dynamodb", region_name="us-east-1")
     dynamodb.put_item(TableName=table_name, Item={"pk": {"S": "foo2"}})

tests/test_dynamodb/test_dynamodb_create_table.py

@@ -3,9 +3,10 @@ from botocore.exceptions import ClientError
 from datetime import datetime
 import pytest
 
-from moto import mock_dynamodb, settings
+from moto import mock_dynamodb
 from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
-from moto.dynamodb.models import dynamodb_backends
+
+from . import dynamodb_aws_verified
 
 
 @mock_dynamodb
@@ -399,36 +400,94 @@ def test_create_table_with_ssespecification__custom_kms_key():
     assert actual["SSEDescription"]["KMSMasterKeyArn"] == "custom-kms-key"
 
 
-@mock_dynamodb
+@pytest.mark.aws_verified
+@dynamodb_aws_verified(create_table=False)
 def test_create_table__specify_non_key_column():
-    client = boto3.client("dynamodb", "us-east-2")
+    dynamodb = boto3.client("dynamodb", region_name="us-east-1")
-    client.create_table(
+    with pytest.raises(ClientError) as exc:
-        TableName="tab",
+        dynamodb.create_table(
-        KeySchema=[
+            TableName="unknown-key-type",
-            {"AttributeName": "PK", "KeyType": "HASH"},
+            KeySchema=[
-            {"AttributeName": "SomeColumn", "KeyType": "N"},
+                {"AttributeName": "pk", "KeyType": "HASH"},
-        ],
+                {"AttributeName": "sk", "KeyType": "SORT"},
-        BillingMode="PAY_PER_REQUEST",
+            ],
-        AttributeDefinitions=[
+            AttributeDefinitions=[
-            {"AttributeName": "PK", "AttributeType": "S"},
+                {"AttributeName": "pk", "AttributeType": "S"},
-            {"AttributeName": "SomeColumn", "AttributeType": "N"},
+                {"AttributeName": "sk", "AttributeType": "S"},
-        ],
+            ],
+            ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+        )
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ValidationException"
+    assert (
+        err["Message"]
+        == "1 validation error detected: Value 'SORT' at 'keySchema.2.member.keyType' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]"
     )
 
-    actual = client.describe_table(TableName="tab")["Table"]
+    # Verify we get the same message for Global Secondary Indexes
-    assert actual["KeySchema"] == [
+    with pytest.raises(ClientError) as exc:
-        {"AttributeName": "PK", "KeyType": "HASH"},
+        dynamodb.create_table(
-        {"AttributeName": "SomeColumn", "KeyType": "N"},
+            TableName="unknown-key-type",
-    ]
+            KeySchema=[
+                {"AttributeName": "pk", "KeyType": "HASH"},
+                {"AttributeName": "sk", "KeyType": "RANGE"},
+            ],
+            AttributeDefinitions=[
+                {"AttributeName": "pk", "AttributeType": "S"},
+                {"AttributeName": "sk", "AttributeType": "S"},
+            ],
+            ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+            GlobalSecondaryIndexes=[
+                {
+                    "IndexName": "TestGSI",
+                    # Note that the attributes are not declared, which is also invalid
+                    # But AWS trips over the KeyType=SORT first
+                    "KeySchema": [
+                        {"AttributeName": "n/a", "KeyType": "HASH"},
+                        {"AttributeName": "sth", "KeyType": "SORT"},
+                    ],
+                    "Projection": {"ProjectionType": "ALL"},
+                    "ProvisionedThroughput": {
+                        "ReadCapacityUnits": 5,
+                        "WriteCapacityUnits": 5,
+                    },
+                }
+            ],
+        )
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ValidationException"
+    assert (
+        err["Message"]
+        == "1 validation error detected: Value 'SORT' at 'globalSecondaryIndexes.1.member.keySchema.2.member.keyType' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]"
+    )
 
-    if not settings.TEST_SERVER_MODE:
+    # Verify we get the same message for Local Secondary Indexes
-        ddb = dynamodb_backends[ACCOUNT_ID]["us-east-2"]
+    with pytest.raises(ClientError) as exc:
-        assert {"AttributeName": "PK", "AttributeType": "S"} in ddb.tables["tab"].attr
+        dynamodb.create_table(
-        assert {"AttributeName": "SomeColumn", "AttributeType": "N"} in ddb.tables[
+            TableName="unknown-key-type",
-            "tab"
+            KeySchema=[
-        ].attr
+                {"AttributeName": "pk", "KeyType": "HASH"},
-        # It should recognize PK is the Hash Key
+                {"AttributeName": "sk", "KeyType": "RANGE"},
-        assert ddb.tables["tab"].hash_key_attr == "PK"
+            ],
-        # It should recognize that SomeColumn is not a Range Key
+            AttributeDefinitions=[
-        assert ddb.tables["tab"].has_range_key is False
+                {"AttributeName": "pk", "AttributeType": "S"},
-        assert ddb.tables["tab"].range_key_names == []
+                {"AttributeName": "sk", "AttributeType": "S"},
+            ],
+            ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+            LocalSecondaryIndexes=[
+                {
+                    "IndexName": "test_lsi",
+                    "KeySchema": [
+                        {"AttributeName": "pk", "KeyType": "HASH"},
+                        {"AttributeName": "lsi_range_key", "KeyType": "SORT"},
+                    ],
+                    "Projection": {"ProjectionType": "ALL"},
+                }
+            ],
+        )
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ValidationException"
+    assert (
+        err["Message"]
+        == "1 validation error detected: Value 'SORT' at 'localSecondaryIndexes.1.member.keySchema.2.member.keyType' failed to satisfy constraint: Member must satisfy enum value set: [HASH, RANGE]"
+    )

tests/test_dynamodb/test_dynamodb_statements.py

@@ -25,7 +25,7 @@ def create_items(table_name):
 
 
 @pytest.mark.aws_verified
-@dynamodb_aws_verified
+@dynamodb_aws_verified()
 def test_execute_statement_select_star(table_name=None):
     client = boto3.client("dynamodb", "us-east-1")
     create_items(table_name)
@@ -35,7 +35,7 @@ def test_execute_statement_select_star(table_name=None):
 
 
 @pytest.mark.aws_verified
-@dynamodb_aws_verified
+@dynamodb_aws_verified()
 def test_execute_statement_select_attr(table_name=None):
     client = boto3.client("dynamodb", "us-east-1")
     create_items(table_name)
@@ -47,7 +47,7 @@ def test_execute_statement_select_attr(table_name=None):
 
 
 @pytest.mark.aws_verified
-@dynamodb_aws_verified
+@dynamodb_aws_verified()
 def test_execute_statement_with_quoted_table(table_name=None):
     client = boto3.client("dynamodb", "us-east-1")
     create_items(table_name)
@@ -57,7 +57,7 @@ def test_execute_statement_with_quoted_table(table_name=None):
 
 
 @pytest.mark.aws_verified
-@dynamodb_aws_verified
+@dynamodb_aws_verified()
 def test_execute_statement_with_parameter(table_name=None):
     client = boto3.client("dynamodb", "us-east-1")
     create_items(table_name)
@@ -77,7 +77,7 @@ def test_execute_statement_with_parameter(table_name=None):
 
 
 @pytest.mark.aws_verified
-@dynamodb_aws_verified
+@dynamodb_aws_verified()
 def test_execute_statement_with_no_results(table_name=None):
     client = boto3.client("dynamodb", "us-east-1")
     create_items(table_name)
@@ -201,7 +201,7 @@ class TestBatchExecuteStatement(TestCase):
 
 
 @pytest.mark.aws_verified
-@dynamodb_aws_verified
+@dynamodb_aws_verified()
 def test_execute_statement_with_all_clauses(table_name=None):
     dynamodb_client = boto3.client("dynamodb", "us-east-1")