Enforce dynamodb key size limit [solves #3866] (#3888)

* add tests for dynamodb max key size

correct too-large error for ddb key

* remove unnecessary requires_boto_gte decorator from ddb tests

* remove literal emoji from ddb test

* implement dynamodb key limits, WIP

* correct direction of dynamodb range key length check

* fix tests for dynamodb max key size check

* catch ddb validation errors and rethrow properly

* finish ddb key size limit fixes

* fix linting

* handle unicode in v2.7 tests

* fix encoding issue in py2.7 for ddb

* linting

* Python2/3 compatability

Co-authored-by: Bert Blommers <info@bertblommers.nl>

.gitignore

@@ -24,3 +24,4 @@ tests/file.tmp
 *.tmp
 .venv/
 htmlcov/
+.~c9_*

moto/core/responses.py

@@ -149,7 +149,7 @@ class ActionAuthenticatorMixin(object):
                 if settings.TEST_SERVER_MODE:
                     response = requests.post(
                         "http://localhost:5000/moto-api/reset-auth",
-                        data=str(initial_no_auth_action_count).encode(),
+                        data=str(initial_no_auth_action_count).encode("utf-8"),
                     )
                     original_initial_no_auth_action_count = response.json()[
                         "PREVIOUS_INITIAL_NO_AUTH_ACTION_COUNT"
@@ -167,7 +167,9 @@ class ActionAuthenticatorMixin(object):
                     if settings.TEST_SERVER_MODE:
                         requests.post(
                             "http://localhost:5000/moto-api/reset-auth",
-                            data=str(original_initial_no_auth_action_count).encode(),
+                            data=str(original_initial_no_auth_action_count).encode(
+                                "utf-8"
+                            ),
                         )
                     else:
                         ActionAuthenticatorMixin.request_count = original_request_count

moto/core/utils.py

@@ -230,7 +230,7 @@ def unix_time_millis(dt=None):
 
 def gen_amz_crc32(response, headerdict=None):
     if not isinstance(response, bytes):
-        response = response.encode()
+        response = response.encode("utf-8")
 
     crc = binascii.crc32(response)
     if six.PY2:

moto/dynamodb2/exceptions.py

@@ -1,3 +1,6 @@
+from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH
+
+
 class InvalidIndexNameError(ValueError):
     pass
 
@@ -133,6 +136,25 @@ class ItemSizeToUpdateTooLarge(MockValidationException):
         )
 
 
+class HashKeyTooLong(MockValidationException):
+    # deliberately no space between of and {lim}
+    key_too_large_msg = "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of{lim} bytes".format(
+        lim=HASH_KEY_MAX_LENGTH
+    )
+
+    def __init__(self):
+        super(HashKeyTooLong, self).__init__(self.key_too_large_msg)
+
+
+class RangeKeyTooLong(MockValidationException):
+    key_too_large_msg = "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of {lim} bytes".format(
+        lim=RANGE_KEY_MAX_LENGTH
+    )
+
+    def __init__(self):
+        super(RangeKeyTooLong, self).__init__(self.key_too_large_msg)
+
+
 class IncorrectOperandType(InvalidUpdateExpression):
     inv_operand_msg = "Incorrect operand type for operator or function; operator or function: {f}, operand type: {t}"
 

moto/dynamodb2/limits.py

@@ -0,0 +1,5 @@
+# https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-partition-sort-keys
+# measured in bytes
+# <= not <
+HASH_KEY_MAX_LENGTH = 2048
+RANGE_KEY_MAX_LENGTH = 1024

moto/dynamodb2/models/__init__.py

@@ -18,6 +18,8 @@ from moto.dynamodb2.exceptions import (
     InvalidIndexNameError,
     ItemSizeTooLarge,
     ItemSizeToUpdateTooLarge,
+    HashKeyTooLong,
+    RangeKeyTooLong,
     ConditionalCheckFailed,
     TransactionCanceledException,
     EmptyKeyAttributeException,
@@ -27,6 +29,7 @@ from moto.dynamodb2.models.dynamo_type import DynamoType
 from moto.dynamodb2.parsing.executors import UpdateExpressionExecutor
 from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
 from moto.dynamodb2.parsing.validators import UpdateExpressionValidator
+from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH
 
 
 class DynamoJsonEncoder(json.JSONEncoder):
@@ -70,6 +73,11 @@ class Item(BaseModel):
         self.range_key = range_key
         self.range_key_type = range_key_type
 
+        if hash_key and hash_key.size() > HASH_KEY_MAX_LENGTH:
+            raise HashKeyTooLong
+        if range_key and (range_key.size() > RANGE_KEY_MAX_LENGTH):
+            raise RangeKeyTooLong
+
         self.attrs = LimitedSizeDict()
         for key, value in attrs.items():
             self.attrs[key] = DynamoType(value)
@@ -1309,13 +1317,14 @@ class DynamoDBBackend(BaseBackend):
             item.validate_no_empty_key_values(attribute_updates, table.key_attributes)
 
         if update_expression:
-            validated_ast = UpdateExpressionValidator(
+            validator = UpdateExpressionValidator(
                 update_expression_ast,
                 expression_attribute_names=expression_attribute_names,
                 expression_attribute_values=expression_attribute_values,
                 item=item,
                 table=table,
-            ).validate()
+            )
+            validated_ast = validator.validate()
             try:
                 UpdateExpressionExecutor(
                     validated_ast, item, expression_attribute_names

moto/dynamodb2/models/utilities.py

@@ -2,7 +2,7 @@ import re
 
 
 def bytesize(val):
-    return len(str(val).encode("utf-8"))
+    return len(val.encode("utf-8"))
 
 
 def attribute_is_list(attr):

moto/dynamodb2/responses.py

@@ -11,7 +11,6 @@ from moto.core.responses import BaseResponse
 from moto.core.utils import camelcase_to_underscores, amz_crc32, amzn_request_id
 from .exceptions import (
     InvalidIndexNameError,
-    ItemSizeTooLarge,
     MockValidationException,
     TransactionCanceledException,
 )
@@ -296,9 +295,9 @@ class DynamoHandler(BaseResponse):
                 expression_attribute_values,
                 overwrite,
             )
-        except ItemSizeTooLarge:
+        except MockValidationException as mve:
             er = "com.amazonaws.dynamodb.v20111205#ValidationException"
-            return self.error(er, ItemSizeTooLarge.item_size_too_large_msg)
+            return self.error(er, mve.exception_msg)
         except KeyError as ke:
             er = "com.amazonaws.dynamodb.v20111205#ValidationException"
             return self.error(er, ke.args[0])

tests/test_dynamodb2/test_dynamodb.py

@@ -16,6 +16,7 @@ from tests.helpers import requires_boto_gte
 
 import moto.dynamodb2.comparisons
 import moto.dynamodb2.models
+from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH
 
 import pytest
 
@@ -184,9 +185,8 @@ def test_list_not_found_table_tags():
         assert exception.response["Error"]["Code"] == "ResourceNotFoundException"
 
 
-@requires_boto_gte("2.9")
 @mock_dynamodb2
-def test_item_add_empty_string_in_key_exception():
+def test_item_add_empty_string_hash_key_exception():
     name = "TestTable"
     conn = boto3.client(
         "dynamodb",
@@ -220,9 +220,49 @@ def test_item_add_empty_string_in_key_exception():
     )
 
 
-@requires_boto_gte("2.9")
 @mock_dynamodb2
-def test_item_add_empty_string_no_exception():
+def test_item_add_empty_string_range_key_exception():
+    name = "TestTable"
+    conn = boto3.client(
+        "dynamodb",
+        region_name="us-west-2",
+        aws_access_key_id="ak",
+        aws_secret_access_key="sk",
+    )
+    conn.create_table(
+        TableName=name,
+        KeySchema=[
+            {"AttributeName": "forum_name", "KeyType": "HASH"},
+            {"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "forum_name", "AttributeType": "S"},
+            {"AttributeName": "ReceivedTime", "AttributeType": "S"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+
+    with pytest.raises(ClientError) as ex:
+        conn.put_item(
+            TableName=name,
+            Item={
+                "forum_name": {"S": "LOLCat Forum"},
+                "subject": {"S": "Check this out!"},
+                "Body": {"S": "http://url_to_lolcat.gif"},
+                "SentBy": {"S": "someone@somewhere.edu"},
+                "ReceivedTime": {"S": ""},
+            },
+        )
+
+    ex.value.response["Error"]["Code"].should.equal("ValidationException")
+    ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    ex.value.response["Error"]["Message"].should.equal(
+        "One or more parameter values were invalid: An AttributeValue may not contain an empty string"
+    )
+
+
+@mock_dynamodb2
+def test_item_add_empty_string_attr_no_exception():
     name = "TestTable"
     conn = boto3.client(
         "dynamodb",
@@ -249,52 +289,8 @@ def test_item_add_empty_string_no_exception():
     )
 
 
-@requires_boto_gte("2.9")
 @mock_dynamodb2
-def test_update_item_with_empty_string_in_key_exception():
+def test_update_item_with_empty_string_attr_no_exception():
-    name = "TestTable"
-    conn = boto3.client(
-        "dynamodb",
-        region_name="us-west-2",
-        aws_access_key_id="ak",
-        aws_secret_access_key="sk",
-    )
-    conn.create_table(
-        TableName=name,
-        KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
-        AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
-        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
-    )
-
-    conn.put_item(
-        TableName=name,
-        Item={
-            "forum_name": {"S": "LOLCat Forum"},
-            "subject": {"S": "Check this out!"},
-            "Body": {"S": "http://url_to_lolcat.gif"},
-            "SentBy": {"S": "test"},
-            "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
-        },
-    )
-
-    with pytest.raises(ClientError) as ex:
-        conn.update_item(
-            TableName=name,
-            Key={"forum_name": {"S": "LOLCat Forum"}},
-            UpdateExpression="set forum_name=:NewName",
-            ExpressionAttributeValues={":NewName": {"S": ""}},
-        )
-
-    ex.value.response["Error"]["Code"].should.equal("ValidationException")
-    ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
-    ex.value.response["Error"]["Message"].should.equal(
-        "One or more parameter values were invalid: An AttributeValue may not contain an empty string"
-    )
-
-
-@requires_boto_gte("2.9")
-@mock_dynamodb2
-def test_update_item_with_empty_string_no_exception():
     name = "TestTable"
     conn = boto3.client(
         "dynamodb",
@@ -328,6 +324,303 @@ def test_update_item_with_empty_string_no_exception():
     )
 
 
+@mock_dynamodb2
+def test_item_add_long_string_hash_key_exception():
+    name = "TestTable"
+    conn = boto3.client(
+        "dynamodb",
+        region_name="us-west-2",
+        aws_access_key_id="ak",
+        aws_secret_access_key="sk",
+    )
+    conn.create_table(
+        TableName=name,
+        KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+
+    conn.put_item(
+        TableName=name,
+        Item={
+            "forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH},
+            "subject": {"S": "Check this out!"},
+            "Body": {"S": "http://url_to_lolcat.gif"},
+            "SentBy": {"S": "test"},
+            "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
+        },
+    )
+
+    with pytest.raises(ClientError) as ex:
+        conn.put_item(
+            TableName=name,
+            Item={
+                "forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)},
+                "subject": {"S": "Check this out!"},
+                "Body": {"S": "http://url_to_lolcat.gif"},
+                "SentBy": {"S": "test"},
+                "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
+            },
+        )
+    ex.value.response["Error"]["Code"].should.equal("ValidationException")
+    ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    # deliberately no space between "of" and "2048"
+    ex.value.response["Error"]["Message"].should.equal(
+        "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
+    )
+
+
+@mock_dynamodb2
+def test_item_add_long_string_nonascii_hash_key_exception():
+    name = "TestTable"
+    conn = boto3.client(
+        "dynamodb",
+        region_name="us-west-2",
+        aws_access_key_id="ak",
+        aws_secret_access_key="sk",
+    )
+    conn.create_table(
+        TableName=name,
+        KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+
+    emoji_b = b"\xf0\x9f\x98\x83"  # smile emoji
+    emoji = emoji_b.decode("utf-8")  # 1 character, but 4 bytes
+    short_enough = emoji * int(HASH_KEY_MAX_LENGTH / len(emoji.encode("utf-8")))
+    too_long = "x" + short_enough
+
+    conn.put_item(
+        TableName=name,
+        Item={
+            "forum_name": {"S": short_enough},
+            "subject": {"S": "Check this out!"},
+            "Body": {"S": "http://url_to_lolcat.gif"},
+            "SentBy": {"S": "test"},
+            "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
+        },
+    )
+
+    with pytest.raises(ClientError) as ex:
+        conn.put_item(
+            TableName=name,
+            Item={
+                "forum_name": {"S": too_long},
+                "subject": {"S": "Check this out!"},
+                "Body": {"S": "http://url_to_lolcat.gif"},
+                "SentBy": {"S": "test"},
+                "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
+            },
+        )
+
+    ex.value.response["Error"]["Code"].should.equal("ValidationException")
+    ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    # deliberately no space between "of" and "2048"
+    ex.value.response["Error"]["Message"].should.equal(
+        "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
+    )
+
+
+@mock_dynamodb2
+def test_item_add_long_string_range_key_exception():
+    name = "TestTable"
+    conn = boto3.client(
+        "dynamodb",
+        region_name="us-west-2",
+        aws_access_key_id="ak",
+        aws_secret_access_key="sk",
+    )
+    conn.create_table(
+        TableName=name,
+        KeySchema=[
+            {"AttributeName": "forum_name", "KeyType": "HASH"},
+            {"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "forum_name", "AttributeType": "S"},
+            {"AttributeName": "ReceivedTime", "AttributeType": "S"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+
+    conn.put_item(
+        TableName=name,
+        Item={
+            "forum_name": {"S": "LOLCat Forum"},
+            "subject": {"S": "Check this out!"},
+            "Body": {"S": "http://url_to_lolcat.gif"},
+            "SentBy": {"S": "someone@somewhere.edu"},
+            "ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH},
+        },
+    )
+
+    with pytest.raises(ClientError) as ex:
+        conn.put_item(
+            TableName=name,
+            Item={
+                "forum_name": {"S": "LOLCat Forum"},
+                "subject": {"S": "Check this out!"},
+                "Body": {"S": "http://url_to_lolcat.gif"},
+                "SentBy": {"S": "someone@somewhere.edu"},
+                "ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
+            },
+        )
+
+    ex.value.response["Error"]["Code"].should.equal("ValidationException")
+    ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    ex.value.response["Error"]["Message"].should.equal(
+        "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
+    )
+
+
+@mock_dynamodb2
+def test_item_add_long_string_range_key_exception():
+    name = "TestTable"
+    conn = boto3.client(
+        "dynamodb",
+        region_name="us-west-2",
+        aws_access_key_id="ak",
+        aws_secret_access_key="sk",
+    )
+    conn.create_table(
+        TableName=name,
+        KeySchema=[
+            {"AttributeName": "forum_name", "KeyType": "HASH"},
+            {"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "forum_name", "AttributeType": "S"},
+            {"AttributeName": "ReceivedTime", "AttributeType": "S"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+
+    conn.put_item(
+        TableName=name,
+        Item={
+            "forum_name": {"S": "LOLCat Forum"},
+            "subject": {"S": "Check this out!"},
+            "Body": {"S": "http://url_to_lolcat.gif"},
+            "SentBy": {"S": "someone@somewhere.edu"},
+            "ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH},
+        },
+    )
+
+    with pytest.raises(ClientError) as ex:
+        conn.put_item(
+            TableName=name,
+            Item={
+                "forum_name": {"S": "LOLCat Forum"},
+                "subject": {"S": "Check this out!"},
+                "Body": {"S": "http://url_to_lolcat.gif"},
+                "SentBy": {"S": "someone@somewhere.edu"},
+                "ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
+            },
+        )
+
+    ex.value.response["Error"]["Code"].should.equal("ValidationException")
+    ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    ex.value.response["Error"]["Message"].should.equal(
+        "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
+    )
+
+
+@mock_dynamodb2
+def test_update_item_with_long_string_hash_key_exception():
+    name = "TestTable"
+    conn = boto3.client(
+        "dynamodb",
+        region_name="us-west-2",
+        aws_access_key_id="ak",
+        aws_secret_access_key="sk",
+    )
+    conn.create_table(
+        TableName=name,
+        KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+
+    conn.update_item(
+        TableName=name,
+        Key={
+            "forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH},
+            "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
+        },
+        UpdateExpression="set body=:New",
+        ExpressionAttributeValues={":New": {"S": "hello"}},
+    )
+
+    with pytest.raises(ClientError) as ex:
+        conn.update_item(
+            TableName=name,
+            Key={
+                "forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)},
+                "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
+            },
+            UpdateExpression="set body=:New",
+            ExpressionAttributeValues={":New": {"S": "hello"}},
+        )
+
+    ex.value.response["Error"]["Code"].should.equal("ValidationException")
+    ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    # deliberately no space between "of" and "2048"
+    ex.value.response["Error"]["Message"].should.equal(
+        "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
+    )
+
+
+@mock_dynamodb2
+def test_update_item_with_long_string_range_key_exception():
+    name = "TestTable"
+    conn = boto3.client(
+        "dynamodb",
+        region_name="us-west-2",
+        aws_access_key_id="ak",
+        aws_secret_access_key="sk",
+    )
+    conn.create_table(
+        TableName=name,
+        KeySchema=[
+            {"AttributeName": "forum_name", "KeyType": "HASH"},
+            {"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "forum_name", "AttributeType": "S"},
+            {"AttributeName": "ReceivedTime", "AttributeType": "S"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
+    )
+    conn.update_item(
+        TableName=name,
+        Key={
+            "forum_name": {"S": "Lolcat Forum"},
+            "ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH},
+        },
+        UpdateExpression="set body=:New",
+        ExpressionAttributeValues={":New": {"S": "hello"}},
+    )
+
+    with pytest.raises(ClientError) as ex:
+        conn.update_item(
+            TableName=name,
+            Key={
+                "forum_name": {"S": "Lolcat Forum"},
+                "ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
+            },
+            UpdateExpression="set body=:New",
+            ExpressionAttributeValues={":New": {"S": "hello"}},
+        )
+
+    ex.value.response["Error"]["Code"].should.equal("ValidationException")
+    ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    # deliberately no space between "of" and "2048"
+    ex.value.response["Error"]["Message"].should.equal(
+        "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
+    )
+
+
 @requires_boto_gte("2.9")
 @mock_dynamodb2
 def test_query_invalid_table():