From a02bdd91ff22e35d6838c4ad347e54abc96b50b9 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 24 Nov 2021 12:22:35 -0100 Subject: [PATCH] DynamoDB - also look at Global indexes to see if the value of a sort key is too big (#4632) --- moto/dynamodb2/models/__init__.py | 20 +- tests/test_dynamodb2/exceptions/__init__.py | 0 .../test_dynamodb_exceptions.py | 0 .../exceptions/test_key_length_exceptions.py | 298 ++++++++++++++++++ tests/test_dynamodb2/test_dynamodb.py | 246 --------------- 5 files changed, 313 insertions(+), 251 deletions(-) create mode 100644 tests/test_dynamodb2/exceptions/__init__.py rename tests/test_dynamodb2/{ => exceptions}/test_dynamodb_exceptions.py (100%) create mode 100644 tests/test_dynamodb2/exceptions/test_key_length_exceptions.py diff --git a/moto/dynamodb2/models/__init__.py b/moto/dynamodb2/models/__init__.py index d4614d6ea..388d82067 100644 --- a/moto/dynamodb2/models/__init__.py +++ b/moto/dynamodb2/models/__init__.py @@ -72,11 +72,6 @@ class Item(BaseModel): self.hash_key = hash_key self.range_key = range_key - if hash_key and hash_key.size() > HASH_KEY_MAX_LENGTH: - raise HashKeyTooLong - if range_key and (range_key.size() > RANGE_KEY_MAX_LENGTH): - raise RangeKeyTooLong - self.attrs = LimitedSizeDict() for key, value in attrs.items(): self.attrs[key] = DynamoType(value) @@ -595,6 +590,18 @@ class Table(CloudFormationModel): keys.append(range_key) return keys + def _validate_key_sizes(self, item_attrs): + for hash_name in self.hash_key_names: + hash_value = item_attrs.get(hash_name) + if hash_value: + if DynamoType(hash_value).size() > HASH_KEY_MAX_LENGTH: + raise HashKeyTooLong + for range_name in self.range_key_names: + range_value = item_attrs.get(range_name) + if range_value: + if DynamoType(range_value).size() > RANGE_KEY_MAX_LENGTH: + raise RangeKeyTooLong + def put_item( self, item_attrs, @@ -634,6 +641,9 @@ class Table(CloudFormationModel): expected_type=self.range_key_type, actual_type=range_value.type, ) + + self._validate_key_sizes(item_attrs) + if expected is None: expected = {} lookup_range_value = range_value diff --git a/tests/test_dynamodb2/exceptions/__init__.py b/tests/test_dynamodb2/exceptions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_dynamodb2/test_dynamodb_exceptions.py b/tests/test_dynamodb2/exceptions/test_dynamodb_exceptions.py similarity index 100% rename from tests/test_dynamodb2/test_dynamodb_exceptions.py rename to tests/test_dynamodb2/exceptions/test_dynamodb_exceptions.py diff --git a/tests/test_dynamodb2/exceptions/test_key_length_exceptions.py b/tests/test_dynamodb2/exceptions/test_key_length_exceptions.py new file mode 100644 index 000000000..12299e7e6 --- /dev/null +++ b/tests/test_dynamodb2/exceptions/test_key_length_exceptions.py @@ -0,0 +1,298 @@ +from __future__ import print_function + +import boto3 +import sure # noqa # pylint: disable=unused-import +import pytest + +from moto import mock_dynamodb2 +from botocore.exceptions import ClientError +from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH + + +@mock_dynamodb2 +def test_item_add_long_string_hash_key_exception(): + name = "TestTable" + conn = boto3.client("dynamodb", region_name="us-west-2") + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # deliberately no space between "of" and "2048" + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes" + ) + + +@mock_dynamodb2 +def test_item_add_long_string_nonascii_hash_key_exception(): + name = "TestTable" + conn = boto3.client("dynamodb", region_name="us-west-2") + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + emoji_b = b"\xf0\x9f\x98\x83" # smile emoji + emoji = emoji_b.decode("utf-8") # 1 character, but 4 bytes + short_enough = emoji * int(HASH_KEY_MAX_LENGTH / len(emoji.encode("utf-8"))) + too_long = "x" + short_enough + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": short_enough}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": too_long}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # deliberately no space between "of" and "2048" + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes" + ) + + +@mock_dynamodb2 +def test_item_add_long_string_range_key_exception(): + name = "TestTable" + conn = boto3.client("dynamodb", region_name="us-west-2") + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "ReceivedTime", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "ReceivedTime", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "someone@somewhere.edu"}, + "ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH}, + }, + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "someone@somewhere.edu"}, + "ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)}, + }, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes" + ) + + +@mock_dynamodb2 +def test_put_long_string_gsi_range_key_exception(): + name = "TestTable" + conn = boto3.client("dynamodb", region_name="us-west-2") + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "partition_key", "KeyType": "HASH"}, + {"AttributeName": "sort_key", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "partition_key", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + # partition_key is only used as the HASH key + # so we can set it to range key length + "partition_key": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)}, + "sort_key": {"S": "sk"}, + }, + ) + + conn.update_table( + TableName=name, + AttributeDefinitions=[ + {"AttributeName": "partition_key", "AttributeType": "S"}, + {"AttributeName": "sort_key", "AttributeType": "S"}, + ], + GlobalSecondaryIndexUpdates=[ + { + "Create": { + "IndexName": "random-table-index", + "KeySchema": [ + {"AttributeName": "sort_key", "KeyType": "HASH",}, + {"AttributeName": "partition_key", "KeyType": "RANGE",}, + ], + "Projection": {"ProjectionType": "KEYS_ONLY",}, + "ProvisionedThroughput": { + "ReadCapacityUnits": 20, + "WriteCapacityUnits": 20, + }, + } + }, + ], + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + # partition_key is used as a range key in the GSI + # so updating this should still fail + "partition_key": {"S": "y" * (RANGE_KEY_MAX_LENGTH + 1)}, + "sort_key": {"S": "sk2"}, + }, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes" + ) + + +@mock_dynamodb2 +def test_update_item_with_long_string_hash_key_exception(): + name = "TestTable" + conn = boto3.client("dynamodb", region_name="us-west-2") + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.update_item( + TableName=name, + Key={ + "forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + UpdateExpression="set body=:New", + ExpressionAttributeValues={":New": {"S": "hello"}}, + ) + + with pytest.raises(ClientError) as ex: + conn.update_item( + TableName=name, + Key={ + "forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + UpdateExpression="set body=:New", + ExpressionAttributeValues={":New": {"S": "hello"}}, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # deliberately no space between "of" and "2048" + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes" + ) + + +@mock_dynamodb2 +def test_update_item_with_long_string_range_key_exception(): + name = "TestTable" + conn = boto3.client("dynamodb", region_name="us-west-2") + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "ReceivedTime", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "ReceivedTime", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + conn.update_item( + TableName=name, + Key={ + "forum_name": {"S": "Lolcat Forum"}, + "ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH}, + }, + UpdateExpression="set body=:New", + ExpressionAttributeValues={":New": {"S": "hello"}}, + ) + + with pytest.raises(ClientError) as ex: + conn.update_item( + TableName=name, + Key={ + "forum_name": {"S": "Lolcat Forum"}, + "ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)}, + }, + UpdateExpression="set body=:New", + ExpressionAttributeValues={":New": {"S": "hello"}}, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # deliberately no space between "of" and "2048" + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes" + ) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 1dc808e7d..dbe238b23 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -17,7 +17,6 @@ from tests.helpers import requires_boto_gte import moto.dynamodb2.comparisons import moto.dynamodb2.models -from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH import pytest @@ -385,251 +384,6 @@ def test_update_item_with_empty_string_attr_no_exception(): ) -@mock_dynamodb2 -def test_item_add_long_string_hash_key_exception(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "test"}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - }, - ) - - with pytest.raises(ClientError) as ex: - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "test"}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - }, - ) - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - # deliberately no space between "of" and "2048" - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes" - ) - - -@mock_dynamodb2 -def test_item_add_long_string_nonascii_hash_key_exception(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - emoji_b = b"\xf0\x9f\x98\x83" # smile emoji - emoji = emoji_b.decode("utf-8") # 1 character, but 4 bytes - short_enough = emoji * int(HASH_KEY_MAX_LENGTH / len(emoji.encode("utf-8"))) - too_long = "x" + short_enough - - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": short_enough}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "test"}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - }, - ) - - with pytest.raises(ClientError) as ex: - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": too_long}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "test"}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - }, - ) - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - # deliberately no space between "of" and "2048" - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes" - ) - - -@mock_dynamodb2 -def test_item_add_long_string_range_key_exception(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[ - {"AttributeName": "forum_name", "KeyType": "HASH"}, - {"AttributeName": "ReceivedTime", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "forum_name", "AttributeType": "S"}, - {"AttributeName": "ReceivedTime", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": "LOLCat Forum"}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "someone@somewhere.edu"}, - "ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH}, - }, - ) - - with pytest.raises(ClientError) as ex: - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": "LOLCat Forum"}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "someone@somewhere.edu"}, - "ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)}, - }, - ) - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes" - ) - - -@mock_dynamodb2 -def test_update_item_with_long_string_hash_key_exception(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - conn.update_item( - TableName=name, - Key={ - "forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - }, - UpdateExpression="set body=:New", - ExpressionAttributeValues={":New": {"S": "hello"}}, - ) - - with pytest.raises(ClientError) as ex: - conn.update_item( - TableName=name, - Key={ - "forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - }, - UpdateExpression="set body=:New", - ExpressionAttributeValues={":New": {"S": "hello"}}, - ) - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - # deliberately no space between "of" and "2048" - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes" - ) - - -@mock_dynamodb2 -def test_update_item_with_long_string_range_key_exception(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[ - {"AttributeName": "forum_name", "KeyType": "HASH"}, - {"AttributeName": "ReceivedTime", "KeyType": "RANGE"}, - ], - AttributeDefinitions=[ - {"AttributeName": "forum_name", "AttributeType": "S"}, - {"AttributeName": "ReceivedTime", "AttributeType": "S"}, - ], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - conn.update_item( - TableName=name, - Key={ - "forum_name": {"S": "Lolcat Forum"}, - "ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH}, - }, - UpdateExpression="set body=:New", - ExpressionAttributeValues={":New": {"S": "hello"}}, - ) - - with pytest.raises(ClientError) as ex: - conn.update_item( - TableName=name, - Key={ - "forum_name": {"S": "Lolcat Forum"}, - "ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)}, - }, - UpdateExpression="set body=:New", - ExpressionAttributeValues={":New": {"S": "hello"}}, - ) - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - # deliberately no space between "of" and "2048" - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes" - ) - - @requires_boto_gte("2.9") @mock_dynamodb2 def test_query_invalid_table():