DynamoDB - also look at Global indexes to see if the value of a sort key is too big (#4632)

This commit is contained in:
Bert Blommers 2021-11-24 12:22:35 -01:00 committed by GitHub
parent 71f831dae6
commit a02bdd91ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 313 additions and 251 deletions

View File

@ -72,11 +72,6 @@ class Item(BaseModel):
self.hash_key = hash_key
self.range_key = range_key
if hash_key and hash_key.size() > HASH_KEY_MAX_LENGTH:
raise HashKeyTooLong
if range_key and (range_key.size() > RANGE_KEY_MAX_LENGTH):
raise RangeKeyTooLong
self.attrs = LimitedSizeDict()
for key, value in attrs.items():
self.attrs[key] = DynamoType(value)
@ -595,6 +590,18 @@ class Table(CloudFormationModel):
keys.append(range_key)
return keys
def _validate_key_sizes(self, item_attrs):
for hash_name in self.hash_key_names:
hash_value = item_attrs.get(hash_name)
if hash_value:
if DynamoType(hash_value).size() > HASH_KEY_MAX_LENGTH:
raise HashKeyTooLong
for range_name in self.range_key_names:
range_value = item_attrs.get(range_name)
if range_value:
if DynamoType(range_value).size() > RANGE_KEY_MAX_LENGTH:
raise RangeKeyTooLong
def put_item(
self,
item_attrs,
@ -634,6 +641,9 @@ class Table(CloudFormationModel):
expected_type=self.range_key_type,
actual_type=range_value.type,
)
self._validate_key_sizes(item_attrs)
if expected is None:
expected = {}
lookup_range_value = range_value

View File

@ -0,0 +1,298 @@
from __future__ import print_function
import boto3
import sure # noqa # pylint: disable=unused-import
import pytest
from moto import mock_dynamodb2
from botocore.exceptions import ClientError
from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH
@mock_dynamodb2
def test_item_add_long_string_hash_key_exception():
name = "TestTable"
conn = boto3.client("dynamodb", region_name="us-west-2")
conn.create_table(
TableName=name,
KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
)
@mock_dynamodb2
def test_item_add_long_string_nonascii_hash_key_exception():
name = "TestTable"
conn = boto3.client("dynamodb", region_name="us-west-2")
conn.create_table(
TableName=name,
KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
emoji_b = b"\xf0\x9f\x98\x83" # smile emoji
emoji = emoji_b.decode("utf-8") # 1 character, but 4 bytes
short_enough = emoji * int(HASH_KEY_MAX_LENGTH / len(emoji.encode("utf-8")))
too_long = "x" + short_enough
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": short_enough},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": too_long},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
)
@mock_dynamodb2
def test_item_add_long_string_range_key_exception():
name = "TestTable"
conn = boto3.client("dynamodb", region_name="us-west-2")
conn.create_table(
TableName=name,
KeySchema=[
{"AttributeName": "forum_name", "KeyType": "HASH"},
{"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "forum_name", "AttributeType": "S"},
{"AttributeName": "ReceivedTime", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "LOLCat Forum"},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "someone@somewhere.edu"},
"ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH},
},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "LOLCat Forum"},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "someone@somewhere.edu"},
"ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
)
@mock_dynamodb2
def test_put_long_string_gsi_range_key_exception():
name = "TestTable"
conn = boto3.client("dynamodb", region_name="us-west-2")
conn.create_table(
TableName=name,
KeySchema=[
{"AttributeName": "partition_key", "KeyType": "HASH"},
{"AttributeName": "sort_key", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "partition_key", "AttributeType": "S"},
{"AttributeName": "sort_key", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.put_item(
TableName=name,
Item={
# partition_key is only used as the HASH key
# so we can set it to range key length
"partition_key": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
"sort_key": {"S": "sk"},
},
)
conn.update_table(
TableName=name,
AttributeDefinitions=[
{"AttributeName": "partition_key", "AttributeType": "S"},
{"AttributeName": "sort_key", "AttributeType": "S"},
],
GlobalSecondaryIndexUpdates=[
{
"Create": {
"IndexName": "random-table-index",
"KeySchema": [
{"AttributeName": "sort_key", "KeyType": "HASH",},
{"AttributeName": "partition_key", "KeyType": "RANGE",},
],
"Projection": {"ProjectionType": "KEYS_ONLY",},
"ProvisionedThroughput": {
"ReadCapacityUnits": 20,
"WriteCapacityUnits": 20,
},
}
},
],
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
# partition_key is used as a range key in the GSI
# so updating this should still fail
"partition_key": {"S": "y" * (RANGE_KEY_MAX_LENGTH + 1)},
"sort_key": {"S": "sk2"},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
)
@mock_dynamodb2
def test_update_item_with_long_string_hash_key_exception():
name = "TestTable"
conn = boto3.client("dynamodb", region_name="us-west-2")
conn.create_table(
TableName=name,
KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
with pytest.raises(ClientError) as ex:
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
)
@mock_dynamodb2
def test_update_item_with_long_string_range_key_exception():
name = "TestTable"
conn = boto3.client("dynamodb", region_name="us-west-2")
conn.create_table(
TableName=name,
KeySchema=[
{"AttributeName": "forum_name", "KeyType": "HASH"},
{"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "forum_name", "AttributeType": "S"},
{"AttributeName": "ReceivedTime", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "Lolcat Forum"},
"ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
with pytest.raises(ClientError) as ex:
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "Lolcat Forum"},
"ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
)

View File

@ -17,7 +17,6 @@ from tests.helpers import requires_boto_gte
import moto.dynamodb2.comparisons
import moto.dynamodb2.models
from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH
import pytest
@ -385,251 +384,6 @@ def test_update_item_with_empty_string_attr_no_exception():
)
@mock_dynamodb2
def test_item_add_long_string_hash_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
)
@mock_dynamodb2
def test_item_add_long_string_nonascii_hash_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
emoji_b = b"\xf0\x9f\x98\x83" # smile emoji
emoji = emoji_b.decode("utf-8") # 1 character, but 4 bytes
short_enough = emoji * int(HASH_KEY_MAX_LENGTH / len(emoji.encode("utf-8")))
too_long = "x" + short_enough
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": short_enough},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": too_long},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
)
@mock_dynamodb2
def test_item_add_long_string_range_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[
{"AttributeName": "forum_name", "KeyType": "HASH"},
{"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "forum_name", "AttributeType": "S"},
{"AttributeName": "ReceivedTime", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "LOLCat Forum"},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "someone@somewhere.edu"},
"ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH},
},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "LOLCat Forum"},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "someone@somewhere.edu"},
"ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
)
@mock_dynamodb2
def test_update_item_with_long_string_hash_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
with pytest.raises(ClientError) as ex:
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
)
@mock_dynamodb2
def test_update_item_with_long_string_range_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[
{"AttributeName": "forum_name", "KeyType": "HASH"},
{"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "forum_name", "AttributeType": "S"},
{"AttributeName": "ReceivedTime", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "Lolcat Forum"},
"ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
with pytest.raises(ClientError) as ex:
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "Lolcat Forum"},
"ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
)
@requires_boto_gte("2.9")
@mock_dynamodb2
def test_query_invalid_table():