Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enforce dynamodb key size limit [solves #3866] (#3888)

Browse Source 
* add tests for dynamodb max key size

correct too-large error for ddb key

* remove unnecessary requires_boto_gte decorator from ddb tests

* remove literal emoji from ddb test

* implement dynamodb key limits, WIP

* correct direction of dynamodb range key length check

* fix tests for dynamodb max key size check

* catch ddb validation errors and rethrow properly

* finish ddb key size limit fixes

* fix linting

* handle unicode in v2.7 tests

* fix encoding issue in py2.7 for ddb

* linting

* Python2/3 compatability

Co-authored-by: Bert Blommers <info@bertblommers.nl>
This commit is contained in:
Matthew Davis 2021-04-30 22:47:47 +10:00 committed by GitHub
parent 58381cce8f
commit d6384fcb35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 390 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -23,4 +23,5 @@ tests/file.tmp
.mypy_cache/
*.tmp
.venv/
htmlcov/
htmlcov/
.~c9_*

6
moto/core/responses.py
View File

@ -149,7 +149,7 @@ class ActionAuthenticatorMixin(object):
if settings.TEST_SERVER_MODE:
response = requests.post(
"http://localhost:5000/moto-api/reset-auth",
data=str(initial_no_auth_action_count).encode(),
data=str(initial_no_auth_action_count).encode("utf-8"),
)
original_initial_no_auth_action_count = response.json()[
"PREVIOUS_INITIAL_NO_AUTH_ACTION_COUNT"
@ -167,7 +167,9 @@ class ActionAuthenticatorMixin(object):
if settings.TEST_SERVER_MODE:
requests.post(
"http://localhost:5000/moto-api/reset-auth",
data=str(original_initial_no_auth_action_count).encode(),
data=str(original_initial_no_auth_action_count).encode(
"utf-8"
),
)
else:
ActionAuthenticatorMixin.request_count = original_request_count

2
moto/core/utils.py
View File

@ -230,7 +230,7 @@ def unix_time_millis(dt=None):
def gen_amz_crc32(response, headerdict=None):
if not isinstance(response, bytes):
response = response.encode()
response = response.encode("utf-8")
crc = binascii.crc32(response)
if six.PY2:

22
moto/dynamodb2/exceptions.py
View File

@ -1,3 +1,6 @@
from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH
class InvalidIndexNameError(ValueError):
pass
@ -133,6 +136,25 @@ class ItemSizeToUpdateTooLarge(MockValidationException):
)
class HashKeyTooLong(MockValidationException):
# deliberately no space between of and {lim}
key_too_large_msg = "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of{lim} bytes".format(
lim=HASH_KEY_MAX_LENGTH
)
def __init__(self):
super(HashKeyTooLong, self).__init__(self.key_too_large_msg)
class RangeKeyTooLong(MockValidationException):
key_too_large_msg = "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of {lim} bytes".format(
lim=RANGE_KEY_MAX_LENGTH
)
def __init__(self):
super(RangeKeyTooLong, self).__init__(self.key_too_large_msg)
class IncorrectOperandType(InvalidUpdateExpression):
inv_operand_msg = "Incorrect operand type for operator or function; operator or function: {f}, operand type: {t}"

5
moto/dynamodb2/limits.py Normal file
View File

@ -0,0 +1,5 @@
# https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-partition-sort-keys
# measured in bytes
# <= not <
HASH_KEY_MAX_LENGTH = 2048
RANGE_KEY_MAX_LENGTH = 1024

13
moto/dynamodb2/models/__init__.py
View File

@ -18,6 +18,8 @@ from moto.dynamodb2.exceptions import (
InvalidIndexNameError,
ItemSizeTooLarge,
ItemSizeToUpdateTooLarge,
HashKeyTooLong,
RangeKeyTooLong,
ConditionalCheckFailed,
TransactionCanceledException,
EmptyKeyAttributeException,
@ -27,6 +29,7 @@ from moto.dynamodb2.models.dynamo_type import DynamoType
from moto.dynamodb2.parsing.executors import UpdateExpressionExecutor
from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
from moto.dynamodb2.parsing.validators import UpdateExpressionValidator
from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH
class DynamoJsonEncoder(json.JSONEncoder):
@ -70,6 +73,11 @@ class Item(BaseModel):
self.range_key = range_key
self.range_key_type = range_key_type
if hash_key and hash_key.size() > HASH_KEY_MAX_LENGTH:
raise HashKeyTooLong
if range_key and (range_key.size() > RANGE_KEY_MAX_LENGTH):
raise RangeKeyTooLong
self.attrs = LimitedSizeDict()
for key, value in attrs.items():
self.attrs[key] = DynamoType(value)
@ -1309,13 +1317,14 @@ class DynamoDBBackend(BaseBackend):
item.validate_no_empty_key_values(attribute_updates, table.key_attributes)
if update_expression:
validated_ast = UpdateExpressionValidator(
validator = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=expression_attribute_names,
expression_attribute_values=expression_attribute_values,
item=item,
table=table,
).validate()
)
validated_ast = validator.validate()
try:
UpdateExpressionExecutor(
validated_ast, item, expression_attribute_names

2
moto/dynamodb2/models/utilities.py
View File

@ -2,7 +2,7 @@ import re
def bytesize(val):
return len(str(val).encode("utf-8"))
return len(val.encode("utf-8"))
def attribute_is_list(attr):

5
moto/dynamodb2/responses.py
View File

@ -11,7 +11,6 @@ from moto.core.responses import BaseResponse
from moto.core.utils import camelcase_to_underscores, amz_crc32, amzn_request_id
from .exceptions import (
InvalidIndexNameError,
ItemSizeTooLarge,
MockValidationException,
TransactionCanceledException,
)
@ -296,9 +295,9 @@ class DynamoHandler(BaseResponse):
expression_attribute_values,
overwrite,
)
except ItemSizeTooLarge:
except MockValidationException as mve:
er = "com.amazonaws.dynamodb.v20111205#ValidationException"
return self.error(er, ItemSizeTooLarge.item_size_too_large_msg)
return self.error(er, mve.exception_msg)
except KeyError as ke:
er = "com.amazonaws.dynamodb.v20111205#ValidationException"
return self.error(er, ke.args[0])

391
tests/test_dynamodb2/test_dynamodb.py
View File

@ -16,6 +16,7 @@ from tests.helpers import requires_boto_gte
import moto.dynamodb2.comparisons
import moto.dynamodb2.models
from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH
import pytest
@ -184,9 +185,8 @@ def test_list_not_found_table_tags():
assert exception.response["Error"]["Code"] == "ResourceNotFoundException"
@requires_boto_gte("2.9")
@mock_dynamodb2
def test_item_add_empty_string_in_key_exception():
def test_item_add_empty_string_hash_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
@ -220,9 +220,49 @@ def test_item_add_empty_string_in_key_exception():
)
@requires_boto_gte("2.9")
@mock_dynamodb2
def test_item_add_empty_string_no_exception():
def test_item_add_empty_string_range_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[
{"AttributeName": "forum_name", "KeyType": "HASH"},
{"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "forum_name", "AttributeType": "S"},
{"AttributeName": "ReceivedTime", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "LOLCat Forum"},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "someone@somewhere.edu"},
"ReceivedTime": {"S": ""},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: An AttributeValue may not contain an empty string"
)
@mock_dynamodb2
def test_item_add_empty_string_attr_no_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
@ -249,52 +289,8 @@ def test_item_add_empty_string_no_exception():
)
@requires_boto_gte("2.9")
@mock_dynamodb2
def test_update_item_with_empty_string_in_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "LOLCat Forum"},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
with pytest.raises(ClientError) as ex:
conn.update_item(
TableName=name,
Key={"forum_name": {"S": "LOLCat Forum"}},
UpdateExpression="set forum_name=:NewName",
ExpressionAttributeValues={":NewName": {"S": ""}},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: An AttributeValue may not contain an empty string"
)
@requires_boto_gte("2.9")
@mock_dynamodb2
def test_update_item_with_empty_string_no_exception():
def test_update_item_with_empty_string_attr_no_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
@ -328,6 +324,303 @@ def test_update_item_with_empty_string_no_exception():
)
@mock_dynamodb2
def test_item_add_long_string_hash_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
)
@mock_dynamodb2
def test_item_add_long_string_nonascii_hash_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
emoji_b = b"\xf0\x9f\x98\x83" # smile emoji
emoji = emoji_b.decode("utf-8") # 1 character, but 4 bytes
short_enough = emoji * int(HASH_KEY_MAX_LENGTH / len(emoji.encode("utf-8")))
too_long = "x" + short_enough
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": short_enough},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": too_long},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "test"},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
)
@mock_dynamodb2
def test_item_add_long_string_range_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[
{"AttributeName": "forum_name", "KeyType": "HASH"},
{"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "forum_name", "AttributeType": "S"},
{"AttributeName": "ReceivedTime", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "LOLCat Forum"},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "someone@somewhere.edu"},
"ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH},
},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "LOLCat Forum"},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "someone@somewhere.edu"},
"ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
)
@mock_dynamodb2
def test_item_add_long_string_range_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[
{"AttributeName": "forum_name", "KeyType": "HASH"},
{"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "forum_name", "AttributeType": "S"},
{"AttributeName": "ReceivedTime", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "LOLCat Forum"},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "someone@somewhere.edu"},
"ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH},
},
)
with pytest.raises(ClientError) as ex:
conn.put_item(
TableName=name,
Item={
"forum_name": {"S": "LOLCat Forum"},
"subject": {"S": "Check this out!"},
"Body": {"S": "http://url_to_lolcat.gif"},
"SentBy": {"S": "someone@somewhere.edu"},
"ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
)
@mock_dynamodb2
def test_update_item_with_long_string_hash_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
with pytest.raises(ClientError) as ex:
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)},
"ReceivedTime": {"S": "12/9/2011 11:36:03 PM"},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes"
)
@mock_dynamodb2
def test_update_item_with_long_string_range_key_exception():
name = "TestTable"
conn = boto3.client(
"dynamodb",
region_name="us-west-2",
aws_access_key_id="ak",
aws_secret_access_key="sk",
)
conn.create_table(
TableName=name,
KeySchema=[
{"AttributeName": "forum_name", "KeyType": "HASH"},
{"AttributeName": "ReceivedTime", "KeyType": "RANGE"},
],
AttributeDefinitions=[
{"AttributeName": "forum_name", "AttributeType": "S"},
{"AttributeName": "ReceivedTime", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
)
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "Lolcat Forum"},
"ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
with pytest.raises(ClientError) as ex:
conn.update_item(
TableName=name,
Key={
"forum_name": {"S": "Lolcat Forum"},
"ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)},
},
UpdateExpression="set body=:New",
ExpressionAttributeValues={":New": {"S": "hello"}},
)
ex.value.response["Error"]["Code"].should.equal("ValidationException")
ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
# deliberately no space between "of" and "2048"
ex.value.response["Error"]["Message"].should.equal(
"One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes"
)
@requires_boto_gte("2.9")
@mock_dynamodb2
def test_query_invalid_table():