Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enable AST Validation

Browse Source 
This commit puts AST validation on the execution path. This means updates get
validated prior to being executed. There were quite a few tests that were not
working against Amazon DDB. These tests I considered broken and as such this
commit adapts them such that they pass against Amazon DDB.

test_update_item_on_map()
=> One of the SET actions would try to set a nested element by specifying the nesting on the path
   rather than by putting a map as a value for a non-existent key. This got changed.

test_item_size_is_under_400KB
=> Used the keyword "item" which DDB doesn't like. Change to cont in order to keep the same sizings.
=> Secondly the size error messages differs a bit depending whether it is part of the update or part
   of a put_item. For an update it should be:
   Item size to update has exceeded the maximum allowed size
   otherwise it is
   Item size has exceeded the maximum allowed size'

test_remove_top_level_attribute
=> Used a keyword item.  Use ExpressionAttributeNames

test_update_item_double_nested_remove
=> Used keywords name & first. Migrated to non-deprecated API and use ExpressionAttributeNames

test_update_item_set &
test_boto3_update_item_conditions_pass &
test_boto3_update_item_conditions_pass_because_expect_not_exists &
test_boto3_update_item_conditions_pass_because_expect_not_exists_by_compare_to_null &
test_boto3_update_item_conditions_pass_because_expect_exists_by_compare_to_not_null &
test_boto3_update_item_conditions_fail &
test_boto3_update_item_conditions_fail_because_expect_not_exists &
test_boto3_update_item_conditions_fail_because_expect_not_exists_by_compare_to_null
=> Were broken tests which had string literal instead of value placeholder
This commit is contained in:
pvbouwel 2020-04-19 16:50:53 +01:00
parent fc4d88401d
commit e6b51a28ee
4 changed files with 117 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
moto/dynamodb2/exceptions.py
View File

@ -111,6 +111,17 @@ class ItemSizeTooLarge(MockValidationException):
super(ItemSizeTooLarge, self).__init__(self.item_size_too_large_msg)
class ItemSizeToUpdateTooLarge(MockValidationException):
item_size_to_update_too_large_msg = (
"Item size to update has exceeded the maximum allowed size"
)
def __init__(self):
super(ItemSizeToUpdateTooLarge, self).__init__(
self.item_size_to_update_too_large_msg
)
class IncorrectOperandType(InvalidUpdateExpression):
inv_operand_msg = "Incorrect operand type for operator or function; operator or function: {f}, operand type: {t}"

22
moto/dynamodb2/models/__init__.py
View File

@ -14,11 +14,16 @@ from moto.core import BaseBackend, BaseModel
from moto.core.utils import unix_time
from moto.core.exceptions import JsonRESTError
from moto.dynamodb2.comparisons import get_filter_expression
from moto.dynamodb2.comparisons import get_expected, get_comparison_func
from moto.dynamodb2.exceptions import InvalidIndexNameError, ItemSizeTooLarge, InvalidUpdateExpression
from moto.dynamodb2.comparisons import get_expected
from moto.dynamodb2.exceptions import (
InvalidIndexNameError,
ItemSizeTooLarge,
ItemSizeToUpdateTooLarge,
)
from moto.dynamodb2.models.utilities import bytesize, attribute_is_list
from moto.dynamodb2.models.dynamo_type import DynamoType
from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
from moto.dynamodb2.parsing.validators import UpdateExpressionValidator
class DynamoJsonEncoder(json.JSONEncoder):
@ -151,7 +156,10 @@ class Item(BaseModel):
if "." in key and attr not in self.attrs:
raise ValueError # Setting nested attr not allowed if first attr does not exist yet
elif attr not in self.attrs:
self.attrs[attr] = dyn_value # set new top-level attribute
try:
self.attrs[attr] = dyn_value # set new top-level attribute
except ItemSizeTooLarge:
raise ItemSizeToUpdateTooLarge()
else:
self.attrs[attr].set(
".".join(key.split(".")[1:]), dyn_value, list_index
@ -1202,7 +1210,7 @@ class DynamoDBBackend(BaseBackend):
# E.g. `a = b + c` -> `a=b+c`
if update_expression:
# Parse expression to get validation errors
UpdateExpressionParser.make(update_expression)
update_expression_ast = UpdateExpressionParser.make(update_expression)
update_expression = re.sub(r"\s*([=\+-])\s*", "\\1", update_expression)
if all([table.hash_key_attr in key, table.range_key_attr in key]):
@ -1247,6 +1255,12 @@ class DynamoDBBackend(BaseBackend):
item = table.get_item(hash_value, range_value)
if update_expression:
UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=expression_attribute_names,
expression_attribute_values=expression_attribute_values,
item=item,
).validate()
item.update(
update_expression,
expression_attribute_names,

54
tests/test_dynamodb2/test_dynamodb.py
View File

@ -2147,13 +2147,33 @@ def test_update_item_on_map():
# Nonexistent nested attributes are supported for existing top-level attributes.
table.update_item(
Key={"forum_name": "the-key", "subject": "123"},
UpdateExpression="SET body.#nested.#data = :tb, body.nested.#nonexistentnested.#data = :tb2",
UpdateExpression="SET body.#nested.#data = :tb",
ExpressionAttributeNames={"#nested": "nested", "#data": "data",},
ExpressionAttributeValues={":tb": "new_value"},
)
# Running this against AWS DDB gives an exception so make sure it also fails.:
with assert_raises(client.exceptions.ClientError):
# botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the UpdateItem
# operation: The document path provided in the update expression is invalid for update
table.update_item(
Key={"forum_name": "the-key", "subject": "123"},
UpdateExpression="SET body.#nested.#nonexistentnested.#data = :tb2",
ExpressionAttributeNames={
"#nested": "nested",
"#nonexistentnested": "nonexistentnested",
"#data": "data",
},
ExpressionAttributeValues={":tb2": "other_value"},
)
table.update_item(
Key={"forum_name": "the-key", "subject": "123"},
UpdateExpression="SET body.#nested.#nonexistentnested = :tb2",
ExpressionAttributeNames={
"#nested": "nested",
"#nonexistentnested": "nonexistentnested",
"#data": "data",
},
ExpressionAttributeValues={":tb": "new_value", ":tb2": "other_value"},
ExpressionAttributeValues={":tb2": {"data": "other_value"}},
)
resp = table.scan()
@ -2161,8 +2181,8 @@ def test_update_item_on_map():
{"nested": {"data": "new_value", "nonexistentnested": {"data": "other_value"}}}
)
# Test nested value for a nonexistent attribute.
with assert_raises(client.exceptions.ConditionalCheckFailedException):
# Test nested value for a nonexistent attribute throws a ClientError.
with assert_raises(client.exceptions.ClientError):
table.update_item(
Key={"forum_name": "the-key", "subject": "123"},
UpdateExpression="SET nonexistent.#nested = :tb",
@ -3184,7 +3204,10 @@ def test_remove_top_level_attribute():
TableName=table_name, Item={"id": {"S": "foo"}, "item": {"S": "bar"}}
)
client.update_item(
TableName=table_name, Key={"id": {"S": "foo"}}, UpdateExpression="REMOVE item"
TableName=table_name,
Key={"id": {"S": "foo"}},
UpdateExpression="REMOVE #i",
ExpressionAttributeNames={"#i": "item"},
)
#
result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"]
@ -3359,21 +3382,21 @@ def test_item_size_is_under_400KB():
assert_failure_due_to_item_size(
func=client.put_item,
TableName="moto-test",
Item={"id": {"S": "foo"}, "item": {"S": large_item}},
Item={"id": {"S": "foo"}, "cont": {"S": large_item}},
)
assert_failure_due_to_item_size(
func=table.put_item, Item={"id": "bar", "item": large_item}
func=table.put_item, Item={"id": "bar", "cont": large_item}
)
assert_failure_due_to_item_size(
assert_failure_due_to_item_size_to_update(
func=client.update_item,
TableName="moto-test",
Key={"id": {"S": "foo2"}},
UpdateExpression="set item=:Item",
UpdateExpression="set cont=:Item",
ExpressionAttributeValues={":Item": {"S": large_item}},
)
# Assert op fails when updating a nested item
assert_failure_due_to_item_size(
func=table.put_item, Item={"id": "bar", "itemlist": [{"item": large_item}]}
func=table.put_item, Item={"id": "bar", "itemlist": [{"cont": large_item}]}
)
assert_failure_due_to_item_size(
func=client.put_item,
@ -3394,6 +3417,15 @@ def assert_failure_due_to_item_size(func, **kwargs):
)
def assert_failure_due_to_item_size_to_update(func, **kwargs):
with assert_raises(ClientError) as ex:
func(**kwargs)
ex.exception.response["Error"]["Code"].should.equal("ValidationException")
ex.exception.response["Error"]["Message"].should.equal(
"Item size to update has exceeded the maximum allowed size"
)
@mock_dynamodb2
# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html#DDB-Query-request-KeyConditionExpression
def test_hash_key_cannot_use_begins_with_operations():

63
tests/test_dynamodb2/test_dynamodb_table_without_range_key.py
View File

@ -443,23 +443,40 @@ def test_update_item_nested_remove():
dict(returned_item).should.equal({"username": "steve", "Meta": {}})
@mock_dynamodb2_deprecated
@mock_dynamodb2
def test_update_item_double_nested_remove():
conn = boto.dynamodb2.connect_to_region("us-east-1")
table = Table.create("messages", schema=[HashKey("username")])
conn = boto3.client("dynamodb", region_name="us-east-1")
conn.create_table(
TableName="messages",
KeySchema=[{"AttributeName": "username", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "username", "AttributeType": "S"}],
BillingMode="PAY_PER_REQUEST",
)
data = {"username": "steve", "Meta": {"Name": {"First": "Steve", "Last": "Urkel"}}}
table.put_item(data=data)
item = {
"username": {"S": "steve"},
"Meta": {
"M": {"Name": {"M": {"First": {"S": "Steve"}, "Last": {"S": "Urkel"}}}}
},
}
conn.put_item(TableName="messages", Item=item)
key_map = {"username": {"S": "steve"}}
# Then remove the Meta.FullName field
conn.update_item("messages", key_map, update_expression="REMOVE Meta.Name.First")
returned_item = table.get_item(username="steve")
dict(returned_item).should.equal(
{"username": "steve", "Meta": {"Name": {"Last": "Urkel"}}}
conn.update_item(
TableName="messages",
Key=key_map,
UpdateExpression="REMOVE Meta.#N.#F",
ExpressionAttributeNames={"#N": "Name", "#F": "First"},
)
returned_item = conn.get_item(TableName="messages", Key=key_map)
expected_item = {
"username": {"S": "steve"},
"Meta": {"M": {"Name": {"M": {"Last": {"S": "Urkel"}}}}},
}
dict(returned_item["Item"]).should.equal(expected_item)
@mock_dynamodb2_deprecated
def test_update_item_set():
@ -471,7 +488,10 @@ def test_update_item_set():
key_map = {"username": {"S": "steve"}}
conn.update_item(
"messages", key_map, update_expression="SET foo=bar, blah=baz REMOVE SentBy"
"messages",
key_map,
update_expression="SET foo=:bar, blah=:baz REMOVE SentBy",
expression_attribute_values={":bar": {"S": "bar"}, ":baz": {"S": "baz"}},
)
returned_item = table.get_item(username="steve")
@ -616,8 +636,9 @@ def test_boto3_update_item_conditions_fail():
table.put_item(Item={"username": "johndoe", "foo": "baz"})
table.update_item.when.called_with(
Key={"username": "johndoe"},
UpdateExpression="SET foo=bar",
UpdateExpression="SET foo=:bar",
Expected={"foo": {"Value": "bar"}},
ExpressionAttributeValues={":bar": "bar"},
).should.throw(botocore.client.ClientError)
@ -627,8 +648,9 @@ def test_boto3_update_item_conditions_fail_because_expect_not_exists():
table.put_item(Item={"username": "johndoe", "foo": "baz"})
table.update_item.when.called_with(
Key={"username": "johndoe"},
UpdateExpression="SET foo=bar",
UpdateExpression="SET foo=:bar",
Expected={"foo": {"Exists": False}},
ExpressionAttributeValues={":bar": "bar"},
).should.throw(botocore.client.ClientError)
@ -638,8 +660,9 @@ def test_boto3_update_item_conditions_fail_because_expect_not_exists_by_compare_
table.put_item(Item={"username": "johndoe", "foo": "baz"})
table.update_item.when.called_with(
Key={"username": "johndoe"},
UpdateExpression="SET foo=bar",
UpdateExpression="SET foo=:bar",
Expected={"foo": {"ComparisonOperator": "NULL"}},
ExpressionAttributeValues={":bar": "bar"},
).should.throw(botocore.client.ClientError)
@ -649,8 +672,9 @@ def test_boto3_update_item_conditions_pass():
table.put_item(Item={"username": "johndoe", "foo": "bar"})
table.update_item(
Key={"username": "johndoe"},
UpdateExpression="SET foo=baz",
UpdateExpression="SET foo=:baz",
Expected={"foo": {"Value": "bar"}},
ExpressionAttributeValues={":baz": "baz"},
)
returned_item = table.get_item(Key={"username": "johndoe"})
assert dict(returned_item)["Item"]["foo"].should.equal("baz")
@ -662,8 +686,9 @@ def test_boto3_update_item_conditions_pass_because_expect_not_exists():
table.put_item(Item={"username": "johndoe", "foo": "bar"})
table.update_item(
Key={"username": "johndoe"},
UpdateExpression="SET foo=baz",
UpdateExpression="SET foo=:baz",
Expected={"whatever": {"Exists": False}},
ExpressionAttributeValues={":baz": "baz"},
)
returned_item = table.get_item(Key={"username": "johndoe"})
assert dict(returned_item)["Item"]["foo"].should.equal("baz")
@ -675,8 +700,9 @@ def test_boto3_update_item_conditions_pass_because_expect_not_exists_by_compare_
table.put_item(Item={"username": "johndoe", "foo": "bar"})
table.update_item(
Key={"username": "johndoe"},
UpdateExpression="SET foo=baz",
UpdateExpression="SET foo=:baz",
Expected={"whatever": {"ComparisonOperator": "NULL"}},
ExpressionAttributeValues={":baz": "baz"},
)
returned_item = table.get_item(Key={"username": "johndoe"})
assert dict(returned_item)["Item"]["foo"].should.equal("baz")
@ -688,8 +714,9 @@ def test_boto3_update_item_conditions_pass_because_expect_exists_by_compare_to_n
table.put_item(Item={"username": "johndoe", "foo": "bar"})
table.update_item(
Key={"username": "johndoe"},
UpdateExpression="SET foo=baz",
UpdateExpression="SET foo=:baz",
Expected={"foo": {"ComparisonOperator": "NOT_NULL"}},
ExpressionAttributeValues={":baz": "baz"},
)
returned_item = table.get_item(Key={"username": "johndoe"})
assert dict(returned_item)["Item"]["foo"].should.equal("baz")