Improve DDB expressions support4: Execution using AST
Part of structured approach for UpdateExpressions: 1) Expression gets parsed into a tokenlist (tokenized) 2) Tokenlist get transformed to expression tree (AST) 3) The AST gets validated (full semantic correctness) 4) AST gets processed to perform the update -> this commit This commit uses the AST to execute the UpdateExpression. All the existing tests pass. The only tests that have been updated are in test_dynamodb_table_with_range_key.py because they wrongly allow adding a set to a path that doesn't exist. This has been alligend to correspond to the behavior of AWS DynamoDB. This commit will resolve https://github.com/spulec/moto/issues/2806 Multiple tests have been implemented that verify this.
This commit is contained in:
pvbouwel
parent
6a41573eb8
commit
ec731ac901
@ -39,6 +39,17 @@ class AttributeDoesNotExist(MockValidationException):
|
||||
super(AttributeDoesNotExist, self).__init__(self.attr_does_not_exist_msg)
|
||||
|
||||
|
||||
class ProvidedKeyDoesNotExist(MockValidationException):
|
||||
provided_key_does_not_exist_msg = (
|
||||
"The provided key element does not match the schema"
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
super(ProvidedKeyDoesNotExist, self).__init__(
|
||||
self.provided_key_does_not_exist_msg
|
||||
)
|
||||
|
||||
|
||||
class ExpressionAttributeNameNotDefined(InvalidUpdateExpression):
|
||||
name_not_defined_msg = "An expression attribute name used in the document path is not defined; attribute name: {n}"
|
||||
|
||||
@ -131,3 +142,10 @@ class IncorrectOperandType(InvalidUpdateExpression):
|
||||
super(IncorrectOperandType, self).__init__(
|
||||
self.inv_operand_msg.format(f=operator_or_function, t=operand_type)
|
||||
)
|
||||
|
||||
|
||||
class IncorrectDataType(MockValidationException):
|
||||
inc_data_type_msg = "An operand in the update expression has an incorrect data type"
|
||||
|
||||
def __init__(self):
|
||||
super(IncorrectDataType, self).__init__(self.inc_data_type_msg)
|
||||
|
||||
@ -8,7 +8,6 @@ import re
|
||||
import uuid
|
||||
|
||||
from boto3 import Session
|
||||
from botocore.exceptions import ParamValidationError
|
||||
from moto.compat import OrderedDict
|
||||
from moto.core import BaseBackend, BaseModel
|
||||
from moto.core.utils import unix_time
|
||||
@ -20,8 +19,9 @@ from moto.dynamodb2.exceptions import (
|
||||
ItemSizeTooLarge,
|
||||
ItemSizeToUpdateTooLarge,
|
||||
)
|
||||
from moto.dynamodb2.models.utilities import bytesize, attribute_is_list
|
||||
from moto.dynamodb2.models.utilities import bytesize
|
||||
from moto.dynamodb2.models.dynamo_type import DynamoType
|
||||
from moto.dynamodb2.parsing.executors import UpdateExpressionExecutor
|
||||
from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
|
||||
from moto.dynamodb2.parsing.validators import UpdateExpressionValidator
|
||||
|
||||
@ -71,6 +71,17 @@ class Item(BaseModel):
|
||||
for key, value in attrs.items():
|
||||
self.attrs[key] = DynamoType(value)
|
||||
|
||||
def __eq__(self, other):
|
||||
return all(
|
||||
[
|
||||
self.hash_key == other.hash_key,
|
||||
self.hash_key_type == other.hash_key_type,
|
||||
self.range_key == other.range_key,
|
||||
self.range_key_type == other.range_key_type,
|
||||
self.attrs == other.attrs,
|
||||
]
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return "Item: {0}".format(self.to_json())
|
||||
|
||||
@ -94,192 +105,6 @@ class Item(BaseModel):
|
||||
included = self.attrs
|
||||
return {"Item": included}
|
||||
|
||||
def update(
|
||||
self, update_expression, expression_attribute_names, expression_attribute_values
|
||||
):
|
||||
# Update subexpressions are identifiable by the operator keyword, so split on that and
|
||||
# get rid of the empty leading string.
|
||||
parts = [
|
||||
p
|
||||
for p in re.split(
|
||||
r"\b(SET|REMOVE|ADD|DELETE)\b", update_expression, flags=re.I
|
||||
)
|
||||
if p
|
||||
]
|
||||
# make sure that we correctly found only operator/value pairs
|
||||
assert (
|
||||
len(parts) % 2 == 0
|
||||
), "Mismatched operators and values in update expression: '{}'".format(
|
||||
update_expression
|
||||
)
|
||||
for action, valstr in zip(parts[:-1:2], parts[1::2]):
|
||||
action = action.upper()
|
||||
|
||||
# "Should" retain arguments in side (...)
|
||||
values = re.split(r",(?![^(]*\))", valstr)
|
||||
for value in values:
|
||||
# A Real value
|
||||
value = value.lstrip(":").rstrip(",").strip()
|
||||
for k, v in expression_attribute_names.items():
|
||||
value = re.sub(r"{0}\b".format(k), v, value)
|
||||
|
||||
if action == "REMOVE":
|
||||
key = value
|
||||
attr, list_index = attribute_is_list(key.split(".")[0])
|
||||
if "." not in key:
|
||||
if list_index:
|
||||
new_list = DynamoType(self.attrs[attr])
|
||||
new_list.delete(None, list_index)
|
||||
self.attrs[attr] = new_list
|
||||
else:
|
||||
self.attrs.pop(value, None)
|
||||
else:
|
||||
# Handle nested dict updates
|
||||
self.attrs[attr].delete(".".join(key.split(".")[1:]))
|
||||
elif action == "SET":
|
||||
key, value = value.split("=", 1)
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
|
||||
# check whether key is a list
|
||||
attr, list_index = attribute_is_list(key.split(".")[0])
|
||||
# If value not exists, changes value to a default if needed, else its the same as it was
|
||||
value = self._get_default(value)
|
||||
# If operation == list_append, get the original value and append it
|
||||
value = self._get_appended_list(value, expression_attribute_values)
|
||||
|
||||
if type(value) != DynamoType:
|
||||
if value in expression_attribute_values:
|
||||
dyn_value = DynamoType(expression_attribute_values[value])
|
||||
else:
|
||||
dyn_value = DynamoType({"S": value})
|
||||
else:
|
||||
dyn_value = value
|
||||
|
||||
if "." in key and attr not in self.attrs:
|
||||
raise ValueError # Setting nested attr not allowed if first attr does not exist yet
|
||||
elif attr not in self.attrs:
|
||||
try:
|
||||
self.attrs[attr] = dyn_value # set new top-level attribute
|
||||
except ItemSizeTooLarge:
|
||||
raise ItemSizeToUpdateTooLarge()
|
||||
else:
|
||||
self.attrs[attr].set(
|
||||
".".join(key.split(".")[1:]), dyn_value, list_index
|
||||
) # set value recursively
|
||||
|
||||
elif action == "ADD":
|
||||
key, value = value.split(" ", 1)
|
||||
key = key.strip()
|
||||
value_str = value.strip()
|
||||
if value_str in expression_attribute_values:
|
||||
dyn_value = DynamoType(expression_attribute_values[value])
|
||||
else:
|
||||
raise TypeError
|
||||
|
||||
# Handle adding numbers - value gets added to existing value,
|
||||
# or added to 0 if it doesn't exist yet
|
||||
if dyn_value.is_number():
|
||||
existing = self.attrs.get(key, DynamoType({"N": "0"}))
|
||||
if not existing.same_type(dyn_value):
|
||||
raise TypeError()
|
||||
self.attrs[key] = DynamoType(
|
||||
{
|
||||
"N": str(
|
||||
decimal.Decimal(existing.value)
|
||||
+ decimal.Decimal(dyn_value.value)
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
# Handle adding sets - value is added to the set, or set is
|
||||
# created with only this value if it doesn't exist yet
|
||||
# New value must be of same set type as previous value
|
||||
elif dyn_value.is_set():
|
||||
key_head = key.split(".")[0]
|
||||
key_tail = ".".join(key.split(".")[1:])
|
||||
if key_head not in self.attrs:
|
||||
self.attrs[key_head] = DynamoType({dyn_value.type: {}})
|
||||
existing = self.attrs.get(key_head)
|
||||
existing = existing.get(key_tail)
|
||||
if existing.value and not existing.same_type(dyn_value):
|
||||
raise TypeError()
|
||||
new_set = set(existing.value or []).union(dyn_value.value)
|
||||
existing.set(
|
||||
key=None,
|
||||
new_value=DynamoType({dyn_value.type: list(new_set)}),
|
||||
)
|
||||
else: # Number and Sets are the only supported types for ADD
|
||||
raise TypeError
|
||||
|
||||
elif action == "DELETE":
|
||||
key, value = value.split(" ", 1)
|
||||
key = key.strip()
|
||||
value_str = value.strip()
|
||||
if value_str in expression_attribute_values:
|
||||
dyn_value = DynamoType(expression_attribute_values[value])
|
||||
else:
|
||||
raise TypeError
|
||||
|
||||
if not dyn_value.is_set():
|
||||
raise TypeError
|
||||
key_head = key.split(".")[0]
|
||||
key_tail = ".".join(key.split(".")[1:])
|
||||
existing = self.attrs.get(key_head)
|
||||
existing = existing.get(key_tail)
|
||||
if existing:
|
||||
if not existing.same_type(dyn_value):
|
||||
raise TypeError
|
||||
new_set = set(existing.value).difference(dyn_value.value)
|
||||
existing.set(
|
||||
key=None,
|
||||
new_value=DynamoType({existing.type: list(new_set)}),
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"{} update action not yet supported".format(action)
|
||||
)
|
||||
|
||||
def _get_appended_list(self, value, expression_attribute_values):
|
||||
if type(value) != DynamoType:
|
||||
list_append_re = re.match("list_append\\((.+),(.+)\\)", value)
|
||||
if list_append_re:
|
||||
new_value = expression_attribute_values[list_append_re.group(2).strip()]
|
||||
old_list_key = list_append_re.group(1)
|
||||
# old_key could be a function itself (if_not_exists)
|
||||
if old_list_key.startswith("if_not_exists"):
|
||||
old_list = self._get_default(old_list_key)
|
||||
if not isinstance(old_list, DynamoType):
|
||||
old_list = DynamoType(expression_attribute_values[old_list])
|
||||
else:
|
||||
old_list = self.attrs[old_list_key.split(".")[0]]
|
||||
if "." in old_list_key:
|
||||
# Value is nested inside a map - find the appropriate child attr
|
||||
old_list = old_list.child_attr(
|
||||
".".join(old_list_key.split(".")[1:])
|
||||
)
|
||||
if not old_list.is_list():
|
||||
raise ParamValidationError
|
||||
old_list.value.extend([DynamoType(v) for v in new_value["L"]])
|
||||
value = old_list
|
||||
return value
|
||||
|
||||
def _get_default(self, value):
|
||||
if value.startswith("if_not_exists"):
|
||||
# Function signature
|
||||
match = re.match(
|
||||
r".*if_not_exists\s*\((?P<path>.+),\s*(?P<default>.+)\).*", value
|
||||
)
|
||||
if not match:
|
||||
raise TypeError
|
||||
|
||||
path, value = match.groups()
|
||||
|
||||
# If it already exists, get its value so we dont overwrite it
|
||||
if path in self.attrs:
|
||||
value = self.attrs[path]
|
||||
return value
|
||||
|
||||
def update_with_attribute_updates(self, attribute_updates):
|
||||
for attribute_name, update_action in attribute_updates.items():
|
||||
action = update_action["Action"]
|
||||
@ -1266,17 +1091,18 @@ class DynamoDBBackend(BaseBackend):
|
||||
item = table.get_item(hash_value, range_value)
|
||||
|
||||
if update_expression:
|
||||
UpdateExpressionValidator(
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=expression_attribute_names,
|
||||
expression_attribute_values=expression_attribute_values,
|
||||
item=item,
|
||||
).validate()
|
||||
item.update(
|
||||
update_expression,
|
||||
expression_attribute_names,
|
||||
expression_attribute_values,
|
||||
)
|
||||
try:
|
||||
UpdateExpressionExecutor(
|
||||
validated_ast, item, expression_attribute_names
|
||||
).execute()
|
||||
except ItemSizeTooLarge:
|
||||
raise ItemSizeToUpdateTooLarge()
|
||||
else:
|
||||
item.update_with_attribute_updates(attribute_updates)
|
||||
if table.stream_shard is not None:
|
||||
|
||||
@ -1,10 +1,53 @@
|
||||
import six
|
||||
|
||||
from moto.dynamodb2.comparisons import get_comparison_func
|
||||
from moto.dynamodb2.exceptions import InvalidUpdateExpression
|
||||
from moto.dynamodb2.exceptions import InvalidUpdateExpression, IncorrectDataType
|
||||
from moto.dynamodb2.models.utilities import attribute_is_list, bytesize
|
||||
|
||||
|
||||
class DDBType(object):
|
||||
"""
|
||||
Official documentation at https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_AttributeValue.html
|
||||
"""
|
||||
|
||||
BINARY_SET = "BS"
|
||||
NUMBER_SET = "NS"
|
||||
STRING_SET = "SS"
|
||||
STRING = "S"
|
||||
NUMBER = "N"
|
||||
MAP = "M"
|
||||
LIST = "L"
|
||||
BOOLEAN = "BOOL"
|
||||
BINARY = "B"
|
||||
NULL = "NULL"
|
||||
|
||||
|
||||
class DDBTypeConversion(object):
|
||||
_human_type_mapping = {
|
||||
val: key.replace("_", " ")
|
||||
for key, val in DDBType.__dict__.items()
|
||||
if key.upper() == key
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def get_human_type(cls, abbreviated_type):
|
||||
"""
|
||||
Args:
|
||||
abbreviated_type(str): An attribute of DDBType
|
||||
|
||||
Returns:
|
||||
str: The human readable form of the DDBType.
|
||||
"""
|
||||
try:
|
||||
human_type_str = cls._human_type_mapping[abbreviated_type]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
"Invalid abbreviated_type {at}".format(at=abbreviated_type)
|
||||
)
|
||||
|
||||
return human_type_str
|
||||
|
||||
|
||||
class DynamoType(object):
|
||||
"""
|
||||
http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html#DataModelDataTypes
|
||||
@ -50,13 +93,22 @@ class DynamoType(object):
|
||||
self.value = new_value.value
|
||||
else:
|
||||
if attr not in self.value: # nonexistingattribute
|
||||
type_of_new_attr = "M" if "." in key else new_value.type
|
||||
type_of_new_attr = DDBType.MAP if "." in key else new_value.type
|
||||
self.value[attr] = DynamoType({type_of_new_attr: {}})
|
||||
# {'M': {'foo': DynamoType}} ==> DynamoType.set(new_value)
|
||||
self.value[attr].set(
|
||||
".".join(key.split(".")[1:]), new_value, list_index
|
||||
)
|
||||
|
||||
def __contains__(self, item):
|
||||
if self.type == DDBType.STRING:
|
||||
return False
|
||||
try:
|
||||
self.__getitem__(item)
|
||||
return True
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
def delete(self, key, index=None):
|
||||
if index:
|
||||
if not key:
|
||||
@ -126,27 +178,35 @@ class DynamoType(object):
|
||||
def __add__(self, other):
|
||||
if self.type != other.type:
|
||||
raise TypeError("Different types of operandi is not allowed.")
|
||||
if self.type == "N":
|
||||
return DynamoType({"N": "{v}".format(v=int(self.value) + int(other.value))})
|
||||
if self.is_number():
|
||||
self_value = float(self.value) if "." in self.value else int(self.value)
|
||||
other_value = float(other.value) if "." in other.value else int(other.value)
|
||||
return DynamoType(
|
||||
{DDBType.NUMBER: "{v}".format(v=self_value + other_value)}
|
||||
)
|
||||
else:
|
||||
raise TypeError("Sum only supported for Numbers.")
|
||||
raise IncorrectDataType()
|
||||
|
||||
def __sub__(self, other):
|
||||
if self.type != other.type:
|
||||
raise TypeError("Different types of operandi is not allowed.")
|
||||
if self.type == "N":
|
||||
return DynamoType({"N": "{v}".format(v=int(self.value) - int(other.value))})
|
||||
if self.type == DDBType.NUMBER:
|
||||
self_value = float(self.value) if "." in self.value else int(self.value)
|
||||
other_value = float(other.value) if "." in other.value else int(other.value)
|
||||
return DynamoType(
|
||||
{DDBType.NUMBER: "{v}".format(v=self_value - other_value)}
|
||||
)
|
||||
else:
|
||||
raise TypeError("Sum only supported for Numbers.")
|
||||
|
||||
def __getitem__(self, item):
|
||||
if isinstance(item, six.string_types):
|
||||
# If our DynamoType is a map it should be subscriptable with a key
|
||||
if self.type == "M":
|
||||
if self.type == DDBType.MAP:
|
||||
return self.value[item]
|
||||
elif isinstance(item, int):
|
||||
# If our DynamoType is a list is should be subscriptable with an index
|
||||
if self.type == "L":
|
||||
if self.type == DDBType.LIST:
|
||||
return self.value[item]
|
||||
raise TypeError(
|
||||
"This DynamoType {dt} is not subscriptable by a {it}".format(
|
||||
@ -154,6 +214,20 @@ class DynamoType(object):
|
||||
)
|
||||
)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
if isinstance(key, int):
|
||||
if self.is_list():
|
||||
if key >= len(self.value):
|
||||
# DynamoDB doesn't care you are out of box just add it to the end.
|
||||
self.value.append(value)
|
||||
else:
|
||||
self.value[key] = value
|
||||
elif isinstance(key, six.string_types):
|
||||
if self.is_map():
|
||||
self.value[key] = value
|
||||
else:
|
||||
raise NotImplementedError("No set_item for {t}".format(t=type(key)))
|
||||
|
||||
@property
|
||||
def cast_value(self):
|
||||
if self.is_number():
|
||||
@ -222,16 +296,22 @@ class DynamoType(object):
|
||||
return comparison_func(self.cast_value, *range_values)
|
||||
|
||||
def is_number(self):
|
||||
return self.type == "N"
|
||||
return self.type == DDBType.NUMBER
|
||||
|
||||
def is_set(self):
|
||||
return self.type == "SS" or self.type == "NS" or self.type == "BS"
|
||||
return self.type in (DDBType.STRING_SET, DDBType.NUMBER_SET, DDBType.BINARY_SET)
|
||||
|
||||
def is_list(self):
|
||||
return self.type == "L"
|
||||
return self.type == DDBType.LIST
|
||||
|
||||
def is_map(self):
|
||||
return self.type == "M"
|
||||
return self.type == DDBType.MAP
|
||||
|
||||
def same_type(self, other):
|
||||
return self.type == other.type
|
||||
|
||||
def pop(self, key, *args, **kwargs):
|
||||
if self.is_map() or self.is_list():
|
||||
self.value.pop(key, *args, **kwargs)
|
||||
else:
|
||||
raise TypeError("pop not supported for DynamoType {t}".format(t=self.type))
|
||||
|
||||
262
moto/dynamodb2/parsing/executors.py
Normal file
262
moto/dynamodb2/parsing/executors.py
Normal file
@ -0,0 +1,262 @@
|
||||
from abc import abstractmethod
|
||||
|
||||
from moto.dynamodb2.exceptions import IncorrectOperandType, IncorrectDataType
|
||||
from moto.dynamodb2.models import DynamoType
|
||||
from moto.dynamodb2.models.dynamo_type import DDBTypeConversion, DDBType
|
||||
from moto.dynamodb2.parsing.ast_nodes import (
|
||||
UpdateExpressionSetAction,
|
||||
UpdateExpressionDeleteAction,
|
||||
UpdateExpressionRemoveAction,
|
||||
UpdateExpressionAddAction,
|
||||
UpdateExpressionPath,
|
||||
DDBTypedValue,
|
||||
ExpressionAttribute,
|
||||
ExpressionSelector,
|
||||
ExpressionAttributeName,
|
||||
)
|
||||
from moto.dynamodb2.parsing.validators import ExpressionPathResolver
|
||||
|
||||
|
||||
class NodeExecutor(object):
|
||||
def __init__(self, ast_node, expression_attribute_names):
|
||||
self.node = ast_node
|
||||
self.expression_attribute_names = expression_attribute_names
|
||||
|
||||
@abstractmethod
|
||||
def execute(self, item):
|
||||
pass
|
||||
|
||||
def get_item_part_for_path_nodes(self, item, path_nodes):
|
||||
"""
|
||||
For a list of path nodes travers the item by following the path_nodes
|
||||
Args:
|
||||
item(Item):
|
||||
path_nodes(list):
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
if len(path_nodes) == 0:
|
||||
return item.attrs
|
||||
else:
|
||||
return ExpressionPathResolver(
|
||||
self.expression_attribute_names
|
||||
).resolve_expression_path_nodes_to_dynamo_type(item, path_nodes)
|
||||
|
||||
def get_item_before_end_of_path(self, item):
|
||||
"""
|
||||
Get the part ot the item where the item will perform the action. For most actions this should be the parent. As
|
||||
that element will need to be modified by the action.
|
||||
Args:
|
||||
item(Item):
|
||||
|
||||
Returns:
|
||||
DynamoType or dict: The path to be set
|
||||
"""
|
||||
return self.get_item_part_for_path_nodes(
|
||||
item, self.get_path_expression_nodes()[:-1]
|
||||
)
|
||||
|
||||
def get_item_at_end_of_path(self, item):
|
||||
"""
|
||||
For a DELETE the path points at the stringset so we need to evaluate the full path.
|
||||
Args:
|
||||
item(Item):
|
||||
|
||||
Returns:
|
||||
DynamoType or dict: The path to be set
|
||||
"""
|
||||
return self.get_item_part_for_path_nodes(item, self.get_path_expression_nodes())
|
||||
|
||||
# Get the part ot the item where the item will perform the action. For most actions this should be the parent. As
|
||||
# that element will need to be modified by the action.
|
||||
get_item_part_in_which_to_perform_action = get_item_before_end_of_path
|
||||
|
||||
def get_path_expression_nodes(self):
|
||||
update_expression_path = self.node.children[0]
|
||||
assert isinstance(update_expression_path, UpdateExpressionPath)
|
||||
return update_expression_path.children
|
||||
|
||||
def get_element_to_action(self):
|
||||
return self.get_path_expression_nodes()[-1]
|
||||
|
||||
def get_action_value(self):
|
||||
"""
|
||||
|
||||
Returns:
|
||||
DynamoType: The value to be set
|
||||
"""
|
||||
ddb_typed_value = self.node.children[1]
|
||||
assert isinstance(ddb_typed_value, DDBTypedValue)
|
||||
dynamo_type_value = ddb_typed_value.children[0]
|
||||
assert isinstance(dynamo_type_value, DynamoType)
|
||||
return dynamo_type_value
|
||||
|
||||
|
||||
class SetExecutor(NodeExecutor):
|
||||
def execute(self, item):
|
||||
self.set(
|
||||
item_part_to_modify_with_set=self.get_item_part_in_which_to_perform_action(
|
||||
item
|
||||
),
|
||||
element_to_set=self.get_element_to_action(),
|
||||
value_to_set=self.get_action_value(),
|
||||
expression_attribute_names=self.expression_attribute_names,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def set(
|
||||
cls,
|
||||
item_part_to_modify_with_set,
|
||||
element_to_set,
|
||||
value_to_set,
|
||||
expression_attribute_names,
|
||||
):
|
||||
if isinstance(element_to_set, ExpressionAttribute):
|
||||
attribute_name = element_to_set.get_attribute_name()
|
||||
item_part_to_modify_with_set[attribute_name] = value_to_set
|
||||
elif isinstance(element_to_set, ExpressionSelector):
|
||||
index = element_to_set.get_index()
|
||||
item_part_to_modify_with_set[index] = value_to_set
|
||||
elif isinstance(element_to_set, ExpressionAttributeName):
|
||||
attribute_name = expression_attribute_names[
|
||||
element_to_set.get_attribute_name_placeholder()
|
||||
]
|
||||
item_part_to_modify_with_set[attribute_name] = value_to_set
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"Moto does not support setting {t} yet".format(t=type(element_to_set))
|
||||
)
|
||||
|
||||
|
||||
class DeleteExecutor(NodeExecutor):
|
||||
operator = "operator: DELETE"
|
||||
|
||||
def execute(self, item):
|
||||
string_set_to_remove = self.get_action_value()
|
||||
assert isinstance(string_set_to_remove, DynamoType)
|
||||
if not string_set_to_remove.is_set():
|
||||
raise IncorrectOperandType(
|
||||
self.operator,
|
||||
DDBTypeConversion.get_human_type(string_set_to_remove.type),
|
||||
)
|
||||
|
||||
string_set = self.get_item_at_end_of_path(item)
|
||||
assert isinstance(string_set, DynamoType)
|
||||
if string_set.type != string_set_to_remove.type:
|
||||
raise IncorrectDataType()
|
||||
# String set is currently implemented as a list
|
||||
string_set_list = string_set.value
|
||||
|
||||
stringset_to_remove_list = string_set_to_remove.value
|
||||
|
||||
for value in stringset_to_remove_list:
|
||||
try:
|
||||
string_set_list.remove(value)
|
||||
except (KeyError, ValueError):
|
||||
# DynamoDB does not mind if value is not present
|
||||
pass
|
||||
|
||||
|
||||
class RemoveExecutor(NodeExecutor):
|
||||
def execute(self, item):
|
||||
element_to_remove = self.get_element_to_action()
|
||||
if isinstance(element_to_remove, ExpressionAttribute):
|
||||
attribute_name = element_to_remove.get_attribute_name()
|
||||
self.get_item_part_in_which_to_perform_action(item).pop(
|
||||
attribute_name, None
|
||||
)
|
||||
elif isinstance(element_to_remove, ExpressionAttributeName):
|
||||
attribute_name = self.expression_attribute_names[
|
||||
element_to_remove.get_attribute_name_placeholder()
|
||||
]
|
||||
self.get_item_part_in_which_to_perform_action(item).pop(
|
||||
attribute_name, None
|
||||
)
|
||||
elif isinstance(element_to_remove, ExpressionSelector):
|
||||
index = element_to_remove.get_index()
|
||||
try:
|
||||
self.get_item_part_in_which_to_perform_action(item).pop(index)
|
||||
except IndexError:
|
||||
# DynamoDB does not care that index is out of bounds, it will just do nothing.
|
||||
pass
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"Moto does not support setting {t} yet".format(
|
||||
t=type(element_to_remove)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class AddExecutor(NodeExecutor):
|
||||
def execute(self, item):
|
||||
value_to_add = self.get_action_value()
|
||||
if isinstance(value_to_add, DynamoType):
|
||||
if value_to_add.is_set():
|
||||
current_string_set = self.get_item_at_end_of_path(item)
|
||||
assert isinstance(current_string_set, DynamoType)
|
||||
if not current_string_set.type == value_to_add.type:
|
||||
raise IncorrectDataType()
|
||||
# Sets are implemented as list
|
||||
for value in value_to_add.value:
|
||||
if value in current_string_set.value:
|
||||
continue
|
||||
else:
|
||||
current_string_set.value.append(value)
|
||||
elif value_to_add.type == DDBType.NUMBER:
|
||||
existing_value = self.get_item_at_end_of_path(item)
|
||||
assert isinstance(existing_value, DynamoType)
|
||||
if not existing_value.type == DDBType.NUMBER:
|
||||
raise IncorrectDataType()
|
||||
new_value = existing_value + value_to_add
|
||||
SetExecutor.set(
|
||||
item_part_to_modify_with_set=self.get_item_before_end_of_path(item),
|
||||
element_to_set=self.get_element_to_action(),
|
||||
value_to_set=new_value,
|
||||
expression_attribute_names=self.expression_attribute_names,
|
||||
)
|
||||
else:
|
||||
raise IncorrectDataType()
|
||||
|
||||
|
||||
class UpdateExpressionExecutor(object):
|
||||
execution_map = {
|
||||
UpdateExpressionSetAction: SetExecutor,
|
||||
UpdateExpressionAddAction: AddExecutor,
|
||||
UpdateExpressionRemoveAction: RemoveExecutor,
|
||||
UpdateExpressionDeleteAction: DeleteExecutor,
|
||||
}
|
||||
|
||||
def __init__(self, update_ast, item, expression_attribute_names):
|
||||
self.update_ast = update_ast
|
||||
self.item = item
|
||||
self.expression_attribute_names = expression_attribute_names
|
||||
|
||||
def execute(self, node=None):
|
||||
"""
|
||||
As explained in moto.dynamodb2.parsing.expressions.NestableExpressionParserMixin._create_node the order of nodes
|
||||
in the AST can be translated of the order of statements in the expression. As such we can start at the root node
|
||||
and process the nodes 1-by-1. If no specific execution for the node type is defined we can execute the children
|
||||
in order since it will be a container node that is expandable and left child will be first in the statement.
|
||||
|
||||
Args:
|
||||
node(Node):
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
if node is None:
|
||||
node = self.update_ast
|
||||
|
||||
node_executor = self.get_specific_execution(node)
|
||||
if node_executor is None:
|
||||
for node in node.children:
|
||||
self.execute(node)
|
||||
else:
|
||||
node_executor(node, self.expression_attribute_names).execute(self.item)
|
||||
|
||||
def get_specific_execution(self, node):
|
||||
for node_class in self.execution_map:
|
||||
if isinstance(node, node_class):
|
||||
return self.execution_map[node_class]
|
||||
return None
|
||||
@ -11,6 +11,7 @@ from moto.dynamodb2.exceptions import (
|
||||
ExpressionAttributeNameNotDefined,
|
||||
IncorrectOperandType,
|
||||
InvalidUpdateExpressionInvalidDocumentPath,
|
||||
ProvidedKeyDoesNotExist,
|
||||
)
|
||||
from moto.dynamodb2.models import DynamoType
|
||||
from moto.dynamodb2.parsing.ast_nodes import (
|
||||
@ -56,6 +57,76 @@ class ExpressionAttributeValueProcessor(DepthFirstTraverser):
|
||||
return DDBTypedValue(DynamoType(target))
|
||||
|
||||
|
||||
class ExpressionPathResolver(object):
|
||||
def __init__(self, expression_attribute_names):
|
||||
self.expression_attribute_names = expression_attribute_names
|
||||
|
||||
@classmethod
|
||||
def raise_exception_if_keyword(cls, attribute):
|
||||
if attribute.upper() in ReservedKeywords.get_reserved_keywords():
|
||||
raise AttributeIsReservedKeyword(attribute)
|
||||
|
||||
def resolve_expression_path(self, item, update_expression_path):
|
||||
assert isinstance(update_expression_path, UpdateExpressionPath)
|
||||
return self.resolve_expression_path_nodes(item, update_expression_path.children)
|
||||
|
||||
def resolve_expression_path_nodes(self, item, update_expression_path_nodes):
|
||||
target = item.attrs
|
||||
|
||||
for child in update_expression_path_nodes:
|
||||
# First replace placeholder with attribute_name
|
||||
attr_name = None
|
||||
if isinstance(child, ExpressionAttributeName):
|
||||
attr_placeholder = child.get_attribute_name_placeholder()
|
||||
try:
|
||||
attr_name = self.expression_attribute_names[attr_placeholder]
|
||||
except KeyError:
|
||||
raise ExpressionAttributeNameNotDefined(attr_placeholder)
|
||||
elif isinstance(child, ExpressionAttribute):
|
||||
attr_name = child.get_attribute_name()
|
||||
self.raise_exception_if_keyword(attr_name)
|
||||
if attr_name is not None:
|
||||
# Resolv attribute_name
|
||||
try:
|
||||
target = target[attr_name]
|
||||
except (KeyError, TypeError):
|
||||
if child == update_expression_path_nodes[-1]:
|
||||
return NoneExistingPath(creatable=True)
|
||||
return NoneExistingPath()
|
||||
else:
|
||||
if isinstance(child, ExpressionPathDescender):
|
||||
continue
|
||||
elif isinstance(child, ExpressionSelector):
|
||||
index = child.get_index()
|
||||
if target.is_list():
|
||||
try:
|
||||
target = target[index]
|
||||
except IndexError:
|
||||
# When a list goes out of bounds when assigning that is no problem when at the assignment
|
||||
# side. It will just append to the list.
|
||||
if child == update_expression_path_nodes[-1]:
|
||||
return NoneExistingPath(creatable=True)
|
||||
return NoneExistingPath()
|
||||
else:
|
||||
raise InvalidUpdateExpressionInvalidDocumentPath
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"Path resolution for {t}".format(t=type(child))
|
||||
)
|
||||
if not isinstance(target, DynamoType):
|
||||
print(target)
|
||||
return DDBTypedValue(target)
|
||||
|
||||
def resolve_expression_path_nodes_to_dynamo_type(
|
||||
self, item, update_expression_path_nodes
|
||||
):
|
||||
node = self.resolve_expression_path_nodes(item, update_expression_path_nodes)
|
||||
if isinstance(node, NoneExistingPath):
|
||||
raise ProvidedKeyDoesNotExist()
|
||||
assert isinstance(node, DDBTypedValue)
|
||||
return node.get_value()
|
||||
|
||||
|
||||
class ExpressionAttributeResolvingProcessor(DepthFirstTraverser):
|
||||
def _processing_map(self):
|
||||
return {
|
||||
@ -107,55 +178,9 @@ class ExpressionAttributeResolvingProcessor(DepthFirstTraverser):
|
||||
return node
|
||||
|
||||
def resolve_expression_path(self, node):
|
||||
assert isinstance(node, UpdateExpressionPath)
|
||||
|
||||
target = deepcopy(self.item.attrs)
|
||||
for child in node.children:
|
||||
# First replace placeholder with attribute_name
|
||||
attr_name = None
|
||||
if isinstance(child, ExpressionAttributeName):
|
||||
attr_placeholder = child.get_attribute_name_placeholder()
|
||||
try:
|
||||
attr_name = self.expression_attribute_names[attr_placeholder]
|
||||
except KeyError:
|
||||
raise ExpressionAttributeNameNotDefined(attr_placeholder)
|
||||
elif isinstance(child, ExpressionAttribute):
|
||||
attr_name = child.get_attribute_name()
|
||||
self.raise_exception_if_keyword(attr_name)
|
||||
if attr_name is not None:
|
||||
# Resolv attribute_name
|
||||
try:
|
||||
target = target[attr_name]
|
||||
except (KeyError, TypeError):
|
||||
if child == node.children[-1]:
|
||||
return NoneExistingPath(creatable=True)
|
||||
return NoneExistingPath()
|
||||
else:
|
||||
if isinstance(child, ExpressionPathDescender):
|
||||
continue
|
||||
elif isinstance(child, ExpressionSelector):
|
||||
index = child.get_index()
|
||||
if target.is_list():
|
||||
try:
|
||||
target = target[index]
|
||||
except IndexError:
|
||||
# When a list goes out of bounds when assigning that is no problem when at the assignment
|
||||
# side. It will just append to the list.
|
||||
if child == node.children[-1]:
|
||||
return NoneExistingPath(creatable=True)
|
||||
return NoneExistingPath()
|
||||
else:
|
||||
raise InvalidUpdateExpressionInvalidDocumentPath
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"Path resolution for {t}".format(t=type(child))
|
||||
)
|
||||
return DDBTypedValue(DynamoType(target))
|
||||
|
||||
@classmethod
|
||||
def raise_exception_if_keyword(cls, attribute):
|
||||
if attribute.upper() in ReservedKeywords.get_reserved_keywords():
|
||||
raise AttributeIsReservedKeyword(attribute)
|
||||
return ExpressionPathResolver(
|
||||
self.expression_attribute_names
|
||||
).resolve_expression_path(self.item, node)
|
||||
|
||||
|
||||
class UpdateExpressionFunctionEvaluator(DepthFirstTraverser):
|
||||
@ -183,7 +208,9 @@ class UpdateExpressionFunctionEvaluator(DepthFirstTraverser):
|
||||
assert isinstance(result, (DDBTypedValue, NoneExistingPath))
|
||||
return result
|
||||
elif function_name == "list_append":
|
||||
first_arg = self.get_list_from_ddb_typed_value(first_arg, function_name)
|
||||
first_arg = deepcopy(
|
||||
self.get_list_from_ddb_typed_value(first_arg, function_name)
|
||||
)
|
||||
second_arg = self.get_list_from_ddb_typed_value(second_arg, function_name)
|
||||
for list_element in second_arg.value:
|
||||
first_arg.value.append(list_element)
|
||||
|
||||
@ -1,21 +1,17 @@
|
||||
from __future__ import unicode_literals, print_function
|
||||
|
||||
import re
|
||||
from decimal import Decimal
|
||||
|
||||
import six
|
||||
import boto
|
||||
import boto3
|
||||
from boto3.dynamodb.conditions import Attr, Key
|
||||
import re
|
||||
import requests
|
||||
import sure # noqa
|
||||
from moto import mock_dynamodb2, mock_dynamodb2_deprecated
|
||||
from moto.dynamodb2 import dynamodb_backend2, dynamodb_backends2
|
||||
from boto.exception import JSONResponseError
|
||||
from botocore.exceptions import ClientError, ParamValidationError
|
||||
from tests.helpers import requires_boto_gte
|
||||
import tests.backport_assert_raises
|
||||
|
||||
import moto.dynamodb2.comparisons
|
||||
import moto.dynamodb2.models
|
||||
@ -3221,6 +3217,25 @@ def test_remove_top_level_attribute():
|
||||
result.should.equal({"id": {"S": "foo"}})
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_remove_top_level_attribute_non_existent():
|
||||
"""
|
||||
Remove statements do not require attribute to exist they silently pass
|
||||
"""
|
||||
table_name = "test_remove"
|
||||
client = create_table_with_list(table_name)
|
||||
ddb_item = {"id": {"S": "foo"}, "item": {"S": "bar"}}
|
||||
client.put_item(TableName=table_name, Item=ddb_item)
|
||||
client.update_item(
|
||||
TableName=table_name,
|
||||
Key={"id": {"S": "foo"}},
|
||||
UpdateExpression="REMOVE non_existent_attribute",
|
||||
ExpressionAttributeNames={"#i": "item"},
|
||||
)
|
||||
result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"]
|
||||
result.should.equal(ddb_item)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_remove_list_index__remove_existing_index():
|
||||
table_name = "test_list_index_access"
|
||||
@ -4331,3 +4346,251 @@ def test_list_tables_exclusive_start_table_name_empty():
|
||||
resp = client.list_tables(Limit=1, ExclusiveStartTableName="whatever")
|
||||
|
||||
len(resp["TableNames"]).should.equal(0)
|
||||
|
||||
|
||||
def assert_correct_client_error(
|
||||
client_error, code, message_template, message_values=None, braces=None
|
||||
):
|
||||
"""
|
||||
Assert whether a client_error is as expected. Allow for a list of values to be passed into the message
|
||||
|
||||
Args:
|
||||
client_error(ClientError): The ClientError exception that was raised
|
||||
code(str): The code for the error (e.g. ValidationException)
|
||||
message_template(str): Error message template. if message_values is not None then this template has a {values}
|
||||
as placeholder. For example:
|
||||
'Value provided in ExpressionAttributeValues unused in expressions: keys: {values}'
|
||||
message_values(list of str|None): The values that are passed in the error message
|
||||
braces(list of str|None): List of length 2 with opening and closing brace for the values. By default it will be
|
||||
surrounded by curly brackets
|
||||
"""
|
||||
braces = braces or ["{", "}"]
|
||||
assert client_error.response["Error"]["Code"] == code
|
||||
if message_values is not None:
|
||||
values_string = "{open_brace}(?P<values>.*){close_brace}".format(
|
||||
open_brace=braces[0], close_brace=braces[1]
|
||||
)
|
||||
re_msg = re.compile(message_template.format(values=values_string))
|
||||
match_result = re_msg.match(client_error.response["Error"]["Message"])
|
||||
assert match_result is not None
|
||||
values_string = match_result.groupdict()["values"]
|
||||
values = [key for key in values_string.split(", ")]
|
||||
assert len(message_values) == len(values)
|
||||
for value in message_values:
|
||||
assert value in values
|
||||
else:
|
||||
assert client_error.response["Error"]["Message"] == message_template
|
||||
|
||||
|
||||
def create_simple_table_and_return_client():
|
||||
dynamodb = boto3.client("dynamodb", region_name="eu-west-1")
|
||||
dynamodb.create_table(
|
||||
TableName="moto-test",
|
||||
KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
|
||||
AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"},],
|
||||
ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
|
||||
)
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "myNum": {"N": "1"}, "MyStr": {"S": "1"},},
|
||||
)
|
||||
return dynamodb
|
||||
|
||||
|
||||
# https://github.com/spulec/moto/issues/2806
|
||||
# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html
|
||||
# #DDB-UpdateItem-request-UpdateExpression
|
||||
@mock_dynamodb2
|
||||
def test_update_item_with_attribute_in_right_hand_side_and_operation():
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET myNum = myNum+:val",
|
||||
ExpressionAttributeValues={":val": {"N": "3"}},
|
||||
)
|
||||
|
||||
result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})
|
||||
assert result["Item"]["myNum"]["N"] == "4"
|
||||
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET myNum = myNum - :val",
|
||||
ExpressionAttributeValues={":val": {"N": "1"}},
|
||||
)
|
||||
result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})
|
||||
assert result["Item"]["myNum"]["N"] == "3"
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_non_existing_attribute_should_raise_exception():
|
||||
"""
|
||||
Does error message get correctly raised if attribute is referenced but it does not exist for the item.
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
try:
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET MyStr = no_attr + MyStr",
|
||||
)
|
||||
assert False, "Validation exception not thrown"
|
||||
except dynamodb.exceptions.ClientError as e:
|
||||
assert_correct_client_error(
|
||||
e,
|
||||
"ValidationException",
|
||||
"The provided expression refers to an attribute that does not exist in the item",
|
||||
)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_expression_with_plus_in_attribute_name():
|
||||
"""
|
||||
Does error message get correctly raised if attribute contains a plus and is passed in without an AttributeName. And
|
||||
lhs & rhs are not attribute IDs by themselve.
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "my+Num": {"S": "1"}, "MyStr": {"S": "aaa"},},
|
||||
)
|
||||
try:
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET MyStr = my+Num",
|
||||
)
|
||||
assert False, "Validation exception not thrown"
|
||||
except dynamodb.exceptions.ClientError as e:
|
||||
assert_correct_client_error(
|
||||
e,
|
||||
"ValidationException",
|
||||
"The provided expression refers to an attribute that does not exist in the item",
|
||||
)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_expression_with_minus_in_attribute_name():
|
||||
"""
|
||||
Does error message get correctly raised if attribute contains a minus and is passed in without an AttributeName. And
|
||||
lhs & rhs are not attribute IDs by themselve.
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "my-Num": {"S": "1"}, "MyStr": {"S": "aaa"},},
|
||||
)
|
||||
try:
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET MyStr = my-Num",
|
||||
)
|
||||
assert False, "Validation exception not thrown"
|
||||
except dynamodb.exceptions.ClientError as e:
|
||||
assert_correct_client_error(
|
||||
e,
|
||||
"ValidationException",
|
||||
"The provided expression refers to an attribute that does not exist in the item",
|
||||
)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_expression_with_space_in_attribute_name():
|
||||
"""
|
||||
Does error message get correctly raised if attribute contains a space and is passed in without an AttributeName. And
|
||||
lhs & rhs are not attribute IDs by themselves.
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "my Num": {"S": "1"}, "MyStr": {"S": "aaa"},},
|
||||
)
|
||||
|
||||
try:
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET MyStr = my Num",
|
||||
)
|
||||
assert False, "Validation exception not thrown"
|
||||
except dynamodb.exceptions.ClientError as e:
|
||||
assert_raise_syntax_error(e, "Num", "my Num")
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_summing_up_2_strings_raises_exception():
|
||||
"""
|
||||
Update set supports different DynamoDB types but some operations are not supported. For example summing up 2 strings
|
||||
raises an exception. It results in ClientError with code ValidationException:
|
||||
Saying An operand in the update expression has an incorrect data type
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
try:
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET MyStr = MyStr + MyStr",
|
||||
)
|
||||
assert False, "Validation exception not thrown"
|
||||
except dynamodb.exceptions.ClientError as e:
|
||||
assert_correct_client_error(
|
||||
e,
|
||||
"ValidationException",
|
||||
"An operand in the update expression has an incorrect data type",
|
||||
)
|
||||
|
||||
|
||||
# https://github.com/spulec/moto/issues/2806
|
||||
@mock_dynamodb2
|
||||
def test_update_item_with_attribute_in_right_hand_side():
|
||||
"""
|
||||
After tokenization and building expression make sure referenced attributes are replaced with their current value
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
# Make sure there are 2 values
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "myVal1": {"S": "Value1"}, "myVal2": {"S": "Value2"}},
|
||||
)
|
||||
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET myVal1 = myVal2",
|
||||
)
|
||||
|
||||
result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})
|
||||
assert result["Item"]["myVal1"]["S"] == result["Item"]["myVal2"]["S"] == "Value2"
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_multiple_updates():
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "myNum": {"N": "1"}, "path": {"N": "6"}},
|
||||
)
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET myNum = #p + :val, newAttr = myNum",
|
||||
ExpressionAttributeValues={":val": {"N": "1"}},
|
||||
ExpressionAttributeNames={"#p": "path"},
|
||||
)
|
||||
result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})["Item"]
|
||||
expected_result = {
|
||||
"myNum": {"N": "7"},
|
||||
"newAttr": {"N": "1"},
|
||||
"path": {"N": "6"},
|
||||
"id": {"S": "1"},
|
||||
}
|
||||
assert result == expected_result
|
||||
|
||||
446
tests/test_dynamodb2/test_dynamodb_executor.py
Normal file
446
tests/test_dynamodb2/test_dynamodb_executor.py
Normal file
@ -0,0 +1,446 @@
|
||||
from moto.dynamodb2.exceptions import IncorrectOperandType, IncorrectDataType
|
||||
from moto.dynamodb2.models import Item, DynamoType
|
||||
from moto.dynamodb2.parsing.executors import UpdateExpressionExecutor
|
||||
from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
|
||||
from moto.dynamodb2.parsing.validators import UpdateExpressionValidator
|
||||
from parameterized import parameterized
|
||||
|
||||
|
||||
def test_execution_of_if_not_exists_not_existing_value():
|
||||
update_expression = "SET a = if_not_exists(b, a)"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "a": {"S": "A"}},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values=None,
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "a": {"S": "A"}},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
def test_execution_of_if_not_exists_with_existing_attribute_should_return_attribute():
|
||||
update_expression = "SET a = if_not_exists(b, a)"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "a": {"S": "A"}, "b": {"S": "B"}},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values=None,
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "a": {"S": "B"}, "b": {"S": "B"}},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
def test_execution_of_if_not_exists_with_existing_attribute_should_return_value():
|
||||
update_expression = "SET a = if_not_exists(b, :val)"
|
||||
update_expression_values = {":val": {"N": "4"}}
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "b": {"N": "3"}},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values=update_expression_values,
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "b": {"N": "3"}, "a": {"N": "3"}},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
def test_execution_of_if_not_exists_with_non_existing_attribute_should_return_value():
|
||||
update_expression = "SET a = if_not_exists(b, :val)"
|
||||
update_expression_values = {":val": {"N": "4"}}
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values=update_expression_values,
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "a": {"N": "4"}},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
def test_execution_of_sum_operation():
|
||||
update_expression = "SET a = a + b"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "a": {"N": "3"}, "b": {"N": "4"}},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values=None,
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "a": {"N": "7"}, "b": {"N": "4"}},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
def test_execution_of_remove():
|
||||
update_expression = "Remove a"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "a": {"N": "3"}, "b": {"N": "4"}},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values=None,
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "1"}, "b": {"N": "4"}},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
def test_execution_of_remove_in_map():
|
||||
update_expression = "Remove itemmap.itemlist[1].foo11"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={
|
||||
"id": {"S": "foo2"},
|
||||
"itemmap": {
|
||||
"M": {
|
||||
"itemlist": {
|
||||
"L": [
|
||||
{"M": {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}}},
|
||||
{"M": {"foo10": {"S": "bar1"}, "foo11": {"S": "bar2"}}},
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values=None,
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={
|
||||
"id": {"S": "foo2"},
|
||||
"itemmap": {
|
||||
"M": {
|
||||
"itemlist": {
|
||||
"L": [
|
||||
{"M": {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}}},
|
||||
{"M": {"foo10": {"S": "bar1"},}},
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
def test_execution_of_remove_in_list():
|
||||
update_expression = "Remove itemmap.itemlist[1]"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={
|
||||
"id": {"S": "foo2"},
|
||||
"itemmap": {
|
||||
"M": {
|
||||
"itemlist": {
|
||||
"L": [
|
||||
{"M": {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}}},
|
||||
{"M": {"foo10": {"S": "bar1"}, "foo11": {"S": "bar2"}}},
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values=None,
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={
|
||||
"id": {"S": "foo2"},
|
||||
"itemmap": {
|
||||
"M": {
|
||||
"itemlist": {
|
||||
"L": [{"M": {"foo00": {"S": "bar1"}, "foo01": {"S": "bar2"}}},]
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
def test_execution_of_delete_element_from_set():
|
||||
update_expression = "delete s :value"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "foo2"}, "s": {"SS": ["value1", "value2", "value3"]},},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values={":value": {"SS": ["value2", "value5"]}},
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "foo2"}, "s": {"SS": ["value1", "value3"]},},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
def test_execution_of_add_number():
|
||||
update_expression = "add s :value"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "foo2"}, "s": {"N": "5"},},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values={":value": {"N": "10"}},
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "foo2"}, "s": {"N": "15"}},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
def test_execution_of_add_set_to_a_number():
|
||||
update_expression = "add s :value"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "foo2"}, "s": {"N": "5"},},
|
||||
)
|
||||
try:
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values={":value": {"SS": ["s1"]}},
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "foo2"}, "s": {"N": "15"}},
|
||||
)
|
||||
assert expected_item == item
|
||||
assert False
|
||||
except IncorrectDataType:
|
||||
assert True
|
||||
|
||||
|
||||
def test_execution_of_add_to_a_set():
|
||||
update_expression = "ADD s :value"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "foo2"}, "s": {"SS": ["value1", "value2", "value3"]},},
|
||||
)
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values={":value": {"SS": ["value2", "value5"]}},
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
expected_item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={
|
||||
"id": {"S": "foo2"},
|
||||
"s": {"SS": ["value1", "value2", "value3", "value5"]},
|
||||
},
|
||||
)
|
||||
assert expected_item == item
|
||||
|
||||
|
||||
@parameterized(
|
||||
[
|
||||
({":value": {"S": "10"}}, "STRING",),
|
||||
({":value": {"N": "10"}}, "NUMBER",),
|
||||
({":value": {"B": "10"}}, "BINARY",),
|
||||
({":value": {"BOOL": True}}, "BOOLEAN",),
|
||||
({":value": {"NULL": True}}, "NULL",),
|
||||
({":value": {"M": {"el0": {"S": "10"}}}}, "MAP",),
|
||||
({":value": {"L": []}}, "LIST",),
|
||||
]
|
||||
)
|
||||
def test_execution_of__delete_element_from_set_invalid_value(
|
||||
expression_attribute_values, unexpected_data_type
|
||||
):
|
||||
"""A delete statement must use a value of type SS in order to delete elements from a set."""
|
||||
update_expression = "delete s :value"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "foo2"}, "s": {"SS": ["value1", "value2", "value3"]},},
|
||||
)
|
||||
try:
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values=expression_attribute_values,
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
assert False, "Must raise exception"
|
||||
except IncorrectOperandType as e:
|
||||
assert e.operator_or_function == "operator: DELETE"
|
||||
assert e.operand_type == unexpected_data_type
|
||||
|
||||
|
||||
def test_execution_of_delete_element_from_a_string_attribute():
|
||||
"""A delete statement must use a value of type SS in order to delete elements from a set."""
|
||||
update_expression = "delete s :value"
|
||||
update_expression_ast = UpdateExpressionParser.make(update_expression)
|
||||
item = Item(
|
||||
hash_key=DynamoType({"S": "id"}),
|
||||
hash_key_type="TYPE",
|
||||
range_key=None,
|
||||
range_key_type=None,
|
||||
attrs={"id": {"S": "foo2"}, "s": {"S": "5"},},
|
||||
)
|
||||
try:
|
||||
validated_ast = UpdateExpressionValidator(
|
||||
update_expression_ast,
|
||||
expression_attribute_names=None,
|
||||
expression_attribute_values={":value": {"SS": ["value2"]}},
|
||||
item=item,
|
||||
).validate()
|
||||
UpdateExpressionExecutor(validated_ast, item, None).execute()
|
||||
assert False, "Must raise exception"
|
||||
except IncorrectDataType:
|
||||
assert True
|
||||
@ -8,6 +8,8 @@ from boto3.dynamodb.conditions import Key
|
||||
from botocore.exceptions import ClientError
|
||||
import sure # noqa
|
||||
from freezegun import freeze_time
|
||||
from nose.tools import assert_raises
|
||||
|
||||
from moto import mock_dynamodb2, mock_dynamodb2_deprecated
|
||||
from boto.exception import JSONResponseError
|
||||
from tests.helpers import requires_boto_gte
|
||||
@ -1273,6 +1275,15 @@ def test_update_item_with_expression():
|
||||
)
|
||||
|
||||
|
||||
def assert_failure_due_to_key_not_in_schema(func, **kwargs):
|
||||
with assert_raises(ClientError) as ex:
|
||||
func(**kwargs)
|
||||
ex.exception.response["Error"]["Code"].should.equal("ValidationException")
|
||||
ex.exception.response["Error"]["Message"].should.equal(
|
||||
"The provided key element does not match the schema"
|
||||
)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_item_add_with_expression():
|
||||
table = _create_table_with_range_key()
|
||||
@ -1299,14 +1310,13 @@ def test_update_item_add_with_expression():
|
||||
dict(table.get_item(Key=item_key)["Item"]).should.equal(current_item)
|
||||
|
||||
# Update item to add a string value to a non-existing set
|
||||
# Should just create the set in the background
|
||||
table.update_item(
|
||||
# Should throw: 'The provided key element does not match the schema'
|
||||
assert_failure_due_to_key_not_in_schema(
|
||||
table.update_item,
|
||||
Key=item_key,
|
||||
UpdateExpression="ADD non_existing_str_set :v",
|
||||
ExpressionAttributeValues={":v": {"item4"}},
|
||||
)
|
||||
current_item["non_existing_str_set"] = {"item4"}
|
||||
dict(table.get_item(Key=item_key)["Item"]).should.equal(current_item)
|
||||
|
||||
# Update item to add a num value to a num set
|
||||
table.update_item(
|
||||
@ -1381,15 +1391,14 @@ def test_update_item_add_with_nested_sets():
|
||||
dict(table.get_item(Key=item_key)["Item"]).should.equal(current_item)
|
||||
|
||||
# Update item to add a string value to a non-existing set
|
||||
# Should just create the set in the background
|
||||
table.update_item(
|
||||
# Should raise
|
||||
assert_failure_due_to_key_not_in_schema(
|
||||
table.update_item,
|
||||
Key=item_key,
|
||||
UpdateExpression="ADD #ns.#ne :v",
|
||||
ExpressionAttributeNames={"#ns": "nested", "#ne": "non_existing_str_set"},
|
||||
ExpressionAttributeValues={":v": {"new_item"}},
|
||||
)
|
||||
current_item["nested"]["non_existing_str_set"] = {"new_item"}
|
||||
dict(table.get_item(Key=item_key)["Item"]).should.equal(current_item)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
|
||||
Loading…
Reference in New Issue
Block a user