Merge branch 'master' into fix/table-filter-value-error

CONTRIBUTING.md

@@ -4,7 +4,10 @@ Moto has a [Code of Conduct](https://github.com/spulec/moto/blob/master/CODE_OF_
 
 ## Running the tests locally
 
-Moto has a Makefile which has some helpful commands for getting setup. You should be able to run `make init` to install the dependencies and then `make test` to run the tests.
+Moto has a Makefile which has some helpful commands for getting setup. You should be able to run `make init` to install the dependencies and then `make test` to run the tests.  
+
+## Linting
+Run `make lint` or `black --check moto tests` to verify whether your code confirms to the guidelines.
 
 ## Is there a missing feature?
 

IMPLEMENTATION_COVERAGE.md

@@ -3351,11 +3351,11 @@
 - [ ] update_listener
 
 ## glue
-4% implemented
+11% implemented
-- [ ] batch_create_partition
+- [X] batch_create_partition
 - [ ] batch_delete_connection
-- [ ] batch_delete_partition
+- [X] batch_delete_partition
-- [ ] batch_delete_table
+- [X] batch_delete_table
 - [ ] batch_delete_table_version
 - [ ] batch_get_crawlers
 - [ ] batch_get_dev_endpoints
@@ -3372,7 +3372,7 @@
 - [ ] create_dev_endpoint
 - [ ] create_job
 - [ ] create_ml_transform
-- [ ] create_partition
+- [X] create_partition
 - [ ] create_script
 - [ ] create_security_configuration
 - [X] create_table
@@ -3404,7 +3404,7 @@
 - [ ] get_crawlers
 - [ ] get_data_catalog_encryption_settings
 - [X] get_database
-- [ ] get_databases
+- [X] get_databases
 - [ ] get_dataflow_graph
 - [ ] get_dev_endpoint
 - [ ] get_dev_endpoints
@@ -3418,7 +3418,7 @@
 - [ ] get_ml_task_runs
 - [ ] get_ml_transform
 - [ ] get_ml_transforms
-- [ ] get_partition
+- [X] get_partition
 - [ ] get_partitions
 - [ ] get_plan
 - [ ] get_resource_policy
@@ -3470,8 +3470,8 @@
 - [ ] update_dev_endpoint
 - [ ] update_job
 - [ ] update_ml_transform
-- [ ] update_partition
+- [X] update_partition
-- [ ] update_table
+- [X] update_table
 - [ ] update_trigger
 - [ ] update_user_defined_function
 - [ ] update_workflow
@@ -7210,13 +7210,13 @@
 - [ ] update_vtl_device_type
 
 ## sts
-50% implemented
+62% implemented
 - [X] assume_role
 - [ ] assume_role_with_saml
 - [X] assume_role_with_web_identity
 - [ ] decode_authorization_message
 - [ ] get_access_key_info
-- [ ] get_caller_identity
+- [X] get_caller_identity
 - [X] get_federation_token
 - [X] get_session_token
 

moto/core/models.py

@@ -12,6 +12,8 @@ from io import BytesIO
 from collections import defaultdict
 from botocore.handlers import BUILTIN_HANDLERS
 from botocore.awsrequest import AWSResponse
+from six.moves.urllib.parse import urlparse
+from werkzeug.wrappers import Request
 
 import mock
 from moto import settings
@@ -175,6 +177,26 @@ class CallbackResponse(responses.CallbackResponse):
         """
         Need to override this so we can pass decode_content=False
         """
+        if not isinstance(request, Request):
+            url = urlparse(request.url)
+            if request.body is None:
+                body = None
+            elif isinstance(request.body, six.text_type):
+                body = six.BytesIO(six.b(request.body))
+            else:
+                body = six.BytesIO(request.body)
+            req = Request.from_values(
+                path="?".join([url.path, url.query]),
+                input_stream=body,
+                content_length=request.headers.get("Content-Length"),
+                content_type=request.headers.get("Content-Type"),
+                method=request.method,
+                base_url="{scheme}://{netloc}".format(
+                    scheme=url.scheme, netloc=url.netloc
+                ),
+                headers=[(k, v) for k, v in six.iteritems(request.headers)],
+            )
+            request = req
         headers = self.get_headers()
 
         result = self.callback(request)

moto/dynamodb2/exceptions.py

@@ -2,9 +2,132 @@ class InvalidIndexNameError(ValueError):
     pass
 
 
-class InvalidUpdateExpression(ValueError):
+class MockValidationException(ValueError):
-    pass
+    def __init__(self, message):
+        self.exception_msg = message
 
 
-class ItemSizeTooLarge(Exception):
+class InvalidUpdateExpressionInvalidDocumentPath(MockValidationException):
-    message = "Item size has exceeded the maximum allowed size"
+    invalid_update_expression_msg = (
+        "The document path provided in the update expression is invalid for update"
+    )
+
+    def __init__(self):
+        super(InvalidUpdateExpressionInvalidDocumentPath, self).__init__(
+            self.invalid_update_expression_msg
+        )
+
+
+class InvalidUpdateExpression(MockValidationException):
+    invalid_update_expr_msg = "Invalid UpdateExpression: {update_expression_error}"
+
+    def __init__(self, update_expression_error):
+        self.update_expression_error = update_expression_error
+        super(InvalidUpdateExpression, self).__init__(
+            self.invalid_update_expr_msg.format(
+                update_expression_error=update_expression_error
+            )
+        )
+
+
+class AttributeDoesNotExist(MockValidationException):
+    attr_does_not_exist_msg = (
+        "The provided expression refers to an attribute that does not exist in the item"
+    )
+
+    def __init__(self):
+        super(AttributeDoesNotExist, self).__init__(self.attr_does_not_exist_msg)
+
+
+class ExpressionAttributeNameNotDefined(InvalidUpdateExpression):
+    name_not_defined_msg = "An expression attribute name used in the document path is not defined; attribute name: {n}"
+
+    def __init__(self, attribute_name):
+        self.not_defined_attribute_name = attribute_name
+        super(ExpressionAttributeNameNotDefined, self).__init__(
+            self.name_not_defined_msg.format(n=attribute_name)
+        )
+
+
+class AttributeIsReservedKeyword(InvalidUpdateExpression):
+    attribute_is_keyword_msg = (
+        "Attribute name is a reserved keyword; reserved keyword: {keyword}"
+    )
+
+    def __init__(self, keyword):
+        self.keyword = keyword
+        super(AttributeIsReservedKeyword, self).__init__(
+            self.attribute_is_keyword_msg.format(keyword=keyword)
+        )
+
+
+class ExpressionAttributeValueNotDefined(InvalidUpdateExpression):
+    attr_value_not_defined_msg = "An expression attribute value used in expression is not defined; attribute value: {attribute_value}"
+
+    def __init__(self, attribute_value):
+        self.attribute_value = attribute_value
+        super(ExpressionAttributeValueNotDefined, self).__init__(
+            self.attr_value_not_defined_msg.format(attribute_value=attribute_value)
+        )
+
+
+class UpdateExprSyntaxError(InvalidUpdateExpression):
+    update_expr_syntax_error_msg = "Syntax error; {error_detail}"
+
+    def __init__(self, error_detail):
+        self.error_detail = error_detail
+        super(UpdateExprSyntaxError, self).__init__(
+            self.update_expr_syntax_error_msg.format(error_detail=error_detail)
+        )
+
+
+class InvalidTokenException(UpdateExprSyntaxError):
+    token_detail_msg = 'token: "{token}", near: "{near}"'
+
+    def __init__(self, token, near):
+        self.token = token
+        self.near = near
+        super(InvalidTokenException, self).__init__(
+            self.token_detail_msg.format(token=token, near=near)
+        )
+
+
+class InvalidExpressionAttributeNameKey(MockValidationException):
+    invalid_expr_attr_name_msg = (
+        'ExpressionAttributeNames contains invalid key: Syntax error; key: "{key}"'
+    )
+
+    def __init__(self, key):
+        self.key = key
+        super(InvalidExpressionAttributeNameKey, self).__init__(
+            self.invalid_expr_attr_name_msg.format(key=key)
+        )
+
+
+class ItemSizeTooLarge(MockValidationException):
+    item_size_too_large_msg = "Item size has exceeded the maximum allowed size"
+
+    def __init__(self):
+        super(ItemSizeTooLarge, self).__init__(self.item_size_too_large_msg)
+
+
+class ItemSizeToUpdateTooLarge(MockValidationException):
+    item_size_to_update_too_large_msg = (
+        "Item size to update has exceeded the maximum allowed size"
+    )
+
+    def __init__(self):
+        super(ItemSizeToUpdateTooLarge, self).__init__(
+            self.item_size_to_update_too_large_msg
+        )
+
+
+class IncorrectOperandType(InvalidUpdateExpression):
+    inv_operand_msg = "Incorrect operand type for operator or function; operator or function: {f}, operand type: {t}"
+
+    def __init__(self, operator_or_function, operand_type):
+        self.operator_or_function = operator_or_function
+        self.operand_type = operand_type
+        super(IncorrectOperandType, self).__init__(
+            self.inv_operand_msg.format(f=operator_or_function, t=operand_type)
+        )

moto/dynamodb2/models/__init__.py

@@ -15,9 +15,15 @@ from moto.core.utils import unix_time
 from moto.core.exceptions import JsonRESTError
 from moto.dynamodb2.comparisons import get_filter_expression
 from moto.dynamodb2.comparisons import get_expected
-from moto.dynamodb2.exceptions import InvalidIndexNameError, ItemSizeTooLarge
+from moto.dynamodb2.exceptions import (
+    InvalidIndexNameError,
+    ItemSizeTooLarge,
+    ItemSizeToUpdateTooLarge,
+)
 from moto.dynamodb2.models.utilities import bytesize, attribute_is_list
 from moto.dynamodb2.models.dynamo_type import DynamoType
+from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
+from moto.dynamodb2.parsing.validators import UpdateExpressionValidator
 
 
 class DynamoJsonEncoder(json.JSONEncoder):
@@ -150,7 +156,10 @@ class Item(BaseModel):
                     if "." in key and attr not in self.attrs:
                         raise ValueError  # Setting nested attr not allowed if first attr does not exist yet
                     elif attr not in self.attrs:
-                        self.attrs[attr] = dyn_value  # set new top-level attribute
+                        try:
+                            self.attrs[attr] = dyn_value  # set new top-level attribute
+                        except ItemSizeTooLarge:
+                            raise ItemSizeToUpdateTooLarge()
                     else:
                         self.attrs[attr].set(
                             ".".join(key.split(".")[1:]), dyn_value, list_index
@@ -1197,6 +1206,13 @@ class DynamoDBBackend(BaseBackend):
     ):
         table = self.get_table(table_name)
 
+        # Support spaces between operators in an update expression
+        # E.g. `a = b + c` -> `a=b+c`
+        if update_expression:
+            # Parse expression to get validation errors
+            update_expression_ast = UpdateExpressionParser.make(update_expression)
+            update_expression = re.sub(r"\s*([=\+-])\s*", "\\1", update_expression)
+
         if all([table.hash_key_attr in key, table.range_key_attr in key]):
             # Covers cases where table has hash and range keys, ``key`` param
             # will be a dict
@@ -1239,6 +1255,12 @@ class DynamoDBBackend(BaseBackend):
             item = table.get_item(hash_value, range_value)
 
         if update_expression:
+            UpdateExpressionValidator(
+                update_expression_ast,
+                expression_attribute_names=expression_attribute_names,
+                expression_attribute_values=expression_attribute_values,
+                item=item,
+            ).validate()
             item.update(
                 update_expression,
                 expression_attribute_names,

moto/dynamodb2/models/dynamo_type.py

@@ -123,6 +123,37 @@ class DynamoType(object):
     def __repr__(self):
         return "DynamoType: {0}".format(self.to_json())
 
+    def __add__(self, other):
+        if self.type != other.type:
+            raise TypeError("Different types of operandi is not allowed.")
+        if self.type == "N":
+            return DynamoType({"N": "{v}".format(v=int(self.value) + int(other.value))})
+        else:
+            raise TypeError("Sum only supported for Numbers.")
+
+    def __sub__(self, other):
+        if self.type != other.type:
+            raise TypeError("Different types of operandi is not allowed.")
+        if self.type == "N":
+            return DynamoType({"N": "{v}".format(v=int(self.value) - int(other.value))})
+        else:
+            raise TypeError("Sum only supported for Numbers.")
+
+    def __getitem__(self, item):
+        if isinstance(item, six.string_types):
+            # If our DynamoType is a map it should be subscriptable with a key
+            if self.type == "M":
+                return self.value[item]
+        elif isinstance(item, int):
+            # If our DynamoType is a list is should be subscriptable with an index
+            if self.type == "L":
+                return self.value[item]
+        raise TypeError(
+            "This DynamoType {dt} is not subscriptable by a {it}".format(
+                dt=self.type, it=type(item)
+            )
+        )
+
     @property
     def cast_value(self):
         if self.is_number():

moto/dynamodb2/parsing/README.md

@@ -0,0 +1,23 @@
+# Parsing dev documentation
+
+Parsing happens in a structured manner and happens in different phases.
+This document explains these phases.
+
+
+## 1) Expression gets parsed into a tokenlist (tokenized)
+A string gets parsed from left to right and gets converted into a list of tokens.
+The tokens are available in `tokens.py`.
+
+## 2) Tokenlist get transformed to expression tree (AST)
+This is the parsing of the token list. This parsing will result in an Abstract Syntax Tree (AST).
+The different node types are available in `ast_nodes.py`.  The AST is a representation that has all
+the information that is in the expression but its tree form allows processing it in a structured manner.
+
+## 3) The AST gets validated (full semantic correctness)
+The AST is used for validation. The paths and attributes are validated to be correct. At the end of the
+validation all the values will be resolved.
+
+## 4) Update Expression gets executed using the validated AST
+Finally the AST is used to execute the update expression. There should be no reason for this step to fail
+since validation has completed. Due to this we have the update expressions behaving atomically (i.e. all the
+actions of the update expresion are performed or none of them are performed).

moto/dynamodb2/parsing/ast_nodes.py

@@ -0,0 +1,360 @@
+import abc
+from abc import abstractmethod
+from collections import deque
+
+import six
+
+from moto.dynamodb2.models import DynamoType
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Node:
+    def __init__(self, children=None):
+        self.type = self.__class__.__name__
+        assert children is None or isinstance(children, list)
+        self.children = children
+        self.parent = None
+
+        if isinstance(children, list):
+            for child in children:
+                if isinstance(child, Node):
+                    child.set_parent(self)
+
+    def set_parent(self, parent_node):
+        self.parent = parent_node
+
+
+class LeafNode(Node):
+    """A LeafNode is a Node where none of the children are Nodes themselves."""
+
+    def __init__(self, children=None):
+        super(LeafNode, self).__init__(children)
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Expression(Node):
+    """
+    Abstract Syntax Tree representing the expression
+
+    For the Grammar start here and jump down into the classes at the righ-hand side to look further. Nodes marked with
+    a star are abstract and won't appear in the final AST.
+
+    Expression* => UpdateExpression
+    Expression* => ConditionExpression
+    """
+
+
+class UpdateExpression(Expression):
+    """
+    UpdateExpression => UpdateExpressionClause*
+    UpdateExpression => UpdateExpressionClause* UpdateExpression
+    """
+
+
+@six.add_metaclass(abc.ABCMeta)
+class UpdateExpressionClause(UpdateExpression):
+    """
+    UpdateExpressionClause* => UpdateExpressionSetClause
+    UpdateExpressionClause* => UpdateExpressionRemoveClause
+    UpdateExpressionClause* => UpdateExpressionAddClause
+    UpdateExpressionClause* => UpdateExpressionDeleteClause
+    """
+
+
+class UpdateExpressionSetClause(UpdateExpressionClause):
+    """
+    UpdateExpressionSetClause => SET SetActions
+    """
+
+
+class UpdateExpressionSetActions(UpdateExpressionClause):
+    """
+    UpdateExpressionSetClause => SET SetActions
+
+    SetActions => SetAction
+    SetActions => SetAction , SetActions
+
+    """
+
+
+class UpdateExpressionSetAction(UpdateExpressionClause):
+    """
+    SetAction => Path = Value
+    """
+
+
+class UpdateExpressionRemoveActions(UpdateExpressionClause):
+    """
+    UpdateExpressionSetClause => REMOVE RemoveActions
+
+    RemoveActions => RemoveAction
+    RemoveActions => RemoveAction , RemoveActions
+    """
+
+
+class UpdateExpressionRemoveAction(UpdateExpressionClause):
+    """
+    RemoveAction => Path
+    """
+
+
+class UpdateExpressionAddActions(UpdateExpressionClause):
+    """
+    UpdateExpressionAddClause => ADD RemoveActions
+
+    AddActions => AddAction
+    AddActions => AddAction , AddActions
+    """
+
+
+class UpdateExpressionAddAction(UpdateExpressionClause):
+    """
+    AddAction => Path Value
+    """
+
+
+class UpdateExpressionDeleteActions(UpdateExpressionClause):
+    """
+    UpdateExpressionDeleteClause => DELETE RemoveActions
+
+    DeleteActions => DeleteAction
+    DeleteActions => DeleteAction , DeleteActions
+    """
+
+
+class UpdateExpressionDeleteAction(UpdateExpressionClause):
+    """
+    DeleteAction => Path Value
+    """
+
+
+class UpdateExpressionPath(UpdateExpressionClause):
+    pass
+
+
+class UpdateExpressionValue(UpdateExpressionClause):
+    """
+    Value => Operand
+    Value => Operand + Value
+    Value => Operand - Value
+    """
+
+
+class UpdateExpressionGroupedValue(UpdateExpressionClause):
+    """
+    GroupedValue => ( Value )
+    """
+
+
+class UpdateExpressionRemoveClause(UpdateExpressionClause):
+    """
+    UpdateExpressionRemoveClause => REMOVE RemoveActions
+    """
+
+
+class UpdateExpressionAddClause(UpdateExpressionClause):
+    """
+    UpdateExpressionAddClause => ADD AddActions
+    """
+
+
+class UpdateExpressionDeleteClause(UpdateExpressionClause):
+    """
+    UpdateExpressionDeleteClause => DELETE DeleteActions
+    """
+
+
+class ExpressionPathDescender(Node):
+    """Node identifying descender into nested structure (.) in expression"""
+
+
+class ExpressionSelector(LeafNode):
+    """Node identifying selector [selection_index] in expresion"""
+
+    def __init__(self, selection_index):
+        try:
+            super(ExpressionSelector, self).__init__(children=[int(selection_index)])
+        except ValueError:
+            assert (
+                False
+            ), "Expression selector must be an int, this is a bug in the moto library."
+
+    def get_index(self):
+        return self.children[0]
+
+
+class ExpressionAttribute(LeafNode):
+    """An attribute identifier as used in the DDB item"""
+
+    def __init__(self, attribute):
+        super(ExpressionAttribute, self).__init__(children=[attribute])
+
+    def get_attribute_name(self):
+        return self.children[0]
+
+
+class ExpressionAttributeName(LeafNode):
+    """An ExpressionAttributeName is an alias for an attribute identifier"""
+
+    def __init__(self, attribute_name):
+        super(ExpressionAttributeName, self).__init__(children=[attribute_name])
+
+    def get_attribute_name_placeholder(self):
+        return self.children[0]
+
+
+class ExpressionAttributeValue(LeafNode):
+    """An ExpressionAttributeValue is an alias for an value"""
+
+    def __init__(self, value):
+        super(ExpressionAttributeValue, self).__init__(children=[value])
+
+    def get_value_name(self):
+        return self.children[0]
+
+
+class ExpressionValueOperator(LeafNode):
+    """An ExpressionValueOperator is an operation that works on 2 values"""
+
+    def __init__(self, value):
+        super(ExpressionValueOperator, self).__init__(children=[value])
+
+    def get_operator(self):
+        return self.children[0]
+
+
+class UpdateExpressionFunction(Node):
+    """
+    A Node representing a function of an Update Expression. The first child is the function name the others are the
+    arguments.
+    """
+
+    def get_function_name(self):
+        return self.children[0]
+
+    def get_nth_argument(self, n=1):
+        """Return nth element where n is a 1-based index."""
+        assert n >= 1
+        return self.children[n]
+
+
+class DDBTypedValue(Node):
+    """
+    A node representing a DDBTyped value. This can be any structure as supported by DyanmoDB. The node only has 1 child
+    which is the value of type `DynamoType`.
+    """
+
+    def __init__(self, value):
+        assert isinstance(value, DynamoType), "DDBTypedValue must be of DynamoType"
+        super(DDBTypedValue, self).__init__(children=[value])
+
+    def get_value(self):
+        return self.children[0]
+
+
+class NoneExistingPath(LeafNode):
+    """A placeholder for Paths that did not exist in the Item."""
+
+    def __init__(self, creatable=False):
+        super(NoneExistingPath, self).__init__(children=[creatable])
+
+    def is_creatable(self):
+        """Can this path be created if need be. For example path creating element in a dictionary or creating a new
+        attribute under root level of an item."""
+        return self.children[0]
+
+
+class DepthFirstTraverser(object):
+    """
+    Helper class that allows depth first traversal and to implement custom processing for certain AST nodes. The
+    processor of a node must return the new resulting node. This node will be placed in the tree. Processing of a
+    node using this traverser should therefore only transform child nodes.  The returned node will get the same parent
+    as the node before processing had.
+    """
+
+    @abstractmethod
+    def _processing_map(self):
+        """
+        A map providing a processing function per node class type to a function that takes in a Node object and
+        processes it. A Node can only be processed by a single function and they are considered in order. Therefore if
+        multiple classes from a single class hierarchy strain are used the more specific classes have to be put before
+        the less specific ones. That requires overriding `nodes_to_be_processed`. If no multiple classes form a single
+        class hierarchy strain are used the default implementation of `nodes_to_be_processed` should be OK.
+        Returns:
+            dict: Mapping a Node Class to a processing function.
+        """
+        pass
+
+    def nodes_to_be_processed(self):
+        """Cached accessor for getting Node types that need to be processed."""
+        return tuple(k for k in self._processing_map().keys())
+
+    def process(self, node):
+        """Process a Node"""
+        for class_key, processor in self._processing_map().items():
+            if isinstance(node, class_key):
+                return processor(node)
+
+    def pre_processing_of_child(self, parent_node, child_id):
+        """Hook that is called pre-processing of the child at position `child_id`"""
+        pass
+
+    def traverse_node_recursively(self, node, child_id=-1):
+        """
+        Traverse nodes depth first processing nodes bottom up (if root node is considered the top).
+
+        Args:
+            node(Node): The node which is the last node to be processed but which allows to identify all the
+                             work (which is in the children)
+            child_id(int): The index in the list of children from the parent that this node corresponds to
+
+        Returns:
+            Node: The node of the new processed AST
+        """
+        if isinstance(node, Node):
+            parent_node = node.parent
+            if node.children is not None:
+                for i, child_node in enumerate(node.children):
+                    self.pre_processing_of_child(node, i)
+                    self.traverse_node_recursively(child_node, i)
+            # noinspection PyTypeChecker
+            if isinstance(node, self.nodes_to_be_processed()):
+                node = self.process(node)
+                node.parent = parent_node
+                parent_node.children[child_id] = node
+        return node
+
+    def traverse(self, node):
+        return self.traverse_node_recursively(node)
+
+
+class NodeDepthLeftTypeFetcher(object):
+    """Helper class to fetch a node of a specific type. Depth left-first traversal"""
+
+    def __init__(self, node_type, root_node):
+        assert issubclass(node_type, Node)
+        self.node_type = node_type
+        self.root_node = root_node
+        self.queue = deque()
+        self.add_nodes_left_to_right_depth_first(self.root_node)
+
+    def add_nodes_left_to_right_depth_first(self, node):
+        if isinstance(node, Node) and node.children is not None:
+            for child_node in node.children:
+                self.add_nodes_left_to_right_depth_first(child_node)
+                self.queue.append(child_node)
+        self.queue.append(node)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        return self.__next__()
+
+    def __next__(self):
+        while len(self.queue) > 0:
+            candidate = self.queue.popleft()
+            if isinstance(candidate, self.node_type):
+                return candidate
+        else:
+            raise StopIteration

moto/dynamodb2/parsing/reserved_keywords.py

@@ -0,0 +1,29 @@
+class ReservedKeywords(list):
+    """
+    DynamoDB has an extensive list of keywords. Keywords are considered when validating the expression Tree.
+    Not earlier since an update expression like "SET path = VALUE 1" fails with:
+        'Invalid UpdateExpression: Syntax error; token: "1", near: "VALUE 1"'
+    """
+
+    KEYWORDS = None
+
+    @classmethod
+    def get_reserved_keywords(cls):
+        if cls.KEYWORDS is None:
+            cls.KEYWORDS = cls._get_reserved_keywords()
+        return cls.KEYWORDS
+
+    @classmethod
+    def _get_reserved_keywords(cls):
+        """
+        Get a list of reserved keywords of DynamoDB
+        """
+        try:
+            import importlib.resources as pkg_resources
+        except ImportError:
+            import importlib_resources as pkg_resources
+
+        reserved_keywords = pkg_resources.read_text(
+            "moto.dynamodb2.parsing", "reserved_keywords.txt"
+        )
+        return reserved_keywords.split()

moto/dynamodb2/parsing/reserved_keywords.txt

@@ -0,0 +1,573 @@
+ABORT
+ABSOLUTE
+ACTION
+ADD
+AFTER
+AGENT
+AGGREGATE
+ALL
+ALLOCATE
+ALTER
+ANALYZE
+AND
+ANY
+ARCHIVE
+ARE
+ARRAY
+AS
+ASC
+ASCII
+ASENSITIVE
+ASSERTION
+ASYMMETRIC
+AT
+ATOMIC
+ATTACH
+ATTRIBUTE
+AUTH
+AUTHORIZATION
+AUTHORIZE
+AUTO
+AVG
+BACK
+BACKUP
+BASE
+BATCH
+BEFORE
+BEGIN
+BETWEEN
+BIGINT
+BINARY
+BIT
+BLOB
+BLOCK
+BOOLEAN
+BOTH
+BREADTH
+BUCKET
+BULK
+BY
+BYTE
+CALL
+CALLED
+CALLING
+CAPACITY
+CASCADE
+CASCADED
+CASE
+CAST
+CATALOG
+CHAR
+CHARACTER
+CHECK
+CLASS
+CLOB
+CLOSE
+CLUSTER
+CLUSTERED
+CLUSTERING
+CLUSTERS
+COALESCE
+COLLATE
+COLLATION
+COLLECTION
+COLUMN
+COLUMNS
+COMBINE
+COMMENT
+COMMIT
+COMPACT
+COMPILE
+COMPRESS
+CONDITION
+CONFLICT
+CONNECT
+CONNECTION
+CONSISTENCY
+CONSISTENT
+CONSTRAINT
+CONSTRAINTS
+CONSTRUCTOR
+CONSUMED
+CONTINUE
+CONVERT
+COPY
+CORRESPONDING
+COUNT
+COUNTER
+CREATE
+CROSS
+CUBE
+CURRENT
+CURSOR
+CYCLE
+DATA
+DATABASE
+DATE
+DATETIME
+DAY
+DEALLOCATE
+DEC
+DECIMAL
+DECLARE
+DEFAULT
+DEFERRABLE
+DEFERRED
+DEFINE
+DEFINED
+DEFINITION
+DELETE
+DELIMITED
+DEPTH
+DEREF
+DESC
+DESCRIBE
+DESCRIPTOR
+DETACH
+DETERMINISTIC
+DIAGNOSTICS
+DIRECTORIES
+DISABLE
+DISCONNECT
+DISTINCT
+DISTRIBUTE
+DO
+DOMAIN
+DOUBLE
+DROP
+DUMP
+DURATION
+DYNAMIC
+EACH
+ELEMENT
+ELSE
+ELSEIF
+EMPTY
+ENABLE
+END
+EQUAL
+EQUALS
+ERROR
+ESCAPE
+ESCAPED
+EVAL
+EVALUATE
+EXCEEDED
+EXCEPT
+EXCEPTION
+EXCEPTIONS
+EXCLUSIVE
+EXEC
+EXECUTE
+EXISTS
+EXIT
+EXPLAIN
+EXPLODE
+EXPORT
+EXPRESSION
+EXTENDED
+EXTERNAL
+EXTRACT
+FAIL
+FALSE
+FAMILY
+FETCH
+FIELDS
+FILE
+FILTER
+FILTERING
+FINAL
+FINISH
+FIRST
+FIXED
+FLATTERN
+FLOAT
+FOR
+FORCE
+FOREIGN
+FORMAT
+FORWARD
+FOUND
+FREE
+FROM
+FULL
+FUNCTION
+FUNCTIONS
+GENERAL
+GENERATE
+GET
+GLOB
+GLOBAL
+GO
+GOTO
+GRANT
+GREATER
+GROUP
+GROUPING
+HANDLER
+HASH
+HAVE
+HAVING
+HEAP
+HIDDEN
+HOLD
+HOUR
+IDENTIFIED
+IDENTITY
+IF
+IGNORE
+IMMEDIATE
+IMPORT
+IN
+INCLUDING
+INCLUSIVE
+INCREMENT
+INCREMENTAL
+INDEX
+INDEXED
+INDEXES
+INDICATOR
+INFINITE
+INITIALLY
+INLINE
+INNER
+INNTER
+INOUT
+INPUT
+INSENSITIVE
+INSERT
+INSTEAD
+INT
+INTEGER
+INTERSECT
+INTERVAL
+INTO
+INVALIDATE
+IS
+ISOLATION
+ITEM
+ITEMS
+ITERATE
+JOIN
+KEY
+KEYS
+LAG
+LANGUAGE
+LARGE
+LAST
+LATERAL
+LEAD
+LEADING
+LEAVE
+LEFT
+LENGTH
+LESS
+LEVEL
+LIKE
+LIMIT
+LIMITED
+LINES
+LIST
+LOAD
+LOCAL
+LOCALTIME
+LOCALTIMESTAMP
+LOCATION
+LOCATOR
+LOCK
+LOCKS
+LOG
+LOGED
+LONG
+LOOP
+LOWER
+MAP
+MATCH
+MATERIALIZED
+MAX
+MAXLEN
+MEMBER
+MERGE
+METHOD
+METRICS
+MIN
+MINUS
+MINUTE
+MISSING
+MOD
+MODE
+MODIFIES
+MODIFY
+MODULE
+MONTH
+MULTI
+MULTISET
+NAME
+NAMES
+NATIONAL
+NATURAL
+NCHAR
+NCLOB
+NEW
+NEXT
+NO
+NONE
+NOT
+NULL
+NULLIF
+NUMBER
+NUMERIC
+OBJECT
+OF
+OFFLINE
+OFFSET
+OLD
+ON
+ONLINE
+ONLY
+OPAQUE
+OPEN
+OPERATOR
+OPTION
+OR
+ORDER
+ORDINALITY
+OTHER
+OTHERS
+OUT
+OUTER
+OUTPUT
+OVER
+OVERLAPS
+OVERRIDE
+OWNER
+PAD
+PARALLEL
+PARAMETER
+PARAMETERS
+PARTIAL
+PARTITION
+PARTITIONED
+PARTITIONS
+PATH
+PERCENT
+PERCENTILE
+PERMISSION
+PERMISSIONS
+PIPE
+PIPELINED
+PLAN
+POOL
+POSITION
+PRECISION
+PREPARE
+PRESERVE
+PRIMARY
+PRIOR
+PRIVATE
+PRIVILEGES
+PROCEDURE
+PROCESSED
+PROJECT
+PROJECTION
+PROPERTY
+PROVISIONING
+PUBLIC
+PUT
+QUERY
+QUIT
+QUORUM
+RAISE
+RANDOM
+RANGE
+RANK
+RAW
+READ
+READS
+REAL
+REBUILD
+RECORD
+RECURSIVE
+REDUCE
+REF
+REFERENCE
+REFERENCES
+REFERENCING
+REGEXP
+REGION
+REINDEX
+RELATIVE
+RELEASE
+REMAINDER
+RENAME
+REPEAT
+REPLACE
+REQUEST
+RESET
+RESIGNAL
+RESOURCE
+RESPONSE
+RESTORE
+RESTRICT
+RESULT
+RETURN
+RETURNING
+RETURNS
+REVERSE
+REVOKE
+RIGHT
+ROLE
+ROLES
+ROLLBACK
+ROLLUP
+ROUTINE
+ROW
+ROWS
+RULE
+RULES
+SAMPLE
+SATISFIES
+SAVE
+SAVEPOINT
+SCAN
+SCHEMA
+SCOPE
+SCROLL
+SEARCH
+SECOND
+SECTION
+SEGMENT
+SEGMENTS
+SELECT
+SELF
+SEMI
+SENSITIVE
+SEPARATE
+SEQUENCE
+SERIALIZABLE
+SESSION
+SET
+SETS
+SHARD
+SHARE
+SHARED
+SHORT
+SHOW
+SIGNAL
+SIMILAR
+SIZE
+SKEWED
+SMALLINT
+SNAPSHOT
+SOME
+SOURCE
+SPACE
+SPACES
+SPARSE
+SPECIFIC
+SPECIFICTYPE
+SPLIT
+SQL
+SQLCODE
+SQLERROR
+SQLEXCEPTION
+SQLSTATE
+SQLWARNING
+START
+STATE
+STATIC
+STATUS
+STORAGE
+STORE
+STORED
+STREAM
+STRING
+STRUCT
+STYLE
+SUB
+SUBMULTISET
+SUBPARTITION
+SUBSTRING
+SUBTYPE
+SUM
+SUPER
+SYMMETRIC
+SYNONYM
+SYSTEM
+TABLE
+TABLESAMPLE
+TEMP
+TEMPORARY
+TERMINATED
+TEXT
+THAN
+THEN
+THROUGHPUT
+TIME
+TIMESTAMP
+TIMEZONE
+TINYINT
+TO
+TOKEN
+TOTAL
+TOUCH
+TRAILING
+TRANSACTION
+TRANSFORM
+TRANSLATE
+TRANSLATION
+TREAT
+TRIGGER
+TRIM
+TRUE
+TRUNCATE
+TTL
+TUPLE
+TYPE
+UNDER
+UNDO
+UNION
+UNIQUE
+UNIT
+UNKNOWN
+UNLOGGED
+UNNEST
+UNPROCESSED
+UNSIGNED
+UNTIL
+UPDATE
+UPPER
+URL
+USAGE
+USE
+USER
+USERS
+USING
+UUID
+VACUUM
+VALUE
+VALUED
+VALUES
+VARCHAR
+VARIABLE
+VARIANCE
+VARINT
+VARYING
+VIEW
+VIEWS
+VIRTUAL
+VOID
+WAIT
+WHEN
+WHENEVER
+WHERE
+WHILE
+WINDOW
+WITH
+WITHIN
+WITHOUT
+WORK
+WRAPPED
+WRITE
+YEAR
+ZONE

moto/dynamodb2/parsing/tokens.py

@@ -0,0 +1,223 @@
+import re
+import sys
+
+from moto.dynamodb2.exceptions import (
+    InvalidTokenException,
+    InvalidExpressionAttributeNameKey,
+)
+
+
+class Token(object):
+    _TOKEN_INSTANCE = None
+    MINUS_SIGN = "-"
+    PLUS_SIGN = "+"
+    SPACE_SIGN = " "
+    EQUAL_SIGN = "="
+    OPEN_ROUND_BRACKET = "("
+    CLOSE_ROUND_BRACKET = ")"
+    COMMA = ","
+    SPACE = " "
+    DOT = "."
+    OPEN_SQUARE_BRACKET = "["
+    CLOSE_SQUARE_BRACKET = "]"
+
+    SPECIAL_CHARACTERS = [
+        MINUS_SIGN,
+        PLUS_SIGN,
+        SPACE_SIGN,
+        EQUAL_SIGN,
+        OPEN_ROUND_BRACKET,
+        CLOSE_ROUND_BRACKET,
+        COMMA,
+        SPACE,
+        DOT,
+        OPEN_SQUARE_BRACKET,
+        CLOSE_SQUARE_BRACKET,
+    ]
+
+    # Attribute: an identifier that is an attribute
+    ATTRIBUTE = 0
+    # Place holder for attribute name
+    ATTRIBUTE_NAME = 1
+    # Placeholder for attribute value starts with :
+    ATTRIBUTE_VALUE = 2
+    # WhiteSpace shall be grouped together
+    WHITESPACE = 3
+    # Placeholder for a number
+    NUMBER = 4
+
+    PLACEHOLDER_NAMES = {
+        ATTRIBUTE: "Attribute",
+        ATTRIBUTE_NAME: "AttributeName",
+        ATTRIBUTE_VALUE: "AttributeValue",
+        WHITESPACE: "Whitespace",
+        NUMBER: "Number",
+    }
+
+    def __init__(self, token_type, value):
+        assert (
+            token_type in self.SPECIAL_CHARACTERS
+            or token_type in self.PLACEHOLDER_NAMES
+        )
+        self.type = token_type
+        self.value = value
+
+    def __repr__(self):
+        if isinstance(self.type, int):
+            return 'Token("{tt}", "{tv}")'.format(
+                tt=self.PLACEHOLDER_NAMES[self.type], tv=self.value
+            )
+        else:
+            return 'Token("{tt}", "{tv}")'.format(tt=self.type, tv=self.value)
+
+    def __eq__(self, other):
+        return self.type == other.type and self.value == other.value
+
+
+class ExpressionTokenizer(object):
+    """
+    Takes a string and returns a list of tokens. While attribute names in DynamoDB must be between 1 and 255 characters
+    long there are no other restrictions for attribute names. For expressions however there are additional rules. If an
+    attribute name does not adhere then it must be passed via an ExpressionAttributeName. This tokenizer is aware of the
+    rules of Expression attributes.
+
+    We consider a Token as a tuple which has the tokenType
+
+    From https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html
+    1) If an attribute name begins with a number or contains a space, a special character, or a reserved word, you
+       must use an expression attribute name to replace that attribute's name in the expression.
+       => So spaces,+,- or other special characters do identify tokens in update expressions
+
+    2) When using a dot (.) in an attribute name you must use expression-attribute-names. A dot in an expression
+       will be interpreted as a separator in a document path
+
+    3) For a nested structure if you want to use expression_attribute_names you must specify one per part of the
+       path. Since for members of expression_attribute_names the . is part of the name
+
+    """
+
+    @classmethod
+    def is_simple_token_character(cls, character):
+        return character.isalnum() or character in ("_", ":", "#")
+
+    @classmethod
+    def is_possible_token_boundary(cls, character):
+        return (
+            character in Token.SPECIAL_CHARACTERS
+            or not cls.is_simple_token_character(character)
+        )
+
+    @classmethod
+    def is_expression_attribute(cls, input_string):
+        return re.compile("^[a-zA-Z][a-zA-Z0-9_]*$").match(input_string) is not None
+
+    @classmethod
+    def is_expression_attribute_name(cls, input_string):
+        """
+        https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html
+        An expression attribute name must begin with a pound sign (#), and be followed by one or more alphanumeric
+         characters.
+        """
+        return input_string.startswith("#") and cls.is_expression_attribute(
+            input_string[1:]
+        )
+
+    @classmethod
+    def is_expression_attribute_value(cls, input_string):
+        return re.compile("^:[a-zA-Z0-9_]*$").match(input_string) is not None
+
+    def raise_unexpected_token(self):
+        """If during parsing an unexpected token is encountered"""
+        if len(self.token_list) == 0:
+            near = ""
+        else:
+            if len(self.token_list) == 1:
+                near = self.token_list[-1].value
+            else:
+                if self.token_list[-1].type == Token.WHITESPACE:
+                    # Last token was whitespace take 2nd last token value as well to help User orientate
+                    near = self.token_list[-2].value + self.token_list[-1].value
+                else:
+                    near = self.token_list[-1].value
+
+        problematic_token = self.staged_characters[0]
+        raise InvalidTokenException(problematic_token, near + self.staged_characters)
+
+    def __init__(self, input_expression_str):
+        self.input_expression_str = input_expression_str
+        self.token_list = []
+        self.staged_characters = ""
+
+    @classmethod
+    def is_py2(cls):
+        return sys.version_info[0] == 2
+
+    @classmethod
+    def make_list(cls, input_expression_str):
+        if cls.is_py2():
+            pass
+        else:
+            assert isinstance(input_expression_str, str)
+
+        return ExpressionTokenizer(input_expression_str)._make_list()
+
+    def add_token(self, token_type, token_value):
+        self.token_list.append(Token(token_type, token_value))
+
+    def add_token_from_stage(self, token_type):
+        self.add_token(token_type, self.staged_characters)
+        self.staged_characters = ""
+
+    @classmethod
+    def is_numeric(cls, input_str):
+        return re.compile("[0-9]+").match(input_str) is not None
+
+    def process_staged_characters(self):
+        if len(self.staged_characters) == 0:
+            return
+        if self.staged_characters.startswith("#"):
+            if self.is_expression_attribute_name(self.staged_characters):
+                self.add_token_from_stage(Token.ATTRIBUTE_NAME)
+            else:
+                raise InvalidExpressionAttributeNameKey(self.staged_characters)
+        elif self.is_numeric(self.staged_characters):
+            self.add_token_from_stage(Token.NUMBER)
+        elif self.is_expression_attribute(self.staged_characters):
+            self.add_token_from_stage(Token.ATTRIBUTE)
+        elif self.is_expression_attribute_value(self.staged_characters):
+            self.add_token_from_stage(Token.ATTRIBUTE_VALUE)
+        else:
+            self.raise_unexpected_token()
+
+    def _make_list(self):
+        """
+        Just go through characters if a character is not a token boundary stage it for adding it as a grouped token
+        later if it is a tokenboundary process staged characters and then process the token boundary as well.
+        """
+        for character in self.input_expression_str:
+            if not self.is_possible_token_boundary(character):
+                self.staged_characters += character
+            else:
+                self.process_staged_characters()
+
+                if character == Token.SPACE:
+                    if (
+                        len(self.token_list) > 0
+                        and self.token_list[-1].type == Token.WHITESPACE
+                    ):
+                        self.token_list[-1].value = (
+                            self.token_list[-1].value + character
+                        )
+                    else:
+                        self.add_token(Token.WHITESPACE, character)
+                elif character in Token.SPECIAL_CHARACTERS:
+                    self.add_token(character, character)
+                elif not self.is_simple_token_character(character):
+                    self.staged_characters += character
+                    self.raise_unexpected_token()
+                else:
+                    raise NotImplementedError(
+                        "Encountered character which was not implemented : " + character
+                    )
+        self.process_staged_characters()
+        return self.token_list

moto/dynamodb2/parsing/validators.py

@@ -0,0 +1,341 @@
+"""
+See docstring class Validator below for more details on validation
+"""
+from abc import abstractmethod
+from copy import deepcopy
+
+from moto.dynamodb2.exceptions import (
+    AttributeIsReservedKeyword,
+    ExpressionAttributeValueNotDefined,
+    AttributeDoesNotExist,
+    ExpressionAttributeNameNotDefined,
+    IncorrectOperandType,
+    InvalidUpdateExpressionInvalidDocumentPath,
+)
+from moto.dynamodb2.models import DynamoType
+from moto.dynamodb2.parsing.ast_nodes import (
+    ExpressionAttribute,
+    UpdateExpressionPath,
+    UpdateExpressionSetAction,
+    UpdateExpressionAddAction,
+    UpdateExpressionDeleteAction,
+    UpdateExpressionRemoveAction,
+    DDBTypedValue,
+    ExpressionAttributeValue,
+    ExpressionAttributeName,
+    DepthFirstTraverser,
+    NoneExistingPath,
+    UpdateExpressionFunction,
+    ExpressionPathDescender,
+    UpdateExpressionValue,
+    ExpressionValueOperator,
+    ExpressionSelector,
+)
+from moto.dynamodb2.parsing.reserved_keywords import ReservedKeywords
+
+
+class ExpressionAttributeValueProcessor(DepthFirstTraverser):
+    def __init__(self, expression_attribute_values):
+        self.expression_attribute_values = expression_attribute_values
+
+    def _processing_map(self):
+        return {
+            ExpressionAttributeValue: self.replace_expression_attribute_value_with_value
+        }
+
+    def replace_expression_attribute_value_with_value(self, node):
+        """A node representing an Expression Attribute Value. Resolve and replace value"""
+        assert isinstance(node, ExpressionAttributeValue)
+        attribute_value_name = node.get_value_name()
+        try:
+            target = self.expression_attribute_values[attribute_value_name]
+        except KeyError:
+            raise ExpressionAttributeValueNotDefined(
+                attribute_value=attribute_value_name
+            )
+        return DDBTypedValue(DynamoType(target))
+
+
+class ExpressionAttributeResolvingProcessor(DepthFirstTraverser):
+    def _processing_map(self):
+        return {
+            UpdateExpressionSetAction: self.disable_resolving,
+            UpdateExpressionPath: self.process_expression_path_node,
+        }
+
+    def __init__(self, expression_attribute_names, item):
+        self.expression_attribute_names = expression_attribute_names
+        self.item = item
+        self.resolving = False
+
+    def pre_processing_of_child(self, parent_node, child_id):
+        """
+        We have to enable resolving if we are processing a child of UpdateExpressionSetAction that is not first.
+        Because first argument is path to be set, 2nd argument would be the value.
+        """
+        if isinstance(
+            parent_node,
+            (
+                UpdateExpressionSetAction,
+                UpdateExpressionRemoveAction,
+                UpdateExpressionDeleteAction,
+                UpdateExpressionAddAction,
+            ),
+        ):
+            if child_id == 0:
+                self.resolving = False
+            else:
+                self.resolving = True
+
+    def disable_resolving(self, node=None):
+        self.resolving = False
+        return node
+
+    def process_expression_path_node(self, node):
+        """Resolve ExpressionAttribute if not part of a path and resolving is enabled."""
+        if self.resolving:
+            return self.resolve_expression_path(node)
+        else:
+            # Still resolve but return original note to make sure path is correct Just make sure nodes are creatable.
+            result_node = self.resolve_expression_path(node)
+            if (
+                isinstance(result_node, NoneExistingPath)
+                and not result_node.is_creatable()
+            ):
+                raise InvalidUpdateExpressionInvalidDocumentPath()
+
+            return node
+
+    def resolve_expression_path(self, node):
+        assert isinstance(node, UpdateExpressionPath)
+
+        target = deepcopy(self.item.attrs)
+        for child in node.children:
+            # First replace placeholder with attribute_name
+            attr_name = None
+            if isinstance(child, ExpressionAttributeName):
+                attr_placeholder = child.get_attribute_name_placeholder()
+                try:
+                    attr_name = self.expression_attribute_names[attr_placeholder]
+                except KeyError:
+                    raise ExpressionAttributeNameNotDefined(attr_placeholder)
+            elif isinstance(child, ExpressionAttribute):
+                attr_name = child.get_attribute_name()
+                self.raise_exception_if_keyword(attr_name)
+            if attr_name is not None:
+                # Resolv attribute_name
+                try:
+                    target = target[attr_name]
+                except (KeyError, TypeError):
+                    if child == node.children[-1]:
+                        return NoneExistingPath(creatable=True)
+                    return NoneExistingPath()
+            else:
+                if isinstance(child, ExpressionPathDescender):
+                    continue
+                elif isinstance(child, ExpressionSelector):
+                    index = child.get_index()
+                    if target.is_list():
+                        try:
+                            target = target[index]
+                        except IndexError:
+                            # When a list goes out of bounds when assigning that is no problem when at the assignment
+                            # side. It will just append to the list.
+                            if child == node.children[-1]:
+                                return NoneExistingPath(creatable=True)
+                            return NoneExistingPath()
+                    else:
+                        raise InvalidUpdateExpressionInvalidDocumentPath
+                else:
+                    raise NotImplementedError(
+                        "Path resolution for {t}".format(t=type(child))
+                    )
+        return DDBTypedValue(DynamoType(target))
+
+    @classmethod
+    def raise_exception_if_keyword(cls, attribute):
+        if attribute.upper() in ReservedKeywords.get_reserved_keywords():
+            raise AttributeIsReservedKeyword(attribute)
+
+
+class UpdateExpressionFunctionEvaluator(DepthFirstTraverser):
+    """
+    At time of writing there are only 2 functions for DDB UpdateExpressions. They both are specific to the SET
+    expression as per the official AWS docs:
+        https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/
+        Expressions.UpdateExpressions.html#Expressions.UpdateExpressions.SET
+    """
+
+    def _processing_map(self):
+        return {UpdateExpressionFunction: self.process_function}
+
+    def process_function(self, node):
+        assert isinstance(node, UpdateExpressionFunction)
+        function_name = node.get_function_name()
+        first_arg = node.get_nth_argument(1)
+        second_arg = node.get_nth_argument(2)
+
+        if function_name == "if_not_exists":
+            if isinstance(first_arg, NoneExistingPath):
+                result = second_arg
+            else:
+                result = first_arg
+            assert isinstance(result, (DDBTypedValue, NoneExistingPath))
+            return result
+        elif function_name == "list_append":
+            first_arg = self.get_list_from_ddb_typed_value(first_arg, function_name)
+            second_arg = self.get_list_from_ddb_typed_value(second_arg, function_name)
+            for list_element in second_arg.value:
+                first_arg.value.append(list_element)
+            return DDBTypedValue(first_arg)
+        else:
+            raise NotImplementedError(
+                "Unsupported function for moto {name}".format(name=function_name)
+            )
+
+    @classmethod
+    def get_list_from_ddb_typed_value(cls, node, function_name):
+        assert isinstance(node, DDBTypedValue)
+        dynamo_value = node.get_value()
+        assert isinstance(dynamo_value, DynamoType)
+        if not dynamo_value.is_list():
+            raise IncorrectOperandType(function_name, dynamo_value.type)
+        return dynamo_value
+
+
+class NoneExistingPathChecker(DepthFirstTraverser):
+    """
+    Pass through the AST and make sure there are no none-existing paths.
+    """
+
+    def _processing_map(self):
+        return {NoneExistingPath: self.raise_none_existing_path}
+
+    def raise_none_existing_path(self, node):
+        raise AttributeDoesNotExist
+
+
+class ExecuteOperations(DepthFirstTraverser):
+    def _processing_map(self):
+        return {UpdateExpressionValue: self.process_update_expression_value}
+
+    def process_update_expression_value(self, node):
+        """
+        If an UpdateExpressionValue only has a single child the node will be replaced with the childe.
+        Otherwise it has 3 children and the middle one is an ExpressionValueOperator which details how to combine them
+        Args:
+            node(Node):
+
+        Returns:
+            Node: The resulting node of the operation if present or the child.
+        """
+        assert isinstance(node, UpdateExpressionValue)
+        if len(node.children) == 1:
+            return node.children[0]
+        elif len(node.children) == 3:
+            operator_node = node.children[1]
+            assert isinstance(operator_node, ExpressionValueOperator)
+            operator = operator_node.get_operator()
+            left_operand = self.get_dynamo_value_from_ddb_typed_value(node.children[0])
+            right_operand = self.get_dynamo_value_from_ddb_typed_value(node.children[2])
+            if operator == "+":
+                return self.get_sum(left_operand, right_operand)
+            elif operator == "-":
+                return self.get_subtraction(left_operand, right_operand)
+            else:
+                raise NotImplementedError(
+                    "Moto does not support operator {operator}".format(
+                        operator=operator
+                    )
+                )
+        else:
+            raise NotImplementedError(
+                "UpdateExpressionValue only has implementations for 1 or 3 children."
+            )
+
+    @classmethod
+    def get_dynamo_value_from_ddb_typed_value(cls, node):
+        assert isinstance(node, DDBTypedValue)
+        dynamo_value = node.get_value()
+        assert isinstance(dynamo_value, DynamoType)
+        return dynamo_value
+
+    @classmethod
+    def get_sum(cls, left_operand, right_operand):
+        """
+        Args:
+            left_operand(DynamoType):
+            right_operand(DynamoType):
+
+        Returns:
+            DDBTypedValue:
+        """
+        try:
+            return DDBTypedValue(left_operand + right_operand)
+        except TypeError:
+            raise IncorrectOperandType("+", left_operand.type)
+
+    @classmethod
+    def get_subtraction(cls, left_operand, right_operand):
+        """
+        Args:
+            left_operand(DynamoType):
+            right_operand(DynamoType):
+
+        Returns:
+            DDBTypedValue:
+        """
+        try:
+            return DDBTypedValue(left_operand - right_operand)
+        except TypeError:
+            raise IncorrectOperandType("-", left_operand.type)
+
+
+class Validator(object):
+    """
+    A validator is used to validate expressions which are passed in as an AST.
+    """
+
+    def __init__(
+        self, expression, expression_attribute_names, expression_attribute_values, item
+    ):
+        """
+        Besides validation the Validator should also replace referenced parts of an item which is cheapest upon
+        validation.
+
+        Args:
+            expression(Node): The root node of the AST representing the expression to be validated
+            expression_attribute_names(ExpressionAttributeNames):
+            expression_attribute_values(ExpressionAttributeValues):
+            item(Item): The item which will be updated (pointed to by Key of update_item)
+        """
+        self.expression_attribute_names = expression_attribute_names
+        self.expression_attribute_values = expression_attribute_values
+        self.item = item
+        self.processors = self.get_ast_processors()
+        self.node_to_validate = deepcopy(expression)
+
+    @abstractmethod
+    def get_ast_processors(self):
+        """Get the different processors that go through the AST tree and processes the nodes."""
+
+    def validate(self):
+        n = self.node_to_validate
+        for processor in self.processors:
+            n = processor.traverse(n)
+        return n
+
+
+class UpdateExpressionValidator(Validator):
+    def get_ast_processors(self):
+        """Get the different processors that go through the AST tree and processes the nodes."""
+        processors = [
+            ExpressionAttributeValueProcessor(self.expression_attribute_values),
+            ExpressionAttributeResolvingProcessor(
+                self.expression_attribute_names, self.item
+            ),
+            UpdateExpressionFunctionEvaluator(),
+            NoneExistingPathChecker(),
+            ExecuteOperations(),
+        ]
+        return processors

moto/dynamodb2/responses.py

@@ -9,7 +9,7 @@ import six
 
 from moto.core.responses import BaseResponse
 from moto.core.utils import camelcase_to_underscores, amzn_request_id
-from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge
+from .exceptions import InvalidIndexNameError, ItemSizeTooLarge, MockValidationException
 from moto.dynamodb2.models import dynamodb_backends, dynamo_json_dump
 
 
@@ -306,7 +306,7 @@ class DynamoHandler(BaseResponse):
             )
         except ItemSizeTooLarge:
             er = "com.amazonaws.dynamodb.v20111205#ValidationException"
-            return self.error(er, ItemSizeTooLarge.message)
+            return self.error(er, ItemSizeTooLarge.item_size_too_large_msg)
         except KeyError as ke:
             er = "com.amazonaws.dynamodb.v20111205#ValidationException"
             return self.error(er, ke.args[0])
@@ -756,11 +756,6 @@ class DynamoHandler(BaseResponse):
         expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
         expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
 
-        # Support spaces between operators in an update expression
-        # E.g. `a = b + c` -> `a=b+c`
-        if update_expression:
-            update_expression = re.sub(r"\s*([=\+-])\s*", "\\1", update_expression)
-
         try:
             item = self.dynamodb_backend.update_item(
                 name,
@@ -772,15 +767,9 @@ class DynamoHandler(BaseResponse):
                 expected,
                 condition_expression,
             )
-        except InvalidUpdateExpression:
+        except MockValidationException as mve:
             er = "com.amazonaws.dynamodb.v20111205#ValidationException"
-            return self.error(
+            return self.error(er, mve.exception_msg)
-                er,
-                "The document path provided in the update expression is invalid for update",
-            )
-        except ItemSizeTooLarge:
-            er = "com.amazonaws.dynamodb.v20111205#ValidationException"
-            return self.error(er, ItemSizeTooLarge.message)
         except ValueError:
             er = "com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException"
             return self.error(

moto/ec2/urls.py

@@ -2,6 +2,6 @@ from __future__ import unicode_literals
 from .responses import EC2Response
 
 
-url_bases = ["https?://ec2\.(.+)\.amazonaws\.com(|\.cn)"]
+url_bases = [r"https?://ec2\.(.+)\.amazonaws\.com(|\.cn)"]
 
 url_paths = {"{0}/": EC2Response.dispatch}

moto/elb/models.py

@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
 import datetime
+
+import pytz
+
 from boto.ec2.elb.attributes import (
     LbAttributes,
     ConnectionSettingAttribute,
@@ -83,7 +86,7 @@ class FakeLoadBalancer(BaseModel):
         self.zones = zones
         self.listeners = []
         self.backends = []
-        self.created_time = datetime.datetime.now()
+        self.created_time = datetime.datetime.now(pytz.utc)
         self.scheme = scheme
         self.attributes = FakeLoadBalancer.get_default_attributes()
         self.policies = Policies()

moto/elb/responses.py

@@ -442,7 +442,7 @@ DESCRIBE_LOAD_BALANCERS_TEMPLATE = """<DescribeLoadBalancersResponse xmlns="http
             {% endfor %}
           </SecurityGroups>
           <LoadBalancerName>{{ load_balancer.name }}</LoadBalancerName>
-          <CreatedTime>{{ load_balancer.created_time }}</CreatedTime>
+          <CreatedTime>{{ load_balancer.created_time.isoformat() }}</CreatedTime>
           <HealthCheck>
             {% if load_balancer.health_check %}
               <Interval>{{ load_balancer.health_check.interval }}</Interval>

moto/glue/models.py

@@ -34,6 +34,9 @@ class GlueBackend(BaseBackend):
         except KeyError:
             raise DatabaseNotFoundException(database_name)
 
+    def get_databases(self):
+        return [self.databases[key] for key in self.databases] if self.databases else []
+
     def create_table(self, database_name, table_name, table_input):
         database = self.get_database(database_name)
 

moto/glue/responses.py

@@ -30,6 +30,12 @@ class GlueResponse(BaseResponse):
         database = self.glue_backend.get_database(database_name)
         return json.dumps({"Database": {"Name": database.name}})
 
+    def get_databases(self):
+        database_list = self.glue_backend.get_databases()
+        return json.dumps(
+            {"DatabaseList": [{"Name": database.name} for database in database_list]}
+        )
+
     def create_table(self):
         database_name = self.parameters.get("DatabaseName")
         table_input = self.parameters.get("TableInput")

moto/iot/exceptions.py

@@ -7,10 +7,10 @@ class IoTClientError(JsonRESTError):
 
 
 class ResourceNotFoundException(IoTClientError):
-    def __init__(self):
+    def __init__(self, msg=None):
         self.code = 404
         super(ResourceNotFoundException, self).__init__(
-            "ResourceNotFoundException", "The specified resource does not exist"
+            "ResourceNotFoundException", msg or "The specified resource does not exist"
         )
 
 

moto/iot/models.py

@@ -805,6 +805,14 @@ class IoTBackend(BaseBackend):
         return thing_names
 
     def list_thing_principals(self, thing_name):
+
+        things = [_ for _ in self.things.values() if _.thing_name == thing_name]
+        if len(things) == 0:
+            raise ResourceNotFoundException(
+                "Failed to list principals for thing %s because the thing does not exist in your account"
+                % thing_name
+            )
+
         principals = [
             k[0] for k, v in self.principal_things.items() if k[1] == thing_name
         ]

moto/s3/responses.py

@@ -7,7 +7,7 @@ import six
 from botocore.awsrequest import AWSPreparedRequest
 
 from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys
-from six.moves.urllib.parse import parse_qs, urlparse, unquote
+from six.moves.urllib.parse import parse_qs, urlparse, unquote, parse_qsl
 
 import xmltodict
 
@@ -777,6 +777,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
             return 409, {}, template.render(bucket=removed_bucket)
 
     def _bucket_response_post(self, request, body, bucket_name):
+        response_headers = {}
         if not request.headers.get("Content-Length"):
             return 411, {}, "Content-Length required"
 
@@ -798,11 +799,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
         else:
             # HTTPretty, build new form object
             body = body.decode()
-
+            form = dict(parse_qsl(body))
-            form = {}
-            for kv in body.split("&"):
-                k, v = kv.split("=")
-                form[k] = v
 
         key = form["key"]
         if "file" in form:
@@ -810,13 +807,23 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
         else:
             f = request.files["file"].stream.read()
 
+        if "success_action_redirect" in form:
+            response_headers["Location"] = form["success_action_redirect"]
+
+        if "success_action_status" in form:
+            status_code = form["success_action_status"]
+        elif "success_action_redirect" in form:
+            status_code = 303
+        else:
+            status_code = 204
+
         new_key = self.backend.set_key(bucket_name, key, f)
 
         # Metadata
         metadata = metadata_from_headers(form)
         new_key.set_metadata(metadata)
 
-        return 200, {}, ""
+        return status_code, response_headers, ""
 
     @staticmethod
     def _get_path(request):

moto/ssm/models.py

@@ -651,7 +651,7 @@ class SimpleSystemManagerBackend(BaseBackend):
                 label.startswith("aws")
                 or label.startswith("ssm")
                 or label[:1].isdigit()
-                or not re.match("^[a-zA-z0-9_\.\-]*$", label)
+                or not re.match(r"^[a-zA-z0-9_\.\-]*$", label)
             ):
                 invalid_labels.append(label)
                 continue

setup.py

@@ -94,10 +94,12 @@ setup(
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
         "License :: OSI Approved :: Apache Software License",
         "Topic :: Software Development :: Testing",
     ],
     project_urls={
         "Documentation": "http://docs.getmoto.org/en/latest/",
     },
+    data_files=[('', ['moto/dynamodb2/parsing/reserved_keywords.txt'])],
 )

tests/test_dynamodb2/test_dynamodb.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals, print_function
 
+import re
 from decimal import Decimal
 
 import six
@@ -2146,13 +2147,33 @@ def test_update_item_on_map():
     # Nonexistent nested attributes are supported for existing top-level attributes.
     table.update_item(
         Key={"forum_name": "the-key", "subject": "123"},
-        UpdateExpression="SET body.#nested.#data = :tb, body.nested.#nonexistentnested.#data = :tb2",
+        UpdateExpression="SET body.#nested.#data = :tb",
+        ExpressionAttributeNames={"#nested": "nested", "#data": "data",},
+        ExpressionAttributeValues={":tb": "new_value"},
+    )
+    # Running this against AWS DDB gives an exception so make sure it also fails.:
+    with assert_raises(client.exceptions.ClientError):
+        # botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the UpdateItem
+        # operation: The document path provided in the update expression is invalid for update
+        table.update_item(
+            Key={"forum_name": "the-key", "subject": "123"},
+            UpdateExpression="SET body.#nested.#nonexistentnested.#data = :tb2",
+            ExpressionAttributeNames={
+                "#nested": "nested",
+                "#nonexistentnested": "nonexistentnested",
+                "#data": "data",
+            },
+            ExpressionAttributeValues={":tb2": "other_value"},
+        )
+
+    table.update_item(
+        Key={"forum_name": "the-key", "subject": "123"},
+        UpdateExpression="SET body.#nested.#nonexistentnested = :tb2",
         ExpressionAttributeNames={
             "#nested": "nested",
             "#nonexistentnested": "nonexistentnested",
-            "#data": "data",
         },
-        ExpressionAttributeValues={":tb": "new_value", ":tb2": "other_value"},
+        ExpressionAttributeValues={":tb2": {"data": "other_value"}},
     )
 
     resp = table.scan()
@@ -2160,8 +2181,8 @@ def test_update_item_on_map():
         {"nested": {"data": "new_value", "nonexistentnested": {"data": "other_value"}}}
     )
 
-    # Test nested value for a nonexistent attribute.
+    # Test nested value for a nonexistent attribute throws a ClientError.
-    with assert_raises(client.exceptions.ConditionalCheckFailedException):
+    with assert_raises(client.exceptions.ClientError):
         table.update_item(
             Key={"forum_name": "the-key", "subject": "123"},
             UpdateExpression="SET nonexistent.#nested = :tb",
@@ -3183,7 +3204,10 @@ def test_remove_top_level_attribute():
         TableName=table_name, Item={"id": {"S": "foo"}, "item": {"S": "bar"}}
     )
     client.update_item(
-        TableName=table_name, Key={"id": {"S": "foo"}}, UpdateExpression="REMOVE item"
+        TableName=table_name,
+        Key={"id": {"S": "foo"}},
+        UpdateExpression="REMOVE #i",
+        ExpressionAttributeNames={"#i": "item"},
     )
     #
     result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"]
@@ -3358,21 +3382,21 @@ def test_item_size_is_under_400KB():
     assert_failure_due_to_item_size(
         func=client.put_item,
         TableName="moto-test",
-        Item={"id": {"S": "foo"}, "item": {"S": large_item}},
+        Item={"id": {"S": "foo"}, "cont": {"S": large_item}},
     )
     assert_failure_due_to_item_size(
-        func=table.put_item, Item={"id": "bar", "item": large_item}
+        func=table.put_item, Item={"id": "bar", "cont": large_item}
     )
-    assert_failure_due_to_item_size(
+    assert_failure_due_to_item_size_to_update(
         func=client.update_item,
         TableName="moto-test",
         Key={"id": {"S": "foo2"}},
-        UpdateExpression="set item=:Item",
+        UpdateExpression="set cont=:Item",
         ExpressionAttributeValues={":Item": {"S": large_item}},
     )
     # Assert op fails when updating a nested item
     assert_failure_due_to_item_size(
-        func=table.put_item, Item={"id": "bar", "itemlist": [{"item": large_item}]}
+        func=table.put_item, Item={"id": "bar", "itemlist": [{"cont": large_item}]}
     )
     assert_failure_due_to_item_size(
         func=client.put_item,
@@ -3393,6 +3417,15 @@ def assert_failure_due_to_item_size(func, **kwargs):
     )
 
 
+def assert_failure_due_to_item_size_to_update(func, **kwargs):
+    with assert_raises(ClientError) as ex:
+        func(**kwargs)
+    ex.exception.response["Error"]["Code"].should.equal("ValidationException")
+    ex.exception.response["Error"]["Message"].should.equal(
+        "Item size to update has exceeded the maximum allowed size"
+    )
+
+
 @mock_dynamodb2
 # https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html#DDB-Query-request-KeyConditionExpression
 def test_hash_key_cannot_use_begins_with_operations():
@@ -4179,6 +4212,73 @@ def test_gsi_verify_negative_number_order():
     )
 
 
+def assert_raise_syntax_error(client_error, token, near):
+    """
+    Assert whether a client_error is as expected Syntax error. Syntax error looks like: `syntax_error_template`
+
+    Args:
+        client_error(ClientError): The ClientError exception that was raised
+        token(str): The token that ws unexpected
+        near(str): The part in the expression that shows where the error occurs it generally has the preceding token the
+        optional separation and the problematic token.
+    """
+    syntax_error_template = (
+        'Invalid UpdateExpression: Syntax error; token: "{token}", near: "{near}"'
+    )
+    expected_syntax_error = syntax_error_template.format(token=token, near=near)
+    assert client_error.response["Error"]["Code"] == "ValidationException"
+    assert expected_syntax_error == client_error.response["Error"]["Message"]
+
+
+@mock_dynamodb2
+def test_update_expression_with_numeric_literal_instead_of_value():
+    """
+    DynamoDB requires literals to be passed in as values. If they are put literally in the expression a token error will
+    be raised
+    """
+    dynamodb = boto3.client("dynamodb", region_name="eu-west-1")
+
+    dynamodb.create_table(
+        TableName="moto-test",
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
+    )
+
+    try:
+        dynamodb.update_item(
+            TableName="moto-test",
+            Key={"id": {"S": "1"}},
+            UpdateExpression="SET MyStr = myNum + 1",
+        )
+        assert False, "Validation exception not thrown"
+    except dynamodb.exceptions.ClientError as e:
+        assert_raise_syntax_error(e, "1", "+ 1")
+
+
+@mock_dynamodb2
+def test_update_expression_with_multiple_set_clauses_must_be_comma_separated():
+    """
+    An UpdateExpression can have multiple set clauses but if they are passed in without the separating comma.
+    """
+    dynamodb = boto3.client("dynamodb", region_name="eu-west-1")
+
+    dynamodb.create_table(
+        TableName="moto-test",
+        KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
+    )
+
+    try:
+        dynamodb.update_item(
+            TableName="moto-test",
+            Key={"id": {"S": "1"}},
+            UpdateExpression="SET MyStr = myNum Mystr2 myNum2",
+        )
+        assert False, "Validation exception not thrown"
+    except dynamodb.exceptions.ClientError as e:
+        assert_raise_syntax_error(e, "Mystr2", "myNum Mystr2 myNum2")
+
+
 @mock_dynamodb2
 def test_list_tables_exclusive_start_table_name_empty():
     client = boto3.client("dynamodb", region_name="us-east-1")

tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py

@@ -0,0 +1,259 @@
+from moto.dynamodb2.exceptions import (
+    InvalidTokenException,
+    InvalidExpressionAttributeNameKey,
+)
+from moto.dynamodb2.parsing.tokens import ExpressionTokenizer, Token
+
+
+def test_expression_tokenizer_single_set_action():
+    set_action = "SET attrName = :attrValue"
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE, "attrName"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
+    ]
+
+
+def test_expression_tokenizer_single_set_action_leading_space():
+    set_action = "Set attrName = :attrValue"
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "Set"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE, "attrName"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
+    ]
+
+
+def test_expression_tokenizer_single_set_action_attribute_name_leading_space():
+    set_action = "SET #a = :attrValue"
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE_NAME, "#a"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
+    ]
+
+
+def test_expression_tokenizer_single_set_action_trailing_space():
+    set_action = "SET attrName = :attrValue "
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE, "attrName"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
+        Token(Token.WHITESPACE, " "),
+    ]
+
+
+def test_expression_tokenizer_single_set_action_multi_spaces():
+    set_action = "SET    attrName    =  :attrValue  "
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, "    "),
+        Token(Token.ATTRIBUTE, "attrName"),
+        Token(Token.WHITESPACE, "    "),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.WHITESPACE, "  "),
+        Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
+        Token(Token.WHITESPACE, "  "),
+    ]
+
+
+def test_expression_tokenizer_single_set_action_with_numbers_in_identifiers():
+    set_action = "SET attrName3 = :attr3Value"
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE, "attrName3"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE_VALUE, ":attr3Value"),
+    ]
+
+
+def test_expression_tokenizer_single_set_action_with_underscore_in_identifier():
+    set_action = "SET attr_Name = :attr_Value"
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE, "attr_Name"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE_VALUE, ":attr_Value"),
+    ]
+
+
+def test_expression_tokenizer_leading_underscore_in_attribute_name_expression():
+    """Leading underscore is not allowed for an attribute name"""
+    set_action = "SET attrName = _idid"
+    try:
+        ExpressionTokenizer.make_list(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "_"
+        assert te.near == "= _idid"
+
+
+def test_expression_tokenizer_leading_underscore_in_attribute_value_expression():
+    """Leading underscore is allowed in an attribute value"""
+    set_action = "SET attrName = :_attrValue"
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE, "attrName"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE_VALUE, ":_attrValue"),
+    ]
+
+
+def test_expression_tokenizer_single_set_action_nested_attribute():
+    set_action = "SET attrName.elem = :attrValue"
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE, "attrName"),
+        Token(Token.DOT, "."),
+        Token(Token.ATTRIBUTE, "elem"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
+    ]
+
+
+def test_expression_tokenizer_list_index_with_sub_attribute():
+    set_action = "SET itemmap.itemlist[1].foos=:Item"
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE, "itemmap"),
+        Token(Token.DOT, "."),
+        Token(Token.ATTRIBUTE, "itemlist"),
+        Token(Token.OPEN_SQUARE_BRACKET, "["),
+        Token(Token.NUMBER, "1"),
+        Token(Token.CLOSE_SQUARE_BRACKET, "]"),
+        Token(Token.DOT, "."),
+        Token(Token.ATTRIBUTE, "foos"),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.ATTRIBUTE_VALUE, ":Item"),
+    ]
+
+
+def test_expression_tokenizer_list_index_surrounded_with_whitespace():
+    set_action = "SET itemlist[ 1  ]=:Item"
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE, "itemlist"),
+        Token(Token.OPEN_SQUARE_BRACKET, "["),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.NUMBER, "1"),
+        Token(Token.WHITESPACE, "  "),
+        Token(Token.CLOSE_SQUARE_BRACKET, "]"),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.ATTRIBUTE_VALUE, ":Item"),
+    ]
+
+
+def test_expression_tokenizer_single_set_action_attribute_name_invalid_key():
+    """
+    ExpressionAttributeNames contains invalid key: Syntax error; key: "#va#l2"
+    """
+    set_action = "SET #va#l2 = 3"
+    try:
+        ExpressionTokenizer.make_list(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidExpressionAttributeNameKey as e:
+        assert e.key == "#va#l2"
+
+
+def test_expression_tokenizer_single_set_action_attribute_name_invalid_key_double_hash():
+    """
+    ExpressionAttributeNames contains invalid key: Syntax error; key: "#va#l"
+    """
+    set_action = "SET #va#l = 3"
+    try:
+        ExpressionTokenizer.make_list(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidExpressionAttributeNameKey as e:
+        assert e.key == "#va#l"
+
+
+def test_expression_tokenizer_single_set_action_attribute_name_valid_key():
+    set_action = "SET attr=#val2"
+    token_list = ExpressionTokenizer.make_list(set_action)
+    assert token_list == [
+        Token(Token.ATTRIBUTE, "SET"),
+        Token(Token.WHITESPACE, " "),
+        Token(Token.ATTRIBUTE, "attr"),
+        Token(Token.EQUAL_SIGN, "="),
+        Token(Token.ATTRIBUTE_NAME, "#val2"),
+    ]
+
+
+def test_expression_tokenizer_just_a_pipe():
+    set_action = "|"
+    try:
+        ExpressionTokenizer.make_list(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "|"
+        assert te.near == "|"
+
+
+def test_expression_tokenizer_just_a_pipe_with_leading_white_spaces():
+    set_action = "   |"
+    try:
+        ExpressionTokenizer.make_list(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "|"
+        assert te.near == "   |"
+
+
+def test_expression_tokenizer_just_a_pipe_for_set_expression():
+    set_action = "SET|"
+    try:
+        ExpressionTokenizer.make_list(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "|"
+        assert te.near == "SET|"
+
+
+def test_expression_tokenizer_just_an_attribute_and_a_pipe_for_set_expression():
+    set_action = "SET a|"
+    try:
+        ExpressionTokenizer.make_list(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "|"
+        assert te.near == "a|"

tests/test_dynamodb2/test_dynamodb_expressions.py

@@ -0,0 +1,405 @@
+from moto.dynamodb2.exceptions import InvalidTokenException
+from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
+from moto.dynamodb2.parsing.reserved_keywords import ReservedKeywords
+
+
+def test_get_reserved_keywords():
+    reserved_keywords = ReservedKeywords.get_reserved_keywords()
+    assert "SET" in reserved_keywords
+    assert "DELETE" in reserved_keywords
+    assert "ADD" in reserved_keywords
+    # REMOVE is not part of the list of reserved keywords.
+    assert "REMOVE" not in reserved_keywords
+
+
+def test_update_expression_numeric_literal_in_expression():
+    set_action = "SET attrName = 3"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "3"
+        assert te.near == "= 3"
+
+
+def test_expression_tokenizer_multi_number_numeric_literal_in_expression():
+    set_action = "SET attrName = 34"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "34"
+        assert te.near == "= 34"
+
+
+def test_expression_tokenizer_numeric_literal_unclosed_square_bracket():
+    set_action = "SET MyStr[ 3"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "<EOF>"
+        assert te.near == "3"
+
+
+def test_expression_tokenizer_wrong_closing_bracket_with_space():
+    set_action = "SET MyStr[3 )"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == ")"
+        assert te.near == "3 )"
+
+
+def test_expression_tokenizer_wrong_closing_bracket():
+    set_action = "SET MyStr[3)"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == ")"
+        assert te.near == "3)"
+
+
+def test_expression_tokenizer_only_numeric_literal_for_set():
+    set_action = "SET 2"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "2"
+        assert te.near == "SET 2"
+
+
+def test_expression_tokenizer_only_numeric_literal():
+    set_action = "2"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "2"
+        assert te.near == "2"
+
+
+def test_expression_tokenizer_set_closing_round_bracket():
+    set_action = "SET )"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == ")"
+        assert te.near == "SET )"
+
+
+def test_expression_tokenizer_set_closing_followed_by_numeric_literal():
+    set_action = "SET ) 3"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == ")"
+        assert te.near == "SET ) 3"
+
+
+def test_expression_tokenizer_numeric_literal_unclosed_square_bracket_trailing_space():
+    set_action = "SET MyStr[ 3 "
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "<EOF>"
+        assert te.near == "3 "
+
+
+def test_expression_tokenizer_unbalanced_round_brackets_only_opening():
+    set_action = "SET MyStr = (:_val"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "<EOF>"
+        assert te.near == ":_val"
+
+
+def test_expression_tokenizer_unbalanced_round_brackets_only_opening_trailing_space():
+    set_action = "SET MyStr = (:_val "
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "<EOF>"
+        assert te.near == ":_val "
+
+
+def test_expression_tokenizer_unbalanced_square_brackets_only_opening():
+    set_action = "SET MyStr = [:_val"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "["
+        assert te.near == "= [:_val"
+
+
+def test_expression_tokenizer_unbalanced_square_brackets_only_opening_trailing_spaces():
+    set_action = "SET MyStr = [:_val  "
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "["
+        assert te.near == "= [:_val"
+
+
+def test_expression_tokenizer_unbalanced_round_brackets_multiple_opening():
+    set_action = "SET MyStr = (:_val + (:val2"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "<EOF>"
+        assert te.near == ":val2"
+
+
+def test_expression_tokenizer_unbalanced_round_brackets_only_closing():
+    set_action = "SET MyStr = ):_val"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == ")"
+        assert te.near == "= ):_val"
+
+
+def test_expression_tokenizer_unbalanced_square_brackets_only_closing():
+    set_action = "SET MyStr = ]:_val"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "]"
+        assert te.near == "= ]:_val"
+
+
+def test_expression_tokenizer_unbalanced_round_brackets_only_closing_followed_by_other_parts():
+    set_action = "SET MyStr = ):_val + :val2"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == ")"
+        assert te.near == "= ):_val"
+
+
+def test_update_expression_starts_with_keyword_reset_followed_by_identifier():
+    update_expression = "RESET NonExistent"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "RESET"
+        assert te.near == "RESET NonExistent"
+
+
+def test_update_expression_starts_with_keyword_reset_followed_by_identifier_and_value():
+    update_expression = "RESET NonExistent value"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "RESET"
+        assert te.near == "RESET NonExistent"
+
+
+def test_update_expression_starts_with_leading_spaces_and_keyword_reset_followed_by_identifier_and_value():
+    update_expression = "  RESET NonExistent value"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "RESET"
+        assert te.near == "  RESET NonExistent"
+
+
+def test_update_expression_with_only_keyword_reset():
+    update_expression = "RESET"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "RESET"
+        assert te.near == "RESET"
+
+
+def test_update_nested_expression_with_selector_just_should_fail_parsing_at_numeric_literal_value():
+    update_expression = "SET a[0].b = 5"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "5"
+        assert te.near == "= 5"
+
+
+def test_update_nested_expression_with_selector_and_spaces_should_only_fail_parsing_at_numeric_literal_value():
+    update_expression = "SET a [  2 ]. b = 5"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "5"
+        assert te.near == "= 5"
+
+
+def test_update_nested_expression_with_double_selector_and_spaces_should_only_fail_parsing_at_numeric_literal_value():
+    update_expression = "SET a [2][ 3  ]. b = 5"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "5"
+        assert te.near == "= 5"
+
+
+def test_update_nested_expression_should_only_fail_parsing_at_numeric_literal_value():
+    update_expression = "SET a . b = 5"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "5"
+        assert te.near == "= 5"
+
+
+def test_nested_selectors_in_update_expression_should_fail_at_nesting():
+    update_expression = "SET a [  [2] ]. b = 5"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "["
+        assert te.near == "[  [2"
+
+
+def test_update_expression_number_in_selector_cannot_be_splite():
+    update_expression = "SET a [2 1]. b = 5"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "1"
+        assert te.near == "2 1]"
+
+
+def test_update_expression_cannot_have_successive_attributes():
+    update_expression = "SET #a a = 5"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "a"
+        assert te.near == "#a a ="
+
+
+def test_update_expression_path_with_both_attribute_and_attribute_name_should_only_fail_at_numeric_value():
+    update_expression = "SET #a.a = 5"
+    try:
+        UpdateExpressionParser.make(update_expression)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "5"
+        assert te.near == "= 5"
+
+
+def test_expression_tokenizer_2_same_operators_back_to_back():
+    set_action = "SET MyStr = NoExist + + :_val "
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "+"
+        assert te.near == "+ + :_val"
+
+
+def test_expression_tokenizer_2_different_operators_back_to_back():
+    set_action = "SET MyStr = NoExist + - :_val "
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "-"
+        assert te.near == "+ - :_val"
+
+
+def test_update_expression_remove_does_not_allow_operations():
+    remove_action = "REMOVE NoExist + "
+    try:
+        UpdateExpressionParser.make(remove_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "+"
+        assert te.near == "NoExist + "
+
+
+def test_update_expression_add_does_not_allow_attribute_after_path():
+    """value here is not really a value since a value starts with a colon (:)"""
+    add_expr = "ADD attr val foobar"
+    try:
+        UpdateExpressionParser.make(add_expr)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "val"
+        assert te.near == "attr val foobar"
+
+
+def test_update_expression_add_does_not_allow_attribute_foobar_after_value():
+    add_expr = "ADD attr :val foobar"
+    try:
+        UpdateExpressionParser.make(add_expr)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "foobar"
+        assert te.near == ":val foobar"
+
+
+def test_update_expression_delete_does_not_allow_attribute_after_path():
+    """value here is not really a value since a value starts with a colon (:)"""
+    delete_expr = "DELETE attr val"
+    try:
+        UpdateExpressionParser.make(delete_expr)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "val"
+        assert te.near == "attr val"
+
+
+def test_update_expression_delete_does_not_allow_attribute_foobar_after_value():
+    delete_expr = "DELETE attr :val foobar"
+    try:
+        UpdateExpressionParser.make(delete_expr)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "foobar"
+        assert te.near == ":val foobar"
+
+
+def test_update_expression_parsing_is_not_keyword_aware():
+    """path and VALUE are keywords. Yet a token error will be thrown for the numeric literal 1."""
+    delete_expr = "SET path = VALUE 1"
+    try:
+        UpdateExpressionParser.make(delete_expr)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "1"
+        assert te.near == "VALUE 1"
+
+
+def test_expression_if_not_exists_is_not_valid_in_remove_statement():
+    set_action = "REMOVE if_not_exists(a,b)"
+    try:
+        UpdateExpressionParser.make(set_action)
+        assert False, "Exception not raised correctly"
+    except InvalidTokenException as te:
+        assert te.token == "("
+        assert te.near == "if_not_exists(a"

tests/test_dynamodb2/test_dynamodb_table_with_range_key.py

@@ -1254,14 +1254,22 @@ def test_update_item_with_expression():
 
     item_key = {"forum_name": "the-key", "subject": "123"}
 
-    table.update_item(Key=item_key, UpdateExpression="SET field=2")
+    table.update_item(
+        Key=item_key,
+        UpdateExpression="SET field = :field_value",
+        ExpressionAttributeValues={":field_value": 2},
+    )
     dict(table.get_item(Key=item_key)["Item"]).should.equal(
-        {"field": "2", "forum_name": "the-key", "subject": "123"}
+        {"field": Decimal("2"), "forum_name": "the-key", "subject": "123"}
     )
 
-    table.update_item(Key=item_key, UpdateExpression="SET field  = 3")
+    table.update_item(
+        Key=item_key,
+        UpdateExpression="SET field = :field_value",
+        ExpressionAttributeValues={":field_value": 3},
+    )
     dict(table.get_item(Key=item_key)["Item"]).should.equal(
-        {"field": "3", "forum_name": "the-key", "subject": "123"}
+        {"field": Decimal("3"), "forum_name": "the-key", "subject": "123"}
     )
 
 

tests/test_dynamodb2/test_dynamodb_table_without_range_key.py

@@ -443,23 +443,40 @@ def test_update_item_nested_remove():
     dict(returned_item).should.equal({"username": "steve", "Meta": {}})
 
 
-@mock_dynamodb2_deprecated
+@mock_dynamodb2
 def test_update_item_double_nested_remove():
-    conn = boto.dynamodb2.connect_to_region("us-east-1")
+    conn = boto3.client("dynamodb", region_name="us-east-1")
-    table = Table.create("messages", schema=[HashKey("username")])
+    conn.create_table(
+        TableName="messages",
+        KeySchema=[{"AttributeName": "username", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "username", "AttributeType": "S"}],
+        BillingMode="PAY_PER_REQUEST",
+    )
 
-    data = {"username": "steve", "Meta": {"Name": {"First": "Steve", "Last": "Urkel"}}}
+    item = {
-    table.put_item(data=data)
+        "username": {"S": "steve"},
+        "Meta": {
+            "M": {"Name": {"M": {"First": {"S": "Steve"}, "Last": {"S": "Urkel"}}}}
+        },
+    }
+    conn.put_item(TableName="messages", Item=item)
     key_map = {"username": {"S": "steve"}}
 
     # Then remove the Meta.FullName field
-    conn.update_item("messages", key_map, update_expression="REMOVE Meta.Name.First")
+    conn.update_item(
-
+        TableName="messages",
-    returned_item = table.get_item(username="steve")
+        Key=key_map,
-    dict(returned_item).should.equal(
+        UpdateExpression="REMOVE Meta.#N.#F",
-        {"username": "steve", "Meta": {"Name": {"Last": "Urkel"}}}
+        ExpressionAttributeNames={"#N": "Name", "#F": "First"},
     )
 
+    returned_item = conn.get_item(TableName="messages", Key=key_map)
+    expected_item = {
+        "username": {"S": "steve"},
+        "Meta": {"M": {"Name": {"M": {"Last": {"S": "Urkel"}}}}},
+    }
+    dict(returned_item["Item"]).should.equal(expected_item)
+
 
 @mock_dynamodb2_deprecated
 def test_update_item_set():
@@ -471,7 +488,10 @@ def test_update_item_set():
     key_map = {"username": {"S": "steve"}}
 
     conn.update_item(
-        "messages", key_map, update_expression="SET foo=bar, blah=baz REMOVE SentBy"
+        "messages",
+        key_map,
+        update_expression="SET foo=:bar, blah=:baz REMOVE SentBy",
+        expression_attribute_values={":bar": {"S": "bar"}, ":baz": {"S": "baz"}},
     )
 
     returned_item = table.get_item(username="steve")
@@ -616,8 +636,9 @@ def test_boto3_update_item_conditions_fail():
     table.put_item(Item={"username": "johndoe", "foo": "baz"})
     table.update_item.when.called_with(
         Key={"username": "johndoe"},
-        UpdateExpression="SET foo=bar",
+        UpdateExpression="SET foo=:bar",
         Expected={"foo": {"Value": "bar"}},
+        ExpressionAttributeValues={":bar": "bar"},
     ).should.throw(botocore.client.ClientError)
 
 
@@ -627,8 +648,9 @@ def test_boto3_update_item_conditions_fail_because_expect_not_exists():
     table.put_item(Item={"username": "johndoe", "foo": "baz"})
     table.update_item.when.called_with(
         Key={"username": "johndoe"},
-        UpdateExpression="SET foo=bar",
+        UpdateExpression="SET foo=:bar",
         Expected={"foo": {"Exists": False}},
+        ExpressionAttributeValues={":bar": "bar"},
     ).should.throw(botocore.client.ClientError)
 
 
@@ -638,8 +660,9 @@ def test_boto3_update_item_conditions_fail_because_expect_not_exists_by_compare_
     table.put_item(Item={"username": "johndoe", "foo": "baz"})
     table.update_item.when.called_with(
         Key={"username": "johndoe"},
-        UpdateExpression="SET foo=bar",
+        UpdateExpression="SET foo=:bar",
         Expected={"foo": {"ComparisonOperator": "NULL"}},
+        ExpressionAttributeValues={":bar": "bar"},
     ).should.throw(botocore.client.ClientError)
 
 
@@ -649,8 +672,9 @@ def test_boto3_update_item_conditions_pass():
     table.put_item(Item={"username": "johndoe", "foo": "bar"})
     table.update_item(
         Key={"username": "johndoe"},
-        UpdateExpression="SET foo=baz",
+        UpdateExpression="SET foo=:baz",
         Expected={"foo": {"Value": "bar"}},
+        ExpressionAttributeValues={":baz": "baz"},
     )
     returned_item = table.get_item(Key={"username": "johndoe"})
     assert dict(returned_item)["Item"]["foo"].should.equal("baz")
@@ -662,8 +686,9 @@ def test_boto3_update_item_conditions_pass_because_expect_not_exists():
     table.put_item(Item={"username": "johndoe", "foo": "bar"})
     table.update_item(
         Key={"username": "johndoe"},
-        UpdateExpression="SET foo=baz",
+        UpdateExpression="SET foo=:baz",
         Expected={"whatever": {"Exists": False}},
+        ExpressionAttributeValues={":baz": "baz"},
     )
     returned_item = table.get_item(Key={"username": "johndoe"})
     assert dict(returned_item)["Item"]["foo"].should.equal("baz")
@@ -675,8 +700,9 @@ def test_boto3_update_item_conditions_pass_because_expect_not_exists_by_compare_
     table.put_item(Item={"username": "johndoe", "foo": "bar"})
     table.update_item(
         Key={"username": "johndoe"},
-        UpdateExpression="SET foo=baz",
+        UpdateExpression="SET foo=:baz",
         Expected={"whatever": {"ComparisonOperator": "NULL"}},
+        ExpressionAttributeValues={":baz": "baz"},
     )
     returned_item = table.get_item(Key={"username": "johndoe"})
     assert dict(returned_item)["Item"]["foo"].should.equal("baz")
@@ -688,8 +714,9 @@ def test_boto3_update_item_conditions_pass_because_expect_exists_by_compare_to_n
     table.put_item(Item={"username": "johndoe", "foo": "bar"})
     table.update_item(
         Key={"username": "johndoe"},
-        UpdateExpression="SET foo=baz",
+        UpdateExpression="SET foo=:baz",
         Expected={"foo": {"ComparisonOperator": "NOT_NULL"}},
+        ExpressionAttributeValues={":baz": "baz"},
     )
     returned_item = table.get_item(Key={"username": "johndoe"})
     assert dict(returned_item)["Item"]["foo"].should.equal("baz")

tests/test_dynamodb2/test_dynamodb_validation.py

@@ -0,0 +1,464 @@
+from moto.dynamodb2.exceptions import (
+    AttributeIsReservedKeyword,
+    ExpressionAttributeValueNotDefined,
+    AttributeDoesNotExist,
+    ExpressionAttributeNameNotDefined,
+    IncorrectOperandType,
+    InvalidUpdateExpressionInvalidDocumentPath,
+)
+from moto.dynamodb2.models import Item, DynamoType
+from moto.dynamodb2.parsing.ast_nodes import (
+    NodeDepthLeftTypeFetcher,
+    UpdateExpressionSetAction,
+    UpdateExpressionValue,
+    DDBTypedValue,
+)
+from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
+from moto.dynamodb2.parsing.validators import UpdateExpressionValidator
+from parameterized import parameterized
+
+
+def test_validation_of_update_expression_with_keyword():
+    try:
+        update_expression = "SET myNum = path + :val"
+        update_expression_values = {":val": {"N": "3"}}
+        update_expression_ast = UpdateExpressionParser.make(update_expression)
+        item = Item(
+            hash_key=DynamoType({"S": "id"}),
+            hash_key_type="TYPE",
+            range_key=None,
+            range_key_type=None,
+            attrs={"id": {"S": "1"}, "path": {"N": "3"}},
+        )
+        UpdateExpressionValidator(
+            update_expression_ast,
+            expression_attribute_names=None,
+            expression_attribute_values=update_expression_values,
+            item=item,
+        ).validate()
+        assert False, "No exception raised"
+    except AttributeIsReservedKeyword as e:
+        assert e.keyword == "path"
+
+
+@parameterized(
+    ["SET a = #b + :val2", "SET a = :val2 + #b",]
+)
+def test_validation_of_a_set_statement_with_incorrect_passed_value(update_expression):
+    """
+    By running permutations it shows that values are replaced prior to resolving attributes.
+
+    An error occurred (ValidationException) when calling the UpdateItem operation: Invalid UpdateExpression:
+    An expression attribute value used in expression is not defined; attribute value: :val2
+    """
+    update_expression_ast = UpdateExpressionParser.make(update_expression)
+    item = Item(
+        hash_key=DynamoType({"S": "id"}),
+        hash_key_type="TYPE",
+        range_key=None,
+        range_key_type=None,
+        attrs={"id": {"S": "1"}, "b": {"N": "3"}},
+    )
+    try:
+        UpdateExpressionValidator(
+            update_expression_ast,
+            expression_attribute_names={"#b": "ok"},
+            expression_attribute_values={":val": {"N": "3"}},
+            item=item,
+        ).validate()
+    except ExpressionAttributeValueNotDefined as e:
+        assert e.attribute_value == ":val2"
+
+
+def test_validation_of_update_expression_with_attribute_that_does_not_exist_in_item():
+    """
+    When an update expression tries to get an attribute that does not exist it must throw the appropriate exception.
+
+    An error occurred (ValidationException) when calling the UpdateItem operation:
+    The provided expression refers to an attribute that does not exist in the item
+    """
+    try:
+        update_expression = "SET a = nonexistent"
+        update_expression_ast = UpdateExpressionParser.make(update_expression)
+        item = Item(
+            hash_key=DynamoType({"S": "id"}),
+            hash_key_type="TYPE",
+            range_key=None,
+            range_key_type=None,
+            attrs={"id": {"S": "1"}, "path": {"N": "3"}},
+        )
+        UpdateExpressionValidator(
+            update_expression_ast,
+            expression_attribute_names=None,
+            expression_attribute_values=None,
+            item=item,
+        ).validate()
+        assert False, "No exception raised"
+    except AttributeDoesNotExist:
+        assert True
+
+
+@parameterized(
+    ["SET a = #c", "SET a = #c + #d",]
+)
+def test_validation_of_update_expression_with_attribute_name_that_is_not_defined(
+    update_expression,
+):
+    """
+    When an update expression tries to get an attribute name that is not provided it must throw an exception.
+
+    An error occurred (ValidationException) when calling the UpdateItem operation: Invalid UpdateExpression:
+    An expression attribute name used in the document path is not defined; attribute name: #c
+    """
+    try:
+        update_expression_ast = UpdateExpressionParser.make(update_expression)
+        item = Item(
+            hash_key=DynamoType({"S": "id"}),
+            hash_key_type="TYPE",
+            range_key=None,
+            range_key_type=None,
+            attrs={"id": {"S": "1"}, "path": {"N": "3"}},
+        )
+        UpdateExpressionValidator(
+            update_expression_ast,
+            expression_attribute_names={"#b": "ok"},
+            expression_attribute_values=None,
+            item=item,
+        ).validate()
+        assert False, "No exception raised"
+    except ExpressionAttributeNameNotDefined as e:
+        assert e.not_defined_attribute_name == "#c"
+
+
+def test_validation_of_if_not_exists_not_existing_invalid_replace_value():
+    try:
+        update_expression = "SET a = if_not_exists(b, a.c)"
+        update_expression_ast = UpdateExpressionParser.make(update_expression)
+        item = Item(
+            hash_key=DynamoType({"S": "id"}),
+            hash_key_type="TYPE",
+            range_key=None,
+            range_key_type=None,
+            attrs={"id": {"S": "1"}, "a": {"S": "A"}},
+        )
+        UpdateExpressionValidator(
+            update_expression_ast,
+            expression_attribute_names=None,
+            expression_attribute_values=None,
+            item=item,
+        ).validate()
+        assert False, "No exception raised"
+    except AttributeDoesNotExist:
+        assert True
+
+
+def get_first_node_of_type(ast, node_type):
+    return next(NodeDepthLeftTypeFetcher(node_type, ast))
+
+
+def get_set_action_value(ast):
+    """
+    Helper that takes an AST and gets the first UpdateExpressionSetAction and retrieves the value of that action.
+    This should only be called on validated expressions.
+    Args:
+        ast(Node):
+
+    Returns:
+        DynamoType: The DynamoType object representing the Dynamo value.
+    """
+    set_action = get_first_node_of_type(ast, UpdateExpressionSetAction)
+    typed_value = set_action.children[1]
+    assert isinstance(typed_value, DDBTypedValue)
+    dynamo_value = typed_value.children[0]
+    assert isinstance(dynamo_value, DynamoType)
+    return dynamo_value
+
+
+def test_validation_of_if_not_exists_not_existing_value():
+    update_expression = "SET a = if_not_exists(b, a)"
+    update_expression_ast = UpdateExpressionParser.make(update_expression)
+    item = Item(
+        hash_key=DynamoType({"S": "id"}),
+        hash_key_type="TYPE",
+        range_key=None,
+        range_key_type=None,
+        attrs={"id": {"S": "1"}, "a": {"S": "A"}},
+    )
+    validated_ast = UpdateExpressionValidator(
+        update_expression_ast,
+        expression_attribute_names=None,
+        expression_attribute_values=None,
+        item=item,
+    ).validate()
+    dynamo_value = get_set_action_value(validated_ast)
+    assert dynamo_value == DynamoType({"S": "A"})
+
+
+def test_validation_of_if_not_exists_with_existing_attribute_should_return_attribute():
+    update_expression = "SET a = if_not_exists(b, a)"
+    update_expression_ast = UpdateExpressionParser.make(update_expression)
+    item = Item(
+        hash_key=DynamoType({"S": "id"}),
+        hash_key_type="TYPE",
+        range_key=None,
+        range_key_type=None,
+        attrs={"id": {"S": "1"}, "a": {"S": "A"}, "b": {"S": "B"}},
+    )
+    validated_ast = UpdateExpressionValidator(
+        update_expression_ast,
+        expression_attribute_names=None,
+        expression_attribute_values=None,
+        item=item,
+    ).validate()
+    dynamo_value = get_set_action_value(validated_ast)
+    assert dynamo_value == DynamoType({"S": "B"})
+
+
+def test_validation_of_if_not_exists_with_existing_attribute_should_return_value():
+    update_expression = "SET a = if_not_exists(b, :val)"
+    update_expression_values = {":val": {"N": "4"}}
+    update_expression_ast = UpdateExpressionParser.make(update_expression)
+    item = Item(
+        hash_key=DynamoType({"S": "id"}),
+        hash_key_type="TYPE",
+        range_key=None,
+        range_key_type=None,
+        attrs={"id": {"S": "1"}, "b": {"N": "3"}},
+    )
+    validated_ast = UpdateExpressionValidator(
+        update_expression_ast,
+        expression_attribute_names=None,
+        expression_attribute_values=update_expression_values,
+        item=item,
+    ).validate()
+    dynamo_value = get_set_action_value(validated_ast)
+    assert dynamo_value == DynamoType({"N": "3"})
+
+
+def test_validation_of_if_not_exists_with_non_existing_attribute_should_return_value():
+    update_expression = "SET a = if_not_exists(b, :val)"
+    update_expression_values = {":val": {"N": "4"}}
+    update_expression_ast = UpdateExpressionParser.make(update_expression)
+    item = Item(
+        hash_key=DynamoType({"S": "id"}),
+        hash_key_type="TYPE",
+        range_key=None,
+        range_key_type=None,
+        attrs={"id": {"S": "1"}},
+    )
+    validated_ast = UpdateExpressionValidator(
+        update_expression_ast,
+        expression_attribute_names=None,
+        expression_attribute_values=update_expression_values,
+        item=item,
+    ).validate()
+    dynamo_value = get_set_action_value(validated_ast)
+    assert dynamo_value == DynamoType({"N": "4"})
+
+
+def test_validation_of_sum_operation():
+    update_expression = "SET a = a + b"
+    update_expression_ast = UpdateExpressionParser.make(update_expression)
+    item = Item(
+        hash_key=DynamoType({"S": "id"}),
+        hash_key_type="TYPE",
+        range_key=None,
+        range_key_type=None,
+        attrs={"id": {"S": "1"}, "a": {"N": "3"}, "b": {"N": "4"}},
+    )
+    validated_ast = UpdateExpressionValidator(
+        update_expression_ast,
+        expression_attribute_names=None,
+        expression_attribute_values=None,
+        item=item,
+    ).validate()
+    dynamo_value = get_set_action_value(validated_ast)
+    assert dynamo_value == DynamoType({"N": "7"})
+
+
+def test_validation_homogeneous_list_append_function():
+    update_expression = "SET ri = list_append(ri, :vals)"
+    update_expression_ast = UpdateExpressionParser.make(update_expression)
+    item = Item(
+        hash_key=DynamoType({"S": "id"}),
+        hash_key_type="TYPE",
+        range_key=None,
+        range_key_type=None,
+        attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}},
+    )
+    validated_ast = UpdateExpressionValidator(
+        update_expression_ast,
+        expression_attribute_names=None,
+        expression_attribute_values={":vals": {"L": [{"S": "i3"}, {"S": "i4"}]}},
+        item=item,
+    ).validate()
+    dynamo_value = get_set_action_value(validated_ast)
+    assert dynamo_value == DynamoType(
+        {"L": [{"S": "i1"}, {"S": "i2"}, {"S": "i3"}, {"S": "i4"}]}
+    )
+
+
+def test_validation_hetereogenous_list_append_function():
+    update_expression = "SET ri = list_append(ri, :vals)"
+    update_expression_ast = UpdateExpressionParser.make(update_expression)
+    item = Item(
+        hash_key=DynamoType({"S": "id"}),
+        hash_key_type="TYPE",
+        range_key=None,
+        range_key_type=None,
+        attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}},
+    )
+    validated_ast = UpdateExpressionValidator(
+        update_expression_ast,
+        expression_attribute_names=None,
+        expression_attribute_values={":vals": {"L": [{"N": "3"}]}},
+        item=item,
+    ).validate()
+    dynamo_value = get_set_action_value(validated_ast)
+    assert dynamo_value == DynamoType({"L": [{"S": "i1"}, {"S": "i2"}, {"N": "3"}]})
+
+
+def test_validation_list_append_function_with_non_list_arg():
+    """
+    Must error out:
+    Invalid UpdateExpression: Incorrect operand type for operator or function;
+     operator or function: list_append, operand type: S'
+    Returns:
+
+    """
+    try:
+        update_expression = "SET ri = list_append(ri, :vals)"
+        update_expression_ast = UpdateExpressionParser.make(update_expression)
+        item = Item(
+            hash_key=DynamoType({"S": "id"}),
+            hash_key_type="TYPE",
+            range_key=None,
+            range_key_type=None,
+            attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}},
+        )
+        UpdateExpressionValidator(
+            update_expression_ast,
+            expression_attribute_names=None,
+            expression_attribute_values={":vals": {"S": "N"}},
+            item=item,
+        ).validate()
+    except IncorrectOperandType as e:
+        assert e.operand_type == "S"
+        assert e.operator_or_function == "list_append"
+
+
+def test_sum_with_incompatible_types():
+    """
+    Must error out:
+    Invalid UpdateExpression: Incorrect operand type for operator or function; operator or function: +, operand type: S'
+    Returns:
+
+    """
+    try:
+        update_expression = "SET ri = :val + :val2"
+        update_expression_ast = UpdateExpressionParser.make(update_expression)
+        item = Item(
+            hash_key=DynamoType({"S": "id"}),
+            hash_key_type="TYPE",
+            range_key=None,
+            range_key_type=None,
+            attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}},
+        )
+        UpdateExpressionValidator(
+            update_expression_ast,
+            expression_attribute_names=None,
+            expression_attribute_values={":val": {"S": "N"}, ":val2": {"N": "3"}},
+            item=item,
+        ).validate()
+    except IncorrectOperandType as e:
+        assert e.operand_type == "S"
+        assert e.operator_or_function == "+"
+
+
+def test_validation_of_subraction_operation():
+    update_expression = "SET ri = :val - :val2"
+    update_expression_ast = UpdateExpressionParser.make(update_expression)
+    item = Item(
+        hash_key=DynamoType({"S": "id"}),
+        hash_key_type="TYPE",
+        range_key=None,
+        range_key_type=None,
+        attrs={"id": {"S": "1"}, "a": {"N": "3"}, "b": {"N": "4"}},
+    )
+    validated_ast = UpdateExpressionValidator(
+        update_expression_ast,
+        expression_attribute_names=None,
+        expression_attribute_values={":val": {"N": "1"}, ":val2": {"N": "3"}},
+        item=item,
+    ).validate()
+    dynamo_value = get_set_action_value(validated_ast)
+    assert dynamo_value == DynamoType({"N": "-2"})
+
+
+def test_cannot_index_into_a_string():
+    """
+    Must error out:
+    The document path provided in the update expression is invalid for update'
+    """
+    try:
+        update_expression = "set itemstr[1]=:Item"
+        update_expression_ast = UpdateExpressionParser.make(update_expression)
+        item = Item(
+            hash_key=DynamoType({"S": "id"}),
+            hash_key_type="TYPE",
+            range_key=None,
+            range_key_type=None,
+            attrs={"id": {"S": "foo2"}, "itemstr": {"S": "somestring"}},
+        )
+        UpdateExpressionValidator(
+            update_expression_ast,
+            expression_attribute_names=None,
+            expression_attribute_values={":Item": {"S": "string_update"}},
+            item=item,
+        ).validate()
+        assert False, "Must raise exception"
+    except InvalidUpdateExpressionInvalidDocumentPath:
+        assert True
+
+
+def test_validation_set_path_does_not_need_to_be_resolvable_when_setting_a_new_attribute():
+    """If this step just passes we are happy enough"""
+    update_expression = "set d=a"
+    update_expression_ast = UpdateExpressionParser.make(update_expression)
+    item = Item(
+        hash_key=DynamoType({"S": "id"}),
+        hash_key_type="TYPE",
+        range_key=None,
+        range_key_type=None,
+        attrs={"id": {"S": "foo2"}, "a": {"N": "3"}},
+    )
+    validated_ast = UpdateExpressionValidator(
+        update_expression_ast,
+        expression_attribute_names=None,
+        expression_attribute_values=None,
+        item=item,
+    ).validate()
+    dynamo_value = get_set_action_value(validated_ast)
+    assert dynamo_value == DynamoType({"N": "3"})
+
+
+def test_validation_set_path_does_not_need_to_be_resolvable_but_must_be_creatable_when_setting_a_new_attribute():
+    try:
+        update_expression = "set d.e=a"
+        update_expression_ast = UpdateExpressionParser.make(update_expression)
+        item = Item(
+            hash_key=DynamoType({"S": "id"}),
+            hash_key_type="TYPE",
+            range_key=None,
+            range_key_type=None,
+            attrs={"id": {"S": "foo2"}, "a": {"N": "3"}},
+        )
+        UpdateExpressionValidator(
+            update_expression_ast,
+            expression_attribute_names=None,
+            expression_attribute_values=None,
+            item=item,
+        ).validate()
+        assert False, "Must raise exception"
+    except InvalidUpdateExpressionInvalidDocumentPath:
+        assert True

tests/test_glue/test_datacatalog.py

@@ -52,6 +52,29 @@ def test_get_database_not_exits():
     )
 
 
+@mock_glue
+def test_get_databases_empty():
+    client = boto3.client("glue", region_name="us-east-1")
+    response = client.get_databases()
+    response["DatabaseList"].should.have.length_of(0)
+
+
+@mock_glue
+def test_get_databases_several_items():
+    client = boto3.client("glue", region_name="us-east-1")
+    database_name_1, database_name_2 = "firstdatabase", "seconddatabase"
+
+    helpers.create_database(client, database_name_1)
+    helpers.create_database(client, database_name_2)
+
+    database_list = sorted(
+        client.get_databases()["DatabaseList"], key=lambda x: x["Name"]
+    )
+    database_list.should.have.length_of(2)
+    database_list[0].should.equal({"Name": database_name_1})
+    database_list[1].should.equal({"Name": database_name_2})
+
+
 @mock_glue
 def test_create_table():
     client = boto3.client("glue", region_name="us-east-1")

tests/test_iot/test_iot.py

@@ -728,6 +728,14 @@ def test_principal_thing():
     res = client.list_thing_principals(thingName=thing_name)
     res.should.have.key("principals").which.should.have.length_of(0)
 
+    with assert_raises(ClientError) as e:
+        client.list_thing_principals(thingName="xxx")
+
+    e.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+    e.exception.response["Error"]["Message"].should.equal(
+        "Failed to list principals for thing xxx because the thing does not exist in your account"
+    )
+
 
 @mock_iot
 def test_delete_principal_thing():

tests/test_s3/test_s3.py

@@ -14,6 +14,7 @@ from io import BytesIO
 import mimetypes
 import zlib
 import pickle
+import uuid
 
 import json
 import boto
@@ -4428,3 +4429,41 @@ def test_s3_config_dict():
     assert not logging_bucket["supplementaryConfiguration"].get(
         "BucketTaggingConfiguration"
     )
+
+
+@mock_s3
+def test_creating_presigned_post():
+    bucket = "presigned-test"
+    s3 = boto3.client("s3", region_name="us-east-1")
+    s3.create_bucket(Bucket=bucket)
+    success_url = "http://localhost/completed"
+    fdata = b"test data\n"
+    file_uid = uuid.uuid4()
+    conditions = [
+        {"Content-Type": "text/plain"},
+        {"x-amz-server-side-encryption": "AES256"},
+        {"success_action_redirect": success_url},
+    ]
+    conditions.append(["content-length-range", 1, 30])
+    data = s3.generate_presigned_post(
+        Bucket=bucket,
+        Key="{file_uid}.txt".format(file_uid=file_uid),
+        Fields={
+            "content-type": "text/plain",
+            "success_action_redirect": success_url,
+            "x-amz-server-side-encryption": "AES256",
+        },
+        Conditions=conditions,
+        ExpiresIn=1000,
+    )
+    resp = requests.post(
+        data["url"], data=data["fields"], files={"file": fdata}, allow_redirects=False
+    )
+    assert resp.headers["Location"] == success_url
+    assert resp.status_code == 303
+    assert (
+        s3.get_object(Bucket=bucket, Key="{file_uid}.txt".format(file_uid=file_uid))[
+            "Body"
+        ].read()
+        == fdata
+    )