Better DDB expressions support2: ExpressionTree

Part of structured approach for UpdateExpressions:
 1) Expression gets parsed into a tokenlist (tokenized)
 2) Tokenlist get transformed to expression tree (AST) -> This commit
 3) The AST gets validated (full semantic correctness)
 4) AST gets processed to perform the update

This commit uses the tokenlist to build an expression tree. This tree is not
yet used. Still it allows to raise additional Validation Exceptions which
previously were missed silently therefore it allows tests to catch these type of
ValidationException. For that reason DDB UpdateExpressions will be parsed
already. It also makes sure we won't break existing tests.

One of the existing tests had to be changed in order to still pass:
 - test_dynamodb_table_with_range_key.test_update_item_with_expression

This test passed in a numeric literal which is not supported by DynamoDB
and with the current tokenization it would get the same error as in AWS
DynamoDB.
This commit is contained in:
pvbouwel 2020-04-11 21:17:16 +01:00
parent 7ea419dd54
commit 9ed613e197
10 changed files with 2317 additions and 13 deletions

View File

@ -14,10 +14,11 @@ from moto.core import BaseBackend, BaseModel
from moto.core.utils import unix_time
from moto.core.exceptions import JsonRESTError
from moto.dynamodb2.comparisons import get_filter_expression
from moto.dynamodb2.comparisons import get_expected
from moto.dynamodb2.exceptions import InvalidIndexNameError, ItemSizeTooLarge
from moto.dynamodb2.comparisons import get_expected, get_comparison_func
from moto.dynamodb2.exceptions import InvalidIndexNameError, ItemSizeTooLarge, InvalidUpdateExpression
from moto.dynamodb2.models.utilities import bytesize, attribute_is_list
from moto.dynamodb2.models.dynamo_type import DynamoType
from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
class DynamoJsonEncoder(json.JSONEncoder):
@ -1197,6 +1198,13 @@ class DynamoDBBackend(BaseBackend):
):
table = self.get_table(table_name)
# Support spaces between operators in an update expression
# E.g. `a = b + c` -> `a=b+c`
if update_expression:
# Parse expression to get validation errors
UpdateExpressionParser.make(update_expression)
update_expression = re.sub(r"\s*([=\+-])\s*", "\\1", update_expression)
if all([table.hash_key_attr in key, table.range_key_attr in key]):
# Covers cases where table has hash and range keys, ``key`` param
# will be a dict

View File

@ -0,0 +1,205 @@
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class Node:
def __init__(self, children=None):
self.type = self.__class__.__name__
assert children is None or isinstance(children, list)
self.children = children
self.parent = None
if isinstance(children, list):
for child in children:
if isinstance(child, Node):
child.set_parent(self)
def set_parent(self, parent_node):
self.parent = parent_node
class LeafNode(Node):
"""A LeafNode is a Node where none of the children are Nodes themselves."""
def __init__(self, children=None):
super(LeafNode, self).__init__(children)
@six.add_metaclass(abc.ABCMeta)
class Expression(Node):
"""
Abstract Syntax Tree representing the expression
For the Grammar start here and jump down into the classes at the righ-hand side to look further. Nodes marked with
a star are abstract and won't appear in the final AST.
Expression* => UpdateExpression
Expression* => ConditionExpression
"""
class UpdateExpression(Expression):
"""
UpdateExpression => UpdateExpressionClause*
UpdateExpression => UpdateExpressionClause* UpdateExpression
"""
@six.add_metaclass(abc.ABCMeta)
class UpdateExpressionClause(UpdateExpression):
"""
UpdateExpressionClause* => UpdateExpressionSetClause
UpdateExpressionClause* => UpdateExpressionRemoveClause
UpdateExpressionClause* => UpdateExpressionAddClause
UpdateExpressionClause* => UpdateExpressionDeleteClause
"""
class UpdateExpressionSetClause(UpdateExpressionClause):
"""
UpdateExpressionSetClause => SET SetActions
"""
class UpdateExpressionSetActions(UpdateExpressionClause):
"""
UpdateExpressionSetClause => SET SetActions
SetActions => SetAction
SetActions => SetAction , SetActions
"""
class UpdateExpressionSetAction(UpdateExpressionClause):
"""
SetAction => Path = Value
"""
class UpdateExpressionRemoveActions(UpdateExpressionClause):
"""
UpdateExpressionSetClause => REMOVE RemoveActions
RemoveActions => RemoveAction
RemoveActions => RemoveAction , RemoveActions
"""
class UpdateExpressionRemoveAction(UpdateExpressionClause):
"""
RemoveAction => Path
"""
class UpdateExpressionAddActions(UpdateExpressionClause):
"""
UpdateExpressionAddClause => ADD RemoveActions
AddActions => AddAction
AddActions => AddAction , AddActions
"""
class UpdateExpressionAddAction(UpdateExpressionClause):
"""
AddAction => Path Value
"""
class UpdateExpressionDeleteActions(UpdateExpressionClause):
"""
UpdateExpressionDeleteClause => DELETE RemoveActions
DeleteActions => DeleteAction
DeleteActions => DeleteAction , DeleteActions
"""
class UpdateExpressionDeleteAction(UpdateExpressionClause):
"""
DeleteAction => Path Value
"""
class UpdateExpressionPath(UpdateExpressionClause):
pass
class UpdateExpressionValue(UpdateExpressionClause):
"""
Value => Operand
Value => Operand + Value
Value => Operand - Value
"""
class UpdateExpressionGroupedValue(UpdateExpressionClause):
"""
GroupedValue => ( Value )
"""
class UpdateExpressionRemoveClause(UpdateExpressionClause):
"""
UpdateExpressionRemoveClause => REMOVE RemoveActions
"""
class UpdateExpressionAddClause(UpdateExpressionClause):
"""
UpdateExpressionAddClause => ADD AddActions
"""
class UpdateExpressionDeleteClause(UpdateExpressionClause):
"""
UpdateExpressionDeleteClause => DELETE DeleteActions
"""
class ExpressionPathDescender(Node):
"""Node identifying descender into nested structure (.) in expression"""
class ExpressionSelector(LeafNode):
"""Node identifying selector [selection_index] in expresion"""
def __init__(self, selection_index):
super(ExpressionSelector, self).__init__(children=[selection_index])
class ExpressionAttribute(LeafNode):
"""An attribute identifier as used in the DDB item"""
def __init__(self, attribute):
super(ExpressionAttribute, self).__init__(children=[attribute])
class ExpressionAttributeName(LeafNode):
"""An ExpressionAttributeName is an alias for an attribute identifier"""
def __init__(self, attribute_name):
super(ExpressionAttributeName, self).__init__(children=[attribute_name])
class ExpressionAttributeValue(LeafNode):
"""An ExpressionAttributeValue is an alias for an value"""
def __init__(self, value):
super(ExpressionAttributeValue, self).__init__(children=[value])
class ExpressionValueOperator(LeafNode):
"""An ExpressionValueOperator is an operation that works on 2 values"""
def __init__(self, value):
super(ExpressionValueOperator, self).__init__(children=[value])
class UpdateExpressionFunction(Node):
"""
A Node representing a function of an Update Expression. The first child is the function name the others are the
arguments.
"""

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,29 @@
class ReservedKeywords(list):
"""
DynamoDB has an extensive list of keywords. Keywords are considered when validating the expression Tree.
Not earlier since an update expression like "SET path = VALUE 1" fails with:
'Invalid UpdateExpression: Syntax error; token: "1", near: "VALUE 1"'
"""
KEYWORDS = None
@classmethod
def get_reserved_keywords(cls):
if cls.KEYWORDS is None:
cls.KEYWORDS = cls._get_reserved_keywords()
return cls.KEYWORDS
@classmethod
def _get_reserved_keywords(cls):
"""
Get a list of reserved keywords of DynamoDB
"""
try:
import importlib.resources as pkg_resources
except ImportError:
import importlib_resources as pkg_resources
reserved_keywords = pkg_resources.read_text(
"moto.dynamodb2.parsing", "reserved_keywords.txt"
)
return reserved_keywords.split()

View File

@ -0,0 +1,573 @@
ABORT
ABSOLUTE
ACTION
ADD
AFTER
AGENT
AGGREGATE
ALL
ALLOCATE
ALTER
ANALYZE
AND
ANY
ARCHIVE
ARE
ARRAY
AS
ASC
ASCII
ASENSITIVE
ASSERTION
ASYMMETRIC
AT
ATOMIC
ATTACH
ATTRIBUTE
AUTH
AUTHORIZATION
AUTHORIZE
AUTO
AVG
BACK
BACKUP
BASE
BATCH
BEFORE
BEGIN
BETWEEN
BIGINT
BINARY
BIT
BLOB
BLOCK
BOOLEAN
BOTH
BREADTH
BUCKET
BULK
BY
BYTE
CALL
CALLED
CALLING
CAPACITY
CASCADE
CASCADED
CASE
CAST
CATALOG
CHAR
CHARACTER
CHECK
CLASS
CLOB
CLOSE
CLUSTER
CLUSTERED
CLUSTERING
CLUSTERS
COALESCE
COLLATE
COLLATION
COLLECTION
COLUMN
COLUMNS
COMBINE
COMMENT
COMMIT
COMPACT
COMPILE
COMPRESS
CONDITION
CONFLICT
CONNECT
CONNECTION
CONSISTENCY
CONSISTENT
CONSTRAINT
CONSTRAINTS
CONSTRUCTOR
CONSUMED
CONTINUE
CONVERT
COPY
CORRESPONDING
COUNT
COUNTER
CREATE
CROSS
CUBE
CURRENT
CURSOR
CYCLE
DATA
DATABASE
DATE
DATETIME
DAY
DEALLOCATE
DEC
DECIMAL
DECLARE
DEFAULT
DEFERRABLE
DEFERRED
DEFINE
DEFINED
DEFINITION
DELETE
DELIMITED
DEPTH
DEREF
DESC
DESCRIBE
DESCRIPTOR
DETACH
DETERMINISTIC
DIAGNOSTICS
DIRECTORIES
DISABLE
DISCONNECT
DISTINCT
DISTRIBUTE
DO
DOMAIN
DOUBLE
DROP
DUMP
DURATION
DYNAMIC
EACH
ELEMENT
ELSE
ELSEIF
EMPTY
ENABLE
END
EQUAL
EQUALS
ERROR
ESCAPE
ESCAPED
EVAL
EVALUATE
EXCEEDED
EXCEPT
EXCEPTION
EXCEPTIONS
EXCLUSIVE
EXEC
EXECUTE
EXISTS
EXIT
EXPLAIN
EXPLODE
EXPORT
EXPRESSION
EXTENDED
EXTERNAL
EXTRACT
FAIL
FALSE
FAMILY
FETCH
FIELDS
FILE
FILTER
FILTERING
FINAL
FINISH
FIRST
FIXED
FLATTERN
FLOAT
FOR
FORCE
FOREIGN
FORMAT
FORWARD
FOUND
FREE
FROM
FULL
FUNCTION
FUNCTIONS
GENERAL
GENERATE
GET
GLOB
GLOBAL
GO
GOTO
GRANT
GREATER
GROUP
GROUPING
HANDLER
HASH
HAVE
HAVING
HEAP
HIDDEN
HOLD
HOUR
IDENTIFIED
IDENTITY
IF
IGNORE
IMMEDIATE
IMPORT
IN
INCLUDING
INCLUSIVE
INCREMENT
INCREMENTAL
INDEX
INDEXED
INDEXES
INDICATOR
INFINITE
INITIALLY
INLINE
INNER
INNTER
INOUT
INPUT
INSENSITIVE
INSERT
INSTEAD
INT
INTEGER
INTERSECT
INTERVAL
INTO
INVALIDATE
IS
ISOLATION
ITEM
ITEMS
ITERATE
JOIN
KEY
KEYS
LAG
LANGUAGE
LARGE
LAST
LATERAL
LEAD
LEADING
LEAVE
LEFT
LENGTH
LESS
LEVEL
LIKE
LIMIT
LIMITED
LINES
LIST
LOAD
LOCAL
LOCALTIME
LOCALTIMESTAMP
LOCATION
LOCATOR
LOCK
LOCKS
LOG
LOGED
LONG
LOOP
LOWER
MAP
MATCH
MATERIALIZED
MAX
MAXLEN
MEMBER
MERGE
METHOD
METRICS
MIN
MINUS
MINUTE
MISSING
MOD
MODE
MODIFIES
MODIFY
MODULE
MONTH
MULTI
MULTISET
NAME
NAMES
NATIONAL
NATURAL
NCHAR
NCLOB
NEW
NEXT
NO
NONE
NOT
NULL
NULLIF
NUMBER
NUMERIC
OBJECT
OF
OFFLINE
OFFSET
OLD
ON
ONLINE
ONLY
OPAQUE
OPEN
OPERATOR
OPTION
OR
ORDER
ORDINALITY
OTHER
OTHERS
OUT
OUTER
OUTPUT
OVER
OVERLAPS
OVERRIDE
OWNER
PAD
PARALLEL
PARAMETER
PARAMETERS
PARTIAL
PARTITION
PARTITIONED
PARTITIONS
PATH
PERCENT
PERCENTILE
PERMISSION
PERMISSIONS
PIPE
PIPELINED
PLAN
POOL
POSITION
PRECISION
PREPARE
PRESERVE
PRIMARY
PRIOR
PRIVATE
PRIVILEGES
PROCEDURE
PROCESSED
PROJECT
PROJECTION
PROPERTY
PROVISIONING
PUBLIC
PUT
QUERY
QUIT
QUORUM
RAISE
RANDOM
RANGE
RANK
RAW
READ
READS
REAL
REBUILD
RECORD
RECURSIVE
REDUCE
REF
REFERENCE
REFERENCES
REFERENCING
REGEXP
REGION
REINDEX
RELATIVE
RELEASE
REMAINDER
RENAME
REPEAT
REPLACE
REQUEST
RESET
RESIGNAL
RESOURCE
RESPONSE
RESTORE
RESTRICT
RESULT
RETURN
RETURNING
RETURNS
REVERSE
REVOKE
RIGHT
ROLE
ROLES
ROLLBACK
ROLLUP
ROUTINE
ROW
ROWS
RULE
RULES
SAMPLE
SATISFIES
SAVE
SAVEPOINT
SCAN
SCHEMA
SCOPE
SCROLL
SEARCH
SECOND
SECTION
SEGMENT
SEGMENTS
SELECT
SELF
SEMI
SENSITIVE
SEPARATE
SEQUENCE
SERIALIZABLE
SESSION
SET
SETS
SHARD
SHARE
SHARED
SHORT
SHOW
SIGNAL
SIMILAR
SIZE
SKEWED
SMALLINT
SNAPSHOT
SOME
SOURCE
SPACE
SPACES
SPARSE
SPECIFIC
SPECIFICTYPE
SPLIT
SQL
SQLCODE
SQLERROR
SQLEXCEPTION
SQLSTATE
SQLWARNING
START
STATE
STATIC
STATUS
STORAGE
STORE
STORED
STREAM
STRING
STRUCT
STYLE
SUB
SUBMULTISET
SUBPARTITION
SUBSTRING
SUBTYPE
SUM
SUPER
SYMMETRIC
SYNONYM
SYSTEM
TABLE
TABLESAMPLE
TEMP
TEMPORARY
TERMINATED
TEXT
THAN
THEN
THROUGHPUT
TIME
TIMESTAMP
TIMEZONE
TINYINT
TO
TOKEN
TOTAL
TOUCH
TRAILING
TRANSACTION
TRANSFORM
TRANSLATE
TRANSLATION
TREAT
TRIGGER
TRIM
TRUE
TRUNCATE
TTL
TUPLE
TYPE
UNDER
UNDO
UNION
UNIQUE
UNIT
UNKNOWN
UNLOGGED
UNNEST
UNPROCESSED
UNSIGNED
UNTIL
UPDATE
UPPER
URL
USAGE
USE
USER
USERS
USING
UUID
VACUUM
VALUE
VALUED
VALUES
VARCHAR
VARIABLE
VARIANCE
VARINT
VARYING
VIEW
VIEWS
VIRTUAL
VOID
WAIT
WHEN
WHENEVER
WHERE
WHILE
WINDOW
WITH
WITHIN
WITHOUT
WORK
WRAPPED
WRITE
YEAR
ZONE

View File

@ -1,4 +1,5 @@
import re
import sys
from moto.dynamodb2.exceptions import (
InvalidTokenException,
@ -147,9 +148,17 @@ class ExpressionTokenizer(object):
self.token_list = []
self.staged_characters = ""
@classmethod
def is_py2(cls):
return sys.version_info[0] == 2
@classmethod
def make_list(cls, input_expression_str):
assert isinstance(input_expression_str, str)
if cls.is_py2():
pass
else:
assert isinstance(input_expression_str, str)
return ExpressionTokenizer(input_expression_str)._make_list()
def add_token(self, token_type, token_value):
@ -159,6 +168,10 @@ class ExpressionTokenizer(object):
self.add_token(token_type, self.staged_characters)
self.staged_characters = ""
@classmethod
def is_numeric(cls, input_str):
return re.compile("[0-9]+").match(input_str) is not None
def process_staged_characters(self):
if len(self.staged_characters) == 0:
return
@ -167,7 +180,7 @@ class ExpressionTokenizer(object):
self.add_token_from_stage(Token.ATTRIBUTE_NAME)
else:
raise InvalidExpressionAttributeNameKey(self.staged_characters)
elif self.staged_characters.isnumeric():
elif self.is_numeric(self.staged_characters):
self.add_token_from_stage(Token.NUMBER)
elif self.is_expression_attribute(self.staged_characters):
self.add_token_from_stage(Token.ATTRIBUTE)

View File

@ -748,11 +748,6 @@ class DynamoHandler(BaseResponse):
expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
# Support spaces between operators in an update expression
# E.g. `a = b + c` -> `a=b+c`
if update_expression:
update_expression = re.sub(r"\s*([=\+-])\s*", "\\1", update_expression)
try:
item = self.dynamodb_backend.update_item(
name,

View File

@ -1,5 +1,6 @@
from __future__ import unicode_literals, print_function
import re
from decimal import Decimal
import six
@ -4177,3 +4178,70 @@ def test_gsi_verify_negative_number_order():
[float(item["gsiK1SortKey"]) for item in resp["Items"]].should.equal(
[-0.7, -0.6, 0.7]
)
def assert_raise_syntax_error(client_error, token, near):
"""
Assert whether a client_error is as expected Syntax error. Syntax error looks like: `syntax_error_template`
Args:
client_error(ClientError): The ClientError exception that was raised
token(str): The token that ws unexpected
near(str): The part in the expression that shows where the error occurs it generally has the preceding token the
optional separation and the problematic token.
"""
syntax_error_template = (
'Invalid UpdateExpression: Syntax error; token: "{token}", near: "{near}"'
)
expected_syntax_error = syntax_error_template.format(token=token, near=near)
assert client_error.response["Error"]["Code"] == "ValidationException"
assert expected_syntax_error == client_error.response["Error"]["Message"]
@mock_dynamodb2
def test_update_expression_with_numeric_literal_instead_of_value():
"""
DynamoDB requires literals to be passed in as values. If they are put literally in the expression a token error will
be raised
"""
dynamodb = boto3.client("dynamodb", region_name="eu-west-1")
dynamodb.create_table(
TableName="moto-test",
KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
)
try:
dynamodb.update_item(
TableName="moto-test",
Key={"id": {"S": "1"}},
UpdateExpression="SET MyStr = myNum + 1",
)
assert False, "Validation exception not thrown"
except dynamodb.exceptions.ClientError as e:
assert_raise_syntax_error(e, "1", "+ 1")
@mock_dynamodb2
def test_update_expression_with_multiple_set_clauses_must_be_comma_separated():
"""
An UpdateExpression can have multiple set clauses but if they are passed in without the separating comma.
"""
dynamodb = boto3.client("dynamodb", region_name="eu-west-1")
dynamodb.create_table(
TableName="moto-test",
KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
)
try:
dynamodb.update_item(
TableName="moto-test",
Key={"id": {"S": "1"}},
UpdateExpression="SET MyStr = myNum Mystr2 myNum2",
)
assert False, "Validation exception not thrown"
except dynamodb.exceptions.ClientError as e:
assert_raise_syntax_error(e, "Mystr2", "myNum Mystr2 myNum2")

View File

@ -0,0 +1,395 @@
from moto.dynamodb2.exceptions import InvalidTokenException
from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
from moto.dynamodb2.parsing.reserved_keywords import ReservedKeywords
def test_get_reserved_keywords():
reserved_keywords = ReservedKeywords.get_reserved_keywords()
assert "SET" in reserved_keywords
assert "DELETE" in reserved_keywords
assert "ADD" in reserved_keywords
# REMOVE is not part of the list of reserved keywords.
assert "REMOVE" not in reserved_keywords
def test_update_expression_numeric_literal_in_expression():
set_action = "SET attrName = 3"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "3"
assert te.near == "= 3"
def test_expression_tokenizer_multi_number_numeric_literal_in_expression():
set_action = "SET attrName = 34"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "34"
assert te.near == "= 34"
def test_expression_tokenizer_numeric_literal_unclosed_square_bracket():
set_action = "SET MyStr[ 3"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "<EOF>"
assert te.near == "3"
def test_expression_tokenizer_wrong_closing_bracket_with_space():
set_action = "SET MyStr[3 )"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "3 )"
def test_expression_tokenizer_wrong_closing_bracket():
set_action = "SET MyStr[3)"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "3)"
def test_expression_tokenizer_only_numeric_literal_for_set():
set_action = "SET 2"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "2"
assert te.near == "SET 2"
def test_expression_tokenizer_only_numeric_literal():
set_action = "2"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "2"
assert te.near == "2"
def test_expression_tokenizer_set_closing_round_bracket():
set_action = "SET )"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "SET )"
def test_expression_tokenizer_set_closing_followed_by_numeric_literal():
set_action = "SET ) 3"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "SET ) 3"
def test_expression_tokenizer_numeric_literal_unclosed_square_bracket_trailing_space():
set_action = "SET MyStr[ 3 "
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "<EOF>"
assert te.near == "3 "
def test_expression_tokenizer_unbalanced_round_brackets_only_opening():
set_action = "SET MyStr = (:_val"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "<EOF>"
assert te.near == ":_val"
def test_expression_tokenizer_unbalanced_round_brackets_only_opening_trailing_space():
set_action = "SET MyStr = (:_val "
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "<EOF>"
assert te.near == ":_val "
def test_expression_tokenizer_unbalanced_square_brackets_only_opening():
set_action = "SET MyStr = [:_val"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "["
assert te.near == "= [:_val"
def test_expression_tokenizer_unbalanced_square_brackets_only_opening_trailing_spaces():
set_action = "SET MyStr = [:_val "
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "["
assert te.near == "= [:_val"
def test_expression_tokenizer_unbalanced_round_brackets_multiple_opening():
set_action = "SET MyStr = (:_val + (:val2"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "<EOF>"
assert te.near == ":val2"
def test_expression_tokenizer_unbalanced_round_brackets_only_closing():
set_action = "SET MyStr = ):_val"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "= ):_val"
def test_expression_tokenizer_unbalanced_square_brackets_only_closing():
set_action = "SET MyStr = ]:_val"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "]"
assert te.near == "= ]:_val"
def test_expression_tokenizer_unbalanced_round_brackets_only_closing_followed_by_other_parts():
set_action = "SET MyStr = ):_val + :val2"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "= ):_val"
def test_update_expression_starts_with_keyword_reset_followed_by_identifier():
update_expression = "RESET NonExistent"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "RESET"
assert te.near == "RESET NonExistent"
def test_update_expression_starts_with_keyword_reset_followed_by_identifier_and_value():
update_expression = "RESET NonExistent value"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "RESET"
assert te.near == "RESET NonExistent"
def test_update_expression_starts_with_leading_spaces_and_keyword_reset_followed_by_identifier_and_value():
update_expression = " RESET NonExistent value"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "RESET"
assert te.near == " RESET NonExistent"
def test_update_expression_with_only_keyword_reset():
update_expression = "RESET"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "RESET"
assert te.near == "RESET"
def test_update_nested_expression_with_selector_just_should_fail_parsing_at_numeric_literal_value():
update_expression = "SET a[0].b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "5"
assert te.near == "= 5"
def test_update_nested_expression_with_selector_and_spaces_should_only_fail_parsing_at_numeric_literal_value():
update_expression = "SET a [ 2 ]. b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "5"
assert te.near == "= 5"
def test_update_nested_expression_with_double_selector_and_spaces_should_only_fail_parsing_at_numeric_literal_value():
update_expression = "SET a [2][ 3 ]. b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "5"
assert te.near == "= 5"
def test_update_nested_expression_should_only_fail_parsing_at_numeric_literal_value():
update_expression = "SET a . b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "5"
assert te.near == "= 5"
def test_nested_selectors_in_update_expression_should_fail_at_nesting():
update_expression = "SET a [ [2] ]. b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "["
assert te.near == "[ [2"
def test_update_expression_number_in_selector_cannot_be_splite():
update_expression = "SET a [2 1]. b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "1"
assert te.near == "2 1]"
def test_update_expression_cannot_have_successive_attributes():
update_expression = "SET #a a = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "a"
assert te.near == "#a a ="
def test_update_expression_path_with_both_attribute_and_attribute_name_should_only_fail_at_numeric_value():
update_expression = "SET #a.a = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "5"
assert te.near == "= 5"
def test_expression_tokenizer_2_same_operators_back_to_back():
set_action = "SET MyStr = NoExist + + :_val "
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "+"
assert te.near == "+ + :_val"
def test_expression_tokenizer_2_different_operators_back_to_back():
set_action = "SET MyStr = NoExist + - :_val "
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "-"
assert te.near == "+ - :_val"
def test_update_expression_remove_does_not_allow_operations():
remove_action = "REMOVE NoExist + "
try:
UpdateExpressionParser.make(remove_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "+"
assert te.near == "NoExist + "
def test_update_expression_add_does_not_allow_attribute_after_path():
"""value here is not really a value since a value starts with a colon (:)"""
add_expr = "ADD attr val foobar"
try:
UpdateExpressionParser.make(add_expr)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "val"
assert te.near == "attr val foobar"
def test_update_expression_add_does_not_allow_attribute_foobar_after_value():
add_expr = "ADD attr :val foobar"
try:
UpdateExpressionParser.make(add_expr)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "foobar"
assert te.near == ":val foobar"
def test_update_expression_delete_does_not_allow_attribute_after_path():
"""value here is not really a value since a value starts with a colon (:)"""
delete_expr = "DELETE attr val"
try:
UpdateExpressionParser.make(delete_expr)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "val"
assert te.near == "attr val"
def test_update_expression_delete_does_not_allow_attribute_foobar_after_value():
delete_expr = "DELETE attr :val foobar"
try:
UpdateExpressionParser.make(delete_expr)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "foobar"
assert te.near == ":val foobar"
def test_update_expression_parsing_is_not_keyword_aware():
"""path and VALUE are keywords. Yet a token error will be thrown for the numeric literal 1."""
delete_expr = "SET path = VALUE 1"
try:
UpdateExpressionParser.make(delete_expr)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "1"
assert te.near == "VALUE 1"

View File

@ -1254,14 +1254,22 @@ def test_update_item_with_expression():
item_key = {"forum_name": "the-key", "subject": "123"}
table.update_item(Key=item_key, UpdateExpression="SET field=2")
table.update_item(
Key=item_key,
UpdateExpression="SET field = :field_value",
ExpressionAttributeValues={":field_value": 2},
)
dict(table.get_item(Key=item_key)["Item"]).should.equal(
{"field": "2", "forum_name": "the-key", "subject": "123"}
{"field": Decimal("2"), "forum_name": "the-key", "subject": "123"}
)
table.update_item(Key=item_key, UpdateExpression="SET field = 3")
table.update_item(
Key=item_key,
UpdateExpression="SET field = :field_value",
ExpressionAttributeValues={":field_value": 3},
)
dict(table.get_item(Key=item_key)["Item"]).should.equal(
{"field": "3", "forum_name": "the-key", "subject": "123"}
{"field": Decimal("3"), "forum_name": "the-key", "subject": "123"}
)