Merge pull request #2467 from bblommers/bugfix/1823
DynamoDB: Add List index operations
This commit is contained in:
Mike Grima
GitHub
commit
4497fb132b
@ -2,6 +2,9 @@ class InvalidIndexNameError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidUpdateExpression(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ItemSizeTooLarge(Exception):
|
||||
message = 'Item size has exceeded the maximum allowed size'
|
||||
pass
|
||||
|
||||
@ -16,7 +16,7 @@ from moto.core.exceptions import JsonRESTError
|
||||
from .comparisons import get_comparison_func
|
||||
from .comparisons import get_filter_expression
|
||||
from .comparisons import get_expected
|
||||
from .exceptions import InvalidIndexNameError, ItemSizeTooLarge
|
||||
from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge
|
||||
|
||||
|
||||
class DynamoJsonEncoder(json.JSONEncoder):
|
||||
@ -237,7 +237,16 @@ class Item(BaseModel):
|
||||
if action == "REMOVE":
|
||||
key = value
|
||||
if '.' not in key:
|
||||
self.attrs.pop(value, None)
|
||||
list_index_update = re.match('(.+)\\[([0-9]+)\\]', key)
|
||||
if list_index_update:
|
||||
# We need to remove an item from a list (REMOVE listattr[0])
|
||||
key_attr = self.attrs[list_index_update.group(1)]
|
||||
list_index = int(list_index_update.group(2))
|
||||
if key_attr.is_list():
|
||||
if len(key_attr.value) > list_index:
|
||||
del key_attr.value[list_index]
|
||||
else:
|
||||
self.attrs.pop(value, None)
|
||||
else:
|
||||
# Handle nested dict updates
|
||||
key_parts = key.split('.')
|
||||
@ -247,6 +256,9 @@ class Item(BaseModel):
|
||||
|
||||
last_val = self.attrs[attr].value
|
||||
for key_part in key_parts[:-1]:
|
||||
list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_part)
|
||||
if list_index_update:
|
||||
key_part = list_index_update.group(1) # listattr[1] ==> listattr
|
||||
# Hack but it'll do, traverses into a dict
|
||||
last_val_type = list(last_val.keys())
|
||||
if last_val_type and last_val_type[0] == 'M':
|
||||
@ -256,12 +268,23 @@ class Item(BaseModel):
|
||||
last_val[key_part] = {'M': {}}
|
||||
|
||||
last_val = last_val[key_part]
|
||||
if list_index_update:
|
||||
last_val = last_val['L'][int(list_index_update.group(2))]
|
||||
|
||||
last_val_type = list(last_val.keys())
|
||||
if last_val_type and last_val_type[0] == 'M':
|
||||
last_val['M'].pop(key_parts[-1], None)
|
||||
list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_parts[-1])
|
||||
if list_index_update:
|
||||
# We need to remove an item from a list (REMOVE attr.listattr[0])
|
||||
key_part = list_index_update.group(1) # listattr[1] ==> listattr
|
||||
list_to_update = last_val[key_part]['L']
|
||||
index_to_remove = int(list_index_update.group(2))
|
||||
if index_to_remove < len(list_to_update):
|
||||
del list_to_update[index_to_remove]
|
||||
else:
|
||||
last_val.pop(key_parts[-1], None)
|
||||
if last_val_type and last_val_type[0] == 'M':
|
||||
last_val['M'].pop(key_parts[-1], None)
|
||||
else:
|
||||
last_val.pop(key_parts[-1], None)
|
||||
elif action == 'SET':
|
||||
key, value = value.split("=", 1)
|
||||
key = key.strip()
|
||||
@ -282,38 +305,61 @@ class Item(BaseModel):
|
||||
|
||||
if type(value) != DynamoType:
|
||||
if value in expression_attribute_values:
|
||||
value = DynamoType(expression_attribute_values[value])
|
||||
dyn_value = DynamoType(expression_attribute_values[value])
|
||||
else:
|
||||
value = DynamoType({"S": value})
|
||||
dyn_value = DynamoType({"S": value})
|
||||
else:
|
||||
dyn_value = value
|
||||
|
||||
if '.' not in key:
|
||||
self.attrs[key] = value
|
||||
list_index_update = re.match('(.+)\\[([0-9]+)\\]', key)
|
||||
if list_index_update:
|
||||
key_attr = self.attrs[list_index_update.group(1)]
|
||||
list_index = int(list_index_update.group(2))
|
||||
if key_attr.is_list():
|
||||
if len(key_attr.value) > list_index:
|
||||
key_attr.value[list_index] = expression_attribute_values[value]
|
||||
else:
|
||||
key_attr.value.append(expression_attribute_values[value])
|
||||
else:
|
||||
raise InvalidUpdateExpression
|
||||
else:
|
||||
self.attrs[key] = dyn_value
|
||||
else:
|
||||
# Handle nested dict updates
|
||||
key_parts = key.split('.')
|
||||
attr = key_parts.pop(0)
|
||||
if attr not in self.attrs:
|
||||
raise ValueError
|
||||
|
||||
last_val = self.attrs[attr].value
|
||||
for key_part in key_parts:
|
||||
list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_part)
|
||||
if list_index_update:
|
||||
key_part = list_index_update.group(1) # listattr[1] ==> listattr
|
||||
# Hack but it'll do, traverses into a dict
|
||||
last_val_type = list(last_val.keys())
|
||||
if last_val_type and last_val_type[0] == 'M':
|
||||
last_val = last_val['M']
|
||||
last_val = last_val['M']
|
||||
|
||||
if key_part not in last_val:
|
||||
last_val[key_part] = {'M': {}}
|
||||
|
||||
last_val = last_val[key_part]
|
||||
|
||||
# We have reference to a nested object but we cant just assign to it
|
||||
current_type = list(last_val.keys())[0]
|
||||
if current_type == value.type:
|
||||
last_val[current_type] = value.value
|
||||
if list_index_update:
|
||||
# We need to add an item to a list
|
||||
list_index = int(list_index_update.group(2))
|
||||
if len(last_val['L']) > list_index:
|
||||
last_val['L'][list_index] = expression_attribute_values[value]
|
||||
else:
|
||||
last_val['L'].append(expression_attribute_values[value])
|
||||
else:
|
||||
last_val[value.type] = value.value
|
||||
del last_val[current_type]
|
||||
# We have reference to a nested object but we cant just assign to it
|
||||
if current_type == dyn_value.type:
|
||||
last_val[current_type] = dyn_value.value
|
||||
else:
|
||||
last_val[dyn_value.type] = dyn_value.value
|
||||
del last_val[current_type]
|
||||
|
||||
elif action == 'ADD':
|
||||
key, value = value.split(" ", 1)
|
||||
|
||||
@ -6,7 +6,7 @@ import re
|
||||
|
||||
from moto.core.responses import BaseResponse
|
||||
from moto.core.utils import camelcase_to_underscores, amzn_request_id
|
||||
from .exceptions import InvalidIndexNameError, ItemSizeTooLarge
|
||||
from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge
|
||||
from .models import dynamodb_backends, dynamo_json_dump
|
||||
|
||||
|
||||
@ -683,6 +683,9 @@ class DynamoHandler(BaseResponse):
|
||||
name, key, update_expression, attribute_updates, expression_attribute_names,
|
||||
expression_attribute_values, expected, condition_expression
|
||||
)
|
||||
except InvalidUpdateExpression:
|
||||
er = 'com.amazonaws.dynamodb.v20111205#ValidationException'
|
||||
return self.error(er, 'The document path provided in the update expression is invalid for update')
|
||||
except ItemSizeTooLarge:
|
||||
er = 'com.amazonaws.dynamodb.v20111205#ValidationException'
|
||||
return self.error(er, ItemSizeTooLarge.message)
|
||||
|
||||
@ -2506,6 +2506,144 @@ def test_index_with_unknown_attributes_should_fail():
|
||||
ex.exception.response['Error']['Message'].should.contain(expected_exception)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_list_index__set_existing_index():
|
||||
table_name = 'test_list_index_access'
|
||||
client = create_table_with_list(table_name)
|
||||
client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}},
|
||||
UpdateExpression='set itemlist[1]=:Item',
|
||||
ExpressionAttributeValues={':Item': {'S': 'bar2_update'}})
|
||||
#
|
||||
result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
|
||||
assert result['id'] == {'S': 'foo'}
|
||||
assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2_update'}, {'S': 'bar3'}]}
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_list_index__set_existing_nested_index():
|
||||
table_name = 'test_list_index_access'
|
||||
client = create_table_with_list(table_name)
|
||||
client.put_item(TableName=table_name,
|
||||
Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}}}})
|
||||
client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
|
||||
UpdateExpression='set itemmap.itemlist[1]=:Item',
|
||||
ExpressionAttributeValues={':Item': {'S': 'bar2_update'}})
|
||||
#
|
||||
result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
|
||||
assert result['id'] == {'S': 'foo2'}
|
||||
assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}, {'S': 'bar2_update'}, {'S': 'bar3'}]
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_list_index__set_index_out_of_range():
|
||||
table_name = 'test_list_index_access'
|
||||
client = create_table_with_list(table_name)
|
||||
client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}},
|
||||
UpdateExpression='set itemlist[10]=:Item',
|
||||
ExpressionAttributeValues={':Item': {'S': 'bar10'}})
|
||||
#
|
||||
result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
|
||||
assert result['id'] == {'S': 'foo'}
|
||||
assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}, {'S': 'bar10'}]}
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_list_index__set_nested_index_out_of_range():
|
||||
table_name = 'test_list_index_access'
|
||||
client = create_table_with_list(table_name)
|
||||
client.put_item(TableName=table_name,
|
||||
Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}}}})
|
||||
client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
|
||||
UpdateExpression='set itemmap.itemlist[10]=:Item',
|
||||
ExpressionAttributeValues={':Item': {'S': 'bar10'}})
|
||||
#
|
||||
result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
|
||||
assert result['id'] == {'S': 'foo2'}
|
||||
assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}, {'S': 'bar10'}]
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_list_index__set_index_of_a_string():
|
||||
table_name = 'test_list_index_access'
|
||||
client = create_table_with_list(table_name)
|
||||
client.put_item(TableName=table_name, Item={'id': {'S': 'foo2'}, 'itemstr': {'S': 'somestring'}})
|
||||
with assert_raises(ClientError) as ex:
|
||||
client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
|
||||
UpdateExpression='set itemstr[1]=:Item',
|
||||
ExpressionAttributeValues={':Item': {'S': 'string_update'}})
|
||||
result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
|
||||
|
||||
ex.exception.response['Error']['Code'].should.equal('ValidationException')
|
||||
ex.exception.response['Error']['Message'].should.equal(
|
||||
'The document path provided in the update expression is invalid for update')
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_remove_list_index__remove_existing_index():
|
||||
table_name = 'test_list_index_access'
|
||||
client = create_table_with_list(table_name)
|
||||
client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}}, UpdateExpression='REMOVE itemlist[1]')
|
||||
#
|
||||
result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
|
||||
assert result['id'] == {'S': 'foo'}
|
||||
assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar3'}]}
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_remove_list_index__remove_existing_nested_index():
|
||||
table_name = 'test_list_index_access'
|
||||
client = create_table_with_list(table_name)
|
||||
client.put_item(TableName=table_name,
|
||||
Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}]}}}})
|
||||
client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}}, UpdateExpression='REMOVE itemmap.itemlist[1]')
|
||||
#
|
||||
result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
|
||||
assert result['id'] == {'S': 'foo2'}
|
||||
assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}]
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_remove_list_index__remove_existing_double_nested_index():
|
||||
table_name = 'test_list_index_access'
|
||||
client = create_table_with_list(table_name)
|
||||
client.put_item(TableName=table_name,
|
||||
Item={'id': {'S': 'foo2'},
|
||||
'itemmap': {'M': {'itemlist': {'L': [{'M': {'foo00': {'S': 'bar1'},
|
||||
'foo01': {'S': 'bar2'}}},
|
||||
{'M': {'foo10': {'S': 'bar1'},
|
||||
'foo11': {'S': 'bar2'}}}]}}}})
|
||||
client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
|
||||
UpdateExpression='REMOVE itemmap.itemlist[1].foo10')
|
||||
#
|
||||
result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
|
||||
assert result['id'] == {'S': 'foo2'}
|
||||
assert result['itemmap']['M']['itemlist']['L'][0]['M'].should.equal({'foo00': {'S': 'bar1'},
|
||||
'foo01': {'S': 'bar2'}}) # untouched
|
||||
assert result['itemmap']['M']['itemlist']['L'][1]['M'].should.equal({'foo11': {'S': 'bar2'}}) # changed
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_remove_list_index__remove_index_out_of_range():
|
||||
table_name = 'test_list_index_access'
|
||||
client = create_table_with_list(table_name)
|
||||
client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}}, UpdateExpression='REMOVE itemlist[10]')
|
||||
#
|
||||
result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
|
||||
assert result['id'] == {'S': 'foo'}
|
||||
assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}
|
||||
|
||||
|
||||
def create_table_with_list(table_name):
|
||||
client = boto3.client('dynamodb', region_name='us-east-1')
|
||||
client.create_table(TableName=table_name,
|
||||
KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}],
|
||||
AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}],
|
||||
BillingMode='PAY_PER_REQUEST')
|
||||
client.put_item(TableName=table_name,
|
||||
Item={'id': {'S': 'foo'}, 'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}})
|
||||
return client
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_sorted_query_with_numerical_sort_key():
|
||||
dynamodb = boto3.resource('dynamodb', region_name='us-east-1')
|
||||
|
||||
Loading…
Reference in New Issue
Block a user