From afb8f71e9fb443a8aa1468834ad99ec9f3d832a2 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 9 Oct 2019 17:25:50 +0100 Subject: [PATCH 1/2] #1822 - DynamoDB: Add List index operations --- moto/dynamodb2/exceptions.py | 4 + moto/dynamodb2/models.py | 73 ++++++++++++---- moto/dynamodb2/responses.py | 5 +- tests/test_dynamodb2/test_dynamodb.py | 119 ++++++++++++++++++++++++++ 4 files changed, 184 insertions(+), 17 deletions(-) diff --git a/moto/dynamodb2/exceptions.py b/moto/dynamodb2/exceptions.py index 9df973292..1dab2a56f 100644 --- a/moto/dynamodb2/exceptions.py +++ b/moto/dynamodb2/exceptions.py @@ -1,2 +1,6 @@ class InvalidIndexNameError(ValueError): pass + + +class InvalidUpdateExpression(ValueError): + pass diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index b2ff7ffc3..df8c89b72 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -16,7 +16,7 @@ from moto.core.exceptions import JsonRESTError from .comparisons import get_comparison_func from .comparisons import get_filter_expression from .comparisons import get_expected -from .exceptions import InvalidIndexNameError +from .exceptions import InvalidIndexNameError, InvalidUpdateExpression class DynamoJsonEncoder(json.JSONEncoder): @@ -184,7 +184,16 @@ class Item(BaseModel): if action == "REMOVE": key = value if '.' not in key: - self.attrs.pop(value, None) + list_index_update = re.match('(.+)\\[([0-9]+)\\]', key) + if list_index_update: + # We need to remove an item from a list (REMOVE listattr[0]) + key_attr = self.attrs[list_index_update.group(1)] + list_index = int(list_index_update.group(2)) + if key_attr.is_list(): + if len(key_attr.value) > list_index: + del key_attr.value[list_index] + else: + self.attrs.pop(value, None) else: # Handle nested dict updates key_parts = key.split('.') @@ -205,10 +214,19 @@ class Item(BaseModel): last_val = last_val[key_part] last_val_type = list(last_val.keys()) - if last_val_type and last_val_type[0] == 'M': - last_val['M'].pop(key_parts[-1], None) + list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_parts[-1]) + if list_index_update: + # We need to remove an item from a list (REMOVE attr.listattr[0]) + key_part = list_index_update.group(1) # listattr[1] ==> listattr + list_to_update = last_val[key_part]['L'] + index_to_remove = int(list_index_update.group(2)) + if index_to_remove < len(list_to_update): + del list_to_update[index_to_remove] else: - last_val.pop(key_parts[-1], None) + if last_val_type and last_val_type[0] == 'M': + last_val['M'].pop(key_parts[-1], None) + else: + last_val.pop(key_parts[-1], None) elif action == 'SET': key, value = value.split("=", 1) key = key.strip() @@ -229,38 +247,61 @@ class Item(BaseModel): if type(value) != DynamoType: if value in expression_attribute_values: - value = DynamoType(expression_attribute_values[value]) + dyn_value = DynamoType(expression_attribute_values[value]) else: - value = DynamoType({"S": value}) + dyn_value = DynamoType({"S": value}) + else: + dyn_value = value if '.' not in key: - self.attrs[key] = value + list_index_update = re.match('(.+)\\[([0-9]+)\\]', key) + if list_index_update: + key_attr = self.attrs[list_index_update.group(1)] + list_index = int(list_index_update.group(2)) + if key_attr.is_list(): + if len(key_attr.value) > list_index: + key_attr.value[list_index] = expression_attribute_values[value] + else: + key_attr.value.append(expression_attribute_values[value]) + else: + raise InvalidUpdateExpression + else: + self.attrs[key] = dyn_value else: # Handle nested dict updates key_parts = key.split('.') attr = key_parts.pop(0) if attr not in self.attrs: raise ValueError - last_val = self.attrs[attr].value for key_part in key_parts: + list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_part) + if list_index_update: + key_part = list_index_update.group(1) # listattr[1] ==> listattr # Hack but it'll do, traverses into a dict last_val_type = list(last_val.keys()) if last_val_type and last_val_type[0] == 'M': - last_val = last_val['M'] + last_val = last_val['M'] if key_part not in last_val: last_val[key_part] = {'M': {}} - last_val = last_val[key_part] - # We have reference to a nested object but we cant just assign to it current_type = list(last_val.keys())[0] - if current_type == value.type: - last_val[current_type] = value.value + if list_index_update: + # We need to add an item to a list + list_index = int(list_index_update.group(2)) + if len(last_val['L']) > list_index: + last_val['L'][list_index] = expression_attribute_values[value] + else: + last_val['L'].append(expression_attribute_values[value]) else: - last_val[value.type] = value.value - del last_val[current_type] + # We have reference to a nested object but we cant just assign to it + if current_type == dyn_value.type: + last_val[current_type] = dyn_value.value + else: + last_val[dyn_value.type] = dyn_value.value + del last_val[current_type] elif action == 'ADD': key, value = value.split(" ", 1) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 9e66a3342..7f4c7d214 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -6,7 +6,7 @@ import re from moto.core.responses import BaseResponse from moto.core.utils import camelcase_to_underscores, amzn_request_id -from .exceptions import InvalidIndexNameError +from .exceptions import InvalidIndexNameError, InvalidUpdateExpression from .models import dynamodb_backends, dynamo_json_dump @@ -658,6 +658,9 @@ class DynamoHandler(BaseResponse): name, key, update_expression, attribute_updates, expression_attribute_names, expression_attribute_values, expected, condition_expression ) + except InvalidUpdateExpression: + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er, 'The document path provided in the update expression is invalid for update') except ValueError: er = 'com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException' return self.error(er, 'A condition specified in the operation could not be evaluated.') diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index c4cf05c59..356be7817 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2295,6 +2295,125 @@ def test_index_with_unknown_attributes_should_fail(): ex.exception.response['Error']['Message'].should.contain(expected_exception) +@mock_dynamodb2 +def test_update_list_index__set_existing_index(): + table_name = 'test_list_index_access' + client = create_table_with_list(table_name) + client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}}, + UpdateExpression='set itemlist[1]=:Item', + ExpressionAttributeValues={':Item': {'S': 'bar2_update'}}) + # + result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item'] + assert result['id'] == {'S': 'foo'} + assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2_update'}, {'S': 'bar3'}]} + + +@mock_dynamodb2 +def test_update_list_index__set_existing_nested_index(): + table_name = 'test_list_index_access' + client = create_table_with_list(table_name) + client.put_item(TableName=table_name, + Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}}}}) + client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}}, + UpdateExpression='set itemmap.itemlist[1]=:Item', + ExpressionAttributeValues={':Item': {'S': 'bar2_update'}}) + # + result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item'] + assert result['id'] == {'S': 'foo2'} + assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}, {'S': 'bar2_update'}, {'S': 'bar3'}] + + +@mock_dynamodb2 +def test_update_list_index__set_index_out_of_range(): + table_name = 'test_list_index_access' + client = create_table_with_list(table_name) + client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}}, + UpdateExpression='set itemlist[10]=:Item', + ExpressionAttributeValues={':Item': {'S': 'bar10'}}) + # + result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item'] + assert result['id'] == {'S': 'foo'} + assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}, {'S': 'bar10'}]} + + +@mock_dynamodb2 +def test_update_list_index__set_nested_index_out_of_range(): + table_name = 'test_list_index_access' + client = create_table_with_list(table_name) + client.put_item(TableName=table_name, + Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}}}}) + client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}}, + UpdateExpression='set itemmap.itemlist[10]=:Item', + ExpressionAttributeValues={':Item': {'S': 'bar10'}}) + # + result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item'] + assert result['id'] == {'S': 'foo2'} + assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}, {'S': 'bar10'}] + + +@mock_dynamodb2 +def test_update_list_index__set_index_of_a_string(): + table_name = 'test_list_index_access' + client = create_table_with_list(table_name) + client.put_item(TableName=table_name, Item={'id': {'S': 'foo2'}, 'itemstr': {'S': 'somestring'}}) + with assert_raises(ClientError) as ex: + client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}}, + UpdateExpression='set itemstr[1]=:Item', + ExpressionAttributeValues={':Item': {'S': 'string_update'}}) + result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item'] + print(result) + + ex.exception.response['Error']['Code'].should.equal('ValidationException') + ex.exception.response['Error']['Message'].should.equal( + 'The document path provided in the update expression is invalid for update') + + +@mock_dynamodb2 +def test_remove_list_index__remove_existing_index(): + table_name = 'test_list_index_access' + client = create_table_with_list(table_name) + client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}}, UpdateExpression='REMOVE itemlist[1]') + # + result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item'] + assert result['id'] == {'S': 'foo'} + assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar3'}]} + + +@mock_dynamodb2 +def test_remove_list_index__remove_existing_nested_index(): + table_name = 'test_list_index_access' + client = create_table_with_list(table_name) + client.put_item(TableName=table_name, + Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}]}}}}) + client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}}, UpdateExpression='REMOVE itemmap.itemlist[1]') + # + result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item'] + assert result['id'] == {'S': 'foo2'} + assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}] + + +@mock_dynamodb2 +def test_remove_list_index__remove_index_out_of_range(): + table_name = 'test_list_index_access' + client = create_table_with_list(table_name) + client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}}, UpdateExpression='REMOVE itemlist[10]') + # + result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item'] + assert result['id'] == {'S': 'foo'} + assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]} + + +def create_table_with_list(table_name): + client = boto3.client('dynamodb', region_name='us-east-1') + client.create_table(TableName=table_name, + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + BillingMode='PAY_PER_REQUEST') + client.put_item(TableName=table_name, + Item={'id': {'S': 'foo'}, 'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}}) + return client + + def _create_user_table(): client = boto3.client('dynamodb', region_name='us-east-1') client.create_table( From 106692ed880a3cf58b9bdcfd59cf5c87f01e71a0 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Mon, 14 Oct 2019 09:59:52 +0100 Subject: [PATCH 2/2] #1823 - Add support to delete items from a nested list --- moto/dynamodb2/models.py | 5 +++++ tests/test_dynamodb2/test_dynamodb.py | 21 ++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index df8c89b72..5b1992cdc 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -203,6 +203,9 @@ class Item(BaseModel): last_val = self.attrs[attr].value for key_part in key_parts[:-1]: + list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_part) + if list_index_update: + key_part = list_index_update.group(1) # listattr[1] ==> listattr # Hack but it'll do, traverses into a dict last_val_type = list(last_val.keys()) if last_val_type and last_val_type[0] == 'M': @@ -212,6 +215,8 @@ class Item(BaseModel): last_val[key_part] = {'M': {}} last_val = last_val[key_part] + if list_index_update: + last_val = last_val['L'][int(list_index_update.group(2))] last_val_type = list(last_val.keys()) list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_parts[-1]) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 356be7817..b4fb5e8c8 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2361,7 +2361,6 @@ def test_update_list_index__set_index_of_a_string(): UpdateExpression='set itemstr[1]=:Item', ExpressionAttributeValues={':Item': {'S': 'string_update'}}) result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item'] - print(result) ex.exception.response['Error']['Code'].should.equal('ValidationException') ex.exception.response['Error']['Message'].should.equal( @@ -2392,6 +2391,26 @@ def test_remove_list_index__remove_existing_nested_index(): assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}] +@mock_dynamodb2 +def test_remove_list_index__remove_existing_double_nested_index(): + table_name = 'test_list_index_access' + client = create_table_with_list(table_name) + client.put_item(TableName=table_name, + Item={'id': {'S': 'foo2'}, + 'itemmap': {'M': {'itemlist': {'L': [{'M': {'foo00': {'S': 'bar1'}, + 'foo01': {'S': 'bar2'}}}, + {'M': {'foo10': {'S': 'bar1'}, + 'foo11': {'S': 'bar2'}}}]}}}}) + client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}}, + UpdateExpression='REMOVE itemmap.itemlist[1].foo10') + # + result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item'] + assert result['id'] == {'S': 'foo2'} + assert result['itemmap']['M']['itemlist']['L'][0]['M'].should.equal({'foo00': {'S': 'bar1'}, + 'foo01': {'S': 'bar2'}}) # untouched + assert result['itemmap']['M']['itemlist']['L'][1]['M'].should.equal({'foo11': {'S': 'bar2'}}) # changed + + @mock_dynamodb2 def test_remove_list_index__remove_index_out_of_range(): table_name = 'test_list_index_access'