Merge pull request #2467 from bblommers/bugfix/1823

DynamoDB: Add List index operations

moto/dynamodb2/exceptions.py

@@ -2,6 +2,9 @@ class InvalidIndexNameError(ValueError):
     pass
 
 
+class InvalidUpdateExpression(ValueError):
+    pass
+
+
 class ItemSizeTooLarge(Exception):
     message = 'Item size has exceeded the maximum allowed size'
-    pass

moto/dynamodb2/models.py

@@ -16,7 +16,7 @@ from moto.core.exceptions import JsonRESTError
 from .comparisons import get_comparison_func
 from .comparisons import get_filter_expression
 from .comparisons import get_expected
-from .exceptions import InvalidIndexNameError, ItemSizeTooLarge
+from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge
 
 
 class DynamoJsonEncoder(json.JSONEncoder):
@@ -237,7 +237,16 @@ class Item(BaseModel):
                 if action == "REMOVE":
                     key = value
                     if '.' not in key:
-                        self.attrs.pop(value, None)
+                        list_index_update = re.match('(.+)\\[([0-9]+)\\]', key)
+                        if list_index_update:
+                            # We need to remove an item from a list (REMOVE listattr[0])
+                            key_attr = self.attrs[list_index_update.group(1)]
+                            list_index = int(list_index_update.group(2))
+                            if key_attr.is_list():
+                                if len(key_attr.value) > list_index:
+                                    del key_attr.value[list_index]
+                        else:
+                            self.attrs.pop(value, None)
                     else:
                         # Handle nested dict updates
                         key_parts = key.split('.')
@@ -247,6 +256,9 @@ class Item(BaseModel):
 
                         last_val = self.attrs[attr].value
                         for key_part in key_parts[:-1]:
+                            list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_part)
+                            if list_index_update:
+                                key_part = list_index_update.group(1)  # listattr[1] ==> listattr
                             # Hack but it'll do, traverses into a dict
                             last_val_type = list(last_val.keys())
                             if last_val_type and last_val_type[0] == 'M':
@@ -256,12 +268,23 @@ class Item(BaseModel):
                                 last_val[key_part] = {'M': {}}
 
                             last_val = last_val[key_part]
+                            if list_index_update:
+                                last_val = last_val['L'][int(list_index_update.group(2))]
 
                         last_val_type = list(last_val.keys())
-                        if last_val_type and last_val_type[0] == 'M':
+                        list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_parts[-1])
-                            last_val['M'].pop(key_parts[-1], None)
+                        if list_index_update:
+                            # We need to remove an item from a list (REMOVE attr.listattr[0])
+                            key_part = list_index_update.group(1)  # listattr[1] ==> listattr
+                            list_to_update = last_val[key_part]['L']
+                            index_to_remove = int(list_index_update.group(2))
+                            if index_to_remove < len(list_to_update):
+                                del list_to_update[index_to_remove]
                         else:
-                            last_val.pop(key_parts[-1], None)
+                            if last_val_type and last_val_type[0] == 'M':
+                                last_val['M'].pop(key_parts[-1], None)
+                            else:
+                                last_val.pop(key_parts[-1], None)
                 elif action == 'SET':
                     key, value = value.split("=", 1)
                     key = key.strip()
@@ -282,38 +305,61 @@ class Item(BaseModel):
 
                     if type(value) != DynamoType:
                         if value in expression_attribute_values:
-                            value = DynamoType(expression_attribute_values[value])
+                            dyn_value = DynamoType(expression_attribute_values[value])
                         else:
-                            value = DynamoType({"S": value})
+                            dyn_value = DynamoType({"S": value})
+                    else:
+                        dyn_value = value
 
                     if '.' not in key:
-                        self.attrs[key] = value
+                        list_index_update = re.match('(.+)\\[([0-9]+)\\]', key)
+                        if list_index_update:
+                            key_attr = self.attrs[list_index_update.group(1)]
+                            list_index = int(list_index_update.group(2))
+                            if key_attr.is_list():
+                                if len(key_attr.value) > list_index:
+                                    key_attr.value[list_index] = expression_attribute_values[value]
+                                else:
+                                    key_attr.value.append(expression_attribute_values[value])
+                            else:
+                                raise InvalidUpdateExpression
+                        else:
+                            self.attrs[key] = dyn_value
                     else:
                         # Handle nested dict updates
                         key_parts = key.split('.')
                         attr = key_parts.pop(0)
                         if attr not in self.attrs:
                             raise ValueError
-
                         last_val = self.attrs[attr].value
                         for key_part in key_parts:
+                            list_index_update = re.match('(.+)\\[([0-9]+)\\]', key_part)
+                            if list_index_update:
+                                key_part = list_index_update.group(1)  # listattr[1] ==> listattr
                             # Hack but it'll do, traverses into a dict
                             last_val_type = list(last_val.keys())
                             if last_val_type and last_val_type[0] == 'M':
-                                    last_val = last_val['M']
+                                last_val = last_val['M']
 
                             if key_part not in last_val:
                                 last_val[key_part] = {'M': {}}
-
                             last_val = last_val[key_part]
 
-                        # We have reference to a nested object but we cant just assign to it
                         current_type = list(last_val.keys())[0]
-                        if current_type == value.type:
+                        if list_index_update:
-                            last_val[current_type] = value.value
+                            # We need to add an item to a list
+                            list_index = int(list_index_update.group(2))
+                            if len(last_val['L']) > list_index:
+                                last_val['L'][list_index] = expression_attribute_values[value]
+                            else:
+                                last_val['L'].append(expression_attribute_values[value])
                         else:
-                            last_val[value.type] = value.value
+                            # We have reference to a nested object but we cant just assign to it
-                            del last_val[current_type]
+                            if current_type == dyn_value.type:
+                                last_val[current_type] = dyn_value.value
+                            else:
+                                last_val[dyn_value.type] = dyn_value.value
+                                del last_val[current_type]
 
                 elif action == 'ADD':
                     key, value = value.split(" ", 1)

moto/dynamodb2/responses.py

@@ -6,7 +6,7 @@ import re
 
 from moto.core.responses import BaseResponse
 from moto.core.utils import camelcase_to_underscores, amzn_request_id
-from .exceptions import InvalidIndexNameError, ItemSizeTooLarge
+from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge
 from .models import dynamodb_backends, dynamo_json_dump
 
 
@@ -683,6 +683,9 @@ class DynamoHandler(BaseResponse):
                 name, key, update_expression, attribute_updates, expression_attribute_names,
                 expression_attribute_values, expected, condition_expression
             )
+        except InvalidUpdateExpression:
+            er = 'com.amazonaws.dynamodb.v20111205#ValidationException'
+            return self.error(er, 'The document path provided in the update expression is invalid for update')
         except ItemSizeTooLarge:
             er = 'com.amazonaws.dynamodb.v20111205#ValidationException'
             return self.error(er, ItemSizeTooLarge.message)

tests/test_dynamodb2/test_dynamodb.py

@@ -2506,6 +2506,144 @@ def test_index_with_unknown_attributes_should_fail():
     ex.exception.response['Error']['Message'].should.contain(expected_exception)
 
 
+@mock_dynamodb2
+def test_update_list_index__set_existing_index():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}},
+                       UpdateExpression='set itemlist[1]=:Item',
+                       ExpressionAttributeValues={':Item': {'S': 'bar2_update'}})
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
+    assert result['id'] == {'S': 'foo'}
+    assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2_update'}, {'S': 'bar3'}]}
+
+
+@mock_dynamodb2
+def test_update_list_index__set_existing_nested_index():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.put_item(TableName=table_name,
+                    Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}}}})
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
+                       UpdateExpression='set itemmap.itemlist[1]=:Item',
+                       ExpressionAttributeValues={':Item': {'S': 'bar2_update'}})
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
+    assert result['id'] == {'S': 'foo2'}
+    assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}, {'S': 'bar2_update'}, {'S': 'bar3'}]
+
+
+@mock_dynamodb2
+def test_update_list_index__set_index_out_of_range():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}},
+                       UpdateExpression='set itemlist[10]=:Item',
+                       ExpressionAttributeValues={':Item': {'S': 'bar10'}})
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
+    assert result['id'] == {'S': 'foo'}
+    assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}, {'S': 'bar10'}]}
+
+
+@mock_dynamodb2
+def test_update_list_index__set_nested_index_out_of_range():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.put_item(TableName=table_name,
+                    Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}}}})
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
+                       UpdateExpression='set itemmap.itemlist[10]=:Item',
+                       ExpressionAttributeValues={':Item': {'S': 'bar10'}})
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
+    assert result['id'] == {'S': 'foo2'}
+    assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}, {'S': 'bar10'}]
+
+
+@mock_dynamodb2
+def test_update_list_index__set_index_of_a_string():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.put_item(TableName=table_name, Item={'id': {'S': 'foo2'}, 'itemstr': {'S': 'somestring'}})
+    with assert_raises(ClientError) as ex:
+        client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
+                           UpdateExpression='set itemstr[1]=:Item',
+                           ExpressionAttributeValues={':Item': {'S': 'string_update'}})
+        result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
+
+    ex.exception.response['Error']['Code'].should.equal('ValidationException')
+    ex.exception.response['Error']['Message'].should.equal(
+        'The document path provided in the update expression is invalid for update')
+
+
+@mock_dynamodb2
+def test_remove_list_index__remove_existing_index():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}}, UpdateExpression='REMOVE itemlist[1]')
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
+    assert result['id'] == {'S': 'foo'}
+    assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar3'}]}
+
+
+@mock_dynamodb2
+def test_remove_list_index__remove_existing_nested_index():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.put_item(TableName=table_name,
+                    Item={'id': {'S': 'foo2'}, 'itemmap': {'M': {'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}]}}}})
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}}, UpdateExpression='REMOVE itemmap.itemlist[1]')
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
+    assert result['id'] == {'S': 'foo2'}
+    assert result['itemmap']['M']['itemlist']['L'] == [{'S': 'bar1'}]
+
+
+@mock_dynamodb2
+def test_remove_list_index__remove_existing_double_nested_index():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.put_item(TableName=table_name,
+                    Item={'id': {'S': 'foo2'},
+                          'itemmap': {'M': {'itemlist': {'L': [{'M': {'foo00': {'S': 'bar1'},
+                                                                      'foo01': {'S': 'bar2'}}},
+                                                               {'M': {'foo10': {'S': 'bar1'},
+                                                                      'foo11': {'S': 'bar2'}}}]}}}})
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo2'}},
+                       UpdateExpression='REMOVE itemmap.itemlist[1].foo10')
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo2'}})['Item']
+    assert result['id'] == {'S': 'foo2'}
+    assert result['itemmap']['M']['itemlist']['L'][0]['M'].should.equal({'foo00': {'S': 'bar1'},
+                                                                         'foo01': {'S': 'bar2'}})  # untouched
+    assert result['itemmap']['M']['itemlist']['L'][1]['M'].should.equal({'foo11': {'S': 'bar2'}})  # changed
+
+
+@mock_dynamodb2
+def test_remove_list_index__remove_index_out_of_range():
+    table_name = 'test_list_index_access'
+    client = create_table_with_list(table_name)
+    client.update_item(TableName=table_name, Key={'id': {'S': 'foo'}}, UpdateExpression='REMOVE itemlist[10]')
+    #
+    result = client.get_item(TableName=table_name, Key={'id': {'S': 'foo'}})['Item']
+    assert result['id'] == {'S': 'foo'}
+    assert result['itemlist'] == {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}
+
+
+def create_table_with_list(table_name):
+    client = boto3.client('dynamodb', region_name='us-east-1')
+    client.create_table(TableName=table_name,
+                        KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}],
+                        AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}],
+                        BillingMode='PAY_PER_REQUEST')
+    client.put_item(TableName=table_name,
+                    Item={'id': {'S': 'foo'}, 'itemlist': {'L': [{'S': 'bar1'}, {'S': 'bar2'}, {'S': 'bar3'}]}})
+    return client
+
+
 @mock_dynamodb2
 def test_sorted_query_with_numerical_sort_key():
     dynamodb = boto3.resource('dynamodb', region_name='us-east-1')