Fix UPDATED_NEW return values differences between moto and dynamoDB

moto/dynamodb2/responses.py

@@ -1,9 +1,12 @@
 from __future__ import unicode_literals
-import itertools
+
+import copy
 import json
-import six
 import re
 
+import itertools
+import six
+
 from moto.core.responses import BaseResponse
 from moto.core.utils import camelcase_to_underscores, amzn_request_id
 from .exceptions import InvalidIndexNameError, InvalidUpdateExpression, ItemSizeTooLarge
@@ -710,7 +713,8 @@ class DynamoHandler(BaseResponse):
         attribute_updates = self.body.get("AttributeUpdates")
         expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
         expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
-        existing_item = self.dynamodb_backend.get_item(name, key)
+        # We need to copy the item in order to avoid it being modified by the update_item operation
+        existing_item = copy.deepcopy(self.dynamodb_backend.get_item(name, key))
         if existing_item:
             existing_attributes = existing_item.to_json()["Attributes"]
         else:
@@ -796,14 +800,37 @@ class DynamoHandler(BaseResponse):
                 k: v for k, v in existing_attributes.items() if k in changed_attributes
             }
         elif return_values == "UPDATED_NEW":
-            item_dict["Attributes"] = {
+            item_dict["Attributes"] = self._build_updated_new_attributes(
-                k: v
+                existing_attributes, item_dict["Attributes"]
-                for k, v in item_dict["Attributes"].items()
+            )
-                if k in changed_attributes
-            }
 
         return dynamo_json_dump(item_dict)
 
+    def _build_updated_new_attributes(self, original, changed):
+        if type(changed) != type(original):
+            return changed
+        else:
+            if type(changed) is dict:
+                return {
+                    key: self._build_updated_new_attributes(
+                        original.get(key, None), changed[key]
+                    )
+                    for key in changed.keys()
+                    if changed[key] != original.get(key, None)
+                }
+            elif type(changed) in (set, list):
+                if len(changed) != len(original):
+                    return changed
+                else:
+                    return [
+                        self._build_updated_new_attributes(original[index], changed[index])
+                        for index in range(len(changed))
+                    ]
+            elif changed != original:
+                return changed
+            else:
+                return None
+
     def describe_limits(self):
         return json.dumps(
             {

tests/test_dynamodb2/test_dynamodb.py

@@ -3412,13 +3412,18 @@ def test_update_supports_list_append():
     )
 
     # Update item using list_append expression
-    client.update_item(
+    updated_item = client.update_item(
         TableName="TestTable",
         Key={"SHA256": {"S": "sha-of-file"}},
         UpdateExpression="SET crontab = list_append(crontab, :i)",
         ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}},
+        ReturnValues="UPDATED_NEW",
     )
 
+    # Verify updated item is correct
+    updated_item["Attributes"].should.equal(
+        {"crontab": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}
+    )
     # Verify item is appended to the existing list
     result = client.get_item(
         TableName="TestTable", Key={"SHA256": {"S": "sha-of-file"}}
@@ -3451,15 +3456,19 @@ def test_update_supports_nested_list_append():
     )
 
     # Update item using list_append expression
-    client.update_item(
+    updated_item = client.update_item(
         TableName="TestTable",
         Key={"id": {"S": "nested_list_append"}},
         UpdateExpression="SET a.#b = list_append(a.#b, :i)",
         ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}},
         ExpressionAttributeNames={"#b": "b"},
+        ReturnValues="UPDATED_NEW",
     )
 
-    # Verify item is appended to the existing list
+    # Verify updated item is correct
+    updated_item["Attributes"].should.equal(
+        {"a": {"M": {"b": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}}
+    )
     result = client.get_item(
         TableName="TestTable", Key={"id": {"S": "nested_list_append"}}
     )["Item"]
@@ -3491,14 +3500,19 @@ def test_update_supports_multiple_levels_nested_list_append():
     )
 
     # Update item using list_append expression
-    client.update_item(
+    updated_item = client.update_item(
         TableName="TestTable",
         Key={"id": {"S": "nested_list_append"}},
         UpdateExpression="SET a.#b.c = list_append(a.#b.#c, :i)",
         ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}},
         ExpressionAttributeNames={"#b": "b", "#c": "c"},
+        ReturnValues="UPDATED_NEW",
     )
 
+    # Verify updated item is correct
+    updated_item["Attributes"].should.equal(
+        {"a": {"M": {"b": {"M": {"c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}}}}
+    )
     # Verify item is appended to the existing list
     result = client.get_item(
         TableName="TestTable", Key={"id": {"S": "nested_list_append"}}
@@ -3532,14 +3546,19 @@ def test_update_supports_nested_list_append_onto_another_list():
     )
 
     # Update item using list_append expression
-    client.update_item(
+    updated_item = client.update_item(
         TableName="TestTable",
         Key={"id": {"S": "list_append_another"}},
         UpdateExpression="SET a.#c = list_append(a.#b, :i)",
         ExpressionAttributeValues={":i": {"L": [{"S": "bar2"}]}},
         ExpressionAttributeNames={"#b": "b", "#c": "c"},
+        ReturnValues="UPDATED_NEW",
     )
 
+    # Verify updated item is correct
+    updated_item["Attributes"].should.equal(
+        {"a": {"M": {"c": {"L": [{"S": "bar1"}, {"S": "bar2"}]}}}}
+    )
     # Verify item is appended to the existing list
     result = client.get_item(
         TableName="TestTable", Key={"id": {"S": "list_append_another"}}
@@ -3582,13 +3601,18 @@ def test_update_supports_list_append_maps():
     )
 
     # Update item using list_append expression
-    client.update_item(
+    updated_item = client.update_item(
         TableName="TestTable",
         Key={"id": {"S": "nested_list_append"}, "rid": {"S": "range_key"}},
         UpdateExpression="SET a = list_append(a, :i)",
         ExpressionAttributeValues={":i": {"L": [{"M": {"b": {"S": "bar2"}}}]}},
+        ReturnValues="UPDATED_NEW",
     )
 
+    # Verify updated item is correct
+    updated_item["Attributes"].should.equal(
+        {"a": {"L": [{"M": {"b": {"S": "bar1"}}}, {"M": {"b": {"S": "bar2"}}}]}}
+    )
     # Verify item is appended to the existing list
     result = client.query(
         TableName="TestTable",