Fix: iot:UpdateThingShadow does not properly maintain state document (#4045)

Device shadow updates affect only the fields specified in the request state document.
Any field with a value of null is removed from the device's shadow.[1]

Verified behavior against a real AWS backend.

[1]: https://docs.aws.amazon.com/iot/latest/developerguide/device-shadow-rest-api.html#API_UpdateThingShadow

moto/core/utils.py

@@ -408,3 +408,26 @@ def remap_nested_keys(root, key_transform):
             for k, v in six.iteritems(root)
         }
     return root
+
+
+def merge_dicts(dict1, dict2, remove_nulls=False):
+    """Given two arbitrarily nested dictionaries, merge the second dict into the first.
+
+    :param dict dict1: the dictionary to be updated.
+    :param dict dict2: a dictionary of keys/values to be merged into dict1.
+
+    :param bool remove_nulls: If true, updated values equal to None or an empty dictionary
+        will be removed from dict1.
+    """
+    for key in dict2:
+        if isinstance(dict2[key], dict):
+            if key in dict1 and key in dict2:
+                merge_dicts(dict1[key], dict2[key], remove_nulls)
+            else:
+                dict1[key] = dict2[key]
+            if dict1[key] == {} and remove_nulls:
+                dict1.pop(key)
+        else:
+            dict1[key] = dict2[key]
+            if dict1[key] is None and remove_nulls:
+                dict1.pop(key)

moto/iotdata/models.py

@@ -5,6 +5,7 @@ import jsondiff
 from boto3 import Session
 
 from moto.core import BaseBackend, BaseModel
+from moto.core.utils import merge_dicts
 from moto.iot import iot_backends
 from .exceptions import (
     ConflictException,
@@ -50,13 +51,12 @@ class FakeShadow(BaseModel):
             shadow = FakeShadow(None, None, None, version, deleted=True)
             return shadow
 
-        # we can make sure that payload has 'state' key
-        desired = payload["state"].get(
-            "desired", previous_payload.get("state", {}).get("desired", None)
-        )
-        reported = payload["state"].get(
-            "reported", previous_payload.get("state", {}).get("reported", None)
-        )
+        # Updates affect only the fields specified in the request state document.
+        # Any field with a value of None is removed from the device's shadow.
+        state_document = previous_payload.copy()
+        merge_dicts(state_document, payload, remove_nulls=True)
+        desired = state_document.get("state", {}).get("desired")
+        reported = state_document.get("state", {}).get("reported")
         shadow = FakeShadow(desired, reported, payload, version)
         return shadow
 

tests/test_iotdata/test_iotdata.py

@@ -109,3 +109,41 @@ def test_update():
 def test_publish():
     client = boto3.client("iot-data", region_name="ap-northeast-1")
     client.publish(topic="test/topic", qos=1, payload=b"")
+
+
+@mock_iot
+@mock_iotdata
+def test_delete_field_from_device_shadow():
+    test_thing_name = "TestThing"
+
+    iot_raw_client = boto3.client("iot", region_name="eu-central-1")
+    iot_raw_client.create_thing(thingName=test_thing_name)
+    iot = boto3.client("iot-data", region_name="eu-central-1")
+
+    iot.update_thing_shadow(
+        thingName=test_thing_name,
+        payload=json.dumps({"state": {"desired": {"state1": 1, "state2": 2}}}),
+    )
+    response = json.loads(
+        iot.get_thing_shadow(thingName=test_thing_name)["payload"].read()
+    )
+    assert len(response["state"]["desired"]) == 2
+
+    iot.update_thing_shadow(
+        thingName=test_thing_name,
+        payload=json.dumps({"state": {"desired": {"state1": None}}}),
+    )
+    response = json.loads(
+        iot.get_thing_shadow(thingName=test_thing_name)["payload"].read()
+    )
+    assert len(response["state"]["desired"]) == 1
+    assert "state2" in response["state"]["desired"]
+
+    iot.update_thing_shadow(
+        thingName=test_thing_name,
+        payload=json.dumps({"state": {"desired": {"state2": None}}}),
+    )
+    response = json.loads(
+        iot.get_thing_shadow(thingName=test_thing_name)["payload"].read()
+    )
+    assert "desired" not in response["state"]