Add group features to iot (#1402)

* Add thing group features

* thing thing-group relation

* clean up comments

moto/iot/exceptions.py

@@ -16,9 +16,17 @@ class ResourceNotFoundException(IoTClientError):
 
 
 class InvalidRequestException(IoTClientError):
-    def __init__(self):
+    def __init__(self, msg=None):
         self.code = 400
         super(InvalidRequestException, self).__init__(
             "InvalidRequestException",
-            "The request is not valid."
+            msg or "The request is not valid."
+        )
+
+
+class VersionConflictException(IoTClientError):
+    def __init__(self, name):
+        self.code = 409
+        super(VersionConflictException, self).__init__(
+            'The version for thing %s does not match the expected version.' % name
         )

moto/iot/models.py

@@ -9,7 +9,8 @@ from moto.core import BaseBackend, BaseModel
 from collections import OrderedDict
 from .exceptions import (
     ResourceNotFoundException,
-    InvalidRequestException
+    InvalidRequestException,
+    VersionConflictException
 )
 
 
@@ -44,6 +45,7 @@ class FakeThingType(BaseModel):
         self.region_name = region_name
         self.thing_type_name = thing_type_name
         self.thing_type_properties = thing_type_properties
+        self.thing_type_id = str(uuid.uuid4())  # I don't know the rule of id
         t = time.time()
         self.metadata = {
             'deprecated': False,
@@ -54,11 +56,37 @@ class FakeThingType(BaseModel):
     def to_dict(self):
         return {
             'thingTypeName': self.thing_type_name,
+            'thingTypeId': self.thing_type_id,
             'thingTypeProperties': self.thing_type_properties,
             'thingTypeMetadata': self.metadata
         }
 
 
+class FakeThingGroup(BaseModel):
+    def __init__(self, thing_group_name, parent_group_name, thing_group_properties, region_name):
+        self.region_name = region_name
+        self.thing_group_name = thing_group_name
+        self.thing_group_id = str(uuid.uuid4())  # I don't know the rule of id
+        self.version = 1  # TODO: tmp
+        self.parent_group_name = parent_group_name
+        self.thing_group_properties = thing_group_properties or {}
+        t = time.time()
+        self.metadata = {
+            'creationData': int(t * 1000) / 1000.0
+        }
+        self.arn = 'arn:aws:iot:%s:1:thinggroup/%s' % (self.region_name, thing_group_name)
+        self.things = OrderedDict()
+
+    def to_dict(self):
+        return {
+            'thingGroupName': self.thing_group_name,
+            'thingGroupId': self.thing_group_id,
+            'version': self.version,
+            'thingGroupProperties': self.thing_group_properties,
+            'thingGroupMetadata': self.metadata
+        }
+
+
 class FakeCertificate(BaseModel):
     def __init__(self, certificate_pem, status, region_name):
         m = hashlib.sha256()
@@ -137,6 +165,7 @@ class IoTBackend(BaseBackend):
         self.region_name = region_name
         self.things = OrderedDict()
         self.thing_types = OrderedDict()
+        self.thing_groups = OrderedDict()
         self.certificates = OrderedDict()
         self.policies = OrderedDict()
         self.principal_policies = OrderedDict()
@@ -359,6 +388,125 @@ class IoTBackend(BaseBackend):
         principals = [k[0] for k, v in self.principal_things.items() if k[1] == thing_name]
         return principals
 
+    def describe_thing_group(self, thing_group_name):
+        thing_groups = [_ for _ in self.thing_groups.values() if _.thing_group_name == thing_group_name]
+        if len(thing_groups) == 0:
+            raise ResourceNotFoundException()
+        return thing_groups[0]
+
+    def create_thing_group(self, thing_group_name, parent_group_name, thing_group_properties):
+        thing_group = FakeThingGroup(thing_group_name, parent_group_name, thing_group_properties, self.region_name)
+        self.thing_groups[thing_group.arn] = thing_group
+        return thing_group.thing_group_name, thing_group.arn, thing_group.thing_group_id
+
+    def delete_thing_group(self, thing_group_name, expected_version):
+        thing_group = self.describe_thing_group(thing_group_name)
+        del self.thing_groups[thing_group.arn]
+
+    def list_thing_groups(self, parent_group, name_prefix_filter, recursive):
+        thing_groups = self.thing_groups.values()
+        return thing_groups
+
+    def update_thing_group(self, thing_group_name, thing_group_properties, expected_version):
+        thing_group = self.describe_thing_group(thing_group_name)
+        if expected_version and expected_version != thing_group.version:
+            raise VersionConflictException(thing_group_name)
+        attribute_payload = thing_group_properties.get('attributePayload', None)
+        if attribute_payload is not None and 'attributes' in attribute_payload:
+            do_merge = attribute_payload.get('merge', False)
+            attributes = attribute_payload['attributes']
+            if not do_merge:
+                thing_group.thing_group_properties['attributePayload']['attributes'] = attributes
+            else:
+                thing_group.thing_group_properties['attributePayload']['attributes'].update(attributes)
+        elif attribute_payload is not None and 'attributes' not in attribute_payload:
+            thing_group.attributes = {}
+        thing_group.version = thing_group.version + 1
+        return thing_group.version
+
+    def _identify_thing_group(self, thing_group_name, thing_group_arn):
+        # identify thing group
+        if thing_group_name is None and thing_group_arn is None:
+            raise InvalidRequestException(
+                ' Both thingGroupArn and thingGroupName are empty. Need to specify at least one of them'
+            )
+        if thing_group_name is not None:
+            thing_group = self.describe_thing_group(thing_group_name)
+            if thing_group_arn and thing_group.arn != thing_group_arn:
+                raise InvalidRequestException(
+                    'ThingGroupName thingGroupArn does not match specified thingGroupName in request'
+                )
+        elif thing_group_arn is not None:
+            if thing_group_arn not in self.thing_groups:
+                raise InvalidRequestException()
+            thing_group = self.thing_groups[thing_group_arn]
+        return thing_group
+
+    def _identify_thing(self, thing_name, thing_arn):
+        # identify thing
+        if thing_name is None and thing_arn is None:
+            raise InvalidRequestException(
+                'Both thingArn and thingName are empty. Need to specify at least one of them'
+            )
+        if thing_name is not None:
+            thing = self.describe_thing(thing_name)
+            if thing_arn and thing.arn != thing_arn:
+                raise InvalidRequestException(
+                    'ThingName thingArn does not match specified thingName in request'
+                )
+        elif thing_arn is not None:
+            if thing_arn not in self.things:
+                raise InvalidRequestException()
+            thing = self.things[thing_arn]
+        return thing
+
+    def add_thing_to_thing_group(self, thing_group_name, thing_group_arn, thing_name, thing_arn):
+        thing_group = self._identify_thing_group(thing_group_name, thing_group_arn)
+        thing = self._identify_thing(thing_name, thing_arn)
+        if thing.arn in thing_group.things:
+            # aws ignores duplicate registration
+            return
+        thing_group.things[thing.arn] = thing
+
+    def remove_thing_from_thing_group(self, thing_group_name, thing_group_arn, thing_name, thing_arn):
+        thing_group = self._identify_thing_group(thing_group_name, thing_group_arn)
+        thing = self._identify_thing(thing_name, thing_arn)
+        if thing.arn not in thing_group.things:
+            # aws ignores non-registered thing
+            return
+        del thing_group.things[thing.arn]
+
+    def list_things_in_thing_group(self, thing_group_name, recursive):
+        thing_group = self.describe_thing_group(thing_group_name)
+        return thing_group.things.values()
+
+    def list_thing_groups_for_thing(self, thing_name):
+        thing = self.describe_thing(thing_name)
+        all_thing_groups = self.list_thing_groups(None, None, None)
+        ret = []
+        for thing_group in all_thing_groups:
+            if thing.arn in thing_group.things:
+                ret.append({
+                    'groupName': thing_group.thing_group_name,
+                    'groupArn': thing_group.arn
+                })
+        return ret
+
+    def update_thing_groups_for_thing(self, thing_name, thing_groups_to_add, thing_groups_to_remove):
+        thing = self.describe_thing(thing_name)
+        for thing_group_name in thing_groups_to_add:
+            thing_group = self.describe_thing_group(thing_group_name)
+            self.add_thing_to_thing_group(
+                thing_group.thing_group_name, None,
+                thing.thing_name, None
+            )
+        for thing_group_name in thing_groups_to_remove:
+            thing_group = self.describe_thing_group(thing_group_name)
+            self.remove_thing_from_thing_group(
+                thing_group.thing_group_name, None,
+                thing.thing_name, None
+            )
+
 
 available_regions = boto3.session.Session().get_available_regions("iot")
 iot_backends = {region: IoTBackend(region) for region in available_regions}

moto/iot/responses.py

@@ -38,8 +38,7 @@ class IoTResponse(BaseResponse):
         thing_types = self.iot_backend.list_thing_types(
             thing_type_name=thing_type_name
         )
-
+        # TODO: implement pagination in the future
-        # TODO: support next_token and max_results
         next_token = None
         return json.dumps(dict(thingTypes=[_.to_dict() for _ in thing_types], nextToken=next_token))
 
@@ -54,7 +53,7 @@ class IoTResponse(BaseResponse):
             attribute_value=attribute_value,
             thing_type_name=thing_type_name,
         )
-        # TODO: support next_token and max_results
+        # TODO: implement pagination in the future
         next_token = None
         return json.dumps(dict(things=[_.to_dict() for _ in things], nextToken=next_token))
 
@@ -63,7 +62,6 @@ class IoTResponse(BaseResponse):
         thing = self.iot_backend.describe_thing(
             thing_name=thing_name,
         )
-        print(thing.to_dict(include_default_client_id=True))
         return json.dumps(thing.to_dict(include_default_client_id=True))
 
     def describe_thing_type(self):
@@ -135,7 +133,7 @@ class IoTResponse(BaseResponse):
         # marker = self._get_param("marker")
         # ascending_order = self._get_param("ascendingOrder")
         certificates = self.iot_backend.list_certificates()
-        # TODO: handle pagination
+        # TODO: implement pagination in the future
         return json.dumps(dict(certificates=[_.to_dict() for _ in certificates]))
 
     def update_certificate(self):
@@ -162,7 +160,7 @@ class IoTResponse(BaseResponse):
         # ascending_order = self._get_param("ascendingOrder")
         policies = self.iot_backend.list_policies()
 
-        # TODO: handle pagination
+        # TODO: implement pagination in the future
         return json.dumps(dict(policies=[_.to_dict() for _ in policies]))
 
     def get_policy(self):
@@ -205,7 +203,7 @@ class IoTResponse(BaseResponse):
         policies = self.iot_backend.list_principal_policies(
             principal_arn=principal
         )
-        # TODO: handle pagination
+        # TODO: implement pagination in the future
         next_marker = None
         return json.dumps(dict(policies=[_.to_dict() for _ in policies], nextMarker=next_marker))
 
@@ -217,7 +215,7 @@ class IoTResponse(BaseResponse):
         principals = self.iot_backend.list_policy_principals(
             policy_name=policy_name,
         )
-        # TODO: handle pagination
+        # TODO: implement pagination in the future
         next_marker = None
         return json.dumps(dict(principals=principals, nextMarker=next_marker))
 
@@ -246,7 +244,7 @@ class IoTResponse(BaseResponse):
         things = self.iot_backend.list_principal_things(
             principal_arn=principal,
         )
-        # TODO: handle pagination
+        # TODO: implement pagination in the future
         next_token = None
         return json.dumps(dict(things=things, nextToken=next_token))
 
@@ -256,3 +254,123 @@ class IoTResponse(BaseResponse):
             thing_name=thing_name,
         )
         return json.dumps(dict(principals=principals))
+
+    def describe_thing_group(self):
+        thing_group_name = self._get_param("thingGroupName")
+        thing_group = self.iot_backend.describe_thing_group(
+            thing_group_name=thing_group_name,
+        )
+        return json.dumps(thing_group.to_dict())
+
+    def create_thing_group(self):
+        thing_group_name = self._get_param("thingGroupName")
+        parent_group_name = self._get_param("parentGroupName")
+        thing_group_properties = self._get_param("thingGroupProperties")
+        thing_group_name, thing_group_arn, thing_group_id = self.iot_backend.create_thing_group(
+            thing_group_name=thing_group_name,
+            parent_group_name=parent_group_name,
+            thing_group_properties=thing_group_properties,
+        )
+        return json.dumps(dict(
+            thingGroupName=thing_group_name,
+            thingGroupArn=thing_group_arn,
+            thingGroupId=thing_group_id)
+        )
+
+    def delete_thing_group(self):
+        thing_group_name = self._get_param("thingGroupName")
+        expected_version = self._get_param("expectedVersion")
+        self.iot_backend.delete_thing_group(
+            thing_group_name=thing_group_name,
+            expected_version=expected_version,
+        )
+        return json.dumps(dict())
+
+    def list_thing_groups(self):
+        # next_token = self._get_param("nextToken")
+        # max_results = self._get_int_param("maxResults")
+        parent_group = self._get_param("parentGroup")
+        name_prefix_filter = self._get_param("namePrefixFilter")
+        recursive = self._get_param("recursive")
+        thing_groups = self.iot_backend.list_thing_groups(
+            parent_group=parent_group,
+            name_prefix_filter=name_prefix_filter,
+            recursive=recursive,
+        )
+        next_token = None
+        rets = [{'groupName': _.thing_group_name, 'groupArn': _.arn} for _ in thing_groups]
+        # TODO: implement pagination in the future
+        return json.dumps(dict(thingGroups=rets, nextToken=next_token))
+
+    def update_thing_group(self):
+        thing_group_name = self._get_param("thingGroupName")
+        thing_group_properties = self._get_param("thingGroupProperties")
+        expected_version = self._get_param("expectedVersion")
+        version = self.iot_backend.update_thing_group(
+            thing_group_name=thing_group_name,
+            thing_group_properties=thing_group_properties,
+            expected_version=expected_version,
+        )
+        return json.dumps(dict(version=version))
+
+    def add_thing_to_thing_group(self):
+        thing_group_name = self._get_param("thingGroupName")
+        thing_group_arn = self._get_param("thingGroupArn")
+        thing_name = self._get_param("thingName")
+        thing_arn = self._get_param("thingArn")
+        self.iot_backend.add_thing_to_thing_group(
+            thing_group_name=thing_group_name,
+            thing_group_arn=thing_group_arn,
+            thing_name=thing_name,
+            thing_arn=thing_arn,
+        )
+        return json.dumps(dict())
+
+    def remove_thing_from_thing_group(self):
+        thing_group_name = self._get_param("thingGroupName")
+        thing_group_arn = self._get_param("thingGroupArn")
+        thing_name = self._get_param("thingName")
+        thing_arn = self._get_param("thingArn")
+        self.iot_backend.remove_thing_from_thing_group(
+            thing_group_name=thing_group_name,
+            thing_group_arn=thing_group_arn,
+            thing_name=thing_name,
+            thing_arn=thing_arn,
+        )
+        return json.dumps(dict())
+
+    def list_things_in_thing_group(self):
+        thing_group_name = self._get_param("thingGroupName")
+        recursive = self._get_param("recursive")
+        # next_token = self._get_param("nextToken")
+        # max_results = self._get_int_param("maxResults")
+        things = self.iot_backend.list_things_in_thing_group(
+            thing_group_name=thing_group_name,
+            recursive=recursive,
+        )
+        next_token = None
+        thing_names = [_.thing_name for _ in things]
+        # TODO: implement pagination in the future
+        return json.dumps(dict(things=thing_names, nextToken=next_token))
+
+    def list_thing_groups_for_thing(self):
+        thing_name = self._get_param("thingName")
+        # next_token = self._get_param("nextToken")
+        # max_results = self._get_int_param("maxResults")
+        thing_groups = self.iot_backend.list_thing_groups_for_thing(
+            thing_name=thing_name
+        )
+        next_token = None
+        # TODO: implement pagination in the future
+        return json.dumps(dict(thingGroups=thing_groups, nextToken=next_token))
+
+    def update_thing_groups_for_thing(self):
+        thing_name = self._get_param("thingName")
+        thing_groups_to_add = self._get_param("thingGroupsToAdd") or []
+        thing_groups_to_remove = self._get_param("thingGroupsToRemove") or []
+        self.iot_backend.update_thing_groups_for_thing(
+            thing_name=thing_name,
+            thing_groups_to_add=thing_groups_to_add,
+            thing_groups_to_remove=thing_groups_to_remove,
+        )
+        return json.dumps(dict())

tests/test_iot/test_iot.py

@@ -177,3 +177,192 @@ def test_principal_thing():
     res.should.have.key('things').which.should.have.length_of(0)
     res = client.list_thing_principals(thingName=thing_name)
     res.should.have.key('principals').which.should.have.length_of(0)
+
+
+@mock_iot
+def test_thing_groups():
+    client = boto3.client('iot', region_name='ap-northeast-1')
+    name = 'my-thing'
+    group_name = 'my-group-name'
+
+    # thing group
+    thing_group = client.create_thing_group(thingGroupName=group_name)
+    thing_group.should.have.key('thingGroupName').which.should.equal(group_name)
+    thing_group.should.have.key('thingGroupArn')
+
+    res = client.list_thing_groups()
+    res.should.have.key('thingGroups').which.should.have.length_of(1)
+    for thing_group in res['thingGroups']:
+        thing_group.should.have.key('groupName').which.should_not.be.none
+        thing_group.should.have.key('groupArn').which.should_not.be.none
+
+    thing_group = client.describe_thing_group(thingGroupName=group_name)
+    thing_group.should.have.key('thingGroupName').which.should.equal(group_name)
+    thing_group.should.have.key('thingGroupProperties')
+    thing_group.should.have.key('thingGroupMetadata')
+    thing_group.should.have.key('version')
+
+    # delete thing group
+    client.delete_thing_group(thingGroupName=group_name)
+    res = client.list_thing_groups()
+    res.should.have.key('thingGroups').which.should.have.length_of(0)
+
+    # props create test
+    props = {
+        'thingGroupDescription': 'my first thing group',
+        'attributePayload': {
+            'attributes': {
+                'key1': 'val01',
+                'Key02': 'VAL2'
+            }
+        }
+    }
+    thing_group = client.create_thing_group(thingGroupName=group_name, thingGroupProperties=props)
+    thing_group.should.have.key('thingGroupName').which.should.equal(group_name)
+    thing_group.should.have.key('thingGroupArn')
+
+    thing_group = client.describe_thing_group(thingGroupName=group_name)
+    thing_group.should.have.key('thingGroupProperties')\
+                           .which.should.have.key('attributePayload')\
+                           .which.should.have.key('attributes')
+    res_props = thing_group['thingGroupProperties']['attributePayload']['attributes']
+    res_props.should.have.key('key1').which.should.equal('val01')
+    res_props.should.have.key('Key02').which.should.equal('VAL2')
+
+    # props update test with merge
+    new_props = {
+        'attributePayload': {
+            'attributes': {
+                'k3': 'v3'
+            },
+            'merge': True
+        }
+    }
+    client.update_thing_group(
+        thingGroupName=group_name,
+        thingGroupProperties=new_props
+    )
+    thing_group = client.describe_thing_group(thingGroupName=group_name)
+    thing_group.should.have.key('thingGroupProperties')\
+                           .which.should.have.key('attributePayload')\
+                           .which.should.have.key('attributes')
+    res_props = thing_group['thingGroupProperties']['attributePayload']['attributes']
+    res_props.should.have.key('key1').which.should.equal('val01')
+    res_props.should.have.key('Key02').which.should.equal('VAL2')
+
+    res_props.should.have.key('k3').which.should.equal('v3')
+
+    # props update test
+    new_props = {
+        'attributePayload': {
+            'attributes': {
+                'k4': 'v4'
+            }
+        }
+    }
+    client.update_thing_group(
+        thingGroupName=group_name,
+        thingGroupProperties=new_props
+    )
+    thing_group = client.describe_thing_group(thingGroupName=group_name)
+    thing_group.should.have.key('thingGroupProperties')\
+                           .which.should.have.key('attributePayload')\
+                           .which.should.have.key('attributes')
+    res_props = thing_group['thingGroupProperties']['attributePayload']['attributes']
+    res_props.should.have.key('k4').which.should.equal('v4')
+    res_props.should_not.have.key('key1')
+
+
+@mock_iot
+def test_thing_group_relations():
+    client = boto3.client('iot', region_name='ap-northeast-1')
+    name = 'my-thing'
+    group_name = 'my-group-name'
+
+    # thing group
+    thing_group = client.create_thing_group(thingGroupName=group_name)
+    thing_group.should.have.key('thingGroupName').which.should.equal(group_name)
+    thing_group.should.have.key('thingGroupArn')
+
+    # thing
+    thing = client.create_thing(thingName=name)
+    thing.should.have.key('thingName').which.should.equal(name)
+    thing.should.have.key('thingArn')
+
+    # add in 4 way
+    client.add_thing_to_thing_group(
+        thingGroupName=group_name,
+        thingName=name
+    )
+    client.add_thing_to_thing_group(
+        thingGroupArn=thing_group['thingGroupArn'],
+        thingArn=thing['thingArn']
+    )
+    client.add_thing_to_thing_group(
+        thingGroupName=group_name,
+        thingArn=thing['thingArn']
+    )
+    client.add_thing_to_thing_group(
+        thingGroupArn=thing_group['thingGroupArn'],
+        thingName=name
+    )
+
+    things = client.list_things_in_thing_group(
+        thingGroupName=group_name
+    )
+    things.should.have.key('things')
+    things['things'].should.have.length_of(1)
+
+    thing_groups = client.list_thing_groups_for_thing(
+        thingName=name
+    )
+    thing_groups.should.have.key('thingGroups')
+    thing_groups['thingGroups'].should.have.length_of(1)
+
+    # remove in 4 way
+    client.remove_thing_from_thing_group(
+        thingGroupName=group_name,
+        thingName=name
+    )
+    client.remove_thing_from_thing_group(
+        thingGroupArn=thing_group['thingGroupArn'],
+        thingArn=thing['thingArn']
+    )
+    client.remove_thing_from_thing_group(
+        thingGroupName=group_name,
+        thingArn=thing['thingArn']
+    )
+    client.remove_thing_from_thing_group(
+        thingGroupArn=thing_group['thingGroupArn'],
+        thingName=name
+    )
+    things = client.list_things_in_thing_group(
+        thingGroupName=group_name
+    )
+    things.should.have.key('things')
+    things['things'].should.have.length_of(0)
+
+    # update thing group for thing
+    client.update_thing_groups_for_thing(
+        thingName=name,
+        thingGroupsToAdd=[
+            group_name
+        ]
+    )
+    things = client.list_things_in_thing_group(
+        thingGroupName=group_name
+    )
+    things.should.have.key('things')
+    things['things'].should.have.length_of(1)
+
+    client.update_thing_groups_for_thing(
+        thingName=name,
+        thingGroupsToRemove=[
+            group_name
+        ]
+    )
+    things = client.list_things_in_thing_group(
+        thingGroupName=group_name
+    )
+    things.should.have.key('things')
+    things['things'].should.have.length_of(0)