From 3a355f126cea926fd03815ad240e9e29839a5228 Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Wed, 13 Jun 2018 16:14:18 +0200 Subject: [PATCH 1/8] first steps undertaken to fix spulec/moto#1684 and spulec/moto#1685 --- moto/ecr/models.py | 29 +++++++++----- tests/test_ecr/test_ecr_boto3.py | 68 +++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 15 deletions(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index e20c550c9..87d02c3b1 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -4,12 +4,12 @@ import hashlib from copy import copy from random import random +from botocore.exceptions import ParamValidationError + from moto.core import BaseBackend, BaseModel from moto.ec2 import ec2_backends from moto.ecr.exceptions import ImageNotFoundException, RepositoryNotFoundException -from botocore.exceptions import ParamValidationError - DEFAULT_REGISTRY_ID = '012345678910' @@ -97,13 +97,13 @@ class Repository(BaseObject): class Image(BaseObject): - def __init__(self, tag, manifest, repository, registry_id=DEFAULT_REGISTRY_ID): + def __init__(self, tag, manifest, repository, digest=None, registry_id=DEFAULT_REGISTRY_ID): self.image_tag = tag self.image_manifest = manifest self.image_size_in_bytes = 50 * 1024 * 1024 self.repository = repository self.registry_id = registry_id - self.image_digest = None + self.image_digest = digest self.image_pushed_at = None def _create_digest(self): @@ -115,6 +115,9 @@ class Image(BaseObject): self._create_digest() return self.image_digest + def get_image_manifest(self): + return self.image_manifest + @property def response_object(self): response_object = self.gen_response_object() @@ -124,14 +127,14 @@ class Image(BaseObject): response_object['imageManifest'] = self.image_manifest response_object['repositoryName'] = self.repository response_object['registryId'] = self.registry_id - return response_object + return {k: v for k, v in response_object.items() if v is not None and v != [None]} @property def response_list_object(self): response_object = self.gen_response_object() response_object['imageTag'] = self.image_tag response_object['imageDigest'] = "i don't know" - return response_object + return {k: v for k, v in response_object.items() if v is not None and v != [None]} @property def response_describe_object(self): @@ -143,7 +146,7 @@ class Image(BaseObject): response_object['registryId'] = self.registry_id response_object['imageSizeInBytes'] = self.image_size_in_bytes response_object['imagePushedAt'] = '2017-05-09' - return response_object + return {k: v for k, v in response_object.items() if v is not None and v != [None]} @property def response_batch_get_image(self): @@ -154,7 +157,7 @@ class Image(BaseObject): response_object['imageManifest'] = self.image_manifest response_object['repositoryName'] = self.repository response_object['registryId'] = self.registry_id - return response_object + return {k: v for k, v in response_object.items() if v is not None and v != [None]} class ECRBackend(BaseBackend): @@ -252,8 +255,14 @@ class ECRBackend(BaseBackend): else: raise Exception("{0} is not a repository".format(repository_name)) - image = Image(image_tag, image_manifest, repository_name) - repository.images.append(image) + existing_image = list(filter(lambda x: x.response_object['imageManifest'] == image_manifest, repository.images)) + if not existing_image: + image = Image(image_tag, image_manifest, repository_name) + repository.images.append(image) + else: + image = Image(image_tag, image_manifest, repository_name, existing_image[0].get_image_digest()) + repository.images.append(image) + return image def batch_get_image(self, repository_name, registry_id=None, image_ids=None, accepted_media_types=None): diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 7651dc832..43a41a4d5 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -197,6 +197,54 @@ def test_put_image(): response['image']['repositoryName'].should.equal('test_repository') response['image']['registryId'].should.equal('012345678910') +@mock_ecr +def test_put_image_with_multiple_tags(): + client = boto3.client('ecr', region_name='us-east-1') + _ = client.create_repository( + repositoryName='test_repository' + ) + manifest = _create_image_manifest() + response = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest), + imageTag='v1' + ) + + response['image']['imageId']['imageTag'].should.equal('v1') + response['image']['imageId']['imageDigest'].should.contain("sha") + response['image']['repositoryName'].should.equal('test_repository') + response['image']['registryId'].should.equal('012345678910') + + response1 = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest), + imageTag='latest' + ) + + response1['image']['imageId']['imageTag'].should.equal('latest') + response1['image']['imageId']['imageDigest'].should.contain("sha") + response1['image']['repositoryName'].should.equal('test_repository') + response1['image']['registryId'].should.equal('012345678910') + + response2 = client.describe_images(repositoryName='test_repository') + type(response2['imageDetails']).should.be(list) + len(response2['imageDetails']).should.be(1) + + response['imageDetails'][0]['imageDigest'].should.contain("sha") + + # response['imageDetails'][0]['registryId'].should.equal("012345678910") + # response['imageDetails'][1]['registryId'].should.equal("012345678910") + # response['imageDetails'][2]['registryId'].should.equal("012345678910") + # response['imageDetails'][3]['registryId'].should.equal("012345678910") + # + # response['imageDetails'][0]['repositoryName'].should.equal("test_repository") + # response['imageDetails'][1]['repositoryName'].should.equal("test_repository") + # response['imageDetails'][2]['repositoryName'].should.equal("test_repository") + # response['imageDetails'][3]['repositoryName'].should.equal("test_repository") + # + # response['imageDetails'][0].should_not.have.key('imageTags') + # len(response['imageDetails'][1]['imageTags']).should.be(1) + @mock_ecr def test_list_images(): @@ -259,6 +307,11 @@ def test_describe_images(): repositoryName='test_repository' ) + _ = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(_create_image_manifest()) + ) + _ = client.put_image( repositoryName='test_repository', imageManifest=json.dumps(_create_image_manifest()), @@ -279,32 +332,37 @@ def test_describe_images(): response = client.describe_images(repositoryName='test_repository') type(response['imageDetails']).should.be(list) - len(response['imageDetails']).should.be(3) + len(response['imageDetails']).should.be(4) response['imageDetails'][0]['imageDigest'].should.contain("sha") response['imageDetails'][1]['imageDigest'].should.contain("sha") response['imageDetails'][2]['imageDigest'].should.contain("sha") + response['imageDetails'][3]['imageDigest'].should.contain("sha") response['imageDetails'][0]['registryId'].should.equal("012345678910") response['imageDetails'][1]['registryId'].should.equal("012345678910") response['imageDetails'][2]['registryId'].should.equal("012345678910") + response['imageDetails'][3]['registryId'].should.equal("012345678910") response['imageDetails'][0]['repositoryName'].should.equal("test_repository") response['imageDetails'][1]['repositoryName'].should.equal("test_repository") response['imageDetails'][2]['repositoryName'].should.equal("test_repository") + response['imageDetails'][3]['repositoryName'].should.equal("test_repository") - len(response['imageDetails'][0]['imageTags']).should.be(1) + response['imageDetails'][0].should_not.have.key('imageTags') len(response['imageDetails'][1]['imageTags']).should.be(1) len(response['imageDetails'][2]['imageTags']).should.be(1) + len(response['imageDetails'][3]['imageTags']).should.be(1) image_tags = ['latest', 'v1', 'v2'] - set([response['imageDetails'][0]['imageTags'][0], - response['imageDetails'][1]['imageTags'][0], - response['imageDetails'][2]['imageTags'][0]]).should.equal(set(image_tags)) + set([response['imageDetails'][1]['imageTags'][0], + response['imageDetails'][2]['imageTags'][0], + response['imageDetails'][3]['imageTags'][0]]).should.equal(set(image_tags)) response['imageDetails'][0]['imageSizeInBytes'].should.equal(52428800) response['imageDetails'][1]['imageSizeInBytes'].should.equal(52428800) response['imageDetails'][2]['imageSizeInBytes'].should.equal(52428800) + response['imageDetails'][3]['imageSizeInBytes'].should.equal(52428800) @mock_ecr From cc799b55daf1f3c05238f1f2a6dc52903f89c9df Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 14 Jun 2018 09:07:09 +0200 Subject: [PATCH 2/8] fixed spulec/moto#1684 and fixed spulec/moto#1685 --- moto/ecr/models.py | 21 ++++++++++++++------- tests/test_ecr/test_ecr_boto3.py | 22 ++++++++-------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index 87d02c3b1..a5502df47 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -99,6 +99,7 @@ class Image(BaseObject): def __init__(self, tag, manifest, repository, digest=None, registry_id=DEFAULT_REGISTRY_ID): self.image_tag = tag + self.image_tags = [tag] self.image_manifest = manifest self.image_size_in_bytes = 50 * 1024 * 1024 self.repository = repository @@ -118,6 +119,11 @@ class Image(BaseObject): def get_image_manifest(self): return self.image_manifest + def update_tag(self, tag): + self.image_tag = tag + if tag not in self.image_tags: + self.image_tags.append(tag) + @property def response_object(self): response_object = self.gen_response_object() @@ -139,7 +145,7 @@ class Image(BaseObject): @property def response_describe_object(self): response_object = self.gen_response_object() - response_object['imageTags'] = [self.image_tag] + response_object['imageTags'] = self.image_tags response_object['imageDigest'] = self.get_image_digest() response_object['imageManifest'] = self.image_manifest response_object['repositoryName'] = self.repository @@ -255,15 +261,16 @@ class ECRBackend(BaseBackend): else: raise Exception("{0} is not a repository".format(repository_name)) - existing_image = list(filter(lambda x: x.response_object['imageManifest'] == image_manifest, repository.images)) - if not existing_image: + existing_images = list(filter(lambda x: x.response_object['imageManifest'] == image_manifest, repository.images)) + if not existing_images: + # this image is not in ECR yet image = Image(image_tag, image_manifest, repository_name) repository.images.append(image) + return image else: - image = Image(image_tag, image_manifest, repository_name, existing_image[0].get_image_digest()) - repository.images.append(image) - - return image + # update existing image + existing_images[0].update_tag(image_tag) + return existing_images[0] def batch_get_image(self, repository_name, registry_id=None, image_ids=None, accepted_media_types=None): if repository_name in self.repositories: diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 43a41a4d5..8ecab9001 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -45,7 +45,8 @@ def _create_image_manifest(): { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", "size": 73109, - "digest": _create_image_digest("layer3") + # randomize image digest + "digest": _create_image_digest() } ] } @@ -230,21 +231,14 @@ def test_put_image_with_multiple_tags(): type(response2['imageDetails']).should.be(list) len(response2['imageDetails']).should.be(1) - response['imageDetails'][0]['imageDigest'].should.contain("sha") + response2['imageDetails'][0]['imageDigest'].should.contain("sha") - # response['imageDetails'][0]['registryId'].should.equal("012345678910") - # response['imageDetails'][1]['registryId'].should.equal("012345678910") - # response['imageDetails'][2]['registryId'].should.equal("012345678910") - # response['imageDetails'][3]['registryId'].should.equal("012345678910") - # - # response['imageDetails'][0]['repositoryName'].should.equal("test_repository") - # response['imageDetails'][1]['repositoryName'].should.equal("test_repository") - # response['imageDetails'][2]['repositoryName'].should.equal("test_repository") - # response['imageDetails'][3]['repositoryName'].should.equal("test_repository") - # - # response['imageDetails'][0].should_not.have.key('imageTags') - # len(response['imageDetails'][1]['imageTags']).should.be(1) + response2['imageDetails'][0]['registryId'].should.equal("012345678910") + response2['imageDetails'][0]['repositoryName'].should.equal("test_repository") + + len(response2['imageDetails'][0]['imageTags']).should.be(2) + response2['imageDetails'][0]['imageTags'].should.be.equal(['v1', 'latest']) @mock_ecr def test_list_images(): From 6e269d1e3146bfabfd34c43840e8a31e4b01c644 Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 14 Jun 2018 09:10:06 +0200 Subject: [PATCH 3/8] fixes spulec/moto#1673 and updated IMPLEMENTATION_COVERAGE.md --- IMPLEMENTATION_COVERAGE.md | 78 +++++++++++++++--------------- scripts/implementation_coverage.py | 7 +-- 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 411f55a8b..75bf254ef 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -814,17 +814,17 @@ - [ ] update_team_member - [ ] update_user_profile -## cognito-identity - 0% implemented -- [ ] create_identity_pool +## cognito-identity - 22% implemented +- [X] create_identity_pool - [ ] delete_identities - [ ] delete_identity_pool - [ ] describe_identity - [ ] describe_identity_pool -- [ ] get_credentials_for_identity -- [ ] get_id +- [X] get_credentials_for_identity +- [X] get_id - [ ] get_identity_pool_roles - [ ] get_open_id_token -- [ ] get_open_id_token_for_developer_identity +- [X] get_open_id_token_for_developer_identity - [ ] list_identities - [ ] list_identity_pools - [ ] lookup_developer_identity @@ -834,20 +834,20 @@ - [ ] unlink_identity - [ ] update_identity_pool -## cognito-idp - 0% implemented +## cognito-idp - 25% implemented - [ ] add_custom_attributes - [ ] admin_add_user_to_group - [ ] admin_confirm_sign_up -- [ ] admin_create_user -- [ ] admin_delete_user +- [X] admin_create_user +- [X] admin_delete_user - [ ] admin_delete_user_attributes - [ ] admin_disable_provider_for_user - [ ] admin_disable_user - [ ] admin_enable_user - [ ] admin_forget_device - [ ] admin_get_device -- [ ] admin_get_user -- [ ] admin_initiate_auth +- [X] admin_get_user +- [X] admin_initiate_auth - [ ] admin_link_provider_for_user - [ ] admin_list_devices - [ ] admin_list_groups_for_user @@ -862,32 +862,32 @@ - [ ] admin_update_user_attributes - [ ] admin_user_global_sign_out - [ ] associate_software_token -- [ ] change_password +- [X] change_password - [ ] confirm_device -- [ ] confirm_forgot_password +- [X] confirm_forgot_password - [ ] confirm_sign_up - [ ] create_group -- [ ] create_identity_provider +- [X] create_identity_provider - [ ] create_resource_server - [ ] create_user_import_job -- [ ] create_user_pool -- [ ] create_user_pool_client -- [ ] create_user_pool_domain +- [X] create_user_pool +- [X] create_user_pool_client +- [X] create_user_pool_domain - [ ] delete_group -- [ ] delete_identity_provider +- [X] delete_identity_provider - [ ] delete_resource_server - [ ] delete_user - [ ] delete_user_attributes -- [ ] delete_user_pool -- [ ] delete_user_pool_client -- [ ] delete_user_pool_domain -- [ ] describe_identity_provider +- [X] delete_user_pool +- [X] delete_user_pool_client +- [X] delete_user_pool_domain +- [X] describe_identity_provider - [ ] describe_resource_server - [ ] describe_risk_configuration - [ ] describe_user_import_job -- [ ] describe_user_pool -- [ ] describe_user_pool_client -- [ ] describe_user_pool_domain +- [X] describe_user_pool +- [X] describe_user_pool_client +- [X] describe_user_pool_domain - [ ] forget_device - [ ] forgot_password - [ ] get_csv_header @@ -903,15 +903,15 @@ - [ ] initiate_auth - [ ] list_devices - [ ] list_groups -- [ ] list_identity_providers +- [X] list_identity_providers - [ ] list_resource_servers - [ ] list_user_import_jobs -- [ ] list_user_pool_clients -- [ ] list_user_pools -- [ ] list_users +- [X] list_user_pool_clients +- [X] list_user_pools +- [X] list_users - [ ] list_users_in_group - [ ] resend_confirmation_code -- [ ] respond_to_auth_challenge +- [X] respond_to_auth_challenge - [ ] set_risk_configuration - [ ] set_ui_customization - [ ] set_user_mfa_preference @@ -927,7 +927,7 @@ - [ ] update_resource_server - [ ] update_user_attributes - [ ] update_user_pool -- [ ] update_user_pool_client +- [X] update_user_pool_client - [ ] verify_software_token - [ ] verify_user_attribute @@ -2524,11 +2524,11 @@ - [X] update_thing_group - [X] update_thing_groups_for_thing -## iot-data - 0% implemented -- [ ] delete_thing_shadow -- [ ] get_thing_shadow -- [ ] publish -- [ ] update_thing_shadow +## iot-data - 100% implemented +- [X] delete_thing_shadow +- [X] get_thing_shadow +- [X] publish +- [X] update_thing_shadow ## iot-jobs-data - 0% implemented - [ ] describe_job_execution @@ -2815,7 +2815,7 @@ - [ ] update_domain_entry - [ ] update_load_balancer_attribute -## logs - 24% implemented +## logs - 27% implemented - [ ] associate_kms_key - [ ] cancel_export_task - [ ] create_export_task @@ -2830,7 +2830,7 @@ - [ ] delete_subscription_filter - [ ] describe_destinations - [ ] describe_export_tasks -- [ ] describe_log_groups +- [X] describe_log_groups - [X] describe_log_streams - [ ] describe_metric_filters - [ ] describe_resource_policies @@ -3703,13 +3703,13 @@ - [ ] put_attributes - [ ] select -## secretsmanager - 0% implemented +## secretsmanager - 6% implemented - [ ] cancel_rotate_secret - [ ] create_secret - [ ] delete_secret - [ ] describe_secret - [ ] get_random_password -- [ ] get_secret_value +- [X] get_secret_value - [ ] list_secret_version_ids - [ ] list_secrets - [ ] put_secret_value diff --git a/scripts/implementation_coverage.py b/scripts/implementation_coverage.py index 74ce9590d..1541c4c75 100755 --- a/scripts/implementation_coverage.py +++ b/scripts/implementation_coverage.py @@ -7,12 +7,13 @@ import boto3 def get_moto_implementation(service_name): - if not hasattr(moto, service_name): + service_name_standardized = service_name.replace("-", "") if "-" in service_name else service_name + if not hasattr(moto, service_name_standardized): return None - module = getattr(moto, service_name) + module = getattr(moto, service_name_standardized) if module is None: return None - mock = getattr(module, "mock_{}".format(service_name)) + mock = getattr(module, "mock_{}".format(service_name_standardized)) if mock is None: return None backends = list(mock().backends.values()) From ea3366be35d2345d1769637da9e6a697fbbd87f9 Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 14 Jun 2018 09:53:11 +0200 Subject: [PATCH 4/8] do not allow None as value of image_tags --- moto/ecr/models.py | 6 ++-- tests/test_ecr/test_ecr_boto3.py | 62 ++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index a5502df47..d3e8aa219 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -99,7 +99,7 @@ class Image(BaseObject): def __init__(self, tag, manifest, repository, digest=None, registry_id=DEFAULT_REGISTRY_ID): self.image_tag = tag - self.image_tags = [tag] + self.image_tags = [tag] if tag is not None else [] self.image_manifest = manifest self.image_size_in_bytes = 50 * 1024 * 1024 self.repository = repository @@ -121,7 +121,7 @@ class Image(BaseObject): def update_tag(self, tag): self.image_tag = tag - if tag not in self.image_tags: + if tag not in self.image_tags and tag is not None: self.image_tags.append(tag) @property @@ -235,7 +235,7 @@ class ECRBackend(BaseBackend): found = False for image in repository.images: if (('imageDigest' in image_id and image.get_image_digest() == image_id['imageDigest']) or - ('imageTag' in image_id and image.image_tag == image_id['imageTag'])): + ('imageTag' in image_id and image_id['imageTag'] in image.image_tags)): found = True response.add(image) if not found: diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 8ecab9001..d542184a7 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -385,6 +385,68 @@ def test_describe_images_by_tag(): image_detail['imageDigest'].should.equal(put_response['imageId']['imageDigest']) +@mock_ecr +def test_describe_images_tags_should_not_contain_empty_tag1(): + client = boto3.client('ecr', region_name='us-east-1') + _ = client.create_repository( + repositoryName='test_repository' + ) + + manifest = _create_image_manifest() + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest) + ) + + tags = ['v1', 'v2', 'latest'] + for tag in tags: + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest), + imageTag=tag + ) + + response = client.describe_images(repositoryName='test_repository', imageIds=[{'imageTag': tag}]) + len(response['imageDetails']).should.be(1) + image_detail = response['imageDetails'][0] + len(image_detail['imageTags']).should.equal(3) + image_detail['imageTags'].should.be.equal(tags) + + +@mock_ecr +def test_describe_images_tags_should_not_contain_empty_tag2(): + client = boto3.client('ecr', region_name='us-east-1') + _ = client.create_repository( + repositoryName='test_repository' + ) + + manifest = _create_image_manifest() + tags = ['v1', 'v2'] + for tag in tags: + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest), + imageTag=tag + ) + + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest) + ) + + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest), + imageTag='latest' + ) + + response = client.describe_images(repositoryName='test_repository', imageIds=[{'imageTag': tag}]) + len(response['imageDetails']).should.be(1) + image_detail = response['imageDetails'][0] + len(image_detail['imageTags']).should.equal(3) + image_detail['imageTags'].should.be.equal(['v1', 'v2', 'latest']) + + @mock_ecr def test_describe_repository_that_doesnt_exist(): client = boto3.client('ecr', region_name='us-east-1') From 56ff66394d00f5a7d33d65df865c35f3126818fb Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 14 Jun 2018 09:56:53 +0200 Subject: [PATCH 5/8] updated to make sure that tests still run correctly --- moto/ecr/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index d3e8aa219..a8ee60c5a 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -152,7 +152,7 @@ class Image(BaseObject): response_object['registryId'] = self.registry_id response_object['imageSizeInBytes'] = self.image_size_in_bytes response_object['imagePushedAt'] = '2017-05-09' - return {k: v for k, v in response_object.items() if v is not None and v != [None]} + return {k: v for k, v in response_object.items() if v is not None and v != []} @property def response_batch_get_image(self): From db3593575f0ddeb699c1230c293f0db1ec1a48be Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 28 Jun 2018 10:32:51 +0200 Subject: [PATCH 6/8] list_thing_types and list_things now uses pagination --- moto/iot/models.py | 52 ++++++++++---- moto/iot/responses.py | 35 +++++---- tests/test_iot/test_iot.py | 144 ++++++++++++++++++++++++++++++++++++- 3 files changed, 202 insertions(+), 29 deletions(-) diff --git a/moto/iot/models.py b/moto/iot/models.py index ce7a4cf57..0ef53bbb5 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -1,14 +1,17 @@ from __future__ import unicode_literals -import time -import boto3 -import string -import random + import hashlib -import uuid +import random import re -from datetime import datetime -from moto.core import BaseBackend, BaseModel +import string +import time +import uuid from collections import OrderedDict +from datetime import datetime + +import boto3 + +from moto.core import BaseBackend, BaseModel from .exceptions import ( ResourceNotFoundException, InvalidRequestException, @@ -271,15 +274,36 @@ class IoTBackend(BaseBackend): def list_thing_types(self, thing_type_name=None): if thing_type_name: - # It's wierd but thing_type_name is filterd by forward match, not complete match + # It's weird but thing_type_name is filtered by forward match, not complete match return [_ for _ in self.thing_types.values() if _.thing_type_name.startswith(thing_type_name)] - thing_types = self.thing_types.values() - return thing_types + return self.thing_types.values() - def list_things(self, attribute_name, attribute_value, thing_type_name): - # TODO: filter by attributess or thing_type - things = self.things.values() - return things + def list_things(self, attribute_name, attribute_value, thing_type_name, max_results, token): + all_things = [_.to_dict() for _ in self.things.values()] + if attribute_name is not None and thing_type_name is not None: + filtered_things = list( + filter( + lambda elem: attribute_name in elem["attributes"] and elem["attributes"][ + attribute_name] == attribute_value and "thingTypeName" in elem and elem[ + "thingTypeName"] == thing_type_name, all_things)) + elif attribute_name is not None and thing_type_name is None: + filtered_things = list(filter(lambda elem: attribute_name in elem["attributes"] and elem["attributes"][ + attribute_name] == attribute_value, all_things)) + elif attribute_name is None and thing_type_name is not None: + filtered_things = list( + filter(lambda elem: "thingTypeName" in elem and elem["thingTypeName"] == thing_type_name, all_things)) + else: + filtered_things = all_things + + if token is None: + things = filtered_things[0:max_results] + next_token = str(max_results) if len(filtered_things) > max_results else None + else: + token = int(token) + things = filtered_things[token:token + max_results] + next_token = str(token + max_results) if len(filtered_things) > token + max_results else None + + return things, next_token def describe_thing(self, thing_name): things = [_ for _ in self.things.values() if _.thing_name == thing_name] diff --git a/moto/iot/responses.py b/moto/iot/responses.py index fcdf12f78..006c4c4cc 100644 --- a/moto/iot/responses.py +++ b/moto/iot/responses.py @@ -1,7 +1,9 @@ from __future__ import unicode_literals + +import json + from moto.core.responses import BaseResponse from .models import iot_backends -import json class IoTResponse(BaseResponse): @@ -32,30 +34,39 @@ class IoTResponse(BaseResponse): return json.dumps(dict(thingTypeName=thing_type_name, thingTypeArn=thing_type_arn)) def list_thing_types(self): - # previous_next_token = self._get_param("nextToken") - # max_results = self._get_int_param("maxResults") + previous_next_token = self._get_param("nextToken") + max_results = self._get_int_param("maxResults", 50) # not the default, but makes testing easier thing_type_name = self._get_param("thingTypeName") thing_types = self.iot_backend.list_thing_types( thing_type_name=thing_type_name ) - # TODO: implement pagination in the future - next_token = None - return json.dumps(dict(thingTypes=[_.to_dict() for _ in thing_types], nextToken=next_token)) + + thing_types = [_.to_dict() for _ in thing_types] + if previous_next_token is None: + result = thing_types[0:max_results] + next_token = str(max_results) if len(thing_types) > max_results else None + else: + token = int(previous_next_token) + result = thing_types[token:token + max_results] + next_token = str(token + max_results) if len(thing_types) > token + max_results else None + + return json.dumps(dict(thingTypes=result, nextToken=next_token)) def list_things(self): - # previous_next_token = self._get_param("nextToken") - # max_results = self._get_int_param("maxResults") + previous_next_token = self._get_param("nextToken") + max_results = self._get_int_param("maxResults", 50) # not the default, but makes testing easier attribute_name = self._get_param("attributeName") attribute_value = self._get_param("attributeValue") thing_type_name = self._get_param("thingTypeName") - things = self.iot_backend.list_things( + things, next_token = self.iot_backend.list_things( attribute_name=attribute_name, attribute_value=attribute_value, thing_type_name=thing_type_name, + max_results=max_results, + token=previous_next_token ) - # TODO: implement pagination in the future - next_token = None - return json.dumps(dict(things=[_.to_dict() for _ in things], nextToken=next_token)) + + return json.dumps(dict(things=things, nextToken=next_token)) def describe_thing(self): thing_name = self._get_param("thingName") diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 213615790..5e78f6e49 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -1,8 +1,9 @@ from __future__ import unicode_literals -import boto3 -import sure # noqa import json +import sure # noqa +import boto3 + from moto import mock_iot @@ -63,6 +64,143 @@ def test_things(): res.should.have.key('thingTypes').which.should.have.length_of(0) +@mock_iot +def test_list_thing_types(): + client = boto3.client('iot', region_name='ap-northeast-1') + + for i in range(0, 100): + client.create_thing_type(thingTypeName=str(i + 1)) + + thing_types = client.list_thing_types() + thing_types.should.have.key('nextToken') + thing_types.should.have.key('thingTypes').which.should.have.length_of(50) + thing_types['thingTypes'][0]['thingTypeName'].should.equal('1') + thing_types['thingTypes'][-1]['thingTypeName'].should.equal('50') + + thing_types = client.list_thing_types(nextToken=thing_types['nextToken']) + thing_types.should.have.key('thingTypes').which.should.have.length_of(50) + thing_types.should_not.have.key('nextToken') + thing_types['thingTypes'][0]['thingTypeName'].should.equal('51') + thing_types['thingTypes'][-1]['thingTypeName'].should.equal('100') + # TODO test list_thing_types with filters + + +@mock_iot +def test_list_things_with_next_token(): + client = boto3.client('iot', region_name='ap-northeast-1') + + for i in range(0, 200): + client.create_thing(thingName=str(i + 1)) + + things = client.list_things() + things.should.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('1') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/1') + things['things'][-1]['thingName'].should.equal('50') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/50') + + things = client.list_things(nextToken=things['nextToken']) + things.should.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('51') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/51') + things['things'][-1]['thingName'].should.equal('100') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/100') + + things = client.list_things(nextToken=things['nextToken']) + things.should.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('101') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/101') + things['things'][-1]['thingName'].should.equal('150') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/150') + + things = client.list_things(nextToken=things['nextToken']) + things.should_not.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('151') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/151') + things['things'][-1]['thingName'].should.equal('200') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/200') + + +@mock_iot +def test_list_things_with_attribute_and_thing_type_filter_and_next_token(): + client = boto3.client('iot', region_name='ap-northeast-1') + client.create_thing_type(thingTypeName='my-thing-type') + + for i in range(0, 200): + if not (i + 1) % 3: + attribute_payload = { + 'attributes': { + 'foo': 'bar' + } + } + elif not (i + 1) % 5: + attribute_payload = { + 'attributes': { + 'bar': 'foo' + } + } + else: + attribute_payload = {} + + if not (i + 1) % 2: + thing_type_name = 'my-thing-type' + client.create_thing(thingName=str(i + 1), thingTypeName=thing_type_name, attributePayload=attribute_payload) + else: + client.create_thing(thingName=str(i + 1), attributePayload=attribute_payload) + + # Test filter for thingTypeName + things = client.list_things(thingTypeName=thing_type_name) + things.should.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('2') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/2') + things['things'][-1]['thingName'].should.equal('100') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/100') + all(item['thingTypeName'] == thing_type_name for item in things['things']) + + things = client.list_things(nextToken=things['nextToken'], thingTypeName=thing_type_name) + things.should_not.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('102') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/102') + things['things'][-1]['thingName'].should.equal('200') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/200') + all(item['thingTypeName'] == thing_type_name for item in things['things']) + + # Test filter for attributes + things = client.list_things(attributeName='foo', attributeValue='bar') + things.should.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('3') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/3') + things['things'][-1]['thingName'].should.equal('150') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/150') + all(item['attributes'] == {'foo': 'bar'} for item in things['things']) + + things = client.list_things(nextToken=things['nextToken'], attributeName='foo', attributeValue='bar') + things.should_not.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(16) + things['things'][0]['thingName'].should.equal('153') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/153') + things['things'][-1]['thingName'].should.equal('198') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/198') + all(item['attributes'] == {'foo': 'bar'} for item in things['things']) + + # Test filter for attributes and thingTypeName + things = client.list_things(thingTypeName=thing_type_name, attributeName='foo', attributeValue='bar') + things.should_not.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(33) + things['things'][0]['thingName'].should.equal('6') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/6') + things['things'][-1]['thingName'].should.equal('198') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/198') + all(item['attributes'] == {'foo': 'bar'} and item['thingTypeName'] == thing_type_name for item in things['things']) + + @mock_iot def test_certs(): client = boto3.client('iot', region_name='ap-northeast-1') @@ -204,7 +342,6 @@ def test_principal_thing(): @mock_iot def test_thing_groups(): client = boto3.client('iot', region_name='ap-northeast-1') - name = 'my-thing' group_name = 'my-group-name' # thing group @@ -424,6 +561,7 @@ def test_create_job(): job.should.have.key('jobArn') job.should.have.key('description') + @mock_iot def test_describe_job(): client = boto3.client('iot', region_name='eu-west-1') From 10f96b2ccfa55114c9f2b32355efc4decd0ebb0f Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 28 Jun 2018 12:59:08 +0200 Subject: [PATCH 7/8] next_token (pagination) added for `list_thing_types` --- tests/test_iot/test_iot.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 5e78f6e49..5c6effd7a 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -82,7 +82,30 @@ def test_list_thing_types(): thing_types.should_not.have.key('nextToken') thing_types['thingTypes'][0]['thingTypeName'].should.equal('51') thing_types['thingTypes'][-1]['thingTypeName'].should.equal('100') - # TODO test list_thing_types with filters + + +@mock_iot +def test_list_thing_types_with_typename_filter(): + client = boto3.client('iot', region_name='ap-northeast-1') + + client.create_thing_type(thingTypeName='thing') + client.create_thing_type(thingTypeName='thingType') + client.create_thing_type(thingTypeName='thingTypeName') + client.create_thing_type(thingTypeName='thingTypeNameGroup') + client.create_thing_type(thingTypeName='shouldNotFind') + client.create_thing_type(thingTypeName='find me it shall not') + + thing_types = client.list_thing_types(thingTypeName='thing') + thing_types.should_not.have.key('nextToken') + thing_types.should.have.key('thingTypes').which.should.have.length_of(4) + thing_types['thingTypes'][0]['thingTypeName'].should.equal('thing') + thing_types['thingTypes'][-1]['thingTypeName'].should.equal('thingTypeNameGroup') + + thing_types = client.list_thing_types(thingTypeName='thingTypeName') + thing_types.should_not.have.key('nextToken') + thing_types.should.have.key('thingTypes').which.should.have.length_of(2) + thing_types['thingTypes'][0]['thingTypeName'].should.equal('thingTypeName') + thing_types['thingTypes'][-1]['thingTypeName'].should.equal('thingTypeNameGroup') @mock_iot From a73dc492584cf0438dc92db38f80bbc966982384 Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 28 Jun 2018 13:10:09 +0200 Subject: [PATCH 8/8] fix linting error --- moto/iot/models.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/moto/iot/models.py b/moto/iot/models.py index 0ef53bbb5..c36bb985f 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -281,14 +281,15 @@ class IoTBackend(BaseBackend): def list_things(self, attribute_name, attribute_value, thing_type_name, max_results, token): all_things = [_.to_dict() for _ in self.things.values()] if attribute_name is not None and thing_type_name is not None: - filtered_things = list( - filter( - lambda elem: attribute_name in elem["attributes"] and elem["attributes"][ - attribute_name] == attribute_value and "thingTypeName" in elem and elem[ - "thingTypeName"] == thing_type_name, all_things)) + filtered_things = list(filter(lambda elem: + attribute_name in elem["attributes"] and + elem["attributes"][attribute_name] == attribute_value and + "thingTypeName" in elem and + elem["thingTypeName"] == thing_type_name, all_things)) elif attribute_name is not None and thing_type_name is None: - filtered_things = list(filter(lambda elem: attribute_name in elem["attributes"] and elem["attributes"][ - attribute_name] == attribute_value, all_things)) + filtered_things = list(filter(lambda elem: + attribute_name in elem["attributes"] and + elem["attributes"][attribute_name] == attribute_value, all_things)) elif attribute_name is None and thing_type_name is not None: filtered_things = list( filter(lambda elem: "thingTypeName" in elem and elem["thingTypeName"] == thing_type_name, all_things))