From 3a355f126cea926fd03815ad240e9e29839a5228 Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Wed, 13 Jun 2018 16:14:18 +0200 Subject: [PATCH 01/55] first steps undertaken to fix spulec/moto#1684 and spulec/moto#1685 --- moto/ecr/models.py | 29 +++++++++----- tests/test_ecr/test_ecr_boto3.py | 68 +++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 15 deletions(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index e20c550c9..87d02c3b1 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -4,12 +4,12 @@ import hashlib from copy import copy from random import random +from botocore.exceptions import ParamValidationError + from moto.core import BaseBackend, BaseModel from moto.ec2 import ec2_backends from moto.ecr.exceptions import ImageNotFoundException, RepositoryNotFoundException -from botocore.exceptions import ParamValidationError - DEFAULT_REGISTRY_ID = '012345678910' @@ -97,13 +97,13 @@ class Repository(BaseObject): class Image(BaseObject): - def __init__(self, tag, manifest, repository, registry_id=DEFAULT_REGISTRY_ID): + def __init__(self, tag, manifest, repository, digest=None, registry_id=DEFAULT_REGISTRY_ID): self.image_tag = tag self.image_manifest = manifest self.image_size_in_bytes = 50 * 1024 * 1024 self.repository = repository self.registry_id = registry_id - self.image_digest = None + self.image_digest = digest self.image_pushed_at = None def _create_digest(self): @@ -115,6 +115,9 @@ class Image(BaseObject): self._create_digest() return self.image_digest + def get_image_manifest(self): + return self.image_manifest + @property def response_object(self): response_object = self.gen_response_object() @@ -124,14 +127,14 @@ class Image(BaseObject): response_object['imageManifest'] = self.image_manifest response_object['repositoryName'] = self.repository response_object['registryId'] = self.registry_id - return response_object + return {k: v for k, v in response_object.items() if v is not None and v != [None]} @property def response_list_object(self): response_object = self.gen_response_object() response_object['imageTag'] = self.image_tag response_object['imageDigest'] = "i don't know" - return response_object + return {k: v for k, v in response_object.items() if v is not None and v != [None]} @property def response_describe_object(self): @@ -143,7 +146,7 @@ class Image(BaseObject): response_object['registryId'] = self.registry_id response_object['imageSizeInBytes'] = self.image_size_in_bytes response_object['imagePushedAt'] = '2017-05-09' - return response_object + return {k: v for k, v in response_object.items() if v is not None and v != [None]} @property def response_batch_get_image(self): @@ -154,7 +157,7 @@ class Image(BaseObject): response_object['imageManifest'] = self.image_manifest response_object['repositoryName'] = self.repository response_object['registryId'] = self.registry_id - return response_object + return {k: v for k, v in response_object.items() if v is not None and v != [None]} class ECRBackend(BaseBackend): @@ -252,8 +255,14 @@ class ECRBackend(BaseBackend): else: raise Exception("{0} is not a repository".format(repository_name)) - image = Image(image_tag, image_manifest, repository_name) - repository.images.append(image) + existing_image = list(filter(lambda x: x.response_object['imageManifest'] == image_manifest, repository.images)) + if not existing_image: + image = Image(image_tag, image_manifest, repository_name) + repository.images.append(image) + else: + image = Image(image_tag, image_manifest, repository_name, existing_image[0].get_image_digest()) + repository.images.append(image) + return image def batch_get_image(self, repository_name, registry_id=None, image_ids=None, accepted_media_types=None): diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 7651dc832..43a41a4d5 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -197,6 +197,54 @@ def test_put_image(): response['image']['repositoryName'].should.equal('test_repository') response['image']['registryId'].should.equal('012345678910') +@mock_ecr +def test_put_image_with_multiple_tags(): + client = boto3.client('ecr', region_name='us-east-1') + _ = client.create_repository( + repositoryName='test_repository' + ) + manifest = _create_image_manifest() + response = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest), + imageTag='v1' + ) + + response['image']['imageId']['imageTag'].should.equal('v1') + response['image']['imageId']['imageDigest'].should.contain("sha") + response['image']['repositoryName'].should.equal('test_repository') + response['image']['registryId'].should.equal('012345678910') + + response1 = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest), + imageTag='latest' + ) + + response1['image']['imageId']['imageTag'].should.equal('latest') + response1['image']['imageId']['imageDigest'].should.contain("sha") + response1['image']['repositoryName'].should.equal('test_repository') + response1['image']['registryId'].should.equal('012345678910') + + response2 = client.describe_images(repositoryName='test_repository') + type(response2['imageDetails']).should.be(list) + len(response2['imageDetails']).should.be(1) + + response['imageDetails'][0]['imageDigest'].should.contain("sha") + + # response['imageDetails'][0]['registryId'].should.equal("012345678910") + # response['imageDetails'][1]['registryId'].should.equal("012345678910") + # response['imageDetails'][2]['registryId'].should.equal("012345678910") + # response['imageDetails'][3]['registryId'].should.equal("012345678910") + # + # response['imageDetails'][0]['repositoryName'].should.equal("test_repository") + # response['imageDetails'][1]['repositoryName'].should.equal("test_repository") + # response['imageDetails'][2]['repositoryName'].should.equal("test_repository") + # response['imageDetails'][3]['repositoryName'].should.equal("test_repository") + # + # response['imageDetails'][0].should_not.have.key('imageTags') + # len(response['imageDetails'][1]['imageTags']).should.be(1) + @mock_ecr def test_list_images(): @@ -259,6 +307,11 @@ def test_describe_images(): repositoryName='test_repository' ) + _ = client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(_create_image_manifest()) + ) + _ = client.put_image( repositoryName='test_repository', imageManifest=json.dumps(_create_image_manifest()), @@ -279,32 +332,37 @@ def test_describe_images(): response = client.describe_images(repositoryName='test_repository') type(response['imageDetails']).should.be(list) - len(response['imageDetails']).should.be(3) + len(response['imageDetails']).should.be(4) response['imageDetails'][0]['imageDigest'].should.contain("sha") response['imageDetails'][1]['imageDigest'].should.contain("sha") response['imageDetails'][2]['imageDigest'].should.contain("sha") + response['imageDetails'][3]['imageDigest'].should.contain("sha") response['imageDetails'][0]['registryId'].should.equal("012345678910") response['imageDetails'][1]['registryId'].should.equal("012345678910") response['imageDetails'][2]['registryId'].should.equal("012345678910") + response['imageDetails'][3]['registryId'].should.equal("012345678910") response['imageDetails'][0]['repositoryName'].should.equal("test_repository") response['imageDetails'][1]['repositoryName'].should.equal("test_repository") response['imageDetails'][2]['repositoryName'].should.equal("test_repository") + response['imageDetails'][3]['repositoryName'].should.equal("test_repository") - len(response['imageDetails'][0]['imageTags']).should.be(1) + response['imageDetails'][0].should_not.have.key('imageTags') len(response['imageDetails'][1]['imageTags']).should.be(1) len(response['imageDetails'][2]['imageTags']).should.be(1) + len(response['imageDetails'][3]['imageTags']).should.be(1) image_tags = ['latest', 'v1', 'v2'] - set([response['imageDetails'][0]['imageTags'][0], - response['imageDetails'][1]['imageTags'][0], - response['imageDetails'][2]['imageTags'][0]]).should.equal(set(image_tags)) + set([response['imageDetails'][1]['imageTags'][0], + response['imageDetails'][2]['imageTags'][0], + response['imageDetails'][3]['imageTags'][0]]).should.equal(set(image_tags)) response['imageDetails'][0]['imageSizeInBytes'].should.equal(52428800) response['imageDetails'][1]['imageSizeInBytes'].should.equal(52428800) response['imageDetails'][2]['imageSizeInBytes'].should.equal(52428800) + response['imageDetails'][3]['imageSizeInBytes'].should.equal(52428800) @mock_ecr From cc799b55daf1f3c05238f1f2a6dc52903f89c9df Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 14 Jun 2018 09:07:09 +0200 Subject: [PATCH 02/55] fixed spulec/moto#1684 and fixed spulec/moto#1685 --- moto/ecr/models.py | 21 ++++++++++++++------- tests/test_ecr/test_ecr_boto3.py | 22 ++++++++-------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index 87d02c3b1..a5502df47 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -99,6 +99,7 @@ class Image(BaseObject): def __init__(self, tag, manifest, repository, digest=None, registry_id=DEFAULT_REGISTRY_ID): self.image_tag = tag + self.image_tags = [tag] self.image_manifest = manifest self.image_size_in_bytes = 50 * 1024 * 1024 self.repository = repository @@ -118,6 +119,11 @@ class Image(BaseObject): def get_image_manifest(self): return self.image_manifest + def update_tag(self, tag): + self.image_tag = tag + if tag not in self.image_tags: + self.image_tags.append(tag) + @property def response_object(self): response_object = self.gen_response_object() @@ -139,7 +145,7 @@ class Image(BaseObject): @property def response_describe_object(self): response_object = self.gen_response_object() - response_object['imageTags'] = [self.image_tag] + response_object['imageTags'] = self.image_tags response_object['imageDigest'] = self.get_image_digest() response_object['imageManifest'] = self.image_manifest response_object['repositoryName'] = self.repository @@ -255,15 +261,16 @@ class ECRBackend(BaseBackend): else: raise Exception("{0} is not a repository".format(repository_name)) - existing_image = list(filter(lambda x: x.response_object['imageManifest'] == image_manifest, repository.images)) - if not existing_image: + existing_images = list(filter(lambda x: x.response_object['imageManifest'] == image_manifest, repository.images)) + if not existing_images: + # this image is not in ECR yet image = Image(image_tag, image_manifest, repository_name) repository.images.append(image) + return image else: - image = Image(image_tag, image_manifest, repository_name, existing_image[0].get_image_digest()) - repository.images.append(image) - - return image + # update existing image + existing_images[0].update_tag(image_tag) + return existing_images[0] def batch_get_image(self, repository_name, registry_id=None, image_ids=None, accepted_media_types=None): if repository_name in self.repositories: diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 43a41a4d5..8ecab9001 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -45,7 +45,8 @@ def _create_image_manifest(): { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", "size": 73109, - "digest": _create_image_digest("layer3") + # randomize image digest + "digest": _create_image_digest() } ] } @@ -230,21 +231,14 @@ def test_put_image_with_multiple_tags(): type(response2['imageDetails']).should.be(list) len(response2['imageDetails']).should.be(1) - response['imageDetails'][0]['imageDigest'].should.contain("sha") + response2['imageDetails'][0]['imageDigest'].should.contain("sha") - # response['imageDetails'][0]['registryId'].should.equal("012345678910") - # response['imageDetails'][1]['registryId'].should.equal("012345678910") - # response['imageDetails'][2]['registryId'].should.equal("012345678910") - # response['imageDetails'][3]['registryId'].should.equal("012345678910") - # - # response['imageDetails'][0]['repositoryName'].should.equal("test_repository") - # response['imageDetails'][1]['repositoryName'].should.equal("test_repository") - # response['imageDetails'][2]['repositoryName'].should.equal("test_repository") - # response['imageDetails'][3]['repositoryName'].should.equal("test_repository") - # - # response['imageDetails'][0].should_not.have.key('imageTags') - # len(response['imageDetails'][1]['imageTags']).should.be(1) + response2['imageDetails'][0]['registryId'].should.equal("012345678910") + response2['imageDetails'][0]['repositoryName'].should.equal("test_repository") + + len(response2['imageDetails'][0]['imageTags']).should.be(2) + response2['imageDetails'][0]['imageTags'].should.be.equal(['v1', 'latest']) @mock_ecr def test_list_images(): From 6e269d1e3146bfabfd34c43840e8a31e4b01c644 Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 14 Jun 2018 09:10:06 +0200 Subject: [PATCH 03/55] fixes spulec/moto#1673 and updated IMPLEMENTATION_COVERAGE.md --- IMPLEMENTATION_COVERAGE.md | 78 +++++++++++++++--------------- scripts/implementation_coverage.py | 7 +-- 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 411f55a8b..75bf254ef 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -814,17 +814,17 @@ - [ ] update_team_member - [ ] update_user_profile -## cognito-identity - 0% implemented -- [ ] create_identity_pool +## cognito-identity - 22% implemented +- [X] create_identity_pool - [ ] delete_identities - [ ] delete_identity_pool - [ ] describe_identity - [ ] describe_identity_pool -- [ ] get_credentials_for_identity -- [ ] get_id +- [X] get_credentials_for_identity +- [X] get_id - [ ] get_identity_pool_roles - [ ] get_open_id_token -- [ ] get_open_id_token_for_developer_identity +- [X] get_open_id_token_for_developer_identity - [ ] list_identities - [ ] list_identity_pools - [ ] lookup_developer_identity @@ -834,20 +834,20 @@ - [ ] unlink_identity - [ ] update_identity_pool -## cognito-idp - 0% implemented +## cognito-idp - 25% implemented - [ ] add_custom_attributes - [ ] admin_add_user_to_group - [ ] admin_confirm_sign_up -- [ ] admin_create_user -- [ ] admin_delete_user +- [X] admin_create_user +- [X] admin_delete_user - [ ] admin_delete_user_attributes - [ ] admin_disable_provider_for_user - [ ] admin_disable_user - [ ] admin_enable_user - [ ] admin_forget_device - [ ] admin_get_device -- [ ] admin_get_user -- [ ] admin_initiate_auth +- [X] admin_get_user +- [X] admin_initiate_auth - [ ] admin_link_provider_for_user - [ ] admin_list_devices - [ ] admin_list_groups_for_user @@ -862,32 +862,32 @@ - [ ] admin_update_user_attributes - [ ] admin_user_global_sign_out - [ ] associate_software_token -- [ ] change_password +- [X] change_password - [ ] confirm_device -- [ ] confirm_forgot_password +- [X] confirm_forgot_password - [ ] confirm_sign_up - [ ] create_group -- [ ] create_identity_provider +- [X] create_identity_provider - [ ] create_resource_server - [ ] create_user_import_job -- [ ] create_user_pool -- [ ] create_user_pool_client -- [ ] create_user_pool_domain +- [X] create_user_pool +- [X] create_user_pool_client +- [X] create_user_pool_domain - [ ] delete_group -- [ ] delete_identity_provider +- [X] delete_identity_provider - [ ] delete_resource_server - [ ] delete_user - [ ] delete_user_attributes -- [ ] delete_user_pool -- [ ] delete_user_pool_client -- [ ] delete_user_pool_domain -- [ ] describe_identity_provider +- [X] delete_user_pool +- [X] delete_user_pool_client +- [X] delete_user_pool_domain +- [X] describe_identity_provider - [ ] describe_resource_server - [ ] describe_risk_configuration - [ ] describe_user_import_job -- [ ] describe_user_pool -- [ ] describe_user_pool_client -- [ ] describe_user_pool_domain +- [X] describe_user_pool +- [X] describe_user_pool_client +- [X] describe_user_pool_domain - [ ] forget_device - [ ] forgot_password - [ ] get_csv_header @@ -903,15 +903,15 @@ - [ ] initiate_auth - [ ] list_devices - [ ] list_groups -- [ ] list_identity_providers +- [X] list_identity_providers - [ ] list_resource_servers - [ ] list_user_import_jobs -- [ ] list_user_pool_clients -- [ ] list_user_pools -- [ ] list_users +- [X] list_user_pool_clients +- [X] list_user_pools +- [X] list_users - [ ] list_users_in_group - [ ] resend_confirmation_code -- [ ] respond_to_auth_challenge +- [X] respond_to_auth_challenge - [ ] set_risk_configuration - [ ] set_ui_customization - [ ] set_user_mfa_preference @@ -927,7 +927,7 @@ - [ ] update_resource_server - [ ] update_user_attributes - [ ] update_user_pool -- [ ] update_user_pool_client +- [X] update_user_pool_client - [ ] verify_software_token - [ ] verify_user_attribute @@ -2524,11 +2524,11 @@ - [X] update_thing_group - [X] update_thing_groups_for_thing -## iot-data - 0% implemented -- [ ] delete_thing_shadow -- [ ] get_thing_shadow -- [ ] publish -- [ ] update_thing_shadow +## iot-data - 100% implemented +- [X] delete_thing_shadow +- [X] get_thing_shadow +- [X] publish +- [X] update_thing_shadow ## iot-jobs-data - 0% implemented - [ ] describe_job_execution @@ -2815,7 +2815,7 @@ - [ ] update_domain_entry - [ ] update_load_balancer_attribute -## logs - 24% implemented +## logs - 27% implemented - [ ] associate_kms_key - [ ] cancel_export_task - [ ] create_export_task @@ -2830,7 +2830,7 @@ - [ ] delete_subscription_filter - [ ] describe_destinations - [ ] describe_export_tasks -- [ ] describe_log_groups +- [X] describe_log_groups - [X] describe_log_streams - [ ] describe_metric_filters - [ ] describe_resource_policies @@ -3703,13 +3703,13 @@ - [ ] put_attributes - [ ] select -## secretsmanager - 0% implemented +## secretsmanager - 6% implemented - [ ] cancel_rotate_secret - [ ] create_secret - [ ] delete_secret - [ ] describe_secret - [ ] get_random_password -- [ ] get_secret_value +- [X] get_secret_value - [ ] list_secret_version_ids - [ ] list_secrets - [ ] put_secret_value diff --git a/scripts/implementation_coverage.py b/scripts/implementation_coverage.py index 74ce9590d..1541c4c75 100755 --- a/scripts/implementation_coverage.py +++ b/scripts/implementation_coverage.py @@ -7,12 +7,13 @@ import boto3 def get_moto_implementation(service_name): - if not hasattr(moto, service_name): + service_name_standardized = service_name.replace("-", "") if "-" in service_name else service_name + if not hasattr(moto, service_name_standardized): return None - module = getattr(moto, service_name) + module = getattr(moto, service_name_standardized) if module is None: return None - mock = getattr(module, "mock_{}".format(service_name)) + mock = getattr(module, "mock_{}".format(service_name_standardized)) if mock is None: return None backends = list(mock().backends.values()) From ea3366be35d2345d1769637da9e6a697fbbd87f9 Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 14 Jun 2018 09:53:11 +0200 Subject: [PATCH 04/55] do not allow None as value of image_tags --- moto/ecr/models.py | 6 ++-- tests/test_ecr/test_ecr_boto3.py | 62 ++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index a5502df47..d3e8aa219 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -99,7 +99,7 @@ class Image(BaseObject): def __init__(self, tag, manifest, repository, digest=None, registry_id=DEFAULT_REGISTRY_ID): self.image_tag = tag - self.image_tags = [tag] + self.image_tags = [tag] if tag is not None else [] self.image_manifest = manifest self.image_size_in_bytes = 50 * 1024 * 1024 self.repository = repository @@ -121,7 +121,7 @@ class Image(BaseObject): def update_tag(self, tag): self.image_tag = tag - if tag not in self.image_tags: + if tag not in self.image_tags and tag is not None: self.image_tags.append(tag) @property @@ -235,7 +235,7 @@ class ECRBackend(BaseBackend): found = False for image in repository.images: if (('imageDigest' in image_id and image.get_image_digest() == image_id['imageDigest']) or - ('imageTag' in image_id and image.image_tag == image_id['imageTag'])): + ('imageTag' in image_id and image_id['imageTag'] in image.image_tags)): found = True response.add(image) if not found: diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 8ecab9001..d542184a7 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -385,6 +385,68 @@ def test_describe_images_by_tag(): image_detail['imageDigest'].should.equal(put_response['imageId']['imageDigest']) +@mock_ecr +def test_describe_images_tags_should_not_contain_empty_tag1(): + client = boto3.client('ecr', region_name='us-east-1') + _ = client.create_repository( + repositoryName='test_repository' + ) + + manifest = _create_image_manifest() + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest) + ) + + tags = ['v1', 'v2', 'latest'] + for tag in tags: + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest), + imageTag=tag + ) + + response = client.describe_images(repositoryName='test_repository', imageIds=[{'imageTag': tag}]) + len(response['imageDetails']).should.be(1) + image_detail = response['imageDetails'][0] + len(image_detail['imageTags']).should.equal(3) + image_detail['imageTags'].should.be.equal(tags) + + +@mock_ecr +def test_describe_images_tags_should_not_contain_empty_tag2(): + client = boto3.client('ecr', region_name='us-east-1') + _ = client.create_repository( + repositoryName='test_repository' + ) + + manifest = _create_image_manifest() + tags = ['v1', 'v2'] + for tag in tags: + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest), + imageTag=tag + ) + + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest) + ) + + client.put_image( + repositoryName='test_repository', + imageManifest=json.dumps(manifest), + imageTag='latest' + ) + + response = client.describe_images(repositoryName='test_repository', imageIds=[{'imageTag': tag}]) + len(response['imageDetails']).should.be(1) + image_detail = response['imageDetails'][0] + len(image_detail['imageTags']).should.equal(3) + image_detail['imageTags'].should.be.equal(['v1', 'v2', 'latest']) + + @mock_ecr def test_describe_repository_that_doesnt_exist(): client = boto3.client('ecr', region_name='us-east-1') From 56ff66394d00f5a7d33d65df865c35f3126818fb Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 14 Jun 2018 09:56:53 +0200 Subject: [PATCH 05/55] updated to make sure that tests still run correctly --- moto/ecr/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/ecr/models.py b/moto/ecr/models.py index d3e8aa219..a8ee60c5a 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -152,7 +152,7 @@ class Image(BaseObject): response_object['registryId'] = self.registry_id response_object['imageSizeInBytes'] = self.image_size_in_bytes response_object['imagePushedAt'] = '2017-05-09' - return {k: v for k, v in response_object.items() if v is not None and v != [None]} + return {k: v for k, v in response_object.items() if v is not None and v != []} @property def response_batch_get_image(self): From db3593575f0ddeb699c1230c293f0db1ec1a48be Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 28 Jun 2018 10:32:51 +0200 Subject: [PATCH 06/55] list_thing_types and list_things now uses pagination --- moto/iot/models.py | 52 ++++++++++---- moto/iot/responses.py | 35 +++++---- tests/test_iot/test_iot.py | 144 ++++++++++++++++++++++++++++++++++++- 3 files changed, 202 insertions(+), 29 deletions(-) diff --git a/moto/iot/models.py b/moto/iot/models.py index ce7a4cf57..0ef53bbb5 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -1,14 +1,17 @@ from __future__ import unicode_literals -import time -import boto3 -import string -import random + import hashlib -import uuid +import random import re -from datetime import datetime -from moto.core import BaseBackend, BaseModel +import string +import time +import uuid from collections import OrderedDict +from datetime import datetime + +import boto3 + +from moto.core import BaseBackend, BaseModel from .exceptions import ( ResourceNotFoundException, InvalidRequestException, @@ -271,15 +274,36 @@ class IoTBackend(BaseBackend): def list_thing_types(self, thing_type_name=None): if thing_type_name: - # It's wierd but thing_type_name is filterd by forward match, not complete match + # It's weird but thing_type_name is filtered by forward match, not complete match return [_ for _ in self.thing_types.values() if _.thing_type_name.startswith(thing_type_name)] - thing_types = self.thing_types.values() - return thing_types + return self.thing_types.values() - def list_things(self, attribute_name, attribute_value, thing_type_name): - # TODO: filter by attributess or thing_type - things = self.things.values() - return things + def list_things(self, attribute_name, attribute_value, thing_type_name, max_results, token): + all_things = [_.to_dict() for _ in self.things.values()] + if attribute_name is not None and thing_type_name is not None: + filtered_things = list( + filter( + lambda elem: attribute_name in elem["attributes"] and elem["attributes"][ + attribute_name] == attribute_value and "thingTypeName" in elem and elem[ + "thingTypeName"] == thing_type_name, all_things)) + elif attribute_name is not None and thing_type_name is None: + filtered_things = list(filter(lambda elem: attribute_name in elem["attributes"] and elem["attributes"][ + attribute_name] == attribute_value, all_things)) + elif attribute_name is None and thing_type_name is not None: + filtered_things = list( + filter(lambda elem: "thingTypeName" in elem and elem["thingTypeName"] == thing_type_name, all_things)) + else: + filtered_things = all_things + + if token is None: + things = filtered_things[0:max_results] + next_token = str(max_results) if len(filtered_things) > max_results else None + else: + token = int(token) + things = filtered_things[token:token + max_results] + next_token = str(token + max_results) if len(filtered_things) > token + max_results else None + + return things, next_token def describe_thing(self, thing_name): things = [_ for _ in self.things.values() if _.thing_name == thing_name] diff --git a/moto/iot/responses.py b/moto/iot/responses.py index fcdf12f78..006c4c4cc 100644 --- a/moto/iot/responses.py +++ b/moto/iot/responses.py @@ -1,7 +1,9 @@ from __future__ import unicode_literals + +import json + from moto.core.responses import BaseResponse from .models import iot_backends -import json class IoTResponse(BaseResponse): @@ -32,30 +34,39 @@ class IoTResponse(BaseResponse): return json.dumps(dict(thingTypeName=thing_type_name, thingTypeArn=thing_type_arn)) def list_thing_types(self): - # previous_next_token = self._get_param("nextToken") - # max_results = self._get_int_param("maxResults") + previous_next_token = self._get_param("nextToken") + max_results = self._get_int_param("maxResults", 50) # not the default, but makes testing easier thing_type_name = self._get_param("thingTypeName") thing_types = self.iot_backend.list_thing_types( thing_type_name=thing_type_name ) - # TODO: implement pagination in the future - next_token = None - return json.dumps(dict(thingTypes=[_.to_dict() for _ in thing_types], nextToken=next_token)) + + thing_types = [_.to_dict() for _ in thing_types] + if previous_next_token is None: + result = thing_types[0:max_results] + next_token = str(max_results) if len(thing_types) > max_results else None + else: + token = int(previous_next_token) + result = thing_types[token:token + max_results] + next_token = str(token + max_results) if len(thing_types) > token + max_results else None + + return json.dumps(dict(thingTypes=result, nextToken=next_token)) def list_things(self): - # previous_next_token = self._get_param("nextToken") - # max_results = self._get_int_param("maxResults") + previous_next_token = self._get_param("nextToken") + max_results = self._get_int_param("maxResults", 50) # not the default, but makes testing easier attribute_name = self._get_param("attributeName") attribute_value = self._get_param("attributeValue") thing_type_name = self._get_param("thingTypeName") - things = self.iot_backend.list_things( + things, next_token = self.iot_backend.list_things( attribute_name=attribute_name, attribute_value=attribute_value, thing_type_name=thing_type_name, + max_results=max_results, + token=previous_next_token ) - # TODO: implement pagination in the future - next_token = None - return json.dumps(dict(things=[_.to_dict() for _ in things], nextToken=next_token)) + + return json.dumps(dict(things=things, nextToken=next_token)) def describe_thing(self): thing_name = self._get_param("thingName") diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 213615790..5e78f6e49 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -1,8 +1,9 @@ from __future__ import unicode_literals -import boto3 -import sure # noqa import json +import sure # noqa +import boto3 + from moto import mock_iot @@ -63,6 +64,143 @@ def test_things(): res.should.have.key('thingTypes').which.should.have.length_of(0) +@mock_iot +def test_list_thing_types(): + client = boto3.client('iot', region_name='ap-northeast-1') + + for i in range(0, 100): + client.create_thing_type(thingTypeName=str(i + 1)) + + thing_types = client.list_thing_types() + thing_types.should.have.key('nextToken') + thing_types.should.have.key('thingTypes').which.should.have.length_of(50) + thing_types['thingTypes'][0]['thingTypeName'].should.equal('1') + thing_types['thingTypes'][-1]['thingTypeName'].should.equal('50') + + thing_types = client.list_thing_types(nextToken=thing_types['nextToken']) + thing_types.should.have.key('thingTypes').which.should.have.length_of(50) + thing_types.should_not.have.key('nextToken') + thing_types['thingTypes'][0]['thingTypeName'].should.equal('51') + thing_types['thingTypes'][-1]['thingTypeName'].should.equal('100') + # TODO test list_thing_types with filters + + +@mock_iot +def test_list_things_with_next_token(): + client = boto3.client('iot', region_name='ap-northeast-1') + + for i in range(0, 200): + client.create_thing(thingName=str(i + 1)) + + things = client.list_things() + things.should.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('1') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/1') + things['things'][-1]['thingName'].should.equal('50') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/50') + + things = client.list_things(nextToken=things['nextToken']) + things.should.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('51') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/51') + things['things'][-1]['thingName'].should.equal('100') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/100') + + things = client.list_things(nextToken=things['nextToken']) + things.should.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('101') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/101') + things['things'][-1]['thingName'].should.equal('150') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/150') + + things = client.list_things(nextToken=things['nextToken']) + things.should_not.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('151') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/151') + things['things'][-1]['thingName'].should.equal('200') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/200') + + +@mock_iot +def test_list_things_with_attribute_and_thing_type_filter_and_next_token(): + client = boto3.client('iot', region_name='ap-northeast-1') + client.create_thing_type(thingTypeName='my-thing-type') + + for i in range(0, 200): + if not (i + 1) % 3: + attribute_payload = { + 'attributes': { + 'foo': 'bar' + } + } + elif not (i + 1) % 5: + attribute_payload = { + 'attributes': { + 'bar': 'foo' + } + } + else: + attribute_payload = {} + + if not (i + 1) % 2: + thing_type_name = 'my-thing-type' + client.create_thing(thingName=str(i + 1), thingTypeName=thing_type_name, attributePayload=attribute_payload) + else: + client.create_thing(thingName=str(i + 1), attributePayload=attribute_payload) + + # Test filter for thingTypeName + things = client.list_things(thingTypeName=thing_type_name) + things.should.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('2') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/2') + things['things'][-1]['thingName'].should.equal('100') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/100') + all(item['thingTypeName'] == thing_type_name for item in things['things']) + + things = client.list_things(nextToken=things['nextToken'], thingTypeName=thing_type_name) + things.should_not.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('102') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/102') + things['things'][-1]['thingName'].should.equal('200') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/200') + all(item['thingTypeName'] == thing_type_name for item in things['things']) + + # Test filter for attributes + things = client.list_things(attributeName='foo', attributeValue='bar') + things.should.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(50) + things['things'][0]['thingName'].should.equal('3') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/3') + things['things'][-1]['thingName'].should.equal('150') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/150') + all(item['attributes'] == {'foo': 'bar'} for item in things['things']) + + things = client.list_things(nextToken=things['nextToken'], attributeName='foo', attributeValue='bar') + things.should_not.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(16) + things['things'][0]['thingName'].should.equal('153') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/153') + things['things'][-1]['thingName'].should.equal('198') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/198') + all(item['attributes'] == {'foo': 'bar'} for item in things['things']) + + # Test filter for attributes and thingTypeName + things = client.list_things(thingTypeName=thing_type_name, attributeName='foo', attributeValue='bar') + things.should_not.have.key('nextToken') + things.should.have.key('things').which.should.have.length_of(33) + things['things'][0]['thingName'].should.equal('6') + things['things'][0]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/6') + things['things'][-1]['thingName'].should.equal('198') + things['things'][-1]['thingArn'].should.equal('arn:aws:iot:ap-northeast-1:1:thing/198') + all(item['attributes'] == {'foo': 'bar'} and item['thingTypeName'] == thing_type_name for item in things['things']) + + @mock_iot def test_certs(): client = boto3.client('iot', region_name='ap-northeast-1') @@ -204,7 +342,6 @@ def test_principal_thing(): @mock_iot def test_thing_groups(): client = boto3.client('iot', region_name='ap-northeast-1') - name = 'my-thing' group_name = 'my-group-name' # thing group @@ -424,6 +561,7 @@ def test_create_job(): job.should.have.key('jobArn') job.should.have.key('description') + @mock_iot def test_describe_job(): client = boto3.client('iot', region_name='eu-west-1') From 10f96b2ccfa55114c9f2b32355efc4decd0ebb0f Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 28 Jun 2018 12:59:08 +0200 Subject: [PATCH 07/55] next_token (pagination) added for `list_thing_types` --- tests/test_iot/test_iot.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 5e78f6e49..5c6effd7a 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -82,7 +82,30 @@ def test_list_thing_types(): thing_types.should_not.have.key('nextToken') thing_types['thingTypes'][0]['thingTypeName'].should.equal('51') thing_types['thingTypes'][-1]['thingTypeName'].should.equal('100') - # TODO test list_thing_types with filters + + +@mock_iot +def test_list_thing_types_with_typename_filter(): + client = boto3.client('iot', region_name='ap-northeast-1') + + client.create_thing_type(thingTypeName='thing') + client.create_thing_type(thingTypeName='thingType') + client.create_thing_type(thingTypeName='thingTypeName') + client.create_thing_type(thingTypeName='thingTypeNameGroup') + client.create_thing_type(thingTypeName='shouldNotFind') + client.create_thing_type(thingTypeName='find me it shall not') + + thing_types = client.list_thing_types(thingTypeName='thing') + thing_types.should_not.have.key('nextToken') + thing_types.should.have.key('thingTypes').which.should.have.length_of(4) + thing_types['thingTypes'][0]['thingTypeName'].should.equal('thing') + thing_types['thingTypes'][-1]['thingTypeName'].should.equal('thingTypeNameGroup') + + thing_types = client.list_thing_types(thingTypeName='thingTypeName') + thing_types.should_not.have.key('nextToken') + thing_types.should.have.key('thingTypes').which.should.have.length_of(2) + thing_types['thingTypes'][0]['thingTypeName'].should.equal('thingTypeName') + thing_types['thingTypes'][-1]['thingTypeName'].should.equal('thingTypeNameGroup') @mock_iot From a73dc492584cf0438dc92db38f80bbc966982384 Mon Sep 17 00:00:00 2001 From: Stephan Huber Date: Thu, 28 Jun 2018 13:10:09 +0200 Subject: [PATCH 08/55] fix linting error --- moto/iot/models.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/moto/iot/models.py b/moto/iot/models.py index 0ef53bbb5..c36bb985f 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -281,14 +281,15 @@ class IoTBackend(BaseBackend): def list_things(self, attribute_name, attribute_value, thing_type_name, max_results, token): all_things = [_.to_dict() for _ in self.things.values()] if attribute_name is not None and thing_type_name is not None: - filtered_things = list( - filter( - lambda elem: attribute_name in elem["attributes"] and elem["attributes"][ - attribute_name] == attribute_value and "thingTypeName" in elem and elem[ - "thingTypeName"] == thing_type_name, all_things)) + filtered_things = list(filter(lambda elem: + attribute_name in elem["attributes"] and + elem["attributes"][attribute_name] == attribute_value and + "thingTypeName" in elem and + elem["thingTypeName"] == thing_type_name, all_things)) elif attribute_name is not None and thing_type_name is None: - filtered_things = list(filter(lambda elem: attribute_name in elem["attributes"] and elem["attributes"][ - attribute_name] == attribute_value, all_things)) + filtered_things = list(filter(lambda elem: + attribute_name in elem["attributes"] and + elem["attributes"][attribute_name] == attribute_value, all_things)) elif attribute_name is None and thing_type_name is not None: filtered_things = list( filter(lambda elem: "thingTypeName" in elem and elem["thingTypeName"] == thing_type_name, all_things)) From 12807bb6f0ad1840881b077175409c0c8dacbfb5 Mon Sep 17 00:00:00 2001 From: Mike Liu Date: Tue, 24 Jul 2018 16:15:53 -0400 Subject: [PATCH 09/55] Add get_command_invocation endpoint for AWS SSM. Users can make send_command requests and then retrieve the invocations of those commands with get_command_invocation. Getting a command invocation by instance and command id is supported but only the 'aws:runShellScript' plugin name is supported and only one plugin in a document is supported. --- moto/ssm/models.py | 71 ++++++++++++++++++++++++++++++-- moto/ssm/responses.py | 5 +++ tests/test_ssm/test_ssm_boto3.py | 40 ++++++++++++++++++ 3 files changed, 112 insertions(+), 4 deletions(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 656a14839..b3e34eae4 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -88,9 +88,9 @@ class Command(BaseModel): self.status = 'Success' self.status_details = 'Details placeholder' - now = datetime.datetime.now() - self.requested_date_time = now.isoformat() - expires_after = now + datetime.timedelta(0, timeout_seconds) + self.requested_date_time = datetime.datetime.now() + self.requested_date_time_iso = self.requested_date_time.isoformat() + expires_after = self.requested_date_time + datetime.timedelta(0, timeout_seconds) self.expires_after = expires_after.isoformat() self.comment = comment @@ -106,6 +106,12 @@ class Command(BaseModel): self.service_role_arn = service_role_arn self.targets = targets + # Create invocations with a single run command plugin. + self.invocations = [] + for instance_id in self.instance_ids: + self.invocations.append( + self.invocation_response(instance_id, "aws:runShellScript")) + def response_object(self): r = { 'CommandId': self.command_id, @@ -122,7 +128,7 @@ class Command(BaseModel): 'OutputS3BucketName': self.output_s3_bucket_name, 'OutputS3KeyPrefix': self.output_s3_key_prefix, 'Parameters': self.parameters, - 'RequestedDateTime': self.requested_date_time, + 'RequestedDateTime': self.requested_date_time_iso, 'ServiceRole': self.service_role_arn, 'Status': self.status, 'StatusDetails': self.status_details, @@ -132,6 +138,51 @@ class Command(BaseModel): return r + def invocation_response(self, instance_id, plugin_name): + # Calculate elapsed time from requested time and now. Use a hardcoded + # elapsed time since there is no easy way to convert a timedelta to + # an ISO 8601 duration string. + elapsed_time_iso = "PT5M" + elapsed_time_delta = datetime.timedelta(minutes=5) + end_time = self.requested_date_time + elapsed_time_delta + + r = { + 'CommandId': self.command_id, + 'InstanceId': instance_id, + 'Comment': self.comment, + 'DocumentName': self.document_name, + 'PluginName': plugin_name, + 'ResponseCode': 0, + 'ExecutionStartDateTime': self.requested_date_time_iso, + 'ExecutionElapsedTime': elapsed_time_iso, + 'ExecutionEndDateTime': end_time.isoformat(), + 'Status': 'Success', + 'StatusDetails': 'Success', + 'StandardOutputContent': '', + 'StandardOutputUrl': '', + 'StandardErrorContent': '', + } + + return r + + def get_invocation(self, instance_id, plugin_name): + invocation = next( + (invocation for invocation in self.invocations + if invocation['InstanceId'] == instance_id), None) + + if invocation is None: + raise RESTError( + 'InvocationDoesNotExist', + 'An error occurred (InvocationDoesNotExist) when calling the GetCommandInvocation operation') + + if plugin_name is not None and invocation['PluginName'] != plugin_name: + raise RESTError( + 'InvocationDoesNotExist', + 'An error occurred (InvocationDoesNotExist) when calling the GetCommandInvocation operation') + + + return invocation + class SimpleSystemManagerBackend(BaseBackend): @@ -298,6 +349,18 @@ class SimpleSystemManagerBackend(BaseBackend): command for command in self._commands if instance_id in command.instance_ids] + def get_command_invocation(self, **kwargs): + """ + https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetCommandInvocation.html + """ + + command_id = kwargs.get('CommandId') + instance_id = kwargs.get('InstanceId') + plugin_name = kwargs.get('PluginName', None) + + command = self.get_command_by_id(command_id) + return command.get_invocation(instance_id, plugin_name) + ssm_backends = {} for region, ec2_backend in ec2_backends.items(): diff --git a/moto/ssm/responses.py b/moto/ssm/responses.py index fd0d8b630..eb05e51b6 100644 --- a/moto/ssm/responses.py +++ b/moto/ssm/responses.py @@ -210,3 +210,8 @@ class SimpleSystemManagerResponse(BaseResponse): return json.dumps( self.ssm_backend.list_commands(**self.request_params) ) + + def get_command_invocation(self): + return json.dumps( + self.ssm_backend.get_command_invocation(**self.request_params) + ) diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index 7a0685d56..8df565cf6 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -668,3 +668,43 @@ def test_list_commands(): with assert_raises(ClientError): response = client.list_commands( CommandId=str(uuid.uuid4())) + +@mock_ssm +def test_get_command_invocation(): + client = boto3.client('ssm', region_name='us-east-1') + + ssm_document = 'AWS-RunShellScript' + params = {'commands': ['#!/bin/bash\necho \'hello world\'']} + + response = client.send_command( + InstanceIds=['i-123456', 'i-234567', 'i-345678'], + DocumentName=ssm_document, + Parameters=params, + OutputS3Region='us-east-2', + OutputS3BucketName='the-bucket', + OutputS3KeyPrefix='pref') + + cmd = response['Command'] + cmd_id = cmd['CommandId'] + + instance_id = 'i-345678' + invocation_response = client.get_command_invocation( + CommandId=cmd_id, + InstanceId=instance_id, + PluginName='aws:runShellScript') + + invocation_response['CommandId'].should.equal(cmd_id) + invocation_response['InstanceId'].should.equal(instance_id) + + # test the error case for an invalid instance id + with assert_raises(ClientError): + invocation_response = client.get_command_invocation( + CommandId=cmd_id, + InstanceId='i-FAKE') + + # test the error case for an invalid plugin name + with assert_raises(ClientError): + invocation_response = client.get_command_invocation( + CommandId=cmd_id, + InstanceId=instance_id, + PluginName='FAKE') From 7d78a08f9539b6d3dbeb7b6a9766eb042911943c Mon Sep 17 00:00:00 2001 From: Robert C Jensen Date: Thu, 26 Jul 2018 21:08:31 -0400 Subject: [PATCH 10/55] bugfix: support name-addr mailbox form --- moto/ses/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/ses/models.py b/moto/ses/models.py index 3dced60f2..71fe9d9a1 100644 --- a/moto/ses/models.py +++ b/moto/ses/models.py @@ -49,7 +49,8 @@ class SESBackend(BaseBackend): self.sent_messages = [] self.sent_message_count = 0 - def _is_verified_address(self, address): + def _is_verified_address(self, source): + _, address = parseaddr(source) if address in self.addresses: return True user, host = address.split('@', 1) From 2fad7c72024ad20a95a1fbe4cd0d3daca4698dd9 Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Thu, 2 Aug 2018 12:38:50 +0900 Subject: [PATCH 11/55] update dynamodb2 update_item add empty string handle, fix(#1744) --- moto/dynamodb2/responses.py | 24 ++++++++++----- tests/test_dynamodb2/test_dynamodb.py | 42 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 3c7e7ffc2..493e17833 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -20,6 +20,17 @@ def has_empty_keys_or_values(_dict): ) +def get_empty_str_error(): + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return (400, + {'server': 'amazon.com'}, + dynamo_json_dump({'__type': er, + 'message': ('One or more parameter values were ' + 'invalid: An AttributeValue may not ' + 'contain an empty string')} + )) + + class DynamoHandler(BaseResponse): def get_endpoint_name(self, headers): @@ -174,14 +185,7 @@ class DynamoHandler(BaseResponse): item = self.body['Item'] if has_empty_keys_or_values(item): - er = 'com.amazonaws.dynamodb.v20111205#ValidationException' - return (400, - {'server': 'amazon.com'}, - dynamo_json_dump({'__type': er, - 'message': ('One or more parameter values were ' - 'invalid: An AttributeValue may not ' - 'contain an empty string')} - )) + return get_empty_str_error() overwrite = 'Expected' not in self.body if not overwrite: @@ -523,6 +527,7 @@ class DynamoHandler(BaseResponse): return dynamo_json_dump(item_dict) def update_item(self): + name = self.body['TableName'] key = self.body['Key'] update_expression = self.body.get('UpdateExpression') @@ -533,6 +538,9 @@ class DynamoHandler(BaseResponse): 'ExpressionAttributeValues', {}) existing_item = self.dynamodb_backend.get_item(name, key) + if has_empty_keys_or_values(expression_attribute_values): + return get_empty_str_error() + if 'Expected' in self.body: expected = self.body['Expected'] else: diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index ab8f25856..243de2701 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -201,6 +201,48 @@ def test_item_add_empty_string_exception(): ) +@requires_boto_gte("2.9") +@mock_dynamodb2 +def test_update_item_with_empty_string_exception(): + name = 'TestTable' + conn = boto3.client('dynamodb', + region_name='us-west-2', + aws_access_key_id="ak", + aws_secret_access_key="sk") + conn.create_table(TableName=name, + KeySchema=[{'AttributeName':'forum_name','KeyType':'HASH'}], + AttributeDefinitions=[{'AttributeName':'forum_name','AttributeType':'S'}], + ProvisionedThroughput={'ReadCapacityUnits':5,'WriteCapacityUnits':5}) + + conn.put_item( + TableName=name, + Item={ + 'forum_name': { 'S': 'LOLCat Forum' }, + 'subject': { 'S': 'Check this out!' }, + 'Body': { 'S': 'http://url_to_lolcat.gif'}, + 'SentBy': { 'S': "test" }, + 'ReceivedTime': { 'S': '12/9/2011 11:36:03 PM'}, + } + ) + + with assert_raises(ClientError) as ex: + conn.update_item( + TableName=name, + Key={ + 'forum_name': { 'S': 'LOLCat Forum'}, + }, + UpdateExpression='set Body=:Body', + ExpressionAttributeValues={ + ':Body': {'S': ''} + }) + + ex.exception.response['Error']['Code'].should.equal('ValidationException') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + ex.exception.response['Error']['Message'].should.equal( + 'One or more parameter values were invalid: An AttributeValue may not contain an empty string' + ) + + @requires_boto_gte("2.9") @mock_dynamodb2 def test_query_invalid_table(): From ac4197ca47d6f1a54a21f554154e2a1cc7b87522 Mon Sep 17 00:00:00 2001 From: brett55 Date: Thu, 2 Aug 2018 16:09:10 -0600 Subject: [PATCH 12/55] - Updated ResourceAlreadyExistsException(LogsClientError) to latest Boto3 error msg when using CWL Logs --- moto/logs/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/logs/exceptions.py b/moto/logs/exceptions.py index cc83452ea..bb02eced3 100644 --- a/moto/logs/exceptions.py +++ b/moto/logs/exceptions.py @@ -29,5 +29,5 @@ class ResourceAlreadyExistsException(LogsClientError): self.code = 400 super(ResourceAlreadyExistsException, self).__init__( 'ResourceAlreadyExistsException', - 'The specified resource already exists.' + 'The specified log group already exists' ) From ae2e6fef632c5233639702dc304aa4cf793e4e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Faucon?= Date: Mon, 6 Aug 2018 10:47:00 +0200 Subject: [PATCH 13/55] Add support for unregistered instances in ELB DescribeInstancesHealth API. --- moto/elb/responses.py | 26 ++++++++++++++++++-------- tests/test_elb/test_elb.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/moto/elb/responses.py b/moto/elb/responses.py index 40d6ec2f9..b512f56e9 100644 --- a/moto/elb/responses.py +++ b/moto/elb/responses.py @@ -259,12 +259,22 @@ class ELBResponse(BaseResponse): def describe_instance_health(self): load_balancer_name = self._get_param('LoadBalancerName') - instance_ids = [list(param.values())[0] for param in self._get_list_prefix('Instances.member')] - if len(instance_ids) == 0: - instance_ids = self.elb_backend.get_load_balancer( - load_balancer_name).instance_ids + provided_instance_ids = [ + list(param.values())[0] + for param in self._get_list_prefix('Instances.member') + ] + registered_instances_id = self.elb_backend.get_load_balancer( + load_balancer_name).instance_ids + if len(provided_instance_ids) == 0: + provided_instance_ids = registered_instances_id template = self.response_template(DESCRIBE_INSTANCE_HEALTH_TEMPLATE) - return template.render(instance_ids=instance_ids) + instances = [] + for instance_id in provided_instance_ids: + state = "InService" \ + if instance_id in registered_instances_id\ + else "Unknown" + instances.append({"InstanceId": instance_id, "State": state}) + return template.render(instances=instances) def add_tags(self): @@ -689,11 +699,11 @@ SET_LOAD_BALANCER_POLICIES_FOR_BACKEND_SERVER_TEMPLATE = """ - {% for instance_id in instance_ids %} + {% for instance in instances %} N/A - {{ instance_id }} - InService + {{ instance['InstanceId'] }} + {{ instance['State'] }} N/A {% endfor %} diff --git a/tests/test_elb/test_elb.py b/tests/test_elb/test_elb.py index 5827e70c7..6ffcf84da 100644 --- a/tests/test_elb/test_elb.py +++ b/tests/test_elb/test_elb.py @@ -723,6 +723,40 @@ def test_describe_instance_health(): instances_health[0].state.should.equal('InService') +@mock_ec2 +@mock_elb +def test_describe_instance_health_boto3(): + elb = boto3.client('elb') + ec2 = boto3.client('ec2') + instances = ec2.run_instances(MinCount=2, MaxCount=2)['Instances'] + lb_name = "my_load_balancer" + elb.create_load_balancer( + Listeners=[{ + 'InstancePort': 80, + 'LoadBalancerPort': 8080, + 'Protocol': 'HTTP' + }], + LoadBalancerName=lb_name, + ) + elb.register_instances_with_load_balancer( + LoadBalancerName=lb_name, + Instances=[{'InstanceId': instances[0]['InstanceId']}] + ) + instances_health = elb.describe_instance_health( + LoadBalancerName=lb_name, + Instances=[{'InstanceId': instance['InstanceId']} for instance in instances] + ) + instances_health['InstanceStates'].should.have.length_of(2) + instances_health['InstanceStates'][0]['InstanceId'].\ + should.equal(instances[0]['InstanceId']) + instances_health['InstanceStates'][0]['State'].\ + should.equal('InService') + instances_health['InstanceStates'][1]['InstanceId'].\ + should.equal(instances[1]['InstanceId']) + instances_health['InstanceStates'][1]['State'].\ + should.equal('Unknown') + + @mock_elb def test_add_remove_tags(): client = boto3.client('elb', region_name='us-east-1') From 5d7a432af2dd2b19b671bc33fec42ed0ef8ef5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Faucon?= Date: Mon, 6 Aug 2018 11:01:33 +0200 Subject: [PATCH 14/55] Tests on Travis: specify region name. --- tests/test_elb/test_elb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_elb/test_elb.py b/tests/test_elb/test_elb.py index 6ffcf84da..a67508430 100644 --- a/tests/test_elb/test_elb.py +++ b/tests/test_elb/test_elb.py @@ -726,8 +726,8 @@ def test_describe_instance_health(): @mock_ec2 @mock_elb def test_describe_instance_health_boto3(): - elb = boto3.client('elb') - ec2 = boto3.client('ec2') + elb = boto3.client('elb', region_name="us-east-1") + ec2 = boto3.client('ec2', region_name="us-east-1") instances = ec2.run_instances(MinCount=2, MaxCount=2)['Instances'] lb_name = "my_load_balancer" elb.create_load_balancer( From 9e667d6b2582d0919f8f98dbd241a520124540ef Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Tue, 7 Aug 2018 16:59:15 -0400 Subject: [PATCH 15/55] Add get_policy IAM action --- moto/iam/models.py | 28 +++++++++++----------------- moto/iam/responses.py | 25 +++++++++++++++++++++++++ tests/test_iam/test_iam.py | 9 +++++++++ 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 697be7988..c8f95f3c9 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -72,11 +72,11 @@ class ManagedPolicy(Policy): def attach_to(self, obj): self.attachment_count += 1 - obj.managed_policies[self.name] = self + obj.managed_policies[self.arn] = self def detach_from(self, obj): self.attachment_count -= 1 - del obj.managed_policies[self.name] + del obj.managed_policies[self.arn] @property def arn(self): @@ -477,11 +477,13 @@ class IAMBackend(BaseBackend): document=policy_document, path=path, ) - self.managed_policies[policy.name] = policy + self.managed_policies[policy.arn] = policy return policy - def get_policy(self, policy_name): - return self.managed_policies.get(policy_name) + def get_policy(self, policy_arn): + if policy_arn not in self.managed_policies: + raise IAMNotFoundException("Policy {0} not found".format(policy_arn)) + return self.managed_policies.get(policy_arn) def list_attached_role_policies(self, role_name, marker=None, max_items=100, path_prefix='/'): policies = self.get_role(role_name).managed_policies.values() @@ -575,9 +577,7 @@ class IAMBackend(BaseBackend): return role.policies.keys() def create_policy_version(self, policy_arn, policy_document, set_as_default): - policy_name = policy_arn.split(':')[-1] - policy_name = policy_name.split('/')[1] - policy = self.get_policy(policy_name) + policy = self.get_policy(policy_arn) if not policy: raise IAMNotFoundException("Policy not found") version = PolicyVersion(policy_arn, policy_document, set_as_default) @@ -587,9 +587,7 @@ class IAMBackend(BaseBackend): return version def get_policy_version(self, policy_arn, version_id): - policy_name = policy_arn.split(':')[-1] - policy_name = policy_name.split('/')[1] - policy = self.get_policy(policy_name) + policy = self.get_policy(policy_arn) if not policy: raise IAMNotFoundException("Policy not found") for version in policy.versions: @@ -598,17 +596,13 @@ class IAMBackend(BaseBackend): raise IAMNotFoundException("Policy version not found") def list_policy_versions(self, policy_arn): - policy_name = policy_arn.split(':')[-1] - policy_name = policy_name.split('/')[1] - policy = self.get_policy(policy_name) + policy = self.get_policy(policy_arn) if not policy: raise IAMNotFoundException("Policy not found") return policy.versions def delete_policy_version(self, policy_arn, version_id): - policy_name = policy_arn.split(':')[-1] - policy_name = policy_name.split('/')[1] - policy = self.get_policy(policy_name) + policy = self.get_policy(policy_arn) if not policy: raise IAMNotFoundException("Policy not found") for i, v in enumerate(policy.versions): diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 9c1241c36..e77cec358 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -58,6 +58,12 @@ class IamResponse(BaseResponse): template = self.response_template(CREATE_POLICY_TEMPLATE) return template.render(policy=policy) + def get_policy(self): + policy_arn = self._get_param('PolicyArn') + policy = iam_backend.get_policy(policy_arn) + template = self.response_template(GET_POLICY_TEMPLATE) + return template.render(policy=policy) + def list_attached_role_policies(self): marker = self._get_param('Marker') max_items = self._get_int_param('MaxItems', 100) @@ -601,6 +607,25 @@ CREATE_POLICY_TEMPLATE = """ """ +GET_POLICY_TEMPLATE = """ + + + {{ policy.name }} + {{ policy.description }} + {{ policy.default_version_id }} + {{ policy.id }} + {{ policy.path }} + {{ policy.arn }} + {{ policy.attachment_count }} + {{ policy.create_datetime.isoformat() }} + {{ policy.update_datetime.isoformat() }} + + + + 684f0917-3d22-11e4-a4a0-cffb9EXAMPLE + +""" + LIST_ATTACHED_ROLE_POLICIES_TEMPLATE = """ {% if marker is none %} diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 2225f0644..61cd073fe 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -286,6 +286,15 @@ def test_create_policy_versions(): PolicyDocument='{"some":"policy"}') version.get('PolicyVersion').get('Document').should.equal({'some': 'policy'}) +@mock_iam +def test_get_policy(): + conn = boto3.client('iam', region_name='us-east-1') + response = conn.create_policy( + PolicyName="TestGetPolicy", + PolicyDocument='{"some":"policy"}') + policy = conn.get_policy( + PolicyArn="arn:aws:iam::123456789012:policy/TestGetPolicy") + response['Policy']['Arn'].should.equal("arn:aws:iam::123456789012:policy/TestGetPolicy") @mock_iam def test_get_policy_version(): From 5ae5ae0efae3312eed7da1cff1ea8785b7161b36 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Tue, 7 Aug 2018 17:24:15 -0400 Subject: [PATCH 16/55] Correct deviation in behavior of policy versions from standard API --- moto/iam/models.py | 8 ++++++-- tests/test_iam/test_iam.py | 22 ++++++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index c8f95f3c9..7b7c14b6c 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -37,7 +37,7 @@ class Policy(BaseModel): description=None, document=None, path=None): - self.document = document or {} + #self.document = document or {} self.name = name self.attachment_count = 0 @@ -45,7 +45,7 @@ class Policy(BaseModel): self.id = random_policy_id() self.path = path or '/' self.default_version_id = default_version_id or 'v1' - self.versions = [] + self.versions = [PolicyVersion(self.arn, document, True)] self.create_datetime = datetime.now(pytz.utc) self.update_datetime = datetime.now(pytz.utc) @@ -582,6 +582,7 @@ class IAMBackend(BaseBackend): raise IAMNotFoundException("Policy not found") version = PolicyVersion(policy_arn, policy_document, set_as_default) policy.versions.append(version) + version.version_id = 'v{0}'.format(len(policy.versions)) if set_as_default: policy.default_version_id = version.version_id return version @@ -605,6 +606,9 @@ class IAMBackend(BaseBackend): policy = self.get_policy(policy_arn) if not policy: raise IAMNotFoundException("Policy not found") + if version_id == policy.default_version_id: + raise IAMConflictException( + "Cannot delete the default version of a policy") for i, v in enumerate(policy.versions): if v.version_id == version_id: del policy.versions[i] diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 61cd073fe..bc23ff712 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -296,6 +296,7 @@ def test_get_policy(): PolicyArn="arn:aws:iam::123456789012:policy/TestGetPolicy") response['Policy']['Arn'].should.equal("arn:aws:iam::123456789012:policy/TestGetPolicy") + @mock_iam def test_get_policy_version(): conn = boto3.client('iam', region_name='us-east-1') @@ -323,17 +324,22 @@ def test_list_policy_versions(): PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions") conn.create_policy( PolicyName="TestListPolicyVersions", - PolicyDocument='{"some":"policy"}') - conn.create_policy_version( - PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions", PolicyDocument='{"first":"policy"}') + versions = conn.list_policy_versions( + PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions") + versions.get('Versions')[0].get('VersionId').should.equal('v1') + conn.create_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions", PolicyDocument='{"second":"policy"}') + conn.create_policy_version( + PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions", + PolicyDocument='{"third":"policy"}') versions = conn.list_policy_versions( PolicyArn="arn:aws:iam::123456789012:policy/TestListPolicyVersions") - versions.get('Versions')[0].get('Document').should.equal({'first': 'policy'}) + print(versions.get('Versions')) versions.get('Versions')[1].get('Document').should.equal({'second': 'policy'}) + versions.get('Versions')[2].get('Document').should.equal({'third': 'policy'}) @mock_iam @@ -341,20 +347,20 @@ def test_delete_policy_version(): conn = boto3.client('iam', region_name='us-east-1') conn.create_policy( PolicyName="TestDeletePolicyVersion", - PolicyDocument='{"some":"policy"}') + PolicyDocument='{"first":"policy"}') conn.create_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", - PolicyDocument='{"first":"policy"}') + PolicyDocument='{"second":"policy"}') with assert_raises(ClientError): conn.delete_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", VersionId='v2-nope-this-does-not-exist') conn.delete_policy_version( PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion", - VersionId='v1') + VersionId='v2') versions = conn.list_policy_versions( PolicyArn="arn:aws:iam::123456789012:policy/TestDeletePolicyVersion") - len(versions.get('Versions')).should.equal(0) + len(versions.get('Versions')).should.equal(1) @mock_iam_deprecated() From de4d5d9d6250669b0e0f84ccdbccb879556aad76 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Wed, 8 Aug 2018 11:13:25 -0400 Subject: [PATCH 17/55] Correct flake8 failures --- moto/iam/models.py | 1 - moto/iam/responses.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 7b7c14b6c..4d884fa2f 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -37,7 +37,6 @@ class Policy(BaseModel): description=None, document=None, path=None): - #self.document = document or {} self.name = name self.attachment_count = 0 diff --git a/moto/iam/responses.py b/moto/iam/responses.py index e77cec358..9e8d21396 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -62,7 +62,7 @@ class IamResponse(BaseResponse): policy_arn = self._get_param('PolicyArn') policy = iam_backend.get_policy(policy_arn) template = self.response_template(GET_POLICY_TEMPLATE) - return template.render(policy=policy) + return template.render(policy=policy) def list_attached_role_policies(self): marker = self._get_param('Marker') @@ -611,7 +611,7 @@ GET_POLICY_TEMPLATE = """ {{ policy.name }} - {{ policy.description }} + {{ policy.description }} {{ policy.default_version_id }} {{ policy.id }} {{ policy.path }} From 9d1c66531070774be508f17f8d8dd12429fed2e9 Mon Sep 17 00:00:00 2001 From: Nathan Sutton Date: Wed, 8 Aug 2018 19:14:56 -0500 Subject: [PATCH 18/55] Force receive_message_wait_time_seconds to be int When a queue is created with the ReceiveMessageWaitTimeSeconds attribute the value is never converted to an integer. When the ReceiveMessage action is called it tries to compare the string ReceiveMessageWaitTimeSeconds with the min and max wait times which raises a TypeError. The solution is to convert this value to an integer before comparing. --- moto/sqs/responses.py | 2 +- tests/test_sqs/test_sqs.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index c489d7118..b4f64b14e 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -336,7 +336,7 @@ class SQSResponse(BaseResponse): try: wait_time = int(self.querystring.get("WaitTimeSeconds")[0]) except TypeError: - wait_time = queue.receive_message_wait_time_seconds + wait_time = int(queue.receive_message_wait_time_seconds) if wait_time < 0 or wait_time > 20: return self._error( diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index d3e4ca917..bd6bbf0fb 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -1195,3 +1195,16 @@ def test_receive_messages_with_message_group_id_on_visibility_timeout(): messages = queue.receive_messages() messages.should.have.length_of(1) messages[0].message_id.should.equal(message.message_id) + +@mock_sqs +def test_receive_message_for_queue_with_receive_message_wait_time_seconds_set(): + sqs = boto3.resource('sqs', region_name='us-east-1') + + queue = sqs.create_queue( + QueueName='test-queue', + Attributes={ + 'ReceiveMessageWaitTimeSeconds': '2', + } + ) + + queue.receive_messages() From 8393c7f20be1395fcd1f8d3ccc19e946d679ee0b Mon Sep 17 00:00:00 2001 From: Nathan Sutton Date: Wed, 8 Aug 2018 21:10:13 -0500 Subject: [PATCH 19/55] Don't error on double create queue with same attrs Creating a queue a second time with the same attributes should not raise an error. This change makes it work correctly. --- moto/sqs/models.py | 20 ++++++++++++++++---- tests/test_sqs/test_sqs.py | 27 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index b8db356e9..f3262a988 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -385,10 +385,22 @@ class SQSBackend(BaseBackend): def create_queue(self, name, **kwargs): queue = self.queues.get(name) if queue: - # Queue already exist. If attributes don't match, throw error - for key, value in kwargs.items(): - if getattr(queue, camelcase_to_underscores(key)) != value: - raise QueueAlreadyExists("The specified queue already exists.") + try: + kwargs.pop('region') + except KeyError: + pass + + new_queue = Queue(name, region=self.region_name, **kwargs) + + queue_attributes = queue.attributes + new_queue_attributes = new_queue.attributes + + for key in ['CreatedTimestamp', 'LastModifiedTimestamp']: + queue_attributes.pop(key) + new_queue_attributes.pop(key) + + if queue_attributes != new_queue_attributes: + raise QueueAlreadyExists("The specified queue already exists.") else: try: kwargs.pop('region') diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index d3e4ca917..5d90d38b2 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -40,6 +40,33 @@ def test_create_fifo_queue_fail(): raise RuntimeError('Should of raised InvalidParameterValue Exception') +@mock_sqs +def test_create_queue_with_same_attributes(): + sqs = boto3.client('sqs', region_name='us-east-1') + + dlq_url = sqs.create_queue(QueueName='test-queue-dlq')['QueueUrl'] + dlq_arn = sqs.get_queue_attributes(QueueUrl=dlq_url)['Attributes']['QueueArn'] + + attributes = { + 'DelaySeconds': '900', + 'MaximumMessageSize': '262144', + 'MessageRetentionPeriod': '1209600', + 'ReceiveMessageWaitTimeSeconds': '20', + 'RedrivePolicy': '{"deadLetterTargetArn": "%s", "maxReceiveCount": 100}' % (dlq_arn), + 'VisibilityTimeout': '43200' + } + + sqs.create_queue( + QueueName='test-queue', + Attributes=attributes + ) + + sqs.create_queue( + QueueName='test-queue', + Attributes=attributes + ) + + @mock_sqs def test_create_queue_with_different_attributes_fail(): sqs = boto3.client('sqs', region_name='us-east-1') From 1b1fc4c030c446e23165cf28ad503de340ba645a Mon Sep 17 00:00:00 2001 From: Mike Liu Date: Wed, 25 Jul 2018 14:24:01 -0400 Subject: [PATCH 20/55] Support getting cloudformation targets for the SSM backend. SendCommand allows filtering for instances by tags. This adds support for filtering by a cloud formation stack resource when creating command invocations. --- moto/ssm/models.py | 32 ++++++++++++- tests/test_ssm/test_ssm_boto3.py | 78 +++++++++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 3 deletions(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index b3e34eae4..65d2f7b87 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -5,10 +5,12 @@ from collections import defaultdict from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError from moto.ec2 import ec2_backends +from moto.cloudformation import cloudformation_backends import datetime import time import uuid +import itertools class Parameter(BaseModel): @@ -67,7 +69,7 @@ class Command(BaseModel): instance_ids=None, max_concurrency='', max_errors='', notification_config=None, output_s3_bucket_name='', output_s3_key_prefix='', output_s3_region='', parameters=None, - service_role_arn='', targets=None): + service_role_arn='', targets=None, backend_region='us-east-1'): if instance_ids is None: instance_ids = [] @@ -105,6 +107,14 @@ class Command(BaseModel): self.parameters = parameters self.service_role_arn = service_role_arn self.targets = targets + self.backend_region = backend_region + + # Get instance ids from a cloud formation stack target. + stack_instance_ids = [self.get_instance_ids_by_stack_ids(target['Values']) for + target in self.targets if + target['Key'] == 'tag:aws:cloudformation:stack-name'] + + self.instance_ids += list(itertools.chain.from_iterable(stack_instance_ids)) # Create invocations with a single run command plugin. self.invocations = [] @@ -112,6 +122,18 @@ class Command(BaseModel): self.invocations.append( self.invocation_response(instance_id, "aws:runShellScript")) + def get_instance_ids_by_stack_ids(self, stack_ids): + instance_ids = [] + cloudformation_backend = cloudformation_backends[self.backend_region] + for stack_id in stack_ids: + stack_resources = cloudformation_backend.list_stack_resources(stack_id) + instance_resources = [ + instance.id for instance in stack_resources + if instance.type == "AWS::EC2::Instance"] + instance_ids.extend(instance_resources) + + return instance_ids + def response_object(self): r = { 'CommandId': self.command_id, @@ -191,6 +213,11 @@ class SimpleSystemManagerBackend(BaseBackend): self._resource_tags = defaultdict(lambda: defaultdict(dict)) self._commands = [] + # figure out what region we're in + for region, backend in ssm_backends.items(): + if backend == self: + self._region = region + def delete_parameter(self, name): try: del self._parameters[name] @@ -311,7 +338,8 @@ class SimpleSystemManagerBackend(BaseBackend): output_s3_region=kwargs.get('OutputS3Region', ''), parameters=kwargs.get('Parameters', {}), service_role_arn=kwargs.get('ServiceRoleArn', ''), - targets=kwargs.get('Targets', [])) + targets=kwargs.get('Targets', []), + backend_region=self._region) self._commands.append(command) return { diff --git a/tests/test_ssm/test_ssm_boto3.py b/tests/test_ssm/test_ssm_boto3.py index 8df565cf6..f8ef3a237 100644 --- a/tests/test_ssm/test_ssm_boto3.py +++ b/tests/test_ssm/test_ssm_boto3.py @@ -5,11 +5,12 @@ import botocore.exceptions import sure # noqa import datetime import uuid +import json from botocore.exceptions import ClientError from nose.tools import assert_raises -from moto import mock_ssm +from moto import mock_ssm, mock_cloudformation @mock_ssm @@ -708,3 +709,78 @@ def test_get_command_invocation(): CommandId=cmd_id, InstanceId=instance_id, PluginName='FAKE') + +@mock_ssm +@mock_cloudformation +def test_get_command_invocations_from_stack(): + stack_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Test Stack", + "Resources": { + "EC2Instance1": { + "Type": "AWS::EC2::Instance", + "Properties": { + "ImageId": "ami-test-image-id", + "KeyName": "test", + "InstanceType": "t2.micro", + "Tags": [ + { + "Key": "Test Description", + "Value": "Test tag" + }, + { + "Key": "Test Name", + "Value": "Name tag for tests" + } + ] + } + } + }, + "Outputs": { + "test": { + "Description": "Test Output", + "Value": "Test output value", + "Export": { + "Name": "Test value to export" + } + }, + "PublicIP": { + "Value": "Test public ip" + } + } + } + + cloudformation_client = boto3.client( + 'cloudformation', + region_name='us-east-1') + + stack_template_str = json.dumps(stack_template) + + response = cloudformation_client.create_stack( + StackName='test_stack', + TemplateBody=stack_template_str, + Capabilities=('CAPABILITY_IAM', )) + + client = boto3.client('ssm', region_name='us-east-1') + + ssm_document = 'AWS-RunShellScript' + params = {'commands': ['#!/bin/bash\necho \'hello world\'']} + + response = client.send_command( + Targets=[{ + 'Key': 'tag:aws:cloudformation:stack-name', + 'Values': ('test_stack', )}], + DocumentName=ssm_document, + Parameters=params, + OutputS3Region='us-east-2', + OutputS3BucketName='the-bucket', + OutputS3KeyPrefix='pref') + + cmd = response['Command'] + cmd_id = cmd['CommandId'] + instance_ids = cmd['InstanceIds'] + + invocation_response = client.get_command_invocation( + CommandId=cmd_id, + InstanceId=instance_ids[0], + PluginName='aws:runShellScript') From de532b93b78a01763ac622bd538ce12c41b86893 Mon Sep 17 00:00:00 2001 From: Mike Liu Date: Thu, 9 Aug 2018 10:53:32 -0400 Subject: [PATCH 21/55] Fix flake8. Remove extra whitespace. --- moto/ssm/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/moto/ssm/models.py b/moto/ssm/models.py index 65d2f7b87..f16a7d981 100644 --- a/moto/ssm/models.py +++ b/moto/ssm/models.py @@ -202,7 +202,6 @@ class Command(BaseModel): 'InvocationDoesNotExist', 'An error occurred (InvocationDoesNotExist) when calling the GetCommandInvocation operation') - return invocation From b75e78a3cdca3e5158702623364700ca29400c45 Mon Sep 17 00:00:00 2001 From: Manuel Adarve Date: Mon, 13 Aug 2018 17:09:39 +0200 Subject: [PATCH 22/55] ECS: Added schedulingStrategy support Co-authored-by: Jonas Gratz --- moto/ecs/models.py | 8 ++- moto/ecs/responses.py | 3 +- tests/test_ecs/test_ecs_boto3.py | 115 +++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 4 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 55fb4d4d9..d00853843 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -179,7 +179,7 @@ class Task(BaseObject): class Service(BaseObject): - def __init__(self, cluster, service_name, task_definition, desired_count, load_balancers=None): + def __init__(self, cluster, service_name, task_definition, desired_count, load_balancers=None, scheduling_strategy=None): self.cluster_arn = cluster.arn self.arn = 'arn:aws:ecs:us-east-1:012345678910:service/{0}'.format( service_name) @@ -202,6 +202,7 @@ class Service(BaseObject): } ] self.load_balancers = load_balancers if load_balancers is not None else [] + self.scheduling_strategy = scheduling_strategy if scheduling_strategy is not None else 'REPLICA' self.pending_count = 0 @property @@ -214,6 +215,7 @@ class Service(BaseObject): del response_object['name'], response_object['arn'] response_object['serviceName'] = self.name response_object['serviceArn'] = self.arn + response_object['schedulingStrategy'] = self.scheduling_strategy for deployment in response_object['deployments']: if isinstance(deployment['createdAt'], datetime): @@ -655,7 +657,7 @@ class EC2ContainerServiceBackend(BaseBackend): raise Exception("Could not find task {} on cluster {}".format( task_str, cluster_name)) - def create_service(self, cluster_str, service_name, task_definition_str, desired_count, load_balancers=None): + def create_service(self, cluster_str, service_name, task_definition_str, desired_count, load_balancers=None, scheduling_strategy=None): cluster_name = cluster_str.split('/')[-1] if cluster_name in self.clusters: cluster = self.clusters[cluster_name] @@ -665,7 +667,7 @@ class EC2ContainerServiceBackend(BaseBackend): desired_count = desired_count if desired_count is not None else 0 service = Service(cluster, service_name, - task_definition, desired_count, load_balancers) + task_definition, desired_count, load_balancers, scheduling_strategy) cluster_service_pair = '{0}:{1}'.format(cluster_name, service_name) self.services[cluster_service_pair] = service diff --git a/moto/ecs/responses.py b/moto/ecs/responses.py index 9455d7a28..e0bfefc02 100644 --- a/moto/ecs/responses.py +++ b/moto/ecs/responses.py @@ -154,8 +154,9 @@ class EC2ContainerServiceResponse(BaseResponse): task_definition_str = self._get_param('taskDefinition') desired_count = self._get_int_param('desiredCount') load_balancers = self._get_param('loadBalancers') + scheduling_strategy = self._get_param('schedulingStrategy') service = self.ecs_backend.create_service( - cluster_str, service_name, task_definition_str, desired_count, load_balancers) + cluster_str, service_name, task_definition_str, desired_count, load_balancers, scheduling_strategy) return json.dumps({ 'service': service.response_object }) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index bf72dc230..70c1463ee 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -304,6 +304,52 @@ def test_create_service(): response['service']['status'].should.equal('ACTIVE') response['service']['taskDefinition'].should.equal( 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') + response['service']['schedulingStrategy'].should.equal('REPLICA') + +@mock_ecs +def test_create_service_scheduling_strategy(): + client = boto3.client('ecs', region_name='us-east-1') + _ = client.create_cluster( + clusterName='test_ecs_cluster' + ) + _ = client.register_task_definition( + family='test_ecs_task', + containerDefinitions=[ + { + 'name': 'hello_world', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True, + 'environment': [{ + 'name': 'AWS_ACCESS_KEY_ID', + 'value': 'SOME_ACCESS_KEY' + }], + 'logConfiguration': {'logDriver': 'json-file'} + } + ] + ) + response = client.create_service( + cluster='test_ecs_cluster', + serviceName='test_ecs_service', + taskDefinition='test_ecs_task', + desiredCount=2, + schedulingStrategy='DAEMON', + ) + response['service']['clusterArn'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:cluster/test_ecs_cluster') + response['service']['desiredCount'].should.equal(2) + len(response['service']['events']).should.equal(0) + len(response['service']['loadBalancers']).should.equal(0) + response['service']['pendingCount'].should.equal(0) + response['service']['runningCount'].should.equal(0) + response['service']['serviceArn'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service') + response['service']['serviceName'].should.equal('test_ecs_service') + response['service']['status'].should.equal('ACTIVE') + response['service']['taskDefinition'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') + response['service']['schedulingStrategy'].should.equal('DAEMON') @mock_ecs @@ -411,6 +457,72 @@ def test_describe_services(): response['services'][0]['deployments'][0]['status'].should.equal('PRIMARY') +@mock_ecs +def test_describe_services_scheduling_strategy(): + client = boto3.client('ecs', region_name='us-east-1') + _ = client.create_cluster( + clusterName='test_ecs_cluster' + ) + _ = client.register_task_definition( + family='test_ecs_task', + containerDefinitions=[ + { + 'name': 'hello_world', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True, + 'environment': [{ + 'name': 'AWS_ACCESS_KEY_ID', + 'value': 'SOME_ACCESS_KEY' + }], + 'logConfiguration': {'logDriver': 'json-file'} + } + ] + ) + _ = client.create_service( + cluster='test_ecs_cluster', + serviceName='test_ecs_service1', + taskDefinition='test_ecs_task', + desiredCount=2 + ) + _ = client.create_service( + cluster='test_ecs_cluster', + serviceName='test_ecs_service2', + taskDefinition='test_ecs_task', + desiredCount=2, + schedulingStrategy='DAEMON' + ) + _ = client.create_service( + cluster='test_ecs_cluster', + serviceName='test_ecs_service3', + taskDefinition='test_ecs_task', + desiredCount=2 + ) + response = client.describe_services( + cluster='test_ecs_cluster', + services=['test_ecs_service1', + 'arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service2', + 'test_ecs_service3'] + ) + len(response['services']).should.equal(3) + response['services'][0]['serviceArn'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service1') + response['services'][0]['serviceName'].should.equal('test_ecs_service1') + response['services'][1]['serviceArn'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service2') + response['services'][1]['serviceName'].should.equal('test_ecs_service2') + + response['services'][0]['deployments'][0]['desiredCount'].should.equal(2) + response['services'][0]['deployments'][0]['pendingCount'].should.equal(2) + response['services'][0]['deployments'][0]['runningCount'].should.equal(0) + response['services'][0]['deployments'][0]['status'].should.equal('PRIMARY') + + response['services'][0]['schedulingStrategy'].should.equal('REPLICA') + response['services'][1]['schedulingStrategy'].should.equal('DAEMON') + response['services'][2]['schedulingStrategy'].should.equal('REPLICA') + + @mock_ecs def test_update_service(): client = boto3.client('ecs', region_name='us-east-1') @@ -449,6 +561,7 @@ def test_update_service(): desiredCount=0 ) response['service']['desiredCount'].should.equal(0) + response['service']['schedulingStrategy'].should.equal('REPLICA') @mock_ecs @@ -515,8 +628,10 @@ def test_delete_service(): 'arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service') response['service']['serviceName'].should.equal('test_ecs_service') response['service']['status'].should.equal('ACTIVE') + response['service']['schedulingStrategy'].should.equal('REPLICA') response['service']['taskDefinition'].should.equal( 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') + @mock_ec2 From 354d48fb8da739ee7134df41ff55128b2842e821 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Tue, 24 Jul 2018 21:11:35 +0800 Subject: [PATCH 23/55] Fix HTTPretty on Python 3.7 This is a revised backport of https://github.com/gabrielfalcao/HTTPretty/commit/5776d97da3992b9071db5e21faf175f6e8729060 and the following fixup https://github.com/gabrielfalcao/HTTPretty/pull/341 --- moto/packages/httpretty/core.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/moto/packages/httpretty/core.py b/moto/packages/httpretty/core.py index e0f3a7e69..8ad9168a5 100644 --- a/moto/packages/httpretty/core.py +++ b/moto/packages/httpretty/core.py @@ -85,6 +85,7 @@ old_socksocket = None old_ssl_wrap_socket = None old_sslwrap_simple = None old_sslsocket = None +old_sslcontext_wrap_socket = None if PY3: # pragma: no cover basestring = (bytes, str) @@ -100,6 +101,10 @@ try: # pragma: no cover if not PY3: old_sslwrap_simple = ssl.sslwrap_simple old_sslsocket = ssl.SSLSocket + try: + old_sslcontext_wrap_socket = ssl.SSLContext.wrap_socket + except AttributeError: + pass except ImportError: # pragma: no cover ssl = None @@ -281,7 +286,7 @@ class fakesock(object): return { 'notAfter': shift.strftime('%b %d %H:%M:%S GMT'), 'subjectAltName': ( - ('DNS', '*%s' % self._host), + ('DNS', '*.%s' % self._host), ('DNS', self._host), ('DNS', '*'), ), @@ -772,7 +777,7 @@ class URIMatcher(object): def __init__(self, uri, entries, match_querystring=False): self._match_querystring = match_querystring - if type(uri).__name__ == 'SRE_Pattern': + if type(uri).__name__ in ('SRE_Pattern', 'Pattern'): self.regex = uri result = urlsplit(uri.pattern) if result.scheme == 'https': @@ -1012,6 +1017,10 @@ class httpretty(HttpBaseClass): if ssl: ssl.wrap_socket = old_ssl_wrap_socket ssl.SSLSocket = old_sslsocket + try: + ssl.SSLContext.wrap_socket = old_sslcontext_wrap_socket + except AttributeError: + pass ssl.__dict__['wrap_socket'] = old_ssl_wrap_socket ssl.__dict__['SSLSocket'] = old_sslsocket @@ -1058,6 +1067,14 @@ class httpretty(HttpBaseClass): ssl.wrap_socket = fake_wrap_socket ssl.SSLSocket = FakeSSLSocket + try: + def fake_sslcontext_wrap_socket(cls, *args, **kwargs): + return fake_wrap_socket(*args, **kwargs) + + ssl.SSLContext.wrap_socket = fake_sslcontext_wrap_socket + except AttributeError: + pass + ssl.__dict__['wrap_socket'] = fake_wrap_socket ssl.__dict__['SSLSocket'] = FakeSSLSocket From d00300bc1b79d4f059a60683a56b559a96c6049f Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Tue, 24 Jul 2018 21:14:52 +0800 Subject: [PATCH 24/55] Test Python 3.7 on Travis CI --- .travis.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index f1b7ac40d..de22818b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,19 @@ python: env: - TEST_SERVER_MODE=false - TEST_SERVER_MODE=true +# Due to incomplete Python 3.7 support on Travis CI ( +# https://github.com/travis-ci/travis-ci/issues/9815), +# using a matrix is necessary +matrix: + include: + - python: 3.7 + env: TEST_SERVER_MODE=false + dist: xenial + sudo: true + - python: 3.7 + env: TEST_SERVER_MODE=true + dist: xenial + sudo: true before_install: - export BOTO_CONFIG=/dev/null install: From 52248589bf162d4f74aaad517ebac47e26c0b1a3 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Tue, 24 Jul 2018 21:32:43 +0800 Subject: [PATCH 25/55] Update lxml and sure for Python 3.7 compatibility --- requirements-dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 655be0616..111cd5f3f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ -r requirements.txt mock nose -sure==1.2.24 +sure==1.4.11 coverage flake8==3.5.0 freezegun @@ -13,5 +13,5 @@ six>=1.9 prompt-toolkit==1.0.14 click==6.7 inflection==0.3.1 -lxml==4.0.0 +lxml==4.2.3 beautifulsoup4==4.6.0 From 32c32ee5d764064327ba9881bc7cb310f2fa1296 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 10:29:58 -0700 Subject: [PATCH 26/55] Moto-1781: Implement rotate_secret response. --- moto/secretsmanager/responses.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/moto/secretsmanager/responses.py b/moto/secretsmanager/responses.py index c50c6a6e1..b8b6872a8 100644 --- a/moto/secretsmanager/responses.py +++ b/moto/secretsmanager/responses.py @@ -50,3 +50,15 @@ class SecretsManagerResponse(BaseResponse): return secretsmanager_backends[self.region].describe_secret( secret_id=secret_id ) + + def rotate_secret(self): + client_request_token = self._get_param('ClientRequestToken') + rotation_lambda_arn = self._get_param('RotationLambdaARN') + rotation_rules = self._get_param('RotationRules') + secret_id = self._get_param('SecretId') + return secretsmanager_backends[self.region].rotate_secret( + secret_id=secret_id, + client_request_token=client_request_token, + rotation_lambda_arn=rotation_lambda_arn, + rotation_rules=rotation_rules + ) From 0b0d16e8ec2b7b1b78d6a44a843d52e5db3bbdbc Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 10:33:38 -0700 Subject: [PATCH 27/55] Opportunistic update to unit test for consistency. --- tests/test_secretsmanager/test_secretsmanager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index c631fabb0..d3fc85467 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -26,13 +26,13 @@ def test_get_secret_that_does_not_exist(): result = conn.get_secret_value(SecretId='i-dont-exist') @mock_secretsmanager -def test_get_secret_with_mismatched_id(): +def test_get_secret_that_does_not_match(): conn = boto3.client('secretsmanager', region_name='us-west-2') create_secret = conn.create_secret(Name='java-util-test-password', SecretString="foosecret") with assert_raises(ClientError): - result = conn.get_secret_value(SecretId='i-dont-exist') + result = conn.get_secret_value(SecretId='i-dont-match') @mock_secretsmanager def test_create_secret(): From 69a78ba7c98dba3886a63d7923fa18bea94eaea1 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 10:52:30 -0700 Subject: [PATCH 28/55] Moto-1781: Create initial stub of rotate_secret and tests. --- moto/secretsmanager/models.py | 15 +++++++++++++++ tests/test_secretsmanager/test_secretsmanager.py | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index c60feb530..4d9c4ecbe 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -36,6 +36,7 @@ class SecretsManagerBackend(BaseBackend): self.rotation_enabled = False self.rotation_lambda_arn = '' self.auto_rotate_after_days = 0 + self.version_id = '' def reset(self): region_name = self.region @@ -105,6 +106,20 @@ class SecretsManagerBackend(BaseBackend): return response + def rotate_secret(self, secret_id, client_request_token=None, + rotation_lambda_arn=None, rotation_rules=None): + + if not self._is_valid_identifier(secret_id): + raise ResourceNotFoundException + + response = json.dumps({ + "ARN": secret_arn(self.region, self.secret_id), + "Name": self.name, + "VersionId": self.version_id + }) + + return response + def get_random_password(self, password_length, exclude_characters, exclude_numbers, exclude_punctuation, exclude_uppercase, diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index d3fc85467..bbc87e5c1 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -179,3 +179,19 @@ def test_describe_secret_that_does_not_match(): with assert_raises(ClientError): result = conn.get_secret_value(SecretId='i-dont-match') + +@mock_secretsmanager +def test_rotate_secret_that_does_not_exist(): + conn = boto3.client('secretsmanager', 'us-west-2') + + with assert_raises(ClientError): + result = conn.rotate_secret(SecretId='i-dont-exist') + +@mock_secretsmanager +def test_rotate_secret_that_does_not_match(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name='test-secret', + SecretString='foosecret') + + with assert_raises(ClientError): + result = conn.rotate_secret(SecretId='i-dont-match') From 43277a59b9df04f6017265c7a57fd84e7d697f10 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 11:04:44 -0700 Subject: [PATCH 29/55] Moto-1781: finish implementation of rotate_secret and add happy test. - Implement RotateSecret to support initial setup of secret rotation. Moto's implementation of secrets is currently flat and needs to gain some dimension before full rotation can be simulated. - Add the happy path unit test. --- moto/secretsmanager/models.py | 36 +++++++++++++++++++ .../test_secretsmanager.py | 16 +++++++++ 2 files changed, 52 insertions(+) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 4d9c4ecbe..1404a0ec8 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -109,9 +109,45 @@ class SecretsManagerBackend(BaseBackend): def rotate_secret(self, secret_id, client_request_token=None, rotation_lambda_arn=None, rotation_rules=None): + rotation_days = 'AutomaticallyAfterDays' + if not self._is_valid_identifier(secret_id): raise ResourceNotFoundException + if client_request_token: + token_length = len(client_request_token) + if token_length < 32 or token_length > 64: + msg = ( + 'ClientRequestToken ' + 'must be 32-64 characters long.' + ) + raise InvalidParameterException(msg) + + if rotation_lambda_arn: + if len(rotation_lambda_arn) > 2048: + msg = ( + 'RotationLambdaARN ' + 'must <= 2048 characters long.' + ) + raise InvalidParameterException(msg) + + if rotation_rules: + if rotation_days in rotation_rules: + rotation_period = rotation_rules[rotation_days] + if rotation_period < 1 or rotation_period > 1000: + msg = ( + 'RotationRules.AutomaticallyAfterDays ' + 'must be within 1-1000.' + ) + raise InvalidParameterException(msg) + + self.version_id = client_request_token or '' + self.rotation_lambda_arn = rotation_lambda_arn or '' + if rotation_rules: + self.auto_rotate_after_days = rotation_rules.get(rotation_days, 0) + if self.auto_rotate_after_days > 0: + self.rotation_enabled = True + response = json.dumps({ "ARN": secret_arn(self.region, self.secret_id), "Name": self.name, diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index bbc87e5c1..4b15a3c01 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -180,6 +180,22 @@ def test_describe_secret_that_does_not_match(): with assert_raises(ClientError): result = conn.get_secret_value(SecretId='i-dont-match') +@mock_secretsmanager +def test_rotate_secret(): + secret_name = 'test-secret' + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name=secret_name, + SecretString='foosecret') + + rotated_secret = conn.rotate_secret(SecretId=secret_name) + + assert rotated_secret + assert rotated_secret['ARN'] == ( + 'arn:aws:secretsmanager:us-west-2:1234567890:secret:test-secret-rIjad' + ) + assert rotated_secret['Name'] == secret_name + assert rotated_secret['VersionId'] != '' + @mock_secretsmanager def test_rotate_secret_that_does_not_exist(): conn = boto3.client('secretsmanager', 'us-west-2') From 6e716a5926f12e9292125db516b3ab8614ed224f Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 11:06:26 -0700 Subject: [PATCH 30/55] Moto-1781: Update implementation coverage. --- IMPLEMENTATION_COVERAGE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 938cc3549..b6524b451 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3645,7 +3645,7 @@ - [ ] put_attributes - [ ] select -## secretsmanager - 27% implemented +## secretsmanager - 33% implemented - [ ] cancel_rotate_secret - [X] create_secret - [ ] delete_secret @@ -3656,7 +3656,7 @@ - [ ] list_secrets - [ ] put_secret_value - [ ] restore_secret -- [ ] rotate_secret +- [X] rotate_secret - [ ] tag_resource - [ ] untag_resource - [ ] update_secret From b2997304b4dd76040f8bca005683427d4ce5fdea Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 11:20:29 -0700 Subject: [PATCH 31/55] Moto-1781: Implement standalone unit tests to cover bad parameters. --- .../test_secretsmanager.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 4b15a3c01..3e7b31c72 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -211,3 +211,56 @@ def test_rotate_secret_that_does_not_match(): with assert_raises(ClientError): result = conn.rotate_secret(SecretId='i-dont-match') + +@mock_secretsmanager +def test_rotate_secret_client_request_token_too_short(): + # Test is intentionally empty. Boto3 catches too short ClientRequestToken + # and raises ParamValidationError before Moto can see it. + # test_server actually handles this error. + assert True + +@mock_secretsmanager +def test_rotate_secret_client_request_token_too_long(): + secret_name = 'test-secret' + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name=secret_name, + SecretString='foosecret') + + client_request_token = ( + 'ED9F8B6C-85B7-446A-B7E4-38F2A3BEB13C-' + 'ED9F8B6C-85B7-446A-B7E4-38F2A3BEB13C' + ) + with assert_raises(ClientError): + result = conn.rotate_secret(SecretId=secret_name, + ClientRequestToken=client_request_token) + +@mock_secretsmanager +def test_rotate_secret_rotation_lambda_arn_too_long(): + secret_name = 'test-secret' + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name=secret_name, + SecretString='foosecret') + + rotation_lambda_arn = '85B7-446A-B7E4' * 147 # == 2058 characters + with assert_raises(ClientError): + result = conn.rotate_secret(SecretId=secret_name, + RotationLambdaARN=rotation_lambda_arn) + +@mock_secretsmanager +def test_rotate_secret_rotation_period_zero(): + # Test is intentionally empty. Boto3 catches zero day rotation period + # and raises ParamValidationError before Moto can see it. + # test_server actually handles this error. + assert True + +@mock_secretsmanager +def test_rotate_secret_rotation_period_too_long(): + secret_name = 'test-secret' + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name=secret_name, + SecretString='foosecret') + + rotation_rules = {'AutomaticallyAfterDays': 1001} + with assert_raises(ClientError): + result = conn.rotate_secret(SecretId=secret_name, + RotationRules=rotation_rules) From 4ced0ce0dbae6a3d488681bb3d6546b39f04db85 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 11:22:07 -0700 Subject: [PATCH 32/55] Opportunistic update to unit test for consistency. --- tests/test_secretsmanager/test_server.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index 8c6f7b970..a45263304 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -49,6 +49,27 @@ def test_get_secret_that_does_not_exist(): assert json_data['message'] == "Secrets Manager can't find the specified secret" assert json_data['__type'] == 'ResourceNotFoundException' +@mock_secretsmanager +def test_get_secret_that_does_not_match(): + backend = server.create_backend_app("secretsmanager") + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foo-secret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret"}, + ) + get_secret = test_client.post('/', + data={"SecretId": "i-dont-match", + "VersionStage": "AWSCURRENT"}, + headers={ + "X-Amz-Target": "secretsmanager.GetSecretValue"}, + ) + json_data = json.loads(get_secret.data.decode("utf-8")) + assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['__type'] == 'ResourceNotFoundException' + @mock_secretsmanager def test_create_secret(): From 71ed78141ab03f634c49c52d109740d7f545ce1b Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 11:32:54 -0700 Subject: [PATCH 33/55] Moto-1781: Implement server unit tests. *** Includes 2 commented out tests that generate errors pointing deeper in the code. *** --- tests/test_secretsmanager/test_server.py | 208 +++++++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index a45263304..b5494ef96 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -154,3 +154,211 @@ def test_describe_secret_that_does_not_match(): json_data = json.loads(describe_secret.data.decode("utf-8")) assert json_data['message'] == "Secrets Manager can't find the specified secret" assert json_data['__type'] == 'ResourceNotFoundException' + +@mock_secretsmanager +def test_rotate_secret(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + + client_request_token = "EXAMPLE2-90ab-cdef-fedc-ba987SECRET2" + rotate_secret = test_client.post('/', + data={"SecretId": "test-secret", + "ClientRequestToken": client_request_token}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data # Returned dict is not empty + assert json_data['ARN'] == ( + 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad' + ) + assert json_data['Name'] == 'test-secret' + assert json_data['VersionId'] == client_request_token + +@mock_secretsmanager +def test_rotate_secret_that_does_not_exist(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + rotate_secret = test_client.post('/', + data={"SecretId": "i-dont-exist"}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['__type'] == 'ResourceNotFoundException' + +@mock_secretsmanager +def test_rotate_secret_that_does_not_match(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + + rotate_secret = test_client.post('/', + data={"SecretId": "i-dont-match"}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['__type'] == 'ResourceNotFoundException' + +@mock_secretsmanager +def test_rotate_secret_client_request_token_too_short(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + + client_request_token = "ED9F8B6C-85B7-B7E4-38F2A3BEB13C" + rotate_secret = test_client.post('/', + data={"SecretId": "test-secret", + "ClientRequestToken": client_request_token}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data['message'] == "ClientRequestToken must be 32-64 characters long." + assert json_data['__type'] == 'InvalidParameterException' + +@mock_secretsmanager +def test_rotate_secret_client_request_token_too_long(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + + client_request_token = ( + 'ED9F8B6C-85B7-446A-B7E4-38F2A3BEB13C-' + 'ED9F8B6C-85B7-446A-B7E4-38F2A3BEB13C' + ) + rotate_secret = test_client.post('/', + data={"SecretId": "test-secret", + "ClientRequestToken": client_request_token}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data['message'] == "ClientRequestToken must be 32-64 characters long." + assert json_data['__type'] == 'InvalidParameterException' + +@mock_secretsmanager +def test_rotate_secret_rotation_lambda_arn_too_long(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": "test-secret", + "SecretString": "foosecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + + rotation_lambda_arn = '85B7-446A-B7E4' * 147 # == 2058 characters + rotate_secret = test_client.post('/', + data={"SecretId": "test-secret", + "RotationLambdaARN": rotation_lambda_arn}, + headers={ + "X-Amz-Target": "secretsmanager.RotateSecret" + }, + ) + + json_data = json.loads(rotate_secret.data.decode("utf-8")) + assert json_data['message'] == "RotationLambdaARN must <= 2048 characters long." + assert json_data['__type'] == 'InvalidParameterException' + + +# +# The following tests should work, but fail on the embedded dict in +# RotationRules. The error message suggests a problem deeper in the code, which +# needs further investigation. +# + +# @mock_secretsmanager +# def test_rotate_secret_rotation_period_zero(): +# backend = server.create_backend_app('secretsmanager') +# test_client = backend.test_client() + +# create_secret = test_client.post('/', +# data={"Name": "test-secret", +# "SecretString": "foosecret"}, +# headers={ +# "X-Amz-Target": "secretsmanager.CreateSecret" +# }, +# ) + +# rotate_secret = test_client.post('/', +# data={"SecretId": "test-secret", +# "RotationRules": {"AutomaticallyAfterDays": 0}}, +# headers={ +# "X-Amz-Target": "secretsmanager.RotateSecret" +# }, +# ) + +# json_data = json.loads(rotate_secret.data.decode("utf-8")) +# assert json_data['message'] == "RotationLambdaARN must <= 2048 characters long." +# assert json_data['__type'] == 'InvalidParameterException' + +# @mock_secretsmanager +# def test_rotate_secret_rotation_period_too_long(): +# backend = server.create_backend_app('secretsmanager') +# test_client = backend.test_client() + +# create_secret = test_client.post('/', +# data={"Name": "test-secret", +# "SecretString": "foosecret"}, +# headers={ +# "X-Amz-Target": "secretsmanager.CreateSecret" +# }, +# ) + +# rotate_secret = test_client.post('/', +# data={"SecretId": "test-secret", +# "RotationRules": {"AutomaticallyAfterDays": 1001}}, +# headers={ +# "X-Amz-Target": "secretsmanager.RotateSecret" +# }, +# ) + +# json_data = json.loads(rotate_secret.data.decode("utf-8")) +# assert json_data['message'] == "RotationLambdaARN must <= 2048 characters long." +# assert json_data['__type'] == 'InvalidParameterException' From 6985f27167ad7323dad4f9896b42a32a9b8e6836 Mon Sep 17 00:00:00 2001 From: Neil Roberts Date: Wed, 15 Aug 2018 17:11:58 -0700 Subject: [PATCH 34/55] Moto-1781: Add unit tests to verify that rotation is enabled. - Add standalone unit test to verify that rotation is enabled and the rotation interval is correct. - Add server test to verify that rotation is enabled and the rotation interval is correct. Commented out until nested dict error is sorted. - Fix incorrectly asserted message strings. --- .../test_secretsmanager.py | 20 ++++++ tests/test_secretsmanager/test_server.py | 61 ++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 3e7b31c72..ec384a660 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -196,6 +196,26 @@ def test_rotate_secret(): assert rotated_secret['Name'] == secret_name assert rotated_secret['VersionId'] != '' +@mock_secretsmanager +def test_rotate_secret_enable_rotation(): + secret_name = 'test-secret' + conn = boto3.client('secretsmanager', region_name='us-west-2') + conn.create_secret(Name=secret_name, + SecretString='foosecret') + + initial_description = conn.describe_secret(SecretId=secret_name) + assert initial_description + assert initial_description['RotationEnabled'] is False + assert initial_description['RotationRules']['AutomaticallyAfterDays'] == 0 + + conn.rotate_secret(SecretId=secret_name, + RotationRules={'AutomaticallyAfterDays': 42}) + + rotated_description = conn.describe_secret(SecretId=secret_name) + assert rotated_description + assert rotated_description['RotationEnabled'] is True + assert rotated_description['RotationRules']['AutomaticallyAfterDays'] == 42 + @mock_secretsmanager def test_rotate_secret_that_does_not_exist(): conn = boto3.client('secretsmanager', 'us-west-2') diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index b5494ef96..e573f9b67 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -185,6 +185,63 @@ def test_rotate_secret(): assert json_data['Name'] == 'test-secret' assert json_data['VersionId'] == client_request_token +# @mock_secretsmanager +# def test_rotate_secret_enable_rotation(): +# backend = server.create_backend_app('secretsmanager') +# test_client = backend.test_client() + +# create_secret = test_client.post( +# '/', +# data={ +# "Name": "test-secret", +# "SecretString": "foosecret" +# }, +# headers={ +# "X-Amz-Target": "secretsmanager.CreateSecret" +# }, +# ) + +# initial_description = test_client.post( +# '/', +# data={ +# "SecretId": "test-secret" +# }, +# headers={ +# "X-Amz-Target": "secretsmanager.DescribeSecret" +# }, +# ) + +# json_data = json.loads(initial_description.data.decode("utf-8")) +# assert json_data # Returned dict is not empty +# assert json_data['RotationEnabled'] is False +# assert json_data['RotationRules']['AutomaticallyAfterDays'] == 0 + +# rotate_secret = test_client.post( +# '/', +# data={ +# "SecretId": "test-secret", +# "RotationRules": {"AutomaticallyAfterDays": 42} +# }, +# headers={ +# "X-Amz-Target": "secretsmanager.RotateSecret" +# }, +# ) + +# rotated_description = test_client.post( +# '/', +# data={ +# "SecretId": "test-secret" +# }, +# headers={ +# "X-Amz-Target": "secretsmanager.DescribeSecret" +# }, +# ) + +# json_data = json.loads(rotated_description.data.decode("utf-8")) +# assert json_data # Returned dict is not empty +# assert json_data['RotationEnabled'] is True +# assert json_data['RotationRules']['AutomaticallyAfterDays'] == 42 + @mock_secretsmanager def test_rotate_secret_that_does_not_exist(): backend = server.create_backend_app('secretsmanager') @@ -335,7 +392,7 @@ def test_rotate_secret_rotation_lambda_arn_too_long(): # ) # json_data = json.loads(rotate_secret.data.decode("utf-8")) -# assert json_data['message'] == "RotationLambdaARN must <= 2048 characters long." +# assert json_data['message'] == "RotationRules.AutomaticallyAfterDays must be within 1-1000." # assert json_data['__type'] == 'InvalidParameterException' # @mock_secretsmanager @@ -360,5 +417,5 @@ def test_rotate_secret_rotation_lambda_arn_too_long(): # ) # json_data = json.loads(rotate_secret.data.decode("utf-8")) -# assert json_data['message'] == "RotationLambdaARN must <= 2048 characters long." +# assert json_data['message'] == "RotationRules.AutomaticallyAfterDays must be within 1-1000." # assert json_data['__type'] == 'InvalidParameterException' From 001a7d0278d8827019622541609e036e3cecc6f5 Mon Sep 17 00:00:00 2001 From: chrisLeeTW Date: Sat, 11 Aug 2018 22:14:04 +0800 Subject: [PATCH 35/55] type of json field - nextSequenceToken that return by cloudwatch logs putLogEvents should be string, not int. --- moto/logs/models.py | 2 +- tests/test_logs/test_logs.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/moto/logs/models.py b/moto/logs/models.py index 3e1c7b955..a4ff9db46 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -86,7 +86,7 @@ class LogStream: self.events += [LogEvent(self.lastIngestionTime, log_event) for log_event in log_events] self.uploadSequenceToken += 1 - return self.uploadSequenceToken + return '{:056d}'.format(self.uploadSequenceToken) def get_log_events(self, log_group_name, log_stream_name, start_time, end_time, limit, next_token, start_from_head): def filter_func(event): diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index 3f924cc55..05bd3c823 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -1,5 +1,6 @@ import boto3 import sure # noqa +import six from botocore.exceptions import ClientError from moto import mock_logs, settings @@ -47,7 +48,7 @@ def test_exceptions(): logEvents=[ { 'timestamp': 0, - 'message': 'line' + 'message': 'line' }, ], ) @@ -79,7 +80,7 @@ def test_put_logs(): {'timestamp': 0, 'message': 'hello'}, {'timestamp': 0, 'message': 'world'} ] - conn.put_log_events( + putRes = conn.put_log_events( logGroupName=log_group_name, logStreamName=log_stream_name, logEvents=messages @@ -89,6 +90,9 @@ def test_put_logs(): logStreamName=log_stream_name ) events = res['events'] + nextSequenceToken = putRes['nextSequenceToken'] + assert isinstance(nextSequenceToken, six.string_types) == True + assert len(nextSequenceToken) == 56 events.should.have.length_of(2) From c6f5e816ccda6386976a5e3c9677a6b9839ec108 Mon Sep 17 00:00:00 2001 From: Brian Pandola Date: Mon, 20 Aug 2018 18:48:13 -0700 Subject: [PATCH 36/55] Add `ClusterCreateTime` to Redshift response Fixes #1778 --- moto/redshift/models.py | 2 ++ tests/test_redshift/test_redshift.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 4eafcfc79..70cbb95cb 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -78,6 +78,7 @@ class Cluster(TaggableResourceMixin, BaseModel): super(Cluster, self).__init__(region_name, tags) self.redshift_backend = redshift_backend self.cluster_identifier = cluster_identifier + self.create_time = iso_8601_datetime_with_milliseconds(datetime.datetime.now()) self.status = 'available' self.node_type = node_type self.master_username = master_username @@ -237,6 +238,7 @@ class Cluster(TaggableResourceMixin, BaseModel): "Address": self.endpoint, "Port": self.port }, + 'ClusterCreateTime': self.create_time, "PendingModifiedValues": [], "Tags": self.tags, "IamRoles": [{ diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index 6e027b86c..9208c92dd 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import datetime + import boto import boto3 from boto.redshift.exceptions import ( @@ -32,6 +34,8 @@ def test_create_cluster_boto3(): MasterUserPassword='password', ) response['Cluster']['NodeType'].should.equal('ds2.xlarge') + create_time = response['Cluster']['ClusterCreateTime'] + create_time.should.be.lower_than(datetime.datetime.now(create_time.tzinfo)) @mock_redshift From 50f8d5e13c70686da3b865832ae3e0ec4fa4aad2 Mon Sep 17 00:00:00 2001 From: Matthew Neal Date: Wed, 29 Aug 2018 12:42:31 -0400 Subject: [PATCH 37/55] Pin boto3 version to 1.7.84 Pinning boto3 to this version would remove the mismatch between boto3 and the botocore dependency in setup.py. Closes #1800 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 16aaf1452..bcb48a967 100755 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ import sys install_requires = [ "Jinja2>=2.7.3", "boto>=2.36.0", - "boto3>=1.6.16", + "boto3>=1.6.16,<1.8", "botocore>=1.9.16,<1.11", "cookies", "cryptography>=2.0.0", From 54cbc98506d261be9ee2228b9ccf584e8b723234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Diaz?= Date: Mon, 3 Sep 2018 16:34:19 -0300 Subject: [PATCH 38/55] Return given host for 'moto-api' --- moto/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moto/server.py b/moto/server.py index aad47757a..ba2470478 100644 --- a/moto/server.py +++ b/moto/server.py @@ -34,6 +34,9 @@ class DomainDispatcherApplication(object): self.service = service def get_backend_for_host(self, host): + if host == 'moto_api': + return host + if self.service: return self.service From 28aa5d34b0bf2417089bf49fd1573a620720997a Mon Sep 17 00:00:00 2001 From: Ollie Ford Date: Wed, 5 Sep 2018 10:43:39 +0100 Subject: [PATCH 39/55] Add failing test for #1809 --- tests/test_core/test_decorator_calls.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_core/test_decorator_calls.py b/tests/test_core/test_decorator_calls.py index 9e3638cc2..5d2f6a4ef 100644 --- a/tests/test_core/test_decorator_calls.py +++ b/tests/test_core/test_decorator_calls.py @@ -85,3 +85,14 @@ class TesterWithSetup(unittest.TestCase): def test_still_the_same(self): bucket = self.conn.get_bucket('mybucket') bucket.name.should.equal("mybucket") + + +@mock_s3_deprecated +class TesterWithStaticmethod(object): + + @staticmethod + def static(*args): + assert not args or not isinstance(args[0], TesterWithStaticmethod) + + def test_no_instance_sent_to_staticmethod(self): + self.static() From 0ac989cfd4a0e684df2f0170821d1598526484bf Mon Sep 17 00:00:00 2001 From: Ollie Ford Date: Wed, 5 Sep 2018 10:39:09 +0100 Subject: [PATCH 40/55] Fix #1809: skip patching staticmethods --- moto/core/models.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/moto/core/models.py b/moto/core/models.py index 92dc2a980..adc06a9c0 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -89,6 +89,17 @@ class BaseMockAWS(object): if inspect.ismethod(attr_value) and attr_value.__self__ is klass: continue + # Check if this is a staticmethod. If so, skip patching + for cls in inspect.getmro(klass): + if attr_value.__name__ not in cls.__dict__: + continue + bound_attr_value = cls.__dict__[attr_value.__name__] + if not isinstance(bound_attr_value, staticmethod): + break + else: + # It is a staticmethod, skip patching + continue + try: setattr(klass, attr, self(attr_value, reset=False)) except TypeError: From d60d562c6262e5e2b58a2d8e5ee9f5f12eb09fc3 Mon Sep 17 00:00:00 2001 From: Justin McCormick Date: Sun, 9 Sep 2018 02:14:28 -0400 Subject: [PATCH 41/55] Add AWS::Partition as a variable available to CloudFormation templates --- moto/cloudformation/parsing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index c4059a06b..35b05d101 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -387,6 +387,7 @@ class ResourceMap(collections.Mapping): "AWS::StackName": stack_name, "AWS::URLSuffix": "amazonaws.com", "AWS::NoValue": None, + "AWS::Partition": "aws", } def __getitem__(self, key): From 756b5d6671c04e8a13d86e82224ea35c32a053e4 Mon Sep 17 00:00:00 2001 From: Jordan Upiter Date: Fri, 5 Jan 2018 15:12:45 -0500 Subject: [PATCH 42/55] Add support for multiple delete markers on an s3 object --- moto/s3/models.py | 11 ++++--- moto/s3/responses.py | 6 ++-- tests/test_s3/test_s3.py | 66 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index cf5628141..f3994b5d8 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -27,8 +27,14 @@ class FakeDeleteMarker(BaseModel): def __init__(self, key): self.key = key + self.name = key.name + self.last_modified = datetime.datetime.utcnow() self._version_id = key.version_id + 1 + @property + def last_modified_ISO8601(self): + return iso_8601_datetime_with_milliseconds(self.last_modified) + @property def version_id(self): return self._version_id @@ -630,10 +636,7 @@ class S3Backend(BaseBackend): latest_versions = {} for version in versions: - if isinstance(version, FakeDeleteMarker): - name = version.key.name - else: - name = version.name + name = version.name version_id = version.version_id maximum_version_per_key[name] = max( version_id, diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 5e7cf0fe5..f8dc7e42b 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1273,10 +1273,10 @@ S3_BUCKET_GET_VERSIONS = """ {% endfor %} {% for marker in delete_marker_list %} - {{ marker.key.name }} + {{ marker.name }} {{ marker.version_id }} - {% if latest_versions[marker.key.name] == marker.version_id %}true{% else %}false{% endif %} - {{ marker.key.last_modified_ISO8601 }} + {% if latest_versions[marker.name] == marker.version_id %}true{% else %}false{% endif %} + {{ marker.last_modified_ISO8601 }} 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a webfile diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 9a68d1bbb..6e339abb6 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -2471,6 +2471,72 @@ def test_boto3_delete_markers(): oldest['Key'].should.equal('key-with-versions-and-unicode-ó') +@mock_s3 +def test_boto3_multiple_delete_markers(): + s3 = boto3.client('s3', region_name='us-east-1') + bucket_name = 'mybucket' + key = u'key-with-versions-and-unicode-ó' + s3.create_bucket(Bucket=bucket_name) + s3.put_bucket_versioning( + Bucket=bucket_name, + VersioningConfiguration={ + 'Status': 'Enabled' + } + ) + items = (six.b('v1'), six.b('v2')) + for body in items: + s3.put_object( + Bucket=bucket_name, + Key=key, + Body=body + ) + + # Delete the object twice to add multiple delete markers + s3.delete_object(Bucket=bucket_name, Key=key) + s3.delete_object(Bucket=bucket_name, Key=key) + + response = s3.list_object_versions(Bucket=bucket_name) + response['DeleteMarkers'].should.have.length_of(2) + + with assert_raises(ClientError) as e: + s3.get_object( + Bucket=bucket_name, + Key=key + ) + e.response['Error']['Code'].should.equal('404') + + # Remove both delete markers to restore the object + s3.delete_object( + Bucket=bucket_name, + Key=key, + VersionId='2' + ) + s3.delete_object( + Bucket=bucket_name, + Key=key, + VersionId='3' + ) + + response = s3.get_object( + Bucket=bucket_name, + Key=key + ) + response['Body'].read().should.equal(items[-1]) + response = s3.list_object_versions(Bucket=bucket_name) + response['Versions'].should.have.length_of(2) + + # We've asserted there is only 2 records so one is newest, one is oldest + latest = list(filter(lambda item: item['IsLatest'], response['Versions']))[0] + oldest = list(filter(lambda item: not item['IsLatest'], response['Versions']))[0] + + # Double check ordering of version ID's + latest['VersionId'].should.equal('1') + oldest['VersionId'].should.equal('0') + + # Double check the name is still unicode + latest['Key'].should.equal('key-with-versions-and-unicode-ó') + oldest['Key'].should.equal('key-with-versions-and-unicode-ó') + @mock_s3 def test_get_stream_gzipped(): payload = b"this is some stuff here" From 4ffff9161e1a859f5beef431f576e1151e247d95 Mon Sep 17 00:00:00 2001 From: Daniel Huang Date: Thu, 20 Sep 2018 11:23:20 -0700 Subject: [PATCH 43/55] Remove unused cookies dependency --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 16aaf1452..df00255ed 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ install_requires = [ "boto>=2.36.0", "boto3>=1.6.16", "botocore>=1.9.16,<1.11", - "cookies", "cryptography>=2.0.0", "requests>=2.5", "xmltodict", From 881afc8f4a35564caef6098d383f7213fb14693e Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 08:31:31 -0500 Subject: [PATCH 44/55] update RDS models to include CopyTagsToSnapshot --- AUTHORS.md | 1 + moto/rds/models.py | 6 +++++ moto/rds2/models.py | 8 +++++++ tests/test_rds2/test_rds2.py | 44 ++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index 6b7c96291..0a152505a 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -53,3 +53,4 @@ Moto is written by Steve Pulec with contributions from: * [Jim Shields](https://github.com/jimjshields) * [William Richard](https://github.com/william-richard) * [Alex Casalboni](https://github.com/alexcasalboni) +* [Jon Beilke](https://github.com/jrbeilke) diff --git a/moto/rds/models.py b/moto/rds/models.py index 77deff09d..feecefe0c 100644 --- a/moto/rds/models.py +++ b/moto/rds/models.py @@ -48,6 +48,10 @@ class Database(BaseModel): if self.publicly_accessible is None: self.publicly_accessible = True + self.copy_tags_to_snapshot = kwargs.get("copy_tags_to_snapshot") + if self.copy_tags_to_snapshot is None: + self.copy_tags_to_snapshot = False + self.backup_retention_period = kwargs.get("backup_retention_period") if self.backup_retention_period is None: self.backup_retention_period = 1 @@ -137,6 +141,7 @@ class Database(BaseModel): "multi_az": properties.get("MultiAZ"), "port": properties.get('Port', 3306), "publicly_accessible": properties.get("PubliclyAccessible"), + "copy_tags_to_snapshot": properties.get("CopyTagsToSnapshot"), "region": region_name, "security_groups": security_groups, "storage_encrypted": properties.get("StorageEncrypted"), @@ -217,6 +222,7 @@ class Database(BaseModel): {% endif %} {{ database.publicly_accessible }} + {{ database.copy_tags_to_snapshot }} {{ database.auto_minor_version_upgrade }} {{ database.allocated_storage }} {{ database.storage_encrypted }} diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 3fc4b6d65..c656f5ec3 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -73,6 +73,9 @@ class Database(BaseModel): self.publicly_accessible = kwargs.get("publicly_accessible") if self.publicly_accessible is None: self.publicly_accessible = True + self.copy_tags_to_snapshot = kwargs.get("copy_tags_to_snapshot") + if self.copy_tags_to_snapshot is None: + self.copy_tags_to_snapshot = False self.backup_retention_period = kwargs.get("backup_retention_period") if self.backup_retention_period is None: self.backup_retention_period = 1 @@ -208,6 +211,7 @@ class Database(BaseModel): {% endif %} {{ database.publicly_accessible }} + {{ database.copy_tags_to_snapshot }} {{ database.auto_minor_version_upgrade }} {{ database.allocated_storage }} {{ database.storage_encrypted }} @@ -304,6 +308,7 @@ class Database(BaseModel): "db_parameter_group_name": properties.get('DBParameterGroupName'), "port": properties.get('Port', 3306), "publicly_accessible": properties.get("PubliclyAccessible"), + "copy_tags_to_snapshot": properties.get("CopyTagsToSnapshot"), "region": region_name, "security_groups": security_groups, "storage_encrypted": properties.get("StorageEncrypted"), @@ -362,6 +367,7 @@ class Database(BaseModel): "PreferredBackupWindow": "{{ database.preferred_backup_window }}", "PreferredMaintenanceWindow": "{{ database.preferred_maintenance_window }}", "PubliclyAccessible": "{{ database.publicly_accessible }}", + "CopyTagsToSnapshot": "{{ database.copy_tags_to_snapshot }}", "AllocatedStorage": "{{ database.allocated_storage }}", "Endpoint": { "Address": "{{ database.address }}", @@ -691,6 +697,8 @@ class RDS2Backend(BaseBackend): raise DBSnapshotAlreadyExistsError(db_snapshot_identifier) if len(self.snapshots) >= int(os.environ.get('MOTO_RDS_SNAPSHOT_LIMIT', '100')): raise SnapshotQuotaExceededError() + if not database.copy_tags_to_snapshot: + tags = None snapshot = Snapshot(database, db_snapshot_identifier, tags) self.snapshots[db_snapshot_identifier] = snapshot return snapshot diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 80dcd4f53..7fecfeca9 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -33,6 +33,7 @@ def test_create_database(): db_instance['DBInstanceIdentifier'].should.equal("db-master-1") db_instance['IAMDatabaseAuthenticationEnabled'].should.equal(False) db_instance['DbiResourceId'].should.contain("db-") + db_instance['CopyTagsToSnapshot'].should.equal(False) @mock_rds2 @@ -339,6 +340,49 @@ def test_create_db_snapshots(): snapshot.get('Engine').should.equal('postgres') snapshot.get('DBInstanceIdentifier').should.equal('db-primary-1') snapshot.get('DBSnapshotIdentifier').should.equal('g-1') + result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshot']['DBSnapshotArn']) + result['TagList'].should.equal([]) + + +@mock_rds2 +def test_create_db_snapshots_copy_tags(): + conn = boto3.client('rds', region_name='us-west-2') + conn.create_db_snapshot.when.called_with( + DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-1').should.throw(ClientError) + + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"], + CopyTagsToSnapshot=True, + Tags=[ + { + 'Key': 'foo', + 'Value': 'bar', + }, + { + 'Key': 'foo1', + 'Value': 'bar1', + }, + ]) + + snapshot = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='g-1').get('DBSnapshot') + + snapshot.get('Engine').should.equal('postgres') + snapshot.get('DBInstanceIdentifier').should.equal('db-primary-1') + snapshot.get('DBSnapshotIdentifier').should.equal('g-1') + result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshot']['DBSnapshotArn']) + result['TagList'].should.equal([{'Value': 'bar', + 'Key': 'foo'}, + {'Value': 'bar1', + 'Key': 'foo1'}]) @mock_rds2 From bf9b37142e7d95102dfe11c506d5ae1a3f2779ed Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 08:49:45 -0500 Subject: [PATCH 45/55] no need for [DBSnapshot] with list_tags_for_resource as the retuned snapshot already handles it --- tests/test_rds2/test_rds2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 7fecfeca9..0f802bb5c 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -340,7 +340,7 @@ def test_create_db_snapshots(): snapshot.get('Engine').should.equal('postgres') snapshot.get('DBInstanceIdentifier').should.equal('db-primary-1') snapshot.get('DBSnapshotIdentifier').should.equal('g-1') - result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshot']['DBSnapshotArn']) + result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshotArn']) result['TagList'].should.equal([]) @@ -378,7 +378,7 @@ def test_create_db_snapshots_copy_tags(): snapshot.get('Engine').should.equal('postgres') snapshot.get('DBInstanceIdentifier').should.equal('db-primary-1') snapshot.get('DBSnapshotIdentifier').should.equal('g-1') - result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshot']['DBSnapshotArn']) + result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshotArn']) result['TagList'].should.equal([{'Value': 'bar', 'Key': 'foo'}, {'Value': 'bar1', From cedb89dc3b2a7e4701b68f06a010d17f3a06f98f Mon Sep 17 00:00:00 2001 From: hans Date: Fri, 21 Sep 2018 23:29:04 +0800 Subject: [PATCH 46/55] Fix #1830 Add support for cross-region VPC peering Add a class level store in models/VPCBackend of ec2 for saving vpcs of all regions info. Any instance can correctly find vpc in another region when connecting vpc of cross-region or vpc of same region. Modify vpc_peering_connections in ec2/responses to handle vpc peering of same region or cross region. Update vpc_peering_connections response template content to latest (2016-11-15) . Add vpc cross region peering successful test case. Add vpc cross region peering fail test case. Related: https://github.com/spulec/moto/issues/1830 Reference CreateVpcPeeringConnection Sample Response https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateVpcPeeringConnection.html --- moto/ec2/models.py | 18 +++++++++ moto/ec2/responses/vpc_peering_connections.py | 31 +++++++++------ tests/test_ec2/test_vpc_peering.py | 39 ++++++++++++++++++- 3 files changed, 76 insertions(+), 12 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 4e26f0f65..b94cac479 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -13,6 +13,7 @@ from pkg_resources import resource_filename import boto.ec2 from collections import defaultdict +import weakref from datetime import datetime from boto.ec2.instance import Instance as BotoInstance, Reservation from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType @@ -2115,10 +2116,20 @@ class VPC(TaggedEC2Resource): class VPCBackend(object): + __refs__ = defaultdict(list) + def __init__(self): self.vpcs = {} + self.__refs__[self.__class__].append(weakref.ref(self)) super(VPCBackend, self).__init__() + @classmethod + def get_instances(cls): + for inst_ref in cls.__refs__[cls]: + inst = inst_ref() + if inst is not None: + yield inst + def create_vpc(self, cidr_block, instance_tenancy='default', amazon_provided_ipv6_cidr_block=False): vpc_id = random_vpc_id() vpc = VPC(self, vpc_id, cidr_block, len(self.vpcs) == 0, instance_tenancy, amazon_provided_ipv6_cidr_block) @@ -2142,6 +2153,13 @@ class VPCBackend(object): raise InvalidVPCIdError(vpc_id) return self.vpcs.get(vpc_id) + # get vpc by vpc id and aws region + def get_cross_vpc(self, vpc_id, peer_region): + for vpcs in self.get_instances(): + if vpcs.region_name == peer_region: + match_vpc = vpcs.get_vpc(vpc_id) + return match_vpc + def get_all_vpcs(self, vpc_ids=None, filters=None): matches = self.vpcs.values() if vpc_ids: diff --git a/moto/ec2/responses/vpc_peering_connections.py b/moto/ec2/responses/vpc_peering_connections.py index 1bccce4f6..49d752893 100644 --- a/moto/ec2/responses/vpc_peering_connections.py +++ b/moto/ec2/responses/vpc_peering_connections.py @@ -5,8 +5,12 @@ from moto.core.responses import BaseResponse class VPCPeeringConnections(BaseResponse): def create_vpc_peering_connection(self): + peer_region = self._get_param('PeerRegion') + if peer_region == self.region or peer_region is None: + peer_vpc = self.ec2_backend.get_vpc(self._get_param('PeerVpcId')) + else: + peer_vpc = self.ec2_backend.get_cross_vpc(self._get_param('PeerVpcId'), peer_region) vpc = self.ec2_backend.get_vpc(self._get_param('VpcId')) - peer_vpc = self.ec2_backend.get_vpc(self._get_param('PeerVpcId')) vpc_pcx = self.ec2_backend.create_vpc_peering_connection(vpc, peer_vpc) template = self.response_template( CREATE_VPC_PEERING_CONNECTION_RESPONSE) @@ -41,26 +45,31 @@ class VPCPeeringConnections(BaseResponse): CREATE_VPC_PEERING_CONNECTION_RESPONSE = """ - - 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE - - {{ vpc_pcx.id }} + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + + {{ vpc_pcx.id }} - 777788889999 - {{ vpc_pcx.vpc.id }} - {{ vpc_pcx.vpc.cidr_block }} + 777788889999 + {{ vpc_pcx.vpc.id }} + {{ vpc_pcx.vpc.cidr_block }} + + false + false + false + 123456789012 {{ vpc_pcx.peer_vpc.id }} - initiating-request - Initiating request to {accepter ID}. + initiating-request + Initiating Request to {accepter ID} 2014-02-18T14:37:25.000Z - + """ diff --git a/tests/test_ec2/test_vpc_peering.py b/tests/test_ec2/test_vpc_peering.py index 6722eed60..1f98791b3 100644 --- a/tests/test_ec2/test_vpc_peering.py +++ b/tests/test_ec2/test_vpc_peering.py @@ -2,12 +2,15 @@ from __future__ import unicode_literals # Ensure 'assert_raises' context manager support for Python 2.6 import tests.backport_assert_raises from nose.tools import assert_raises +from moto.ec2.exceptions import EC2ClientError +from botocore.exceptions import ClientError +import boto3 import boto from boto.exception import EC2ResponseError import sure # noqa -from moto import mock_ec2_deprecated +from moto import mock_ec2, mock_ec2_deprecated from tests.helpers import requires_boto_gte @@ -93,3 +96,37 @@ def test_vpc_peering_connections_delete(): cm.exception.code.should.equal('InvalidVpcPeeringConnectionId.NotFound') cm.exception.status.should.equal(400) cm.exception.request_id.should_not.be.none + + +@mock_ec2 +def test_vpc_peering_connections_cross_region(): + # create vpc in us-west-1 and ap-northeast-1 + ec2_usw1 = boto3.resource('ec2', region_name='us-west-1') + vpc_usw1 = ec2_usw1.create_vpc(CidrBlock='10.90.0.0/16') + ec2_apn1 = boto3.resource('ec2', region_name='ap-northeast-1') + vpc_apn1 = ec2_apn1.create_vpc(CidrBlock='10.20.0.0/16') + # create peering + vpc_pcx = ec2_usw1.create_vpc_peering_connection( + VpcId=vpc_usw1.id, + PeerVpcId=vpc_apn1.id, + PeerRegion='ap-northeast-1', + ) + vpc_pcx.status['Code'].should.equal('initiating-request') + vpc_pcx.requester_vpc.id.should.equal(vpc_usw1.id) + vpc_pcx.accepter_vpc.id.should.equal(vpc_apn1.id) + + +@mock_ec2 +def test_vpc_peering_connections_cross_region_fail(): + # create vpc in us-west-1 and ap-northeast-1 + ec2_usw1 = boto3.resource('ec2', region_name='us-west-1') + vpc_usw1 = ec2_usw1.create_vpc(CidrBlock='10.90.0.0/16') + ec2_apn1 = boto3.resource('ec2', region_name='ap-northeast-1') + vpc_apn1 = ec2_apn1.create_vpc(CidrBlock='10.20.0.0/16') + # create peering wrong region with no vpc + with assert_raises(ClientError) as cm: + ec2_usw1.create_vpc_peering_connection( + VpcId=vpc_usw1.id, + PeerVpcId=vpc_apn1.id, + PeerRegion='ap-northeast-2') + cm.exception.response['Error']['Code'].should.equal('InvalidVpcID.NotFound') From 276da0616851183bd5a2cd6dd8f80882816aea4a Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 10:39:42 -0500 Subject: [PATCH 47/55] added new merge_taglists() to moto.core.utils for merging lists of tags with precedence (ie. during rds2.create_snapshot) --- moto/core/utils.py | 9 +++++++++ moto/rds2/models.py | 5 +++-- tests/test_core/test_utils.py | 27 ++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index 86e7632b0..2cdcd07f1 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -286,3 +286,12 @@ def amzn_request_id(f): return status, headers, body return _wrapper + + +def merge_taglists(taglist_a, taglist_b): + ''' Merges two tag lists into a single tag list with Keys in the second list taking precedence''' + tags_a = {t['Key']:t for t in taglist_a} + tags_b = {t['Key']:t for t in taglist_b} + merged_tags = tags_a.copy() + merged_tags.update(tags_b) + return merged_tags.values() diff --git a/moto/rds2/models.py b/moto/rds2/models.py index c656f5ec3..d564e93b1 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -13,6 +13,7 @@ from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import get_random_hex from moto.core.utils import iso_8601_datetime_with_milliseconds +from moto.core.utils import merge_taglists from moto.ec2.models import ec2_backends from .exceptions import (RDSClientError, DBInstanceNotFoundError, @@ -697,8 +698,8 @@ class RDS2Backend(BaseBackend): raise DBSnapshotAlreadyExistsError(db_snapshot_identifier) if len(self.snapshots) >= int(os.environ.get('MOTO_RDS_SNAPSHOT_LIMIT', '100')): raise SnapshotQuotaExceededError() - if not database.copy_tags_to_snapshot: - tags = None + if database.copy_tags_to_snapshot: + tags = merge_taglists(database.tags, tags) snapshot = Snapshot(database, db_snapshot_identifier, tags) self.snapshots[db_snapshot_identifier] = snapshot return snapshot diff --git a/tests/test_core/test_utils.py b/tests/test_core/test_utils.py index 8dbf21716..0fa4f3c59 100644 --- a/tests/test_core/test_utils.py +++ b/tests/test_core/test_utils.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import sure # noqa from freezegun import freeze_time -from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase, unix_time +from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase, unix_time, merge_taglists def test_camelcase_to_underscores(): @@ -28,3 +28,28 @@ def test_underscores_to_camelcase(): @freeze_time("2015-01-01 12:00:00") def test_unix_time(): unix_time().should.equal(1420113600.0) + + +def test_merge_taglists(): + taglist_a = [ + { + 'Key': 'foo', + 'Value': 'bar', + }, + { + 'Key': 'foo1', + 'Value': 'bar1', + }, + ] + taglist_b = [ + { + 'Key': 'foo1', + 'Value': 'bar1b', + }, + ] + taglist_merged = merge_taglists(taglist_a, taglist_b) + len(taglist_merged).should.equal(2) + tag_foo = [t for t in taglist_merged if t['Key']=='foo'] + tag_foo1 = [t for t in taglist_merged if t['Key']=='foo1'] + tag_foo[0].should.equal({'Key': 'foo','Value': 'bar',}) + tag_foo1[0].should.equal({'Key': 'foo1','Value': 'bar1b',}) From 1729681106203f4b692d12283ec01ffadaca20c4 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 10:45:22 -0500 Subject: [PATCH 48/55] formatting fix for E231 missing whitespace after : --- moto/core/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index 2cdcd07f1..21ac8c2e8 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -290,8 +290,8 @@ def amzn_request_id(f): def merge_taglists(taglist_a, taglist_b): ''' Merges two tag lists into a single tag list with Keys in the second list taking precedence''' - tags_a = {t['Key']:t for t in taglist_a} - tags_b = {t['Key']:t for t in taglist_b} + tags_a = {t['Key']: t for t in taglist_a} + tags_b = {t['Key']: t for t in taglist_b} merged_tags = tags_a.copy() merged_tags.update(tags_b) return merged_tags.values() From 1b8b32a663dc52a3a138331bc79fce4ffa38d316 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 11:13:33 -0500 Subject: [PATCH 49/55] add CopyTagsToSnapshot to db_kwargs --- moto/rds2/responses.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index eddb0042b..6b1da103b 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -19,6 +19,7 @@ class RDS2Response(BaseResponse): "allocated_storage": self._get_int_param('AllocatedStorage'), "availability_zone": self._get_param("AvailabilityZone"), "backup_retention_period": self._get_param("BackupRetentionPeriod"), + "copy_tags_to_snapshot": self._get_param("CopyTagsToSnapshot"), "db_instance_class": self._get_param('DBInstanceClass'), "db_instance_identifier": self._get_param('DBInstanceIdentifier'), "db_name": self._get_param("DBName"), From 6eb490ac78d217a2c570db997c926ba08b5c9503 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 12:03:13 -0500 Subject: [PATCH 50/55] add support for tags to rds snapshots --- moto/rds2/models.py | 18 +++++- tests/test_rds2/test_rds2.py | 111 +++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 2 deletions(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index d564e93b1..b85fe8b05 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -463,6 +463,20 @@ class Snapshot(BaseModel): """) return template.render(snapshot=self, database=self.database) + def get_tags(self): + return self.tags + + def add_tags(self, tags): + new_keys = [tag_set['Key'] for tag_set in tags] + self.tags = [tag_set for tag_set in self.tags if tag_set[ + 'Key'] not in new_keys] + self.tags.extend(tags) + return self.tags + + def remove_tags(self, tag_keys): + self.tags = [tag_set for tag_set in self.tags if tag_set[ + 'Key'] not in tag_keys] + class SecurityGroup(BaseModel): @@ -1037,8 +1051,8 @@ class RDS2Backend(BaseBackend): if resource_name in self.security_groups: return self.security_groups[resource_name].get_tags() elif resource_type == 'snapshot': # DB Snapshot - # TODO: Complete call to tags on resource type DB Snapshot - return [] + if resource_name in self.snapshots: + return self.snapshots[resource_name].get_tags() elif resource_type == 'subgrp': # DB subnet group if resource_name in self.subnet_groups: return self.subnet_groups[resource_name].get_tags() diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 0f802bb5c..cf9805444 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -700,6 +700,117 @@ def test_remove_tags_db(): len(result['TagList']).should.equal(1) +@mock_rds2 +def test_list_tags_snapshot(): + conn = boto3.client('rds', region_name='us-west-2') + result = conn.list_tags_for_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:foo') + result['TagList'].should.equal([]) + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"]) + snapshot = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-with-tags', + Tags=[ + { + 'Key': 'foo', + 'Value': 'bar', + }, + { + 'Key': 'foo1', + 'Value': 'bar1', + }, + ]) + result = conn.list_tags_for_resource(ResourceName=snapshot['DBSnapshot']['DBSnapshotArn']) + result['TagList'].should.equal([{'Value': 'bar', + 'Key': 'foo'}, + {'Value': 'bar1', + 'Key': 'foo1'}]) + + +@mock_rds2 +def test_add_tags_snapshot(): + conn = boto3.client('rds', region_name='us-west-2') + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"]) + snapshot = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-without-tags', + Tags=[ + { + 'Key': 'foo', + 'Value': 'bar', + }, + { + 'Key': 'foo1', + 'Value': 'bar1', + }, + ]) + result = conn.list_tags_for_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-without-tags') + list(result['TagList']).should.have.length_of(2) + conn.add_tags_to_resource(ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-without-tags', + Tags=[ + { + 'Key': 'foo', + 'Value': 'fish', + }, + { + 'Key': 'foo2', + 'Value': 'bar2', + }, + ]) + result = conn.list_tags_for_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-without-tags') + list(result['TagList']).should.have.length_of(3) + + +@mock_rds2 +def test_remove_tags_snapshot(): + conn = boto3.client('rds', region_name='us-west-2') + conn.create_db_instance(DBInstanceIdentifier='db-primary-1', + AllocatedStorage=10, + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small', + MasterUsername='root', + MasterUserPassword='hunter2', + Port=1234, + DBSecurityGroups=["my_sg"]) + snapshot = conn.create_db_snapshot(DBInstanceIdentifier='db-primary-1', + DBSnapshotIdentifier='snapshot-with-tags', + Tags=[ + { + 'Key': 'foo', + 'Value': 'bar', + }, + { + 'Key': 'foo1', + 'Value': 'bar1', + }, + ]) + result = conn.list_tags_for_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-with-tags') + list(result['TagList']).should.have.length_of(2) + conn.remove_tags_from_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-with-tags', TagKeys=['foo']) + result = conn.list_tags_for_resource( + ResourceName='arn:aws:rds:us-west-2:1234567890:snapshot:snapshot-with-tags') + len(result['TagList']).should.equal(1) + + @mock_rds2 def test_add_tags_option_group(): conn = boto3.client('rds', region_name='us-west-2') From 7daee905a5a62b534d3e098558ce29d40278751f Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 13:28:13 -0500 Subject: [PATCH 51/55] remove merge_taglists as AWS will only take submitted tags or tags from db but not both when creating snapshot --- moto/core/utils.py | 9 --------- moto/rds2/models.py | 15 ++++++++------- moto/rds2/responses.py | 2 +- tests/test_core/test_utils.py | 27 +-------------------------- 4 files changed, 10 insertions(+), 43 deletions(-) diff --git a/moto/core/utils.py b/moto/core/utils.py index 21ac8c2e8..86e7632b0 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -286,12 +286,3 @@ def amzn_request_id(f): return status, headers, body return _wrapper - - -def merge_taglists(taglist_a, taglist_b): - ''' Merges two tag lists into a single tag list with Keys in the second list taking precedence''' - tags_a = {t['Key']: t for t in taglist_a} - tags_b = {t['Key']: t for t in taglist_b} - merged_tags = tags_a.copy() - merged_tags.update(tags_b) - return merged_tags.values() diff --git a/moto/rds2/models.py b/moto/rds2/models.py index b85fe8b05..b37306b12 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -13,7 +13,6 @@ from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import get_random_hex from moto.core.utils import iso_8601_datetime_with_milliseconds -from moto.core.utils import merge_taglists from moto.ec2.models import ec2_backends from .exceptions import (RDSClientError, DBInstanceNotFoundError, @@ -418,10 +417,10 @@ class Database(BaseModel): class Snapshot(BaseModel): - def __init__(self, database, snapshot_id, tags=None): + def __init__(self, database, snapshot_id, tags): self.database = database self.snapshot_id = snapshot_id - self.tags = tags or [] + self.tags = tags self.created_at = iso_8601_datetime_with_milliseconds(datetime.datetime.now()) @property @@ -712,8 +711,10 @@ class RDS2Backend(BaseBackend): raise DBSnapshotAlreadyExistsError(db_snapshot_identifier) if len(self.snapshots) >= int(os.environ.get('MOTO_RDS_SNAPSHOT_LIMIT', '100')): raise SnapshotQuotaExceededError() - if database.copy_tags_to_snapshot: - tags = merge_taglists(database.tags, tags) + if tags is None: + tags = list() + if database.copy_tags_to_snapshot and not tags: + tags = database.get_tags() snapshot = Snapshot(database, db_snapshot_identifier, tags) self.snapshots[db_snapshot_identifier] = snapshot return snapshot @@ -810,13 +811,13 @@ class RDS2Backend(BaseBackend): def delete_database(self, db_instance_identifier, db_snapshot_name=None): if db_instance_identifier in self.databases: + if db_snapshot_name: + self.create_snapshot(db_instance_identifier, db_snapshot_name) database = self.databases.pop(db_instance_identifier) if database.is_replica: primary = self.find_db_from_id(database.source_db_identifier) primary.remove_replica(database) database.status = 'deleting' - if db_snapshot_name: - self.snapshots[db_snapshot_name] = Snapshot(database, db_snapshot_name) return database else: raise DBInstanceNotFoundError(db_instance_identifier) diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index 6b1da103b..66d4e0c52 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -160,7 +160,7 @@ class RDS2Response(BaseResponse): def create_db_snapshot(self): db_instance_identifier = self._get_param('DBInstanceIdentifier') db_snapshot_identifier = self._get_param('DBSnapshotIdentifier') - tags = self._get_param('Tags', []) + tags = self.unpack_complex_list_params('Tags.Tag', ('Key', 'Value')) snapshot = self.backend.create_snapshot(db_instance_identifier, db_snapshot_identifier, tags) template = self.response_template(CREATE_SNAPSHOT_TEMPLATE) return template.render(snapshot=snapshot) diff --git a/tests/test_core/test_utils.py b/tests/test_core/test_utils.py index 0fa4f3c59..8dbf21716 100644 --- a/tests/test_core/test_utils.py +++ b/tests/test_core/test_utils.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import sure # noqa from freezegun import freeze_time -from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase, unix_time, merge_taglists +from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase, unix_time def test_camelcase_to_underscores(): @@ -28,28 +28,3 @@ def test_underscores_to_camelcase(): @freeze_time("2015-01-01 12:00:00") def test_unix_time(): unix_time().should.equal(1420113600.0) - - -def test_merge_taglists(): - taglist_a = [ - { - 'Key': 'foo', - 'Value': 'bar', - }, - { - 'Key': 'foo1', - 'Value': 'bar1', - }, - ] - taglist_b = [ - { - 'Key': 'foo1', - 'Value': 'bar1b', - }, - ] - taglist_merged = merge_taglists(taglist_a, taglist_b) - len(taglist_merged).should.equal(2) - tag_foo = [t for t in taglist_merged if t['Key']=='foo'] - tag_foo1 = [t for t in taglist_merged if t['Key']=='foo1'] - tag_foo[0].should.equal({'Key': 'foo','Value': 'bar',}) - tag_foo1[0].should.equal({'Key': 'foo1','Value': 'bar1b',}) From 245e3a5f719bcf74846e789d3c3f1b4f9b56ca53 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 13:33:10 -0500 Subject: [PATCH 52/55] formatting fix for E111 indentation is not a multiple of four --- moto/rds2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index b37306b12..159a56ebd 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -712,7 +712,7 @@ class RDS2Backend(BaseBackend): if len(self.snapshots) >= int(os.environ.get('MOTO_RDS_SNAPSHOT_LIMIT', '100')): raise SnapshotQuotaExceededError() if tags is None: - tags = list() + tags = list() if database.copy_tags_to_snapshot and not tags: tags = database.get_tags() snapshot = Snapshot(database, db_snapshot_identifier, tags) From 67a0e06059101e90a0370bc1f7b10bbe1414821c Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 21 Sep 2018 13:54:07 -0500 Subject: [PATCH 53/55] allow for adding and removing tags on rds snapshots --- moto/rds2/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 159a56ebd..fee004f76 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -1083,7 +1083,8 @@ class RDS2Backend(BaseBackend): if resource_name in self.security_groups: return self.security_groups[resource_name].remove_tags(tag_keys) elif resource_type == 'snapshot': # DB Snapshot - return None + if resource_name in self.snapshots: + return self.snapshots[resource_name].remove_tags(tag_keys) elif resource_type == 'subgrp': # DB subnet group if resource_name in self.subnet_groups: return self.subnet_groups[resource_name].remove_tags(tag_keys) @@ -1112,7 +1113,8 @@ class RDS2Backend(BaseBackend): if resource_name in self.security_groups: return self.security_groups[resource_name].add_tags(tags) elif resource_type == 'snapshot': # DB Snapshot - return [] + if resource_name in self.snapshots: + return self.snapshots[resource_name].add_tags(tags) elif resource_type == 'subgrp': # DB subnet group if resource_name in self.subnet_groups: return self.subnet_groups[resource_name].add_tags(tags) From 57f9691a52245ffd9d7c0317e90953d054070c65 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sat, 22 Sep 2018 15:36:25 -0400 Subject: [PATCH 54/55] Version 1.3.6 --- CHANGELOG.md | 5 +++++ moto/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 202da6ce6..7f7ee4448 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ Moto Changelog =================== +1.3.6 +----- + + * Fix boto3 pinning. + 1.3.5 ----- diff --git a/moto/__init__.py b/moto/__init__.py index b7b653200..37628e6e4 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.5' +__version__ = '1.3.6' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/setup.py b/setup.py index 919b9849b..dad9ab9bb 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ else: setup( name='moto', - version='1.3.5', + version='1.3.6', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From 8bdcc6244d118ab58d8eb50ceb0a15dfbeeecdf6 Mon Sep 17 00:00:00 2001 From: Justin Eyster Date: Mon, 24 Sep 2018 15:58:35 -0400 Subject: [PATCH 55/55] Addresses security vulnerability in cryptography<2.3 Discovered using pipenv's security check feature that there's a vulnerability in the cryptography package versions<2.3. > Checking installed package safety... 36351: cryptography >=1.9.0,<2.3 resolved (2.2.2 installed)! python-cryptography versions >=1.9.0 and <2.3 did not enforce a minimum tag length for finalize_with_tag API. If a user did not validate the input length prior to passing it to finalize_with_tag an attacker could craft an invalid payload with a shortened tag (e.g. 1 byte) such that they would have a 1 in 256 chance of passing the MAC check. GCM tag forgeries can cause key leakage. More details here: http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-10903 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dad9ab9bb..98780dd5a 100755 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ install_requires = [ "boto>=2.36.0", "boto3>=1.6.16,<1.8", "botocore>=1.9.16,<1.11", - "cryptography>=2.0.0", + "cryptography>=2.3.0", "requests>=2.5", "xmltodict", "six>1.9",