From 83c8740b910e178d93fc26b02d3557d4ad36bbd3 Mon Sep 17 00:00:00 2001 From: Ivan Dromigny Date: Wed, 2 Oct 2019 18:06:34 +0200 Subject: [PATCH 01/32] Add informations in subscriptions `attributes` --- moto/sns/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/moto/sns/models.py b/moto/sns/models.py index 92e6c61de..515fe116d 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -339,6 +339,14 @@ class SNSBackend(BaseBackend): return old_subscription topic = self.get_topic(topic_arn) subscription = Subscription(topic, endpoint, protocol) + attributes = { + 'PendingConfirmation' : 'false', + 'Endpoint' : endpoint, + 'TopicArn' : topic_arn, + 'Protocol': protocol, + 'SubscriptionArn': subscription.arn + } + subscription.attributes = attributes self.subscriptions[subscription.arn] = subscription return subscription From 03986df929a2dd59e845bf4be90b1acf3b09e0f2 Mon Sep 17 00:00:00 2001 From: Ivan Dromigny Date: Thu, 3 Oct 2019 09:53:04 +0200 Subject: [PATCH 02/32] Update syntax --- moto/sns/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/sns/models.py b/moto/sns/models.py index 515fe116d..cead172e9 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -340,9 +340,9 @@ class SNSBackend(BaseBackend): topic = self.get_topic(topic_arn) subscription = Subscription(topic, endpoint, protocol) attributes = { - 'PendingConfirmation' : 'false', - 'Endpoint' : endpoint, - 'TopicArn' : topic_arn, + 'PendingConfirmation': 'false', + 'Endpoint': endpoint, + 'TopicArn': topic_arn, 'Protocol': protocol, 'SubscriptionArn': subscription.arn } From 02fc1fbcef69a4de1a98bf808a15beb71ebfb280 Mon Sep 17 00:00:00 2001 From: Ivan Dromigny Date: Mon, 14 Oct 2019 18:03:01 +0200 Subject: [PATCH 03/32] Add a test --- tests/test_sns/test_subscriptions_boto3.py | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_sns/test_subscriptions_boto3.py b/tests/test_sns/test_subscriptions_boto3.py index 012cd6470..d09f625fd 100644 --- a/tests/test_sns/test_subscriptions_boto3.py +++ b/tests/test_sns/test_subscriptions_boto3.py @@ -181,6 +181,30 @@ def test_subscription_paging(): int(DEFAULT_PAGE_SIZE / 3)) topic1_subscriptions.shouldnt.have("NextToken") +@mock_sns +def test_subscribe_attributes(): + client = boto3.client('sns', region_name='us-east-1') + client.create_topic(Name="some-topic") + resp = client.create_topic(Name="some-topic") + arn = resp['TopicArn'] + + resp = client.subscribe( + TopicArn=arn, + Protocol='http', + Endpoint='http://test.com' + ) + + attributes = client.get_subscription_attributes( + SubscriptionArn=resp['SubscriptionArn'] + ) + + attributes.should.contain('Attributes') + attributes['Attributes'].should.contain('PendingConfirmation') + attributes['Attributes'].should.contain('Endpoint') + attributes['Attributes'].should.contain('TopicArn') + attributes['Attributes'].should.contain('Protocol') + attributes['Attributes'].should.contain('SubscriptionArn') + @mock_sns def test_creating_subscription_with_attributes(): From 123209515c7338b3324e29b51d478b48afbca25c Mon Sep 17 00:00:00 2001 From: Ivan Dromigny Date: Tue, 15 Oct 2019 16:12:22 +0200 Subject: [PATCH 04/32] Change test --- tests/test_sns/test_subscriptions_boto3.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_sns/test_subscriptions_boto3.py b/tests/test_sns/test_subscriptions_boto3.py index d09f625fd..282ec4652 100644 --- a/tests/test_sns/test_subscriptions_boto3.py +++ b/tests/test_sns/test_subscriptions_boto3.py @@ -200,10 +200,15 @@ def test_subscribe_attributes(): attributes.should.contain('Attributes') attributes['Attributes'].should.contain('PendingConfirmation') + attributes['Attributes']['PendingConfirmation'].should.equal('false') attributes['Attributes'].should.contain('Endpoint') + attributes['Attributes']['Endpoint'].should.equal('http://test.com') attributes['Attributes'].should.contain('TopicArn') + attributes['Attributes']['TopicArn'].should.equal(arn) attributes['Attributes'].should.contain('Protocol') + attributes['Attributes']['Protocol'].should.equal('http') attributes['Attributes'].should.contain('SubscriptionArn') + attributes['Attributes']['SubscriptionArn'].should.equal(resp['SubscriptionArn']) @mock_sns From 7b1cf9eecd7e6da7939e13839d7f6a594fe82235 Mon Sep 17 00:00:00 2001 From: Stephen Huff Date: Tue, 15 Oct 2019 16:18:37 -0400 Subject: [PATCH 05/32] fix(rds2): handle create_db_instance when AllocatedStorage is not specified In all of the tests of `create_db_instance()`, the `AllocatedStorage` parameter is provided. The [RDS API reference](https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html) says this parameter is optional; however, when none is provided, moto returns an obscure error message: ``` self = , shape = , text = 'None' @_text_content def _handle_integer(self, shape, text): > return int(text) E ValueError: invalid literal for int() with base 10: 'None' /usr/local/Cellar/pyenv/1.2.13_1/versions/3.7.4/envs/rds_encrypt/lib/python3.7/site-packages/botocore/parsers.py:466: ValueError ``` This PR adds default values that correspond to the current default API behaviors. --- moto/rds2/models.py | 77 ++++++++++++++++++++++++++++++++++++ moto/rds2/responses.py | 2 +- tests/test_rds2/test_rds2.py | 14 +++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 4c0daa230..18f605511 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -56,6 +56,8 @@ class Database(BaseModel): else: self.kms_key_id = kwargs.get("kms_key_id") self.storage_type = kwargs.get("storage_type") + if self.storage_type is None: + self.storage_type = Database.default_storage_type(iops=self.iops) self.master_username = kwargs.get('master_username') self.master_user_password = kwargs.get('master_user_password') self.auto_minor_version_upgrade = kwargs.get( @@ -63,6 +65,8 @@ class Database(BaseModel): if self.auto_minor_version_upgrade is None: self.auto_minor_version_upgrade = True self.allocated_storage = kwargs.get('allocated_storage') + if self.allocated_storage is None: + self.allocated_storage = Database.default_allocated_storage(engine=self.engine,storage_type=self.storage_type) self.db_instance_identifier = kwargs.get('db_instance_identifier') self.source_db_identifier = kwargs.get("source_db_identifier") self.db_instance_class = kwargs.get('db_instance_class') @@ -292,6 +296,79 @@ class Database(BaseModel): 'sqlserver-web': 1433, }[engine] + @staticmethod + def default_storage_type(iops): + if iops is None: + return 'gp2' + else: + return 'io1' + + + @staticmethod + def default_allocated_storage(engine,storage_type): + return { + 'aurora': { + 'gp2': 0, + 'io1': 0, + 'standard': 0, + }, + 'mysql': { + 'gp2': 20, + 'io1': 100, + 'standard': 5, + }, + 'mariadb': { + 'gp2': 20, + 'io1': 100, + 'standard': 5, + }, + 'postgres': { + 'gp2': 20, + 'io1': 100, + 'standard': 5, + }, + 'oracle-ee': { + 'gp2': 20, + 'io1': 100, + 'standard': 10, + }, + 'oracle-se2': { + 'gp2': 20, + 'io1': 100, + 'standard': 10, + }, + 'oracle-se1': { + 'gp2': 20, + 'io1': 100, + 'standard': 10, + }, + 'oracle-se': { + 'gp2': 20, + 'io1': 100, + 'standard': 10, + }, + 'sqlserver-ee': { + 'gp2': 200, + 'io1': 200, + 'standard': 200, + }, + 'sqlserver-ex': { + 'gp2': 20, + 'io1': 100, + 'standard': 20, + }, + 'sqlserver-se': { + 'gp2': 200, + 'io1': 200, + 'standard': 200, + }, + 'sqlserver-web': { + 'gp2': 20, + 'io1': 100, + 'standard': 20, + }, + }[engine][storage_type] + @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): properties = cloudformation_json['Properties'] diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index 7b8d0b63a..fdba73248 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -42,7 +42,7 @@ class RDS2Response(BaseResponse): "region": self.region, "security_groups": self._get_multi_param('DBSecurityGroups.DBSecurityGroupName'), "storage_encrypted": self._get_param("StorageEncrypted"), - "storage_type": self._get_param("StorageType", 'standard'), + "storage_type": self._get_param("StorageType", None), "vpc_security_group_ids": self._get_multi_param("VpcSecurityGroupIds.VpcSecurityGroupId"), "tags": list(), } diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index aacaf04f1..911f682a8 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -39,6 +39,20 @@ def test_create_database(): db_instance['VpcSecurityGroups'][0]['VpcSecurityGroupId'].should.equal('sg-123456') +@mock_rds2 +def test_create_database_no_allocated_storage(): + conn = boto3.client('rds', region_name='us-west-2') + database = conn.create_db_instance( + DBInstanceIdentifier='db-master-1', + Engine='postgres', + DBName='staging-postgres', + DBInstanceClass='db.m1.small') + db_instance = database['DBInstance'] + db_instance['Engine'].should.equal('postgres') + db_instance['StorageType'].should.equal('gp2') + db_instance['AllocatedStorage'].should.equal(20) + + @mock_rds2 def test_create_database_non_existing_option_group(): conn = boto3.client('rds', region_name='us-west-2') From 381e7b165fdd18c7aebc7cfd1379d44468e87861 Mon Sep 17 00:00:00 2001 From: Alexander Campbell Date: Tue, 15 Oct 2019 21:57:16 +1100 Subject: [PATCH 06/32] Raise appropriate error when secret exists but has no value --- moto/secretsmanager/exceptions.py | 4 ++-- moto/secretsmanager/models.py | 20 ++++++++++++------- .../test_secretsmanager.py | 17 +++++++++++++++- tests/test_secretsmanager/test_server.py | 20 +++++++++++++++++++ 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/moto/secretsmanager/exceptions.py b/moto/secretsmanager/exceptions.py index fa81b6d8b..746368a6b 100644 --- a/moto/secretsmanager/exceptions.py +++ b/moto/secretsmanager/exceptions.py @@ -7,11 +7,11 @@ class SecretsManagerClientError(JsonRESTError): class ResourceNotFoundException(SecretsManagerClientError): - def __init__(self): + def __init__(self, message): self.code = 404 super(ResourceNotFoundException, self).__init__( "ResourceNotFoundException", - "Secrets Manager can't find the specified secret" + message, ) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 63d847c49..f7cb855eb 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -46,7 +46,7 @@ class SecretsManagerBackend(BaseBackend): def get_secret_value(self, secret_id, version_id, version_stage): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException() + raise ResourceNotFoundException("Secrets Manager can't find the specified secret") if not version_id and version_stage: # set version_id to match version_stage @@ -56,7 +56,7 @@ class SecretsManagerBackend(BaseBackend): version_id = ver_id break if not version_id: - raise ResourceNotFoundException() + raise ResourceNotFoundException("Secrets Manager can't find the specified secret") # TODO check this part if 'deleted_date' in self.secrets[secret_id]: @@ -84,6 +84,12 @@ class SecretsManagerBackend(BaseBackend): if 'secret_binary' in secret_version: response_data["SecretBinary"] = secret_version['secret_binary'] + if 'secret_string' not in secret_version and 'secret_binary' not in secret_version: + raise ResourceNotFoundException( + "Secrets Manager can’t find the specified secret value for staging label: %s" % + (version_stage or "AWSCURRENT") + ) + response = json.dumps(response_data) return response @@ -169,7 +175,7 @@ class SecretsManagerBackend(BaseBackend): def describe_secret(self, secret_id): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException + raise ResourceNotFoundException("Secrets Manager can't find the specified secret") secret = self.secrets[secret_id] @@ -198,7 +204,7 @@ class SecretsManagerBackend(BaseBackend): rotation_days = 'AutomaticallyAfterDays' if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException + raise ResourceNotFoundException("Secrets Manager can't find the specified secret") if 'deleted_date' in self.secrets[secret_id]: raise InvalidRequestException( @@ -340,7 +346,7 @@ class SecretsManagerBackend(BaseBackend): def delete_secret(self, secret_id, recovery_window_in_days, force_delete_without_recovery): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException + raise ResourceNotFoundException("Secrets Manager can't find the specified secret") if 'deleted_date' in self.secrets[secret_id]: raise InvalidRequestException( @@ -370,7 +376,7 @@ class SecretsManagerBackend(BaseBackend): secret = self.secrets.get(secret_id, None) if not secret: - raise ResourceNotFoundException + raise ResourceNotFoundException("Secrets Manager can't find the specified secret") arn = secret_arn(self.region, secret['secret_id']) name = secret['name'] @@ -380,7 +386,7 @@ class SecretsManagerBackend(BaseBackend): def restore_secret(self, secret_id): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException + raise ResourceNotFoundException("Secrets Manager can't find the specified secret") self.secrets[secret_id].pop('deleted_date', None) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 62de93bab..b77fa70d0 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -8,7 +8,7 @@ import string import pytz from datetime import datetime import sure # noqa -from nose.tools import assert_raises +from nose.tools import assert_raises, assert_raises_regexp from six import b DEFAULT_SECRET_NAME = 'test-secret' @@ -65,6 +65,21 @@ def test_get_secret_value_that_is_marked_deleted(): result = conn.get_secret_value(SecretId='test-secret') +@mock_secretsmanager +def test_get_secret_that_has_no_value(): + conn = boto3.client('secretsmanager', region_name='us-west-2') + + create_secret = conn.create_secret(Name="java-util-test-password") + + with assert_raises_regexp( + ClientError, + r"An error occurred \(ResourceNotFoundException\) when calling the GetSecretValue " + r"operation: Secrets Manager can’t find the specified secret value for staging label: " + r"AWSCURRENT" + ): + result = conn.get_secret_value(SecretId='java-util-test-password') + + @mock_secretsmanager def test_create_secret(): conn = boto3.client('secretsmanager', region_name='us-east-1') diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index 23d823239..25fc1676b 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -73,6 +73,26 @@ def test_get_secret_that_does_not_match(): assert json_data['message'] == "Secrets Manager can't find the specified secret" assert json_data['__type'] == 'ResourceNotFoundException' +@mock_secretsmanager +def test_get_secret_that_has_no_value(): + backend = server.create_backend_app('secretsmanager') + test_client = backend.test_client() + + create_secret = test_client.post('/', + data={"Name": DEFAULT_SECRET_NAME}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret"}, + ) + get_secret = test_client.post('/', + data={"SecretId": DEFAULT_SECRET_NAME}, + headers={ + "X-Amz-Target": "secretsmanager.GetSecretValue"}, + ) + + json_data = json.loads(get_secret.data.decode("utf-8")) + assert json_data['message'] == "Secrets Manager can’t find the specified secret value for staging label: AWSCURRENT" + assert json_data['__type'] == 'ResourceNotFoundException' + @mock_secretsmanager def test_create_secret(): From 9d6a1ca81d3296a465cbe038ce4e12eeb57f2c7a Mon Sep 17 00:00:00 2001 From: Alexander Campbell Date: Wed, 16 Oct 2019 10:58:59 +1100 Subject: [PATCH 07/32] Fix slightly incorrect message for some errors --- moto/secretsmanager/models.py | 14 +++++++------- tests/test_secretsmanager/test_secretsmanager.py | 12 ++++++++++-- tests/test_secretsmanager/test_server.py | 12 ++++++------ 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index f7cb855eb..b46e04d6c 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -46,7 +46,7 @@ class SecretsManagerBackend(BaseBackend): def get_secret_value(self, secret_id, version_id, version_stage): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException("Secrets Manager can't find the specified secret") + raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") if not version_id and version_stage: # set version_id to match version_stage @@ -56,7 +56,7 @@ class SecretsManagerBackend(BaseBackend): version_id = ver_id break if not version_id: - raise ResourceNotFoundException("Secrets Manager can't find the specified secret") + raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") # TODO check this part if 'deleted_date' in self.secrets[secret_id]: @@ -175,7 +175,7 @@ class SecretsManagerBackend(BaseBackend): def describe_secret(self, secret_id): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException("Secrets Manager can't find the specified secret") + raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") secret = self.secrets[secret_id] @@ -204,7 +204,7 @@ class SecretsManagerBackend(BaseBackend): rotation_days = 'AutomaticallyAfterDays' if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException("Secrets Manager can't find the specified secret") + raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") if 'deleted_date' in self.secrets[secret_id]: raise InvalidRequestException( @@ -346,7 +346,7 @@ class SecretsManagerBackend(BaseBackend): def delete_secret(self, secret_id, recovery_window_in_days, force_delete_without_recovery): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException("Secrets Manager can't find the specified secret") + raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") if 'deleted_date' in self.secrets[secret_id]: raise InvalidRequestException( @@ -376,7 +376,7 @@ class SecretsManagerBackend(BaseBackend): secret = self.secrets.get(secret_id, None) if not secret: - raise ResourceNotFoundException("Secrets Manager can't find the specified secret") + raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") arn = secret_arn(self.region, secret['secret_id']) name = secret['name'] @@ -386,7 +386,7 @@ class SecretsManagerBackend(BaseBackend): def restore_secret(self, secret_id): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException("Secrets Manager can't find the specified secret") + raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") self.secrets[secret_id].pop('deleted_date', None) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index b77fa70d0..7273b5c21 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -38,7 +38,11 @@ def test_get_secret_value_binary(): def test_get_secret_that_does_not_exist(): conn = boto3.client('secretsmanager', region_name='us-west-2') - with assert_raises(ClientError): + with assert_raises_regexp( + ClientError, + r"An error occurred \(ResourceNotFoundException\) when calling the GetSecretValue " + r"operation: Secrets Manager can’t find the specified secret." + ): result = conn.get_secret_value(SecretId='i-dont-exist') @@ -48,7 +52,11 @@ def test_get_secret_that_does_not_match(): create_secret = conn.create_secret(Name='java-util-test-password', SecretString="foosecret") - with assert_raises(ClientError): + with assert_raises_regexp( + ClientError, + r"An error occurred \(ResourceNotFoundException\) when calling the GetSecretValue " + r"operation: Secrets Manager can’t find the specified secret." + ): result = conn.get_secret_value(SecretId='i-dont-match') diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index 25fc1676b..a1491dc8f 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -49,7 +49,7 @@ def test_get_secret_that_does_not_exist(): "X-Amz-Target": "secretsmanager.GetSecretValue"}, ) json_data = json.loads(get_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['message'] == "Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -70,7 +70,7 @@ def test_get_secret_that_does_not_match(): "X-Amz-Target": "secretsmanager.GetSecretValue"}, ) json_data = json.loads(get_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['message'] == "Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -178,7 +178,7 @@ def test_describe_secret_that_does_not_exist(): ) json_data = json.loads(describe_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['message'] == "Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -202,7 +202,7 @@ def test_describe_secret_that_does_not_match(): ) json_data = json.loads(describe_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['message'] == "Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -303,7 +303,7 @@ def test_rotate_secret_that_does_not_exist(): ) json_data = json.loads(rotate_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['message'] == "Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -327,7 +327,7 @@ def test_rotate_secret_that_does_not_match(): ) json_data = json.loads(rotate_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can't find the specified secret" + assert json_data['message'] == "Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager From d74f9e47c834de7e3527f25619ed6326f5686ba5 Mon Sep 17 00:00:00 2001 From: Alexander Campbell Date: Wed, 16 Oct 2019 12:44:30 +1100 Subject: [PATCH 08/32] Add coding hint for python2 compatibility --- moto/secretsmanager/models.py | 1 + tests/test_secretsmanager/test_secretsmanager.py | 1 + tests/test_secretsmanager/test_server.py | 1 + 3 files changed, 3 insertions(+) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index b46e04d6c..7040d4136 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import unicode_literals import time diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 7273b5c21..606f775b2 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import unicode_literals import boto3 diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index a1491dc8f..bc8b9700d 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import unicode_literals import json From 9a54cea4f1fbd66710ec428f45aaaadd30a2d746 Mon Sep 17 00:00:00 2001 From: Alexander Campbell Date: Wed, 16 Oct 2019 14:44:41 +1100 Subject: [PATCH 09/32] Work around python2 unicode exception str() issues --- moto/secretsmanager/models.py | 18 +++++++++--------- tests/test_secretsmanager/test_server.py | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 7040d4136..6ecc32935 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -47,7 +47,7 @@ class SecretsManagerBackend(BaseBackend): def get_secret_value(self, secret_id, version_id, version_stage): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") + raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") if not version_id and version_stage: # set version_id to match version_stage @@ -57,7 +57,7 @@ class SecretsManagerBackend(BaseBackend): version_id = ver_id break if not version_id: - raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") + raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") # TODO check this part if 'deleted_date' in self.secrets[secret_id]: @@ -87,8 +87,8 @@ class SecretsManagerBackend(BaseBackend): if 'secret_string' not in secret_version and 'secret_binary' not in secret_version: raise ResourceNotFoundException( - "Secrets Manager can’t find the specified secret value for staging label: %s" % - (version_stage or "AWSCURRENT") + u"Secrets Manager can’t find the specified secret value for staging label: %s" % + (version_stage or u"AWSCURRENT") ) response = json.dumps(response_data) @@ -176,7 +176,7 @@ class SecretsManagerBackend(BaseBackend): def describe_secret(self, secret_id): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") + raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") secret = self.secrets[secret_id] @@ -205,7 +205,7 @@ class SecretsManagerBackend(BaseBackend): rotation_days = 'AutomaticallyAfterDays' if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") + raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") if 'deleted_date' in self.secrets[secret_id]: raise InvalidRequestException( @@ -347,7 +347,7 @@ class SecretsManagerBackend(BaseBackend): def delete_secret(self, secret_id, recovery_window_in_days, force_delete_without_recovery): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") + raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") if 'deleted_date' in self.secrets[secret_id]: raise InvalidRequestException( @@ -377,7 +377,7 @@ class SecretsManagerBackend(BaseBackend): secret = self.secrets.get(secret_id, None) if not secret: - raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") + raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") arn = secret_arn(self.region, secret['secret_id']) name = secret['name'] @@ -387,7 +387,7 @@ class SecretsManagerBackend(BaseBackend): def restore_secret(self, secret_id): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException("Secrets Manager can’t find the specified secret.") + raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") self.secrets[secret_id].pop('deleted_date', None) diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index bc8b9700d..552ed6a92 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -50,7 +50,7 @@ def test_get_secret_that_does_not_exist(): "X-Amz-Target": "secretsmanager.GetSecretValue"}, ) json_data = json.loads(get_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -71,7 +71,7 @@ def test_get_secret_that_does_not_match(): "X-Amz-Target": "secretsmanager.GetSecretValue"}, ) json_data = json.loads(get_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -91,7 +91,7 @@ def test_get_secret_that_has_no_value(): ) json_data = json.loads(get_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can’t find the specified secret value for staging label: AWSCURRENT" + assert json_data['message'] == u"Secrets Manager can’t find the specified secret value for staging label: AWSCURRENT" assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -179,7 +179,7 @@ def test_describe_secret_that_does_not_exist(): ) json_data = json.loads(describe_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -203,7 +203,7 @@ def test_describe_secret_that_does_not_match(): ) json_data = json.loads(describe_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -304,7 +304,7 @@ def test_rotate_secret_that_does_not_exist(): ) json_data = json.loads(rotate_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -328,7 +328,7 @@ def test_rotate_secret_that_does_not_match(): ) json_data = json.loads(rotate_secret.data.decode("utf-8")) - assert json_data['message'] == "Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can’t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager From 8eba88d1afd314581705cb02b4fcc1ebcdb7eca5 Mon Sep 17 00:00:00 2001 From: Alexander Campbell Date: Wed, 16 Oct 2019 15:16:38 +1100 Subject: [PATCH 10/32] Fix python2 unicode-in-exceptions issue for tests --- .../test_secretsmanager.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 606f775b2..7319241d3 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -9,7 +9,7 @@ import string import pytz from datetime import datetime import sure # noqa -from nose.tools import assert_raises, assert_raises_regexp +from nose.tools import assert_raises, assert_equal from six import b DEFAULT_SECRET_NAME = 'test-secret' @@ -39,13 +39,14 @@ def test_get_secret_value_binary(): def test_get_secret_that_does_not_exist(): conn = boto3.client('secretsmanager', region_name='us-west-2') - with assert_raises_regexp( - ClientError, - r"An error occurred \(ResourceNotFoundException\) when calling the GetSecretValue " - r"operation: Secrets Manager can’t find the specified secret." - ): + with assert_raises(ClientError) as cm: result = conn.get_secret_value(SecretId='i-dont-exist') + assert_equal( + u"Secrets Manager can’t find the specified secret.", + cm.exception.response['Error']['Message'] + ) + @mock_secretsmanager def test_get_secret_that_does_not_match(): @@ -53,13 +54,13 @@ def test_get_secret_that_does_not_match(): create_secret = conn.create_secret(Name='java-util-test-password', SecretString="foosecret") - with assert_raises_regexp( - ClientError, - r"An error occurred \(ResourceNotFoundException\) when calling the GetSecretValue " - r"operation: Secrets Manager can’t find the specified secret." - ): + with assert_raises(ClientError) as cm: result = conn.get_secret_value(SecretId='i-dont-match') + assert_equal( + u"Secrets Manager can’t find the specified secret.", + cm.exception.response['Error']['Message'] + ) @mock_secretsmanager def test_get_secret_value_that_is_marked_deleted(): @@ -80,14 +81,14 @@ def test_get_secret_that_has_no_value(): create_secret = conn.create_secret(Name="java-util-test-password") - with assert_raises_regexp( - ClientError, - r"An error occurred \(ResourceNotFoundException\) when calling the GetSecretValue " - r"operation: Secrets Manager can’t find the specified secret value for staging label: " - r"AWSCURRENT" - ): + with assert_raises(ClientError) as cm: result = conn.get_secret_value(SecretId='java-util-test-password') + assert_equal( + u"Secrets Manager can’t find the specified secret value for staging label: AWSCURRENT", + cm.exception.response['Error']['Message'] + ) + @mock_secretsmanager def test_create_secret(): From 98f33740e117d20656a4ab2557f30a0d0e481cc0 Mon Sep 17 00:00:00 2001 From: koshigoe Date: Wed, 16 Oct 2019 18:13:59 +0900 Subject: [PATCH 11/32] fix(s3): check whether key is None or not to avoid exception. ``` AttributeError: 'NoneType' object has no attribute 'multipart' ``` --- moto/s3/models.py | 2 +- tests/test_s3/test_s3.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index ef49d7f95..39c4b1edd 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -1107,7 +1107,7 @@ class S3Backend(BaseBackend): key = key_version break - if part_number and key.multipart: + if part_number and key and key.multipart: key = key.multipart.parts[part_number] if isinstance(key, FakeKey): diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 292093893..0d8f3385d 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1534,6 +1534,18 @@ def test_boto3_get_object(): e.exception.response['Error']['Code'].should.equal('NoSuchKey') +@mock_s3 +def test_boto3_get_missing_object_with_part_number(): + s3 = boto3.resource('s3', region_name='us-east-1') + s3.create_bucket(Bucket="blah") + + with assert_raises(ClientError) as e: + s3.Object('blah', 'hello.txt').meta.client.head_object( + Bucket='blah', Key='hello.txt', PartNumber=123) + + e.exception.response['Error']['Code'].should.equal('404') + + @mock_s3 def test_boto3_head_object_with_versioning(): s3 = boto3.resource('s3', region_name='us-east-1') From d1a13ed7827f521dd67af8f6f8a0e623afda628d Mon Sep 17 00:00:00 2001 From: Stephen Huff Date: Wed, 16 Oct 2019 09:10:56 -0400 Subject: [PATCH 12/32] chore(rds2): make flake8 happy --- moto/rds2/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 18f605511..cd56599e6 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -66,7 +66,7 @@ class Database(BaseModel): self.auto_minor_version_upgrade = True self.allocated_storage = kwargs.get('allocated_storage') if self.allocated_storage is None: - self.allocated_storage = Database.default_allocated_storage(engine=self.engine,storage_type=self.storage_type) + self.allocated_storage = Database.default_allocated_storage(engine=self.engine, storage_type=self.storage_type) self.db_instance_identifier = kwargs.get('db_instance_identifier') self.source_db_identifier = kwargs.get("source_db_identifier") self.db_instance_class = kwargs.get('db_instance_class') @@ -303,9 +303,8 @@ class Database(BaseModel): else: return 'io1' - @staticmethod - def default_allocated_storage(engine,storage_type): + def default_allocated_storage(engine, storage_type): return { 'aurora': { 'gp2': 0, From df2de373b236f82a8e9ac2611e34cc387a9250ce Mon Sep 17 00:00:00 2001 From: Harrison Termotto Date: Thu, 17 Oct 2019 00:16:16 -0400 Subject: [PATCH 13/32] Add tagging to versioned s3 objects. --- moto/s3/models.py | 4 +- moto/s3/responses.py | 6 +- tests/test_s3/test_s3.py | 155 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 39c4b1edd..6210aecd6 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -1115,8 +1115,8 @@ class S3Backend(BaseBackend): else: return None - def set_key_tagging(self, bucket_name, key_name, tagging): - key = self.get_key(bucket_name, key_name) + def set_key_tagging(self, bucket_name, key_name, tagging, version_id=None): + key = self.get_key(bucket_name, key_name, version_id) if key is None: raise MissingKey(key_name) key.set_tagging(tagging) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index ae6662579..e0e65850c 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -905,8 +905,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return 200, response_headers, "" if 'tagging' in query: + if 'versionId' in query: + version_id = query['versionId'][0] + else: + version_id = None tagging = self._tagging_from_xml(body) - self.backend.set_key_tagging(bucket_name, key_name, tagging) + self.backend.set_key_tagging(bucket_name, key_name, tagging, version_id) return 200, response_headers, "" if 'x-amz-copy-source' in request.headers: diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 0d8f3385d..8e22bfdad 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -2599,6 +2599,161 @@ def test_boto3_put_object_tagging(): resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) +@mock_s3 +def test_boto3_put_object_tagging_on_earliest_version(): + s3 = boto3.client('s3', region_name='us-east-1') + bucket_name = 'mybucket' + key = 'key-with-tags' + s3.create_bucket(Bucket=bucket_name) + s3_resource = boto3.resource('s3') + bucket_versioning = s3_resource.BucketVersioning(bucket_name) + bucket_versioning.enable() + bucket_versioning.status.should.equal('Enabled') + + with assert_raises(ClientError) as err: + s3.put_object_tagging( + Bucket=bucket_name, + Key=key, + Tagging={'TagSet': [ + {'Key': 'item1', 'Value': 'foo'}, + {'Key': 'item2', 'Value': 'bar'}, + ]} + ) + + e = err.exception + e.response['Error'].should.equal({ + 'Code': 'NoSuchKey', + 'Message': 'The specified key does not exist.', + 'RequestID': '7a62c49f-347e-4fc4-9331-6e8eEXAMPLE', + }) + + s3.put_object( + Bucket=bucket_name, + Key=key, + Body='test' + ) + s3.put_object( + Bucket=bucket_name, + Key=key, + Body='test_updated' + ) + + object_versions = list(s3_resource.Bucket(bucket_name).object_versions.all()) + first_object = object_versions[0] + second_object = object_versions[1] + + resp = s3.put_object_tagging( + Bucket=bucket_name, + Key=key, + Tagging={'TagSet': [ + {'Key': 'item1', 'Value': 'foo'}, + {'Key': 'item2', 'Value': 'bar'}, + ]}, + VersionId=first_object.id + ) + + resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + + # Older version has tags while the most recent does not + resp = s3.get_object_tagging(Bucket=bucket_name, Key=key, VersionId=first_object.id) + resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + resp['TagSet'].should.equal( + [ + {'Key': 'item1', 'Value': 'foo'}, + {'Key': 'item2', 'Value': 'bar'} + ] + ) + + resp = s3.get_object_tagging(Bucket=bucket_name, Key=key, VersionId=second_object.id) + resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + resp['TagSet'].should.equal([]) + + +@mock_s3 +def test_boto3_put_object_tagging_on_both_version(): + s3 = boto3.client('s3', region_name='us-east-1') + bucket_name = 'mybucket' + key = 'key-with-tags' + s3.create_bucket(Bucket=bucket_name) + s3_resource = boto3.resource('s3') + bucket_versioning = s3_resource.BucketVersioning(bucket_name) + bucket_versioning.enable() + bucket_versioning.status.should.equal('Enabled') + + with assert_raises(ClientError) as err: + s3.put_object_tagging( + Bucket=bucket_name, + Key=key, + Tagging={'TagSet': [ + {'Key': 'item1', 'Value': 'foo'}, + {'Key': 'item2', 'Value': 'bar'}, + ]} + ) + + e = err.exception + e.response['Error'].should.equal({ + 'Code': 'NoSuchKey', + 'Message': 'The specified key does not exist.', + 'RequestID': '7a62c49f-347e-4fc4-9331-6e8eEXAMPLE', + }) + + s3.put_object( + Bucket=bucket_name, + Key=key, + Body='test' + ) + s3.put_object( + Bucket=bucket_name, + Key=key, + Body='test_updated' + ) + + object_versions = list(s3_resource.Bucket(bucket_name).object_versions.all()) + first_object = object_versions[0] + second_object = object_versions[1] + + resp = s3.put_object_tagging( + Bucket=bucket_name, + Key=key, + Tagging={'TagSet': [ + {'Key': 'item1', 'Value': 'foo'}, + {'Key': 'item2', 'Value': 'bar'}, + ]}, + VersionId=first_object.id + ) + resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + + resp = s3.put_object_tagging( + Bucket=bucket_name, + Key=key, + Tagging={'TagSet': [ + {'Key': 'item1', 'Value': 'baz'}, + {'Key': 'item2', 'Value': 'bin'}, + ]}, + VersionId=second_object.id + ) + resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + + # Older version has tags while the most recent does not + resp = s3.get_object_tagging(Bucket=bucket_name, Key=key, VersionId=first_object.id) + resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + resp['TagSet'].should.equal( + [ + {'Key': 'item1', 'Value': 'foo'}, + {'Key': 'item2', 'Value': 'bar'} + ] + ) + + resp = s3.get_object_tagging(Bucket=bucket_name, Key=key, VersionId=second_object.id) + resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + resp['TagSet'].should.equal( + [ + {'Key': 'item1', 'Value': 'baz'}, + {'Key': 'item2', 'Value': 'bin'} + ] + ) + + @mock_s3 def test_boto3_put_object_tagging_with_single_tag(): s3 = boto3.client('s3', region_name='us-east-1') From f788fd5c8cd0251fa0ffca57925045999020cbf6 Mon Sep 17 00:00:00 2001 From: Harrison Termotto Date: Thu, 17 Oct 2019 00:17:45 -0400 Subject: [PATCH 14/32] Remove erroneous comment --- tests/test_s3/test_s3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 8e22bfdad..8d535420a 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -2734,7 +2734,6 @@ def test_boto3_put_object_tagging_on_both_version(): ) resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) - # Older version has tags while the most recent does not resp = s3.get_object_tagging(Bucket=bucket_name, Key=key, VersionId=first_object.id) resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) resp['TagSet'].should.equal( From 856a06a7785d3afa7e6b932c9a628bd1f4986299 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 17 Oct 2019 09:28:19 +0100 Subject: [PATCH 15/32] IAM - Delete Policy implementation --- IMPLEMENTATION_COVERAGE.md | 2 +- moto/iam/models.py | 3 +++ moto/iam/responses.py | 6 +++++ .../test_lambda_eventsourcemappings.py | 0 tests/test_iam/test_iam.py | 25 ++++++++++++++++++- 5 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 tests/test_awslambda/test_lambda_eventsourcemappings.py diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 57f169b8a..e304904c7 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3194,7 +3194,7 @@ - [ ] delete_instance_profile - [X] delete_login_profile - [ ] delete_open_id_connect_provider -- [ ] delete_policy +- [X] delete_policy - [X] delete_policy_version - [X] delete_role - [ ] delete_role_permissions_boundary diff --git a/moto/iam/models.py b/moto/iam/models.py index d76df8a28..506f2a942 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -1099,6 +1099,9 @@ class IAMBackend(BaseBackend): user = self.get_user(user_name) user.delete_policy(policy_name) + def delete_policy(self, policy_arn): + del self.managed_policies[policy_arn] + def create_access_key(self, user_name=None): user = self.get_user(user_name) key = user.create_access_key() diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 806dd37f4..8e63c1075 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -604,6 +604,12 @@ class IamResponse(BaseResponse): template = self.response_template(GENERIC_EMPTY_TEMPLATE) return template.render(name='DeleteUser') + def delete_policy(self): + policy_arn = self._get_param('PolicyArn') + iam_backend.delete_policy(policy_arn) + template = self.response_template(GENERIC_EMPTY_TEMPLATE) + return template.render(name='DeletePolicy') + def delete_login_profile(self): user_name = self._get_param('UserName') iam_backend.delete_login_profile(user_name) diff --git a/tests/test_awslambda/test_lambda_eventsourcemappings.py b/tests/test_awslambda/test_lambda_eventsourcemappings.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index fe2117a3a..36c9b1910 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -338,6 +338,16 @@ def test_create_policy(): response['Policy']['Arn'].should.equal("arn:aws:iam::123456789012:policy/TestCreatePolicy") +@mock_iam +def test_delete_policy(): + conn = boto3.client('iam', region_name='us-east-1') + response = conn.create_policy(PolicyName="TestCreatePolicy", PolicyDocument=MOCK_POLICY) + [pol['PolicyName'] for pol in conn.list_policies(Scope='Local')['Policies']].should.equal(['TestCreatePolicy']) + # + conn.delete_policy(PolicyArn=response['Policy']['Arn']) + assert conn.list_policies(Scope='Local')['Policies'].should.be.empty + + @mock_iam def test_create_policy_versions(): conn = boto3.client('iam', region_name='us-east-1') @@ -713,7 +723,7 @@ def test_mfa_devices(): @mock_iam_deprecated() -def test_delete_user(): +def test_delete_user_deprecated(): conn = boto.connect_iam() with assert_raises(BotoServerError): conn.delete_user('my-user') @@ -721,6 +731,19 @@ def test_delete_user(): conn.delete_user('my-user') +@mock_iam() +def test_delete_user(): + conn = boto3.client('iam', region_name='us-east-1') + with assert_raises(ClientError): + conn.delete_user(UserName='my-user') + # + conn.create_user(UserName='my-user') + [user['UserName'] for user in conn.list_users()['Users']].should.equal(['my-user']) + # + conn.delete_user(UserName='my-user') + assert conn.list_users()['Users'].should.be.empty + + @mock_iam_deprecated() def test_generate_credential_report(): conn = boto.connect_iam() From 65c5502a62e81308d3907002f6f6b827a8f1c342 Mon Sep 17 00:00:00 2001 From: gruebel Date: Thu, 17 Oct 2019 22:09:14 +0200 Subject: [PATCH 16/32] Add error handling for sqs.send_message_batch --- moto/sqs/responses.py | 70 ++++++++++++++++----- tests/test_sqs/test_sqs.py | 121 +++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 14 deletions(-) diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index 747fa2363..0ddf5a314 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -242,25 +242,59 @@ class SQSResponse(BaseResponse): queue_name = self._get_queue_name() - messages = [] - for index in range(1, 11): - # Loop through looking for messages - message_key = 'SendMessageBatchRequestEntry.{0}.MessageBody'.format( - index) - message_body = self.querystring.get(message_key) - if not message_body: - # Found all messages - break + try: + self.sqs_backend.get_queue(queue_name) + except QueueDoesNotExist as e: + return self._error('AWS.SimpleQueueService.NonExistentQueue', + e.description) - message_user_id_key = 'SendMessageBatchRequestEntry.{0}.Id'.format( - index) - message_user_id = self.querystring.get(message_user_id_key)[0] + if self.querystring.get('Entries'): + return self._error('AWS.SimpleQueueService.EmptyBatchRequest', + 'There should be at least one SendMessageBatchRequestEntry in the request.') + + entries = {} + for key, value in self.querystring.items(): + match = re.match(r'^SendMessageBatchRequestEntry\.(\d+)\.Id', key) + if match: + entries[match.group(1)] = { + 'Id': value[0], + 'MessageBody': self.querystring.get( + 'SendMessageBatchRequestEntry.{}.MessageBody'.format(match.group(1)))[0] + } + + if any(not re.match(r'^[\w-]{1,80}$', entry['Id']) for entry in entries.values()): + return self._error('AWS.SimpleQueueService.InvalidBatchEntryId', + 'A batch entry id can only contain alphanumeric characters, ' + 'hyphens and underscores. It can be at most 80 letters long.') + + body_length = next( + (len(entry['MessageBody']) for entry in entries.values() if len(entry['MessageBody']) > 262144), + False + ) + if body_length: + return self._error('AWS.SimpleQueueService.BatchRequestTooLong', + 'Batch requests cannot be longer than 262144 bytes. ' + 'You have sent {} bytes.'.format(body_length)) + + duplicate_id = self._get_first_duplicate_id([entry['Id'] for entry in entries.values()]) + if duplicate_id: + return self._error('AWS.SimpleQueueService.BatchEntryIdsNotDistinct', + 'Id {} repeated.'.format(duplicate_id)) + + if len(entries) > 10: + return self._error('AWS.SimpleQueueService.TooManyEntriesInBatchRequest', + 'Maximum number of entries per request are 10. ' + 'You have sent 11.') + + messages = [] + for index, entry in entries.items(): + # Loop through looking for messages delay_key = 'SendMessageBatchRequestEntry.{0}.DelaySeconds'.format( index) delay_seconds = self.querystring.get(delay_key, [None])[0] message = self.sqs_backend.send_message( - queue_name, message_body[0], delay_seconds=delay_seconds) - message.user_id = message_user_id + queue_name, entry['MessageBody'], delay_seconds=delay_seconds) + message.user_id = entry['Id'] message_attributes = parse_message_attributes( self.querystring, base='SendMessageBatchRequestEntry.{0}.'.format(index)) @@ -273,6 +307,14 @@ class SQSResponse(BaseResponse): template = self.response_template(SEND_MESSAGE_BATCH_RESPONSE) return template.render(messages=messages) + def _get_first_duplicate_id(self, ids): + unique_ids = set() + for id in ids: + if id in unique_ids: + return id + unique_ids.add(id) + return None + def delete_message(self): queue_name = self._get_queue_name() receipt_handle = self.querystring.get("ReceiptHandle")[0] diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index bc9fa8e4d..4d06411e6 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -882,6 +882,127 @@ def test_delete_message_after_visibility_timeout(): assert new_queue.count() == 0 +@mock_sqs +def test_send_message_batch_errors(): + client = boto3.client('sqs', region_name = 'us-east-1') + + response = client.create_queue(QueueName='test-queue-with-tags') + queue_url = response['QueueUrl'] + + client.send_message_batch.when.called_with( + QueueUrl=queue_url + '-not-existing', + Entries=[ + { + 'Id': 'id_1', + 'MessageBody': 'body_1' + } + ] + ).should.throw( + ClientError, + 'The specified queue does not exist for this wsdl version.' + ) + + client.send_message_batch.when.called_with( + QueueUrl=queue_url, + Entries=[] + ).should.throw( + ClientError, + 'There should be at least one SendMessageBatchRequestEntry in the request.' + ) + + client.send_message_batch.when.called_with( + QueueUrl=queue_url, + Entries=[ + { + 'Id': '', + 'MessageBody': 'body_1' + } + ] + ).should.throw( + ClientError, + 'A batch entry id can only contain alphanumeric characters, ' + 'hyphens and underscores. It can be at most 80 letters long.' + ) + + client.send_message_batch.when.called_with( + QueueUrl=queue_url, + Entries=[ + { + 'Id': '.!@#$%^&*()+=', + 'MessageBody': 'body_1' + } + ] + ).should.throw( + ClientError, + 'A batch entry id can only contain alphanumeric characters, ' + 'hyphens and underscores. It can be at most 80 letters long.' + ) + + client.send_message_batch.when.called_with( + QueueUrl=queue_url, + Entries=[ + { + 'Id': 'i' * 81, + 'MessageBody': 'body_1' + } + ] + ).should.throw( + ClientError, + 'A batch entry id can only contain alphanumeric characters, ' + 'hyphens and underscores. It can be at most 80 letters long.' + ) + + client.send_message_batch.when.called_with( + QueueUrl=queue_url, + Entries=[ + { + 'Id': 'id_1', + 'MessageBody': 'b' * 262145 + } + ] + ).should.throw( + ClientError, + 'Batch requests cannot be longer than 262144 bytes. ' + 'You have sent 262145 bytes.' + ) + + # only the first duplicated Id is reported + client.send_message_batch.when.called_with( + QueueUrl=queue_url, + Entries=[ + { + 'Id': 'id_1', + 'MessageBody': 'body_1' + }, + { + 'Id': 'id_2', + 'MessageBody': 'body_2' + }, + { + 'Id': 'id_2', + 'MessageBody': 'body_2' + }, + { + 'Id': 'id_1', + 'MessageBody': 'body_1' + } + ] + ).should.throw( + ClientError, + 'Id id_2 repeated.' + ) + + entries = [{'Id': 'id_{}'.format(i), 'MessageBody': 'body_{}'.format(i)} for i in range(11)] + client.send_message_batch.when.called_with( + QueueUrl=queue_url, + Entries=entries + ).should.throw( + ClientError, + 'Maximum number of entries per request are 10. ' + 'You have sent 11.' + ) + + @mock_sqs def test_batch_change_message_visibility(): if os.environ.get('TEST_SERVER_MODE', 'false').lower() == 'true': From dbfb319defaba8de98ae0d305900fee2a14c3e84 Mon Sep 17 00:00:00 2001 From: gruebel Date: Thu, 17 Oct 2019 22:36:37 +0200 Subject: [PATCH 17/32] Add error handling for sqs.tag_queue --- moto/sqs/responses.py | 14 +++++++++++ tests/test_sqs/test_sqs.py | 49 +++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index 0ddf5a314..f84d293d3 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -439,6 +439,20 @@ class SQSResponse(BaseResponse): queue_name = self._get_queue_name() tags = self._get_map_prefix('Tag', key_end='.Key', value_end='.Value') + try: + self.sqs_backend.get_queue(queue_name) + except QueueDoesNotExist as e: + return self._error('AWS.SimpleQueueService.NonExistentQueue', + e.description) + + if len(tags) == 0: + return self._error('MissingParameter', + 'The request must contain the parameter Tags.') + + if len(tags) > 50: + return self._error('InvalidParameterValue', + 'Too many tags added for queue {}.'.format(queue_name)) + self.sqs_backend.tag_queue(queue_name, tags) template = self.response_template(TAG_QUEUE_RESPONSE) diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 4d06411e6..f5d09ba4b 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -1108,6 +1108,53 @@ def test_tags(): }) +@mock_sqs +def test_tag_queue_errors(): + client = boto3.client('sqs', region_name='us-east-1') + + response = client.create_queue( + QueueName='test-queue-with-tags', + tags={ + 'tag_key_1': 'tag_value_X' + } + ) + queue_url = response['QueueUrl'] + + client.tag_queue.when.called_with( + QueueUrl=queue_url + '-not-existing', + Tags={ + 'tag_key_1': 'tag_value_1' + } + ).should.throw( + ClientError, + 'The specified queue does not exist for this wsdl version.' + ) + + client.tag_queue.when.called_with( + QueueUrl=queue_url, + Tags={} + ).should.throw( + ClientError, + 'The request must contain the parameter Tags.' + ) + + too_many_tags = {'tag_key_{}'.format(i): 'tag_value_{}'.format(i) for i in range(51)} + client.tag_queue.when.called_with( + QueueUrl=queue_url, + Tags=too_many_tags + ).should.throw( + ClientError, + 'Too many tags added for queue test-queue-with-tags.' + ) + + # when the request fails, the tags should not be updated + client.list_queue_tags(QueueUrl=queue_url)['Tags'].should.equal( + { + 'tag_key_1': 'tag_value_X' + } + ) + + @mock_sqs def test_untag_queue_errors(): client = boto3.client('sqs', region_name='us-east-1') @@ -1127,7 +1174,7 @@ def test_untag_queue_errors(): ] ).should.throw( ClientError, - "The specified queue does not exist for this wsdl version." + 'The specified queue does not exist for this wsdl version.' ) client.untag_queue.when.called_with( From 19a34ea57ab4455d1c37982e48018b16f1ceab9f Mon Sep 17 00:00:00 2001 From: gruebel Date: Thu, 17 Oct 2019 22:38:16 +0200 Subject: [PATCH 18/32] Add error handling for sqs.list_queue_tags --- moto/sqs/responses.py | 6 +++++- tests/test_sqs/test_sqs.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index f84d293d3..75f121b76 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -474,7 +474,11 @@ class SQSResponse(BaseResponse): def list_queue_tags(self): queue_name = self._get_queue_name() - queue = self.sqs_backend.get_queue(queue_name) + try: + queue = self.sqs_backend.get_queue(queue_name) + except QueueDoesNotExist as e: + return self._error('AWS.SimpleQueueService.NonExistentQueue', + e.description) template = self.response_template(LIST_QUEUE_TAGS_RESPONSE) return template.render(tags=queue.tags) diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index f5d09ba4b..1ad2e1a80 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -1108,6 +1108,26 @@ def test_tags(): }) +@mock_sqs +def test_list_queue_tags_errors(): + client = boto3.client('sqs', region_name='us-east-1') + + response = client.create_queue( + QueueName='test-queue-with-tags', + tags={ + 'tag_key_1': 'tag_value_X' + } + ) + queue_url = response['QueueUrl'] + + client.list_queue_tags.when.called_with( + QueueUrl=queue_url + '-not-existing', + ).should.throw( + ClientError, + 'The specified queue does not exist for this wsdl version.' + ) + + @mock_sqs def test_tag_queue_errors(): client = boto3.client('sqs', region_name='us-east-1') From 05dc97b468cdfbb05686966444e2ed9c4dff4e3d Mon Sep 17 00:00:00 2001 From: gruebel Date: Thu, 17 Oct 2019 22:41:46 +0200 Subject: [PATCH 19/32] Update implementation coverage --- IMPLEMENTATION_COVERAGE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 57f169b8a..060f9f7a7 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -5902,7 +5902,7 @@ - [x] untag_resource ## sqs -65% implemented +75% implemented - [X] add_permission - [X] change_message_visibility - [ ] change_message_visibility_batch @@ -5913,13 +5913,13 @@ - [ ] get_queue_attributes - [ ] get_queue_url - [X] list_dead_letter_source_queues -- [ ] list_queue_tags +- [x] list_queue_tags - [X] list_queues - [X] purge_queue - [ ] receive_message - [X] remove_permission - [X] send_message -- [ ] send_message_batch +- [x] send_message_batch - [X] set_queue_attributes - [X] tag_queue - [X] untag_queue From 4eb921480eaafffa33ac010d268d6a5e4f12c301 Mon Sep 17 00:00:00 2001 From: Alexander Campbell Date: Fri, 18 Oct 2019 10:09:16 +1100 Subject: [PATCH 20/32] Use specific exception to prevent repetition --- moto/secretsmanager/exceptions.py | 9 +++++++++ moto/secretsmanager/models.py | 15 ++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/moto/secretsmanager/exceptions.py b/moto/secretsmanager/exceptions.py index 746368a6b..d4563dc42 100644 --- a/moto/secretsmanager/exceptions.py +++ b/moto/secretsmanager/exceptions.py @@ -15,6 +15,15 @@ class ResourceNotFoundException(SecretsManagerClientError): ) +class SecretNotFoundException(SecretsManagerClientError): + def __init__(self): + self.code = 404 + super(SecretNotFoundException, self).__init__( + "ResourceNotFoundException", + message=u"Secrets Manager can\u2019t find the specified secret." + ) + + class ClientError(SecretsManagerClientError): def __init__(self, message): super(ClientError, self).__init__( diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 6ecc32935..2a3e7ede4 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -11,6 +11,7 @@ import boto3 from moto.core import BaseBackend, BaseModel from .exceptions import ( ResourceNotFoundException, + SecretNotFoundException, InvalidParameterException, ResourceExistsException, InvalidRequestException, @@ -47,7 +48,7 @@ class SecretsManagerBackend(BaseBackend): def get_secret_value(self, secret_id, version_id, version_stage): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") + raise SecretNotFoundException() if not version_id and version_stage: # set version_id to match version_stage @@ -57,7 +58,7 @@ class SecretsManagerBackend(BaseBackend): version_id = ver_id break if not version_id: - raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") + raise SecretNotFoundException() # TODO check this part if 'deleted_date' in self.secrets[secret_id]: @@ -176,7 +177,7 @@ class SecretsManagerBackend(BaseBackend): def describe_secret(self, secret_id): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") + raise SecretNotFoundException() secret = self.secrets[secret_id] @@ -205,7 +206,7 @@ class SecretsManagerBackend(BaseBackend): rotation_days = 'AutomaticallyAfterDays' if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") + raise SecretNotFoundException() if 'deleted_date' in self.secrets[secret_id]: raise InvalidRequestException( @@ -347,7 +348,7 @@ class SecretsManagerBackend(BaseBackend): def delete_secret(self, secret_id, recovery_window_in_days, force_delete_without_recovery): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") + raise SecretNotFoundException() if 'deleted_date' in self.secrets[secret_id]: raise InvalidRequestException( @@ -377,7 +378,7 @@ class SecretsManagerBackend(BaseBackend): secret = self.secrets.get(secret_id, None) if not secret: - raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") + raise SecretNotFoundException() arn = secret_arn(self.region, secret['secret_id']) name = secret['name'] @@ -387,7 +388,7 @@ class SecretsManagerBackend(BaseBackend): def restore_secret(self, secret_id): if not self._is_valid_identifier(secret_id): - raise ResourceNotFoundException(u"Secrets Manager can’t find the specified secret.") + raise SecretNotFoundException() self.secrets[secret_id].pop('deleted_date', None) From 6120a60263b215862941dfc0e2b5fd132bd00261 Mon Sep 17 00:00:00 2001 From: Alexander Campbell Date: Fri, 18 Oct 2019 10:13:22 +1100 Subject: [PATCH 21/32] Use escape sequence to express non-ASCII character --- moto/secretsmanager/models.py | 2 +- tests/test_secretsmanager/test_secretsmanager.py | 7 ++++--- tests/test_secretsmanager/test_server.py | 14 +++++++------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 2a3e7ede4..1e9d6a518 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -88,7 +88,7 @@ class SecretsManagerBackend(BaseBackend): if 'secret_string' not in secret_version and 'secret_binary' not in secret_version: raise ResourceNotFoundException( - u"Secrets Manager can’t find the specified secret value for staging label: %s" % + u"Secrets Manager can\u2019t find the specified secret value for staging label: %s" % (version_stage or u"AWSCURRENT") ) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 7319241d3..e2fc266ea 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -43,7 +43,7 @@ def test_get_secret_that_does_not_exist(): result = conn.get_secret_value(SecretId='i-dont-exist') assert_equal( - u"Secrets Manager can’t find the specified secret.", + u"Secrets Manager can\u2019t find the specified secret.", cm.exception.response['Error']['Message'] ) @@ -58,10 +58,11 @@ def test_get_secret_that_does_not_match(): result = conn.get_secret_value(SecretId='i-dont-match') assert_equal( - u"Secrets Manager can’t find the specified secret.", + u"Secrets Manager can\u2019t find the specified secret.", cm.exception.response['Error']['Message'] ) + @mock_secretsmanager def test_get_secret_value_that_is_marked_deleted(): conn = boto3.client('secretsmanager', region_name='us-west-2') @@ -85,7 +86,7 @@ def test_get_secret_that_has_no_value(): result = conn.get_secret_value(SecretId='java-util-test-password') assert_equal( - u"Secrets Manager can’t find the specified secret value for staging label: AWSCURRENT", + u"Secrets Manager can\u2019t find the specified secret value for staging label: AWSCURRENT", cm.exception.response['Error']['Message'] ) diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index 552ed6a92..6955d8232 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -50,7 +50,7 @@ def test_get_secret_that_does_not_exist(): "X-Amz-Target": "secretsmanager.GetSecretValue"}, ) json_data = json.loads(get_secret.data.decode("utf-8")) - assert json_data['message'] == u"Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can\u2019t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -71,7 +71,7 @@ def test_get_secret_that_does_not_match(): "X-Amz-Target": "secretsmanager.GetSecretValue"}, ) json_data = json.loads(get_secret.data.decode("utf-8")) - assert json_data['message'] == u"Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can\u2019t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -91,7 +91,7 @@ def test_get_secret_that_has_no_value(): ) json_data = json.loads(get_secret.data.decode("utf-8")) - assert json_data['message'] == u"Secrets Manager can’t find the specified secret value for staging label: AWSCURRENT" + assert json_data['message'] == u"Secrets Manager can\u2019t find the specified secret value for staging label: AWSCURRENT" assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -179,7 +179,7 @@ def test_describe_secret_that_does_not_exist(): ) json_data = json.loads(describe_secret.data.decode("utf-8")) - assert json_data['message'] == u"Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can\u2019t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -203,7 +203,7 @@ def test_describe_secret_that_does_not_match(): ) json_data = json.loads(describe_secret.data.decode("utf-8")) - assert json_data['message'] == u"Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can\u2019t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -304,7 +304,7 @@ def test_rotate_secret_that_does_not_exist(): ) json_data = json.loads(rotate_secret.data.decode("utf-8")) - assert json_data['message'] == u"Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can\u2019t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager @@ -328,7 +328,7 @@ def test_rotate_secret_that_does_not_match(): ) json_data = json.loads(rotate_secret.data.decode("utf-8")) - assert json_data['message'] == u"Secrets Manager can’t find the specified secret." + assert json_data['message'] == u"Secrets Manager can\u2019t find the specified secret." assert json_data['__type'] == 'ResourceNotFoundException' @mock_secretsmanager From 30853a0b5cde01495adc371c9612574c5e640d2d Mon Sep 17 00:00:00 2001 From: Alexander Campbell Date: Fri, 18 Oct 2019 12:06:12 +1100 Subject: [PATCH 22/32] Use specialised exception for "secret has no value" scenario --- moto/secretsmanager/exceptions.py | 12 ++++++++++++ moto/secretsmanager/models.py | 7 ++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/moto/secretsmanager/exceptions.py b/moto/secretsmanager/exceptions.py index d4563dc42..7ef1a9239 100644 --- a/moto/secretsmanager/exceptions.py +++ b/moto/secretsmanager/exceptions.py @@ -15,6 +15,7 @@ class ResourceNotFoundException(SecretsManagerClientError): ) +# Using specialised exception due to the use of a non-ASCII character class SecretNotFoundException(SecretsManagerClientError): def __init__(self): self.code = 404 @@ -24,6 +25,17 @@ class SecretNotFoundException(SecretsManagerClientError): ) +# Using specialised exception due to the use of a non-ASCII character +class SecretHasNoValueException(SecretsManagerClientError): + def __init__(self, version_stage): + self.code = 404 + super(SecretHasNoValueException, self).__init__( + "ResourceNotFoundException", + message=u"Secrets Manager can\u2019t find the specified secret " + u"value for staging label: {}".format(version_stage) + ) + + class ClientError(SecretsManagerClientError): def __init__(self, message): super(ClientError, self).__init__( diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 1e9d6a518..e1a380c39 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -10,8 +10,8 @@ import boto3 from moto.core import BaseBackend, BaseModel from .exceptions import ( - ResourceNotFoundException, SecretNotFoundException, + SecretHasNoValueException, InvalidParameterException, ResourceExistsException, InvalidRequestException, @@ -87,10 +87,7 @@ class SecretsManagerBackend(BaseBackend): response_data["SecretBinary"] = secret_version['secret_binary'] if 'secret_string' not in secret_version and 'secret_binary' not in secret_version: - raise ResourceNotFoundException( - u"Secrets Manager can\u2019t find the specified secret value for staging label: %s" % - (version_stage or u"AWSCURRENT") - ) + raise SecretHasNoValueException(version_stage or u"AWSCURRENT") response = json.dumps(response_data) From c3c75c12d956071650ba7f6ce45cf383e41572d7 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 18 Oct 2019 09:03:31 +0200 Subject: [PATCH 23/32] Use ==/!= to compare str, bytes, and int literals Python 3.8 will probably [raise SyntaxWarnings](https://docs.python.org/3/whatsnew/3.8.html#changes-in-python-behavior) on the flake8 F632 issue raised below. [flake8](http://flake8.pycqa.org) testing of https://github.com/spulec/moto on Python 3.7.1 $ __flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics__ ``` ./tests/test_awslambda/test_lambda.py:1137:24: F632 use ==/!= to compare str, bytes, and int literals assert len(messages) is 3 ^ 1 F632 use ==/!= to compare str, bytes, and int literals 1 ``` --- tests/test_awslambda/test_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 2f47b6284..d57722074 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -1134,7 +1134,7 @@ def test_invoke_function_from_sqs_exception(): if 'I failed!' in event['message']: messages = queue.receive_messages(MaxNumberOfMessages=10) # Verify messages are still visible and unprocessed - assert len(messages) is 3 + assert len(messages) == 3 return time.sleep(1) From ed1c799bdcd2c6bd11d8a679014cbd21d0d36a12 Mon Sep 17 00:00:00 2001 From: gruebel Date: Fri, 18 Oct 2019 09:04:29 +0200 Subject: [PATCH 24/32] CR fix --- moto/sqs/exceptions.py | 9 ++++--- moto/sqs/models.py | 11 ++++++++- moto/sqs/responses.py | 53 +++++++----------------------------------- 3 files changed, 24 insertions(+), 49 deletions(-) diff --git a/moto/sqs/exceptions.py b/moto/sqs/exceptions.py index 5f1cc46b2..02f28b2d2 100644 --- a/moto/sqs/exceptions.py +++ b/moto/sqs/exceptions.py @@ -19,9 +19,12 @@ class MessageAttributesInvalid(Exception): self.description = description -class QueueDoesNotExist(Exception): - status_code = 404 - description = "The specified queue does not exist for this wsdl version." +class QueueDoesNotExist(RESTError): + code = 404 + + def __init__(self): + super(QueueDoesNotExist, self).__init__( + "QueueDoesNotExist", "The specified queue does not exist for this wsdl version.") class QueueAlreadyExists(RESTError): diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 922da704f..eb237e437 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -654,12 +654,21 @@ class SQSBackend(BaseBackend): def tag_queue(self, queue_name, tags): queue = self.get_queue(queue_name) + + if not len(tags): + raise RESTError('MissingParameter', + 'The request must contain the parameter Tags.') + + if len(tags) > 50: + raise RESTError('InvalidParameterValue', + 'Too many tags added for queue {}.'.format(queue_name)) + queue.tags.update(tags) def untag_queue(self, queue_name, tag_keys): queue = self.get_queue(queue_name) - if len(tag_keys) == 0: + if not len(tag_keys): raise RESTError('InvalidParameterValue', 'Tag keys must be between 1 and 128 characters in length.') for key in tag_keys: diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index 75f121b76..b6f717f3b 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -10,7 +10,6 @@ from .models import sqs_backends from .exceptions import ( MessageAttributesInvalid, MessageNotInflight, - QueueDoesNotExist, ReceiptHandleIsInvalid, ) @@ -90,11 +89,7 @@ class SQSResponse(BaseResponse): request_url = urlparse(self.uri) queue_name = self._get_param("QueueName") - try: - queue = self.sqs_backend.get_queue(queue_name) - except QueueDoesNotExist as e: - return self._error('AWS.SimpleQueueService.NonExistentQueue', - e.description) + queue = self.sqs_backend.get_queue(queue_name) if queue: template = self.response_template(GET_QUEUE_URL_RESPONSE) @@ -175,11 +170,8 @@ class SQSResponse(BaseResponse): def get_queue_attributes(self): queue_name = self._get_queue_name() - try: - queue = self.sqs_backend.get_queue(queue_name) - except QueueDoesNotExist as e: - return self._error('AWS.SimpleQueueService.NonExistentQueue', - e.description) + + queue = self.sqs_backend.get_queue(queue_name) template = self.response_template(GET_QUEUE_ATTRIBUTES_RESPONSE) return template.render(queue=queue) @@ -242,11 +234,7 @@ class SQSResponse(BaseResponse): queue_name = self._get_queue_name() - try: - self.sqs_backend.get_queue(queue_name) - except QueueDoesNotExist as e: - return self._error('AWS.SimpleQueueService.NonExistentQueue', - e.description) + self.sqs_backend.get_queue(queue_name) if self.querystring.get('Entries'): return self._error('AWS.SimpleQueueService.EmptyBatchRequest', @@ -268,7 +256,7 @@ class SQSResponse(BaseResponse): 'hyphens and underscores. It can be at most 80 letters long.') body_length = next( - (len(entry['MessageBody']) for entry in entries.values() if len(entry['MessageBody']) > 262144), + (len(entry['MessageBody']) for entry in entries.values() if len(entry['MessageBody']) > MAXIMUM_MESSAGE_LENGTH), False ) if body_length: @@ -363,10 +351,7 @@ class SQSResponse(BaseResponse): def receive_message(self): queue_name = self._get_queue_name() - try: - queue = self.sqs_backend.get_queue(queue_name) - except QueueDoesNotExist as e: - return self._error('QueueDoesNotExist', e.description) + queue = self.sqs_backend.get_queue(queue_name) try: message_count = int(self.querystring.get("MaxNumberOfMessages")[0]) @@ -439,20 +424,6 @@ class SQSResponse(BaseResponse): queue_name = self._get_queue_name() tags = self._get_map_prefix('Tag', key_end='.Key', value_end='.Value') - try: - self.sqs_backend.get_queue(queue_name) - except QueueDoesNotExist as e: - return self._error('AWS.SimpleQueueService.NonExistentQueue', - e.description) - - if len(tags) == 0: - return self._error('MissingParameter', - 'The request must contain the parameter Tags.') - - if len(tags) > 50: - return self._error('InvalidParameterValue', - 'Too many tags added for queue {}.'.format(queue_name)) - self.sqs_backend.tag_queue(queue_name, tags) template = self.response_template(TAG_QUEUE_RESPONSE) @@ -462,11 +433,7 @@ class SQSResponse(BaseResponse): queue_name = self._get_queue_name() tag_keys = self._get_multi_param('TagKey') - try: - self.sqs_backend.untag_queue(queue_name, tag_keys) - except QueueDoesNotExist as e: - return self._error('AWS.SimpleQueueService.NonExistentQueue', - e.description) + self.sqs_backend.untag_queue(queue_name, tag_keys) template = self.response_template(UNTAG_QUEUE_RESPONSE) return template.render() @@ -474,11 +441,7 @@ class SQSResponse(BaseResponse): def list_queue_tags(self): queue_name = self._get_queue_name() - try: - queue = self.sqs_backend.get_queue(queue_name) - except QueueDoesNotExist as e: - return self._error('AWS.SimpleQueueService.NonExistentQueue', - e.description) + queue = self.sqs_backend.get_queue(queue_name) template = self.response_template(LIST_QUEUE_TAGS_RESPONSE) return template.render(tags=queue.tags) From deffefbfb880644260f4ca3bb922b272e61f4024 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 18 Oct 2019 09:18:12 +0100 Subject: [PATCH 25/32] PR changes --- tests/test_iam/test_iam.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 36c9b1910..23846712d 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -343,7 +343,6 @@ def test_delete_policy(): conn = boto3.client('iam', region_name='us-east-1') response = conn.create_policy(PolicyName="TestCreatePolicy", PolicyDocument=MOCK_POLICY) [pol['PolicyName'] for pol in conn.list_policies(Scope='Local')['Policies']].should.equal(['TestCreatePolicy']) - # conn.delete_policy(PolicyArn=response['Policy']['Arn']) assert conn.list_policies(Scope='Local')['Policies'].should.be.empty @@ -736,10 +735,8 @@ def test_delete_user(): conn = boto3.client('iam', region_name='us-east-1') with assert_raises(ClientError): conn.delete_user(UserName='my-user') - # conn.create_user(UserName='my-user') [user['UserName'] for user in conn.list_users()['Users']].should.equal(['my-user']) - # conn.delete_user(UserName='my-user') assert conn.list_users()['Users'].should.be.empty From db206e994b89de8b1a99bff4b905a320369b2bbf Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Fri, 18 Oct 2019 09:58:09 +0100 Subject: [PATCH 26/32] #250 - DynamoDB - Add check for valid query keyconditionexpression --- moto/dynamodb2/responses.py | 3 +++ tests/test_dynamodb2/test_dynamodb.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 333213c2a..b5e5e11a8 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -457,6 +457,9 @@ class DynamoHandler(BaseResponse): range_comparison = None range_values = [] + if '=' not in hash_key_expression: + return self.error('com.amazonaws.dynamodb.v20111205#ValidationException', + 'Query key condition not supported') hash_key_value_alias = hash_key_expression.split("=")[1].strip() # Temporary fix until we get proper KeyConditionExpression function hash_key = value_alias_map.get(hash_key_value_alias, {'S': hash_key_value_alias}) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 9debe49d6..3d9914f14 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2705,6 +2705,31 @@ def test_item_size_is_under_400KB(): Item={'id': {'S': 'foo'}, 'itemlist': {'L': [{'M': {'item1': {'S': large_item}}}]}}) +@mock_dynamodb2 +# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html#DDB-Query-request-KeyConditionExpression +def test_hash_key_cannot_use_begins_with_operations(): + dynamodb = boto3.resource('dynamodb') + table = dynamodb.create_table( + TableName='test-table', + KeySchema=[{'AttributeName': 'key', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'key', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1}) + + items = [{'key': 'prefix-$LATEST', 'value': '$LATEST'}, + {'key': 'prefix-DEV', 'value': 'DEV'}, + {'key': 'prefix-PROD', 'value': 'PROD'}] + + with table.batch_writer() as batch: + for item in items: + batch.put_item(Item=item) + + table = dynamodb.Table('test-table') + with assert_raises(ClientError) as ex: + table.query(KeyConditionExpression=Key('key').begins_with('prefix-')) + ex.exception.response['Error']['Code'].should.equal('ValidationException') + ex.exception.response['Error']['Message'].should.equal('Query key condition not supported') + + def assert_failure_due_to_item_size(func, **kwargs): with assert_raises(ClientError) as ex: func(**kwargs) From e64b9ca0ef81f739dbdba05ba370dd37e0b2314c Mon Sep 17 00:00:00 2001 From: Selena Date: Fri, 18 Oct 2019 12:52:05 +0200 Subject: [PATCH 27/32] fix deprication warning invalid escape sequence fix #2489 --- moto/ec2/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index e67cb39f4..a718c7812 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -50,7 +50,7 @@ EC2_PREFIX_TO_RESOURCE = dict((v, k) for (k, v) in EC2_RESOURCE_TO_PREFIX.items( def random_resource_id(size=8): chars = list(range(10)) + ['a', 'b', 'c', 'd', 'e', 'f'] - resource_id = ''.join(six.text_type(random.choice(chars)) for x in range(size)) + resource_id = ''.join(six.text_type(random.choice(chars)) for _ in range(size)) return resource_id @@ -460,8 +460,8 @@ def generic_filter(filters, objects): def simple_aws_filter_to_re(filter_string): - tmp_filter = filter_string.replace('\?', '[?]') - tmp_filter = tmp_filter.replace('\*', '[*]') + tmp_filter = filter_string.replace(r'\?', '[?]') + tmp_filter = tmp_filter.replace(r'\*', '[*]') tmp_filter = fnmatch.translate(tmp_filter) return tmp_filter @@ -491,7 +491,7 @@ def get_prefix(resource_id): 'network-interface-attachment'] if resource_id_prefix not in EC2_RESOURCE_TO_PREFIX.values(): uuid4hex = re.compile( - '[0-9a-f]{12}4[0-9a-f]{3}[89ab][0-9a-f]{15}\Z', re.I) + r'[0-9a-f]{12}4[0-9a-f]{3}[89ab][0-9a-f]{15}\Z', re.I) if uuid4hex.match(resource_id) is not None: resource_id_prefix = EC2_RESOURCE_TO_PREFIX['reserved-instance'] else: @@ -510,7 +510,7 @@ def is_valid_resource_id(resource_id): def is_valid_cidr(cird): - cidr_pattern = '^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$' + cidr_pattern = r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$' cidr_pattern_re = re.compile(cidr_pattern) return cidr_pattern_re.match(cird) is not None From bd627b65f7c53713fc5e2b0d6eea1e90ffb941f8 Mon Sep 17 00:00:00 2001 From: gruebel Date: Fri, 18 Oct 2019 17:29:15 +0200 Subject: [PATCH 28/32] Add iam.create_open_id_connect_provider --- IMPLEMENTATION_COVERAGE.md | 4 +- moto/iam/exceptions.py | 24 +++++++ moto/iam/models.py | 87 ++++++++++++++++++++++++- moto/iam/responses.py | 19 ++++++ tests/test_iam/test_iam.py | 130 +++++++++++++++++++++++++++++++++++++ 5 files changed, 260 insertions(+), 4 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 4fd909d67..3bea66408 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3163,7 +3163,7 @@ - [ ] describe_events ## iam -55% implemented +56% implemented - [ ] add_client_id_to_open_id_connect_provider - [X] add_role_to_instance_profile - [X] add_user_to_group @@ -3176,7 +3176,7 @@ - [X] create_group - [X] create_instance_profile - [X] create_login_profile -- [ ] create_open_id_connect_provider +- [X] create_open_id_connect_provider - [X] create_policy - [X] create_policy_version - [X] create_role diff --git a/moto/iam/exceptions.py b/moto/iam/exceptions.py index ac08e0d88..0511fa144 100644 --- a/moto/iam/exceptions.py +++ b/moto/iam/exceptions.py @@ -93,3 +93,27 @@ class TooManyTags(RESTError): super(TooManyTags, self).__init__( 'ValidationError', "1 validation error detected: Value '{}' at '{}' failed to satisfy " "constraint: Member must have length less than or equal to 50.".format(tags, param)) + + +class EntityAlreadyExists(RESTError): + code = 409 + + def __init__(self): + super(EntityAlreadyExists, self).__init__( + 'EntityAlreadyExists', "Unknown") + + +class ValidationError(RESTError): + code = 400 + + def __init__(self, message): + super(ValidationError, self).__init__( + 'ValidationError', message) + + +class InvalidInput(RESTError): + code = 400 + + def __init__(self, message): + super(InvalidInput, self).__init__( + 'InvalidInput', message) diff --git a/moto/iam/models.py b/moto/iam/models.py index 506f2a942..2a7d185a5 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -7,6 +7,7 @@ import re from cryptography import x509 from cryptography.hazmat.backends import default_backend +from six.moves.urllib.parse import urlparse from moto.core.exceptions import RESTError from moto.core import BaseBackend, BaseModel @@ -14,8 +15,9 @@ from moto.core.utils import iso_8601_datetime_without_milliseconds, iso_8601_dat from moto.iam.policy_validation import IAMPolicyDocumentValidator from .aws_managed_policies import aws_managed_policies_data -from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException, IAMLimitExceededException, \ - MalformedCertificate, DuplicateTags, TagKeyTooBig, InvalidTagCharacters, TooManyTags, TagValueTooBig +from .exceptions import (IAMNotFoundException, IAMConflictException, IAMReportNotPresentException, IAMLimitExceededException, + MalformedCertificate, DuplicateTags, TagKeyTooBig, InvalidTagCharacters, TooManyTags, TagValueTooBig, + EntityAlreadyExists, ValidationError, InvalidInput) from .utils import random_access_key, random_alphanumeric, random_resource_id, random_policy_id ACCOUNT_ID = 123456789012 @@ -93,6 +95,77 @@ class SAMLProvider(BaseModel): return "arn:aws:iam::{0}:saml-provider/{1}".format(ACCOUNT_ID, self.name) +class OpenIDConnectProvider(BaseModel): + def __init__(self, url, thumbprint_list, client_id_list=None): + self._errors = [] + self._validate(url, thumbprint_list, client_id_list) + + parsed_url = urlparse(url) + self.url = parsed_url.netloc + parsed_url.path + self.thumbprint_list = thumbprint_list + self.client_id_list = client_id_list + + @property + def arn(self): + return 'arn:aws:iam::{0}:oidc-provider/{1}'.format(ACCOUNT_ID, self.url) + + def _validate(self, url, thumbprint_list, client_id_list): + if any(len(client_id) > 255 for client_id in client_id_list): + self._errors.append(self._format_error( + key='clientIDList', + value=client_id_list, + constraint='Member must satisfy constraint: ' + '[Member must have length less than or equal to 255, ' + 'Member must have length greater than or equal to 1]', + )) + + if any(len(thumbprint) > 40 for thumbprint in thumbprint_list): + self._errors.append(self._format_error( + key='thumbprintList', + value=thumbprint_list, + constraint='Member must satisfy constraint: ' + '[Member must have length less than or equal to 40, ' + 'Member must have length greater than or equal to 40]', + )) + + if len(url) > 255: + self._errors.append(self._format_error( + key='url', + value=url, + constraint='Member must have length less than or equal to 255', + )) + + self._raise_errors() + + parsed_url = urlparse(url) + if not parsed_url.scheme or not parsed_url.netloc: + raise ValidationError('Invalid Open ID Connect Provider URL') + + if len(thumbprint_list) > 5: + raise InvalidInput('Thumbprint list must contain fewer than 5 entries.') + + if len(client_id_list) > 100: + raise IAMLimitExceededException('Cannot exceed quota for ClientIdsPerOpenIdConnectProvider: 100') + + def _format_error(self, key, value, constraint): + return 'Value "{value}" at "{key}" failed to satisfy constraint: {constraint}'.format( + constraint=constraint, + key=key, + value=value, + ) + + def _raise_errors(self): + if self._errors: + count = len(self._errors) + plural = "s" if len(self._errors) > 1 else "" + errors = "; ".join(self._errors) + self._errors = [] # reset collected errors + + raise ValidationError('{count} validation error{plural} detected: {errors}'.format( + count=count, plural=plural, errors=errors, + )) + + class PolicyVersion(object): def __init__(self, @@ -515,6 +588,7 @@ class IAMBackend(BaseBackend): self.managed_policies = self._init_managed_policies() self.account_aliases = [] self.saml_providers = {} + self.open_id_providers = {} self.policy_arn_regex = re.compile( r'^arn:aws:iam::[0-9]*:policy/.*$') super(IAMBackend, self).__init__() @@ -1264,5 +1338,14 @@ class IAMBackend(BaseBackend): return user return None + def create_open_id_connect_provider(self, url, thumbprint_list, client_id_list): + open_id_provider = OpenIDConnectProvider(url, thumbprint_list, client_id_list) + + if open_id_provider.arn in self.open_id_providers: + raise EntityAlreadyExists + + self.open_id_providers[open_id_provider.arn] = open_id_provider + return open_id_provider + iam_backend = IAMBackend() diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 8e63c1075..a4ed38666 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -755,6 +755,15 @@ class IamResponse(BaseResponse): template = self.response_template(UNTAG_ROLE_TEMPLATE) return template.render() + def create_open_id_connect_provider(self): + open_id_provider_url = self._get_param('Url') + thumbprint_list = self._get_multi_param('ThumbprintList.member') + client_id_list = self._get_multi_param('ClientIDList.member') + open_id_provider = iam_backend.create_open_id_connect_provider(open_id_provider_url, thumbprint_list, client_id_list) + + template = self.response_template(CREATE_OPEN_ID_CONNECT_PROVIDER_TEMPLATE) + return template.render(open_id_provider=open_id_provider) + LIST_ENTITIES_FOR_POLICY_TEMPLATE = """ @@ -1974,3 +1983,13 @@ UNTAG_ROLE_TEMPLATE = """ + + {{ open_id_provider.arn }} + + + f248366a-4f64-11e4-aefa-bfd6aEXAMPLE + +""" diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 23846712d..deec15285 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -1565,3 +1565,133 @@ def test_create_role_with_permissions_boundary(): # Ensure the PermissionsBoundary is included in role listing as well conn.list_roles().get('Roles')[0].get('PermissionsBoundary').should.equal(expected) + + +@mock_iam +def test_create_open_id_connect_provider(): + client = boto3.client('iam', region_name='us-east-1') + response = client.create_open_id_connect_provider( + Url='https://example.com', + ThumbprintList=[] # even it is required to provide at least one thumbprint, AWS accepts an empty list + ) + + response['OpenIDConnectProviderArn'].should.equal( + 'arn:aws:iam::123456789012:oidc-provider/example.com' + ) + + response = client.create_open_id_connect_provider( + Url='http://example.org', + ThumbprintList=[ + 'b' * 40 + ], + ClientIDList=[ + 'b' + ] + ) + + response['OpenIDConnectProviderArn'].should.equal( + 'arn:aws:iam::123456789012:oidc-provider/example.org' + ) + + response = client.create_open_id_connect_provider( + Url='http://example.org/oidc', + ThumbprintList=[] + ) + + response['OpenIDConnectProviderArn'].should.equal( + 'arn:aws:iam::123456789012:oidc-provider/example.org/oidc' + ) + + response = client.create_open_id_connect_provider( + Url='http://example.org/oidc-query?test=true', + ThumbprintList=[] + ) + + response['OpenIDConnectProviderArn'].should.equal( + 'arn:aws:iam::123456789012:oidc-provider/example.org/oidc-query' + ) + + +@mock_iam +def test_create_open_id_connect_provider_errors(): + client = boto3.client('iam', region_name='us-east-1') + response = client.create_open_id_connect_provider( + Url='https://example.com', + ThumbprintList=[] + ) + open_id_arn = response['OpenIDConnectProviderArn'] + + client.create_open_id_connect_provider.when.called_with( + Url='https://example.com', + ThumbprintList=[] + ).should.throw( + ClientError, + 'Unknown' + ) + + client.create_open_id_connect_provider.when.called_with( + Url='example.org', + ThumbprintList=[] + ).should.throw( + ClientError, + 'Invalid Open ID Connect Provider URL' + ) + + client.create_open_id_connect_provider.when.called_with( + Url='example', + ThumbprintList=[] + ).should.throw( + ClientError, + 'Invalid Open ID Connect Provider URL' + ) + + client.create_open_id_connect_provider.when.called_with( + Url='http://example.org', + ThumbprintList=[ + 'a' * 40, + 'b' * 40, + 'c' * 40, + 'd' * 40, + 'e' * 40, + 'f' * 40, + ] + ).should.throw( + ClientError, + 'Thumbprint list must contain fewer than 5 entries.' + ) + + too_many_client_ids = ['{}'.format(i) for i in range(101)] + client.create_open_id_connect_provider.when.called_with( + Url='http://example.org', + ThumbprintList=[], + ClientIDList=too_many_client_ids + ).should.throw( + ClientError, + 'Cannot exceed quota for ClientIdsPerOpenIdConnectProvider: 100' + ) + + too_long_url = 'b' * 256 + too_long_thumbprint = 'b' * 41 + too_long_client_id = 'b' * 256 + client.create_open_id_connect_provider.when.called_with( + Url=too_long_url, + ThumbprintList=[ + too_long_thumbprint + ], + ClientIDList=[ + too_long_client_id + ] + ).should.throw( + ClientError, + '3 validation errors detected: ' + 'Value "{0}" at "clientIDList" failed to satisfy constraint: ' + 'Member must satisfy constraint: ' + '[Member must have length less than or equal to 255, ' + 'Member must have length greater than or equal to 1]; ' + 'Value "{1}" at "thumbprintList" failed to satisfy constraint: ' + 'Member must satisfy constraint: ' + '[Member must have length less than or equal to 40, ' + 'Member must have length greater than or equal to 40]; ' + 'Value "{2}" at "url" failed to satisfy constraint: ' + 'Member must have length less than or equal to 255'.format([too_long_client_id], [too_long_thumbprint], too_long_url) + ) From c492c5c2289abcf7b6aef8596764a020563d8b42 Mon Sep 17 00:00:00 2001 From: gruebel Date: Fri, 18 Oct 2019 20:37:35 +0200 Subject: [PATCH 29/32] Add iam.get_open_id_connect_provider --- IMPLEMENTATION_COVERAGE.md | 2 +- moto/iam/models.py | 13 +++++++++ moto/iam/responses.py | 30 ++++++++++++++++++++ tests/test_iam/test_iam.py | 57 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 99 insertions(+), 3 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 3bea66408..e91376eda 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3227,7 +3227,7 @@ - [X] get_group_policy - [X] get_instance_profile - [X] get_login_profile -- [ ] get_open_id_connect_provider +- [X] get_open_id_connect_provider - [ ] get_organizations_access_report - [X] get_policy - [X] get_policy_version diff --git a/moto/iam/models.py b/moto/iam/models.py index 2a7d185a5..b95c37df7 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -104,11 +104,16 @@ class OpenIDConnectProvider(BaseModel): self.url = parsed_url.netloc + parsed_url.path self.thumbprint_list = thumbprint_list self.client_id_list = client_id_list + self.create_date = datetime.utcnow() @property def arn(self): return 'arn:aws:iam::{0}:oidc-provider/{1}'.format(ACCOUNT_ID, self.url) + @property + def created_iso_8601(self): + return iso_8601_datetime_without_milliseconds(self.create_date) + def _validate(self, url, thumbprint_list, client_id_list): if any(len(client_id) > 255 for client_id in client_id_list): self._errors.append(self._format_error( @@ -1347,5 +1352,13 @@ class IAMBackend(BaseBackend): self.open_id_providers[open_id_provider.arn] = open_id_provider return open_id_provider + def get_open_id_connect_provider(self, arn): + open_id_provider = self.open_id_providers.get(arn) + + if not open_id_provider: + raise IAMNotFoundException('OpenIDConnect Provider not found for arn {}'.format(arn)) + + return open_id_provider + iam_backend = IAMBackend() diff --git a/moto/iam/responses.py b/moto/iam/responses.py index a4ed38666..06ad7b7ce 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -759,11 +759,20 @@ class IamResponse(BaseResponse): open_id_provider_url = self._get_param('Url') thumbprint_list = self._get_multi_param('ThumbprintList.member') client_id_list = self._get_multi_param('ClientIDList.member') + open_id_provider = iam_backend.create_open_id_connect_provider(open_id_provider_url, thumbprint_list, client_id_list) template = self.response_template(CREATE_OPEN_ID_CONNECT_PROVIDER_TEMPLATE) return template.render(open_id_provider=open_id_provider) + def get_open_id_connect_provider(self): + open_id_provider_arn = self._get_param('OpenIDConnectProviderArn') + + open_id_provider = iam_backend.get_open_id_connect_provider(open_id_provider_arn) + + template = self.response_template(GET_OPEN_ID_CONNECT_PROVIDER_TEMPLATE) + return template.render(open_id_provider=open_id_provider) + LIST_ENTITIES_FOR_POLICY_TEMPLATE = """ @@ -1993,3 +2002,24 @@ CREATE_OPEN_ID_CONNECT_PROVIDER_TEMPLATE = """f248366a-4f64-11e4-aefa-bfd6aEXAMPLE """ + + +GET_OPEN_ID_CONNECT_PROVIDER_TEMPLATE = """ + + + {% for thumbprint in open_id_provider.thumbprint_list %} + {{ thumbprint }} + {% endfor %} + + {{ open_id_provider.created_iso_8601 }} + + {% for client_id in open_id_provider.client_id_list %} + {{ client_id }} + {% endfor %} + + {{ open_id_provider.url }} + + + 2c91531b-4f65-11e4-aefa-bfd6aEXAMPLE + +""" diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index deec15285..ea0434bff 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -9,6 +9,9 @@ import sure # noqa import sys from boto.exception import BotoServerError from botocore.exceptions import ClientError +from dateutil.tz import tzutc +from freezegun import freeze_time + from moto import mock_iam, mock_iam_deprecated from moto.iam.models import aws_managed_policies from nose.tools import assert_raises, assert_equals @@ -1615,11 +1618,10 @@ def test_create_open_id_connect_provider(): @mock_iam def test_create_open_id_connect_provider_errors(): client = boto3.client('iam', region_name='us-east-1') - response = client.create_open_id_connect_provider( + client.create_open_id_connect_provider( Url='https://example.com', ThumbprintList=[] ) - open_id_arn = response['OpenIDConnectProviderArn'] client.create_open_id_connect_provider.when.called_with( Url='https://example.com', @@ -1695,3 +1697,54 @@ def test_create_open_id_connect_provider_errors(): 'Value "{2}" at "url" failed to satisfy constraint: ' 'Member must have length less than or equal to 255'.format([too_long_client_id], [too_long_thumbprint], too_long_url) ) + + +@freeze_time('2019-01-01 00:00:00') +@mock_iam +def test_get_open_id_connect_provider(): + client = boto3.client('iam', region_name='us-east-1') + response = client.create_open_id_connect_provider( + Url='https://example.com', + ThumbprintList=[ + 'b' * 40 + ], + ClientIDList=[ + 'b' + ] + ) + open_id_arn = response['OpenIDConnectProviderArn'] + + response = client.get_open_id_connect_provider( + OpenIDConnectProviderArn=open_id_arn + ) + + response['Url'].should.equal('example.com') + response['ThumbprintList'].should.equal([ + 'b' * 40 + ]) + response['ClientIDList'].should.equal([ + 'b' + ]) + response['CreateDate'].should.equal(datetime.now(tzutc())) + + +@mock_iam +def test_get_open_id_connect_provider_errors(): + client = boto3.client('iam', region_name = 'us-east-1') + response = client.create_open_id_connect_provider( + Url='https://example.com', + ThumbprintList=[ + 'b' * 40 + ], + ClientIDList=[ + 'b' + ] + ) + open_id_arn = response['OpenIDConnectProviderArn'] + + client.get_open_id_connect_provider.when.called_with( + OpenIDConnectProviderArn = open_id_arn + '-not-existing' + ).should.throw( + ClientError, + 'OpenIDConnect Provider not found for arn {}'.format(open_id_arn + '-not-existing') + ) From f4af9a1d548110ce763e4422150b8459e3c69bf9 Mon Sep 17 00:00:00 2001 From: gruebel Date: Fri, 18 Oct 2019 20:51:22 +0200 Subject: [PATCH 30/32] Add iam.delete_open_id_connect_provider --- IMPLEMENTATION_COVERAGE.md | 2 +- moto/iam/models.py | 3 +++ moto/iam/responses.py | 15 +++++++++++++++ tests/test_iam/test_iam.py | 30 ++++++++++++++++++++++++++++-- 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index e91376eda..8ccfb2ccf 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3193,7 +3193,7 @@ - [ ] delete_group_policy - [ ] delete_instance_profile - [X] delete_login_profile -- [ ] delete_open_id_connect_provider +- [X] delete_open_id_connect_provider - [X] delete_policy - [X] delete_policy_version - [X] delete_role diff --git a/moto/iam/models.py b/moto/iam/models.py index b95c37df7..6c790c15f 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -1352,6 +1352,9 @@ class IAMBackend(BaseBackend): self.open_id_providers[open_id_provider.arn] = open_id_provider return open_id_provider + def delete_open_id_connect_provider(self, arn): + self.open_id_providers.pop(arn, None) + def get_open_id_connect_provider(self, arn): open_id_provider = self.open_id_providers.get(arn) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 06ad7b7ce..819f0e649 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -765,6 +765,14 @@ class IamResponse(BaseResponse): template = self.response_template(CREATE_OPEN_ID_CONNECT_PROVIDER_TEMPLATE) return template.render(open_id_provider=open_id_provider) + def delete_open_id_connect_provider(self): + open_id_provider_arn = self._get_param('OpenIDConnectProviderArn') + + iam_backend.delete_open_id_connect_provider(open_id_provider_arn) + + template = self.response_template(DELETE_OPEN_ID_CONNECT_PROVIDER_TEMPLATE) + return template.render() + def get_open_id_connect_provider(self): open_id_provider_arn = self._get_param('OpenIDConnectProviderArn') @@ -2004,6 +2012,13 @@ CREATE_OPEN_ID_CONNECT_PROVIDER_TEMPLATE = """""" +DELETE_OPEN_ID_CONNECT_PROVIDER_TEMPLATE = """ + + b5e49e29-4f64-11e4-aefa-bfd6aEXAMPLE + +""" + + GET_OPEN_ID_CONNECT_PROVIDER_TEMPLATE = """ diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index ea0434bff..cffa00a37 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -1699,6 +1699,32 @@ def test_create_open_id_connect_provider_errors(): ) +@mock_iam +def test_delete_open_id_connect_provider(): + client = boto3.client('iam', region_name='us-east-1') + response = client.create_open_id_connect_provider( + Url='https://example.com', + ThumbprintList=[] + ) + open_id_arn = response['OpenIDConnectProviderArn'] + + client.delete_open_id_connect_provider( + OpenIDConnectProviderArn=open_id_arn + ) + + client.get_open_id_connect_provider.when.called_with( + OpenIDConnectProviderArn=open_id_arn + ).should.throw( + ClientError, + 'OpenIDConnect Provider not found for arn {}'.format(open_id_arn) + ) + + # deleting a non existing provider should be successful + client.delete_open_id_connect_provider( + OpenIDConnectProviderArn=open_id_arn + ) + + @freeze_time('2019-01-01 00:00:00') @mock_iam def test_get_open_id_connect_provider(): @@ -1730,7 +1756,7 @@ def test_get_open_id_connect_provider(): @mock_iam def test_get_open_id_connect_provider_errors(): - client = boto3.client('iam', region_name = 'us-east-1') + client = boto3.client('iam', region_name='us-east-1') response = client.create_open_id_connect_provider( Url='https://example.com', ThumbprintList=[ @@ -1743,7 +1769,7 @@ def test_get_open_id_connect_provider_errors(): open_id_arn = response['OpenIDConnectProviderArn'] client.get_open_id_connect_provider.when.called_with( - OpenIDConnectProviderArn = open_id_arn + '-not-existing' + OpenIDConnectProviderArn=open_id_arn + '-not-existing' ).should.throw( ClientError, 'OpenIDConnect Provider not found for arn {}'.format(open_id_arn + '-not-existing') From cd8027ce9db403ac2460d8bd1e534a300ae232db Mon Sep 17 00:00:00 2001 From: gruebel Date: Fri, 18 Oct 2019 21:12:44 +0200 Subject: [PATCH 31/32] Add iam.list_open_id_connect_providers --- IMPLEMENTATION_COVERAGE.md | 4 ++-- moto/iam/models.py | 3 +++ moto/iam/responses.py | 22 ++++++++++++++++++++ tests/test_iam/test_iam.py | 41 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 8ccfb2ccf..284f4d68a 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3163,7 +3163,7 @@ - [ ] describe_events ## iam -56% implemented +57% implemented - [ ] add_client_id_to_open_id_connect_provider - [X] add_role_to_instance_profile - [X] add_user_to_group @@ -3253,7 +3253,7 @@ - [ ] list_instance_profiles - [ ] list_instance_profiles_for_role - [X] list_mfa_devices -- [ ] list_open_id_connect_providers +- [X] list_open_id_connect_providers - [X] list_policies - [ ] list_policies_granting_service_access - [X] list_policy_versions diff --git a/moto/iam/models.py b/moto/iam/models.py index 6c790c15f..ba7d0ac82 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -1363,5 +1363,8 @@ class IAMBackend(BaseBackend): return open_id_provider + def list_open_id_connect_providers(self): + return list(self.open_id_providers.keys()) + iam_backend = IAMBackend() diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 819f0e649..1c00d211c 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -781,6 +781,12 @@ class IamResponse(BaseResponse): template = self.response_template(GET_OPEN_ID_CONNECT_PROVIDER_TEMPLATE) return template.render(open_id_provider=open_id_provider) + def list_open_id_connect_providers(self): + open_id_provider_arns = iam_backend.list_open_id_connect_providers() + + template = self.response_template(LIST_OPEN_ID_CONNECT_PROVIDERS_TEMPLATE) + return template.render(open_id_provider_arns=open_id_provider_arns) + LIST_ENTITIES_FOR_POLICY_TEMPLATE = """ @@ -2038,3 +2044,19 @@ GET_OPEN_ID_CONNECT_PROVIDER_TEMPLATE = """2c91531b-4f65-11e4-aefa-bfd6aEXAMPLE """ + + +LIST_OPEN_ID_CONNECT_PROVIDERS_TEMPLATE = """ + + + {% for open_id_provider_arn in open_id_provider_arns %} + + {{ open_id_provider_arn }} + + {% endfor %} + + + + de2c0228-4f63-11e4-aefa-bfd6aEXAMPLE + +""" diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index cffa00a37..f87af0d16 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -1774,3 +1774,44 @@ def test_get_open_id_connect_provider_errors(): ClientError, 'OpenIDConnect Provider not found for arn {}'.format(open_id_arn + '-not-existing') ) + + +@mock_iam +def test_list_open_id_connect_providers(): + client = boto3.client('iam', region_name='us-east-1') + response = client.create_open_id_connect_provider( + Url='https://example.com', + ThumbprintList=[] + ) + open_id_arn_1 = response['OpenIDConnectProviderArn'] + + response = client.create_open_id_connect_provider( + Url='http://example.org', + ThumbprintList=[ + 'b' * 40 + ], + ClientIDList=[ + 'b' + ] + ) + open_id_arn_2 = response['OpenIDConnectProviderArn'] + + response = client.create_open_id_connect_provider( + Url='http://example.org/oidc', + ThumbprintList=[] + ) + open_id_arn_3 = response['OpenIDConnectProviderArn'] + + response = client.list_open_id_connect_providers() + + response['OpenIDConnectProviderList'].should.equal([ + { + 'Arn': open_id_arn_1 + }, + { + 'Arn': open_id_arn_2 + }, + { + 'Arn': open_id_arn_3 + } + ]) From 3f3feb5bdb9140e9b7f0aab1a8a873daa2b579a3 Mon Sep 17 00:00:00 2001 From: gruebel Date: Sat, 19 Oct 2019 14:23:35 +0200 Subject: [PATCH 32/32] Fix tests --- tests/test_iam/test_iam.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index f87af0d16..2374fb599 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -10,7 +10,6 @@ import sys from boto.exception import BotoServerError from botocore.exceptions import ClientError from dateutil.tz import tzutc -from freezegun import freeze_time from moto import mock_iam, mock_iam_deprecated from moto.iam.models import aws_managed_policies @@ -1725,7 +1724,6 @@ def test_delete_open_id_connect_provider(): ) -@freeze_time('2019-01-01 00:00:00') @mock_iam def test_get_open_id_connect_provider(): client = boto3.client('iam', region_name='us-east-1') @@ -1751,7 +1749,7 @@ def test_get_open_id_connect_provider(): response['ClientIDList'].should.equal([ 'b' ]) - response['CreateDate'].should.equal(datetime.now(tzutc())) + response.should.have.key('CreateDate').should.be.a(datetime) @mock_iam @@ -1804,14 +1802,16 @@ def test_list_open_id_connect_providers(): response = client.list_open_id_connect_providers() - response['OpenIDConnectProviderList'].should.equal([ - { - 'Arn': open_id_arn_1 - }, - { - 'Arn': open_id_arn_2 - }, - { - 'Arn': open_id_arn_3 - } - ]) + sorted(response['OpenIDConnectProviderList'], key=lambda i: i['Arn']).should.equal( + [ + { + 'Arn': open_id_arn_1 + }, + { + 'Arn': open_id_arn_2 + }, + { + 'Arn': open_id_arn_3 + } + ] + )