From 79cd1e609c5ff7153e11956c1f164cda504732fe Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 7 Jul 2019 22:32:46 -0500 Subject: [PATCH 01/63] Move env variable mocking and undo when stopping. CC #2058, #2172. --- moto/core/models.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index 9fe1e96bd..ad700b0be 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -12,6 +12,7 @@ from collections import defaultdict from botocore.handlers import BUILTIN_HANDLERS from botocore.awsrequest import AWSResponse +import mock from moto import settings import responses from moto.packages.httpretty import HTTPretty @@ -22,11 +23,6 @@ from .utils import ( ) -# "Mock" the AWS credentials as they can't be mocked in Botocore currently -os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") -os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") - - class BaseMockAWS(object): nested_count = 0 @@ -42,6 +38,10 @@ class BaseMockAWS(object): self.backends_for_urls.update(self.backends) self.backends_for_urls.update(default_backends) + # "Mock" the AWS credentials as they can't be mocked in Botocore currently + FAKE_KEYS = {"AWS_ACCESS_KEY_ID": "foobar_key", "AWS_SECRET_ACCESS_KEY": "foobar_secret"} + self.env_variables_mocks = mock.patch.dict(os.environ, FAKE_KEYS) + if self.__class__.nested_count == 0: self.reset() @@ -57,6 +57,8 @@ class BaseMockAWS(object): self.stop() def start(self, reset=True): + self.env_variables_mocks.start() + self.__class__.nested_count += 1 if reset: for backend in self.backends.values(): @@ -65,6 +67,7 @@ class BaseMockAWS(object): self.enable_patching() def stop(self): + self.env_variables_mocks.stop() self.__class__.nested_count -= 1 if self.__class__.nested_count < 0: From 308712841cb1e9021fce9615d035017421d953b4 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 9 Jul 2019 20:31:43 -0500 Subject: [PATCH 02/63] Have context manager return mock. --- moto/core/models.py | 1 + tests/test_core/test_context_manager.py | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 tests/test_core/test_context_manager.py diff --git a/moto/core/models.py b/moto/core/models.py index 9fe1e96bd..75a5ecce1 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -52,6 +52,7 @@ class BaseMockAWS(object): def __enter__(self): self.start() + return self def __exit__(self, *args): self.stop() diff --git a/tests/test_core/test_context_manager.py b/tests/test_core/test_context_manager.py new file mode 100644 index 000000000..c5a22558f --- /dev/null +++ b/tests/test_core/test_context_manager.py @@ -0,0 +1,10 @@ +import sure # noqa +import boto3 +from moto import mock_sqs + +def test_reset_api(): + with mock_sqs() as sqs_mock: + conn = boto3.client("sqs", region_name='us-west-1') + conn.create_queue(QueueName="queue1") + + list(sqs_mock.backends['us-west-1'].queues.keys()).should.equal(['queue1']) From b19c201975a1cf243d6441fbe7def08466b099e1 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 10 Jul 2019 21:16:11 -0500 Subject: [PATCH 03/63] Cleanup model ref resetting. --- moto/core/models.py | 6 +++++- moto/sqs/models.py | 1 + tests/test_core/test_context_manager.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index 75a5ecce1..1b0f2d0dd 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -466,10 +466,14 @@ class BaseModel(object): class BaseBackend(object): - def reset(self): + def _reset_model_refs(self): + # Remove all references to the models stored for service, models in model_data.items(): for model_name, model in models.items(): model.instances = [] + + def reset(self): + self._reset_model_refs() self.__dict__ = {} self.__init__() diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 1404ded75..f2e3ed400 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -379,6 +379,7 @@ class SQSBackend(BaseBackend): def reset(self): region_name = self.region_name + self._reset_model_refs() self.__dict__ = {} self.__init__(region_name) diff --git a/tests/test_core/test_context_manager.py b/tests/test_core/test_context_manager.py index c5a22558f..96d2bec9a 100644 --- a/tests/test_core/test_context_manager.py +++ b/tests/test_core/test_context_manager.py @@ -2,7 +2,7 @@ import sure # noqa import boto3 from moto import mock_sqs -def test_reset_api(): +def test_context_manager_returns_mock(): with mock_sqs() as sqs_mock: conn = boto3.client("sqs", region_name='us-west-1') conn.create_queue(QueueName="queue1") From c2e382f537b87cdc898924eeea2ef66a2d4f1741 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Wed, 10 Jul 2019 21:40:49 -0500 Subject: [PATCH 04/63] Dont test context manager in server mode. --- tests/test_core/test_context_manager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_core/test_context_manager.py b/tests/test_core/test_context_manager.py index 96d2bec9a..4824e021f 100644 --- a/tests/test_core/test_context_manager.py +++ b/tests/test_core/test_context_manager.py @@ -1,10 +1,12 @@ import sure # noqa import boto3 -from moto import mock_sqs +from moto import mock_sqs, settings + def test_context_manager_returns_mock(): with mock_sqs() as sqs_mock: conn = boto3.client("sqs", region_name='us-west-1') conn.create_queue(QueueName="queue1") - list(sqs_mock.backends['us-west-1'].queues.keys()).should.equal(['queue1']) + if not settings.TEST_SERVER_MODE: + list(sqs_mock.backends['us-west-1'].queues.keys()).should.equal(['queue1']) From f32db6e64aaa2f72cc8f8095fdfd79c9800ebf12 Mon Sep 17 00:00:00 2001 From: Mariusz Strzelecki Date: Tue, 18 Jun 2019 12:36:32 +0000 Subject: [PATCH 05/63] Raising MalformedXML exception when using boto3 client and s3.delete_objects() --- moto/s3/responses.py | 2 ++ tests/test_s3/test_s3.py | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 40449dbf9..704ac398c 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -566,6 +566,8 @@ class ResponseObject(_TemplateEnvironmentMixin): keys = minidom.parseString(body).getElementsByTagName('Key') deleted_names = [] error_names = [] + if len(keys) == 0: + raise MalformedXML() for k in keys: key_name = k.firstChild.nodeValue diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 697c47865..f0997b8d6 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -648,6 +648,7 @@ def test_delete_keys_with_invalid(): Key(bucket=bucket, name='file3').set_contents_from_string('abc') Key(bucket=bucket, name='file4').set_contents_from_string('abc') + # non-existing key case result = bucket.delete_keys(['abc', 'file3']) result.deleted.should.have.length_of(1) @@ -656,6 +657,17 @@ def test_delete_keys_with_invalid(): keys.should.have.length_of(3) keys[0].name.should.equal('file1') + # empty keys and boto2 client + result = bucket.delete_keys([]) + + result.deleted.should.have.length_of(0) + result.errors.should.have.length_of(0) + + # empty keys and boto3 client + with assert_raises(ClientError) as err: + boto3.client('s3').delete_objects(Bucket='foobar', Delete={'Objects': []}) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + @mock_s3_deprecated def test_bucket_name_with_dot(): From 0093a7992fe6d833055b4b63a5ac6652aa245604 Mon Sep 17 00:00:00 2001 From: Garrett Heel Date: Thu, 11 Jul 2019 08:27:05 -0400 Subject: [PATCH 06/63] dynamodb2: Defer evaluation of the OR RHS in condition expr --- moto/dynamodb2/comparisons.py | 3 +-- tests/test_dynamodb2/test_dynamodb.py | 30 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/comparisons.py b/moto/dynamodb2/comparisons.py index 1a4633e64..151a314f1 100644 --- a/moto/dynamodb2/comparisons.py +++ b/moto/dynamodb2/comparisons.py @@ -1004,8 +1004,7 @@ class OpOr(Op): def expr(self, item): lhs = self.lhs.expr(item) - rhs = self.rhs.expr(item) - return lhs or rhs + return lhs or self.rhs.expr(item) class Func(object): diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index f5afc1e7e..7746cf66b 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1964,6 +1964,36 @@ def test_condition_expression__attr_doesnt_exist(): update_if_attr_doesnt_exist() +@mock_dynamodb2 +def test_condition_expression__or_order(): + client = boto3.client('dynamodb', region_name='us-east-1') + + client.create_table( + TableName='test', + KeySchema=[{'AttributeName': 'forum_name', 'KeyType': 'HASH'}], + AttributeDefinitions=[ + {'AttributeName': 'forum_name', 'AttributeType': 'S'}, + ], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1}, + ) + + # ensure that the RHS of the OR expression is not evaluated if the LHS + # returns true (as it would result an error) + client.update_item( + TableName='test', + Key={ + 'forum_name': {'S': 'the-key'}, + }, + UpdateExpression='set #ttl=:ttl', + ConditionExpression='attribute_not_exists(#ttl) OR #ttl <= :old_ttl', + ExpressionAttributeNames={'#ttl': 'ttl'}, + ExpressionAttributeValues={ + ':ttl': {'N': '6'}, + ':old_ttl': {'N': '5'}, + } + ) + + @mock_dynamodb2 def test_query_gsi_with_range_key(): dynamodb = boto3.client('dynamodb', region_name='us-east-1') From 92bf8eff12b49def8e9689c57d26184c548546d0 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Fri, 12 Jul 2019 11:48:07 +0800 Subject: [PATCH 07/63] setup.py: remove unnecessary 'datetime' entry This entry is added in [1]. New statements in [1] use the built-in datetime module, which is available since Python 2.3 [2]. On the other hand, when running `pip install moto`, pip installs Zope Datetime module [3], which is unnecessary. [1] https://github.com/spulec/moto/commit/ed93821621af337b701319823f9cb022e49c7a6d [2] https://docs.python.org/2.7/library/datetime.html [3] https://pypi.org/project/DateTime/ --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 6aab240cf..36f9f8eb4 100755 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ install_requires = [ "boto3>=1.9.86", "botocore>=1.12.86", "cryptography>=2.3.0", - "datetime", "requests>=2.5", "xmltodict", "six>1.9", From 4ed189c45438c9f27ba26a9f6d8ea58b37694eb4 Mon Sep 17 00:00:00 2001 From: Berislav Kovacki Date: Sat, 13 Jul 2019 08:19:23 +0200 Subject: [PATCH 08/63] Add support for VpcSecurityGroups set/update in RDS --- moto/rds2/models.py | 14 ++++++++++++-- moto/rds2/responses.py | 2 +- tests/test_rds2/test_rds2.py | 8 ++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 81b346fdb..4c0daa230 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -149,7 +149,14 @@ class Database(BaseModel): {{ database.status }} {% if database.db_name %}{{ database.db_name }}{% endif %} {{ database.multi_az }} - + + {% for vpc_security_group_id in database.vpc_security_group_ids %} + + active + {{ vpc_security_group_id }} + + {% endfor %} + {{ database.db_instance_identifier }} {{ database.dbi_resource_id }} {{ database.instance_create_time }} @@ -323,6 +330,7 @@ class Database(BaseModel): "storage_encrypted": properties.get("StorageEncrypted"), "storage_type": properties.get("StorageType"), "tags": properties.get("Tags"), + "vpc_security_group_ids": properties.get('VpcSecurityGroupIds', []), } rds2_backend = rds2_backends[region_name] @@ -397,10 +405,12 @@ class Database(BaseModel): "SecondaryAvailabilityZone": null, "StatusInfos": null, "VpcSecurityGroups": [ + {% for vpc_security_group_id in database.vpc_security_group_ids %} { "Status": "active", - "VpcSecurityGroupId": "sg-123456" + "VpcSecurityGroupId": "{{ vpc_security_group_id }}" } + {% endfor %} ], "DBInstanceArn": "{{ database.db_instance_arn }}" }""") diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index e92625635..75d4f4ce0 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -43,7 +43,7 @@ class RDS2Response(BaseResponse): "security_groups": self._get_multi_param('DBSecurityGroups.DBSecurityGroupName'), "storage_encrypted": self._get_param("StorageEncrypted"), "storage_type": self._get_param("StorageType", 'standard'), - # VpcSecurityGroupIds.member.N + "vpc_security_group_ids": self._get_multi_param("VpcSecurityGroupIds.VpcSecurityGroupId"), "tags": list(), } args['tags'] = self.unpack_complex_list_params( diff --git a/tests/test_rds2/test_rds2.py b/tests/test_rds2/test_rds2.py index 8ea296c2c..aacaf04f1 100644 --- a/tests/test_rds2/test_rds2.py +++ b/tests/test_rds2/test_rds2.py @@ -18,7 +18,8 @@ def test_create_database(): MasterUsername='root', MasterUserPassword='hunter2', Port=1234, - DBSecurityGroups=["my_sg"]) + DBSecurityGroups=["my_sg"], + VpcSecurityGroupIds=['sg-123456']) db_instance = database['DBInstance'] db_instance['AllocatedStorage'].should.equal(10) db_instance['DBInstanceClass'].should.equal("db.m1.small") @@ -35,6 +36,7 @@ def test_create_database(): db_instance['DbiResourceId'].should.contain("db-") db_instance['CopyTagsToSnapshot'].should.equal(False) db_instance['InstanceCreateTime'].should.be.a("datetime.datetime") + db_instance['VpcSecurityGroups'][0]['VpcSecurityGroupId'].should.equal('sg-123456') @mock_rds2 @@ -260,9 +262,11 @@ def test_modify_db_instance(): instances['DBInstances'][0]['AllocatedStorage'].should.equal(10) conn.modify_db_instance(DBInstanceIdentifier='db-master-1', AllocatedStorage=20, - ApplyImmediately=True) + ApplyImmediately=True, + VpcSecurityGroupIds=['sg-123456']) instances = conn.describe_db_instances(DBInstanceIdentifier='db-master-1') instances['DBInstances'][0]['AllocatedStorage'].should.equal(20) + instances['DBInstances'][0]['VpcSecurityGroups'][0]['VpcSecurityGroupId'].should.equal('sg-123456') @mock_rds2 From 4a87bdf1f34a0ec238bb33f6d57736b4247c2939 Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Sat, 13 Jul 2019 13:16:12 -0700 Subject: [PATCH 09/63] Update supported python versions, add badages. --- README.md | 3 +++ setup.py | 1 - tox.ini | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e4c88dec8..0a9b843f0 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,9 @@ [![Build Status](https://travis-ci.org/spulec/moto.svg?branch=master)](https://travis-ci.org/spulec/moto) [![Coverage Status](https://coveralls.io/repos/spulec/moto/badge.svg?branch=master)](https://coveralls.io/r/spulec/moto) [![Docs](https://readthedocs.org/projects/pip/badge/?version=stable)](http://docs.getmoto.org) +![PyPI](https://img.shields.io/pypi/v/moto.svg) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/moto.svg) +![PyPI - Downloads](https://img.shields.io/pypi/dw/moto.svg) # In a nutshell diff --git a/setup.py b/setup.py index 6aab240cf..4cf11e247 100755 --- a/setup.py +++ b/setup.py @@ -89,7 +89,6 @@ setup( "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", diff --git a/tox.ini b/tox.ini index 1fea4d81d..570b5790f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py36 +envlist = py27, py36, py37 [testenv] setenv = From 9633917aec064b0aef8624e900275ab55f4f2138 Mon Sep 17 00:00:00 2001 From: jakzo Date: Sun, 14 Jul 2019 16:37:54 +0000 Subject: [PATCH 10/63] Fix ID generation --- moto/apigateway/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/apigateway/utils.py b/moto/apigateway/utils.py index 6d1e6ef19..31f8060b0 100644 --- a/moto/apigateway/utils.py +++ b/moto/apigateway/utils.py @@ -1,9 +1,10 @@ from __future__ import unicode_literals import six import random +import string def create_id(): size = 10 - chars = list(range(10)) + ['A-Z'] + chars = list(range(10)) + list(string.ascii_lowercase) return ''.join(six.text_type(random.choice(chars)) for x in range(size)) From e67e2deee440999fb4a44dc3bec298777e4ed7f7 Mon Sep 17 00:00:00 2001 From: Berislav Kovacki Date: Mon, 15 Jul 2019 00:01:37 +0200 Subject: [PATCH 11/63] Extend EC2 DescribeNetworkInterface filter support * add description property to EC2 NetworkInterface * extend DescribeNetworkInterfaces filter support with description, subnet-id, private-ip-address attributes --- moto/ec2/models.py | 19 ++- .../responses/elastic_network_interfaces.py | 9 +- .../test_elastic_network_interfaces.py | 108 +++++++++++++++++- 3 files changed, 129 insertions(+), 7 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 47f201888..2b56c9921 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -201,7 +201,7 @@ class TaggedEC2Resource(BaseModel): class NetworkInterface(TaggedEC2Resource): def __init__(self, ec2_backend, subnet, private_ip_address, device_index=0, - public_ip_auto_assign=True, group_ids=None): + public_ip_auto_assign=True, group_ids=None, description=None): self.ec2_backend = ec2_backend self.id = random_eni_id() self.device_index = device_index @@ -209,6 +209,7 @@ class NetworkInterface(TaggedEC2Resource): self.subnet = subnet self.instance = None self.attachment_id = None + self.description = description self.public_ip = None self.public_ip_auto_assign = public_ip_auto_assign @@ -246,11 +247,13 @@ class NetworkInterface(TaggedEC2Resource): subnet = None private_ip_address = properties.get('PrivateIpAddress', None) + description = properties.get('Description', None) network_interface = ec2_backend.create_network_interface( subnet, private_ip_address, - group_ids=security_group_ids + group_ids=security_group_ids, + description=description ) return network_interface @@ -298,6 +301,8 @@ class NetworkInterface(TaggedEC2Resource): return [group.id for group in self._group_set] elif filter_name == 'availability-zone': return self.subnet.availability_zone + elif filter_name == 'description': + return self.description else: return super(NetworkInterface, self).get_filter_value( filter_name, 'DescribeNetworkInterfaces') @@ -308,9 +313,9 @@ class NetworkInterfaceBackend(object): self.enis = {} super(NetworkInterfaceBackend, self).__init__() - def create_network_interface(self, subnet, private_ip_address, group_ids=None, **kwargs): + def create_network_interface(self, subnet, private_ip_address, group_ids=None, description=None, **kwargs): eni = NetworkInterface( - self, subnet, private_ip_address, group_ids=group_ids, **kwargs) + self, subnet, private_ip_address, group_ids=group_ids, description=description, **kwargs) self.enis[eni.id] = eni return eni @@ -343,6 +348,12 @@ class NetworkInterfaceBackend(object): if group.id in _filter_value: enis.append(eni) break + elif _filter == 'private-ip-address:': + enis = [eni for eni in enis if eni.private_ip_address in _filter_value] + elif _filter == 'subnet-id': + enis = [eni for eni in enis if eni.subnet.id in _filter_value] + elif _filter == 'description': + enis = [eni for eni in enis if eni.description in _filter_value] else: self.raise_not_implemented_error( "The filter '{0}' for DescribeNetworkInterfaces".format(_filter)) diff --git a/moto/ec2/responses/elastic_network_interfaces.py b/moto/ec2/responses/elastic_network_interfaces.py index dc8b92df8..9c37e70da 100644 --- a/moto/ec2/responses/elastic_network_interfaces.py +++ b/moto/ec2/responses/elastic_network_interfaces.py @@ -10,9 +10,10 @@ class ElasticNetworkInterfaces(BaseResponse): private_ip_address = self._get_param('PrivateIpAddress') groups = self._get_multi_param('SecurityGroupId') subnet = self.ec2_backend.get_subnet(subnet_id) + description = self._get_param('Description') if self.is_not_dryrun('CreateNetworkInterface'): eni = self.ec2_backend.create_network_interface( - subnet, private_ip_address, groups) + subnet, private_ip_address, groups, description) template = self.response_template( CREATE_NETWORK_INTERFACE_RESPONSE) return template.render(eni=eni) @@ -78,7 +79,11 @@ CREATE_NETWORK_INTERFACE_RESPONSE = """ {{ eni.subnet.id }} {{ eni.subnet.vpc_id }} us-west-2a + {% if eni.description %} + {{ eni.description }} + {% else %} + {% endif %} 498654062920 false pending @@ -121,7 +126,7 @@ DESCRIBE_NETWORK_INTERFACES_RESPONSE = """{{ eni.subnet.id }} {{ eni.subnet.vpc_id }} us-west-2a - Primary network interface + {{ eni.description }} 190610284047 false {% if eni.attachment_id %} diff --git a/tests/test_ec2/test_elastic_network_interfaces.py b/tests/test_ec2/test_elastic_network_interfaces.py index 70e78ae12..05b45fda9 100644 --- a/tests/test_ec2/test_elastic_network_interfaces.py +++ b/tests/test_ec2/test_elastic_network_interfaces.py @@ -161,7 +161,7 @@ def test_elastic_network_interfaces_filtering(): subnet.id, groups=[security_group1.id, security_group2.id]) eni2 = conn.create_network_interface( subnet.id, groups=[security_group1.id]) - eni3 = conn.create_network_interface(subnet.id) + eni3 = conn.create_network_interface(subnet.id, description='test description') all_enis = conn.get_all_network_interfaces() all_enis.should.have.length_of(3) @@ -189,6 +189,12 @@ def test_elastic_network_interfaces_filtering(): enis_by_group.should.have.length_of(1) set([eni.id for eni in enis_by_group]).should.equal(set([eni1.id])) + # Filter by Description + enis_by_description = conn.get_all_network_interfaces( + filters={'description': eni3.description }) + enis_by_description.should.have.length_of(1) + enis_by_description[0].description.should.equal(eni3.description) + # Unsupported filter conn.get_all_network_interfaces.when.called_with( filters={'not-implemented-filter': 'foobar'}).should.throw(NotImplementedError) @@ -343,6 +349,106 @@ def test_elastic_network_interfaces_get_by_subnet_id(): enis.should.have.length_of(0) +@mock_ec2 +def test_elastic_network_interfaces_get_by_description(): + ec2 = boto3.resource('ec2', region_name='us-west-2') + ec2_client = boto3.client('ec2', region_name='us-west-2') + + vpc = ec2.create_vpc(CidrBlock='10.0.0.0/16') + subnet = ec2.create_subnet( + VpcId=vpc.id, CidrBlock='10.0.0.0/24', AvailabilityZone='us-west-2a') + + eni1 = ec2.create_network_interface( + SubnetId=subnet.id, PrivateIpAddress='10.0.10.5', Description='test interface') + + # The status of the new interface should be 'available' + waiter = ec2_client.get_waiter('network_interface_available') + waiter.wait(NetworkInterfaceIds=[eni1.id]) + + filters = [{'Name': 'description', 'Values': [eni1.description]}] + enis = list(ec2.network_interfaces.filter(Filters=filters)) + enis.should.have.length_of(1) + + filters = [{'Name': 'description', 'Values': ['bad description']}] + enis = list(ec2.network_interfaces.filter(Filters=filters)) + enis.should.have.length_of(0) + + +@mock_ec2 +def test_elastic_network_interfaces_describe_network_interfaces_with_filter(): + ec2 = boto3.resource('ec2', region_name='us-west-2') + ec2_client = boto3.client('ec2', region_name='us-west-2') + + vpc = ec2.create_vpc(CidrBlock='10.0.0.0/16') + subnet = ec2.create_subnet( + VpcId=vpc.id, CidrBlock='10.0.0.0/24', AvailabilityZone='us-west-2a') + + eni1 = ec2.create_network_interface( + SubnetId=subnet.id, PrivateIpAddress='10.0.10.5', Description='test interface') + + # The status of the new interface should be 'available' + waiter = ec2_client.get_waiter('network_interface_available') + waiter.wait(NetworkInterfaceIds=[eni1.id]) + + # Filter by network-interface-id + response = ec2_client.describe_network_interfaces( + Filters=[{'Name': 'network-interface-id', 'Values': [eni1.id]}]) + response['NetworkInterfaces'].should.have.length_of(1) + response['NetworkInterfaces'][0]['NetworkInterfaceId'].should.equal(eni1.id) + response['NetworkInterfaces'][0]['PrivateIpAddress'].should.equal(eni1.private_ip_address) + response['NetworkInterfaces'][0]['Description'].should.equal(eni1.description) + + response = ec2_client.describe_network_interfaces( + Filters=[{'Name': 'network-interface-id', 'Values': ['bad-id']}]) + response['NetworkInterfaces'].should.have.length_of(0) + + # Filter by private-ip-address + response = ec2_client.describe_network_interfaces( + Filters=[{'Name': 'private-ip-address', 'Values': [eni1.private_ip_address]}]) + response['NetworkInterfaces'].should.have.length_of(1) + response['NetworkInterfaces'][0]['NetworkInterfaceId'].should.equal(eni1.id) + response['NetworkInterfaces'][0]['PrivateIpAddress'].should.equal(eni1.private_ip_address) + response['NetworkInterfaces'][0]['Description'].should.equal(eni1.description) + + response = ec2_client.describe_network_interfaces( + Filters=[{'Name': 'private-ip-address', 'Values': ['11.11.11.11']}]) + response['NetworkInterfaces'].should.have.length_of(0) + + # Filter by sunet-id + response = ec2_client.describe_network_interfaces( + Filters=[{'Name': 'subnet-id', 'Values': [eni1.subnet.id]}]) + response['NetworkInterfaces'].should.have.length_of(1) + response['NetworkInterfaces'][0]['NetworkInterfaceId'].should.equal(eni1.id) + response['NetworkInterfaces'][0]['PrivateIpAddress'].should.equal(eni1.private_ip_address) + response['NetworkInterfaces'][0]['Description'].should.equal(eni1.description) + + response = ec2_client.describe_network_interfaces( + Filters=[{'Name': 'subnet-id', 'Values': ['sn-bad-id']}]) + response['NetworkInterfaces'].should.have.length_of(0) + + # Filter by description + response = ec2_client.describe_network_interfaces( + Filters=[{'Name': 'description', 'Values': [eni1.description]}]) + response['NetworkInterfaces'].should.have.length_of(1) + response['NetworkInterfaces'][0]['NetworkInterfaceId'].should.equal(eni1.id) + response['NetworkInterfaces'][0]['PrivateIpAddress'].should.equal(eni1.private_ip_address) + response['NetworkInterfaces'][0]['Description'].should.equal(eni1.description) + + response = ec2_client.describe_network_interfaces( + Filters=[{'Name': 'description', 'Values': ['bad description']}]) + response['NetworkInterfaces'].should.have.length_of(0) + + # Filter by multiple filters + response = ec2_client.describe_network_interfaces( + Filters=[{'Name': 'private-ip-address', 'Values': [eni1.private_ip_address]}, + {'Name': 'network-interface-id', 'Values': [eni1.id]}, + {'Name': 'subnet-id', 'Values': [eni1.subnet.id]}]) + response['NetworkInterfaces'].should.have.length_of(1) + response['NetworkInterfaces'][0]['NetworkInterfaceId'].should.equal(eni1.id) + response['NetworkInterfaces'][0]['PrivateIpAddress'].should.equal(eni1.private_ip_address) + response['NetworkInterfaces'][0]['Description'].should.equal(eni1.description) + + @mock_ec2_deprecated @mock_cloudformation_deprecated def test_elastic_network_interfaces_cloudformation(): From 57136a5f1b7cd2457cb9ceb4b8fd4b42bbf8a140 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 14 Jul 2019 20:39:51 -0500 Subject: [PATCH 12/63] Prep for release 1.3.12 --- moto/__init__.py | 2 +- update_version_from_git.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/__init__.py b/moto/__init__.py index a6f35069e..ac63e38d8 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.11' +__version__ = '1.3.12' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/update_version_from_git.py b/update_version_from_git.py index 355bc2ba9..7b327727e 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -74,7 +74,7 @@ def prerelease_version(): ver, commits_since, githash = get_git_version_info() initpy_ver = get_version() - assert len(initpy_ver.split('.')) in [3, 4], 'moto/__init__.py version should be like 0.0.2 or 0.0.2.dev' + assert len(initpy_ver.split('.')) == 4, 'moto/__init__.py version should be like 0.0.2.dev' assert initpy_ver > ver, 'the moto/__init__.py version should be newer than the last tagged release.' return '{initpy_ver}.dev{commits_since}'.format(initpy_ver=initpy_ver, commits_since=commits_since) From 81da2f5ff153ae870ff2e2613ee7bc1bd478c205 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 14 Jul 2019 20:49:30 -0500 Subject: [PATCH 13/63] Add more debug logs. --- update_version_from_git.py | 1 + 1 file changed, 1 insertion(+) diff --git a/update_version_from_git.py b/update_version_from_git.py index 7b327727e..6f82293dd 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -74,6 +74,7 @@ def prerelease_version(): ver, commits_since, githash = get_git_version_info() initpy_ver = get_version() + print("Retrieve prerelease version", initpy_ver) assert len(initpy_ver.split('.')) == 4, 'moto/__init__.py version should be like 0.0.2.dev' assert initpy_ver > ver, 'the moto/__init__.py version should be newer than the last tagged release.' return '{initpy_ver}.dev{commits_since}'.format(initpy_ver=initpy_ver, commits_since=commits_since) From 77d5d099b9914a9c0b501914fedc6c01a646ec88 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 14 Jul 2019 20:58:59 -0500 Subject: [PATCH 14/63] Fix where we check version length. --- update_version_from_git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update_version_from_git.py b/update_version_from_git.py index 6f82293dd..8f2e4f90b 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -74,8 +74,7 @@ def prerelease_version(): ver, commits_since, githash = get_git_version_info() initpy_ver = get_version() - print("Retrieve prerelease version", initpy_ver) - assert len(initpy_ver.split('.')) == 4, 'moto/__init__.py version should be like 0.0.2.dev' + assert len(initpy_ver.split('.')) in [3, 4], 'moto/__init__.py version should be like 0.0.2.dev' assert initpy_ver > ver, 'the moto/__init__.py version should be newer than the last tagged release.' return '{initpy_ver}.dev{commits_since}'.format(initpy_ver=initpy_ver, commits_since=commits_since) @@ -109,6 +108,7 @@ def release_version_correct(): new_version = prerelease_version() print('updating version in __init__.py to {new_version}'.format(new_version=new_version)) + assert len(new_version.split('.')) == 4, 'moto/__init__.py version should be like 0.0.2.dev' migrate_version(initpy, new_version) else: # check that we are a tag with the same version as in __init__.py From cf95313b39a601ba0325a582b9428008e73594e0 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 14 Jul 2019 21:14:20 -0500 Subject: [PATCH 15/63] Disable non-prereleases; prep for 1.3.13. --- .travis.yml | 16 ++++++++-------- moto/__init__.py | 2 +- update_version_from_git.py | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8145cfb46..77dd2ae55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,11 +47,11 @@ deploy: - master skip_cleanup: true skip_existing: true - - provider: pypi - distributions: sdist bdist_wheel - user: spulec - password: - secure: NxnPylnTfekJmGyoufCw0lMoYRskSMJzvAIyAlJJVYKwEhmiCPOrdy5qV8i8mRZ1AkUsqU3jBZ/PD56n96clHW0E3d080UleRDj6JpyALVdeLfMqZl9kLmZ8bqakWzYq3VSJKw2zGP/L4tPGf8wTK1SUv9yl/YNDsBdCkjDverw= - on: - tags: true - skip_existing: true + # - provider: pypi + # distributions: sdist bdist_wheel + # user: spulec + # password: + # secure: NxnPylnTfekJmGyoufCw0lMoYRskSMJzvAIyAlJJVYKwEhmiCPOrdy5qV8i8mRZ1AkUsqU3jBZ/PD56n96clHW0E3d080UleRDj6JpyALVdeLfMqZl9kLmZ8bqakWzYq3VSJKw2zGP/L4tPGf8wTK1SUv9yl/YNDsBdCkjDverw= + # on: + # tags: true + # skip_existing: true diff --git a/moto/__init__.py b/moto/__init__.py index ac63e38d8..9f240cce4 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.12' +__version__ = '1.3.13' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/update_version_from_git.py b/update_version_from_git.py index 8f2e4f90b..fcb3b5bab 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -111,6 +111,7 @@ def release_version_correct(): assert len(new_version.split('.')) == 4, 'moto/__init__.py version should be like 0.0.2.dev' migrate_version(initpy, new_version) else: + assert False, "No non-master deployments yet" # check that we are a tag with the same version as in __init__.py assert get_version() == git_tag_name(), 'git tag/branch name not the same as moto/__init__.py __verion__' From 86e650ff0166afef94549e33dc2c795755dfe516 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 14 Jul 2019 22:30:09 -0500 Subject: [PATCH 16/63] Set version to 1.3.14.dev. --- moto/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/__init__.py b/moto/__init__.py index 9f240cce4..8594cedd2 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.13' +__version__ = '1.3.14.dev' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa From 8b3b1f88ab6da6cfedf1cf39fb9eb56c9fc2b137 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 14 Jul 2019 22:40:41 -0500 Subject: [PATCH 17/63] Remove double-dev from release name. --- update_version_from_git.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update_version_from_git.py b/update_version_from_git.py index fcb3b5bab..d72dc4ae9 100644 --- a/update_version_from_git.py +++ b/update_version_from_git.py @@ -76,7 +76,7 @@ def prerelease_version(): assert len(initpy_ver.split('.')) in [3, 4], 'moto/__init__.py version should be like 0.0.2.dev' assert initpy_ver > ver, 'the moto/__init__.py version should be newer than the last tagged release.' - return '{initpy_ver}.dev{commits_since}'.format(initpy_ver=initpy_ver, commits_since=commits_since) + return '{initpy_ver}.{commits_since}'.format(initpy_ver=initpy_ver, commits_since=commits_since) def read(*parts): """ Reads in file from *parts. @@ -108,7 +108,7 @@ def release_version_correct(): new_version = prerelease_version() print('updating version in __init__.py to {new_version}'.format(new_version=new_version)) - assert len(new_version.split('.')) == 4, 'moto/__init__.py version should be like 0.0.2.dev' + assert len(new_version.split('.')) >= 4, 'moto/__init__.py version should be like 0.0.2.dev' migrate_version(initpy, new_version) else: assert False, "No non-master deployments yet" From 2c2dff22bca7587d3e1e5106615cabd995447f69 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 15 Jul 2019 20:08:15 +0300 Subject: [PATCH 18/63] moto: s3: support partNumber for head_object To support it, we need to keep multipart info in the key itself when completing multipart upload. Fixes #2154 Signed-off-by: Ruslan Kuprieiev --- moto/s3/models.py | 40 ++++++++++++++++++++++++++++++++++------ moto/s3/responses.py | 9 ++++++++- tests/test_s3/test_s3.py | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 7488114e3..2a628d681 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -52,8 +52,17 @@ class FakeDeleteMarker(BaseModel): class FakeKey(BaseModel): - def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0, - max_buffer_size=DEFAULT_KEY_BUFFER_SIZE): + def __init__( + self, + name, + value, + storage="STANDARD", + etag=None, + is_versioned=False, + version_id=0, + max_buffer_size=DEFAULT_KEY_BUFFER_SIZE, + multipart=None + ): self.name = name self.last_modified = datetime.datetime.utcnow() self.acl = get_canned_acl('private') @@ -65,6 +74,7 @@ class FakeKey(BaseModel): self._version_id = version_id self._is_versioned = is_versioned self._tagging = FakeTagging() + self.multipart = multipart self._value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size) self._max_buffer_size = max_buffer_size @@ -782,7 +792,15 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) return bucket.website_configuration - def set_key(self, bucket_name, key_name, value, storage=None, etag=None): + def set_key( + self, + bucket_name, + key_name, + value, + storage=None, + etag=None, + multipart=None, + ): key_name = clean_key_name(key_name) if storage is not None and storage not in STORAGE_CLASS: raise InvalidStorageClass(storage=storage) @@ -795,7 +813,9 @@ class S3Backend(BaseBackend): storage=storage, etag=etag, is_versioned=bucket.is_versioned, - version_id=str(uuid.uuid4()) if bucket.is_versioned else None) + version_id=str(uuid.uuid4()) if bucket.is_versioned else None, + multipart=multipart, + ) keys = [ key for key in bucket.keys.getlist(key_name, []) @@ -812,7 +832,7 @@ class S3Backend(BaseBackend): key.append_to_value(value) return key - def get_key(self, bucket_name, key_name, version_id=None): + def get_key(self, bucket_name, key_name, version_id=None, part_number=None): key_name = clean_key_name(key_name) bucket = self.get_bucket(bucket_name) key = None @@ -827,6 +847,9 @@ class S3Backend(BaseBackend): key = key_version break + if part_number and key.multipart: + key = key.multipart.parts[part_number] + if isinstance(key, FakeKey): return key else: @@ -890,7 +913,12 @@ class S3Backend(BaseBackend): return del bucket.multiparts[multipart_id] - key = self.set_key(bucket_name, multipart.key_name, value, etag=etag) + key = self.set_key( + bucket_name, + multipart.key_name, + value, etag=etag, + multipart=multipart + ) key.set_metadata(multipart.metadata) return key diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 40449dbf9..6ba7a52c6 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -809,13 +809,20 @@ class ResponseObject(_TemplateEnvironmentMixin): def _key_response_head(self, bucket_name, query, key_name, headers): response_headers = {} version_id = query.get('versionId', [None])[0] + part_number = query.get('partNumber', [None])[0] + if part_number: + part_number = int(part_number) if_modified_since = headers.get('If-Modified-Since', None) if if_modified_since: if_modified_since = str_to_rfc_1123_datetime(if_modified_since) key = self.backend.get_key( - bucket_name, key_name, version_id=version_id) + bucket_name, + key_name, + version_id=version_id, + part_number=part_number + ) if key: response_headers.update(key.metadata) response_headers.update(key.response_dict) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 697c47865..b6129c542 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1671,6 +1671,42 @@ def test_boto3_multipart_etag(): resp['ETag'].should.equal(EXPECTED_ETAG) +@mock_s3 +@reduced_min_part_size +def test_boto3_multipart_part_size(): + s3 = boto3.client('s3', region_name='us-east-1') + s3.create_bucket(Bucket='mybucket') + + mpu = s3.create_multipart_upload(Bucket='mybucket', Key='the-key') + mpu_id = mpu["UploadId"] + + parts = [] + n_parts = 10 + for i in range(1, n_parts + 1): + part_size = 5 * 1024 * 1024 + body = b'1' * part_size + part = s3.upload_part( + Bucket='mybucket', + Key='the-key', + PartNumber=i, + UploadId=mpu_id, + Body=body, + ContentLength=len(body), + ) + parts.append({"PartNumber": i, "ETag": part["ETag"]}) + + s3.complete_multipart_upload( + Bucket='mybucket', + Key='the-key', + UploadId=mpu_id, + MultipartUpload={"Parts": parts}, + ) + + for i in range(1, n_parts + 1): + obj = s3.head_object(Bucket='mybucket', Key='the-key', PartNumber=i) + assert obj["ContentLength"] == part_size + + @mock_s3 def test_boto3_put_object_with_tagging(): s3 = boto3.client('s3', region_name='us-east-1') From 81980850d4c140849af5ecd09001807434b5bacc Mon Sep 17 00:00:00 2001 From: Carlos Aguado Date: Tue, 16 Jul 2019 13:09:13 +1000 Subject: [PATCH 19/63] Implement update_user_pool_domain Introduce the CognitoIDP's UpdateUserPoolDomain to update configuration options of the associated domain to a Cognito IDP (e.g. ACM certificate). --- IMPLEMENTATION_COVERAGE.md | 1 + moto/cognitoidp/models.py | 54 ++++++++++++++++++------ moto/cognitoidp/responses.py | 19 ++++++++- tests/test_cognitoidp/test_cognitoidp.py | 33 +++++++++++++++ 4 files changed, 93 insertions(+), 14 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 685db7ec4..86d8cecad 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -928,6 +928,7 @@ - [ ] update_user_attributes - [ ] update_user_pool - [X] update_user_pool_client +- [X] update_user_pool_domain - [ ] verify_software_token - [ ] verify_user_attribute diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index ef1377789..2c82367c6 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import datetime import functools +import hashlib import itertools import json import os @@ -154,20 +155,37 @@ class CognitoIdpUserPool(BaseModel): class CognitoIdpUserPoolDomain(BaseModel): - def __init__(self, user_pool_id, domain): + def __init__(self, user_pool_id, domain, custom_domain_config=None): self.user_pool_id = user_pool_id self.domain = domain + self.custom_domain_config = custom_domain_config or {} - def to_json(self): - return { - "UserPoolId": self.user_pool_id, - "AWSAccountId": str(uuid.uuid4()), - "CloudFrontDistribution": None, - "Domain": self.domain, - "S3Bucket": None, - "Status": "ACTIVE", - "Version": None, - } + def _distribution_name(self): + if self.custom_domain_config and \ + 'CertificateArn' in self.custom_domain_config: + hash = hashlib.md5( + self.custom_domain_config['CertificateArn'].encode('utf-8') + ).hexdigest() + return "{hash}.cloudfront.net".format(hash=hash[:16]) + return None + + def to_json(self, extended=True): + distribution = self._distribution_name() + if extended: + return { + "UserPoolId": self.user_pool_id, + "AWSAccountId": str(uuid.uuid4()), + "CloudFrontDistribution": distribution, + "Domain": self.domain, + "S3Bucket": None, + "Status": "ACTIVE", + "Version": None, + } + elif distribution: + return { + "CloudFrontDomain": distribution, + } + return None class CognitoIdpUserPoolClient(BaseModel): @@ -338,11 +356,13 @@ class CognitoIdpBackend(BaseBackend): del self.user_pools[user_pool_id] # User pool domain - def create_user_pool_domain(self, user_pool_id, domain): + def create_user_pool_domain(self, user_pool_id, domain, custom_domain_config=None): if user_pool_id not in self.user_pools: raise ResourceNotFoundError(user_pool_id) - user_pool_domain = CognitoIdpUserPoolDomain(user_pool_id, domain) + user_pool_domain = CognitoIdpUserPoolDomain( + user_pool_id, domain, custom_domain_config=custom_domain_config + ) self.user_pool_domains[domain] = user_pool_domain return user_pool_domain @@ -358,6 +378,14 @@ class CognitoIdpBackend(BaseBackend): del self.user_pool_domains[domain] + def update_user_pool_domain(self, domain, custom_domain_config): + if domain not in self.user_pool_domains: + raise ResourceNotFoundError(domain) + + user_pool_domain = self.user_pool_domains[domain] + user_pool_domain.custom_domain_config = custom_domain_config + return user_pool_domain + # User pool client def create_user_pool_client(self, user_pool_id, extended_config): user_pool = self.user_pools.get(user_pool_id) diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index e9e83695a..75dd8c181 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -50,7 +50,13 @@ class CognitoIdpResponse(BaseResponse): def create_user_pool_domain(self): domain = self._get_param("Domain") user_pool_id = self._get_param("UserPoolId") - cognitoidp_backends[self.region].create_user_pool_domain(user_pool_id, domain) + custom_domain_config = self._get_param("CustomDomainConfig") + user_pool_domain = cognitoidp_backends[self.region].create_user_pool_domain( + user_pool_id, domain, custom_domain_config + ) + domain_description = user_pool_domain.to_json(extended=False) + if domain_description: + return json.dumps(domain_description) return "" def describe_user_pool_domain(self): @@ -69,6 +75,17 @@ class CognitoIdpResponse(BaseResponse): cognitoidp_backends[self.region].delete_user_pool_domain(domain) return "" + def update_user_pool_domain(self): + domain = self._get_param("Domain") + custom_domain_config = self._get_param("CustomDomainConfig") + user_pool_domain = cognitoidp_backends[self.region].update_user_pool_domain( + domain, custom_domain_config + ) + domain_description = user_pool_domain.to_json(extended=False) + if domain_description: + return json.dumps(domain_description) + return "" + # User pool client def create_user_pool_client(self): user_pool_id = self.parameters.pop("UserPoolId") diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 1483fcd0e..774ff7621 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -133,6 +133,22 @@ def test_create_user_pool_domain(): result["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) +@mock_cognitoidp +def test_create_user_pool_domain_custom_domain_config(): + conn = boto3.client("cognito-idp", "us-west-2") + + domain = str(uuid.uuid4()) + custom_domain_config = { + "CertificateArn": "arn:aws:acm:us-east-1:123456789012:certificate/123456789012", + } + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + result = conn.create_user_pool_domain( + UserPoolId=user_pool_id, Domain=domain, CustomDomainConfig=custom_domain_config + ) + result["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + result["CloudFrontDomain"].should.equal("e2c343b3293ee505.cloudfront.net") + + @mock_cognitoidp def test_describe_user_pool_domain(): conn = boto3.client("cognito-idp", "us-west-2") @@ -162,6 +178,23 @@ def test_delete_user_pool_domain(): result["DomainDescription"].keys().should.have.length_of(0) +@mock_cognitoidp +def test_update_user_pool_domain(): + conn = boto3.client("cognito-idp", "us-west-2") + + domain = str(uuid.uuid4()) + custom_domain_config = { + "CertificateArn": "arn:aws:acm:us-east-1:123456789012:certificate/123456789012", + } + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.create_user_pool_domain(UserPoolId=user_pool_id, Domain=domain) + result = conn.update_user_pool_domain( + UserPoolId=user_pool_id, Domain=domain, CustomDomainConfig=custom_domain_config + ) + result["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + result["CloudFrontDomain"].should.equal("e2c343b3293ee505.cloudfront.net") + + @mock_cognitoidp def test_create_user_pool_client(): conn = boto3.client("cognito-idp", "us-west-2") From 7c17fcd21d1ee09bce1c12d31c1a631af1d062ec Mon Sep 17 00:00:00 2001 From: Carlos Aguado Date: Tue, 16 Jul 2019 13:20:31 +1000 Subject: [PATCH 20/63] Implement get_open_id_token Introduce the CognitoIdentity's GetOpenIDToken endpoint to retrieve a JWT tuple from Cognito's Identity Pool for a given IdentityId. --- IMPLEMENTATION_COVERAGE.md | 10 +++++----- moto/cognitoidentity/models.py | 9 +++++++++ moto/cognitoidentity/responses.py | 5 +++++ tests/test_cognitoidentity/test_cognitoidentity.py | 14 +++++++++++++- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 685db7ec4..5c5cebd7b 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -815,16 +815,16 @@ - [ ] update_user_profile ## cognito-identity - 0% implemented -- [ ] create_identity_pool +- [X] create_identity_pool - [ ] delete_identities - [ ] delete_identity_pool - [ ] describe_identity - [ ] describe_identity_pool -- [ ] get_credentials_for_identity -- [ ] get_id +- [X] get_credentials_for_identity +- [X] get_id - [ ] get_identity_pool_roles -- [ ] get_open_id_token -- [ ] get_open_id_token_for_developer_identity +- [X] get_open_id_token +- [X] get_open_id_token_for_developer_identity - [ ] list_identities - [ ] list_identity_pools - [ ] lookup_developer_identity diff --git a/moto/cognitoidentity/models.py b/moto/cognitoidentity/models.py index daa2a4641..c916b7f62 100644 --- a/moto/cognitoidentity/models.py +++ b/moto/cognitoidentity/models.py @@ -95,6 +95,15 @@ class CognitoIdentityBackend(BaseBackend): }) return response + def get_open_id_token(self, identity_id): + response = json.dumps( + { + "IdentityId": identity_id, + "Token": get_random_identity_id(self.region) + } + ) + return response + cognitoidentity_backends = {} for region in boto.cognito.identity.regions(): diff --git a/moto/cognitoidentity/responses.py b/moto/cognitoidentity/responses.py index e7b428329..33faaa300 100644 --- a/moto/cognitoidentity/responses.py +++ b/moto/cognitoidentity/responses.py @@ -35,3 +35,8 @@ class CognitoIdentityResponse(BaseResponse): return cognitoidentity_backends[self.region].get_open_id_token_for_developer_identity( self._get_param('IdentityId') or get_random_identity_id(self.region) ) + + def get_open_id_token(self): + return cognitoidentity_backends[self.region].get_open_id_token( + self._get_param("IdentityId") or get_random_identity_id(self.region) + ) diff --git a/tests/test_cognitoidentity/test_cognitoidentity.py b/tests/test_cognitoidentity/test_cognitoidentity.py index ac79fa223..ea9ccbc78 100644 --- a/tests/test_cognitoidentity/test_cognitoidentity.py +++ b/tests/test_cognitoidentity/test_cognitoidentity.py @@ -68,7 +68,7 @@ def test_get_open_id_token_for_developer_identity(): }, TokenDuration=123 ) - assert len(result['Token']) + assert len(result['Token']) > 0 assert result['IdentityId'] == '12345' @mock_cognitoidentity @@ -83,3 +83,15 @@ def test_get_open_id_token_for_developer_identity_when_no_explicit_identity_id() ) assert len(result['Token']) > 0 assert len(result['IdentityId']) > 0 + +@mock_cognitoidentity +def test_get_open_id_token(): + conn = boto3.client('cognito-identity', 'us-west-2') + result = conn.get_open_id_token( + IdentityId='12345', + Logins={ + 'someurl': '12345' + } + ) + assert len(result['Token']) > 0 + assert result['IdentityId'] == '12345' From e54f74776b0b69a3fa848dba946951a532b30aa1 Mon Sep 17 00:00:00 2001 From: Carlos Aguado Date: Tue, 16 Jul 2019 13:27:47 +1000 Subject: [PATCH 21/63] Implement assume_role_with_web_identity The AssumeRoleWithWebIdentity is a similar endpoint to STS's AssumeRole where the authentication element is a JWT id_token from a configured OP. This commit implements the functionality and relies on the same result generated for the regular AssumeRole. --- IMPLEMENTATION_COVERAGE.md | 2 +- moto/sts/models.py | 3 +++ moto/sts/responses.py | 39 ++++++++++++++++++++++++++++++++++++++ tests/test_sts/test_sts.py | 35 ++++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 1 deletion(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 685db7ec4..fcf1463c5 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -4127,7 +4127,7 @@ ## sts - 42% implemented - [X] assume_role - [ ] assume_role_with_saml -- [ ] assume_role_with_web_identity +- [X] assume_role_with_web_identity - [ ] decode_authorization_message - [ ] get_caller_identity - [X] get_federation_token diff --git a/moto/sts/models.py b/moto/sts/models.py index 983db5602..9ad87358f 100644 --- a/moto/sts/models.py +++ b/moto/sts/models.py @@ -50,5 +50,8 @@ class STSBackend(BaseBackend): role = AssumedRole(**kwargs) return role + def assume_role_with_web_identity(self, **kwargs): + return self.assume_role(**kwargs) + sts_backend = STSBackend() diff --git a/moto/sts/responses.py b/moto/sts/responses.py index 24ec181ba..fd71a963f 100644 --- a/moto/sts/responses.py +++ b/moto/sts/responses.py @@ -39,6 +39,24 @@ class TokenResponse(BaseResponse): template = self.response_template(ASSUME_ROLE_RESPONSE) return template.render(role=role) + def assume_role_with_web_identity(self): + role_session_name = self.querystring.get('RoleSessionName')[0] + role_arn = self.querystring.get('RoleArn')[0] + + policy = self.querystring.get('Policy', [None])[0] + duration = int(self.querystring.get('DurationSeconds', [3600])[0]) + external_id = self.querystring.get('ExternalId', [None])[0] + + role = sts_backend.assume_role_with_web_identity( + role_session_name=role_session_name, + role_arn=role_arn, + policy=policy, + duration=duration, + external_id=external_id, + ) + template = self.response_template(ASSUME_ROLE_WITH_WEB_IDENTITY_RESPONSE) + return template.render(role=role) + def get_caller_identity(self): template = self.response_template(GET_CALLER_IDENTITY_RESPONSE) return template.render() @@ -100,6 +118,27 @@ ASSUME_ROLE_RESPONSE = """ + + + {{ role.session_token }} + {{ role.secret_access_key }} + {{ role.expiration_ISO8601 }} + {{ role.access_key_id }} + + + {{ role.arn }} + ARO123EXAMPLE123:{{ role.session_name }} + + 6 + + + c6104cbe-af31-11e0-8154-cbc7ccf896c7 + +""" + + GET_CALLER_IDENTITY_RESPONSE = """ arn:aws:sts::123456789012:user/moto diff --git a/tests/test_sts/test_sts.py b/tests/test_sts/test_sts.py index 566f17ffe..36c9da258 100644 --- a/tests/test_sts/test_sts.py +++ b/tests/test_sts/test_sts.py @@ -74,6 +74,41 @@ def test_assume_role(): role.user.assume_role_id.should.contain("session-name") +@freeze_time("2012-01-01 12:00:00") +@mock_sts_deprecated +def test_assume_role_with_web_identity(): + conn = boto.connect_sts() + + policy = json.dumps({ + "Statement": [ + { + "Sid": "Stmt13690092345534", + "Action": [ + "S3:ListBucket" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::foobar-tester" + ] + }, + ] + }) + s3_role = "arn:aws:iam::123456789012:role/test-role" + role = conn.assume_role_with_web_identity( + s3_role, "session-name", policy, duration_seconds=123) + + credentials = role.credentials + credentials.expiration.should.equal('2012-01-01T12:02:03.000Z') + credentials.session_token.should.have.length_of(356) + assert credentials.session_token.startswith("FQoGZXIvYXdzE") + credentials.access_key.should.have.length_of(20) + assert credentials.access_key.startswith("ASIA") + credentials.secret_key.should.have.length_of(40) + + role.user.arn.should.equal("arn:aws:iam::123456789012:role/test-role") + role.user.assume_role_id.should.contain("session-name") + + @mock_sts def test_get_caller_identity(): identity = boto3.client( From 1b3157ced0de580921a58a668b944619d0e17a77 Mon Sep 17 00:00:00 2001 From: Berislav Kovacki Date: Tue, 16 Jul 2019 09:12:03 +0200 Subject: [PATCH 22/63] Handle change of ASG desired capacity on min and max size update A change in UpdateAutoScalingGroup: * if a value for MinSize is specified without specifying a value for DesiredCapacity, and the new MinSize is larger than the current size of the group, set the group's DesiredCapacity to the new MinSize value * if a value for MaxSize is specified without specifying a value for DesiredCapacity, and the new MaxSize is smaller than the current size of the group, set the group's DesiredCapacity to the new MaxSize value --- moto/autoscaling/models.py | 6 +++ tests/test_autoscaling/test_autoscaling.py | 56 ++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 24811be73..2f477fbb8 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -279,6 +279,12 @@ class FakeAutoScalingGroup(BaseModel): if min_size is not None: self.min_size = min_size + if desired_capacity is None: + if min_size is not None and min_size > len(self.instance_states): + desired_capacity = min_size + if max_size is not None and max_size < len(self.instance_states): + desired_capacity = max_size + if launch_config_name: self.launch_config = self.autoscaling_backend.launch_configurations[ launch_config_name] diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index 750605c07..49396340c 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -823,6 +823,62 @@ def test_update_autoscaling_group_boto3(): group['NewInstancesProtectedFromScaleIn'].should.equal(False) +@mock_autoscaling +def test_update_autoscaling_group_min_size_desired_capacity_change(): + mocked_networking = setup_networking() + client = boto3.client('autoscaling', region_name='us-east-1') + + client.create_launch_configuration( + LaunchConfigurationName='test_launch_configuration' + ) + client.create_auto_scaling_group( + AutoScalingGroupName='test_asg', + LaunchConfigurationName='test_launch_configuration', + MinSize=2, + MaxSize=20, + DesiredCapacity=3, + VPCZoneIdentifier=mocked_networking['subnet1'], + ) + client.update_auto_scaling_group( + AutoScalingGroupName='test_asg', + MinSize=5, + ) + response = client.describe_auto_scaling_groups( + AutoScalingGroupNames=['test_asg']) + group = response['AutoScalingGroups'][0] + group['DesiredCapacity'].should.equal(5) + group['MinSize'].should.equal(5) + group['Instances'].should.have.length_of(5) + + +@mock_autoscaling +def test_update_autoscaling_group_max_size_desired_capacity_change(): + mocked_networking = setup_networking() + client = boto3.client('autoscaling', region_name='us-east-1') + + client.create_launch_configuration( + LaunchConfigurationName='test_launch_configuration' + ) + client.create_auto_scaling_group( + AutoScalingGroupName='test_asg', + LaunchConfigurationName='test_launch_configuration', + MinSize=2, + MaxSize=20, + DesiredCapacity=10, + VPCZoneIdentifier=mocked_networking['subnet1'], + ) + client.update_auto_scaling_group( + AutoScalingGroupName='test_asg', + MaxSize=5, + ) + response = client.describe_auto_scaling_groups( + AutoScalingGroupNames=['test_asg']) + group = response['AutoScalingGroups'][0] + group['DesiredCapacity'].should.equal(5) + group['MaxSize'].should.equal(5) + group['Instances'].should.have.length_of(5) + + @mock_autoscaling def test_autoscaling_taqs_update_boto3(): mocked_networking = setup_networking() From 19fef76a5faa8cc5e6df5e82a40016f8b4ad91f6 Mon Sep 17 00:00:00 2001 From: Carlos Aguado Date: Wed, 17 Jul 2019 08:47:26 +1000 Subject: [PATCH 23/63] Fix moto_server handling of unsigned requests Certain AWS requests are unsigned. Moto in standalone server mode implements an heuristic to deduce the endpoint and region based on the X-Amz-Target HTTP header. This commit extends this concept to add additional endpoints that used unsigned requests at times. --- moto/server.py | 50 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/moto/server.py b/moto/server.py index 5ad02d383..971589cac 100644 --- a/moto/server.py +++ b/moto/server.py @@ -21,6 +21,16 @@ from moto.core.utils import convert_flask_to_httpretty_response HTTP_METHODS = ["GET", "POST", "PUT", "DELETE", "HEAD", "PATCH"] +DEFAULT_SERVICE_REGION = ('s3', 'us-east-1') + +# Map of unsigned calls to service-region as per AWS API docs +# https://docs.aws.amazon.com/cognito/latest/developerguide/resource-permissions.html#amazon-cognito-signed-versus-unsigned-apis +UNSIGNED_REQUESTS = { + 'AWSCognitoIdentityService': ('cognito-identity', 'us-east-1'), + 'AWSCognitoIdentityProviderService': ('cognito-idp', 'us-east-1'), +} + + class DomainDispatcherApplication(object): """ Dispatch requests to different applications based on the "Host:" header @@ -50,6 +60,32 @@ class DomainDispatcherApplication(object): raise RuntimeError('Invalid host: "%s"' % host) + def infer_service_region(self, environ): + auth = environ.get('HTTP_AUTHORIZATION') + if auth: + # Signed request + # Parse auth header to find service assuming a SigV4 request + # https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html + # ['Credential=sdffdsa', '20170220', 'us-east-1', 'sns', 'aws4_request'] + try: + credential_scope = auth.split(",")[0].split()[1] + _, _, region, service, _ = credential_scope.split("/") + return service, region + except ValueError: + # Signature format does not match, this is exceptional and we can't + # infer a service-region. A reduced set of services still use + # the deprecated SigV2, ergo prefer S3 as most likely default. + # https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html + return DEFAULT_SERVICE_REGION + else: + # Unsigned request + target = environ.get('HTTP_X_AMZ_TARGET') + if target: + service, _ = target.split('.', 1) + return UNSIGNED_REQUESTS.get(service, DEFAULT_SERVICE_REGION) + # S3 is the last resort when the target is also unknown + return DEFAULT_SERVICE_REGION + def get_application(self, environ): path_info = environ.get('PATH_INFO', '') @@ -66,19 +102,7 @@ class DomainDispatcherApplication(object): else: host = environ['HTTP_HOST'].split(':')[0] if host in {'localhost', 'motoserver'} or host.startswith("192.168."): - # Fall back to parsing auth header to find service - # ['Credential=sdffdsa', '20170220', 'us-east-1', 'sns', 'aws4_request'] - try: - _, _, region, service, _ = environ['HTTP_AUTHORIZATION'].split(",")[0].split()[ - 1].split("/") - except (KeyError, ValueError): - # Some cognito-idp endpoints (e.g. change password) do not receive an auth header. - if environ.get('HTTP_X_AMZ_TARGET', '').startswith('AWSCognitoIdentityProviderService'): - service = 'cognito-idp' - else: - service = 's3' - - region = 'us-east-1' + service, region = self.infer_service_region(environ) if service == 'dynamodb': if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'): host = 'dynamodbstreams' From 5d91ce20fc4e48a42810149e1faf455d38a6f456 Mon Sep 17 00:00:00 2001 From: Jongseok Choi Date: Thu, 18 Jul 2019 03:09:53 +0900 Subject: [PATCH 24/63] Fix 'MaxRecords' type issues by _get_param() It alternates _get_param() with _get_int_param() on parsing the parameter 'MaxRecords'. --- moto/autoscaling/responses.py | 2 +- moto/rds/responses.py | 2 +- moto/rds2/responses.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index 985c6f852..8dc3e4b27 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -48,7 +48,7 @@ class AutoScalingResponse(BaseResponse): start = all_names.index(marker) + 1 else: start = 0 - max_records = self._get_param('MaxRecords', 50) # the default is 100, but using 50 to make testing easier + max_records = self._get_int_param('MaxRecords', 50) # the default is 100, but using 50 to make testing easier launch_configurations_resp = all_launch_configurations[start:start + max_records] next_token = None if len(all_launch_configurations) > start + max_records: diff --git a/moto/rds/responses.py b/moto/rds/responses.py index 987a6f21a..0afb03979 100644 --- a/moto/rds/responses.py +++ b/moto/rds/responses.py @@ -95,7 +95,7 @@ class RDSResponse(BaseResponse): start = all_ids.index(marker) + 1 else: start = 0 - page_size = self._get_param('MaxRecords', 50) # the default is 100, but using 50 to make testing easier + page_size = self._get_int_param('MaxRecords', 50) # the default is 100, but using 50 to make testing easier instances_resp = all_instances[start:start + page_size] next_marker = None if len(all_instances) > start + page_size: diff --git a/moto/rds2/responses.py b/moto/rds2/responses.py index 75d4f4ce0..7b8d0b63a 100644 --- a/moto/rds2/responses.py +++ b/moto/rds2/responses.py @@ -280,7 +280,7 @@ class RDS2Response(BaseResponse): def describe_option_groups(self): kwargs = self._get_option_group_kwargs() - kwargs['max_records'] = self._get_param('MaxRecords') + kwargs['max_records'] = self._get_int_param('MaxRecords') kwargs['marker'] = self._get_param('Marker') option_groups = self.backend.describe_option_groups(kwargs) template = self.response_template(DESCRIBE_OPTION_GROUP_TEMPLATE) @@ -329,7 +329,7 @@ class RDS2Response(BaseResponse): def describe_db_parameter_groups(self): kwargs = self._get_db_parameter_group_kwargs() - kwargs['max_records'] = self._get_param('MaxRecords') + kwargs['max_records'] = self._get_int_param('MaxRecords') kwargs['marker'] = self._get_param('Marker') db_parameter_groups = self.backend.describe_db_parameter_groups(kwargs) template = self.response_template( From 4ca0bd807f05657894fad11c8bb25bc6f8c91afc Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 17 Jul 2019 20:15:59 +0200 Subject: [PATCH 25/63] Created tests for calling CreateAutoScalingGroup with an instance id. --- tests/test_autoscaling/test_autoscaling.py | 65 +++++++++++++++++++++- tests/test_autoscaling/utils.py | 15 +++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/tests/test_autoscaling/test_autoscaling.py b/tests/test_autoscaling/test_autoscaling.py index 750605c07..ab894e324 100644 --- a/tests/test_autoscaling/test_autoscaling.py +++ b/tests/test_autoscaling/test_autoscaling.py @@ -7,11 +7,13 @@ from boto.ec2.autoscale.group import AutoScalingGroup from boto.ec2.autoscale import Tag import boto.ec2.elb import sure # noqa +from botocore.exceptions import ClientError +from nose.tools import assert_raises from moto import mock_autoscaling, mock_ec2_deprecated, mock_elb_deprecated, mock_elb, mock_autoscaling_deprecated, mock_ec2 from tests.helpers import requires_boto_gte -from utils import setup_networking, setup_networking_deprecated +from utils import setup_networking, setup_networking_deprecated, setup_instance_with_networking @mock_autoscaling_deprecated @@ -724,6 +726,67 @@ def test_create_autoscaling_group_boto3(): response['ResponseMetadata']['HTTPStatusCode'].should.equal(200) +@mock_autoscaling +def test_create_autoscaling_group_from_instance(): + autoscaling_group_name = 'test_asg' + image_id = 'ami-0cc293023f983ed53' + instance_type = 't2.micro' + + mocked_instance_with_networking = setup_instance_with_networking(image_id, instance_type) + client = boto3.client('autoscaling', region_name='us-east-1') + response = client.create_auto_scaling_group( + AutoScalingGroupName=autoscaling_group_name, + InstanceId=mocked_instance_with_networking['instance'], + MinSize=1, + MaxSize=3, + DesiredCapacity=2, + Tags=[ + {'ResourceId': 'test_asg', + 'ResourceType': 'auto-scaling-group', + 'Key': 'propogated-tag-key', + 'Value': 'propogate-tag-value', + 'PropagateAtLaunch': True + }, + {'ResourceId': 'test_asg', + 'ResourceType': 'auto-scaling-group', + 'Key': 'not-propogated-tag-key', + 'Value': 'not-propogate-tag-value', + 'PropagateAtLaunch': False + }], + VPCZoneIdentifier=mocked_instance_with_networking['subnet1'], + NewInstancesProtectedFromScaleIn=False, + ) + response['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + + describe_launch_configurations_response = client.describe_launch_configurations() + describe_launch_configurations_response['LaunchConfigurations'].should.have.length_of(1) + launch_configuration_from_instance = describe_launch_configurations_response['LaunchConfigurations'][0] + launch_configuration_from_instance['LaunchConfigurationName'].should.equal('test_asg') + launch_configuration_from_instance['ImageId'].should.equal(image_id) + launch_configuration_from_instance['InstanceType'].should.equal(instance_type) + + +@mock_autoscaling +def test_create_autoscaling_group_from_invalid_instance_id(): + invalid_instance_id = 'invalid_instance' + + mocked_networking = setup_networking() + client = boto3.client('autoscaling', region_name='us-east-1') + with assert_raises(ClientError) as ex: + client.create_auto_scaling_group( + AutoScalingGroupName='test_asg', + InstanceId=invalid_instance_id, + MinSize=9, + MaxSize=15, + DesiredCapacity=12, + VPCZoneIdentifier=mocked_networking['subnet1'], + NewInstancesProtectedFromScaleIn=False, + ) + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + ex.exception.response['Error']['Code'].should.equal('ValidationError') + ex.exception.response['Error']['Message'].should.equal('Instance [{0}] is invalid.'.format(invalid_instance_id)) + + @mock_autoscaling def test_describe_autoscaling_groups_boto3(): mocked_networking = setup_networking() diff --git a/tests/test_autoscaling/utils.py b/tests/test_autoscaling/utils.py index ebbffbed3..dc38aba3d 100644 --- a/tests/test_autoscaling/utils.py +++ b/tests/test_autoscaling/utils.py @@ -31,3 +31,18 @@ def setup_networking_deprecated(): "10.11.2.0/24", availability_zone='us-east-1b') return {'vpc': vpc.id, 'subnet1': subnet1.id, 'subnet2': subnet2.id} + + +@mock_ec2 +def setup_instance_with_networking(image_id, instance_type): + mock_data = setup_networking() + ec2 = boto3.resource('ec2', region_name='us-east-1') + instances = ec2.create_instances( + ImageId=image_id, + InstanceType=instance_type, + MaxCount=1, + MinCount=1, + SubnetId=mock_data['subnet1'] + ) + mock_data['instance'] = instances[0].id + return mock_data From c8abd43c881e8f48fd2c82dca0f781aa61662f5c Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 17 Jul 2019 20:58:23 +0200 Subject: [PATCH 26/63] Implemented creating autoscaling group from instance. --- moto/autoscaling/exceptions.py | 9 +++++++ moto/autoscaling/models.py | 45 ++++++++++++++++++++++++++++------ moto/autoscaling/responses.py | 1 + moto/ec2/models.py | 10 ++++---- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/moto/autoscaling/exceptions.py b/moto/autoscaling/exceptions.py index 7dd81e0d6..74f62241d 100644 --- a/moto/autoscaling/exceptions.py +++ b/moto/autoscaling/exceptions.py @@ -13,3 +13,12 @@ class ResourceContentionError(RESTError): super(ResourceContentionError, self).__init__( "ResourceContentionError", "You already have a pending update to an Auto Scaling resource (for example, a group, instance, or load balancer).") + + +class InvalidInstanceError(AutoscalingClientError): + + def __init__(self, instance_id): + super(InvalidInstanceError, self).__init__( + "ValidationError", + "Instance [{0}] is invalid." + .format(instance_id)) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 24811be73..56ee0b4ab 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import random from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping +from moto.ec2.exceptions import InvalidInstanceIdError + from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.ec2 import ec2_backends @@ -10,7 +12,7 @@ from moto.elb import elb_backends from moto.elbv2 import elbv2_backends from moto.elb.exceptions import LoadBalancerNotFoundError from .exceptions import ( - AutoscalingClientError, ResourceContentionError, + AutoscalingClientError, ResourceContentionError, InvalidInstanceError ) # http://docs.aws.amazon.com/AutoScaling/latest/DeveloperGuide/AS_Concepts.html#Cooldown @@ -73,6 +75,26 @@ class FakeLaunchConfiguration(BaseModel): self.associate_public_ip_address = associate_public_ip_address self.block_device_mapping_dict = block_device_mapping_dict + @classmethod + def create_from_instance(cls, name, instance, backend): + config = backend.create_launch_configuration( + name=name, + image_id=instance.image_id, + kernel_id='', + ramdisk_id='', + key_name=instance.key_name, + security_groups=instance.security_groups, + user_data=instance.user_data, + instance_type=instance.instance_type, + instance_monitoring=False, + instance_profile_name=None, + spot_price=None, + ebs_optimized=instance.ebs_optimized, + associate_public_ip_address=instance.associate_public_ip, + block_device_mappings=instance.block_device_mapping + ) + return config + @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): properties = cloudformation_json['Properties'] @@ -408,13 +430,13 @@ class AutoScalingBackend(BaseBackend): self.launch_configurations.pop(launch_configuration_name, None) def create_auto_scaling_group(self, name, availability_zones, - desired_capacity, max_size, min_size, - launch_config_name, vpc_zone_identifier, - default_cooldown, health_check_period, - health_check_type, load_balancers, - target_group_arns, placement_group, - termination_policies, tags, - new_instances_protected_from_scale_in=False): + desired_capacity, max_size, min_size, + instance_id, launch_config_name, + vpc_zone_identifier, default_cooldown, + health_check_period, health_check_type, + load_balancers, target_group_arns, + placement_group,termination_policies, tags, + new_instances_protected_from_scale_in=False): def make_int(value): return int(value) if value is not None else value @@ -427,6 +449,13 @@ class AutoScalingBackend(BaseBackend): health_check_period = 300 else: health_check_period = make_int(health_check_period) + if launch_config_name is None and instance_id is not None: + try: + instance = self.ec2_backend.get_instance(instance_id) + launch_config_name = name + FakeLaunchConfiguration.create_from_instance(launch_config_name, instance, self) + except InvalidInstanceIdError: + raise InvalidInstanceError(instance_id) group = FakeAutoScalingGroup( name=name, diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index 985c6f852..7413e2f78 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -74,6 +74,7 @@ class AutoScalingResponse(BaseResponse): desired_capacity=self._get_int_param('DesiredCapacity'), max_size=self._get_int_param('MaxSize'), min_size=self._get_int_param('MinSize'), + instance_id=self._get_param('InstanceId'), launch_config_name=self._get_param('LaunchConfigurationName'), vpc_zone_identifier=self._get_param('VPCZoneIdentifier'), default_cooldown=self._get_int_param('DefaultCooldown'), diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 811283fe8..fa5994ee2 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -406,10 +406,10 @@ class Instance(TaggedEC2Resource, BotoInstance): self.ami_launch_index = kwargs.get("ami_launch_index", 0) self.disable_api_termination = kwargs.get("disable_api_termination", False) self._spot_fleet_id = kwargs.get("spot_fleet_id", None) - associate_public_ip = kwargs.get("associate_public_ip", False) + self.associate_public_ip = kwargs.get("associate_public_ip", False) if in_ec2_classic: # If we are in EC2-Classic, autoassign a public IP - associate_public_ip = True + self.associate_public_ip = True amis = self.ec2_backend.describe_images(filters={'image-id': image_id}) ami = amis[0] if amis else None @@ -440,9 +440,9 @@ class Instance(TaggedEC2Resource, BotoInstance): self.vpc_id = subnet.vpc_id self._placement.zone = subnet.availability_zone - if associate_public_ip is None: + if self.associate_public_ip is None: # Mapping public ip hasnt been explicitly enabled or disabled - associate_public_ip = subnet.map_public_ip_on_launch == 'true' + self.associate_public_ip = subnet.map_public_ip_on_launch == 'true' elif placement: self._placement.zone = placement else: @@ -454,7 +454,7 @@ class Instance(TaggedEC2Resource, BotoInstance): self.prep_nics( kwargs.get("nics", {}), private_ip=kwargs.get("private_ip"), - associate_public_ip=associate_public_ip + associate_public_ip=self.associate_public_ip ) def __del__(self): From c8d8aa4dd0acf3ba6c62bbf57fa81943d6700f0c Mon Sep 17 00:00:00 2001 From: Don Kuntz Date: Wed, 17 Jul 2019 14:07:19 -0500 Subject: [PATCH 27/63] Add glue.batch_get_partition endpoint --- moto/glue/responses.py | 17 +++++++++ tests/test_glue/test_datacatalog.py | 57 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/moto/glue/responses.py b/moto/glue/responses.py index cb1ecf519..875513e7f 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -141,6 +141,23 @@ class GlueResponse(BaseResponse): return json.dumps({'Partition': p.as_dict()}) + def batch_get_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + partitions_to_get = self.parameters.get('PartitionsToGet') + + table = self.glue_backend.get_table(database_name, table_name) + + partitions = [] + for values in partitions_to_get: + try: + p = table.get_partition(values=values["Values"]) + partitions.append(p.as_dict()) + except PartitionNotFoundException: + continue + + return json.dumps({'Partitions': partitions}) + def create_partition(self): database_name = self.parameters.get('DatabaseName') table_name = self.parameters.get('TableName') diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index 232ab3019..9034feb55 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -419,6 +419,63 @@ def test_get_partition(): partition['Values'].should.equal(values[1]) +@mock_glue +def test_batch_get_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + values = [['2018-10-01'], ['2018-09-01']] + + helpers.create_partition(client, database_name, table_name, values=values[0]) + helpers.create_partition(client, database_name, table_name, values=values[1]) + + partitions_to_get = [ + {'Values': values[0]}, + {'Values': values[1]}, + ] + response = client.batch_get_partition(DatabaseName=database_name, TableName=table_name, PartitionsToGet=partitions_to_get) + + partitions = response['Partitions'] + partitions.should.have.length_of(2) + + partition = partitions[1] + partition['TableName'].should.equal(table_name) + partition['Values'].should.equal(values[1]) + + +@mock_glue +def test_batch_get_partition_missing_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + values = [['2018-10-01'], ['2018-09-01'], ['2018-08-01']] + + helpers.create_partition(client, database_name, table_name, values=values[0]) + helpers.create_partition(client, database_name, table_name, values=values[2]) + + partitions_to_get = [ + {'Values': values[0]}, + {'Values': values[1]}, + {'Values': values[2]}, + ] + response = client.batch_get_partition(DatabaseName=database_name, TableName=table_name, PartitionsToGet=partitions_to_get) + + partitions = response['Partitions'] + partitions.should.have.length_of(2) + + partitions[0]['Values'].should.equal(values[0]) + partitions[1]['Values'].should.equal(values[2]) + + + @mock_glue def test_update_partition_not_found_moving(): client = boto3.client('glue', region_name='us-east-1') From 4834fc41c62421f1f33df027cddaba1b7ea8b222 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 17 Jul 2019 21:15:31 +0200 Subject: [PATCH 28/63] Fixed a linting error. --- moto/autoscaling/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 56ee0b4ab..ff1be9e56 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -435,7 +435,7 @@ class AutoScalingBackend(BaseBackend): vpc_zone_identifier, default_cooldown, health_check_period, health_check_type, load_balancers, target_group_arns, - placement_group,termination_policies, tags, + placement_group, termination_policies, tags, new_instances_protected_from_scale_in=False): def make_int(value): From e4374431d67c6de76a55bbc4d0e9a1de742f9919 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Wed, 17 Jul 2019 21:46:07 +0200 Subject: [PATCH 29/63] Fixed instance_id not set to a default value in create_auto_scaling_group. --- moto/autoscaling/models.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index ff1be9e56..15831ca05 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -430,13 +430,14 @@ class AutoScalingBackend(BaseBackend): self.launch_configurations.pop(launch_configuration_name, None) def create_auto_scaling_group(self, name, availability_zones, - desired_capacity, max_size, min_size, - instance_id, launch_config_name, - vpc_zone_identifier, default_cooldown, - health_check_period, health_check_type, - load_balancers, target_group_arns, - placement_group, termination_policies, tags, - new_instances_protected_from_scale_in=False): + desired_capacity, max_size, min_size, + launch_config_name, vpc_zone_identifier, + default_cooldown, health_check_period, + health_check_type, load_balancers, + target_group_arns, placement_group, + termination_policies, tags, + new_instances_protected_from_scale_in=False, + instance_id=None): def make_int(value): return int(value) if value is not None else value From a2ac341e3db4270acd5abdc5bc20ebf0bdcc4510 Mon Sep 17 00:00:00 2001 From: sblumin Date: Wed, 17 Jul 2019 16:37:47 -0700 Subject: [PATCH 30/63] added support to UPSERT records that are not simple routing policy --- moto/route53/models.py | 2 +- tests/test_route53/test_route53.py | 108 +++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/moto/route53/models.py b/moto/route53/models.py index 681a9d6ff..61a6609aa 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -198,7 +198,7 @@ class FakeZone(BaseModel): def upsert_rrset(self, record_set): new_rrset = RecordSet(record_set) for i, rrset in enumerate(self.rrsets): - if rrset.name == new_rrset.name and rrset.type_ == new_rrset.type_: + if rrset.name == new_rrset.name and rrset.type_ == new_rrset.type_ and rrset.set_identifier == new_rrset.set_identifier: self.rrsets[i] = new_rrset break else: diff --git a/tests/test_route53/test_route53.py b/tests/test_route53/test_route53.py index ca652af88..de9465d6d 100644 --- a/tests/test_route53/test_route53.py +++ b/tests/test_route53/test_route53.py @@ -652,6 +652,114 @@ def test_change_resource_record_sets_crud_valid(): response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) len(response['ResourceRecordSets']).should.equal(0) +@mock_route53 +def test_change_weighted_resource_record_sets(): + conn = boto3.client('route53', region_name='us-east-2') + conn.create_hosted_zone( + Name='test.vpc.internal.', + CallerReference=str(hash('test')) + ) + + zones = conn.list_hosted_zones_by_name( + DNSName='test.vpc.internal.' + ) + + hosted_zone_id = zones['HostedZones'][0]['Id'] + + #Create 2 weighted records + conn.change_resource_record_sets( + HostedZoneId=hosted_zone_id, + ChangeBatch={ + 'Changes': [ + { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'Name': 'test.vpc.internal', + 'Type': 'A', + 'SetIdentifier': 'test1', + 'Weight': 50, + 'AliasTarget': { + 'HostedZoneId': 'Z3AADJGX6KTTL2', + 'DNSName': 'internal-test1lb-447688172.us-east-2.elb.amazonaws.com.', + 'EvaluateTargetHealth': True + } + } + }, + + { + 'Action': 'CREATE', + 'ResourceRecordSet': { + 'Name': 'test.vpc.internal', + 'Type': 'A', + 'SetIdentifier': 'test2', + 'Weight': 50, + 'AliasTarget': { + 'HostedZoneId': 'Z3AADJGX6KTTL2', + 'DNSName': 'internal-testlb2-1116641781.us-east-2.elb.amazonaws.com.', + 'EvaluateTargetHealth': True + } + } + } + ] + } + ) + + response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) + record = response['ResourceRecordSets'][0] + #Update the first record to have a weight of 90 + conn.change_resource_record_sets( + HostedZoneId=hosted_zone_id, + ChangeBatch={ + 'Changes' : [ + { + 'Action' : 'UPSERT', + 'ResourceRecordSet' : { + 'Name' : record['Name'], + 'Type' : record['Type'], + 'SetIdentifier' : record['SetIdentifier'], + 'Weight' : 90, + 'AliasTarget' : { + 'HostedZoneId' : record['AliasTarget']['HostedZoneId'], + 'DNSName' : record['AliasTarget']['DNSName'], + 'EvaluateTargetHealth' : record['AliasTarget']['EvaluateTargetHealth'] + } + } + }, + ] + } + ) + + record = response['ResourceRecordSets'][1] + #Update the second record to have a weight of 10 + conn.change_resource_record_sets( + HostedZoneId=hosted_zone_id, + ChangeBatch={ + 'Changes' : [ + { + 'Action' : 'UPSERT', + 'ResourceRecordSet' : { + 'Name' : record['Name'], + 'Type' : record['Type'], + 'SetIdentifier' : record['SetIdentifier'], + 'Weight' : 10, + 'AliasTarget' : { + 'HostedZoneId' : record['AliasTarget']['HostedZoneId'], + 'DNSName' : record['AliasTarget']['DNSName'], + 'EvaluateTargetHealth' : record['AliasTarget']['EvaluateTargetHealth'] + } + } + }, + ] + } + ) + + response = conn.list_resource_record_sets(HostedZoneId=hosted_zone_id) + for record in response['ResourceRecordSets']: + if record['SetIdentifier'] == 'test1': + record['Weight'].should.equal(90) + if record['SetIdentifier'] == 'test2': + record['Weight'].should.equal(10) + @mock_route53 def test_change_resource_record_invalid(): From 44e93c94e83e88d26d381d32bdc525f610577a82 Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 18 Jul 2019 16:25:41 +0200 Subject: [PATCH 31/63] Created test for describing health of a stopped instance target. --- tests/test_elbv2/test_elbv2.py | 85 ++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/test_elbv2/test_elbv2.py b/tests/test_elbv2/test_elbv2.py index 03273ad3a..879a04cd8 100644 --- a/tests/test_elbv2/test_elbv2.py +++ b/tests/test_elbv2/test_elbv2.py @@ -667,6 +667,91 @@ def test_register_targets(): response.get('TargetHealthDescriptions').should.have.length_of(1) +@mock_ec2 +@mock_elbv2 +def test_stopped_instance_target(): + target_group_port = 8080 + + conn = boto3.client('elbv2', region_name='us-east-1') + ec2 = boto3.resource('ec2', region_name='us-east-1') + + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') + vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.0/26', + AvailabilityZone='us-east-1b') + + conn.create_load_balancer( + Name='my-lb', + Subnets=[subnet1.id, subnet2.id], + SecurityGroups=[security_group.id], + Scheme='internal', + Tags=[{'Key': 'key_name', 'Value': 'a_value'}]) + + response = conn.create_target_group( + Name='a-target', + Protocol='HTTP', + Port=target_group_port, + VpcId=vpc.id, + HealthCheckProtocol='HTTP', + HealthCheckPath='/', + HealthCheckIntervalSeconds=5, + HealthCheckTimeoutSeconds=5, + HealthyThresholdCount=5, + UnhealthyThresholdCount=2, + Matcher={'HttpCode': '200'}) + target_group = response.get('TargetGroups')[0] + + # No targets registered yet + response = conn.describe_target_health( + TargetGroupArn=target_group.get('TargetGroupArn')) + response.get('TargetHealthDescriptions').should.have.length_of(0) + + response = ec2.create_instances( + ImageId='ami-1234abcd', MinCount=1, MaxCount=1) + instance = response[0] + + target_dict = { + 'Id': instance.id, + 'Port': 500 + } + + response = conn.register_targets( + TargetGroupArn=target_group.get('TargetGroupArn'), + Targets=[target_dict]) + + response = conn.describe_target_health( + TargetGroupArn=target_group.get('TargetGroupArn')) + response.get('TargetHealthDescriptions').should.have.length_of(1) + target_health_description = response.get('TargetHealthDescriptions')[0] + + target_health_description['Target'].should.equal(target_dict) + target_health_description['HealthCheckPort'].should.equal(str(target_group_port)) + target_health_description['TargetHealth'].should.equal({ + 'State': 'healthy' + }) + + instance.stop() + + response = conn.describe_target_health( + TargetGroupArn=target_group.get('TargetGroupArn')) + response.get('TargetHealthDescriptions').should.have.length_of(1) + target_health_description = response.get('TargetHealthDescriptions')[0] + target_health_description['Target'].should.equal(target_dict) + target_health_description['HealthCheckPort'].should.equal(str(target_group_port)) + target_health_description['TargetHealth'].should.equal({ + 'State': 'unused', + 'Reason': 'Target.InvalidState', + 'Description': 'Target is in the stopped state' + }) + + @mock_ec2 @mock_elbv2 def test_target_group_attributes(): From 9149852217e10ecf907382d5b5b2e167536336ad Mon Sep 17 00:00:00 2001 From: acsbendi Date: Thu, 18 Jul 2019 16:57:27 +0200 Subject: [PATCH 32/63] Implemented returning correct health for stopped instances. --- moto/elbv2/models.py | 13 +++++++++---- moto/elbv2/responses.py | 6 ++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index 8d98f187d..508541f91 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -35,12 +35,13 @@ from .exceptions import ( class FakeHealthStatus(BaseModel): - def __init__(self, instance_id, port, health_port, status, reason=None): + def __init__(self, instance_id, port, health_port, status, reason=None, description=None): self.instance_id = instance_id self.port = port self.health_port = health_port self.status = status self.reason = reason + self.description = description class FakeTargetGroup(BaseModel): @@ -69,7 +70,7 @@ class FakeTargetGroup(BaseModel): self.protocol = protocol self.port = port self.healthcheck_protocol = healthcheck_protocol or 'HTTP' - self.healthcheck_port = healthcheck_port or 'traffic-port' + self.healthcheck_port = healthcheck_port or str(self.port) self.healthcheck_path = healthcheck_path or '/' self.healthcheck_interval_seconds = healthcheck_interval_seconds or 30 self.healthcheck_timeout_seconds = healthcheck_timeout_seconds or 5 @@ -112,10 +113,14 @@ class FakeTargetGroup(BaseModel): raise TooManyTagsError() self.tags[key] = value - def health_for(self, target): + def health_for(self, target, ec2_backend): t = self.targets.get(target['id']) if t is None: raise InvalidTargetError() + if t['id'].startswith("i-"): # EC2 instance ID + instance = ec2_backend.get_instance_by_id(t['id']) + if instance.state == "stopped": + return FakeHealthStatus(t['id'], t['port'], self.healthcheck_port, 'unused', 'Target.InvalidState', 'Target is in the stopped state') return FakeHealthStatus(t['id'], t['port'], self.healthcheck_port, 'healthy') @classmethod @@ -712,7 +717,7 @@ class ELBv2Backend(BaseBackend): if not targets: targets = target_group.targets.values() - return [target_group.health_for(target) for target in targets] + return [target_group.health_for(target, self.ec2_backend) for target in targets] def set_rule_priorities(self, rule_priorities): # validate diff --git a/moto/elbv2/responses.py b/moto/elbv2/responses.py index 3ca53240b..c98435440 100644 --- a/moto/elbv2/responses.py +++ b/moto/elbv2/responses.py @@ -1208,6 +1208,12 @@ DESCRIBE_TARGET_HEALTH_TEMPLATE = """ MAX_FEDERATION_TOKEN_POLICY_LENGTH: + raise STSValidationError( + "1 validation error detected: Value " + "'{\"Version\": \"2012-10-17\", \"Statement\": [...]}' " + "at 'policy' failed to satisfy constraint: Member must have length less than or " + " equal to %s" % MAX_FEDERATION_TOKEN_POLICY_LENGTH + ) + name = self.querystring.get('Name')[0] token = sts_backend.get_federation_token( duration=duration, name=name, policy=policy) diff --git a/tests/test_sts/test_sts.py b/tests/test_sts/test_sts.py index 36c9da258..7d3586f2c 100644 --- a/tests/test_sts/test_sts.py +++ b/tests/test_sts/test_sts.py @@ -3,10 +3,13 @@ import json import boto import boto3 +from botocore.client import ClientError from freezegun import freeze_time +from nose.tools import assert_raises import sure # noqa from moto import mock_sts, mock_sts_deprecated +from moto.sts.responses import MAX_FEDERATION_TOKEN_POLICY_LENGTH @freeze_time("2012-01-01 12:00:00") @@ -95,7 +98,7 @@ def test_assume_role_with_web_identity(): }) s3_role = "arn:aws:iam::123456789012:role/test-role" role = conn.assume_role_with_web_identity( - s3_role, "session-name", policy, duration_seconds=123) + s3_role, "session-name", policy, duration_seconds=123) credentials = role.credentials credentials.expiration.should.equal('2012-01-01T12:02:03.000Z') @@ -117,3 +120,32 @@ def test_get_caller_identity(): identity['Arn'].should.equal('arn:aws:sts::123456789012:user/moto') identity['UserId'].should.equal('AKIAIOSFODNN7EXAMPLE') identity['Account'].should.equal('123456789012') + + +@mock_sts +def test_federation_token_with_too_long_policy(): + "Trying to get a federation token with a policy longer than 2048 character should fail" + cli = boto3.client("sts", region_name='us-east-1') + resource_tmpl = 'arn:aws:s3:::yyyy-xxxxx-cloud-default/my_default_folder/folder-name-%s/*' + statements = [] + for num in range(30): + statements.append( + { + 'Effect': 'Allow', + 'Action': ['s3:*'], + 'Resource': resource_tmpl % str(num) + } + ) + policy = { + 'Version': '2012-10-17', + 'Statement': statements + } + json_policy = json.dumps(policy) + assert len(json_policy) > MAX_FEDERATION_TOKEN_POLICY_LENGTH + + with assert_raises(ClientError) as exc: + cli.get_federation_token(Name='foo', DurationSeconds=3600, Policy=json_policy) + exc.exception.response['Error']['Code'].should.equal('ValidationError') + exc.exception.response['Error']['Message'].should.contain( + str(MAX_FEDERATION_TOKEN_POLICY_LENGTH) + ) From 615d427c6d552deae845caac44e3185269769fb8 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sat, 20 Jul 2019 16:26:24 +0300 Subject: [PATCH 36/63] tests: multipart: use REDUCED_PART_SIZE and variable part size There is no reason to use 5M chunks, especially with the reduced_part_size decorator. Also made part_size to be variable to add extra layer of testing to make sure that moto is handling that scenario correctly. Signed-off-by: Ruslan Kuprieiev --- tests/test_s3/test_s3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index b6129c542..e2bcb109d 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1683,7 +1683,7 @@ def test_boto3_multipart_part_size(): parts = [] n_parts = 10 for i in range(1, n_parts + 1): - part_size = 5 * 1024 * 1024 + part_size = REDUCED_PART_SIZE + i body = b'1' * part_size part = s3.upload_part( Bucket='mybucket', @@ -1704,7 +1704,7 @@ def test_boto3_multipart_part_size(): for i in range(1, n_parts + 1): obj = s3.head_object(Bucket='mybucket', Key='the-key', PartNumber=i) - assert obj["ContentLength"] == part_size + assert obj["ContentLength"] == REDUCED_PART_SIZE + i @mock_s3 From a2aefc49b486f8191541f033007b6b3cf81ed723 Mon Sep 17 00:00:00 2001 From: PND Date: Mon, 22 Jul 2019 00:34:20 +0900 Subject: [PATCH 37/63] Fixed `update_item` of DynamoDB to deal with the list type. --- moto/dynamodb2/models.py | 4 ++- tests/test_dynamodb2/test_dynamodb.py | 40 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 29e90e7dc..e868caaa8 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -298,7 +298,9 @@ class Item(BaseModel): new_value = list(update_action['Value'].values())[0] if action == 'PUT': # TODO deal with other types - if isinstance(new_value, list) or isinstance(new_value, set): + if isinstance(new_value, list): + self.attrs[attribute_name] = DynamoType({"L": new_value}) + elif isinstance(new_value, set): self.attrs[attribute_name] = DynamoType({"SS": new_value}) elif isinstance(new_value, dict): self.attrs[attribute_name] = DynamoType({"M": new_value}) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 7746cf66b..a8f73bee6 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1342,6 +1342,46 @@ def test_query_missing_expr_names(): resp['Items'][0]['client']['S'].should.equal('test2') +# https://github.com/spulec/moto/issues/2328 +@mock_dynamodb2 +def test_update_item_with_list(): + dynamodb = boto3.resource('dynamodb', region_name='us-east-1') + + # Create the DynamoDB table. + dynamodb.create_table( + TableName='Table', + KeySchema=[ + { + 'AttributeName': 'key', + 'KeyType': 'HASH' + } + ], + AttributeDefinitions=[ + { + 'AttributeName': 'key', + 'AttributeType': 'S' + }, + ], + ProvisionedThroughput={ + 'ReadCapacityUnits': 1, + 'WriteCapacityUnits': 1 + } + ) + table = dynamodb.Table('Table') + table.update_item( + Key={'key': 'the-key'}, + AttributeUpdates={ + 'list': {'Value': [1, 2], 'Action': 'PUT'} + } + ) + + resp = table.get_item(Key={'key': 'the-key'}) + resp['Item'].should.equal({ + 'key': 'the-key', + 'list': [1, 2] + }) + + # https://github.com/spulec/moto/issues/1342 @mock_dynamodb2 def test_update_item_on_map(): From e3eb4d1809149447c87a27fb47828a1ea95fc7d2 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 21 Jul 2019 21:31:19 -0500 Subject: [PATCH 38/63] Cleanup host parsing. --- moto/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/server.py b/moto/server.py index 971589cac..7e251e2f5 100644 --- a/moto/server.py +++ b/moto/server.py @@ -95,13 +95,14 @@ class DomainDispatcherApplication(object): if six.PY3 and isinstance(path_info, six.binary_type): path_info = path_info.decode('utf-8') + host = None if path_info.startswith("/moto-api") or path_info == "/favicon.ico": host = "moto_api" elif path_info.startswith("/latest/meta-data/"): host = "instance_metadata" else: host = environ['HTTP_HOST'].split(':')[0] - if host in {'localhost', 'motoserver'} or host.startswith("192.168."): + if host is None: service, region = self.infer_service_region(environ) if service == 'dynamodb': if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'): From 4dd2b66b04e85937b6390aac80c35d991da71047 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 21 Jul 2019 22:07:58 -0500 Subject: [PATCH 39/63] Refactor backend parsing. --- moto/server.py | 33 +++++++++++++++++---------------- tests/test_core/test_server.py | 6 ------ 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/moto/server.py b/moto/server.py index 7e251e2f5..183c8c1f7 100644 --- a/moto/server.py +++ b/moto/server.py @@ -58,8 +58,6 @@ class DomainDispatcherApplication(object): if re.match(url_base, 'http://%s' % host): return backend_name - raise RuntimeError('Invalid host: "%s"' % host) - def infer_service_region(self, environ): auth = environ.get('HTTP_AUTHORIZATION') if auth: @@ -95,29 +93,32 @@ class DomainDispatcherApplication(object): if six.PY3 and isinstance(path_info, six.binary_type): path_info = path_info.decode('utf-8') - host = None if path_info.startswith("/moto-api") or path_info == "/favicon.ico": host = "moto_api" elif path_info.startswith("/latest/meta-data/"): host = "instance_metadata" else: host = environ['HTTP_HOST'].split(':')[0] - if host is None: - service, region = self.infer_service_region(environ) - if service == 'dynamodb': - if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'): - host = 'dynamodbstreams' - else: - dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0] - # If Newer API version, use dynamodb2 - if dynamo_api_version > "20111205": - host = "dynamodb2" - else: - host = "{service}.{region}.amazonaws.com".format( - service=service, region=region) with self.lock: backend = self.get_backend_for_host(host) + if not backend: + service, region = self.infer_service_region(environ) + if service == 'dynamodb': + if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'): + host = 'dynamodbstreams' + else: + dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0] + # If Newer API version, use dynamodb2 + if dynamo_api_version > "20111205": + host = "dynamodb2" + else: + host = "{service}.{region}.amazonaws.com".format( + service=service, region=region) + backend = self.get_backend_for_host(host) + if not backend: + raise RuntimeError('Invalid host: "%s"' % host) + app = self.app_instances.get(backend, None) if app is None: app = self.create_app(backend) diff --git a/tests/test_core/test_server.py b/tests/test_core/test_server.py index b7290e351..bd00b17c3 100644 --- a/tests/test_core/test_server.py +++ b/tests/test_core/test_server.py @@ -38,12 +38,6 @@ def test_domain_dispatched(): keys[0].should.equal('EmailResponse.dispatch') -def test_domain_without_matches(): - dispatcher = DomainDispatcherApplication(create_backend_app) - dispatcher.get_application.when.called_with( - {"HTTP_HOST": "not-matching-anything.com"}).should.throw(RuntimeError) - - def test_domain_dispatched_with_service(): # If we pass a particular service, always return that. dispatcher = DomainDispatcherApplication(create_backend_app, service="s3") From 5de95b026ac7474d8edbce20666531388babc912 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 21 Jul 2019 22:30:35 -0500 Subject: [PATCH 40/63] More refactoring. --- moto/server.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/moto/server.py b/moto/server.py index 183c8c1f7..907681f09 100644 --- a/moto/server.py +++ b/moto/server.py @@ -58,7 +58,7 @@ class DomainDispatcherApplication(object): if re.match(url_base, 'http://%s' % host): return backend_name - def infer_service_region(self, environ): + def infer_service_region_host(self, environ): auth = environ.get('HTTP_AUTHORIZATION') if auth: # Signed request @@ -68,21 +68,34 @@ class DomainDispatcherApplication(object): try: credential_scope = auth.split(",")[0].split()[1] _, _, region, service, _ = credential_scope.split("/") - return service, region except ValueError: # Signature format does not match, this is exceptional and we can't # infer a service-region. A reduced set of services still use # the deprecated SigV2, ergo prefer S3 as most likely default. # https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html - return DEFAULT_SERVICE_REGION + service, region = DEFAULT_SERVICE_REGION else: # Unsigned request target = environ.get('HTTP_X_AMZ_TARGET') if target: service, _ = target.split('.', 1) - return UNSIGNED_REQUESTS.get(service, DEFAULT_SERVICE_REGION) + service, region = UNSIGNED_REQUESTS.get(service, DEFAULT_SERVICE_REGION) # S3 is the last resort when the target is also unknown - return DEFAULT_SERVICE_REGION + service, region = DEFAULT_SERVICE_REGION + + if service == 'dynamodb': + if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'): + host = 'dynamodbstreams' + else: + dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0] + # If Newer API version, use dynamodb2 + if dynamo_api_version > "20111205": + host = "dynamodb2" + else: + host = "{service}.{region}.amazonaws.com".format( + service=service, region=region) + + return host def get_application(self, environ): path_info = environ.get('PATH_INFO', '') @@ -103,21 +116,9 @@ class DomainDispatcherApplication(object): with self.lock: backend = self.get_backend_for_host(host) if not backend: - service, region = self.infer_service_region(environ) - if service == 'dynamodb': - if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'): - host = 'dynamodbstreams' - else: - dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0] - # If Newer API version, use dynamodb2 - if dynamo_api_version > "20111205": - host = "dynamodb2" - else: - host = "{service}.{region}.amazonaws.com".format( - service=service, region=region) + # No regular backend found; try parsing other headers + host = self.infer_service_region_host(environ) backend = self.get_backend_for_host(host) - if not backend: - raise RuntimeError('Invalid host: "%s"' % host) app = self.app_instances.get(backend, None) if app is None: From 1fb06e6a085d52e7e237a0b3d8f833739a016cfe Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 21 Jul 2019 23:03:36 -0500 Subject: [PATCH 41/63] Cleanup refactor. --- moto/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/moto/server.py b/moto/server.py index 907681f09..89be47093 100644 --- a/moto/server.py +++ b/moto/server.py @@ -80,8 +80,9 @@ class DomainDispatcherApplication(object): if target: service, _ = target.split('.', 1) service, region = UNSIGNED_REQUESTS.get(service, DEFAULT_SERVICE_REGION) - # S3 is the last resort when the target is also unknown - service, region = DEFAULT_SERVICE_REGION + else: + # S3 is the last resort when the target is also unknown + service, region = DEFAULT_SERVICE_REGION if service == 'dynamodb': if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'): From ce4059f6d9902eca6eb0f259118eb5f13706f9da Mon Sep 17 00:00:00 2001 From: Don Kuntz Date: Mon, 22 Jul 2019 21:50:09 -0500 Subject: [PATCH 42/63] Use a consistent owner id between EC2 resources Previously there were a couple models which used different owner ids by default, which could make tests relying on them fail if someone wasn't expecting that. This change ensures a uniform owner id between resources. --- moto/ec2/models.py | 14 ++++++++------ tests/test_ec2/test_elastic_block_store.py | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 60a0257b0..41a84ec48 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -142,6 +142,8 @@ AMIS = json.load( __name__, 'resources/amis.json'), 'r') ) +OWNER_ID = "111122223333" + def utc_date_and_time(): return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.000Z') @@ -1087,7 +1089,7 @@ class TagBackend(object): class Ami(TaggedEC2Resource): def __init__(self, ec2_backend, ami_id, instance=None, source_ami=None, - name=None, description=None, owner_id=111122223333, + name=None, description=None, owner_id=OWNER_ID, public=False, virtualization_type=None, architecture=None, state='available', creation_date=None, platform=None, image_type='machine', image_location=None, hypervisor=None, @@ -1200,7 +1202,7 @@ class AmiBackend(object): ami = Ami(self, ami_id, instance=instance, source_ami=None, name=name, description=description, - owner_id=context.get_current_user() if context else '111122223333') + owner_id=context.get_current_user() if context else OWNER_ID) self.amis[ami_id] = ami return ami @@ -1468,7 +1470,7 @@ class SecurityGroup(TaggedEC2Resource): self.egress_rules = [SecurityRule(-1, None, None, ['0.0.0.0/0'], [])] self.enis = {} self.vpc_id = vpc_id - self.owner_id = "123456789012" + self.owner_id = OWNER_ID @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): @@ -1989,7 +1991,7 @@ class Volume(TaggedEC2Resource): class Snapshot(TaggedEC2Resource): - def __init__(self, ec2_backend, snapshot_id, volume, description, encrypted=False, owner_id='123456789012'): + def __init__(self, ec2_backend, snapshot_id, volume, description, encrypted=False, owner_id=OWNER_ID): self.id = snapshot_id self.volume = volume self.description = description @@ -2491,7 +2493,7 @@ class VPCPeeringConnectionBackend(object): class Subnet(TaggedEC2Resource): def __init__(self, ec2_backend, subnet_id, vpc_id, cidr_block, availability_zone, default_for_az, - map_public_ip_on_launch, owner_id=111122223333, assign_ipv6_address_on_creation=False): + map_public_ip_on_launch, owner_id=OWNER_ID, assign_ipv6_address_on_creation=False): self.ec2_backend = ec2_backend self.id = subnet_id self.vpc_id = vpc_id @@ -2657,7 +2659,7 @@ class SubnetBackend(object): raise InvalidAvailabilityZoneError(availability_zone, ", ".join([zone.name for zones in RegionsAndZonesBackend.zones.values() for zone in zones])) subnet = Subnet(self, subnet_id, vpc_id, cidr_block, availability_zone_data, default_for_az, map_public_ip_on_launch, - owner_id=context.get_current_user() if context else '111122223333', assign_ipv6_address_on_creation=False) + owner_id=context.get_current_user() if context else OWNER_ID, assign_ipv6_address_on_creation=False) # AWS associates a new subnet with the default Network ACL self.associate_default_network_acl_with_subnet(subnet_id, vpc_id) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index ab5b31ba0..9dbaa5ea6 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -12,6 +12,7 @@ from freezegun import freeze_time import sure # noqa from moto import mock_ec2_deprecated, mock_ec2 +from moto.ec2.models import OWNER_ID @mock_ec2_deprecated @@ -395,7 +396,7 @@ def test_snapshot_filters(): ).should.equal({snapshot3.id}) snapshots_by_owner_id = conn.get_all_snapshots( - filters={'owner-id': '123456789012'}) + filters={'owner-id': OWNER_ID}) set([snap.id for snap in snapshots_by_owner_id] ).should.equal({snapshot1.id, snapshot2.id, snapshot3.id}) From abf3db8d8a4287946a7dc0c5dfaa233139e3e71f Mon Sep 17 00:00:00 2001 From: Don Kuntz Date: Mon, 22 Jul 2019 21:57:15 -0500 Subject: [PATCH 43/63] Add a test to ensure that ec2.copy_image sets the proper owner id This test is useful because before the last commit using copy_image would not set the owner_id to the same one used when calling describe_images. For example, this code conn = boto3.client("ec2") copy_resp = conn.copy_image( SourceImageId="ami-whatever", ... ) describe_resp = conn.describe_images( Owners=["self"] ) Would result in describe_resp being empty, when it should contain the image from the copy_resp before it. By ensuring the owner ids are the same (see ce4059f6) the code example now works as expected. --- tests/test_ec2/test_amis.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_ec2/test_amis.py b/tests/test_ec2/test_amis.py index fd7234511..feff4a16c 100644 --- a/tests/test_ec2/test_amis.py +++ b/tests/test_ec2/test_amis.py @@ -10,7 +10,7 @@ from nose.tools import assert_raises import sure # noqa from moto import mock_ec2_deprecated, mock_ec2 -from moto.ec2.models import AMIS +from moto.ec2.models import AMIS, OWNER_ID from tests.helpers import requires_boto_gte @@ -152,6 +152,29 @@ def test_ami_copy(): cm.exception.request_id.should_not.be.none +@mock_ec2 +def test_copy_image_changes_owner_id(): + conn = boto3.client('ec2', region_name='us-east-1') + + # this source AMI ID is from moto/ec2/resources/amis.json + source_ami_id = "ami-03cf127a" + + # confirm the source ami owner id is different from the default owner id. + # if they're ever the same it means this test is invalid. + check_resp = conn.describe_images(ImageIds=[source_ami_id]) + check_resp["Images"][0]["OwnerId"].should_not.equal(OWNER_ID) + + copy_resp = conn.copy_image( + SourceImageId=source_ami_id, + Name="new-image", + Description="a copy of an image", + SourceRegion="us-east-1") + + describe_resp = conn.describe_images(Owners=["self"]) + describe_resp["Images"][0]["OwnerId"].should.equal(OWNER_ID) + describe_resp["Images"][0]["ImageId"].should.equal(copy_resp["ImageId"]) + + @mock_ec2_deprecated def test_ami_tagging(): conn = boto.connect_vpc('the_key', 'the_secret') From 51d96ae8f38b16d96f780b597a40d45a06881c7d Mon Sep 17 00:00:00 2001 From: Mariusz Strzelecki Date: Tue, 23 Jul 2019 20:53:45 +0200 Subject: [PATCH 44/63] Test split into boto and boto3 part --- tests/test_s3/test_s3.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index f0997b8d6..2786ff38a 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -639,7 +639,7 @@ def test_delete_keys(): @mock_s3_deprecated -def test_delete_keys_with_invalid(): +def test_delete_keys_invalid(): conn = boto.connect_s3('the_key', 'the_secret') bucket = conn.create_bucket('foobar') @@ -657,13 +657,14 @@ def test_delete_keys_with_invalid(): keys.should.have.length_of(3) keys[0].name.should.equal('file1') - # empty keys and boto2 client + # empty keys result = bucket.delete_keys([]) result.deleted.should.have.length_of(0) result.errors.should.have.length_of(0) - # empty keys and boto3 client +@mock_s3 +def test_boto3_delete_empty_keys_list(): with assert_raises(ClientError) as err: boto3.client('s3').delete_objects(Bucket='foobar', Delete={'Objects': []}) assert err.exception.response["Error"]["Code"] == "MalformedXML" From 348dc54e6a93b9cd71995f2134cd58c2eb8d23ba Mon Sep 17 00:00:00 2001 From: Jack Danger Date: Wed, 24 Jul 2019 19:15:43 -0700 Subject: [PATCH 45/63] Supporting tags in KMS (#2332) The CreateKey API method accepts tags but does not return them. --- moto/kms/models.py | 13 ++++---- moto/kms/responses.py | 3 +- moto/resourcegroupstaggingapi/models.py | 28 ++++++++++++++--- tests/test_kms/test_kms.py | 18 +++++++---- .../test_resourcegroupstaggingapi.py | 30 +++++++++++++++---- 5 files changed, 71 insertions(+), 21 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 2d6245ad2..577840b06 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import os import boto.kms from moto.core import BaseBackend, BaseModel -from moto.core.utils import iso_8601_datetime_without_milliseconds, unix_time +from moto.core.utils import iso_8601_datetime_without_milliseconds from .utils import generate_key_id from collections import defaultdict from datetime import datetime, timedelta @@ -11,7 +11,7 @@ from datetime import datetime, timedelta class Key(BaseModel): - def __init__(self, policy, key_usage, description, region): + def __init__(self, policy, key_usage, description, tags, region): self.id = generate_key_id() self.policy = policy self.key_usage = key_usage @@ -22,7 +22,7 @@ class Key(BaseModel): self.account_id = "0123456789012" self.key_rotation_status = False self.deletion_date = None - self.tags = {} + self.tags = tags or {} @property def physical_resource_id(self): @@ -37,7 +37,7 @@ class Key(BaseModel): "KeyMetadata": { "AWSAccountId": self.account_id, "Arn": self.arn, - "CreationDate": "%d" % unix_time(), + "CreationDate": iso_8601_datetime_without_milliseconds(datetime.now()), "Description": self.description, "Enabled": self.enabled, "KeyId": self.id, @@ -61,6 +61,7 @@ class Key(BaseModel): policy=properties['KeyPolicy'], key_usage='ENCRYPT_DECRYPT', description=properties['Description'], + tags=properties.get('Tags'), region=region_name, ) key.key_rotation_status = properties['EnableKeyRotation'] @@ -80,8 +81,8 @@ class KmsBackend(BaseBackend): self.keys = {} self.key_to_aliases = defaultdict(set) - def create_key(self, policy, key_usage, description, region): - key = Key(policy, key_usage, description, region) + def create_key(self, policy, key_usage, description, tags, region): + key = Key(policy, key_usage, description, tags, region) self.keys[key.id] = key return key diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 92195ed6b..8cd6e7663 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -31,9 +31,10 @@ class KmsResponse(BaseResponse): policy = self.parameters.get('Policy') key_usage = self.parameters.get('KeyUsage') description = self.parameters.get('Description') + tags = self.parameters.get('Tags') key = self.kms_backend.create_key( - policy, key_usage, description, self.region) + policy, key_usage, description, tags, self.region) return json.dumps(key.to_dict()) def update_key_description(self): diff --git a/moto/resourcegroupstaggingapi/models.py b/moto/resourcegroupstaggingapi/models.py index 4aec63aa6..3f15017cc 100644 --- a/moto/resourcegroupstaggingapi/models.py +++ b/moto/resourcegroupstaggingapi/models.py @@ -10,6 +10,7 @@ from moto.ec2 import ec2_backends from moto.elb import elb_backends from moto.elbv2 import elbv2_backends from moto.kinesis import kinesis_backends +from moto.kms import kms_backends from moto.rds2 import rds2_backends from moto.glacier import glacier_backends from moto.redshift import redshift_backends @@ -71,6 +72,13 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): """ return kinesis_backends[self.region_name] + @property + def kms_backend(self): + """ + :rtype: moto.kms.models.KmsBackend + """ + return kms_backends[self.region_name] + @property def rds_backend(self): """ @@ -221,9 +229,6 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): if not resource_type_filters or 'elasticloadbalancer' in resource_type_filters or 'elasticloadbalancer:loadbalancer' in resource_type_filters: for elb in self.elbv2_backend.load_balancers.values(): tags = get_elbv2_tags(elb.arn) - # if 'elasticloadbalancer:loadbalancer' in resource_type_filters: - # from IPython import embed - # embed() if not tag_filter(tags): # Skip if no tags, or invalid filter continue @@ -235,6 +240,21 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): # Kinesis + # KMS + def get_kms_tags(kms_key_id): + result = [] + for tag in self.kms_backend.list_resource_tags(kms_key_id): + result.append({'Key': tag['TagKey'], 'Value': tag['TagValue']}) + return result + + if not resource_type_filters or 'kms' in resource_type_filters: + for kms_key in self.kms_backend.list_keys(): + tags = get_kms_tags(kms_key.id) + if not tag_filter(tags): # Skip if no tags, or invalid filter + continue + + yield {'ResourceARN': '{0}'.format(kms_key.arn), 'Tags': tags} + # RDS Instance # RDS Reserved Database Instance # RDS Option Group @@ -370,7 +390,7 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend): def get_resources(self, pagination_token=None, resources_per_page=50, tags_per_page=100, tag_filters=None, resource_type_filters=None): - # Simple range checning + # Simple range checking if 100 >= tags_per_page >= 500: raise RESTError('InvalidParameterException', 'TagsPerPage must be between 100 and 500') if 1 >= resources_per_page >= 50: diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index f0d77d3e9..8fe0620f1 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -11,21 +11,29 @@ import sure # noqa from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises from freezegun import freeze_time +from datetime import date from datetime import datetime from dateutil.tz import tzutc -@mock_kms_deprecated +@mock_kms def test_create_key(): - conn = boto.kms.connect_to_region("us-west-2") + conn = boto3.client('kms', region_name='us-east-1') with freeze_time("2015-01-01 00:00:00"): - key = conn.create_key(policy="my policy", - description="my key", key_usage='ENCRYPT_DECRYPT') + key = conn.create_key(Policy="my policy", + Description="my key", + KeyUsage='ENCRYPT_DECRYPT', + Tags=[ + { + 'TagKey': 'project', + 'TagValue': 'moto', + }, + ]) key['KeyMetadata']['Description'].should.equal("my key") key['KeyMetadata']['KeyUsage'].should.equal("ENCRYPT_DECRYPT") key['KeyMetadata']['Enabled'].should.equal(True) - key['KeyMetadata']['CreationDate'].should.equal("1420070400") + key['KeyMetadata']['CreationDate'].should.be.a(date) @mock_kms_deprecated diff --git a/tests/test_resourcegroupstaggingapi/test_resourcegroupstaggingapi.py b/tests/test_resourcegroupstaggingapi/test_resourcegroupstaggingapi.py index 8015472bf..1e42dfe55 100644 --- a/tests/test_resourcegroupstaggingapi/test_resourcegroupstaggingapi.py +++ b/tests/test_resourcegroupstaggingapi/test_resourcegroupstaggingapi.py @@ -2,7 +2,11 @@ from __future__ import unicode_literals import boto3 import sure # noqa -from moto import mock_resourcegroupstaggingapi, mock_s3, mock_ec2, mock_elbv2 +from moto import mock_ec2 +from moto import mock_elbv2 +from moto import mock_kms +from moto import mock_resourcegroupstaggingapi +from moto import mock_s3 @mock_s3 @@ -225,10 +229,12 @@ def test_get_tag_values_ec2(): @mock_ec2 @mock_elbv2 +@mock_kms @mock_resourcegroupstaggingapi -def test_get_resources_elbv2(): - conn = boto3.client('elbv2', region_name='us-east-1') +def test_get_many_resources(): + elbv2 = boto3.client('elbv2', region_name='us-east-1') ec2 = boto3.resource('ec2', region_name='us-east-1') + kms = boto3.client('kms', region_name='us-east-1') security_group = ec2.create_security_group( GroupName='a-security-group', Description='First One') @@ -242,7 +248,7 @@ def test_get_resources_elbv2(): CidrBlock='172.28.7.0/26', AvailabilityZone='us-east-1b') - conn.create_load_balancer( + elbv2.create_load_balancer( Name='my-lb', Subnets=[subnet1.id, subnet2.id], SecurityGroups=[security_group.id], @@ -259,13 +265,27 @@ def test_get_resources_elbv2(): ] ) - conn.create_load_balancer( + elbv2.create_load_balancer( Name='my-other-lb', Subnets=[subnet1.id, subnet2.id], SecurityGroups=[security_group.id], Scheme='internal', ) + kms.create_key( + KeyUsage='ENCRYPT_DECRYPT', + Tags=[ + { + 'TagKey': 'key_name', + 'TagValue': 'a_value' + }, + { + 'TagKey': 'key_2', + 'TagValue': 'val2' + } + ] + ) + rtapi = boto3.client('resourcegroupstaggingapi', region_name='us-east-1') resp = rtapi.get_resources(ResourceTypeFilters=['elasticloadbalancer:loadbalancer']) From 9346a999d8d1ce726b71ad5dc8d95b1dd91b23c6 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Thu, 25 Jul 2019 21:29:56 -0500 Subject: [PATCH 46/63] Force minimum version of cfn-lint. Closes #2336. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3e14101d6..17a4f6691 100755 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ install_requires = [ "aws-xray-sdk!=0.96,>=0.93", "responses>=0.9.0", "idna<2.9,>=2.5", - "cfn-lint", + "cfn-lint>=0.4.0", "sshpubkeys>=3.1.0,<4.0" ] From feef7b2b5ac662b97d331f58aa5e5027e170ed65 Mon Sep 17 00:00:00 2001 From: Michael van Tellingen Date: Fri, 26 Jul 2019 12:55:05 +0200 Subject: [PATCH 47/63] Use a dict instead of a list for storing task definition revisions Before when a task definition revision was de-registered all revisions after that changed their revision id. This doesn't match the way it is handled in AWS. Using a hash and manually increment the revision id solves that. --- moto/ecs/models.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index a314c7776..7f8005a6a 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -422,11 +422,9 @@ class EC2ContainerServiceBackend(BaseBackend): revision = int(revision) else: family = task_definition_name - revision = len(self.task_definitions.get(family, [])) + revision = self._get_last_task_definition_revision_id(family) - if family in self.task_definitions and 0 < revision <= len(self.task_definitions[family]): - return self.task_definitions[family][revision - 1] - elif family in self.task_definitions and revision == -1: + if family in self.task_definitions and revision in self.task_definitions[family]: return self.task_definitions[family][revision] else: raise Exception( @@ -468,13 +466,14 @@ class EC2ContainerServiceBackend(BaseBackend): def register_task_definition(self, family, container_definitions, volumes): if family in self.task_definitions: - revision = len(self.task_definitions[family]) + 1 + last_id = self._get_last_task_definition_revision_id(family) + revision = (last_id or 0) + 1 else: - self.task_definitions[family] = [] + self.task_definitions[family] = {} revision = 1 task_definition = TaskDefinition( family, revision, container_definitions, volumes) - self.task_definitions[family].append(task_definition) + self.task_definitions[family][revision] = task_definition return task_definition @@ -484,16 +483,18 @@ class EC2ContainerServiceBackend(BaseBackend): """ task_arns = [] for task_definition_list in self.task_definitions.values(): - task_arns.extend( - [task_definition.arn for task_definition in task_definition_list]) + task_arns.extend([ + task_definition.arn + for task_definition in task_definition_list.values() + ]) return task_arns def deregister_task_definition(self, task_definition_str): task_definition_name = task_definition_str.split('/')[-1] family, revision = task_definition_name.split(':') revision = int(revision) - if family in self.task_definitions and 0 < revision <= len(self.task_definitions[family]): - return self.task_definitions[family].pop(revision - 1) + if family in self.task_definitions and revision in self.task_definitions[family]: + return self.task_definitions[family].pop(revision) else: raise Exception( "{0} is not a task_definition".format(task_definition_name)) @@ -950,6 +951,11 @@ class EC2ContainerServiceBackend(BaseBackend): yield task_fam + def _get_last_task_definition_revision_id(self, family): + definitions = self.task_definitions.get(family, {}) + if definitions: + return max(definitions.keys()) + available_regions = boto3.session.Session().get_available_regions("ecs") ecs_backends = {region: EC2ContainerServiceBackend(region) for region in available_regions} From 64e2a74e8c174297200dbf642c3a143657e854f4 Mon Sep 17 00:00:00 2001 From: Michael van Tellingen Date: Fri, 26 Jul 2019 15:11:25 +0200 Subject: [PATCH 48/63] Add support for setting tags on ecs task definitions This also implements the ecs.list_tags_for_resources, although the resources it checks for are currently only the task definitions --- moto/ecs/exceptions.py | 12 +++++++- moto/ecs/models.py | 32 ++++++++++++++++--- moto/ecs/responses.py | 8 ++++- tests/test_ecs/test_ecs_boto3.py | 53 ++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 6 deletions(-) diff --git a/moto/ecs/exceptions.py b/moto/ecs/exceptions.py index bb7e685c8..6e329f227 100644 --- a/moto/ecs/exceptions.py +++ b/moto/ecs/exceptions.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals -from moto.core.exceptions import RESTError +from moto.core.exceptions import RESTError, JsonRESTError class ServiceNotFoundException(RESTError): @@ -11,3 +11,13 @@ class ServiceNotFoundException(RESTError): message="The service {0} does not exist".format(service_name), template='error_json', ) + + +class TaskDefinitionNotFoundException(JsonRESTError): + code = 400 + + def __init__(self): + super(TaskDefinitionNotFoundException, self).__init__( + error_type="ClientException", + message="The specified task definition does not exist.", + ) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 7f8005a6a..92759651d 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +import re import uuid from datetime import datetime from random import random, randint @@ -10,7 +11,10 @@ from moto.core import BaseBackend, BaseModel from moto.ec2 import ec2_backends from copy import copy -from .exceptions import ServiceNotFoundException +from .exceptions import ( + ServiceNotFoundException, + TaskDefinitionNotFoundException +) class BaseObject(BaseModel): @@ -103,12 +107,13 @@ class Cluster(BaseObject): class TaskDefinition(BaseObject): - def __init__(self, family, revision, container_definitions, volumes=None): + def __init__(self, family, revision, container_definitions, volumes=None, tags=None): self.family = family self.revision = revision self.arn = 'arn:aws:ecs:us-east-1:012345678910:task-definition/{0}:{1}'.format( family, revision) self.container_definitions = container_definitions + self.tags = tags if tags is not None else [] if volumes is None: self.volumes = [] else: @@ -119,6 +124,7 @@ class TaskDefinition(BaseObject): response_object = self.gen_response_object() response_object['taskDefinitionArn'] = response_object['arn'] del response_object['arn'] + del response_object['tags'] return response_object @property @@ -464,7 +470,7 @@ class EC2ContainerServiceBackend(BaseBackend): else: raise Exception("{0} is not a cluster".format(cluster_name)) - def register_task_definition(self, family, container_definitions, volumes): + def register_task_definition(self, family, container_definitions, volumes, tags=None): if family in self.task_definitions: last_id = self._get_last_task_definition_revision_id(family) revision = (last_id or 0) + 1 @@ -472,7 +478,7 @@ class EC2ContainerServiceBackend(BaseBackend): self.task_definitions[family] = {} revision = 1 task_definition = TaskDefinition( - family, revision, container_definitions, volumes) + family, revision, container_definitions, volumes, tags) self.task_definitions[family][revision] = task_definition return task_definition @@ -951,6 +957,24 @@ class EC2ContainerServiceBackend(BaseBackend): yield task_fam + def list_tags_for_resource(self, resource_arn): + """Currently only implemented for task definitions""" + match = re.match( + "^arn:aws:ecs:(?P[^:]+):(?P[^:]+):(?P[^:]+)/(?P.*)$", + resource_arn) + if not match: + raise JsonRESTError('InvalidParameterException', 'The ARN provided is invalid.') + + service = match.group("service") + if service == "task-definition": + for task_definition in self.task_definitions.values(): + for revision in task_definition.values(): + if revision.arn == resource_arn: + return revision.tags + else: + raise TaskDefinitionNotFoundException() + raise NotImplementedError() + def _get_last_task_definition_revision_id(self, family): definitions = self.task_definitions.get(family, {}) if definitions: diff --git a/moto/ecs/responses.py b/moto/ecs/responses.py index 92b769fad..abb79ea78 100644 --- a/moto/ecs/responses.py +++ b/moto/ecs/responses.py @@ -62,8 +62,9 @@ class EC2ContainerServiceResponse(BaseResponse): family = self._get_param('family') container_definitions = self._get_param('containerDefinitions') volumes = self._get_param('volumes') + tags = self._get_param('tags') task_definition = self.ecs_backend.register_task_definition( - family, container_definitions, volumes) + family, container_definitions, volumes, tags) return json.dumps({ 'taskDefinition': task_definition.response_object }) @@ -313,3 +314,8 @@ class EC2ContainerServiceResponse(BaseResponse): results = self.ecs_backend.list_task_definition_families(family_prefix, status, max_results, next_token) return json.dumps({'families': list(results)}) + + def list_tags_for_resource(self): + resource_arn = self._get_param('resourceArn') + tags = self.ecs_backend.list_tags_for_resource(resource_arn) + return json.dumps({'tags': tags}) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index b147c4159..27f37308e 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -94,6 +94,10 @@ def test_register_task_definition(): }], 'logConfiguration': {'logDriver': 'json-file'} } + ], + tags=[ + {'key': 'createdBy', 'value': 'moto-unittest'}, + {'key': 'foo', 'value': 'bar'}, ] ) type(response['taskDefinition']).should.be(dict) @@ -2304,3 +2308,52 @@ def test_create_service_load_balancing(): response['service']['status'].should.equal('ACTIVE') response['service']['taskDefinition'].should.equal( 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') + + +@mock_ecs +def test_list_tags_for_resource(): + client = boto3.client('ecs', region_name='us-east-1') + response = client.register_task_definition( + family='test_ecs_task', + containerDefinitions=[ + { + 'name': 'hello_world', + 'image': 'docker/hello-world:latest', + 'cpu': 1024, + 'memory': 400, + 'essential': True, + 'environment': [{ + 'name': 'AWS_ACCESS_KEY_ID', + 'value': 'SOME_ACCESS_KEY' + }], + 'logConfiguration': {'logDriver': 'json-file'} + } + ], + tags=[ + {'key': 'createdBy', 'value': 'moto-unittest'}, + {'key': 'foo', 'value': 'bar'}, + ] + ) + type(response['taskDefinition']).should.be(dict) + response['taskDefinition']['revision'].should.equal(1) + response['taskDefinition']['taskDefinitionArn'].should.equal( + 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') + + task_definition_arn = response['taskDefinition']['taskDefinitionArn'] + response = client.list_tags_for_resource(resourceArn=task_definition_arn) + + type(response['tags']).should.be(list) + response['tags'].should.equal([ + {'key': 'createdBy', 'value': 'moto-unittest'}, + {'key': 'foo', 'value': 'bar'}, + ]) + + +@mock_ecs +def test_list_tags_for_resource_unknown(): + client = boto3.client('ecs', region_name='us-east-1') + task_definition_arn = 'arn:aws:ecs:us-east-1:012345678910:task-definition/unknown:1' + try: + client.list_tags_for_resource(resourceArn=task_definition_arn) + except ClientError as err: + err.response['Error']['Code'].should.equal('ClientException') From e4a4e6183560489e98b95e815b439c7a1cf3566c Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 29 Jul 2019 22:52:29 -0500 Subject: [PATCH 49/63] Allow passing of encoding-type for s3 get_bucket_versions without throwing error. This was a change made in https://github.com/boto/botocore/pull/1794. --- moto/s3/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 2a628d681..528eacee3 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -764,7 +764,7 @@ class S3Backend(BaseBackend): prefix=''): bucket = self.get_bucket(bucket_name) - if any((delimiter, encoding_type, key_marker, version_id_marker)): + if any((delimiter, key_marker, version_id_marker)): raise NotImplementedError( "Called get_bucket_versions with some of delimiter, encoding_type, key_marker, version_id_marker") From 8cca33dc42f0217a00068d6727757953db8e1a3a Mon Sep 17 00:00:00 2001 From: Iiro Sulopuisto Date: Tue, 30 Jul 2019 14:14:14 +0300 Subject: [PATCH 50/63] Test redshift cluster creation time more thoroughly --- tests/test_redshift/test_redshift.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index 541614788..2c9b42a1d 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -36,6 +36,7 @@ def test_create_cluster_boto3(): response['Cluster']['NodeType'].should.equal('ds2.xlarge') create_time = response['Cluster']['ClusterCreateTime'] create_time.should.be.lower_than(datetime.datetime.now(create_time.tzinfo)) + create_time.should.be.greater_than(datetime.datetime.now(create_time.tzinfo) - datetime.timedelta(minutes=1)) @mock_redshift From 24bd99b5c4d161b1af22b86d48bc4cc141e7e44a Mon Sep 17 00:00:00 2001 From: Iiro Sulopuisto Date: Tue, 30 Jul 2019 14:16:12 +0300 Subject: [PATCH 51/63] Make cluster creation time UTC --- moto/redshift/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 64e5c5e35..c0b783bde 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -78,7 +78,7 @@ class Cluster(TaggableResourceMixin, BaseModel): super(Cluster, self).__init__(region_name, tags) self.redshift_backend = redshift_backend self.cluster_identifier = cluster_identifier - self.create_time = iso_8601_datetime_with_milliseconds(datetime.datetime.now()) + self.create_time = iso_8601_datetime_with_milliseconds(datetime.datetime.utcnow()) self.status = 'available' self.node_type = node_type self.master_username = master_username From 7fa5ce3dc308c5b1bd8581a39109e14d1977beee Mon Sep 17 00:00:00 2001 From: Chiharu Terashima Date: Sat, 3 Aug 2019 01:28:47 +0900 Subject: [PATCH 52/63] implements APIGateway update_api_key --- .gitignore | 1 + IMPLEMENTATION_COVERAGE.md | 2 +- moto/apigateway/models.py | 23 +++++++++++++++++++++++ moto/apigateway/responses.py | 3 +++ tests/test_apigateway/test_apigateway.py | 21 +++++++++++++++++++-- 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 0a24fe476..0282e3caf 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ python_env .ropeproject/ .pytest_cache/ venv/ +env/ .python-version .vscode/ tests/file.tmp diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 1d9811983..897c3885c 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -181,7 +181,7 @@ - [ ] test_invoke_method - [ ] untag_resource - [ ] update_account -- [ ] update_api_key +- [X] update_api_key - [ ] update_authorizer - [ ] update_base_path_mapping - [ ] update_client_certificate diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 41a49e361..6be062d7f 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -309,6 +309,25 @@ class ApiKey(BaseModel, dict): self['createdDate'] = self['lastUpdatedDate'] = int(time.time()) self['stageKeys'] = stageKeys + def update_operations(self, patch_operations): + for op in patch_operations: + if op['op'] == 'replace': + if '/name' in op['path']: + self['name'] = op['value'] + elif '/customerId' in op['path']: + self['customerId'] = op['value'] + elif '/description' in op['path']: + self['description'] = op['value'] + elif '/enabled' in op['path']: + self['enabled'] = self._str2bool(op['value']) + else: + raise Exception( + 'Patch operation "%s" not implemented' % op['op']) + return self + + def _str2bool(self, v): + return v.lower() == "true" + class UsagePlan(BaseModel, dict): @@ -599,6 +618,10 @@ class APIGatewayBackend(BaseBackend): def get_apikey(self, api_key_id): return self.keys[api_key_id] + def update_apikey(self, api_key_id, patch_operations): + key = self.keys[api_key_id] + return key.update_operations(patch_operations) + def delete_apikey(self, api_key_id): self.keys.pop(api_key_id) return {} diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index bc4d262cd..fa82705b1 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -245,6 +245,9 @@ class APIGatewayResponse(BaseResponse): if self.method == 'GET': apikey_response = self.backend.get_apikey(apikey) + elif self.method == 'PATCH': + patch_operations = self._get_param('patchOperations') + apikey_response = self.backend.update_apikey(apikey, patch_operations) elif self.method == 'DELETE': apikey_response = self.backend.delete_apikey(apikey) return 200, {}, json.dumps(apikey_response) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 5954de8ca..0a33f2f9f 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -988,13 +988,30 @@ def test_api_keys(): apikey['name'].should.equal(apikey_name) len(apikey['value']).should.equal(40) + apikey_name = 'TESTKEY3' + payload = {'name': apikey_name } + response = client.create_api_key(**payload) + apikey_id = response['id'] + + patch_operations = [ + {'op': 'replace', 'path': '/name', 'value': 'TESTKEY3_CHANGE'}, + {'op': 'replace', 'path': '/customerId', 'value': '12345'}, + {'op': 'replace', 'path': '/description', 'value': 'APIKEY UPDATE TEST'}, + {'op': 'replace', 'path': '/enabled', 'value': 'false'}, + ] + response = client.update_api_key(apiKey=apikey_id, patchOperations=patch_operations) + response['name'].should.equal('TESTKEY3_CHANGE') + response['customerId'].should.equal('12345') + response['description'].should.equal('APIKEY UPDATE TEST') + response['enabled'].should.equal(False) + response = client.get_api_keys() - len(response['items']).should.equal(2) + len(response['items']).should.equal(3) client.delete_api_key(apiKey=apikey_id) response = client.get_api_keys() - len(response['items']).should.equal(1) + len(response['items']).should.equal(2) @mock_apigateway def test_usage_plans(): From 3c19f0a02dfc440b989047fee751bb65eabb527f Mon Sep 17 00:00:00 2001 From: wndhydrnt Date: Sat, 3 Aug 2019 13:56:07 +0200 Subject: [PATCH 53/63] Convert fields createdAt and updatedAt of deployment to timestamp --- moto/ecs/models.py | 5 +++-- tests/test_ecs/test_ecs_boto3.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 92759651d..863cfc49e 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -8,6 +8,7 @@ import boto3 import pytz from moto.core.exceptions import JsonRESTError from moto.core import BaseBackend, BaseModel +from moto.core.utils import unix_time from moto.ec2 import ec2_backends from copy import copy @@ -231,9 +232,9 @@ class Service(BaseObject): for deployment in response_object['deployments']: if isinstance(deployment['createdAt'], datetime): - deployment['createdAt'] = deployment['createdAt'].isoformat() + deployment['createdAt'] = unix_time(deployment['createdAt'].replace(tzinfo=None)) if isinstance(deployment['updatedAt'], datetime): - deployment['updatedAt'] = deployment['updatedAt'].isoformat() + deployment['updatedAt'] = unix_time(deployment['updatedAt'].replace(tzinfo=None)) return response_object diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index 27f37308e..9937af26b 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +from datetime import datetime from copy import deepcopy @@ -477,6 +478,8 @@ def test_describe_services(): response['services'][0]['deployments'][0]['pendingCount'].should.equal(2) response['services'][0]['deployments'][0]['runningCount'].should.equal(0) response['services'][0]['deployments'][0]['status'].should.equal('PRIMARY') + (datetime.now() - response['services'][0]['deployments'][0]["createdAt"].replace(tzinfo=None)).seconds.should.be.within(0, 10) + (datetime.now() - response['services'][0]['deployments'][0]["updatedAt"].replace(tzinfo=None)).seconds.should.be.within(0, 10) @mock_ecs From b7884ef9034d79a496e832c4564cd6de6a5d4f75 Mon Sep 17 00:00:00 2001 From: Berislav Kovacki Date: Mon, 5 Aug 2019 17:34:39 +0200 Subject: [PATCH 54/63] Add S3 support for INTELLIGENT_TIERING, GLACIER and DEEP_ARCHIVE storage * Add INTELLIGENT_TIERING, GLACIER and DEEP_ARCHIVE as valid storage classes for objects * Add ObjectNotInActiveTierError error on PUT object copy for GLACIER and DEEP_ARCHIVE storage class objects --- moto/s3/exceptions.py | 11 +++++++ moto/s3/models.py | 3 +- moto/s3/responses.py | 8 +++-- tests/test_s3/test_s3_storageclass.py | 42 +++++++++++++++++++++++---- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index f74fc21ae..8d2326fa1 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -60,6 +60,17 @@ class MissingKey(S3ClientError): ) +class ObjectNotInActiveTierError(S3ClientError): + code = 403 + + def __init__(self, key_name): + super(ObjectNotInActiveTierError, self).__init__( + "ObjectNotInActiveTierError", + "The source object of the COPY operation is not in the active tier and is only stored in Amazon Glacier.", + Key=key_name, + ) + + class InvalidPartOrder(S3ClientError): code = 400 diff --git a/moto/s3/models.py b/moto/s3/models.py index 528eacee3..b5aef34d3 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -28,7 +28,8 @@ MAX_BUCKET_NAME_LENGTH = 63 MIN_BUCKET_NAME_LENGTH = 3 UPLOAD_ID_BYTES = 43 UPLOAD_PART_MIN_SIZE = 5242880 -STORAGE_CLASS = ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"] +STORAGE_CLASS = ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA", + "INTELLIGENT_TIERING", "GLACIER", "DEEP_ARCHIVE"] DEFAULT_KEY_BUFFER_SIZE = 16 * 1024 * 1024 DEFAULT_TEXT_ENCODING = sys.getdefaultencoding() diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 2bd6ed1a3..a05a86de4 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -17,7 +17,7 @@ from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_n parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys from .exceptions import BucketAlreadyExists, S3ClientError, MissingBucket, MissingKey, InvalidPartOrder, MalformedXML, \ - MalformedACLError, InvalidNotificationARN, InvalidNotificationEvent + MalformedACLError, InvalidNotificationARN, InvalidNotificationEvent, ObjectNotInActiveTierError from .models import s3_backend, get_canned_acl, FakeGrantee, FakeGrant, FakeAcl, FakeKey, FakeTagging, FakeTagSet, \ FakeTag from .utils import bucket_name_from_url, clean_key_name, metadata_from_headers, parse_region_from_url @@ -902,7 +902,11 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): src_version_id = parse_qs(src_key_parsed.query).get( 'versionId', [None])[0] - if self.backend.get_key(src_bucket, src_key, version_id=src_version_id): + key = self.backend.get_key(src_bucket, src_key, version_id=src_version_id) + + if key is not None: + if key.storage_class in ["GLACIER", "DEEP_ARCHIVE"]: + raise ObjectNotInActiveTierError(key) self.backend.copy_key(src_bucket, src_key, bucket_name, key_name, storage=storage_class, acl=acl, src_version_id=src_version_id) else: diff --git a/tests/test_s3/test_s3_storageclass.py b/tests/test_s3/test_s3_storageclass.py index 99908c501..c72b773a9 100644 --- a/tests/test_s3/test_s3_storageclass.py +++ b/tests/test_s3/test_s3_storageclass.py @@ -1,16 +1,12 @@ from __future__ import unicode_literals -import boto import boto3 -from boto.exception import S3CreateError, S3ResponseError -from boto.s3.lifecycle import Lifecycle, Transition, Expiration, Rule import sure # noqa from botocore.exceptions import ClientError -from datetime import datetime from nose.tools import assert_raises -from moto import mock_s3_deprecated, mock_s3 +from moto import mock_s3 @mock_s3 @@ -41,6 +37,18 @@ def test_s3_storage_class_infrequent_access(): D['Contents'][0]["StorageClass"].should.equal("STANDARD_IA") +@mock_s3 +def test_s3_storage_class_intelligent_tiering(): + s3 = boto3.client("s3") + + s3.create_bucket(Bucket="Bucket") + s3.put_object(Bucket="Bucket", Key="my_key_infrequent", Body="my_value_infrequent", StorageClass="INTELLIGENT_TIERING") + + objects = s3.list_objects(Bucket="Bucket") + + objects['Contents'][0]["StorageClass"].should.equal("INTELLIGENT_TIERING") + + @mock_s3 def test_s3_storage_class_copy(): s3 = boto3.client("s3") @@ -90,6 +98,7 @@ def test_s3_invalid_storage_class(): e.response["Error"]["Code"].should.equal("InvalidStorageClass") e.response["Error"]["Message"].should.equal("The storage class you specified is not valid") + @mock_s3 def test_s3_default_storage_class(): s3 = boto3.client("s3") @@ -103,4 +112,27 @@ def test_s3_default_storage_class(): list_of_objects["Contents"][0]["StorageClass"].should.equal("STANDARD") +@mock_s3 +def test_s3_copy_object_error_for_glacier_storage_class(): + s3 = boto3.client("s3") + s3.create_bucket(Bucket="Bucket") + s3.put_object(Bucket="Bucket", Key="First_Object", Body="Body", StorageClass="GLACIER") + + with assert_raises(ClientError) as exc: + s3.copy_object(CopySource={"Bucket": "Bucket", "Key": "First_Object"}, Bucket="Bucket", Key="Second_Object") + + exc.exception.response["Error"]["Code"].should.equal("ObjectNotInActiveTierError") + + +@mock_s3 +def test_s3_copy_object_error_for_deep_archive_storage_class(): + s3 = boto3.client("s3") + s3.create_bucket(Bucket="Bucket") + + s3.put_object(Bucket="Bucket", Key="First_Object", Body="Body", StorageClass="DEEP_ARCHIVE") + + with assert_raises(ClientError) as exc: + s3.copy_object(CopySource={"Bucket": "Bucket", "Key": "First_Object"}, Bucket="Bucket", Key="Second_Object") + + exc.exception.response["Error"]["Code"].should.equal("ObjectNotInActiveTierError") From a3794f2701eb54393b44d13da11680ec5314b8b8 Mon Sep 17 00:00:00 2001 From: aksagrimada <1955605+aksagrimada@users.noreply.github.com> Date: Tue, 6 Aug 2019 07:16:00 +0100 Subject: [PATCH 55/63] Resolve invalid escape sequence When run not as a decorator dynamodb2 displays an invalid escape sequence error /moto/dynamodb2/responses.py:603: DeprecationWarning: invalid escape sequence \s '\s*([=\+-])\s*', '\\1', update_expression) --- moto/dynamodb2/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index d34b176a7..86ca9a362 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -600,7 +600,7 @@ class DynamoHandler(BaseResponse): # E.g. `a = b + c` -> `a=b+c` if update_expression: update_expression = re.sub( - '\s*([=\+-])\s*', '\\1', update_expression) + r'\s*([=\+-])\s*', '\\1', update_expression) try: item = self.dynamodb_backend.update_item( From a35a55ec261c2ce8dcbd4d14c77426c5aec06c2f Mon Sep 17 00:00:00 2001 From: Berislav Kovacki Date: Tue, 6 Aug 2019 22:13:52 +0200 Subject: [PATCH 56/63] Add option to call batch submit_job with job definition name only * Add option to call batch submit_job with job definition name only * Fix bug which causes register_job_definition not to increment job revision number after a second revision --- moto/batch/models.py | 26 ++++++----- tests/test_batch/test_batch.py | 81 ++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 11 deletions(-) diff --git a/moto/batch/models.py b/moto/batch/models.py index c47ca6e97..caa442802 100644 --- a/moto/batch/models.py +++ b/moto/batch/models.py @@ -514,10 +514,13 @@ class BatchBackend(BaseBackend): return self._job_definitions.get(arn) def get_job_definition_by_name(self, name): - for comp_env in self._job_definitions.values(): - if comp_env.name == name: - return comp_env - return None + latest_revision = -1 + latest_job = None + for job_def in self._job_definitions.values(): + if job_def.name == name and job_def.revision > latest_revision: + latest_job = job_def + latest_revision = job_def.revision + return latest_job def get_job_definition_by_name_revision(self, name, revision): for job_def in self._job_definitions.values(): @@ -534,10 +537,13 @@ class BatchBackend(BaseBackend): :return: Job definition or None :rtype: JobDefinition or None """ - env = self.get_job_definition_by_arn(identifier) - if env is None: - env = self.get_job_definition_by_name(identifier) - return env + job_def = self.get_job_definition_by_arn(identifier) + if job_def is None: + if ':' in identifier: + job_def = self.get_job_definition_by_name_revision(*identifier.split(':', 1)) + else: + job_def = self.get_job_definition_by_name(identifier) + return job_def def get_job_definitions(self, identifier): """ @@ -984,9 +990,7 @@ class BatchBackend(BaseBackend): # TODO parameters, retries (which is a dict raw from request), job dependancies and container overrides are ignored for now # Look for job definition - job_def = self.get_job_definition_by_arn(job_def_id) - if job_def is None and ':' in job_def_id: - job_def = self.get_job_definition_by_name_revision(*job_def_id.split(':', 1)) + job_def = self.get_job_definition(job_def_id) if job_def is None: raise ClientException('Job definition {0} does not exist'.format(job_def_id)) diff --git a/tests/test_batch/test_batch.py b/tests/test_batch/test_batch.py index 310ac0b48..89a8d4d0e 100644 --- a/tests/test_batch/test_batch.py +++ b/tests/test_batch/test_batch.py @@ -642,6 +642,87 @@ def test_describe_task_definition(): len(resp['jobDefinitions']).should.equal(3) +@mock_logs +@mock_ec2 +@mock_ecs +@mock_iam +@mock_batch +def test_submit_job_by_name(): + ec2_client, iam_client, ecs_client, logs_client, batch_client = _get_clients() + vpc_id, subnet_id, sg_id, iam_arn = _setup(ec2_client, iam_client) + + compute_name = 'test_compute_env' + resp = batch_client.create_compute_environment( + computeEnvironmentName=compute_name, + type='UNMANAGED', + state='ENABLED', + serviceRole=iam_arn + ) + arn = resp['computeEnvironmentArn'] + + resp = batch_client.create_job_queue( + jobQueueName='test_job_queue', + state='ENABLED', + priority=123, + computeEnvironmentOrder=[ + { + 'order': 123, + 'computeEnvironment': arn + }, + ] + ) + queue_arn = resp['jobQueueArn'] + + job_definition_name = 'sleep10' + + batch_client.register_job_definition( + jobDefinitionName=job_definition_name, + type='container', + containerProperties={ + 'image': 'busybox', + 'vcpus': 1, + 'memory': 128, + 'command': ['sleep', '10'] + } + ) + batch_client.register_job_definition( + jobDefinitionName=job_definition_name, + type='container', + containerProperties={ + 'image': 'busybox', + 'vcpus': 1, + 'memory': 256, + 'command': ['sleep', '10'] + } + ) + resp = batch_client.register_job_definition( + jobDefinitionName=job_definition_name, + type='container', + containerProperties={ + 'image': 'busybox', + 'vcpus': 1, + 'memory': 512, + 'command': ['sleep', '10'] + } + ) + job_definition_arn = resp['jobDefinitionArn'] + + resp = batch_client.submit_job( + jobName='test1', + jobQueue=queue_arn, + jobDefinition=job_definition_name + ) + job_id = resp['jobId'] + + resp_jobs = batch_client.describe_jobs(jobs=[job_id]) + + # batch_client.terminate_job(jobId=job_id) + + len(resp_jobs['jobs']).should.equal(1) + resp_jobs['jobs'][0]['jobId'].should.equal(job_id) + resp_jobs['jobs'][0]['jobQueue'].should.equal(queue_arn) + resp_jobs['jobs'][0]['jobDefinition'].should.equal(job_definition_arn) + # SLOW TESTS @expected_failure @mock_logs From 7d453fec9ad57ae39ecad60cbba5f8d471e73925 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Tue, 6 Aug 2019 15:44:49 -0700 Subject: [PATCH 57/63] [Resolves #2355] - create_organization(): add master account, default policy Model: OrganizationsBackend Method: create_organization create_organization now creates master account, root ou, and a default service control policy objects and adds them to the OrganizationsBackend object. the policy is attached to both the master account and the root ou. any subsiquently created accounts or OU also have the default policy attached. --- moto/organizations/models.py | 40 +++++++++++++++++-- moto/organizations/utils.py | 5 ++- .../organizations_test_utils.py | 1 - .../test_organizations_boto3.py | 31 +++++++++++--- 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 91004b9ba..561c6c3a8 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import datetime import re +import json from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError @@ -151,7 +152,6 @@ class FakeRoot(FakeOrganizationalUnit): class FakeServiceControlPolicy(BaseModel): def __init__(self, organization, **kwargs): - self.type = 'POLICY' self.content = kwargs.get('Content') self.description = kwargs.get('Description') self.name = kwargs.get('Name') @@ -197,7 +197,38 @@ class OrganizationsBackend(BaseBackend): def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs['FeatureSet']) - self.ou.append(FakeRoot(self.org)) + root_ou = FakeRoot(self.org) + self.ou.append(root_ou) + master_account = FakeAccount( + self.org, + AccountName='master', + Email=self.org.master_account_email, + ) + master_account.id = self.org.master_account_id + self.accounts.append(master_account) + default_policy = FakeServiceControlPolicy( + self.org, + Name='FullAWSAccess', + Description='Allows access to every operation', + Type='SERVICE_CONTROL_POLICY', + Content=json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "*", + "Resource": "*" + } + ] + } + ) + ) + default_policy.id = utils.DEFAULT_POLICY_ID + default_policy.aws_managed = True + self.policies.append(default_policy) + self.attach_policy(PolicyId=default_policy.id, TargetId=root_ou.id) + self.attach_policy(PolicyId=default_policy.id, TargetId=master_account.id) return self.org.describe() def describe_organization(self): @@ -216,6 +247,7 @@ class OrganizationsBackend(BaseBackend): def create_organizational_unit(self, **kwargs): new_ou = FakeOrganizationalUnit(self.org, **kwargs) self.ou.append(new_ou) + self.attach_policy(PolicyId=utils.DEFAULT_POLICY_ID, TargetId=new_ou.id) return new_ou.describe() def get_organizational_unit_by_id(self, ou_id): @@ -258,6 +290,7 @@ class OrganizationsBackend(BaseBackend): def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) + self.attach_policy(PolicyId=utils.DEFAULT_POLICY_ID, TargetId=new_account.id) return new_account.create_account_status def get_account_by_id(self, account_id): @@ -358,8 +391,7 @@ class OrganizationsBackend(BaseBackend): def attach_policy(self, **kwargs): policy = next((p for p in self.policies if p.id == kwargs['PolicyId']), None) - if (re.compile(utils.ROOT_ID_REGEX).match(kwargs['TargetId']) or - re.compile(utils.OU_ID_REGEX).match(kwargs['TargetId'])): + if (re.compile(utils.ROOT_ID_REGEX).match(kwargs['TargetId']) or re.compile(utils.OU_ID_REGEX).match(kwargs['TargetId'])): ou = next((ou for ou in self.ou if ou.id == kwargs['TargetId']), None) if ou is not None: if ou not in ou.attached_policies: diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py index bde3660d2..5cbe59ada 100644 --- a/moto/organizations/utils.py +++ b/moto/organizations/utils.py @@ -4,7 +4,8 @@ import random import string MASTER_ACCOUNT_ID = '123456789012' -MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' +MASTER_ACCOUNT_EMAIL = 'master@example.com' +DEFAULT_POLICY_ID = 'p-FullAWSAccess' ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' @@ -26,7 +27,7 @@ ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % ROOT_ID_SIZE OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (ROOT_ID_SIZE, OU_ID_SUFFIX_SIZE) ACCOUNT_ID_REGEX = r'[0-9]{%s}' % ACCOUNT_ID_SIZE CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % CREATE_ACCOUNT_STATUS_ID_SIZE -SCP_ID_REGEX = r'p-[a-z0-9]{%s}' % SCP_ID_SIZE +SCP_ID_REGEX = r'%s|p-[a-z0-9]{%s}' % (DEFAULT_POLICY_ID, SCP_ID_SIZE) def make_random_org_id(): diff --git a/tests/test_organizations/organizations_test_utils.py b/tests/test_organizations/organizations_test_utils.py index 36933d41a..83b60b877 100644 --- a/tests/test_organizations/organizations_test_utils.py +++ b/tests/test_organizations/organizations_test_utils.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import six -import sure # noqa import datetime from moto.organizations import utils diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 05f831e62..28f8cca91 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import boto3 import json import six -import sure # noqa from botocore.exceptions import ClientError from nose.tools import assert_raises @@ -27,6 +26,25 @@ def test_create_organization(): validate_organization(response) response['Organization']['FeatureSet'].should.equal('ALL') + response = client.list_accounts() + len(response['Accounts']).should.equal(1) + response['Accounts'][0]['Name'].should.equal('master') + response['Accounts'][0]['Id'].should.equal(utils.MASTER_ACCOUNT_ID) + response['Accounts'][0]['Email'].should.equal(utils.MASTER_ACCOUNT_EMAIL) + + response = client.list_policies(Filter='SERVICE_CONTROL_POLICY') + len(response['Policies']).should.equal(1) + response['Policies'][0]['Name'].should.equal('FullAWSAccess') + response['Policies'][0]['Id'].should.equal(utils.DEFAULT_POLICY_ID) + response['Policies'][0]['AwsManaged'].should.equal(True) + + response = client.list_targets_for_policy(PolicyId=utils.DEFAULT_POLICY_ID) + len(response['Targets']).should.equal(2) + root_ou = [t for t in response['Targets'] if t['Type'] == 'ROOT'][0] + root_ou['Name'].should.equal('Root') + master_account = [t for t in response['Targets'] if t['Type'] == 'ACCOUNT'][0] + master_account['Name'].should.equal('master') + @mock_organizations def test_describe_organization(): @@ -177,11 +195,11 @@ def test_list_accounts(): response = client.list_accounts() response.should.have.key('Accounts') accounts = response['Accounts'] - len(accounts).should.equal(5) + len(accounts).should.equal(6) for account in accounts: validate_account(org, account) - accounts[3]['Name'].should.equal(mockname + '3') - accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) + accounts[4]['Name'].should.equal(mockname + '3') + accounts[3]['Email'].should.equal(mockname + '2' + '@' + mockdomain) @mock_organizations @@ -291,8 +309,10 @@ def test_list_children(): response02 = client.list_children(ParentId=root_id, ChildType='ORGANIZATIONAL_UNIT') response03 = client.list_children(ParentId=ou01_id, ChildType='ACCOUNT') response04 = client.list_children(ParentId=ou01_id, ChildType='ORGANIZATIONAL_UNIT') - response01['Children'][0]['Id'].should.equal(account01_id) + response01['Children'][0]['Id'].should.equal(utils.MASTER_ACCOUNT_ID) response01['Children'][0]['Type'].should.equal('ACCOUNT') + response01['Children'][1]['Id'].should.equal(account01_id) + response01['Children'][1]['Type'].should.equal('ACCOUNT') response02['Children'][0]['Id'].should.equal(ou01_id) response02['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') response03['Children'][0]['Id'].should.equal(account02_id) @@ -591,4 +611,3 @@ def test_list_targets_for_policy_exception(): ex.operation_name.should.equal('ListTargetsForPolicy') ex.response['Error']['Code'].should.equal('400') ex.response['Error']['Message'].should.contain('InvalidInputException') - From 5063ffc837aa755491d15671db16d5ba68fce902 Mon Sep 17 00:00:00 2001 From: Berislav Kovacki Date: Wed, 7 Aug 2019 17:37:53 +0200 Subject: [PATCH 58/63] Implement pagination support for GetLogEvents * Add nextForwardToken and nextBackwardToken to GetLogEvents response * Handle end of stream by returning the same token as passed in --- moto/logs/models.py | 28 ++++++++++++----- tests/test_logs/test_logs.py | 60 ++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/moto/logs/models.py b/moto/logs/models.py index a44b76812..2b8dcfeb4 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -98,17 +98,29 @@ class LogStream: return True + def get_paging_token_from_index(index, back=False): + if index is not None: + return "b/{:056d}".format(index) if back else "f/{:056d}".format(index) + return 0 + + def get_index_from_paging_token(token): + if token is not None: + return int(token[2:]) + return 0 + events = sorted(filter(filter_func, self.events), key=lambda event: event.timestamp, reverse=start_from_head) - back_token = next_token - if next_token is None: - next_token = 0 + next_index = get_index_from_paging_token(next_token) + back_index = next_index - events_page = [event.to_response_dict() for event in events[next_token: next_token + limit]] - next_token += limit - if next_token >= len(self.events): - next_token = None + events_page = [event.to_response_dict() for event in events[next_index: next_index + limit]] + if next_index + limit < len(self.events): + next_index += limit - return events_page, back_token, next_token + back_index -= limit + if back_index <= 0: + back_index = 0 + + return events_page, get_paging_token_from_index(back_index, True), get_paging_token_from_index(next_index) def filter_log_events(self, log_group_name, log_stream_names, start_time, end_time, limit, next_token, filter_pattern, interleaved): def filter_func(event): diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index 7048061f0..49e593fdc 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -162,3 +162,63 @@ def test_delete_retention_policy(): response = conn.delete_log_group(logGroupName=log_group_name) + +@mock_logs +def test_get_log_events(): + conn = boto3.client('logs', 'us-west-2') + log_group_name = 'test' + log_stream_name = 'stream' + conn.create_log_group(logGroupName=log_group_name) + conn.create_log_stream( + logGroupName=log_group_name, + logStreamName=log_stream_name + ) + + events = [{'timestamp': x, 'message': str(x)} for x in range(20)] + + conn.put_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + logEvents=events + ) + + resp = conn.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + limit=10) + + resp['events'].should.have.length_of(10) + resp.should.have.key('nextForwardToken') + resp.should.have.key('nextBackwardToken') + for i in range(10): + resp['events'][i]['timestamp'].should.equal(i) + resp['events'][i]['message'].should.equal(str(i)) + + next_token = resp['nextForwardToken'] + + resp = conn.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + nextToken=next_token, + limit=10) + + resp['events'].should.have.length_of(10) + resp.should.have.key('nextForwardToken') + resp.should.have.key('nextBackwardToken') + resp['nextForwardToken'].should.equal(next_token) + for i in range(10): + resp['events'][i]['timestamp'].should.equal(i+10) + resp['events'][i]['message'].should.equal(str(i+10)) + + resp = conn.get_log_events( + logGroupName=log_group_name, + logStreamName=log_stream_name, + nextToken=resp['nextBackwardToken'], + limit=10) + + resp['events'].should.have.length_of(10) + resp.should.have.key('nextForwardToken') + resp.should.have.key('nextBackwardToken') + for i in range(10): + resp['events'][i]['timestamp'].should.equal(i) + resp['events'][i]['message'].should.equal(str(i)) From a43228c5afdb23b2ca5af1af8ae1ac4a8b665b8c Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 9 Aug 2019 10:15:56 -0500 Subject: [PATCH 59/63] Refactor validating ELB actions. --- moto/elbv2/models.py | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index 508541f91..fe009b84f 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -429,6 +429,17 @@ class ELBv2Backend(BaseBackend): if rule.priority == priority: raise PriorityInUseError() + self._validate_actions(actions) + + # TODO: check for error 'TooManyRegistrationsForTargetId' + # TODO: check for error 'TooManyRules' + + # create rule + rule = FakeRule(listener.arn, conditions, priority, actions, is_default=False) + listener.register(rule) + return [rule] + + def _validate_actions(self, actions): # validate Actions target_group_arns = [target_group.arn for target_group in self.target_groups.values()] for i, action in enumerate(actions): @@ -444,14 +455,6 @@ class ELBv2Backend(BaseBackend): else: raise InvalidActionTypeError(action_type, index) - # TODO: check for error 'TooManyRegistrationsForTargetId' - # TODO: check for error 'TooManyRules' - - # create rule - rule = FakeRule(listener.arn, conditions, priority, actions, is_default=False) - listener.register(rule) - return [rule] - def create_target_group(self, name, **kwargs): if len(name) > 32: raise InvalidTargetGroupNameError( @@ -673,20 +676,7 @@ class ELBv2Backend(BaseBackend): # TODO: check pattern of value for 'path-pattern' # validate Actions - target_group_arns = [target_group.arn for target_group in self.target_groups.values()] - if actions: - for i, action in enumerate(actions): - index = i + 1 - action_type = action['type'] - if action_type == 'forward': - action_target_group_arn = action['target_group_arn'] - if action_target_group_arn not in target_group_arns: - raise ActionTargetGroupNotFoundError(action_target_group_arn) - elif action_type == 'redirect': - # nothing to do - pass - else: - raise InvalidActionTypeError(action_type, index) + self._validate_actions(actions) # TODO: check for error 'TooManyRegistrationsForTargetId' # TODO: check for error 'TooManyRules' From 7b3846f6a1a33c55a4b60837a60dac9ef2f04ee4 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 9 Aug 2019 23:34:52 -0500 Subject: [PATCH 60/63] Refactor Actions to be a real class. Add elb cognito. --- moto/elbv2/models.py | 69 ++++++++++++------ moto/elbv2/responses.py | 47 ++---------- tests/test_elbv2/test_elbv2.py | 129 +++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 64 deletions(-) diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index fe009b84f..7e73c7042 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -2,9 +2,11 @@ from __future__ import unicode_literals import datetime import re +from jinja2 import Template from moto.compat import OrderedDict from moto.core.exceptions import RESTError from moto.core import BaseBackend, BaseModel +from moto.core.utils import camelcase_to_underscores from moto.ec2.models import ec2_backends from moto.acm.models import acm_backends from .utils import make_arn_for_target_group @@ -213,13 +215,12 @@ class FakeListener(BaseModel): action_type = action['Type'] if action_type == 'forward': default_actions.append({'type': action_type, 'target_group_arn': action['TargetGroupArn']}) - elif action_type == 'redirect': - redirect_action = {'type': action_type, } - for redirect_config_key, redirect_config_value in action['RedirectConfig'].items(): + elif action_type in ['redirect', 'authenticate-cognito']: + redirect_action = {'type': action_type} + key = 'RedirectConfig' if action_type == 'redirect' else 'AuthenticateCognitoConfig' + for redirect_config_key, redirect_config_value in action[key].items(): # need to match the output of _get_list_prefix - if redirect_config_key == 'StatusCode': - redirect_config_key = 'status_code' - redirect_action['redirect_config._' + redirect_config_key.lower()] = redirect_config_value + redirect_action[camelcase_to_underscores(key) + '._' + camelcase_to_underscores(redirect_config_key)] = redirect_config_value default_actions.append(redirect_action) else: raise InvalidActionTypeError(action_type, i + 1) @@ -231,6 +232,32 @@ class FakeListener(BaseModel): return listener +class FakeAction(BaseModel): + def __init__(self, data): + self.data = data + self.type = data.get("type") + + def to_xml(self): + template = Template("""{{ action.type }} + {% if action.type == "forward" %} + {{ action.data["target_group_arn"] }} + {% elif action.type == "redirect" %} + + {{ action.data["redirect_config._protocol"] }} + {{ action.data["redirect_config._port"] }} + {{ action.data["redirect_config._status_code"] }} + + {% elif action.type == "authenticate-cognito" %} + + {{ action.data["authenticate_cognito_config._user_pool_arn"] }} + {{ action.data["authenticate_cognito_config._user_pool_client_id"] }} + {{ action.data["authenticate_cognito_config._user_pool_domain"] }} + + {% endif %} + """) + return template.render(action=self) + + class FakeRule(BaseModel): def __init__(self, listener_arn, conditions, priority, actions, is_default): @@ -402,6 +429,7 @@ class ELBv2Backend(BaseBackend): return new_load_balancer def create_rule(self, listener_arn, conditions, priority, actions): + actions = [FakeAction(action) for action in actions] listeners = self.describe_listeners(None, [listener_arn]) if not listeners: raise ListenerNotFoundError() @@ -444,13 +472,12 @@ class ELBv2Backend(BaseBackend): target_group_arns = [target_group.arn for target_group in self.target_groups.values()] for i, action in enumerate(actions): index = i + 1 - action_type = action['type'] + action_type = action.type if action_type == 'forward': - action_target_group_arn = action['target_group_arn'] + action_target_group_arn = action.data['target_group_arn'] if action_target_group_arn not in target_group_arns: raise ActionTargetGroupNotFoundError(action_target_group_arn) - elif action_type == 'redirect': - # nothing to do + elif action_type in ['redirect', 'authenticate-cognito']: pass else: raise InvalidActionTypeError(action_type, index) @@ -498,26 +525,22 @@ class ELBv2Backend(BaseBackend): return target_group def create_listener(self, load_balancer_arn, protocol, port, ssl_policy, certificate, default_actions): + default_actions = [FakeAction(action) for action in default_actions] balancer = self.load_balancers.get(load_balancer_arn) if balancer is None: raise LoadBalancerNotFoundError() if port in balancer.listeners: raise DuplicateListenerError() + self._validate_actions(default_actions) + arn = load_balancer_arn.replace(':loadbalancer/', ':listener/') + "/%s%s" % (port, id(self)) listener = FakeListener(load_balancer_arn, arn, protocol, port, ssl_policy, certificate, default_actions) balancer.listeners[listener.arn] = listener - for i, action in enumerate(default_actions): - action_type = action['type'] - if action_type == 'forward': - if action['target_group_arn'] in self.target_groups.keys(): - target_group = self.target_groups[action['target_group_arn']] - target_group.load_balancer_arns.append(load_balancer_arn) - elif action_type == 'redirect': - # nothing to do - pass - else: - raise InvalidActionTypeError(action_type, i + 1) + for action in default_actions: + if action.type == 'forward': + target_group = self.target_groups[action.data['target_group_arn']] + target_group.load_balancer_arns.append(load_balancer_arn) return listener @@ -651,6 +674,7 @@ class ELBv2Backend(BaseBackend): raise ListenerNotFoundError() def modify_rule(self, rule_arn, conditions, actions): + actions = [FakeAction(action) for action in actions] # if conditions or actions is empty list, do not update the attributes if not conditions and not actions: raise InvalidModifyRuleArgumentsError() @@ -841,6 +865,7 @@ class ELBv2Backend(BaseBackend): return target_group def modify_listener(self, arn, port=None, protocol=None, ssl_policy=None, certificates=None, default_actions=None): + default_actions = [FakeAction(action) for action in default_actions] for load_balancer in self.load_balancers.values(): if arn in load_balancer.listeners: break @@ -907,7 +932,7 @@ class ELBv2Backend(BaseBackend): for listener in load_balancer.listeners.values(): for rule in listener.rules: for action in rule.actions: - if action.get('target_group_arn') == target_group_arn: + if action.data.get('target_group_arn') == target_group_arn: return True return False diff --git a/moto/elbv2/responses.py b/moto/elbv2/responses.py index c98435440..25c23bb17 100644 --- a/moto/elbv2/responses.py +++ b/moto/elbv2/responses.py @@ -775,16 +775,7 @@ CREATE_LISTENER_TEMPLATE = """{{ action["target_group_arn"] }} - {% elif action["type"] == "redirect" %} - - {{ action["redirect_config._protocol"] }} - {{ action["redirect_config._port"] }} - {{ action["redirect_config._status_code"] }} - - {% endif %} + {{ action.to_xml() }} {% endfor %} @@ -888,16 +879,7 @@ DESCRIBE_RULES_TEMPLATE = """ - {% if action["type"] == "forward" %} - {{ action["target_group_arn"] }} - {% elif action["type"] == "redirect" %} - - {{ action["redirect_config._protocol"] }} - {{ action["redirect_config._port"] }} - {{ action["redirect_config._status_code"] }} - - {% endif %} + {{ action.to_xml() }} {% endfor %} @@ -989,16 +971,7 @@ DESCRIBE_LISTENERS_TEMPLATE = """{{ action["target_group_arn"] }}m - {% elif action["type"] == "redirect" %} - - {{ action["redirect_config._protocol"] }} - {{ action["redirect_config._port"] }} - {{ action["redirect_config._status_code"] }} - - {% endif %} + {{ action.to_xml() }} {% endfor %} @@ -1048,8 +1021,7 @@ MODIFY_RULE_TEMPLATE = """ - {{ action["target_group_arn"] }} + {{ action.to_xml() }} {% endfor %} @@ -1432,16 +1404,7 @@ MODIFY_LISTENER_TEMPLATE = """{{ action["target_group_arn"] }} - {% elif action["type"] == "redirect" %} - - {{ action["redirect_config._protocol"] }} - {{ action["redirect_config._port"] }} - {{ action["redirect_config._status_code"] }} - - {% endif %} + {{ action.to_xml() }} {% endfor %} diff --git a/tests/test_elbv2/test_elbv2.py b/tests/test_elbv2/test_elbv2.py index 879a04cd8..36772c02e 100644 --- a/tests/test_elbv2/test_elbv2.py +++ b/tests/test_elbv2/test_elbv2.py @@ -1811,3 +1811,132 @@ def test_redirect_action_listener_rule_cloudformation(): 'Port': '443', 'Protocol': 'HTTPS', 'StatusCode': 'HTTP_301', } },]) + + +@mock_elbv2 +@mock_ec2 +def test_cognito_action_listener_rule(): + conn = boto3.client('elbv2', region_name='us-east-1') + ec2 = boto3.resource('ec2', region_name='us-east-1') + + security_group = ec2.create_security_group( + GroupName='a-security-group', Description='First One') + vpc = ec2.create_vpc(CidrBlock='172.28.7.0/24', InstanceTenancy='default') + subnet1 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.192/26', + AvailabilityZone='us-east-1a') + subnet2 = ec2.create_subnet( + VpcId=vpc.id, + CidrBlock='172.28.7.128/26', + AvailabilityZone='us-east-1b') + + response = conn.create_load_balancer( + Name='my-lb', + Subnets=[subnet1.id, subnet2.id], + SecurityGroups=[security_group.id], + Scheme='internal', + Tags=[{'Key': 'key_name', 'Value': 'a_value'}]) + load_balancer_arn = response.get('LoadBalancers')[0].get('LoadBalancerArn') + + action = { + 'Type': 'authenticate-cognito', + 'AuthenticateCognitoConfig': { + 'UserPoolArn': 'arn:aws:cognito-idp:us-east-1:123456789012:userpool/us-east-1_ABCD1234', + 'UserPoolClientId': 'abcd1234abcd', + 'UserPoolDomain': 'testpool', + } + } + response = conn.create_listener(LoadBalancerArn=load_balancer_arn, + Protocol='HTTP', + Port=80, + DefaultActions=[action]) + + listener = response.get('Listeners')[0] + listener.get('DefaultActions')[0].should.equal(action) + listener_arn = listener.get('ListenerArn') + + describe_rules_response = conn.describe_rules(ListenerArn=listener_arn) + describe_rules_response['Rules'][0]['Actions'][0].should.equal(action) + + describe_listener_response = conn.describe_listeners(ListenerArns=[listener_arn, ]) + describe_listener_actions = describe_listener_response['Listeners'][0]['DefaultActions'][0] + describe_listener_actions.should.equal(action) + + +@mock_elbv2 +@mock_cloudformation +def test_cognito_action_listener_rule_cloudformation(): + cnf_conn = boto3.client('cloudformation', region_name='us-east-1') + elbv2_client = boto3.client('elbv2', region_name='us-east-1') + + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "ECS Cluster Test CloudFormation", + "Resources": { + "testVPC": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + }, + }, + "subnet1": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.0.0/24", + "VpcId": {"Ref": "testVPC"}, + "AvalabilityZone": "us-east-1b", + }, + }, + "subnet2": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.1.0/24", + "VpcId": {"Ref": "testVPC"}, + "AvalabilityZone": "us-east-1b", + }, + }, + "testLb": { + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", + "Properties": { + "Name": "my-lb", + "Subnets": [{"Ref": "subnet1"}, {"Ref": "subnet2"}], + "Type": "application", + "SecurityGroups": [], + } + }, + "testListener": { + "Type": "AWS::ElasticLoadBalancingV2::Listener", + "Properties": { + "LoadBalancerArn": {"Ref": "testLb"}, + "Port": 80, + "Protocol": "HTTP", + "DefaultActions": [{ + "Type": "authenticate-cognito", + "AuthenticateCognitoConfig": { + 'UserPoolArn': 'arn:aws:cognito-idp:us-east-1:123456789012:userpool/us-east-1_ABCD1234', + 'UserPoolClientId': 'abcd1234abcd', + 'UserPoolDomain': 'testpool', + } + }] + } + + } + } + } + template_json = json.dumps(template) + cnf_conn.create_stack(StackName="test-stack", TemplateBody=template_json) + + describe_load_balancers_response = elbv2_client.describe_load_balancers(Names=['my-lb',]) + load_balancer_arn = describe_load_balancers_response['LoadBalancers'][0]['LoadBalancerArn'] + describe_listeners_response = elbv2_client.describe_listeners(LoadBalancerArn=load_balancer_arn) + + describe_listeners_response['Listeners'].should.have.length_of(1) + describe_listeners_response['Listeners'][0]['DefaultActions'].should.equal([{ + 'Type': 'authenticate-cognito', + "AuthenticateCognitoConfig": { + 'UserPoolArn': 'arn:aws:cognito-idp:us-east-1:123456789012:userpool/us-east-1_ABCD1234', + 'UserPoolClientId': 'abcd1234abcd', + 'UserPoolDomain': 'testpool', + } + },]) From 5347a577da36a114cb8931430737bd8d42bf669a Mon Sep 17 00:00:00 2001 From: Earl Robinson Date: Wed, 14 Aug 2019 08:19:32 -0400 Subject: [PATCH 61/63] restore KeyId to kms decrypt response regression introduced in #2071 "KMS generate_data_key" --- moto/kms/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 8cd6e7663..53012b7f8 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -238,7 +238,7 @@ class KmsResponse(BaseResponse): value = self.parameters.get("CiphertextBlob") try: - return json.dumps({"Plaintext": base64.b64decode(value).decode("utf-8")}) + return json.dumps({"Plaintext": base64.b64decode(value).decode("utf-8"), 'KeyId': 'key_id'}) except UnicodeDecodeError: # Generate data key will produce random bytes which when decrypted is still returned as base64 return json.dumps({"Plaintext": value}) From bbaff4b273f37386f660eb3acde7edac1b04854b Mon Sep 17 00:00:00 2001 From: Earl Robinson Date: Wed, 14 Aug 2019 08:39:54 -0400 Subject: [PATCH 62/63] restore KeyId to test_decrypt in test_kms --- tests/test_kms/test_kms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 8fe0620f1..f189fbe41 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -191,6 +191,7 @@ def test_decrypt(): conn = boto.kms.connect_to_region('us-west-2') response = conn.decrypt('ZW5jcnlwdG1l'.encode('utf-8')) response['Plaintext'].should.equal(b'encryptme') + response['KeyId'].should.equal('key_id') @mock_kms_deprecated From 188969a048e01c772121fb8461deb74998e0b90c Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Mon, 29 Jul 2019 16:36:57 -0700 Subject: [PATCH 63/63] AWS Config Aggregator support - Added support for the following APIs: - put_configuration_aggregator - describe_configuration_aggregators - delete_configuration_aggregator - put_aggregation_authorization - describe_aggregation_authorizations - delete_aggregation_authorization --- README.md | 91 ++++++ moto/config/exceptions.py | 83 +++++ moto/config/models.py | 370 +++++++++++++++++++++- moto/config/responses.py | 33 ++ moto/iam/models.py | 1 - setup.py | 4 +- tests/test_config/test_config.py | 520 +++++++++++++++++++++++++++++++ 7 files changed, 1090 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 5a1d7f1f1..4e39ada35 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L | Cognito Identity Provider | @mock_cognitoidp | basic endpoints done | |-------------------------------------------------------------------------------------| | Config | @mock_config | basic endpoints done | +| | | core endpoints done | |-------------------------------------------------------------------------------------| | Data Pipeline | @mock_datapipeline | basic endpoints done | |-------------------------------------------------------------------------------------| @@ -296,6 +297,96 @@ def test_describe_instances_allowed(): See [the related test suite](https://github.com/spulec/moto/blob/master/tests/test_core/test_auth.py) for more examples. +## Very Important -- Recommended Usage +There are some important caveats to be aware of when using moto: + +*Failure to follow these guidelines could result in your tests mutating your __REAL__ infrastructure!* + +### How do I avoid tests from mutating my real infrastructure? +You need to ensure that the mocks are actually in place. Changes made to recent versions of `botocore` +have altered some of the mock behavior. In short, you need to ensure that you _always_ do the following: + +1. Ensure that your tests have dummy environment variables set up: + + export AWS_ACCESS_KEY_ID='testing' + export AWS_SECRET_ACCESS_KEY='testing' + export AWS_SECURITY_TOKEN='testing' + export AWS_SESSION_TOKEN='testing' + +1. __VERY IMPORTANT__: ensure that you have your mocks set up __BEFORE__ your `boto3` client is established. + This can typically happen if you import a module that has a `boto3` client instantiated outside of a function. + See the pesky imports section below on how to work around this. + +### Example on usage? +If you are a user of [pytest](https://pytest.org/en/latest/), you can leverage [pytest fixtures](https://pytest.org/en/latest/fixture.html#fixture) +to help set up your mocks and other AWS resources that you would need. + +Here is an example: +```python +@pytest.fixture(scope='function') +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ['AWS_ACCESS_KEY_ID'] = 'testing' + os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing' + os.environ['AWS_SECURITY_TOKEN'] = 'testing' + os.environ['AWS_SESSION_TOKEN'] = 'testing' + +@pytest.fixture(scope='function') +def s3(aws_credentials): + with mock_s3(): + yield boto3.client('s3', region_name='us-east-1') + + +@pytest.fixture(scope='function') +def sts(aws_credentials): + with mock_sts(): + yield boto3.client('sts', region_name='us-east-1') + + +@pytest.fixture(scope='function') +def cloudwatch(aws_credentials): + with mock_cloudwatch(): + yield boto3.client('cloudwatch', region_name='us-east-1') + +... etc. +``` + +In the code sample above, all of the AWS/mocked fixtures take in a parameter of `aws_credentials`, +which sets the proper fake environment variables. The fake environment variables are used so that `botocore` doesn't try to locate real +credentials on your system. + +Next, once you need to do anything with the mocked AWS environment, do something like: +```python +def test_create_bucket(s3): + # s3 is a fixture defined above that yields a boto3 s3 client. + # Feel free to instantiate another boto3 S3 client -- Keep note of the region though. + s3.create_bucket(Bucket="somebucket") + + result = s3.list_buckets() + assert len(result['Buckets']) == 1 + assert result['Buckets'][0]['Name'] == 'somebucket' +``` + +### What about those pesky imports? +Recall earlier, it was mentioned that mocks should be established __BEFORE__ the clients are set up. One way +to avoid import issues is to make use of local Python imports -- i.e. import the module inside of the unit +test you want to run vs. importing at the top of the file. + +Example: +```python +def test_something(s3): + from some.package.that.does.something.with.s3 import some_func # <-- Local import for unit test + # ^^ Importing here ensures that the mock has been established. + + sume_func() # The mock has been established from the "s3" pytest fixture, so this function that uses + # a package-level S3 client will properly use the mock and not reach out to AWS. +``` + +### Other caveats +For Tox, Travis CI, and other build systems, you might need to also perform a `touch ~/.aws/credentials` +command before running the tests. As long as that file is present (empty preferably) and the environment +variables above are set, you should be good to go. + ## Stand-alone Server Mode Moto also has a stand-alone server mode. This allows you to utilize diff --git a/moto/config/exceptions.py b/moto/config/exceptions.py index b2b01d6a0..25749200f 100644 --- a/moto/config/exceptions.py +++ b/moto/config/exceptions.py @@ -52,6 +52,18 @@ class InvalidResourceTypeException(JsonRESTError): super(InvalidResourceTypeException, self).__init__("ValidationException", message) +class NoSuchConfigurationAggregatorException(JsonRESTError): + code = 400 + + def __init__(self, number=1): + if number == 1: + message = 'The configuration aggregator does not exist. Check the configuration aggregator name and try again.' + else: + message = 'At least one of the configuration aggregators does not exist. Check the configuration aggregator' \ + ' names and try again.' + super(NoSuchConfigurationAggregatorException, self).__init__("NoSuchConfigurationAggregatorException", message) + + class NoSuchConfigurationRecorderException(JsonRESTError): code = 400 @@ -78,6 +90,14 @@ class NoSuchBucketException(JsonRESTError): super(NoSuchBucketException, self).__init__("NoSuchBucketException", message) +class InvalidNextTokenException(JsonRESTError): + code = 400 + + def __init__(self): + message = 'The nextToken provided is invalid' + super(InvalidNextTokenException, self).__init__("InvalidNextTokenException", message) + + class InvalidS3KeyPrefixException(JsonRESTError): code = 400 @@ -147,3 +167,66 @@ class LastDeliveryChannelDeleteFailedException(JsonRESTError): message = 'Failed to delete last specified delivery channel with name \'{name}\', because there, ' \ 'because there is a running configuration recorder.'.format(name=name) super(LastDeliveryChannelDeleteFailedException, self).__init__("LastDeliveryChannelDeleteFailedException", message) + + +class TooManyAccountSources(JsonRESTError): + code = 400 + + def __init__(self, length): + locations = ['com.amazonaws.xyz'] * length + + message = 'Value \'[{locations}]\' at \'accountAggregationSources\' failed to satisfy constraint: ' \ + 'Member must have length less than or equal to 1'.format(locations=', '.join(locations)) + super(TooManyAccountSources, self).__init__("ValidationException", message) + + +class DuplicateTags(JsonRESTError): + code = 400 + + def __init__(self): + super(DuplicateTags, self).__init__( + 'InvalidInput', 'Duplicate tag keys found. Please note that Tag keys are case insensitive.') + + +class TagKeyTooBig(JsonRESTError): + code = 400 + + def __init__(self, tag, param='tags.X.member.key'): + super(TagKeyTooBig, self).__init__( + 'ValidationException', "1 validation error detected: Value '{}' at '{}' failed to satisfy " + "constraint: Member must have length less than or equal to 128".format(tag, param)) + + +class TagValueTooBig(JsonRESTError): + code = 400 + + def __init__(self, tag): + super(TagValueTooBig, self).__init__( + 'ValidationException', "1 validation error detected: Value '{}' at 'tags.X.member.value' failed to satisfy " + "constraint: Member must have length less than or equal to 256".format(tag)) + + +class InvalidParameterValueException(JsonRESTError): + code = 400 + + def __init__(self, message): + super(InvalidParameterValueException, self).__init__('InvalidParameterValueException', message) + + +class InvalidTagCharacters(JsonRESTError): + code = 400 + + def __init__(self, tag, param='tags.X.member.key'): + message = "1 validation error detected: Value '{}' at '{}' failed to satisfy ".format(tag, param) + message += 'constraint: Member must satisfy regular expression pattern: [\\\\p{L}\\\\p{Z}\\\\p{N}_.:/=+\\\\-@]+' + + super(InvalidTagCharacters, self).__init__('ValidationException', message) + + +class TooManyTags(JsonRESTError): + code = 400 + + def __init__(self, tags, param='tags'): + super(TooManyTags, self).__init__( + 'ValidationException', "1 validation error detected: Value '{}' at '{}' failed to satisfy " + "constraint: Member must have length less than or equal to 50.".format(tags, param)) diff --git a/moto/config/models.py b/moto/config/models.py index cd6e07afa..6541fc981 100644 --- a/moto/config/models.py +++ b/moto/config/models.py @@ -1,6 +1,9 @@ import json +import re import time import pkg_resources +import random +import string from datetime import datetime @@ -12,37 +15,125 @@ from moto.config.exceptions import InvalidResourceTypeException, InvalidDelivery NoSuchConfigurationRecorderException, NoAvailableConfigurationRecorderException, \ InvalidDeliveryChannelNameException, NoSuchBucketException, InvalidS3KeyPrefixException, \ InvalidSNSTopicARNException, MaxNumberOfDeliveryChannelsExceededException, NoAvailableDeliveryChannelException, \ - NoSuchDeliveryChannelException, LastDeliveryChannelDeleteFailedException + NoSuchDeliveryChannelException, LastDeliveryChannelDeleteFailedException, TagKeyTooBig, \ + TooManyTags, TagValueTooBig, TooManyAccountSources, InvalidParameterValueException, InvalidNextTokenException, \ + NoSuchConfigurationAggregatorException, InvalidTagCharacters, DuplicateTags from moto.core import BaseBackend, BaseModel DEFAULT_ACCOUNT_ID = 123456789012 +POP_STRINGS = [ + 'capitalizeStart', + 'CapitalizeStart', + 'capitalizeArn', + 'CapitalizeArn', + 'capitalizeARN', + 'CapitalizeARN' +] +DEFAULT_PAGE_SIZE = 100 def datetime2int(date): return int(time.mktime(date.timetuple())) -def snake_to_camels(original): +def snake_to_camels(original, cap_start, cap_arn): parts = original.split('_') camel_cased = parts[0].lower() + ''.join(p.title() for p in parts[1:]) - camel_cased = camel_cased.replace('Arn', 'ARN') # Config uses 'ARN' instead of 'Arn' + + if cap_arn: + camel_cased = camel_cased.replace('Arn', 'ARN') # Some config services use 'ARN' instead of 'Arn' + + if cap_start: + camel_cased = camel_cased[0].upper() + camel_cased[1::] return camel_cased +def random_string(): + """Returns a random set of 8 lowercase letters for the Config Aggregator ARN""" + chars = [] + for x in range(0, 8): + chars.append(random.choice(string.ascii_lowercase)) + + return "".join(chars) + + +def validate_tag_key(tag_key, exception_param='tags.X.member.key'): + """Validates the tag key. + + :param tag_key: The tag key to check against. + :param exception_param: The exception parameter to send over to help format the message. This is to reflect + the difference between the tag and untag APIs. + :return: + """ + # Validate that the key length is correct: + if len(tag_key) > 128: + raise TagKeyTooBig(tag_key, param=exception_param) + + # Validate that the tag key fits the proper Regex: + # [\w\s_.:/=+\-@]+ SHOULD be the same as the Java regex on the AWS documentation: [\p{L}\p{Z}\p{N}_.:/=+\-@]+ + match = re.findall(r'[\w\s_.:/=+\-@]+', tag_key) + # Kudos if you can come up with a better way of doing a global search :) + if not len(match) or len(match[0]) < len(tag_key): + raise InvalidTagCharacters(tag_key, param=exception_param) + + +def check_tag_duplicate(all_tags, tag_key): + """Validates that a tag key is not a duplicate + + :param all_tags: Dict to check if there is a duplicate tag. + :param tag_key: The tag key to check against. + :return: + """ + if all_tags.get(tag_key): + raise DuplicateTags() + + +def validate_tags(tags): + proper_tags = {} + + if len(tags) > 50: + raise TooManyTags(tags) + + for tag in tags: + # Validate the Key: + validate_tag_key(tag['Key']) + check_tag_duplicate(proper_tags, tag['Key']) + + # Validate the Value: + if len(tag['Value']) > 256: + raise TagValueTooBig(tag['Value']) + + proper_tags[tag['Key']] = tag['Value'] + + return proper_tags + + class ConfigEmptyDictable(BaseModel): """Base class to make serialization easy. This assumes that the sub-class will NOT return 'None's in the JSON.""" + def __init__(self, capitalize_start=False, capitalize_arn=True): + """Assists with the serialization of the config object + :param capitalize_start: For some Config services, the first letter is lowercase -- for others it's capital + :param capitalize_arn: For some Config services, the API expects 'ARN' and for others, it expects 'Arn' + """ + self.capitalize_start = capitalize_start + self.capitalize_arn = capitalize_arn + def to_dict(self): data = {} for item, value in self.__dict__.items(): if value is not None: if isinstance(value, ConfigEmptyDictable): - data[snake_to_camels(item)] = value.to_dict() + data[snake_to_camels(item, self.capitalize_start, self.capitalize_arn)] = value.to_dict() else: - data[snake_to_camels(item)] = value + data[snake_to_camels(item, self.capitalize_start, self.capitalize_arn)] = value + + # Cleanse the extra properties: + for prop in POP_STRINGS: + data.pop(prop, None) return data @@ -50,8 +141,9 @@ class ConfigEmptyDictable(BaseModel): class ConfigRecorderStatus(ConfigEmptyDictable): def __init__(self, name): - self.name = name + super(ConfigRecorderStatus, self).__init__() + self.name = name self.recording = False self.last_start_time = None self.last_stop_time = None @@ -75,12 +167,16 @@ class ConfigRecorderStatus(ConfigEmptyDictable): class ConfigDeliverySnapshotProperties(ConfigEmptyDictable): def __init__(self, delivery_frequency): + super(ConfigDeliverySnapshotProperties, self).__init__() + self.delivery_frequency = delivery_frequency class ConfigDeliveryChannel(ConfigEmptyDictable): def __init__(self, name, s3_bucket_name, prefix=None, sns_arn=None, snapshot_properties=None): + super(ConfigDeliveryChannel, self).__init__() + self.name = name self.s3_bucket_name = s3_bucket_name self.s3_key_prefix = prefix @@ -91,6 +187,8 @@ class ConfigDeliveryChannel(ConfigEmptyDictable): class RecordingGroup(ConfigEmptyDictable): def __init__(self, all_supported=True, include_global_resource_types=False, resource_types=None): + super(RecordingGroup, self).__init__() + self.all_supported = all_supported self.include_global_resource_types = include_global_resource_types self.resource_types = resource_types @@ -99,6 +197,8 @@ class RecordingGroup(ConfigEmptyDictable): class ConfigRecorder(ConfigEmptyDictable): def __init__(self, role_arn, recording_group, name='default', status=None): + super(ConfigRecorder, self).__init__() + self.name = name self.role_arn = role_arn self.recording_group = recording_group @@ -109,18 +209,118 @@ class ConfigRecorder(ConfigEmptyDictable): self.status = status +class AccountAggregatorSource(ConfigEmptyDictable): + + def __init__(self, account_ids, aws_regions=None, all_aws_regions=None): + super(AccountAggregatorSource, self).__init__(capitalize_start=True) + + # Can't have both the regions and all_regions flag present -- also can't have them both missing: + if aws_regions and all_aws_regions: + raise InvalidParameterValueException('Your configuration aggregator contains a list of regions and also specifies ' + 'the use of all regions. You must choose one of these options.') + + if not (aws_regions or all_aws_regions): + raise InvalidParameterValueException('Your request does not specify any regions. Select AWS Config-supported ' + 'regions and try again.') + + self.account_ids = account_ids + self.aws_regions = aws_regions + + if not all_aws_regions: + all_aws_regions = False + + self.all_aws_regions = all_aws_regions + + +class OrganizationAggregationSource(ConfigEmptyDictable): + + def __init__(self, role_arn, aws_regions=None, all_aws_regions=None): + super(OrganizationAggregationSource, self).__init__(capitalize_start=True, capitalize_arn=False) + + # Can't have both the regions and all_regions flag present -- also can't have them both missing: + if aws_regions and all_aws_regions: + raise InvalidParameterValueException('Your configuration aggregator contains a list of regions and also specifies ' + 'the use of all regions. You must choose one of these options.') + + if not (aws_regions or all_aws_regions): + raise InvalidParameterValueException('Your request does not specify any regions. Select AWS Config-supported ' + 'regions and try again.') + + self.role_arn = role_arn + self.aws_regions = aws_regions + + if not all_aws_regions: + all_aws_regions = False + + self.all_aws_regions = all_aws_regions + + +class ConfigAggregator(ConfigEmptyDictable): + + def __init__(self, name, region, account_sources=None, org_source=None, tags=None): + super(ConfigAggregator, self).__init__(capitalize_start=True, capitalize_arn=False) + + self.configuration_aggregator_name = name + self.configuration_aggregator_arn = 'arn:aws:config:{region}:{id}:config-aggregator/config-aggregator-{random}'.format( + region=region, + id=DEFAULT_ACCOUNT_ID, + random=random_string() + ) + self.account_aggregation_sources = account_sources + self.organization_aggregation_source = org_source + self.creation_time = datetime2int(datetime.utcnow()) + self.last_updated_time = datetime2int(datetime.utcnow()) + + # Tags are listed in the list_tags_for_resource API call ... not implementing yet -- please feel free to! + self.tags = tags or {} + + # Override the to_dict so that we can format the tags properly... + def to_dict(self): + result = super(ConfigAggregator, self).to_dict() + + # Override the account aggregation sources if present: + if self.account_aggregation_sources: + result['AccountAggregationSources'] = [a.to_dict() for a in self.account_aggregation_sources] + + # Tags are listed in the list_tags_for_resource API call ... not implementing yet -- please feel free to! + # if self.tags: + # result['Tags'] = [{'Key': key, 'Value': value} for key, value in self.tags.items()] + + return result + + +class ConfigAggregationAuthorization(ConfigEmptyDictable): + + def __init__(self, current_region, authorized_account_id, authorized_aws_region, tags=None): + super(ConfigAggregationAuthorization, self).__init__(capitalize_start=True, capitalize_arn=False) + + self.aggregation_authorization_arn = 'arn:aws:config:{region}:{id}:aggregation-authorization/' \ + '{auth_account}/{auth_region}'.format(region=current_region, + id=DEFAULT_ACCOUNT_ID, + auth_account=authorized_account_id, + auth_region=authorized_aws_region) + self.authorized_account_id = authorized_account_id + self.authorized_aws_region = authorized_aws_region + self.creation_time = datetime2int(datetime.utcnow()) + + # Tags are listed in the list_tags_for_resource API call ... not implementing yet -- please feel free to! + self.tags = tags or {} + + class ConfigBackend(BaseBackend): def __init__(self): self.recorders = {} self.delivery_channels = {} + self.config_aggregators = {} + self.aggregation_authorizations = {} @staticmethod def _validate_resource_types(resource_list): # Load the service file: resource_package = 'botocore' resource_path = '/'.join(('data', 'config', '2014-11-12', 'service-2.json')) - conifg_schema = json.loads(pkg_resources.resource_string(resource_package, resource_path)) + config_schema = json.loads(pkg_resources.resource_string(resource_package, resource_path)) # Verify that each entry exists in the supported list: bad_list = [] @@ -128,11 +328,11 @@ class ConfigBackend(BaseBackend): # For PY2: r_str = str(resource) - if r_str not in conifg_schema['shapes']['ResourceType']['enum']: + if r_str not in config_schema['shapes']['ResourceType']['enum']: bad_list.append(r_str) if bad_list: - raise InvalidResourceTypeException(bad_list, conifg_schema['shapes']['ResourceType']['enum']) + raise InvalidResourceTypeException(bad_list, config_schema['shapes']['ResourceType']['enum']) @staticmethod def _validate_delivery_snapshot_properties(properties): @@ -147,6 +347,158 @@ class ConfigBackend(BaseBackend): raise InvalidDeliveryFrequency(properties.get('deliveryFrequency', None), conifg_schema['shapes']['MaximumExecutionFrequency']['enum']) + def put_configuration_aggregator(self, config_aggregator, region): + # Validate the name: + if len(config_aggregator['ConfigurationAggregatorName']) > 256: + raise NameTooLongException(config_aggregator['ConfigurationAggregatorName'], 'configurationAggregatorName') + + account_sources = None + org_source = None + + # Tag validation: + tags = validate_tags(config_aggregator.get('Tags', [])) + + # Exception if both AccountAggregationSources and OrganizationAggregationSource are supplied: + if config_aggregator.get('AccountAggregationSources') and config_aggregator.get('OrganizationAggregationSource'): + raise InvalidParameterValueException('The configuration aggregator cannot be created because your request contains both the' + ' AccountAggregationSource and the OrganizationAggregationSource. Include only ' + 'one aggregation source and try again.') + + # If neither are supplied: + if not config_aggregator.get('AccountAggregationSources') and not config_aggregator.get('OrganizationAggregationSource'): + raise InvalidParameterValueException('The configuration aggregator cannot be created because your request is missing either ' + 'the AccountAggregationSource or the OrganizationAggregationSource. Include the ' + 'appropriate aggregation source and try again.') + + if config_aggregator.get('AccountAggregationSources'): + # Currently, only 1 account aggregation source can be set: + if len(config_aggregator['AccountAggregationSources']) > 1: + raise TooManyAccountSources(len(config_aggregator['AccountAggregationSources'])) + + account_sources = [] + for a in config_aggregator['AccountAggregationSources']: + account_sources.append(AccountAggregatorSource(a['AccountIds'], aws_regions=a.get('AwsRegions'), + all_aws_regions=a.get('AllAwsRegions'))) + + else: + org_source = OrganizationAggregationSource(config_aggregator['OrganizationAggregationSource']['RoleArn'], + aws_regions=config_aggregator['OrganizationAggregationSource'].get('AwsRegions'), + all_aws_regions=config_aggregator['OrganizationAggregationSource'].get( + 'AllAwsRegions')) + + # Grab the existing one if it exists and update it: + if not self.config_aggregators.get(config_aggregator['ConfigurationAggregatorName']): + aggregator = ConfigAggregator(config_aggregator['ConfigurationAggregatorName'], region, account_sources=account_sources, + org_source=org_source, tags=tags) + self.config_aggregators[config_aggregator['ConfigurationAggregatorName']] = aggregator + + else: + aggregator = self.config_aggregators[config_aggregator['ConfigurationAggregatorName']] + aggregator.tags = tags + aggregator.account_aggregation_sources = account_sources + aggregator.organization_aggregation_source = org_source + aggregator.last_updated_time = datetime2int(datetime.utcnow()) + + return aggregator.to_dict() + + def describe_configuration_aggregators(self, names, token, limit): + limit = DEFAULT_PAGE_SIZE if not limit or limit < 0 else limit + agg_list = [] + result = {'ConfigurationAggregators': []} + + if names: + for name in names: + if not self.config_aggregators.get(name): + raise NoSuchConfigurationAggregatorException(number=len(names)) + + agg_list.append(name) + + else: + agg_list = list(self.config_aggregators.keys()) + + # Empty? + if not agg_list: + return result + + # Sort by name: + sorted_aggregators = sorted(agg_list) + + # Get the start: + if not token: + start = 0 + else: + # Tokens for this moto feature are just the next names of the items in the list: + if not self.config_aggregators.get(token): + raise InvalidNextTokenException() + + start = sorted_aggregators.index(token) + + # Get the list of items to collect: + agg_list = sorted_aggregators[start:(start + limit)] + result['ConfigurationAggregators'] = [self.config_aggregators[agg].to_dict() for agg in agg_list] + + if len(sorted_aggregators) > (start + limit): + result['NextToken'] = sorted_aggregators[start + limit] + + return result + + def delete_configuration_aggregator(self, config_aggregator): + if not self.config_aggregators.get(config_aggregator): + raise NoSuchConfigurationAggregatorException() + + del self.config_aggregators[config_aggregator] + + def put_aggregation_authorization(self, current_region, authorized_account, authorized_region, tags): + # Tag validation: + tags = validate_tags(tags or []) + + # Does this already exist? + key = '{}/{}'.format(authorized_account, authorized_region) + agg_auth = self.aggregation_authorizations.get(key) + if not agg_auth: + agg_auth = ConfigAggregationAuthorization(current_region, authorized_account, authorized_region, tags=tags) + self.aggregation_authorizations['{}/{}'.format(authorized_account, authorized_region)] = agg_auth + else: + # Only update the tags: + agg_auth.tags = tags + + return agg_auth.to_dict() + + def describe_aggregation_authorizations(self, token, limit): + limit = DEFAULT_PAGE_SIZE if not limit or limit < 0 else limit + result = {'AggregationAuthorizations': []} + + if not self.aggregation_authorizations: + return result + + # Sort by name: + sorted_authorizations = sorted(self.aggregation_authorizations.keys()) + + # Get the start: + if not token: + start = 0 + else: + # Tokens for this moto feature are just the next names of the items in the list: + if not self.aggregation_authorizations.get(token): + raise InvalidNextTokenException() + + start = sorted_authorizations.index(token) + + # Get the list of items to collect: + auth_list = sorted_authorizations[start:(start + limit)] + result['AggregationAuthorizations'] = [self.aggregation_authorizations[auth].to_dict() for auth in auth_list] + + if len(sorted_authorizations) > (start + limit): + result['NextToken'] = sorted_authorizations[start + limit] + + return result + + def delete_aggregation_authorization(self, authorized_account, authorized_region): + # This will always return a 200 -- regardless if there is or isn't an existing + # aggregation authorization. + key = '{}/{}'.format(authorized_account, authorized_region) + self.aggregation_authorizations.pop(key, None) + def put_configuration_recorder(self, config_recorder): # Validate the name: if not config_recorder.get('name'): diff --git a/moto/config/responses.py b/moto/config/responses.py index 286b2349f..03612d403 100644 --- a/moto/config/responses.py +++ b/moto/config/responses.py @@ -13,6 +13,39 @@ class ConfigResponse(BaseResponse): self.config_backend.put_configuration_recorder(self._get_param('ConfigurationRecorder')) return "" + def put_configuration_aggregator(self): + aggregator = self.config_backend.put_configuration_aggregator(json.loads(self.body), self.region) + schema = {'ConfigurationAggregator': aggregator} + return json.dumps(schema) + + def describe_configuration_aggregators(self): + aggregators = self.config_backend.describe_configuration_aggregators(self._get_param('ConfigurationAggregatorNames'), + self._get_param('NextToken'), + self._get_param('Limit')) + return json.dumps(aggregators) + + def delete_configuration_aggregator(self): + self.config_backend.delete_configuration_aggregator(self._get_param('ConfigurationAggregatorName')) + return "" + + def put_aggregation_authorization(self): + agg_auth = self.config_backend.put_aggregation_authorization(self.region, + self._get_param('AuthorizedAccountId'), + self._get_param('AuthorizedAwsRegion'), + self._get_param('Tags')) + schema = {'AggregationAuthorization': agg_auth} + return json.dumps(schema) + + def describe_aggregation_authorizations(self): + authorizations = self.config_backend.describe_aggregation_authorizations(self._get_param('NextToken'), self._get_param('Limit')) + + return json.dumps(authorizations) + + def delete_aggregation_authorization(self): + self.config_backend.delete_aggregation_authorization(self._get_param('AuthorizedAccountId'), self._get_param('AuthorizedAwsRegion')) + + return "" + def describe_configuration_recorders(self): recorders = self.config_backend.describe_configuration_recorders(self._get_param('ConfigurationRecorderNames')) schema = {'ConfigurationRecorders': recorders} diff --git a/moto/iam/models.py b/moto/iam/models.py index bb19b8cad..21bb87e02 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -694,7 +694,6 @@ class IAMBackend(BaseBackend): def _validate_tag_key(self, tag_key, exception_param='tags.X.member.key'): """Validates the tag key. - :param all_tags: Dict to check if there is a duplicate tag. :param tag_key: The tag key to check against. :param exception_param: The exception parameter to send over to help format the message. This is to reflect the difference between the tag and untag APIs. diff --git a/setup.py b/setup.py index 17a4f6691..ff4d9720a 100755 --- a/setup.py +++ b/setup.py @@ -30,8 +30,8 @@ def get_version(): install_requires = [ "Jinja2>=2.10.1", "boto>=2.36.0", - "boto3>=1.9.86", - "botocore>=1.12.86", + "boto3>=1.9.201", + "botocore>=1.12.201", "cryptography>=2.3.0", "requests>=2.5", "xmltodict", diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index 96c62455c..95e88cab1 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -123,6 +123,526 @@ def test_put_configuration_recorder(): assert "maximum number of configuration recorders: 1 is reached." in ce.exception.response['Error']['Message'] +@mock_config +def test_put_configuration_aggregator(): + client = boto3.client('config', region_name='us-west-2') + + # With too many aggregation sources: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + '111111111111', + '222222222222' + ], + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ] + }, + { + 'AccountIds': [ + '012345678910', + '111111111111', + '222222222222' + ], + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ] + } + ] + ) + assert 'Member must have length less than or equal to 1' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # With an invalid region config (no regions defined): + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + '111111111111', + '222222222222' + ], + 'AllAwsRegions': False + } + ] + ) + assert 'Your request does not specify any regions' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + OrganizationAggregationSource={ + 'RoleArn': 'arn:aws:iam::012345678910:role/SomeRole' + } + ) + assert 'Your request does not specify any regions' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + # With both region flags defined: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + '111111111111', + '222222222222' + ], + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ], + 'AllAwsRegions': True + } + ] + ) + assert 'You must choose one of these options' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + OrganizationAggregationSource={ + 'RoleArn': 'arn:aws:iam::012345678910:role/SomeRole', + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ], + 'AllAwsRegions': True + } + ) + assert 'You must choose one of these options' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + # Name too long: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='a' * 257, + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ] + ) + assert 'configurationAggregatorName' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Too many tags (>50): + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ], + Tags=[{'Key': '{}'.format(x), 'Value': '{}'.format(x)} for x in range(0, 51)] + ) + assert 'Member must have length less than or equal to 50' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Tag key is too big (>128 chars): + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ], + Tags=[{'Key': 'a' * 129, 'Value': 'a'}] + ) + assert 'Member must have length less than or equal to 128' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Tag value is too big (>256 chars): + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ], + Tags=[{'Key': 'tag', 'Value': 'a' * 257}] + ) + assert 'Member must have length less than or equal to 256' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Duplicate Tags: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ], + Tags=[{'Key': 'a', 'Value': 'a'}, {'Key': 'a', 'Value': 'a'}] + ) + assert 'Duplicate tag keys found.' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidInput' + + # Invalid characters in the tag key: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ], + Tags=[{'Key': '!', 'Value': 'a'}] + ) + assert 'Member must satisfy regular expression pattern:' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # If it contains both the AccountAggregationSources and the OrganizationAggregationSource + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': False + } + ], + OrganizationAggregationSource={ + 'RoleArn': 'arn:aws:iam::012345678910:role/SomeRole', + 'AllAwsRegions': False + } + ) + assert 'AccountAggregationSource and the OrganizationAggregationSource' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + # If it contains neither: + with assert_raises(ClientError) as ce: + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + ) + assert 'AccountAggregationSource or the OrganizationAggregationSource' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidParameterValueException' + + # Just make one: + account_aggregation_source = { + 'AccountIds': [ + '012345678910', + '111111111111', + '222222222222' + ], + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ], + 'AllAwsRegions': False + } + + result = client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[account_aggregation_source], + ) + assert result['ConfigurationAggregator']['ConfigurationAggregatorName'] == 'testing' + assert result['ConfigurationAggregator']['AccountAggregationSources'] == [account_aggregation_source] + assert 'arn:aws:config:us-west-2:123456789012:config-aggregator/config-aggregator-' in \ + result['ConfigurationAggregator']['ConfigurationAggregatorArn'] + assert result['ConfigurationAggregator']['CreationTime'] == result['ConfigurationAggregator']['LastUpdatedTime'] + + # Update the existing one: + original_arn = result['ConfigurationAggregator']['ConfigurationAggregatorArn'] + account_aggregation_source.pop('AwsRegions') + account_aggregation_source['AllAwsRegions'] = True + result = client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[account_aggregation_source] + ) + + assert result['ConfigurationAggregator']['ConfigurationAggregatorName'] == 'testing' + assert result['ConfigurationAggregator']['AccountAggregationSources'] == [account_aggregation_source] + assert result['ConfigurationAggregator']['ConfigurationAggregatorArn'] == original_arn + + # Make an org one: + result = client.put_configuration_aggregator( + ConfigurationAggregatorName='testingOrg', + OrganizationAggregationSource={ + 'RoleArn': 'arn:aws:iam::012345678910:role/SomeRole', + 'AwsRegions': ['us-east-1', 'us-west-2'] + } + ) + + assert result['ConfigurationAggregator']['ConfigurationAggregatorName'] == 'testingOrg' + assert result['ConfigurationAggregator']['OrganizationAggregationSource'] == { + 'RoleArn': 'arn:aws:iam::012345678910:role/SomeRole', + 'AwsRegions': [ + 'us-east-1', + 'us-west-2' + ], + 'AllAwsRegions': False + } + + +@mock_config +def test_describe_configuration_aggregators(): + client = boto3.client('config', region_name='us-west-2') + + # Without any config aggregators: + assert not client.describe_configuration_aggregators()['ConfigurationAggregators'] + + # Make 10 config aggregators: + for x in range(0, 10): + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing{}'.format(x), + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ] + ) + + # Describe with an incorrect name: + with assert_raises(ClientError) as ce: + client.describe_configuration_aggregators(ConfigurationAggregatorNames=['DoesNotExist']) + assert 'The configuration aggregator does not exist.' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationAggregatorException' + + # Error describe with more than 1 item in the list: + with assert_raises(ClientError) as ce: + client.describe_configuration_aggregators(ConfigurationAggregatorNames=['testing0', 'DoesNotExist']) + assert 'At least one of the configuration aggregators does not exist.' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationAggregatorException' + + # Get the normal list: + result = client.describe_configuration_aggregators() + assert not result.get('NextToken') + assert len(result['ConfigurationAggregators']) == 10 + + # Test filtered list: + agg_names = ['testing0', 'testing1', 'testing2'] + result = client.describe_configuration_aggregators(ConfigurationAggregatorNames=agg_names) + assert not result.get('NextToken') + assert len(result['ConfigurationAggregators']) == 3 + assert [agg['ConfigurationAggregatorName'] for agg in result['ConfigurationAggregators']] == agg_names + + # Test Pagination: + result = client.describe_configuration_aggregators(Limit=4) + assert len(result['ConfigurationAggregators']) == 4 + assert result['NextToken'] == 'testing4' + assert [agg['ConfigurationAggregatorName'] for agg in result['ConfigurationAggregators']] == \ + ['testing{}'.format(x) for x in range(0, 4)] + result = client.describe_configuration_aggregators(Limit=4, NextToken='testing4') + assert len(result['ConfigurationAggregators']) == 4 + assert result['NextToken'] == 'testing8' + assert [agg['ConfigurationAggregatorName'] for agg in result['ConfigurationAggregators']] == \ + ['testing{}'.format(x) for x in range(4, 8)] + result = client.describe_configuration_aggregators(Limit=4, NextToken='testing8') + assert len(result['ConfigurationAggregators']) == 2 + assert not result.get('NextToken') + assert [agg['ConfigurationAggregatorName'] for agg in result['ConfigurationAggregators']] == \ + ['testing{}'.format(x) for x in range(8, 10)] + + # Test Pagination with Filtering: + result = client.describe_configuration_aggregators(ConfigurationAggregatorNames=['testing2', 'testing4'], Limit=1) + assert len(result['ConfigurationAggregators']) == 1 + assert result['NextToken'] == 'testing4' + assert result['ConfigurationAggregators'][0]['ConfigurationAggregatorName'] == 'testing2' + result = client.describe_configuration_aggregators(ConfigurationAggregatorNames=['testing2', 'testing4'], Limit=1, NextToken='testing4') + assert not result.get('NextToken') + assert result['ConfigurationAggregators'][0]['ConfigurationAggregatorName'] == 'testing4' + + # Test with an invalid filter: + with assert_raises(ClientError) as ce: + client.describe_configuration_aggregators(NextToken='WRONG') + assert 'The nextToken provided is invalid' == ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidNextTokenException' + + +@mock_config +def test_put_aggregation_authorization(): + client = boto3.client('config', region_name='us-west-2') + + # Too many tags (>50): + with assert_raises(ClientError) as ce: + client.put_aggregation_authorization( + AuthorizedAccountId='012345678910', + AuthorizedAwsRegion='us-west-2', + Tags=[{'Key': '{}'.format(x), 'Value': '{}'.format(x)} for x in range(0, 51)] + ) + assert 'Member must have length less than or equal to 50' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Tag key is too big (>128 chars): + with assert_raises(ClientError) as ce: + client.put_aggregation_authorization( + AuthorizedAccountId='012345678910', + AuthorizedAwsRegion='us-west-2', + Tags=[{'Key': 'a' * 129, 'Value': 'a'}] + ) + assert 'Member must have length less than or equal to 128' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Tag value is too big (>256 chars): + with assert_raises(ClientError) as ce: + client.put_aggregation_authorization( + AuthorizedAccountId='012345678910', + AuthorizedAwsRegion='us-west-2', + Tags=[{'Key': 'tag', 'Value': 'a' * 257}] + ) + assert 'Member must have length less than or equal to 256' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Duplicate Tags: + with assert_raises(ClientError) as ce: + client.put_aggregation_authorization( + AuthorizedAccountId='012345678910', + AuthorizedAwsRegion='us-west-2', + Tags=[{'Key': 'a', 'Value': 'a'}, {'Key': 'a', 'Value': 'a'}] + ) + assert 'Duplicate tag keys found.' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidInput' + + # Invalid characters in the tag key: + with assert_raises(ClientError) as ce: + client.put_aggregation_authorization( + AuthorizedAccountId='012345678910', + AuthorizedAwsRegion='us-west-2', + Tags=[{'Key': '!', 'Value': 'a'}] + ) + assert 'Member must satisfy regular expression pattern:' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'ValidationException' + + # Put a normal one there: + result = client.put_aggregation_authorization(AuthorizedAccountId='012345678910', AuthorizedAwsRegion='us-east-1', + Tags=[{'Key': 'tag', 'Value': 'a'}]) + + assert result['AggregationAuthorization']['AggregationAuthorizationArn'] == 'arn:aws:config:us-west-2:123456789012:' \ + 'aggregation-authorization/012345678910/us-east-1' + assert result['AggregationAuthorization']['AuthorizedAccountId'] == '012345678910' + assert result['AggregationAuthorization']['AuthorizedAwsRegion'] == 'us-east-1' + assert isinstance(result['AggregationAuthorization']['CreationTime'], datetime) + + creation_date = result['AggregationAuthorization']['CreationTime'] + + # And again: + result = client.put_aggregation_authorization(AuthorizedAccountId='012345678910', AuthorizedAwsRegion='us-east-1') + assert result['AggregationAuthorization']['AggregationAuthorizationArn'] == 'arn:aws:config:us-west-2:123456789012:' \ + 'aggregation-authorization/012345678910/us-east-1' + assert result['AggregationAuthorization']['AuthorizedAccountId'] == '012345678910' + assert result['AggregationAuthorization']['AuthorizedAwsRegion'] == 'us-east-1' + assert result['AggregationAuthorization']['CreationTime'] == creation_date + + +@mock_config +def test_describe_aggregation_authorizations(): + client = boto3.client('config', region_name='us-west-2') + + # With no aggregation authorizations: + assert not client.describe_aggregation_authorizations()['AggregationAuthorizations'] + + # Make 10 account authorizations: + for i in range(0, 10): + client.put_aggregation_authorization(AuthorizedAccountId='{}'.format(str(i) * 12), AuthorizedAwsRegion='us-west-2') + + result = client.describe_aggregation_authorizations() + assert len(result['AggregationAuthorizations']) == 10 + assert not result.get('NextToken') + for i in range(0, 10): + assert result['AggregationAuthorizations'][i]['AuthorizedAccountId'] == str(i) * 12 + + # Test Pagination: + result = client.describe_aggregation_authorizations(Limit=4) + assert len(result['AggregationAuthorizations']) == 4 + assert result['NextToken'] == ('4' * 12) + '/us-west-2' + assert [auth['AuthorizedAccountId'] for auth in result['AggregationAuthorizations']] == ['{}'.format(str(x) * 12) for x in range(0, 4)] + + result = client.describe_aggregation_authorizations(Limit=4, NextToken=('4' * 12) + '/us-west-2') + assert len(result['AggregationAuthorizations']) == 4 + assert result['NextToken'] == ('8' * 12) + '/us-west-2' + assert [auth['AuthorizedAccountId'] for auth in result['AggregationAuthorizations']] == ['{}'.format(str(x) * 12) for x in range(4, 8)] + + result = client.describe_aggregation_authorizations(Limit=4, NextToken=('8' * 12) + '/us-west-2') + assert len(result['AggregationAuthorizations']) == 2 + assert not result.get('NextToken') + assert [auth['AuthorizedAccountId'] for auth in result['AggregationAuthorizations']] == ['{}'.format(str(x) * 12) for x in range(8, 10)] + + # Test with an invalid filter: + with assert_raises(ClientError) as ce: + client.describe_aggregation_authorizations(NextToken='WRONG') + assert 'The nextToken provided is invalid' == ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'InvalidNextTokenException' + + +@mock_config +def test_delete_aggregation_authorization(): + client = boto3.client('config', region_name='us-west-2') + + client.put_aggregation_authorization(AuthorizedAccountId='012345678910', AuthorizedAwsRegion='us-west-2') + + # Delete it: + client.delete_aggregation_authorization(AuthorizedAccountId='012345678910', AuthorizedAwsRegion='us-west-2') + + # Verify that none are there: + assert not client.describe_aggregation_authorizations()['AggregationAuthorizations'] + + # Try it again -- nothing should happen: + client.delete_aggregation_authorization(AuthorizedAccountId='012345678910', AuthorizedAwsRegion='us-west-2') + + +@mock_config +def test_delete_configuration_aggregator(): + client = boto3.client('config', region_name='us-west-2') + client.put_configuration_aggregator( + ConfigurationAggregatorName='testing', + AccountAggregationSources=[ + { + 'AccountIds': [ + '012345678910', + ], + 'AllAwsRegions': True + } + ] + ) + + client.delete_configuration_aggregator(ConfigurationAggregatorName='testing') + + # And again to confirm that it's deleted: + with assert_raises(ClientError) as ce: + client.delete_configuration_aggregator(ConfigurationAggregatorName='testing') + assert 'The configuration aggregator does not exist.' in ce.exception.response['Error']['Message'] + assert ce.exception.response['Error']['Code'] == 'NoSuchConfigurationAggregatorException' + + @mock_config def test_describe_configurations(): client = boto3.client('config', region_name='us-west-2')