From 943ecb7ea798e6832a72663b66dfaa44d1f3fe3e Mon Sep 17 00:00:00 2001 From: Chris Kilding <590569+chriskilding@users.noreply.github.com> Date: Fri, 31 Jul 2020 15:31:18 +0100 Subject: [PATCH 1/5] Support --filters option in secretsmanager:ListSecrets (#3173) * Feature: Support --filters opton in secretsmanager:ListSecrets * Implement some of the secret filters * Check listSecrets filters combine with an implicit AND operator * Test all filter and multi-value filter and multi-word filter * Fix matcher behavior, restructure code * Implement remaining listSecrets filter cases * Linter fixes * Use contains-in-any-order assertions for test_list_secrets * Linter fix again * Attempt Python 2 fix for assert_items_equal * Remove docstrings from test_list_secrets tests as they make the test reports weird * Test and handle listSecrets filter with no values --- moto/secretsmanager/exceptions.py | 5 + moto/secretsmanager/list_secrets/__init__.py | 0 moto/secretsmanager/list_secrets/filters.py | 44 +++ moto/secretsmanager/models.py | 73 +++-- moto/secretsmanager/responses.py | 31 ++- .../test_secretsmanager/test_list_secrets.py | 251 ++++++++++++++++++ .../test_secretsmanager.py | 30 --- 7 files changed, 377 insertions(+), 57 deletions(-) create mode 100644 moto/secretsmanager/list_secrets/__init__.py create mode 100644 moto/secretsmanager/list_secrets/filters.py create mode 100644 tests/test_secretsmanager/test_list_secrets.py diff --git a/moto/secretsmanager/exceptions.py b/moto/secretsmanager/exceptions.py index bf717e20c..6618cd3ac 100644 --- a/moto/secretsmanager/exceptions.py +++ b/moto/secretsmanager/exceptions.py @@ -57,3 +57,8 @@ class InvalidRequestException(SecretsManagerClientError): super(InvalidRequestException, self).__init__( "InvalidRequestException", message ) + + +class ValidationException(SecretsManagerClientError): + def __init__(self, message): + super(ValidationException, self).__init__("ValidationException", message) diff --git a/moto/secretsmanager/list_secrets/__init__.py b/moto/secretsmanager/list_secrets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/moto/secretsmanager/list_secrets/filters.py b/moto/secretsmanager/list_secrets/filters.py new file mode 100644 index 000000000..813b1f544 --- /dev/null +++ b/moto/secretsmanager/list_secrets/filters.py @@ -0,0 +1,44 @@ +def _matcher(pattern, str): + for word in pattern.split(" "): + if word not in str: + return False + return True + + +def name(secret, names): + for n in names: + if _matcher(n, secret["name"]): + return True + return False + + +def description(secret, descriptions): + for d in descriptions: + if _matcher(d, secret["description"]): + return True + return False + + +def tag_key(secret, tag_keys): + for k in tag_keys: + for tag in secret["tags"]: + if _matcher(k, tag["Key"]): + return True + return False + + +def tag_value(secret, tag_values): + for v in tag_values: + for tag in secret["tags"]: + if _matcher(v, tag["Value"]): + return True + return False + + +def all(secret, values): + return ( + name(secret, values) + or description(secret, values) + or tag_key(secret, values) + or tag_value(secret, values) + ) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 8641916a7..0339dc575 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -18,6 +18,31 @@ from .exceptions import ( ClientError, ) from .utils import random_password, secret_arn, get_secret_name_from_arn +from .list_secrets.filters import all, tag_key, tag_value, description, name + + +_filter_functions = { + "all": all, + "name": name, + "description": description, + "tag-key": tag_key, + "tag-value": tag_value, +} + + +def filter_keys(): + return list(_filter_functions.keys()) + + +def _matches(secret, filters): + is_match = True + + for f in filters: + # Filter names are pre-validated in the resource layer + filter_function = _filter_functions.get(f["Key"]) + is_match = is_match and filter_function(secret, f["Values"]) + + return is_match class SecretsManager(BaseModel): @@ -442,35 +467,35 @@ class SecretsManagerBackend(BaseBackend): return response - def list_secrets(self, max_results, next_token): + def list_secrets(self, filters, max_results, next_token): # TODO implement pagination and limits secret_list = [] for secret in self.secrets.values(): + if _matches(secret, filters): + versions_to_stages = {} + for version_id, version in secret["versions"].items(): + versions_to_stages[version_id] = version["version_stages"] - versions_to_stages = {} - for version_id, version in secret["versions"].items(): - versions_to_stages[version_id] = version["version_stages"] - - secret_list.append( - { - "ARN": secret_arn(self.region, secret["secret_id"]), - "DeletedDate": secret.get("deleted_date", None), - "Description": secret.get("description", ""), - "KmsKeyId": "", - "LastAccessedDate": None, - "LastChangedDate": None, - "LastRotatedDate": None, - "Name": secret["name"], - "RotationEnabled": secret["rotation_enabled"], - "RotationLambdaARN": secret["rotation_lambda_arn"], - "RotationRules": { - "AutomaticallyAfterDays": secret["auto_rotate_after_days"] - }, - "SecretVersionsToStages": versions_to_stages, - "Tags": secret["tags"], - } - ) + secret_list.append( + { + "ARN": secret_arn(self.region, secret["secret_id"]), + "DeletedDate": secret.get("deleted_date", None), + "Description": secret.get("description", ""), + "KmsKeyId": "", + "LastAccessedDate": None, + "LastChangedDate": None, + "LastRotatedDate": None, + "Name": secret["name"], + "RotationEnabled": secret["rotation_enabled"], + "RotationLambdaARN": secret["rotation_lambda_arn"], + "RotationRules": { + "AutomaticallyAfterDays": secret["auto_rotate_after_days"] + }, + "SecretVersionsToStages": versions_to_stages, + "Tags": secret["tags"], + } + ) return secret_list, None diff --git a/moto/secretsmanager/responses.py b/moto/secretsmanager/responses.py index 9a899c90d..fcf991ea2 100644 --- a/moto/secretsmanager/responses.py +++ b/moto/secretsmanager/responses.py @@ -1,13 +1,36 @@ from __future__ import unicode_literals from moto.core.responses import BaseResponse -from moto.secretsmanager.exceptions import InvalidRequestException +from moto.secretsmanager.exceptions import ( + InvalidRequestException, + InvalidParameterException, + ValidationException, +) -from .models import secretsmanager_backends +from .models import secretsmanager_backends, filter_keys import json +def _validate_filters(filters): + for idx, f in enumerate(filters): + filter_key = f.get("Key", None) + filter_values = f.get("Values", None) + if filter_key is None: + raise InvalidParameterException("Invalid filter key") + if filter_key not in filter_keys(): + raise ValidationException( + "1 validation error detected: Value '{}' at 'filters.{}.member.key' failed to satisfy constraint: " + "Member must satisfy enum value set: [all, name, tag-key, description, tag-value]".format( + filter_key, idx + 1 + ) + ) + if filter_values is None: + raise InvalidParameterException( + "Invalid filter values for key: {}".format(filter_key) + ) + + class SecretsManagerResponse(BaseResponse): def get_secret_value(self): secret_id = self._get_param("SecretId") @@ -102,10 +125,12 @@ class SecretsManagerResponse(BaseResponse): ) def list_secrets(self): + filters = self._get_param("Filters", if_none=[]) + _validate_filters(filters) max_results = self._get_int_param("MaxResults") next_token = self._get_param("NextToken") secret_list, next_token = secretsmanager_backends[self.region].list_secrets( - max_results=max_results, next_token=next_token + filters=filters, max_results=max_results, next_token=next_token ) return json.dumps(dict(SecretList=secret_list, NextToken=next_token)) diff --git a/tests/test_secretsmanager/test_list_secrets.py b/tests/test_secretsmanager/test_list_secrets.py new file mode 100644 index 000000000..da3c4eb7e --- /dev/null +++ b/tests/test_secretsmanager/test_list_secrets.py @@ -0,0 +1,251 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import boto3 + +from moto import mock_secretsmanager +from botocore.exceptions import ClientError +import sure # noqa +from nose.tools import assert_raises + +try: + from nose.tools import assert_items_equal +except ImportError: + from nose.tools import assert_count_equal as assert_items_equal + + +def boto_client(): + return boto3.client("secretsmanager", region_name="us-west-2") + + +@mock_secretsmanager +def test_empty(): + conn = boto_client() + + secrets = conn.list_secrets() + + assert_items_equal(secrets["SecretList"], []) + + +@mock_secretsmanager +def test_list_secrets(): + conn = boto_client() + + conn.create_secret(Name="test-secret", SecretString="foosecret") + + conn.create_secret( + Name="test-secret-2", + SecretString="barsecret", + Tags=[{"Key": "a", "Value": "1"}], + ) + + secrets = conn.list_secrets() + + assert secrets["SecretList"][0]["ARN"] is not None + assert secrets["SecretList"][0]["Name"] == "test-secret" + assert secrets["SecretList"][1]["ARN"] is not None + assert secrets["SecretList"][1]["Name"] == "test-secret-2" + assert secrets["SecretList"][1]["Tags"] == [{"Key": "a", "Value": "1"}] + + +@mock_secretsmanager +def test_with_name_filter(): + conn = boto_client() + + conn.create_secret(Name="foo", SecretString="secret") + conn.create_secret(Name="bar", SecretString="secret") + + secrets = conn.list_secrets(Filters=[{"Key": "name", "Values": ["foo"]}]) + + secret_names = list(map(lambda s: s["Name"], secrets["SecretList"])) + assert_items_equal(secret_names, ["foo"]) + + +@mock_secretsmanager +def test_with_tag_key_filter(): + conn = boto_client() + + conn.create_secret( + Name="foo", SecretString="secret", Tags=[{"Key": "baz", "Value": "1"}] + ) + conn.create_secret(Name="bar", SecretString="secret") + + secrets = conn.list_secrets(Filters=[{"Key": "tag-key", "Values": ["baz"]}]) + + secret_names = list(map(lambda s: s["Name"], secrets["SecretList"])) + assert_items_equal(secret_names, ["foo"]) + + +@mock_secretsmanager +def test_with_tag_value_filter(): + conn = boto_client() + + conn.create_secret( + Name="foo", SecretString="secret", Tags=[{"Key": "1", "Value": "baz"}] + ) + conn.create_secret(Name="bar", SecretString="secret") + + secrets = conn.list_secrets(Filters=[{"Key": "tag-value", "Values": ["baz"]}]) + + secret_names = list(map(lambda s: s["Name"], secrets["SecretList"])) + assert_items_equal(secret_names, ["foo"]) + + +@mock_secretsmanager +def test_with_description_filter(): + conn = boto_client() + + conn.create_secret(Name="foo", SecretString="secret", Description="baz qux") + conn.create_secret(Name="bar", SecretString="secret") + + secrets = conn.list_secrets(Filters=[{"Key": "description", "Values": ["baz"]}]) + + secret_names = list(map(lambda s: s["Name"], secrets["SecretList"])) + assert_items_equal(secret_names, ["foo"]) + + +@mock_secretsmanager +def test_with_all_filter(): + # The 'all' filter will match a secret that contains ANY field with the criteria. In other words an implicit OR. + + conn = boto_client() + + conn.create_secret(Name="foo", SecretString="secret") + conn.create_secret(Name="bar", SecretString="secret", Description="foo") + conn.create_secret( + Name="baz", SecretString="secret", Tags=[{"Key": "foo", "Value": "1"}] + ) + conn.create_secret( + Name="qux", SecretString="secret", Tags=[{"Key": "1", "Value": "foo"}] + ) + conn.create_secret( + Name="multi", SecretString="secret", Tags=[{"Key": "foo", "Value": "foo"}] + ) + conn.create_secret(Name="none", SecretString="secret") + + secrets = conn.list_secrets(Filters=[{"Key": "all", "Values": ["foo"]}]) + + secret_names = list(map(lambda s: s["Name"], secrets["SecretList"])) + assert_items_equal(secret_names, ["foo", "bar", "baz", "qux", "multi"]) + + +@mock_secretsmanager +def test_with_no_filter_key(): + conn = boto_client() + + with assert_raises(ClientError) as ire: + conn.list_secrets(Filters=[{"Values": ["foo"]}]) + + ire.exception.response["Error"]["Code"].should.equal("InvalidParameterException") + ire.exception.response["Error"]["Message"].should.equal("Invalid filter key") + + +@mock_secretsmanager +def test_with_no_filter_values(): + conn = boto_client() + + conn.create_secret(Name="foo", SecretString="secret", Description="hello") + + with assert_raises(ClientError) as ire: + conn.list_secrets(Filters=[{"Key": "description"}]) + + ire.exception.response["Error"]["Code"].should.equal("InvalidParameterException") + ire.exception.response["Error"]["Message"].should.equal( + "Invalid filter values for key: description" + ) + + +@mock_secretsmanager +def test_with_invalid_filter_key(): + conn = boto_client() + + with assert_raises(ClientError) as ire: + conn.list_secrets(Filters=[{"Key": "invalid", "Values": ["foo"]}]) + + ire.exception.response["Error"]["Code"].should.equal("ValidationException") + ire.exception.response["Error"]["Message"].should.equal( + "1 validation error detected: Value 'invalid' at 'filters.1.member.key' failed to satisfy constraint: Member " + "must satisfy enum value set: [all, name, tag-key, description, tag-value]" + ) + + +@mock_secretsmanager +def test_with_duplicate_filter_keys(): + # Multiple filters with the same key combine with an implicit AND operator + + conn = boto_client() + + conn.create_secret(Name="foo", SecretString="secret", Description="one two") + conn.create_secret(Name="bar", SecretString="secret", Description="one") + conn.create_secret(Name="baz", SecretString="secret", Description="two") + conn.create_secret(Name="qux", SecretString="secret", Description="unrelated") + + secrets = conn.list_secrets( + Filters=[ + {"Key": "description", "Values": ["one"]}, + {"Key": "description", "Values": ["two"]}, + ] + ) + + secret_names = list(map(lambda s: s["Name"], secrets["SecretList"])) + assert_items_equal(secret_names, ["foo"]) + + +@mock_secretsmanager +def test_with_multiple_filters(): + # Multiple filters combine with an implicit AND operator + + conn = boto_client() + + conn.create_secret( + Name="foo", SecretString="secret", Tags=[{"Key": "right", "Value": "right"}] + ) + conn.create_secret( + Name="bar", SecretString="secret", Tags=[{"Key": "right", "Value": "wrong"}] + ) + conn.create_secret( + Name="baz", SecretString="secret", Tags=[{"Key": "wrong", "Value": "right"}] + ) + conn.create_secret( + Name="qux", SecretString="secret", Tags=[{"Key": "wrong", "Value": "wrong"}] + ) + + secrets = conn.list_secrets( + Filters=[ + {"Key": "tag-key", "Values": ["right"]}, + {"Key": "tag-value", "Values": ["right"]}, + ] + ) + + secret_names = list(map(lambda s: s["Name"], secrets["SecretList"])) + assert_items_equal(secret_names, ["foo"]) + + +@mock_secretsmanager +def test_with_filter_with_multiple_values(): + conn = boto_client() + + conn.create_secret(Name="foo", SecretString="secret") + conn.create_secret(Name="bar", SecretString="secret") + conn.create_secret(Name="baz", SecretString="secret") + + secrets = conn.list_secrets(Filters=[{"Key": "name", "Values": ["foo", "bar"]}]) + + secret_names = list(map(lambda s: s["Name"], secrets["SecretList"])) + assert_items_equal(secret_names, ["foo", "bar"]) + + +@mock_secretsmanager +def test_with_filter_with_value_with_multiple_words(): + conn = boto_client() + + conn.create_secret(Name="foo", SecretString="secret", Description="one two") + conn.create_secret(Name="bar", SecretString="secret", Description="one and two") + conn.create_secret(Name="baz", SecretString="secret", Description="one") + conn.create_secret(Name="qux", SecretString="secret", Description="two") + conn.create_secret(Name="none", SecretString="secret", Description="unrelated") + + secrets = conn.list_secrets(Filters=[{"Key": "description", "Values": ["one two"]}]) + + secret_names = list(map(lambda s: s["Name"], secrets["SecretList"])) + assert_items_equal(secret_names, ["foo", "bar"]) diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 59992e094..0bd66b128 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -459,36 +459,6 @@ def test_describe_secret_that_does_not_match(): result = conn.get_secret_value(SecretId="i-dont-match") -@mock_secretsmanager -def test_list_secrets_empty(): - conn = boto3.client("secretsmanager", region_name="us-west-2") - - secrets = conn.list_secrets() - - assert secrets["SecretList"] == [] - - -@mock_secretsmanager -def test_list_secrets(): - conn = boto3.client("secretsmanager", region_name="us-west-2") - - conn.create_secret(Name="test-secret", SecretString="foosecret") - - conn.create_secret( - Name="test-secret-2", - SecretString="barsecret", - Tags=[{"Key": "a", "Value": "1"}], - ) - - secrets = conn.list_secrets() - - assert secrets["SecretList"][0]["ARN"] is not None - assert secrets["SecretList"][0]["Name"] == "test-secret" - assert secrets["SecretList"][1]["ARN"] is not None - assert secrets["SecretList"][1]["Name"] == "test-secret-2" - assert secrets["SecretList"][1]["Tags"] == [{"Key": "a", "Value": "1"}] - - @mock_secretsmanager def test_restore_secret(): conn = boto3.client("secretsmanager", region_name="us-west-2") From 8162947ebb320c769b951ead443e31acc33c967f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20Gr=C3=BCbel?= Date: Fri, 31 Jul 2020 17:32:57 +0200 Subject: [PATCH 2/5] Organizations - implement Delegated Administrator functionality (#3200) * Add organizations.register_delegated_administrator * Add organizations.list_delegated_administrators * Add organizations.list_delegated_services_for_account * Add organizations.deregister_delegated_administrator * Fix Python2 incompatibility --- moto/organizations/exceptions.py | 48 +++ moto/organizations/models.py | 207 +++++++-- moto/organizations/responses.py | 28 ++ .../test_organizations_boto3.py | 394 +++++++++++++++++- 4 files changed, 619 insertions(+), 58 deletions(-) diff --git a/moto/organizations/exceptions.py b/moto/organizations/exceptions.py index 036eeccbc..2d1ee7328 100644 --- a/moto/organizations/exceptions.py +++ b/moto/organizations/exceptions.py @@ -2,6 +2,54 @@ from __future__ import unicode_literals from moto.core.exceptions import JsonRESTError +class AccountAlreadyRegisteredException(JsonRESTError): + code = 400 + + def __init__(self): + super(AccountAlreadyRegisteredException, self).__init__( + "AccountAlreadyRegisteredException", + "The provided account is already a delegated administrator for your organization.", + ) + + +class AccountNotRegisteredException(JsonRESTError): + code = 400 + + def __init__(self): + super(AccountNotRegisteredException, self).__init__( + "AccountNotRegisteredException", + "The provided account is not a registered delegated administrator for your organization.", + ) + + +class AccountNotFoundException(JsonRESTError): + code = 400 + + def __init__(self): + super(AccountNotFoundException, self).__init__( + "AccountNotFoundException", "You specified an account that doesn't exist." + ) + + +class AWSOrganizationsNotInUseException(JsonRESTError): + code = 400 + + def __init__(self): + super(AWSOrganizationsNotInUseException, self).__init__( + "AWSOrganizationsNotInUseException", + "Your account is not a member of an organization.", + ) + + +class ConstraintViolationException(JsonRESTError): + code = 400 + + def __init__(self, message): + super(ConstraintViolationException, self).__init__( + "ConstraintViolationException", message + ) + + class InvalidInputException(JsonRESTError): code = 400 diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 6c1dab15d..6c8029e3d 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -4,7 +4,7 @@ import datetime import re import json -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, ACCOUNT_ID from moto.core.exceptions import RESTError from moto.core.utils import unix_time from moto.organizations import utils @@ -12,6 +12,11 @@ from moto.organizations.exceptions import ( InvalidInputException, DuplicateOrganizationalUnitException, DuplicatePolicyException, + AccountNotFoundException, + ConstraintViolationException, + AccountAlreadyRegisteredException, + AWSOrganizationsNotInUseException, + AccountNotRegisteredException, ) @@ -85,15 +90,13 @@ class FakeAccount(BaseModel): def describe(self): return { - "Account": { - "Id": self.id, - "Arn": self.arn, - "Email": self.email, - "Name": self.name, - "Status": self.status, - "JoinedMethod": self.joined_method, - "JoinedTimestamp": unix_time(self.create_time), - } + "Id": self.id, + "Arn": self.arn, + "Email": self.email, + "Name": self.name, + "Status": self.status, + "JoinedMethod": self.joined_method, + "JoinedTimestamp": unix_time(self.create_time), } @@ -221,6 +224,56 @@ class FakeServiceAccess(BaseModel): return service_principal in FakeServiceAccess.TRUSTED_SERVICES +class FakeDelegatedAdministrator(BaseModel): + # List of services, which support a different Account to ba a delegated administrator + # https://docs.aws.amazon.com/organizations/latest/userguide/orgs_integrated-services-list.html + SUPPORTED_SERVICES = [ + "config-multiaccountsetup.amazonaws.com", + "guardduty.amazonaws.com", + "access-analyzer.amazonaws.com", + "macie.amazonaws.com", + "servicecatalog.amazonaws.com", + "ssm.amazonaws.com", + ] + + def __init__(self, account): + self.account = account + self.enabled_date = datetime.datetime.utcnow() + self.services = {} + + def add_service_principal(self, service_principal): + if service_principal in self.services: + raise AccountAlreadyRegisteredException + + if not self.supported_service(service_principal): + raise InvalidInputException( + "You specified an unrecognized service principal." + ) + + self.services[service_principal] = { + "ServicePrincipal": service_principal, + "DelegationEnabledDate": unix_time(datetime.datetime.utcnow()), + } + + def remove_service_principal(self, service_principal): + if service_principal not in self.services: + raise InvalidInputException( + "You specified an unrecognized service principal." + ) + + self.services.pop(service_principal) + + def describe(self): + admin = self.account.describe() + admin["DelegationEnabledDate"] = unix_time(self.enabled_date) + + return admin + + @staticmethod + def supported_service(service_principal): + return service_principal in FakeDelegatedAdministrator.SUPPORTED_SERVICES + + class OrganizationsBackend(BaseBackend): def __init__(self): self.org = None @@ -228,6 +281,7 @@ class OrganizationsBackend(BaseBackend): self.ou = [] self.policies = [] self.services = [] + self.admins = [] def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs["FeatureSet"]) @@ -259,10 +313,7 @@ class OrganizationsBackend(BaseBackend): def describe_organization(self): if not self.org: - raise RESTError( - "AWSOrganizationsNotInUseException", - "Your account is not a member of an organization.", - ) + raise AWSOrganizationsNotInUseException return self.org.describe() def list_roots(self): @@ -325,10 +376,7 @@ class OrganizationsBackend(BaseBackend): (account for account in self.accounts if account.id == account_id), None ) if account is None: - raise RESTError( - "AccountNotFoundException", - "You specified an account that doesn't exist.", - ) + raise AccountNotFoundException return account def get_account_by_attr(self, attr, value): @@ -341,15 +389,12 @@ class OrganizationsBackend(BaseBackend): None, ) if account is None: - raise RESTError( - "AccountNotFoundException", - "You specified an account that doesn't exist.", - ) + raise AccountNotFoundException return account def describe_account(self, **kwargs): account = self.get_account_by_id(kwargs["AccountId"]) - return account.describe() + return dict(Account=account.describe()) def describe_create_account_status(self, **kwargs): account = self.get_account_by_attr( @@ -358,15 +403,13 @@ class OrganizationsBackend(BaseBackend): return account.create_account_status def list_accounts(self): - return dict( - Accounts=[account.describe()["Account"] for account in self.accounts] - ) + return dict(Accounts=[account.describe() for account in self.accounts]) def list_accounts_for_parent(self, **kwargs): parent_id = self.validate_parent_id(kwargs["ParentId"]) return dict( Accounts=[ - account.describe()["Account"] + account.describe() for account in self.accounts if account.parent_id == parent_id ] @@ -399,7 +442,7 @@ class OrganizationsBackend(BaseBackend): elif kwargs["ChildType"] == "ORGANIZATIONAL_UNIT": obj_list = self.ou else: - raise RESTError("InvalidInputException", "You specified an invalid value.") + raise InvalidInputException("You specified an invalid value.") return dict( Children=[ {"Id": obj.id, "Type": kwargs["ChildType"]} @@ -427,7 +470,7 @@ class OrganizationsBackend(BaseBackend): "You specified a policy that doesn't exist.", ) else: - raise RESTError("InvalidInputException", "You specified an invalid value.") + raise InvalidInputException("You specified an invalid value.") return policy.describe() def get_policy_by_id(self, policy_id): @@ -472,12 +515,9 @@ class OrganizationsBackend(BaseBackend): account.attached_policies.append(policy) policy.attachments.append(account) else: - raise RESTError( - "AccountNotFoundException", - "You specified an account that doesn't exist.", - ) + raise AccountNotFoundException else: - raise RESTError("InvalidInputException", "You specified an invalid value.") + raise InvalidInputException("You specified an invalid value.") def list_policies(self, **kwargs): return dict( @@ -510,12 +550,9 @@ class OrganizationsBackend(BaseBackend): elif re.compile(utils.ACCOUNT_ID_REGEX).match(kwargs["TargetId"]): obj = next((a for a in self.accounts if a.id == kwargs["TargetId"]), None) if obj is None: - raise RESTError( - "AccountNotFoundException", - "You specified an account that doesn't exist.", - ) + raise AccountNotFoundException else: - raise RESTError("InvalidInputException", "You specified an invalid value.") + raise InvalidInputException("You specified an invalid value.") return dict( Policies=[ p.describe()["Policy"]["PolicySummary"] for p in obj.attached_policies @@ -533,7 +570,7 @@ class OrganizationsBackend(BaseBackend): "You specified a policy that doesn't exist.", ) else: - raise RESTError("InvalidInputException", "You specified an invalid value.") + raise InvalidInputException("You specified an invalid value.") objects = [ {"TargetId": obj.id, "Arn": obj.arn, "Name": obj.name, "Type": obj.type} for obj in policy.attachments @@ -606,5 +643,95 @@ class OrganizationsBackend(BaseBackend): if service_principal: self.services.remove(service_principal) + def register_delegated_administrator(self, **kwargs): + account_id = kwargs["AccountId"] + + if account_id == ACCOUNT_ID: + raise ConstraintViolationException( + "You cannot register master account/yourself as delegated administrator for your organization." + ) + + account = self.get_account_by_id(account_id) + + admin = next( + (admin for admin in self.admins if admin.account.id == account_id), None + ) + if admin is None: + admin = FakeDelegatedAdministrator(account) + self.admins.append(admin) + + admin.add_service_principal(kwargs["ServicePrincipal"]) + + def list_delegated_administrators(self, **kwargs): + admins = self.admins + service = kwargs.get("ServicePrincipal") + + if service: + if not FakeDelegatedAdministrator.supported_service(service): + raise InvalidInputException( + "You specified an unrecognized service principal." + ) + + admins = [admin for admin in admins if service in admin.services] + + delegated_admins = [admin.describe() for admin in admins] + + return dict(DelegatedAdministrators=delegated_admins) + + def list_delegated_services_for_account(self, **kwargs): + admin = next( + (admin for admin in self.admins if admin.account.id == kwargs["AccountId"]), + None, + ) + if admin is None: + account = next( + ( + account + for account in self.accounts + if account.id == kwargs["AccountId"] + ), + None, + ) + if account: + raise AccountNotRegisteredException + + raise AWSOrganizationsNotInUseException + + services = [service for service in admin.services.values()] + + return dict(DelegatedServices=services) + + def deregister_delegated_administrator(self, **kwargs): + account_id = kwargs["AccountId"] + service = kwargs["ServicePrincipal"] + + if account_id == ACCOUNT_ID: + raise ConstraintViolationException( + "You cannot register master account/yourself as delegated administrator for your organization." + ) + + admin = next( + (admin for admin in self.admins if admin.account.id == account_id), None, + ) + if admin is None: + account = next( + ( + account + for account in self.accounts + if account.id == kwargs["AccountId"] + ), + None, + ) + if account: + raise AccountNotRegisteredException + + raise AccountNotFoundException + + admin.remove_service_principal(service) + + # remove account, when no services attached + if not admin.services: + self.admins.remove(admin) + organizations_backend = OrganizationsBackend() diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index a2bd028d9..4689db5d7 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -163,3 +163,31 @@ class OrganizationsResponse(BaseResponse): return json.dumps( self.organizations_backend.disable_aws_service_access(**self.request_params) ) + + def register_delegated_administrator(self): + return json.dumps( + self.organizations_backend.register_delegated_administrator( + **self.request_params + ) + ) + + def list_delegated_administrators(self): + return json.dumps( + self.organizations_backend.list_delegated_administrators( + **self.request_params + ) + ) + + def list_delegated_services_for_account(self): + return json.dumps( + self.organizations_backend.list_delegated_services_for_account( + **self.request_params + ) + ) + + def deregister_delegated_administrator(self): + return json.dumps( + self.organizations_backend.deregister_delegated_administrator( + **self.request_params + ) + ) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index decc0a178..90bee1edb 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -10,6 +10,7 @@ from botocore.exceptions import ClientError from nose.tools import assert_raises from moto import mock_organizations +from moto.core import ACCOUNT_ID from moto.organizations import utils from .organizations_test_utils import ( validate_organization, @@ -64,8 +65,11 @@ def test_describe_organization_exception(): response = client.describe_organization() ex = e.exception ex.operation_name.should.equal("DescribeOrganization") - ex.response["Error"]["Code"].should.equal("400") - ex.response["Error"]["Message"].should.contain("AWSOrganizationsNotInUseException") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("AWSOrganizationsNotInUseException") + ex.response["Error"]["Message"].should.equal( + "Your account is not a member of an organization." + ) # Organizational Units @@ -193,8 +197,11 @@ def test_describe_account_exception(): response = client.describe_account(AccountId=utils.make_random_account_id()) ex = e.exception ex.operation_name.should.equal("DescribeAccount") - ex.response["Error"]["Code"].should.equal("400") - ex.response["Error"]["Message"].should.contain("AccountNotFoundException") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("AccountNotFoundException") + ex.response["Error"]["Message"].should.equal( + "You specified an account that doesn't exist." + ) @mock_organizations @@ -340,8 +347,9 @@ def test_list_children_exception(): response = client.list_children(ParentId=root_id, ChildType="BLEE") ex = e.exception ex.operation_name.should.equal("ListChildren") - ex.response["Error"]["Code"].should.equal("400") - ex.response["Error"]["Message"].should.contain("InvalidInputException") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidInputException") + ex.response["Error"]["Message"].should.equal("You specified an invalid value.") # Service Control Policies @@ -405,8 +413,9 @@ def test_describe_policy_exception(): response = client.describe_policy(PolicyId="meaninglessstring") ex = e.exception ex.operation_name.should.equal("DescribePolicy") - ex.response["Error"]["Code"].should.equal("400") - ex.response["Error"]["Message"].should.contain("InvalidInputException") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidInputException") + ex.response["Error"]["Message"].should.equal("You specified an invalid value.") @mock_organizations @@ -517,16 +526,20 @@ def test_attach_policy_exception(): response = client.attach_policy(PolicyId=policy_id, TargetId=account_id) ex = e.exception ex.operation_name.should.equal("AttachPolicy") - ex.response["Error"]["Code"].should.equal("400") - ex.response["Error"]["Message"].should.contain("AccountNotFoundException") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("AccountNotFoundException") + ex.response["Error"]["Message"].should.equal( + "You specified an account that doesn't exist." + ) with assert_raises(ClientError) as e: response = client.attach_policy( PolicyId=policy_id, TargetId="meaninglessstring" ) ex = e.exception ex.operation_name.should.equal("AttachPolicy") - ex.response["Error"]["Code"].should.equal("400") - ex.response["Error"]["Message"].should.contain("InvalidInputException") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidInputException") + ex.response["Error"]["Message"].should.equal("You specified an invalid value.") @mock_organizations @@ -636,16 +649,20 @@ def test_list_policies_for_target_exception(): ) ex = e.exception ex.operation_name.should.equal("ListPoliciesForTarget") - ex.response["Error"]["Code"].should.equal("400") - ex.response["Error"]["Message"].should.contain("AccountNotFoundException") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("AccountNotFoundException") + ex.response["Error"]["Message"].should.equal( + "You specified an account that doesn't exist." + ) with assert_raises(ClientError) as e: response = client.list_policies_for_target( TargetId="meaninglessstring", Filter="SERVICE_CONTROL_POLICY" ) ex = e.exception ex.operation_name.should.equal("ListPoliciesForTarget") - ex.response["Error"]["Code"].should.equal("400") - ex.response["Error"]["Message"].should.contain("InvalidInputException") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidInputException") + ex.response["Error"]["Message"].should.equal("You specified an invalid value.") @mock_organizations @@ -694,8 +711,9 @@ def test_list_targets_for_policy_exception(): response = client.list_targets_for_policy(PolicyId="meaninglessstring") ex = e.exception ex.operation_name.should.equal("ListTargetsForPolicy") - ex.response["Error"]["Code"].should.equal("400") - ex.response["Error"]["Message"].should.contain("InvalidInputException") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidInputException") + ex.response["Error"]["Message"].should.equal("You specified an invalid value.") @mock_organizations @@ -947,3 +965,343 @@ def test_disable_aws_service_access_errors(): ex.response["Error"]["Message"].should.equal( "You specified an unrecognized service principal." ) + + +@mock_organizations +def test_register_delegated_administrator(): + # given + client = boto3.client("organizations", region_name="us-east-1") + org_id = client.create_organization(FeatureSet="ALL")["Organization"]["Id"] + account_id = client.create_account(AccountName=mockname, Email=mockemail)[ + "CreateAccountStatus" + ]["AccountId"] + + # when + client.register_delegated_administrator( + AccountId=account_id, ServicePrincipal="ssm.amazonaws.com" + ) + + # then + response = client.list_delegated_administrators() + response["DelegatedAdministrators"].should.have.length_of(1) + admin = response["DelegatedAdministrators"][0] + admin["Id"].should.equal(account_id) + admin["Arn"].should.equal( + "arn:aws:organizations::{0}:account/{1}/{2}".format( + ACCOUNT_ID, org_id, account_id + ) + ) + admin["Email"].should.equal(mockemail) + admin["Name"].should.equal(mockname) + admin["Status"].should.equal("ACTIVE") + admin["JoinedMethod"].should.equal("CREATED") + admin["JoinedTimestamp"].should.be.a(datetime) + admin["DelegationEnabledDate"].should.be.a(datetime) + + +@mock_organizations +def test_register_delegated_administrator_errors(): + # given + client = boto3.client("organizations", region_name="us-east-1") + client.create_organization(FeatureSet="ALL") + account_id = client.create_account(AccountName=mockname, Email=mockemail)[ + "CreateAccountStatus" + ]["AccountId"] + client.register_delegated_administrator( + AccountId=account_id, ServicePrincipal="ssm.amazonaws.com" + ) + + # register master Account + # when + with assert_raises(ClientError) as e: + client.register_delegated_administrator( + AccountId=ACCOUNT_ID, ServicePrincipal="ssm.amazonaws.com" + ) + + # then + ex = e.exception + ex.operation_name.should.equal("RegisterDelegatedAdministrator") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("ConstraintViolationException") + ex.response["Error"]["Message"].should.equal( + "You cannot register master account/yourself as delegated administrator for your organization." + ) + + # register not existing Account + # when + with assert_raises(ClientError) as e: + client.register_delegated_administrator( + AccountId="000000000000", ServicePrincipal="ssm.amazonaws.com" + ) + + # then + ex = e.exception + ex.operation_name.should.equal("RegisterDelegatedAdministrator") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("AccountNotFoundException") + ex.response["Error"]["Message"].should.equal( + "You specified an account that doesn't exist." + ) + + # register not supported service + # when + with assert_raises(ClientError) as e: + client.register_delegated_administrator( + AccountId=account_id, ServicePrincipal="moto.amazonaws.com" + ) + + # then + ex = e.exception + ex.operation_name.should.equal("RegisterDelegatedAdministrator") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidInputException") + ex.response["Error"]["Message"].should.equal( + "You specified an unrecognized service principal." + ) + + # register service again + # when + with assert_raises(ClientError) as e: + client.register_delegated_administrator( + AccountId=account_id, ServicePrincipal="ssm.amazonaws.com" + ) + + # then + ex = e.exception + ex.operation_name.should.equal("RegisterDelegatedAdministrator") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("AccountAlreadyRegisteredException") + ex.response["Error"]["Message"].should.equal( + "The provided account is already a delegated administrator for your organization." + ) + + +@mock_organizations +def test_list_delegated_administrators(): + # given + client = boto3.client("organizations", region_name="us-east-1") + org_id = client.create_organization(FeatureSet="ALL")["Organization"]["Id"] + account_id_1 = client.create_account(AccountName=mockname, Email=mockemail)[ + "CreateAccountStatus" + ]["AccountId"] + account_id_2 = client.create_account(AccountName=mockname, Email=mockemail)[ + "CreateAccountStatus" + ]["AccountId"] + client.register_delegated_administrator( + AccountId=account_id_1, ServicePrincipal="ssm.amazonaws.com" + ) + client.register_delegated_administrator( + AccountId=account_id_2, ServicePrincipal="guardduty.amazonaws.com" + ) + + # when + response = client.list_delegated_administrators() + + # then + response["DelegatedAdministrators"].should.have.length_of(2) + sorted([admin["Id"] for admin in response["DelegatedAdministrators"]]).should.equal( + sorted([account_id_1, account_id_2]) + ) + + # when + response = client.list_delegated_administrators( + ServicePrincipal="ssm.amazonaws.com" + ) + + # then + response["DelegatedAdministrators"].should.have.length_of(1) + admin = response["DelegatedAdministrators"][0] + admin["Id"].should.equal(account_id_1) + admin["Arn"].should.equal( + "arn:aws:organizations::{0}:account/{1}/{2}".format( + ACCOUNT_ID, org_id, account_id_1 + ) + ) + admin["Email"].should.equal(mockemail) + admin["Name"].should.equal(mockname) + admin["Status"].should.equal("ACTIVE") + admin["JoinedMethod"].should.equal("CREATED") + admin["JoinedTimestamp"].should.be.a(datetime) + admin["DelegationEnabledDate"].should.be.a(datetime) + + +@mock_organizations +def test_list_delegated_administrators_erros(): + # given + client = boto3.client("organizations", region_name="us-east-1") + client.create_organization(FeatureSet="ALL") + + # list not supported service + # when + with assert_raises(ClientError) as e: + client.list_delegated_administrators(ServicePrincipal="moto.amazonaws.com") + + # then + ex = e.exception + ex.operation_name.should.equal("ListDelegatedAdministrators") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidInputException") + ex.response["Error"]["Message"].should.equal( + "You specified an unrecognized service principal." + ) + + +@mock_organizations +def test_list_delegated_services_for_account(): + # given + client = boto3.client("organizations", region_name="us-east-1") + client.create_organization(FeatureSet="ALL") + account_id = client.create_account(AccountName=mockname, Email=mockemail)[ + "CreateAccountStatus" + ]["AccountId"] + client.register_delegated_administrator( + AccountId=account_id, ServicePrincipal="ssm.amazonaws.com" + ) + client.register_delegated_administrator( + AccountId=account_id, ServicePrincipal="guardduty.amazonaws.com" + ) + + # when + response = client.list_delegated_services_for_account(AccountId=account_id) + + # then + response["DelegatedServices"].should.have.length_of(2) + sorted( + [service["ServicePrincipal"] for service in response["DelegatedServices"]] + ).should.equal(["guardduty.amazonaws.com", "ssm.amazonaws.com"]) + + +@mock_organizations +def test_list_delegated_services_for_account_erros(): + # given + client = boto3.client("organizations", region_name="us-east-1") + client.create_organization(FeatureSet="ALL") + + # list services for not existing Account + # when + with assert_raises(ClientError) as e: + client.list_delegated_services_for_account(AccountId="000000000000") + + # then + ex = e.exception + ex.operation_name.should.equal("ListDelegatedServicesForAccount") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("AWSOrganizationsNotInUseException") + ex.response["Error"]["Message"].should.equal( + "Your account is not a member of an organization." + ) + + # list services for not registered Account + # when + with assert_raises(ClientError) as e: + client.list_delegated_services_for_account(AccountId=ACCOUNT_ID) + + # then + ex = e.exception + ex.operation_name.should.equal("ListDelegatedServicesForAccount") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("AccountNotRegisteredException") + ex.response["Error"]["Message"].should.equal( + "The provided account is not a registered delegated administrator for your organization." + ) + + +@mock_organizations +def test_deregister_delegated_administrator(): + # given + client = boto3.client("organizations", region_name="us-east-1") + client.create_organization(FeatureSet="ALL") + account_id = client.create_account(AccountName=mockname, Email=mockemail)[ + "CreateAccountStatus" + ]["AccountId"] + client.register_delegated_administrator( + AccountId=account_id, ServicePrincipal="ssm.amazonaws.com" + ) + + # when + client.deregister_delegated_administrator( + AccountId=account_id, ServicePrincipal="ssm.amazonaws.com" + ) + + # then + response = client.list_delegated_administrators() + response["DelegatedAdministrators"].should.have.length_of(0) + + +@mock_organizations +def test_deregister_delegated_administrator_erros(): + # given + client = boto3.client("organizations", region_name="us-east-1") + client.create_organization(FeatureSet="ALL") + account_id = client.create_account(AccountName=mockname, Email=mockemail)[ + "CreateAccountStatus" + ]["AccountId"] + + # deregister master Account + # when + with assert_raises(ClientError) as e: + client.deregister_delegated_administrator( + AccountId=ACCOUNT_ID, ServicePrincipal="ssm.amazonaws.com" + ) + + # then + ex = e.exception + ex.operation_name.should.equal("DeregisterDelegatedAdministrator") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("ConstraintViolationException") + ex.response["Error"]["Message"].should.equal( + "You cannot register master account/yourself as delegated administrator for your organization." + ) + + # deregister not existing Account + # when + with assert_raises(ClientError) as e: + client.deregister_delegated_administrator( + AccountId="000000000000", ServicePrincipal="ssm.amazonaws.com" + ) + + # then + ex = e.exception + ex.operation_name.should.equal("DeregisterDelegatedAdministrator") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("AccountNotFoundException") + ex.response["Error"]["Message"].should.equal( + "You specified an account that doesn't exist." + ) + + # deregister not registered Account + # when + with assert_raises(ClientError) as e: + client.deregister_delegated_administrator( + AccountId=account_id, ServicePrincipal="ssm.amazonaws.com" + ) + + # then + ex = e.exception + ex.operation_name.should.equal("DeregisterDelegatedAdministrator") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("AccountNotRegisteredException") + ex.response["Error"]["Message"].should.equal( + "The provided account is not a registered delegated administrator for your organization." + ) + + # given + client.register_delegated_administrator( + AccountId=account_id, ServicePrincipal="ssm.amazonaws.com" + ) + + # deregister not registered service + # when + with assert_raises(ClientError) as e: + client.deregister_delegated_administrator( + AccountId=account_id, ServicePrincipal="guardduty.amazonaws.com" + ) + + # then + ex = e.exception + ex.operation_name.should.equal("DeregisterDelegatedAdministrator") + ex.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.response["Error"]["Code"].should.contain("InvalidInputException") + ex.response["Error"]["Message"].should.equal( + "You specified an unrecognized service principal." + ) From 88a11346576d740983fe648e62d88e5ba780e38d Mon Sep 17 00:00:00 2001 From: Iain Bullard Date: Fri, 31 Jul 2020 17:46:48 +0100 Subject: [PATCH 3/5] Fix DynamoDb2 ExpressionAttributeNames can start with a number (#3206) When using pynamodb's support for transactions it makes use of of ExpressionAttributeNames that look like #0 #1 etc. According to https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html And when testing against dynamodb-local these work without issue, however, when testing with moto they fail. --- moto/dynamodb2/parsing/tokens.py | 2 +- .../test_dynamodb_expression_tokenizer.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/moto/dynamodb2/parsing/tokens.py b/moto/dynamodb2/parsing/tokens.py index 4fbb7883a..34c3151ef 100644 --- a/moto/dynamodb2/parsing/tokens.py +++ b/moto/dynamodb2/parsing/tokens.py @@ -109,7 +109,7 @@ class ExpressionTokenizer(object): @classmethod def is_expression_attribute(cls, input_string): - return re.compile("^[a-zA-Z][a-zA-Z0-9_]*$").match(input_string) is not None + return re.compile("^[a-zA-Z0-9][a-zA-Z0-9_]*$").match(input_string) is not None @classmethod def is_expression_attribute_name(cls, input_string): diff --git a/tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py b/tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py index 3330d431e..ddfb81d1a 100644 --- a/tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py +++ b/tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py @@ -219,6 +219,18 @@ def test_expression_tokenizer_single_set_action_attribute_name_valid_key(): ] +def test_expression_tokenizer_single_set_action_attribute_name_leading_number(): + set_action = "SET attr=#0" + token_list = ExpressionTokenizer.make_list(set_action) + assert token_list == [ + Token(Token.ATTRIBUTE, "SET"), + Token(Token.WHITESPACE, " "), + Token(Token.ATTRIBUTE, "attr"), + Token(Token.EQUAL_SIGN, "="), + Token(Token.ATTRIBUTE_NAME, "#0"), + ] + + def test_expression_tokenizer_just_a_pipe(): set_action = "|" try: From 9a9a1d8413e1c93800b6ae695842a91e9487c6d8 Mon Sep 17 00:00:00 2001 From: Adam Richie-Halford Date: Sat, 1 Aug 2020 07:23:36 -0700 Subject: [PATCH 4/5] Decentralize cloudformation naming responsibilities (#3201) * #3127 - Decentralize CF naming responsibilities * Decentralize CloudFormation naming responsibilities * Update URLs in cloudformation_resource_type functions * Fix flake8 errors * Black formatting * Add a bunch of imports to populate CloudFormationModel.__subclasses__ * Add noqa to s3 models import statement in cloudformation/parsing.py * Black formatting * Remove debugging print statement Co-authored-by: Bert Blommers --- moto/autoscaling/models.py | 24 +++- moto/awslambda/models.py | 35 +++++- moto/batch/models.py | 35 +++++- moto/cloudformation/parsing.py | 160 +++++++------------------ moto/cloudwatch/models.py | 13 +- moto/core/__init__.py | 1 + moto/core/models.py | 42 +++++++ moto/datapipeline/models.py | 13 +- moto/dynamodb/models.py | 13 +- moto/dynamodb2/models/__init__.py | 13 +- moto/ec2/models.py | 189 +++++++++++++++++++++++++++--- moto/ecr/models.py | 13 +- moto/ecs/models.py | 35 +++++- moto/elb/models.py | 13 +- moto/elbv2/models.py | 35 +++++- moto/events/models.py | 24 +++- moto/iam/models.py | 24 +++- moto/kinesis/models.py | 13 +- moto/kms/models.py | 13 +- moto/rds/models.py | 42 +++++-- moto/rds2/models.py | 52 ++++++-- moto/redshift/models.py | 36 +++++- moto/route53/models.py | 46 +++++++- moto/s3/models.py | 13 +- moto/sns/models.py | 15 ++- moto/sqs/models.py | 13 +- 26 files changed, 717 insertions(+), 208 deletions(-) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index f4185da6c..d82f15095 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -6,7 +6,7 @@ from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping from moto.ec2.exceptions import InvalidInstanceIdError from moto.compat import OrderedDict -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.ec2 import ec2_backends from moto.elb import elb_backends from moto.elbv2 import elbv2_backends @@ -74,7 +74,7 @@ class FakeScalingPolicy(BaseModel): ) -class FakeLaunchConfiguration(BaseModel): +class FakeLaunchConfiguration(CloudFormationModel): def __init__( self, name, @@ -127,6 +127,15 @@ class FakeLaunchConfiguration(BaseModel): ) return config + @staticmethod + def cloudformation_name_type(): + return "LaunchConfigurationName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-autoscaling-launchconfiguration.html + return "AWS::AutoScaling::LaunchConfiguration" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -215,7 +224,7 @@ class FakeLaunchConfiguration(BaseModel): return block_device_map -class FakeAutoScalingGroup(BaseModel): +class FakeAutoScalingGroup(CloudFormationModel): def __init__( self, name, @@ -309,6 +318,15 @@ class FakeAutoScalingGroup(BaseModel): tag["PropagateAtLaunch"] = bool_to_string[tag["PropagateAtLaunch"]] return tags + @staticmethod + def cloudformation_name_type(): + return "AutoScalingGroupName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-autoscaling-autoscalinggroup.html + return "AWS::AutoScaling::AutoScalingGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index afbe9775a..a234fbe01 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -28,7 +28,7 @@ import requests.adapters from boto3 import Session from moto.awslambda.policy import Policy -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, CloudFormationModel from moto.core.exceptions import RESTError from moto.iam.models import iam_backend from moto.iam.exceptions import IAMNotFoundException @@ -151,7 +151,7 @@ class _DockerDataVolumeContext: raise # multiple processes trying to use same volume? -class LambdaFunction(BaseModel): +class LambdaFunction(CloudFormationModel): def __init__(self, spec, region, validate_s3=True, version=1): # required self.region = region @@ -492,6 +492,15 @@ class LambdaFunction(BaseModel): return result + @staticmethod + def cloudformation_name_type(): + return "FunctionName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-function.html + return "AWS::Lambda::Function" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -556,7 +565,7 @@ class LambdaFunction(BaseModel): lambda_backends[region].delete_function(self.function_name) -class EventSourceMapping(BaseModel): +class EventSourceMapping(CloudFormationModel): def __init__(self, spec): # required self.function_name = spec["FunctionName"] @@ -633,6 +642,15 @@ class EventSourceMapping(BaseModel): lambda_backend = lambda_backends[region_name] lambda_backend.delete_event_source_mapping(self.uuid) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-eventsourcemapping.html + return "AWS::Lambda::EventSourceMapping" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -667,13 +685,22 @@ class EventSourceMapping(BaseModel): esm.delete(region_name) -class LambdaVersion(BaseModel): +class LambdaVersion(CloudFormationModel): def __init__(self, spec): self.version = spec["Version"] def __repr__(self): return str(self.logical_resource_id) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-version.html + return "AWS::Lambda::Version" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/batch/models.py b/moto/batch/models.py index fde744911..c4bc81a73 100644 --- a/moto/batch/models.py +++ b/moto/batch/models.py @@ -13,7 +13,7 @@ import threading import dateutil.parser from boto3 import Session -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.iam import iam_backends from moto.ec2 import ec2_backends from moto.ecs import ecs_backends @@ -42,7 +42,7 @@ def datetime2int(date): return int(time.mktime(date.timetuple())) -class ComputeEnvironment(BaseModel): +class ComputeEnvironment(CloudFormationModel): def __init__( self, compute_environment_name, @@ -76,6 +76,15 @@ class ComputeEnvironment(BaseModel): def physical_resource_id(self): return self.arn + @staticmethod + def cloudformation_name_type(): + return "ComputeEnvironmentName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-batch-computeenvironment.html + return "AWS::Batch::ComputeEnvironment" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -95,7 +104,7 @@ class ComputeEnvironment(BaseModel): return backend.get_compute_environment_by_arn(arn) -class JobQueue(BaseModel): +class JobQueue(CloudFormationModel): def __init__( self, name, priority, state, environments, env_order_json, region_name ): @@ -139,6 +148,15 @@ class JobQueue(BaseModel): def physical_resource_id(self): return self.arn + @staticmethod + def cloudformation_name_type(): + return "JobQueueName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-batch-jobqueue.html + return "AWS::Batch::JobQueue" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -164,7 +182,7 @@ class JobQueue(BaseModel): return backend.get_job_queue_by_arn(arn) -class JobDefinition(BaseModel): +class JobDefinition(CloudFormationModel): def __init__( self, name, @@ -264,6 +282,15 @@ class JobDefinition(BaseModel): def physical_resource_id(self): return self.arn + @staticmethod + def cloudformation_name_type(): + return "JobDefinitionName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-batch-jobdefinition.html + return "AWS::Batch::JobDefinition" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 58409901d..2c212a148 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -6,31 +6,43 @@ import copy import warnings import re -from moto.autoscaling import models as autoscaling_models -from moto.awslambda import models as lambda_models -from moto.batch import models as batch_models -from moto.cloudwatch import models as cloudwatch_models -from moto.cognitoidentity import models as cognitoidentity_models from moto.compat import collections_abc -from moto.datapipeline import models as datapipeline_models -from moto.dynamodb2 import models as dynamodb2_models + +# This ugly section of imports is necessary because we +# build the list of CloudFormationModel subclasses using +# CloudFormationModel.__subclasses__(). However, if the class +# definition of a subclass hasn't been executed yet - for example, if +# the subclass's module hasn't been imported yet - then that subclass +# doesn't exist yet, and __subclasses__ won't find it. +# So we import here to populate the list of subclasses. +from moto.autoscaling import models as autoscaling_models # noqa +from moto.awslambda import models as awslambda_models # noqa +from moto.batch import models as batch_models # noqa +from moto.cloudwatch import models as cloudwatch_models # noqa +from moto.datapipeline import models as datapipeline_models # noqa +from moto.dynamodb2 import models as dynamodb2_models # noqa +from moto.ecr import models as ecr_models # noqa +from moto.ecs import models as ecs_models # noqa +from moto.elb import models as elb_models # noqa +from moto.elbv2 import models as elbv2_models # noqa +from moto.events import models as events_models # noqa +from moto.iam import models as iam_models # noqa +from moto.kinesis import models as kinesis_models # noqa +from moto.kms import models as kms_models # noqa +from moto.rds import models as rds_models # noqa +from moto.rds2 import models as rds2_models # noqa +from moto.redshift import models as redshift_models # noqa +from moto.route53 import models as route53_models # noqa +from moto.s3 import models as s3_models # noqa +from moto.sns import models as sns_models # noqa +from moto.sqs import models as sqs_models # noqa + +# End ugly list of imports + from moto.ec2 import models as ec2_models -from moto.ecs import models as ecs_models -from moto.elb import models as elb_models -from moto.elbv2 import models as elbv2_models -from moto.events import models as events_models -from moto.iam import models as iam_models -from moto.kinesis import models as kinesis_models -from moto.kms import models as kms_models -from moto.rds import models as rds_models -from moto.rds2 import models as rds2_models -from moto.redshift import models as redshift_models -from moto.route53 import models as route53_models -from moto.s3 import models as s3_models, s3_backend +from moto.s3 import models as _, s3_backend # noqa from moto.s3.utils import bucket_and_name_from_url -from moto.sns import models as sns_models -from moto.sqs import models as sqs_models -from moto.core import ACCOUNT_ID +from moto.core import ACCOUNT_ID, CloudFormationModel from .utils import random_suffix from .exceptions import ( ExportNotFound, @@ -40,105 +52,13 @@ from .exceptions import ( ) from boto.cloudformation.stack import Output -MODEL_MAP = { - "AWS::AutoScaling::AutoScalingGroup": autoscaling_models.FakeAutoScalingGroup, - "AWS::AutoScaling::LaunchConfiguration": autoscaling_models.FakeLaunchConfiguration, - "AWS::Batch::JobDefinition": batch_models.JobDefinition, - "AWS::Batch::JobQueue": batch_models.JobQueue, - "AWS::Batch::ComputeEnvironment": batch_models.ComputeEnvironment, - "AWS::DynamoDB::Table": dynamodb2_models.Table, - "AWS::Kinesis::Stream": kinesis_models.Stream, - "AWS::Lambda::EventSourceMapping": lambda_models.EventSourceMapping, - "AWS::Lambda::Function": lambda_models.LambdaFunction, - "AWS::Lambda::Version": lambda_models.LambdaVersion, - "AWS::EC2::EIP": ec2_models.ElasticAddress, - "AWS::EC2::Instance": ec2_models.Instance, - "AWS::EC2::InternetGateway": ec2_models.InternetGateway, - "AWS::EC2::NatGateway": ec2_models.NatGateway, - "AWS::EC2::NetworkInterface": ec2_models.NetworkInterface, - "AWS::EC2::Route": ec2_models.Route, - "AWS::EC2::RouteTable": ec2_models.RouteTable, - "AWS::EC2::SecurityGroup": ec2_models.SecurityGroup, - "AWS::EC2::SecurityGroupIngress": ec2_models.SecurityGroupIngress, - "AWS::EC2::SpotFleet": ec2_models.SpotFleetRequest, - "AWS::EC2::Subnet": ec2_models.Subnet, - "AWS::EC2::SubnetRouteTableAssociation": ec2_models.SubnetRouteTableAssociation, - "AWS::EC2::Volume": ec2_models.Volume, - "AWS::EC2::VolumeAttachment": ec2_models.VolumeAttachment, - "AWS::EC2::VPC": ec2_models.VPC, - "AWS::EC2::VPCGatewayAttachment": ec2_models.VPCGatewayAttachment, - "AWS::EC2::VPCPeeringConnection": ec2_models.VPCPeeringConnection, - "AWS::ECS::Cluster": ecs_models.Cluster, - "AWS::ECS::TaskDefinition": ecs_models.TaskDefinition, - "AWS::ECS::Service": ecs_models.Service, - "AWS::ElasticLoadBalancing::LoadBalancer": elb_models.FakeLoadBalancer, - "AWS::ElasticLoadBalancingV2::LoadBalancer": elbv2_models.FakeLoadBalancer, - "AWS::ElasticLoadBalancingV2::TargetGroup": elbv2_models.FakeTargetGroup, - "AWS::ElasticLoadBalancingV2::Listener": elbv2_models.FakeListener, - "AWS::Cognito::IdentityPool": cognitoidentity_models.CognitoIdentity, - "AWS::DataPipeline::Pipeline": datapipeline_models.Pipeline, - "AWS::IAM::InstanceProfile": iam_models.InstanceProfile, - "AWS::IAM::Role": iam_models.Role, - "AWS::KMS::Key": kms_models.Key, - "AWS::Logs::LogGroup": cloudwatch_models.LogGroup, - "AWS::RDS::DBInstance": rds_models.Database, - "AWS::RDS::DBSecurityGroup": rds_models.SecurityGroup, - "AWS::RDS::DBSubnetGroup": rds_models.SubnetGroup, - "AWS::RDS::DBParameterGroup": rds2_models.DBParameterGroup, - "AWS::Redshift::Cluster": redshift_models.Cluster, - "AWS::Redshift::ClusterParameterGroup": redshift_models.ParameterGroup, - "AWS::Redshift::ClusterSubnetGroup": redshift_models.SubnetGroup, - "AWS::Route53::HealthCheck": route53_models.HealthCheck, - "AWS::Route53::HostedZone": route53_models.FakeZone, - "AWS::Route53::RecordSet": route53_models.RecordSet, - "AWS::Route53::RecordSetGroup": route53_models.RecordSetGroup, - "AWS::SNS::Topic": sns_models.Topic, - "AWS::S3::Bucket": s3_models.FakeBucket, - "AWS::SQS::Queue": sqs_models.Queue, - "AWS::Events::Rule": events_models.Rule, - "AWS::Events::EventBus": events_models.EventBus, -} - -UNDOCUMENTED_NAME_TYPE_MAP = { - "AWS::AutoScaling::AutoScalingGroup": "AutoScalingGroupName", - "AWS::AutoScaling::LaunchConfiguration": "LaunchConfigurationName", - "AWS::IAM::InstanceProfile": "InstanceProfileName", -} - -# http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-name.html +# List of supported CloudFormation models +MODEL_LIST = CloudFormationModel.__subclasses__() +MODEL_MAP = {model.cloudformation_type(): model for model in MODEL_LIST} NAME_TYPE_MAP = { - "AWS::ApiGateway::ApiKey": "Name", - "AWS::ApiGateway::Model": "Name", - "AWS::CloudWatch::Alarm": "AlarmName", - "AWS::DynamoDB::Table": "TableName", - "AWS::ElasticBeanstalk::Application": "ApplicationName", - "AWS::ElasticBeanstalk::Environment": "EnvironmentName", - "AWS::CodeDeploy::Application": "ApplicationName", - "AWS::CodeDeploy::DeploymentConfig": "DeploymentConfigName", - "AWS::CodeDeploy::DeploymentGroup": "DeploymentGroupName", - "AWS::Config::ConfigRule": "ConfigRuleName", - "AWS::Config::DeliveryChannel": "Name", - "AWS::Config::ConfigurationRecorder": "Name", - "AWS::ElasticLoadBalancing::LoadBalancer": "LoadBalancerName", - "AWS::ElasticLoadBalancingV2::LoadBalancer": "Name", - "AWS::ElasticLoadBalancingV2::TargetGroup": "Name", - "AWS::EC2::SecurityGroup": "GroupName", - "AWS::ElastiCache::CacheCluster": "ClusterName", - "AWS::ECR::Repository": "RepositoryName", - "AWS::ECS::Cluster": "ClusterName", - "AWS::Elasticsearch::Domain": "DomainName", - "AWS::Events::Rule": "Name", - "AWS::IAM::Group": "GroupName", - "AWS::IAM::ManagedPolicy": "ManagedPolicyName", - "AWS::IAM::Role": "RoleName", - "AWS::IAM::User": "UserName", - "AWS::Lambda::Function": "FunctionName", - "AWS::RDS::DBInstance": "DBInstanceIdentifier", - "AWS::S3::Bucket": "BucketName", - "AWS::SNS::Topic": "TopicName", - "AWS::SQS::Queue": "QueueName", + model.cloudformation_type(): model.cloudformation_name_type() + for model in MODEL_LIST } -NAME_TYPE_MAP.update(UNDOCUMENTED_NAME_TYPE_MAP) # Just ignore these models types for now NULL_MODELS = [ @@ -292,9 +212,11 @@ def clean_json(resource_json, resources_map): def resource_class_from_type(resource_type): if resource_type in NULL_MODELS: return None + if resource_type not in MODEL_MAP: logger.warning("No Moto CloudFormation support for %s", resource_type) return None + return MODEL_MAP.get(resource_type) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index f089acb14..d8b28bc97 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -3,7 +3,7 @@ import json from boto3 import Session from moto.core.utils import iso_8601_datetime_without_milliseconds -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.exceptions import RESTError from moto.logs import logs_backends from datetime import datetime, timedelta @@ -490,13 +490,22 @@ class CloudWatchBackend(BaseBackend): return None, metrics -class LogGroup(BaseModel): +class LogGroup(CloudFormationModel): def __init__(self, spec): # required self.name = spec["LogGroupName"] # optional self.tags = spec.get("Tags", []) + @staticmethod + def cloudformation_name_type(): + return "LogGroupName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html + return "AWS::Logs::LogGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/core/__init__.py b/moto/core/__init__.py index 045124fab..09f5b1e16 100644 --- a/moto/core/__init__.py +++ b/moto/core/__init__.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from .models import BaseModel, BaseBackend, moto_api_backend, ACCOUNT_ID # noqa +from .models import CloudFormationModel # noqa from .responses import ActionAuthenticatorMixin moto_api_backends = {"global": moto_api_backend} diff --git a/moto/core/models.py b/moto/core/models.py index 26ee1a1f5..ded6a4fc1 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -8,6 +8,7 @@ import os import re import six import types +from abc import abstractmethod from io import BytesIO from collections import defaultdict from botocore.config import Config @@ -534,6 +535,47 @@ class BaseModel(object): return instance +# Parent class for every Model that can be instantiated by CloudFormation +# On subclasses, implement the two methods as @staticmethod to ensure correct behaviour of the CF parser +class CloudFormationModel(BaseModel): + @abstractmethod + def cloudformation_name_type(self): + # This must be implemented as a staticmethod with no parameters + # Return None for resources that do not have a name property + pass + + @abstractmethod + def cloudformation_type(self): + # This must be implemented as a staticmethod with no parameters + # See for example https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-dynamodb-table.html + return "AWS::SERVICE::RESOURCE" + + @abstractmethod + def create_from_cloudformation_json(self): + # This must be implemented as a classmethod with parameters: + # cls, resource_name, cloudformation_json, region_name + # Extract the resource parameters from the cloudformation json + # and return an instance of the resource class + pass + + @abstractmethod + def update_from_cloudformation_json(self): + # This must be implemented as a classmethod with parameters: + # cls, original_resource, new_resource_name, cloudformation_json, region_name + # Extract the resource parameters from the cloudformation json, + # delete the old resource and return the new one. Optionally inspect + # the change in parameters and no-op when nothing has changed. + pass + + @abstractmethod + def delete_from_cloudformation_json(self): + # This must be implemented as a classmethod with parameters: + # cls, resource_name, cloudformation_json, region_name + # Extract the resource parameters from the cloudformation json + # and delete the resource. Do not include a return statement. + pass + + class BaseBackend(object): def _reset_model_refs(self): # Remove all references to the models stored diff --git a/moto/datapipeline/models.py b/moto/datapipeline/models.py index d93deea61..b17da1f09 100644 --- a/moto/datapipeline/models.py +++ b/moto/datapipeline/models.py @@ -4,7 +4,7 @@ import datetime from boto3 import Session from moto.compat import OrderedDict -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from .utils import get_random_pipeline_id, remove_capitalization_of_dict_keys @@ -18,7 +18,7 @@ class PipelineObject(BaseModel): return {"fields": self.fields, "id": self.object_id, "name": self.name} -class Pipeline(BaseModel): +class Pipeline(CloudFormationModel): def __init__(self, name, unique_id, **kwargs): self.name = name self.unique_id = unique_id @@ -74,6 +74,15 @@ class Pipeline(BaseModel): def activate(self): self.status = "SCHEDULED" + @staticmethod + def cloudformation_name_type(): + return "Name" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-datapipeline-pipeline.html + return "AWS::DataPipeline::Pipeline" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/dynamodb/models.py b/moto/dynamodb/models.py index f5771ec6e..1a3b4afce 100644 --- a/moto/dynamodb/models.py +++ b/moto/dynamodb/models.py @@ -4,7 +4,7 @@ import datetime import json from moto.compat import OrderedDict -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import unix_time from moto.core import ACCOUNT_ID from .comparisons import get_comparison_func @@ -82,7 +82,7 @@ class Item(BaseModel): return {"Item": included} -class Table(BaseModel): +class Table(CloudFormationModel): def __init__( self, name, @@ -135,6 +135,15 @@ class Table(BaseModel): } return results + @staticmethod + def cloudformation_name_type(): + return "TableName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-dynamodb-table.html + return "AWS::DynamoDB::Table" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/dynamodb2/models/__init__.py b/moto/dynamodb2/models/__init__.py index 70fcc5d09..175ed64f8 100644 --- a/moto/dynamodb2/models/__init__.py +++ b/moto/dynamodb2/models/__init__.py @@ -9,7 +9,7 @@ import uuid from boto3 import Session from moto.compat import OrderedDict -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import unix_time from moto.core.exceptions import JsonRESTError from moto.dynamodb2.comparisons import get_filter_expression @@ -359,7 +359,7 @@ class GlobalSecondaryIndex(SecondaryIndex): self.throughput = u.get("ProvisionedThroughput", self.throughput) -class Table(BaseModel): +class Table(CloudFormationModel): def __init__( self, table_name, @@ -431,6 +431,15 @@ class Table(BaseModel): def physical_resource_id(self): return self.name + @staticmethod + def cloudformation_name_type(): + return "TableName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-dynamodb-table.html + return "AWS::DynamoDB::Table" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/ec2/models.py b/moto/ec2/models.py index ad9ae3b1b..3d60654a9 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -22,7 +22,7 @@ from boto.ec2.launchspecification import LaunchSpecification from moto.compat import OrderedDict from moto.core import BaseBackend -from moto.core.models import Model, BaseModel +from moto.core.models import Model, BaseModel, CloudFormationModel from moto.core.utils import ( iso_8601_datetime_with_milliseconds, camelcase_to_underscores, @@ -219,7 +219,7 @@ class TaggedEC2Resource(BaseModel): raise FilterNotImplementedError(filter_name, method_name) -class NetworkInterface(TaggedEC2Resource): +class NetworkInterface(TaggedEC2Resource, CloudFormationModel): def __init__( self, ec2_backend, @@ -268,6 +268,15 @@ class NetworkInterface(TaggedEC2Resource): if group: self._group_set.append(group) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-networkinterface.html + return "AWS::EC2::NetworkInterface" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -454,7 +463,7 @@ class NetworkInterfaceBackend(object): return generic_filter(filters, enis) -class Instance(TaggedEC2Resource, BotoInstance): +class Instance(TaggedEC2Resource, BotoInstance, CloudFormationModel): VALID_ATTRIBUTES = { "instanceType", "kernel", @@ -621,6 +630,15 @@ class Instance(TaggedEC2Resource, BotoInstance): formatted_ip, self.region_name ) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-instance.html + return "AWS::EC2::Instance" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -1843,7 +1861,7 @@ class SecurityRule(object): return True -class SecurityGroup(TaggedEC2Resource): +class SecurityGroup(TaggedEC2Resource, CloudFormationModel): def __init__(self, ec2_backend, group_id, name, description, vpc_id=None): self.ec2_backend = ec2_backend self.id = group_id @@ -1861,6 +1879,15 @@ class SecurityGroup(TaggedEC2Resource): if vpc and len(vpc.get_cidr_block_association_set(ipv6=True)) > 0: self.egress_rules.append(SecurityRule("-1", None, None, [], [])) + @staticmethod + def cloudformation_name_type(): + return "GroupName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-securitygroup.html + return "AWS::EC2::SecurityGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -2260,11 +2287,20 @@ class SecurityGroupBackend(object): raise RulesPerSecurityGroupLimitExceededError -class SecurityGroupIngress(object): +class SecurityGroupIngress(CloudFormationModel): def __init__(self, security_group, properties): self.security_group = security_group self.properties = properties + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-securitygroupingress.html + return "AWS::EC2::SecurityGroupIngress" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -2328,7 +2364,7 @@ class SecurityGroupIngress(object): return cls(security_group, properties) -class VolumeAttachment(object): +class VolumeAttachment(CloudFormationModel): def __init__(self, volume, instance, device, status): self.volume = volume self.attach_time = utc_date_and_time() @@ -2336,6 +2372,15 @@ class VolumeAttachment(object): self.device = device self.status = status + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-volumeattachment.html + return "AWS::EC2::VolumeAttachment" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -2354,7 +2399,7 @@ class VolumeAttachment(object): return attachment -class Volume(TaggedEC2Resource): +class Volume(TaggedEC2Resource, CloudFormationModel): def __init__( self, ec2_backend, volume_id, size, zone, snapshot_id=None, encrypted=False ): @@ -2367,6 +2412,15 @@ class Volume(TaggedEC2Resource): self.ec2_backend = ec2_backend self.encrypted = encrypted + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-volume.html + return "AWS::EC2::Volume" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -2623,7 +2677,7 @@ class EBSBackend(object): return True -class VPC(TaggedEC2Resource): +class VPC(TaggedEC2Resource, CloudFormationModel): def __init__( self, ec2_backend, @@ -2656,6 +2710,15 @@ class VPC(TaggedEC2Resource): amazon_provided_ipv6_cidr_block=amazon_provided_ipv6_cidr_block, ) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-vpc.html + return "AWS::EC2::VPC" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -3022,13 +3085,22 @@ class VPCPeeringConnectionStatus(object): self.message = "Inactive" -class VPCPeeringConnection(TaggedEC2Resource): +class VPCPeeringConnection(TaggedEC2Resource, CloudFormationModel): def __init__(self, vpc_pcx_id, vpc, peer_vpc): self.id = vpc_pcx_id self.vpc = vpc self.peer_vpc = peer_vpc self._status = VPCPeeringConnectionStatus() + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-vpcpeeringconnection.html + return "AWS::EC2::VPCPeeringConnection" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -3114,7 +3186,7 @@ class VPCPeeringConnectionBackend(object): return vpc_pcx -class Subnet(TaggedEC2Resource): +class Subnet(TaggedEC2Resource, CloudFormationModel): def __init__( self, ec2_backend, @@ -3150,6 +3222,15 @@ class Subnet(TaggedEC2Resource): self._unused_ips = set() # if instance is destroyed hold IP here for reuse self._subnet_ips = {} # has IP: instance + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-subnet.html + return "AWS::EC2::Subnet" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -3377,11 +3458,20 @@ class SubnetBackend(object): raise InvalidParameterValueError(attr_name) -class SubnetRouteTableAssociation(object): +class SubnetRouteTableAssociation(CloudFormationModel): def __init__(self, route_table_id, subnet_id): self.route_table_id = route_table_id self.subnet_id = subnet_id + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-subnetroutetableassociation.html + return "AWS::EC2::SubnetRouteTableAssociation" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -3411,7 +3501,7 @@ class SubnetRouteTableAssociationBackend(object): return subnet_association -class RouteTable(TaggedEC2Resource): +class RouteTable(TaggedEC2Resource, CloudFormationModel): def __init__(self, ec2_backend, route_table_id, vpc_id, main=False): self.ec2_backend = ec2_backend self.id = route_table_id @@ -3420,6 +3510,15 @@ class RouteTable(TaggedEC2Resource): self.associations = {} self.routes = {} + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-routetable.html + return "AWS::EC2::RouteTable" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -3555,7 +3654,7 @@ class RouteTableBackend(object): return self.associate_route_table(route_table_id, subnet_id) -class Route(object): +class Route(CloudFormationModel): def __init__( self, route_table, @@ -3581,6 +3680,15 @@ class Route(object): self.interface = interface self.vpc_pcx = vpc_pcx + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-route.html + return "AWS::EC2::Route" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -3748,12 +3856,21 @@ class RouteBackend(object): return deleted -class InternetGateway(TaggedEC2Resource): +class InternetGateway(TaggedEC2Resource, CloudFormationModel): def __init__(self, ec2_backend): self.ec2_backend = ec2_backend self.id = random_internet_gateway_id() self.vpc = None + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-internetgateway.html + return "AWS::EC2::InternetGateway" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -3826,11 +3943,20 @@ class InternetGatewayBackend(object): return self.describe_internet_gateways(internet_gateway_ids=igw_ids)[0] -class VPCGatewayAttachment(BaseModel): +class VPCGatewayAttachment(CloudFormationModel): def __init__(self, gateway_id, vpc_id): self.gateway_id = gateway_id self.vpc_id = vpc_id + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-vpcgatewayattachment.html + return "AWS::EC2::VPCGatewayAttachment" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -4051,7 +4177,7 @@ class SpotFleetLaunchSpec(object): self.weighted_capacity = float(weighted_capacity) -class SpotFleetRequest(TaggedEC2Resource): +class SpotFleetRequest(TaggedEC2Resource, CloudFormationModel): def __init__( self, ec2_backend, @@ -4100,6 +4226,15 @@ class SpotFleetRequest(TaggedEC2Resource): def physical_resource_id(self): return self.id + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-spotfleet.html + return "AWS::EC2::SpotFleet" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -4323,7 +4458,7 @@ class SpotFleetBackend(object): return True -class ElasticAddress(object): +class ElasticAddress(CloudFormationModel): def __init__(self, domain, address=None): if address: self.public_ip = address @@ -4335,6 +4470,15 @@ class ElasticAddress(object): self.eni = None self.association_id = None + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-eip.html + return "AWS::EC2::EIP" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -5095,7 +5239,7 @@ class CustomerGatewayBackend(object): return deleted -class NatGateway(object): +class NatGateway(CloudFormationModel): def __init__(self, backend, subnet_id, allocation_id): # public properties self.id = random_nat_gateway_id() @@ -5133,6 +5277,15 @@ class NatGateway(object): eips = self._backend.address_by_allocation([self.allocation_id]) return eips[0].public_ip + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-natgateway.html + return "AWS::EC2::NatGateway" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/ecr/models.py b/moto/ecr/models.py index 88b058e1e..a1d5aa6e5 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -7,7 +7,7 @@ from random import random from botocore.exceptions import ParamValidationError -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.ec2 import ec2_backends from moto.ecr.exceptions import ImageNotFoundException, RepositoryNotFoundException @@ -38,7 +38,7 @@ class BaseObject(BaseModel): return self.gen_response_object() -class Repository(BaseObject): +class Repository(BaseObject, CloudFormationModel): def __init__(self, repository_name): self.registry_id = DEFAULT_REGISTRY_ID self.arn = "arn:aws:ecr:us-east-1:{0}:repository/{1}".format( @@ -67,6 +67,15 @@ class Repository(BaseObject): del response_object["arn"], response_object["name"], response_object["images"] return response_object + @staticmethod + def cloudformation_name_type(): + return "RepositoryName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ecr-repository.html + return "AWS::ECR::Repository" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/ecs/models.py b/moto/ecs/models.py index a78614cc5..bf20c2245 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -8,7 +8,7 @@ import pytz from boto3 import Session from moto.core.exceptions import JsonRESTError -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import unix_time from moto.ec2 import ec2_backends from copy import copy @@ -44,7 +44,7 @@ class BaseObject(BaseModel): return self.gen_response_object() -class Cluster(BaseObject): +class Cluster(BaseObject, CloudFormationModel): def __init__(self, cluster_name, region_name): self.active_services_count = 0 self.arn = "arn:aws:ecs:{0}:012345678910:cluster/{1}".format( @@ -69,6 +69,15 @@ class Cluster(BaseObject): del response_object["arn"], response_object["name"] return response_object + @staticmethod + def cloudformation_name_type(): + return "ClusterName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ecs-cluster.html + return "AWS::ECS::Cluster" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -116,7 +125,7 @@ class Cluster(BaseObject): raise UnformattedGetAttTemplateException() -class TaskDefinition(BaseObject): +class TaskDefinition(BaseObject, CloudFormationModel): def __init__( self, family, @@ -159,6 +168,15 @@ class TaskDefinition(BaseObject): def physical_resource_id(self): return self.arn + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ecs-taskdefinition.html + return "AWS::ECS::TaskDefinition" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -235,7 +253,7 @@ class Task(BaseObject): return response_object -class Service(BaseObject): +class Service(BaseObject, CloudFormationModel): def __init__( self, cluster, @@ -315,6 +333,15 @@ class Service(BaseObject): return response_object + @staticmethod + def cloudformation_name_type(): + return "ServiceName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ecs-service.html + return "AWS::ECS::Service" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/elb/models.py b/moto/elb/models.py index 4991b0754..715758090 100644 --- a/moto/elb/models.py +++ b/moto/elb/models.py @@ -13,7 +13,7 @@ from boto.ec2.elb.attributes import ( ) from boto.ec2.elb.policies import Policies, OtherPolicy from moto.compat import OrderedDict -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.ec2.models import ec2_backends from .exceptions import ( BadHealthCheckDefinition, @@ -69,7 +69,7 @@ class FakeBackend(BaseModel): ) -class FakeLoadBalancer(BaseModel): +class FakeLoadBalancer(CloudFormationModel): def __init__( self, name, @@ -119,6 +119,15 @@ class FakeLoadBalancer(BaseModel): ) self.backends.append(backend) + @staticmethod + def cloudformation_name_type(): + return "LoadBalancerName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticloadbalancing-loadbalancer.html + return "AWS::ElasticLoadBalancing::LoadBalancer" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/elbv2/models.py b/moto/elbv2/models.py index a6da0d01c..1deaac9c4 100644 --- a/moto/elbv2/models.py +++ b/moto/elbv2/models.py @@ -6,7 +6,7 @@ from jinja2 import Template from botocore.exceptions import ParamValidationError from moto.compat import OrderedDict from moto.core.exceptions import RESTError -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import camelcase_to_underscores, underscores_to_camelcase from moto.ec2.models import ec2_backends from moto.acm.models import acm_backends @@ -50,7 +50,7 @@ class FakeHealthStatus(BaseModel): self.description = description -class FakeTargetGroup(BaseModel): +class FakeTargetGroup(CloudFormationModel): HTTP_CODE_REGEX = re.compile(r"(?:(?:\d+-\d+|\d+),?)+") def __init__( @@ -143,6 +143,15 @@ class FakeTargetGroup(BaseModel): ) return FakeHealthStatus(t["id"], t["port"], self.healthcheck_port, "healthy") + @staticmethod + def cloudformation_name_type(): + return "Name" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticloadbalancingv2-targetgroup.html + return "AWS::ElasticLoadBalancingV2::TargetGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -183,7 +192,7 @@ class FakeTargetGroup(BaseModel): return target_group -class FakeListener(BaseModel): +class FakeListener(CloudFormationModel): def __init__( self, load_balancer_arn, @@ -228,6 +237,15 @@ class FakeListener(BaseModel): self._non_default_rules, key=lambda x: x.priority ) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticloadbalancingv2-listener.html + return "AWS::ElasticLoadBalancingV2::Listener" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -343,7 +361,7 @@ class FakeBackend(BaseModel): ) -class FakeLoadBalancer(BaseModel): +class FakeLoadBalancer(CloudFormationModel): VALID_ATTRS = { "access_logs.s3.enabled", "access_logs.s3.bucket", @@ -402,6 +420,15 @@ class FakeLoadBalancer(BaseModel): """ Not exposed as part of the ELB API - used for CloudFormation. """ elbv2_backends[region].delete_load_balancer(self.arn) + @staticmethod + def cloudformation_name_type(): + return "Name" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticloadbalancingv2-loadbalancer.html + return "AWS::ElasticLoadBalancingV2::LoadBalancer" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/events/models.py b/moto/events/models.py index d70898198..7fa7d225f 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -4,14 +4,14 @@ import json from boto3 import Session from moto.core.exceptions import JsonRESTError -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, CloudFormationModel from moto.sts.models import ACCOUNT_ID from moto.utilities.tagging_service import TaggingService from uuid import uuid4 -class Rule(BaseModel): +class Rule(CloudFormationModel): def _generate_arn(self, name): return "arn:aws:events:{region_name}:111111111111:rule/{name}".format( region_name=self.region_name, name=name @@ -73,6 +73,15 @@ class Rule(BaseModel): raise UnformattedGetAttTemplateException() + @staticmethod + def cloudformation_name_type(): + return "Name" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html + return "AWS::Events::Rule" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -101,7 +110,7 @@ class Rule(BaseModel): event_backend.delete_rule(name=event_name) -class EventBus(BaseModel): +class EventBus(CloudFormationModel): def __init__(self, region_name, name): self.region = region_name self.name = name @@ -152,6 +161,15 @@ class EventBus(BaseModel): raise UnformattedGetAttTemplateException() + @staticmethod + def cloudformation_name_type(): + return "Name" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-eventbus.html + return "AWS::Events::EventBus" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/iam/models.py b/moto/iam/models.py index 49755e57a..16b3ac0ab 100755 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -15,7 +15,7 @@ from six.moves.urllib.parse import urlparse from uuid import uuid4 from moto.core.exceptions import RESTError -from moto.core import BaseBackend, BaseModel, ACCOUNT_ID +from moto.core import BaseBackend, BaseModel, ACCOUNT_ID, CloudFormationModel from moto.core.utils import ( iso_8601_datetime_without_milliseconds, iso_8601_datetime_with_milliseconds, @@ -299,7 +299,7 @@ class InlinePolicy(Policy): """TODO: is this needed?""" -class Role(BaseModel): +class Role(CloudFormationModel): def __init__( self, role_id, @@ -327,6 +327,15 @@ class Role(BaseModel): def created_iso_8601(self): return iso_8601_datetime_with_milliseconds(self.create_date) + @staticmethod + def cloudformation_name_type(): + return "RoleName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html + return "AWS::IAM::Role" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -384,7 +393,7 @@ class Role(BaseModel): return [self.tags[tag] for tag in self.tags] -class InstanceProfile(BaseModel): +class InstanceProfile(CloudFormationModel): def __init__(self, instance_profile_id, name, path, roles): self.id = instance_profile_id self.name = name @@ -396,6 +405,15 @@ class InstanceProfile(BaseModel): def created_iso_8601(self): return iso_8601_datetime_with_milliseconds(self.create_date) + @staticmethod + def cloudformation_name_type(): + return "InstanceProfileName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-instanceprofile.html + return "AWS::IAM::InstanceProfile" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/kinesis/models.py b/moto/kinesis/models.py index ec9655bfa..c4b04d924 100644 --- a/moto/kinesis/models.py +++ b/moto/kinesis/models.py @@ -12,7 +12,7 @@ from hashlib import md5 from boto3 import Session from moto.compat import OrderedDict -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import unix_time from moto.core import ACCOUNT_ID from .exceptions import ( @@ -129,7 +129,7 @@ class Shard(BaseModel): } -class Stream(BaseModel): +class Stream(CloudFormationModel): def __init__(self, stream_name, shard_count, region): self.stream_name = stream_name self.shard_count = shard_count @@ -216,6 +216,15 @@ class Stream(BaseModel): } } + @staticmethod + def cloudformation_name_type(): + return "Name" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-kinesis-stream.html + return "AWS::Kinesis::Stream" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/kms/models.py b/moto/kms/models.py index 36f72e6de..2eb7cb771 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta from boto3 import Session -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, CloudFormationModel from moto.core.utils import unix_time from moto.utilities.tagging_service import TaggingService from moto.core.exceptions import JsonRESTError @@ -15,7 +15,7 @@ from moto.iam.models import ACCOUNT_ID from .utils import decrypt, encrypt, generate_key_id, generate_master_key -class Key(BaseModel): +class Key(CloudFormationModel): def __init__( self, policy, key_usage, customer_master_key_spec, description, region ): @@ -99,6 +99,15 @@ class Key(BaseModel): def delete(self, region_name): kms_backends[region_name].delete_key(self.id) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-kms-key.html + return "AWS::KMS::Key" + @classmethod def create_from_cloudformation_json( self, resource_name, cloudformation_json, region_name diff --git a/moto/rds/models.py b/moto/rds/models.py index 40b1197b6..440da34d2 100644 --- a/moto/rds/models.py +++ b/moto/rds/models.py @@ -3,14 +3,14 @@ from __future__ import unicode_literals import boto.rds from jinja2 import Template -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, CloudFormationModel from moto.core.utils import get_random_hex from moto.ec2.models import ec2_backends from moto.rds.exceptions import UnformattedGetAttTemplateException from moto.rds2.models import rds2_backends -class Database(BaseModel): +class Database(CloudFormationModel): def get_cfn_attribute(self, attribute_name): if attribute_name == "Endpoint.Address": return self.address @@ -18,13 +18,22 @@ class Database(BaseModel): return self.port raise UnformattedGetAttTemplateException() + @staticmethod + def cloudformation_name_type(): + return "DBInstanceIdentifier" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbinstance.html + return "AWS::RDS::DBInstance" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name ): properties = cloudformation_json["Properties"] - db_instance_identifier = properties.get("DBInstanceIdentifier") + db_instance_identifier = properties.get(cls.cloudformation_name_type()) if not db_instance_identifier: db_instance_identifier = resource_name.lower() + get_random_hex(12) db_security_groups = properties.get("DBSecurityGroups") @@ -163,7 +172,7 @@ class Database(BaseModel): backend.delete_database(self.db_instance_identifier) -class SecurityGroup(BaseModel): +class SecurityGroup(CloudFormationModel): def __init__(self, group_name, description): self.group_name = group_name self.description = description @@ -206,6 +215,15 @@ class SecurityGroup(BaseModel): def authorize_security_group(self, security_group): self.ec2_security_groups.append(security_group) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbsecuritygroup.html + return "AWS::RDS::DBSecurityGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -239,7 +257,7 @@ class SecurityGroup(BaseModel): backend.delete_security_group(self.group_name) -class SubnetGroup(BaseModel): +class SubnetGroup(CloudFormationModel): def __init__(self, subnet_name, description, subnets): self.subnet_name = subnet_name self.description = description @@ -271,13 +289,23 @@ class SubnetGroup(BaseModel): ) return template.render(subnet_group=self) + @staticmethod + def cloudformation_name_type(): + return "DBSubnetGroupName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbsubnetgroup.html + return "AWS::RDS::DBSubnetGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name ): properties = cloudformation_json["Properties"] - - subnet_name = resource_name.lower() + get_random_hex(12) + subnet_name = properties.get(cls.cloudformation_name_type()) + if not subnet_name: + subnet_name = resource_name.lower() + get_random_hex(12) description = properties["DBSubnetGroupDescription"] subnet_ids = properties["SubnetIds"] tags = properties.get("Tags") diff --git a/moto/rds2/models.py b/moto/rds2/models.py index 7fa4f3316..5f46311ec 100644 --- a/moto/rds2/models.py +++ b/moto/rds2/models.py @@ -9,7 +9,7 @@ from boto3 import Session from jinja2 import Template from re import compile as re_compile from moto.compat import OrderedDict -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import get_random_hex from moto.core.utils import iso_8601_datetime_with_milliseconds from moto.ec2.models import ec2_backends @@ -28,7 +28,7 @@ from .exceptions import ( ) -class Database(BaseModel): +class Database(CloudFormationModel): def __init__(self, **kwargs): self.status = "available" self.is_replica = False @@ -356,13 +356,22 @@ class Database(BaseModel): "sqlserver-web": {"gp2": 20, "io1": 100, "standard": 20}, }[engine][storage_type] + @staticmethod + def cloudformation_name_type(): + return "DBInstanceIdentifier" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbinstance.html + return "AWS::RDS::DBInstance" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name ): properties = cloudformation_json["Properties"] - db_instance_identifier = properties.get("DBInstanceIdentifier") + db_instance_identifier = properties.get(cls.cloudformation_name_type()) if not db_instance_identifier: db_instance_identifier = resource_name.lower() + get_random_hex(12) db_security_groups = properties.get("DBSecurityGroups") @@ -564,7 +573,7 @@ class Snapshot(BaseModel): self.tags = [tag_set for tag_set in self.tags if tag_set["Key"] not in tag_keys] -class SecurityGroup(BaseModel): +class SecurityGroup(CloudFormationModel): def __init__(self, group_name, description, tags): self.group_name = group_name self.description = description @@ -627,6 +636,15 @@ class SecurityGroup(BaseModel): def authorize_security_group(self, security_group): self.ec2_security_groups.append(security_group) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbsecuritygroup.html + return "AWS::RDS::DBSecurityGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -671,7 +689,7 @@ class SecurityGroup(BaseModel): backend.delete_security_group(self.group_name) -class SubnetGroup(BaseModel): +class SubnetGroup(CloudFormationModel): def __init__(self, subnet_name, description, subnets, tags): self.subnet_name = subnet_name self.description = description @@ -726,13 +744,24 @@ class SubnetGroup(BaseModel): ) return template.render(subnet_group=self) + @staticmethod + def cloudformation_name_type(): + return "DBSubnetGroupName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbsubnetgroup.html + return "AWS::RDS::DBSubnetGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name ): properties = cloudformation_json["Properties"] - subnet_name = resource_name.lower() + get_random_hex(12) + subnet_name = properties.get(cls.cloudformation_name_type()) + if not subnet_name: + subnet_name = resource_name.lower() + get_random_hex(12) description = properties["DBSubnetGroupDescription"] subnet_ids = properties["SubnetIds"] tags = properties.get("Tags") @@ -1441,7 +1470,7 @@ class OptionGroupOptionSetting(object): return template.render(option_group_option_setting=self) -class DBParameterGroup(object): +class DBParameterGroup(CloudFormationModel): def __init__(self, name, description, family, tags): self.name = name self.description = description @@ -1480,6 +1509,15 @@ class DBParameterGroup(object): backend = rds2_backends[region_name] backend.delete_db_parameter_group(self.name) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbparametergroup.html + return "AWS::RDS::DBParameterGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 07baf18c0..0bdb14edc 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -6,7 +6,7 @@ import datetime from boto3 import Session from botocore.exceptions import ClientError from moto.compat import OrderedDict -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import iso_8601_datetime_with_milliseconds from moto.ec2 import ec2_backends from .exceptions import ( @@ -63,7 +63,7 @@ class TaggableResourceMixin(object): return self.tags -class Cluster(TaggableResourceMixin, BaseModel): +class Cluster(TaggableResourceMixin, CloudFormationModel): resource_type = "cluster" @@ -157,6 +157,15 @@ class Cluster(TaggableResourceMixin, BaseModel): self.iam_roles_arn = iam_roles_arn or [] self.restored_from_snapshot = restored_from_snapshot + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-redshift-cluster.html + return "AWS::Redshift::Cluster" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -170,6 +179,7 @@ class Cluster(TaggableResourceMixin, BaseModel): ].cluster_subnet_group_name else: subnet_group_name = None + cluster = redshift_backend.create_cluster( cluster_identifier=resource_name, node_type=properties.get("NodeType"), @@ -321,7 +331,7 @@ class SnapshotCopyGrant(TaggableResourceMixin, BaseModel): } -class SubnetGroup(TaggableResourceMixin, BaseModel): +class SubnetGroup(TaggableResourceMixin, CloudFormationModel): resource_type = "subnetgroup" @@ -342,6 +352,15 @@ class SubnetGroup(TaggableResourceMixin, BaseModel): if not self.subnets: raise InvalidSubnetError(subnet_ids) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-redshift-clustersubnetgroup.html + return "AWS::Redshift::ClusterSubnetGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -412,7 +431,7 @@ class SecurityGroup(TaggableResourceMixin, BaseModel): } -class ParameterGroup(TaggableResourceMixin, BaseModel): +class ParameterGroup(TaggableResourceMixin, CloudFormationModel): resource_type = "parametergroup" @@ -429,6 +448,15 @@ class ParameterGroup(TaggableResourceMixin, BaseModel): self.group_family = group_family self.description = description + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-redshift-clusterparametergroup.html + return "AWS::Redshift::ClusterParameterGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/route53/models.py b/moto/route53/models.py index 0bdefd25b..52f60d971 100644 --- a/moto/route53/models.py +++ b/moto/route53/models.py @@ -7,7 +7,7 @@ import random import uuid from jinja2 import Template -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, CloudFormationModel ROUTE53_ID_CHOICE = string.ascii_uppercase + string.digits @@ -18,7 +18,7 @@ def create_route53_zone_id(): return "".join([random.choice(ROUTE53_ID_CHOICE) for _ in range(0, 15)]) -class HealthCheck(BaseModel): +class HealthCheck(CloudFormationModel): def __init__(self, health_check_id, health_check_args): self.id = health_check_id self.ip_address = health_check_args.get("ip_address") @@ -34,6 +34,15 @@ class HealthCheck(BaseModel): def physical_resource_id(self): return self.id + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-route53-healthcheck.html + return "AWS::Route53::HealthCheck" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -75,7 +84,7 @@ class HealthCheck(BaseModel): return template.render(health_check=self) -class RecordSet(BaseModel): +class RecordSet(CloudFormationModel): def __init__(self, kwargs): self.name = kwargs.get("Name") self.type_ = kwargs.get("Type") @@ -91,6 +100,15 @@ class RecordSet(BaseModel): self.failover = kwargs.get("Failover") self.geo_location = kwargs.get("GeoLocation") + @staticmethod + def cloudformation_name_type(): + return "Name" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-route53-recordset.html + return "AWS::Route53::RecordSet" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -202,7 +220,7 @@ def reverse_domain_name(domain_name): return ".".join(reversed(domain_name.split("."))) -class FakeZone(BaseModel): +class FakeZone(CloudFormationModel): def __init__(self, name, id_, private_zone, comment=None): self.name = name self.id = id_ @@ -267,6 +285,15 @@ class FakeZone(BaseModel): def physical_resource_id(self): return self.id + @staticmethod + def cloudformation_name_type(): + return "Name" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-route53-hostedzone.html + return "AWS::Route53::HostedZone" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -278,7 +305,7 @@ class FakeZone(BaseModel): return hosted_zone -class RecordSetGroup(BaseModel): +class RecordSetGroup(CloudFormationModel): def __init__(self, hosted_zone_id, record_sets): self.hosted_zone_id = hosted_zone_id self.record_sets = record_sets @@ -287,6 +314,15 @@ class RecordSetGroup(BaseModel): def physical_resource_id(self): return "arn:aws:route53:::hostedzone/{0}".format(self.hosted_zone_id) + @staticmethod + def cloudformation_name_type(): + return None + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-route53-recordsetgroup.html + return "AWS::Route53::RecordSetGroup" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/s3/models.py b/moto/s3/models.py index e5237168e..800601690 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -21,7 +21,7 @@ import uuid import six from bisect import insort -from moto.core import ACCOUNT_ID, BaseBackend, BaseModel +from moto.core import ACCOUNT_ID, BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import iso_8601_datetime_without_milliseconds_s3, rfc_1123_datetime from moto.cloudwatch.models import MetricDatum from moto.utilities.tagging_service import TaggingService @@ -763,7 +763,7 @@ class PublicAccessBlock(BaseModel): } -class FakeBucket(BaseModel): +class FakeBucket(CloudFormationModel): def __init__(self, name, region_name): self.name = name self.region_name = region_name @@ -1070,6 +1070,15 @@ class FakeBucket(BaseModel): def physical_resource_id(self): return self.name + @staticmethod + def cloudformation_name_type(): + return "BucketName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-s3-bucket.html + return "AWS::S3::Bucket" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/sns/models.py b/moto/sns/models.py index 76376e58f..8a4771a37 100644 --- a/moto/sns/models.py +++ b/moto/sns/models.py @@ -11,7 +11,7 @@ import re from boto3 import Session from moto.compat import OrderedDict -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import ( iso_8601_datetime_with_milliseconds, camelcase_to_underscores, @@ -37,7 +37,7 @@ DEFAULT_PAGE_SIZE = 100 MAXIMUM_MESSAGE_LENGTH = 262144 # 256 KiB -class Topic(BaseModel): +class Topic(CloudFormationModel): def __init__(self, name, sns_backend): self.name = name self.sns_backend = sns_backend @@ -87,6 +87,15 @@ class Topic(BaseModel): def policy(self, policy): self._policy_json = json.loads(policy) + @staticmethod + def cloudformation_name_type(): + return "TopicName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sns-topic.html + return "AWS::SNS::Topic" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -94,7 +103,7 @@ class Topic(BaseModel): sns_backend = sns_backends[region_name] properties = cloudformation_json["Properties"] - topic = sns_backend.create_topic(properties.get("TopicName")) + topic = sns_backend.create_topic(properties.get(cls.cloudformation_name_type())) for subscription in properties.get("Subscription", []): sns_backend.subscribe( topic.arn, subscription["Endpoint"], subscription["Protocol"] diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 4befbb50a..a3642c17e 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -12,7 +12,7 @@ from xml.sax.saxutils import escape from boto3 import Session from moto.core.exceptions import RESTError -from moto.core import BaseBackend, BaseModel +from moto.core import BaseBackend, BaseModel, CloudFormationModel from moto.core.utils import ( camelcase_to_underscores, get_random_message_id, @@ -188,7 +188,7 @@ class Message(BaseModel): return False -class Queue(BaseModel): +class Queue(CloudFormationModel): BASE_ATTRIBUTES = [ "ApproximateNumberOfMessages", "ApproximateNumberOfMessagesDelayed", @@ -354,6 +354,15 @@ class Queue(BaseModel): ), ) + @staticmethod + def cloudformation_name_type(): + return "QueueName" + + @staticmethod + def cloudformation_type(): + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sqs-queue.html + return "AWS::SQS::Queue" + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name From 06ed67a8e5c6ffddb7b588798141d9c5a70f183f Mon Sep 17 00:00:00 2001 From: Larry Aiello Date: Sat, 1 Aug 2020 12:03:54 -0400 Subject: [PATCH 5/5] Implement UserIds for Snapshot Attributes (#3192) * implement register_image * format code * add user_ids to snapshot model * implement register_image * format code * add user_ids to snapshot model * trying to un-deprecate tests * Write tests and finalize implementation * Add region parameter to boto3 resource call * fixed test error --- moto/ec2/models.py | 41 +++-- moto/ec2/responses/elastic_block_store.py | 35 ++-- tests/test_ec2/test_amis.py | 10 ++ tests/test_ec2/test_elastic_block_store.py | 183 +++++++++++++++++++-- 4 files changed, 221 insertions(+), 48 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 3d60654a9..e6c57dcdd 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -2488,6 +2488,7 @@ class Snapshot(TaggedEC2Resource): self.description = description self.start_time = utc_date_and_time() self.create_volume_permission_groups = set() + self.create_volume_permission_userids = set() self.ec2_backend = ec2_backend self.status = "completed" self.encrypted = encrypted @@ -2652,28 +2653,32 @@ class EBSBackend(object): snapshot = self.get_snapshot(snapshot_id) return snapshot.create_volume_permission_groups - def add_create_volume_permission(self, snapshot_id, user_id=None, group=None): - if user_id: - self.raise_not_implemented_error( - "The UserId parameter for ModifySnapshotAttribute" - ) - - if group != "all": - raise InvalidAMIAttributeItemValueError("UserGroup", group) + def get_create_volume_permission_userids(self, snapshot_id): snapshot = self.get_snapshot(snapshot_id) - snapshot.create_volume_permission_groups.add(group) + return snapshot.create_volume_permission_userids + + def add_create_volume_permission(self, snapshot_id, user_ids=None, groups=None): + snapshot = self.get_snapshot(snapshot_id) + if user_ids: + snapshot.create_volume_permission_userids.update(user_ids) + + if groups and groups != ["all"]: + raise InvalidAMIAttributeItemValueError("UserGroup", groups) + else: + snapshot.create_volume_permission_groups.update(groups) + return True - def remove_create_volume_permission(self, snapshot_id, user_id=None, group=None): - if user_id: - self.raise_not_implemented_error( - "The UserId parameter for ModifySnapshotAttribute" - ) - - if group != "all": - raise InvalidAMIAttributeItemValueError("UserGroup", group) + def remove_create_volume_permission(self, snapshot_id, user_ids=None, groups=None): snapshot = self.get_snapshot(snapshot_id) - snapshot.create_volume_permission_groups.discard(group) + if user_ids: + snapshot.create_volume_permission_userids.difference_update(user_ids) + + if groups and groups != ["all"]: + raise InvalidAMIAttributeItemValueError("UserGroup", groups) + else: + snapshot.create_volume_permission_groups.difference_update(groups) + return True diff --git a/moto/ec2/responses/elastic_block_store.py b/moto/ec2/responses/elastic_block_store.py index d11470242..f7f4df9dc 100644 --- a/moto/ec2/responses/elastic_block_store.py +++ b/moto/ec2/responses/elastic_block_store.py @@ -116,22 +116,23 @@ class ElasticBlockStore(BaseResponse): def describe_snapshot_attribute(self): snapshot_id = self._get_param("SnapshotId") groups = self.ec2_backend.get_create_volume_permission_groups(snapshot_id) + user_ids = self.ec2_backend.get_create_volume_permission_userids(snapshot_id) template = self.response_template(DESCRIBE_SNAPSHOT_ATTRIBUTES_RESPONSE) - return template.render(snapshot_id=snapshot_id, groups=groups) + return template.render(snapshot_id=snapshot_id, groups=groups, userIds=user_ids) def modify_snapshot_attribute(self): snapshot_id = self._get_param("SnapshotId") operation_type = self._get_param("OperationType") - group = self._get_param("UserGroup.1") - user_id = self._get_param("UserId.1") + groups = self._get_multi_param("UserGroup") + user_ids = self._get_multi_param("UserId") if self.is_not_dryrun("ModifySnapshotAttribute"): if operation_type == "add": self.ec2_backend.add_create_volume_permission( - snapshot_id, user_id=user_id, group=group + snapshot_id, user_ids=user_ids, groups=groups ) elif operation_type == "remove": self.ec2_backend.remove_create_volume_permission( - snapshot_id, user_id=user_id, group=group + snapshot_id, user_ids=user_ids, groups=groups ) return MODIFY_SNAPSHOT_ATTRIBUTE_RESPONSE @@ -311,18 +312,18 @@ DESCRIBE_SNAPSHOT_ATTRIBUTES_RESPONSE = """ a9540c9f-161a-45d8-9cc1-1182b89ad69f snap-a0332ee0 - {% if not groups %} - - {% endif %} - {% if groups %} - - {% for group in groups %} - - {{ group }} - - {% endfor %} - - {% endif %} + + {% for group in groups %} + + {{ group }} + + {% endfor %} + {% for userId in userIds %} + + {{ userId }} + + {% endfor %} + """ diff --git a/tests/test_ec2/test_amis.py b/tests/test_ec2/test_amis.py index 220dd143c..e32ef9780 100644 --- a/tests/test_ec2/test_amis.py +++ b/tests/test_ec2/test_amis.py @@ -783,6 +783,16 @@ def test_ami_registration(): assert images[0]["State"] == "available", "State should be available." +@mock_ec2 +def test_ami_registration(): + ec2 = boto3.client("ec2", region_name="us-east-1") + image_id = ec2.register_image(Name="test-register-image").get("ImageId", "") + images = ec2.describe_images(ImageIds=[image_id]).get("Images", []) + assert images[0]["Name"] == "test-register-image", "No image was registered." + assert images[0]["RootDeviceName"] == "/dev/sda1", "Wrong root device name." + assert images[0]["State"] == "available", "State should be available." + + @mock_ec2 def test_ami_filter_wildcard(): ec2_resource = boto3.resource("ec2", region_name="us-west-1") diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 4bd2a8dfa..7f8313da4 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -562,19 +562,176 @@ def test_snapshot_attribute(): cm.exception.status.should.equal(400) cm.exception.request_id.should_not.be.none - # Error: Add or remove with user ID instead of group - conn.modify_snapshot_attribute.when.called_with( - snapshot.id, - attribute="createVolumePermission", - operation="add", - user_ids=["user"], - ).should.throw(NotImplementedError) - conn.modify_snapshot_attribute.when.called_with( - snapshot.id, - attribute="createVolumePermission", - operation="remove", - user_ids=["user"], - ).should.throw(NotImplementedError) + +@mock_ec2 +def test_modify_snapshot_attribute(): + import copy + + ec2_client = boto3.client("ec2", region_name="us-east-1") + response = ec2_client.create_volume(Size=80, AvailabilityZone="us-east-1a") + volume = boto3.resource("ec2", region_name="us-east-1").Volume(response["VolumeId"]) + snapshot = volume.create_snapshot() + + # Baseline + attributes = ec2_client.describe_snapshot_attribute( + SnapshotId=snapshot.id, Attribute="createVolumePermission" + ) + assert not attributes[ + "CreateVolumePermissions" + ], "Snapshot should have no permissions." + + ADD_GROUP_ARGS = { + "SnapshotId": snapshot.id, + "Attribute": "createVolumePermission", + "OperationType": "add", + "GroupNames": ["all"], + } + + REMOVE_GROUP_ARGS = { + "SnapshotId": snapshot.id, + "Attribute": "createVolumePermission", + "OperationType": "remove", + "GroupNames": ["all"], + } + + # Add 'all' group and confirm + with assert_raises(ClientError) as cm: + ec2_client.modify_snapshot_attribute(**dict(ADD_GROUP_ARGS, **{"DryRun": True})) + + cm.exception.response["Error"]["Code"].should.equal("DryRunOperation") + cm.exception.response["ResponseMetadata"]["RequestId"].should_not.be.none + cm.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + ec2_client.modify_snapshot_attribute(**ADD_GROUP_ARGS) + + attributes = ec2_client.describe_snapshot_attribute( + SnapshotId=snapshot.id, Attribute="createVolumePermission" + ) + assert attributes["CreateVolumePermissions"] == [ + {"Group": "all"} + ], "This snapshot should have public group permissions." + + # Add is idempotent + ec2_client.modify_snapshot_attribute.when.called_with( + **ADD_GROUP_ARGS + ).should_not.throw(ClientError) + assert attributes["CreateVolumePermissions"] == [ + {"Group": "all"} + ], "This snapshot should have public group permissions." + + # Remove 'all' group and confirm + with assert_raises(ClientError) as ex: + ec2_client.modify_snapshot_attribute( + **dict(REMOVE_GROUP_ARGS, **{"DryRun": True}) + ) + cm.exception.response["Error"]["Code"].should.equal("DryRunOperation") + cm.exception.response["ResponseMetadata"]["RequestId"].should_not.be.none + cm.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + ec2_client.modify_snapshot_attribute(**REMOVE_GROUP_ARGS) + + attributes = ec2_client.describe_snapshot_attribute( + SnapshotId=snapshot.id, Attribute="createVolumePermission" + ) + assert not attributes[ + "CreateVolumePermissions" + ], "This snapshot should have no permissions." + + # Remove is idempotent + ec2_client.modify_snapshot_attribute.when.called_with( + **REMOVE_GROUP_ARGS + ).should_not.throw(ClientError) + assert not attributes[ + "CreateVolumePermissions" + ], "This snapshot should have no permissions." + + # Error: Add with group != 'all' + with assert_raises(ClientError) as cm: + ec2_client.modify_snapshot_attribute( + SnapshotId=snapshot.id, + Attribute="createVolumePermission", + OperationType="add", + GroupNames=["everyone"], + ) + cm.exception.response["Error"]["Code"].should.equal("InvalidAMIAttributeItemValue") + cm.exception.response["ResponseMetadata"]["RequestId"].should_not.be.none + cm.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + # Error: Add with invalid snapshot ID + with assert_raises(ClientError) as cm: + ec2_client.modify_snapshot_attribute( + SnapshotId="snapshot-abcd1234", + Attribute="createVolumePermission", + OperationType="add", + GroupNames=["all"], + ) + cm.exception.response["Error"]["Code"].should.equal("InvalidSnapshot.NotFound") + cm.exception.response["ResponseMetadata"]["RequestId"].should_not.be.none + cm.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + # Error: Remove with invalid snapshot ID + with assert_raises(ClientError) as cm: + ec2_client.modify_snapshot_attribute( + SnapshotId="snapshot-abcd1234", + Attribute="createVolumePermission", + OperationType="remove", + GroupNames=["all"], + ) + cm.exception.response["Error"]["Code"].should.equal("InvalidSnapshot.NotFound") + cm.exception.response["ResponseMetadata"]["RequestId"].should_not.be.none + cm.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + + # Test adding user id + ec2_client.modify_snapshot_attribute( + SnapshotId=snapshot.id, + Attribute="createVolumePermission", + OperationType="add", + UserIds=["1234567891"], + ) + + attributes = ec2_client.describe_snapshot_attribute( + SnapshotId=snapshot.id, Attribute="createVolumePermission" + ) + assert len(attributes["CreateVolumePermissions"]) == 1 + + # Test adding user id again along with additional. + ec2_client.modify_snapshot_attribute( + SnapshotId=snapshot.id, + Attribute="createVolumePermission", + OperationType="add", + UserIds=["1234567891", "2345678912"], + ) + + attributes = ec2_client.describe_snapshot_attribute( + SnapshotId=snapshot.id, Attribute="createVolumePermission" + ) + assert len(attributes["CreateVolumePermissions"]) == 2 + + # Test removing both user IDs. + ec2_client.modify_snapshot_attribute( + SnapshotId=snapshot.id, + Attribute="createVolumePermission", + OperationType="remove", + UserIds=["1234567891", "2345678912"], + ) + + attributes = ec2_client.describe_snapshot_attribute( + SnapshotId=snapshot.id, Attribute="createVolumePermission" + ) + assert len(attributes["CreateVolumePermissions"]) == 0 + + # Idempotency when removing users. + ec2_client.modify_snapshot_attribute( + SnapshotId=snapshot.id, + Attribute="createVolumePermission", + OperationType="remove", + UserIds=["1234567891"], + ) + + attributes = ec2_client.describe_snapshot_attribute( + SnapshotId=snapshot.id, Attribute="createVolumePermission" + ) + assert len(attributes["CreateVolumePermissions"]) == 0 @mock_ec2_deprecated