From 7bc5b5c08f8f5f7f92312983c2dbccd43c59f41f Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Thu, 30 Jul 2020 08:17:35 -0600 Subject: [PATCH 01/11] Add IAM Role and Policy to Config --- CONFIG_README.md | 4 +- moto/config/models.py | 4 +- moto/core/models.py | 21 ++ moto/iam/config.py | 173 ++++++++++++++ moto/iam/models.py | 148 +++++++++++- tests/test_iam/test_iam.py | 448 +++++++++++++++++++++++++++++++++++++ 6 files changed, 793 insertions(+), 5 deletions(-) create mode 100644 moto/iam/config.py diff --git a/CONFIG_README.md b/CONFIG_README.md index 356bb87a0..e223c8457 100644 --- a/CONFIG_README.md +++ b/CONFIG_README.md @@ -23,8 +23,8 @@ However, this will only work on resource types that have this enabled. ### Current enabled resource types: -1. S3 - +1. S3 (all) +1. IAM (Role, Policy) ## Developer Guide diff --git a/moto/config/models.py b/moto/config/models.py index b6dc4672d..77f46e644 100644 --- a/moto/config/models.py +++ b/moto/config/models.py @@ -47,8 +47,8 @@ from moto.config.exceptions import ( from moto.core import BaseBackend, BaseModel from moto.s3.config import s3_account_public_access_block_query, s3_config_query - from moto.core import ACCOUNT_ID as DEFAULT_ACCOUNT_ID +from moto.iam.config import role_config_query, policy_config_query POP_STRINGS = [ "capitalizeStart", @@ -64,6 +64,8 @@ DEFAULT_PAGE_SIZE = 100 RESOURCE_MAP = { "AWS::S3::Bucket": s3_config_query, "AWS::S3::AccountPublicAccessBlock": s3_account_public_access_block_query, + "AWS::IAM::Role": role_config_query, + "AWS::IAM::Policy": policy_config_query, } diff --git a/moto/core/models.py b/moto/core/models.py index ae241322c..bc7d282fd 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -766,6 +766,27 @@ class ConfigQueryModel(object): """ raise NotImplementedError() + def aggregate_regions(self, path, backend_region, resource_region): + """ + Returns a list of "region\1eresourcename" strings + """ + + filter_region = backend_region or resource_region + if filter_region: + filter_resources = list(self.backends[filter_region].__dict__[path].keys()) + return map( + lambda resource: "{}\1e{}".format(filter_region, resource), + filter_resources, + ) + + # If we don't have a filter region + ret = [] + for region in self.backends: + this_region_resources = list(self.backends[region].__dict__[path].keys()) + for resource in this_region_resources: + ret.append("{}\1e{}".format(region, resource)) + return ret + class base_decorator(object): mock_backend = MockAWS diff --git a/moto/iam/config.py b/moto/iam/config.py new file mode 100644 index 000000000..4bb381248 --- /dev/null +++ b/moto/iam/config.py @@ -0,0 +1,173 @@ +import json + +from moto.core.exceptions import InvalidNextTokenException +from moto.core.models import ConfigQueryModel +from moto.iam import iam_backends + + +class RoleConfigQuery(ConfigQueryModel): + def list_config_service_resources( + self, + resource_ids, + resource_name, + limit, + next_token, + backend_region=None, + resource_region=None, + ): + # For aggregation -- did we get both a resource ID and a resource name? + if resource_ids and resource_name: + # If the values are different, then return an empty list: + if resource_name not in resource_ids: + return [], None + + role_list = self.aggregate_regions("roles", backend_region, resource_region) + + if not role_list: + return [], None + + # Pagination logic: + sorted_roles = sorted(role_list) + new_token = None + + # Get the start: + if not next_token: + start = 0 + else: + # "Tokens" are region + \00 + resource ID. + if next_token not in sorted_roles: + raise InvalidNextTokenException() + + start = sorted_roles.index(next_token) + + # Get the list of items to collect: + role_list = sorted_roles[start : (start + limit)] + + if len(sorted_roles) > (start + limit): + new_token = sorted_roles[start + limit] + + return ( + [ + { + "type": "AWS::IAM::Role", + "id": role.split("\1e")[1], + "name": role.split("\1e")[1], + "region": role.split("\1e")[0], + } + for role in role_list + ], + new_token, + ) + + def get_config_resource( + self, resource_id, resource_name=None, backend_region=None, resource_region=None + ): + + role = self.backends["global"].roles.get(resource_id, {}) + + if not role: + return + + if resource_name and role.name != resource_name: + return + + # Format the bucket to the AWS Config format: + config_data = role.to_config_dict() + + # The 'configuration' field is also a JSON string: + config_data["configuration"] = json.dumps(config_data["configuration"]) + + # Supplementary config need all values converted to JSON strings if they are not strings already: + for field, value in config_data["supplementaryConfiguration"].items(): + if not isinstance(value, str): + config_data["supplementaryConfiguration"][field] = json.dumps(value) + + return config_data + + +class PolicyConfigQuery(ConfigQueryModel): + def list_config_service_resources( + self, + resource_ids, + resource_name, + limit, + next_token, + backend_region=None, + resource_region=None, + ): + # For aggregation -- did we get both a resource ID and a resource name? + if resource_ids and resource_name: + # If the values are different, then return an empty list: + if resource_name not in resource_ids: + return [], None + + # We don't want to include AWS Managed Policies + policy_list = filter( + lambda policy: not policy.split("\1e")[1].startswith("arn:aws:iam::aws"), + self.aggregate_regions("managed_policies", backend_region, resource_region), + ) + + if not policy_list: + return [], None + + # Pagination logic: + sorted_policies = sorted(policy_list) + new_token = None + + # Get the start: + if not next_token: + start = 0 + else: + # "Tokens" are region + \00 + resource ID. + if next_token not in sorted_policies: + raise InvalidNextTokenException() + + start = sorted_policies.index(next_token) + + # Get the list of items to collect: + policy_list = sorted_policies[start : (start + limit)] + + if len(sorted_policies) > (start + limit): + new_token = sorted_policies[start + limit] + + return ( + [ + { + "type": "AWS::IAM::Policy", + "id": policy.split("\1e")[1], + "name": policy.split("\1e")[1], + "region": policy.split("\1e")[0], + } + for policy in policy_list + ], + new_token, + ) + + def get_config_resource( + self, resource_id, resource_name=None, backend_region=None, resource_region=None + ): + + policy = self.backends["global"].managed_policies.get(resource_id, {}) + + if not policy: + return + + if resource_name and policy.name != resource_name: + return + + # Format the bucket to the AWS Config format: + config_data = policy.to_config_dict() + + # The 'configuration' field is also a JSON string: + config_data["configuration"] = json.dumps(config_data["configuration"]) + + # Supplementary config need all values converted to JSON strings if they are not strings already: + for field, value in config_data["supplementaryConfiguration"].items(): + if not isinstance(value, str): + config_data["supplementaryConfiguration"][field] = json.dumps(value) + + return config_data + + +role_config_query = RoleConfigQuery(iam_backends) +policy_config_query = PolicyConfigQuery(iam_backends) diff --git a/moto/iam/models.py b/moto/iam/models.py index 3a174e17b..9ae1ddcdf 100755 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -8,11 +8,13 @@ import sys from datetime import datetime import json import re +import time from cryptography import x509 from cryptography.hazmat.backends import default_backend from six.moves.urllib.parse import urlparse +from six.moves.urllib import parse from moto.core.exceptions import RESTError from moto.core import BaseBackend, BaseModel, ACCOUNT_ID, CloudFormationModel from moto.core.utils import ( @@ -153,7 +155,7 @@ class OpenIDConnectProvider(BaseModel): self._errors = [] self._validate(url, thumbprint_list, client_id_list) - parsed_url = urlparse(url) + parsed_url = parse.urlparse(url) self.url = parsed_url.netloc + parsed_url.path self.thumbprint_list = thumbprint_list self.client_id_list = client_id_list @@ -201,7 +203,7 @@ class OpenIDConnectProvider(BaseModel): self._raise_errors() - parsed_url = urlparse(url) + parsed_url = parse.urlparse(url) if not parsed_url.scheme or not parsed_url.netloc: raise ValidationError("Invalid Open ID Connect Provider URL") @@ -265,6 +267,48 @@ class ManagedPolicy(Policy): def arn(self): return "arn:aws:iam::{0}:policy{1}{2}".format(ACCOUNT_ID, self.path, self.name) + def to_config_dict(self): + return { + "version": "1.3", + "configurationItemCaptureTime": str(self.create_date), + "configurationItemStatus": "OK", + "configurationStateId": str( + int(time.mktime(self.create_date.timetuple())) + ), # PY2 and 3 compatible + "arn": "arn:aws:iam::{}:policy/{}".format(ACCOUNT_ID, self.name), + "resourceType": "AWS::IAM::Policy", + "resourceId": self.id, + "resourceName": self.name, + "awsRegion": "global", + "availabilityZone": "Not Applicable", + "resourceCreationTime": str(self.create_date), + "configuration": { + "policyName": self.name, + "policyId": self.id, + "arn": "arn:aws:iam::{}:policy/{}".format(ACCOUNT_ID, self.name), + "path": self.path, + "defaultVersionId": self.default_version_id, + "attachmentCount": self.attachment_count, + "permissionsBoundaryUsageCount": 0, + "isAttachable": ManagedPolicy.is_attachable, + "description": self.description, + "createDate": str(self.create_date.isoformat()), + "updateDate": str(self.create_date.isoformat()), + "policyVersionList": list( + map( + lambda version: { + "document": parse.quote(version.document), + "versionId": version.version_id, + "isDefaultVersion": version.is_default, + "createDate": str(version.create_date), + }, + self.versions, + ) + ), + }, + "supplementaryConfiguration": {}, + } + class AWSManagedPolicy(ManagedPolicy): """AWS-managed policy.""" @@ -513,6 +557,69 @@ class Role(CloudFormationModel): def arn(self): return "arn:aws:iam::{0}:role{1}{2}".format(ACCOUNT_ID, self.path, self.name) + def to_config_dict(self): + _managed_policies = [] + for key in self.managed_policies.keys(): + _managed_policies.append( + {"policyArn": key, "policyName": iam_backend.managed_policies[key].name} + ) + + _role_policy_list = [] + for key, value in self.policies.items(): + _role_policy_list.append( + {"policyName": key, "policyDocument": parse.quote(value)} + ) + + _instance_profiles = [] + for key, instance_profile in iam_backend.instance_profiles.items(): + for role in instance_profile.roles: + _instance_profiles.append(instance_profile.to_embedded_config_dict()) + break + + config_dict = { + "version": "1.3", + "configurationItemCaptureTime": str(self.create_date), + "configurationItemStatus": "ResourceDiscovered", + "configurationStateId": str( + int(time.mktime(self.create_date.timetuple())) + ), # PY2 and 3 compatible + "arn": "arn:aws:iam::{}:role/{}".format(ACCOUNT_ID, self.name), + "resourceType": "AWS::IAM::Role", + "resourceId": self.name, + "resourceName": self.name, + "awsRegion": "global", + "availabilityZone": "Not Applicable", + "resourceCreationTime": str(self.create_date), + "relatedEvents": [], + "relationships": [], + "tags": self.tags, + "configuration": { + "path": self.path, + "roleName": self.name, + "roleId": self.id, + "arn": "arn:aws:iam::{}:role/{}".format(ACCOUNT_ID, self.name), + "assumeRolePolicyDocument": parse.quote( + self.assume_role_policy_document + ) + if self.assume_role_policy_document + else None, + "instanceProfileList": _instance_profiles, + "rolePolicyList": _role_policy_list, + "createDate": self.create_date.isoformat(), + "attachedManagedPolicies": _managed_policies, + "permissionsBoundary": self.permissions_boundary, + "tags": list( + map( + lambda key: {"key": key, "value": self.tags[key]["Value"]}, + self.tags, + ) + ), + "roleLastUsed": None, + }, + "supplementaryConfiguration": {}, + } + return config_dict + def put_policy(self, policy_name, policy_json): self.policies[policy_name] = policy_json @@ -590,6 +697,43 @@ class InstanceProfile(CloudFormationModel): return self.arn raise UnformattedGetAttTemplateException() + def to_embedded_config_dict(self): + # Instance Profiles aren't a config item itself, but they are returned in IAM roles with + # a "config like" json structure It's also different than Role.to_config_dict() + roles = [] + for role in self.roles: + roles.append( + { + "path": role.path, + "roleName": role.name, + "roleId": role.id, + "arn": "arn:aws:iam::{}:role/{}".format(ACCOUNT_ID, role.name), + "createDate": str(role.create_date), + "assumeRolePolicyDocument": parse.quote( + role.assume_role_policy_document + ), + "description": role.description, + "maxSessionDuration": None, + "permissionsBoundary": role.permissions_boundary, + "tags": list( + map( + lambda key: {"key": key, "value": role.tags[key]["Value"]}, + role.tags, + ) + ), + "roleLastUsed": None, + } + ) + + return { + "path": self.path, + "instanceProfileName": self.name, + "instanceProfileId": self.id, + "arn": "arn:aws:iam::{}:instance-profile/{}".format(ACCOUNT_ID, self.name), + "createDate": str(self.create_date), + "roles": roles, + } + class Certificate(BaseModel): def __init__(self, cert_name, cert_body, private_key, cert_chain=None, path=None): diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 288825d6e..b42f3d76f 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -19,6 +19,7 @@ from nose.tools import raises from datetime import datetime from tests.helpers import requires_boto_gte from uuid import uuid4 +from six.moves.urllib import parse MOCK_CERT = """-----BEGIN CERTIFICATE----- @@ -2882,3 +2883,450 @@ def test_delete_role_with_instance_profiles_present(): role_names = [role["RoleName"] for role in iam.list_roles()["Roles"]] assert "Role1" in role_names assert "Role2" not in role_names + + +@mock_iam +def test_delete_account_password_policy_errors(): + client = boto3.client("iam", region_name="us-east-1") + + client.delete_account_password_policy.when.called_with().should.throw( + ClientError, "The account policy with name PasswordPolicy cannot be found." + ) + + +@mock_iam +def test_role_list_config_discovered_resources(): + from moto.iam.config import role_config_query + from moto.iam.utils import random_resource_id + + # Without any roles + assert role_config_query.list_config_service_resources(None, None, 100, None) == ( + [], + None, + ) + + # Create a role + role_config_query.backends["global"].create_role( + role_name="something", + assume_role_policy_document=None, + path="/", + permissions_boundary=None, + description="something", + tags=[], + max_session_duration=3600, + ) + + result = role_config_query.list_config_service_resources(None, None, 100, None)[0] + assert len(result) == 1 + + # The role gets a random ID, so we have to grab it + role = result[0] + assert role["type"] == "AWS::IAM::Role" + assert len(role["id"]) == len(random_resource_id()) + assert role["id"] == role["name"] + assert role["region"] == "global" + + +@mock_iam +def test_policy_list_config_discovered_resources(): + from moto.iam.config import policy_config_query + + # Without any policies + assert policy_config_query.list_config_service_resources(None, None, 100, None) == ( + [], + None, + ) + + basic_policy = { + "Version": "2012-10-17", + "Statement": [ + {"Action": ["ec2:DeleteKeyPair"], "Effect": "Deny", "Resource": "*"} + ], + } + + # Create a role + policy_config_query.backends["global"].create_policy( + description="mypolicy", + path="", + policy_document=json.dumps(basic_policy), + policy_name="mypolicy", + ) + + result = policy_config_query.list_config_service_resources(None, None, 100, None)[0] + assert len(result) == 1 + + policy = result[0] + assert policy["type"] == "AWS::IAM::Policy" + assert policy["id"] == policy["name"] == "arn:aws:iam::123456789012:policy/mypolicy" + assert policy["region"] == "global" + + +@mock_iam +def test_role_config_dict(): + from moto.iam.config import role_config_query, policy_config_query + from moto.iam.utils import random_resource_id + + # Without any roles + assert not role_config_query.get_config_resource("something") + assert role_config_query.list_config_service_resources(None, None, 100, None) == ( + [], + None, + ) + + basic_assume_role = { + "Version": "2012-10-17", + "Statement": [ + {"Effect": "Allow", "Principal": {"AWS": "*"}, "Action": "sts:AssumeRole"} + ], + } + + basic_policy = { + "Version": "2012-10-17", + "Statement": [{"Action": ["ec2:*"], "Effect": "Allow", "Resource": "*"}], + } + + # Create a policy for use in role permissions boundary + policy_config_query.backends["global"].create_policy( + description="basic_policy", + path="/", + policy_document=json.dumps(basic_policy), + policy_name="basic_policy", + ) + + policy_arn = policy_config_query.list_config_service_resources( + None, None, 100, None + )[0][0]["id"] + assert policy_arn is not None + + # Create some roles (and grab them repeatedly since they create with random names) + role_config_query.backends["global"].create_role( + role_name="plain_role", + assume_role_policy_document=None, + path="/", + permissions_boundary=None, + description="plain_role", + tags=[{"Key": "foo", "Value": "bar"}], + max_session_duration=3600, + ) + + plain_role = role_config_query.list_config_service_resources(None, None, 100, None)[ + 0 + ][0] + assert plain_role is not None + assert len(plain_role["id"]) == len(random_resource_id()) + + role_config_query.backends["global"].create_role( + role_name="assume_role", + assume_role_policy_document=json.dumps(basic_assume_role), + path="/", + permissions_boundary=None, + description="assume_role", + tags=[], + max_session_duration=3600, + ) + + assume_role = next( + role + for role in role_config_query.list_config_service_resources( + None, None, 100, None + )[0] + if role["id"] not in [plain_role["id"]] + ) + assert assume_role is not None + assert len(assume_role["id"]) == len(random_resource_id()) + assert assume_role["id"] is not plain_role["id"] + + role_config_query.backends["global"].create_role( + role_name="assume_and_permission_boundary_role", + assume_role_policy_document=json.dumps(basic_assume_role), + path="/", + permissions_boundary=policy_arn, + description="assume_and_permission_boundary_role", + tags=[], + max_session_duration=3600, + ) + + assume_and_permission_boundary_role = next( + role + for role in role_config_query.list_config_service_resources( + None, None, 100, None + )[0] + if role["id"] not in [plain_role["id"], assume_role["id"]] + ) + assert assume_and_permission_boundary_role is not None + assert len(assume_and_permission_boundary_role["id"]) == len(random_resource_id()) + assert assume_and_permission_boundary_role["id"] is not plain_role["id"] + assert assume_and_permission_boundary_role["id"] is not assume_role["id"] + + role_config_query.backends["global"].create_role( + role_name="role_with_attached_policy", + assume_role_policy_document=json.dumps(basic_assume_role), + path="/", + permissions_boundary=None, + description="role_with_attached_policy", + tags=[], + max_session_duration=3600, + ) + role_config_query.backends["global"].attach_role_policy( + policy_arn, "role_with_attached_policy" + ) + role_with_attached_policy = next( + role + for role in role_config_query.list_config_service_resources( + None, None, 100, None + )[0] + if role["id"] + not in [ + plain_role["id"], + assume_role["id"], + assume_and_permission_boundary_role["id"], + ] + ) + assert role_with_attached_policy is not None + assert len(role_with_attached_policy["id"]) == len(random_resource_id()) + assert role_with_attached_policy["id"] is not plain_role["id"] + assert role_with_attached_policy["id"] is not assume_role["id"] + assert ( + role_with_attached_policy["id"] is not assume_and_permission_boundary_role["id"] + ) + + role_config_query.backends["global"].create_role( + role_name="role_with_inline_policy", + assume_role_policy_document=json.dumps(basic_assume_role), + path="/", + permissions_boundary=None, + description="role_with_inline_policy", + tags=[], + max_session_duration=3600, + ) + role_config_query.backends["global"].put_role_policy( + "role_with_inline_policy", "inline_policy", json.dumps(basic_policy) + ) + + role_with_inline_policy = next( + role + for role in role_config_query.list_config_service_resources( + None, None, 100, None + )[0] + if role["id"] + not in [ + plain_role["id"], + assume_role["id"], + assume_and_permission_boundary_role["id"], + role_with_attached_policy["id"], + ] + ) + assert role_with_inline_policy is not None + assert len(role_with_inline_policy["id"]) == len(random_resource_id()) + assert role_with_inline_policy["id"] is not plain_role["id"] + assert role_with_inline_policy["id"] is not assume_role["id"] + assert ( + role_with_inline_policy["id"] is not assume_and_permission_boundary_role["id"] + ) + assert role_with_inline_policy["id"] is not role_with_attached_policy["id"] + + # plain role + plain_role_config = ( + role_config_query.backends["global"].roles[plain_role["id"]].to_config_dict() + ) + assert plain_role_config["version"] == "1.3" + assert plain_role_config["configurationItemStatus"] == "ResourceDiscovered" + assert plain_role_config["configurationStateId"] is not None + assert plain_role_config["arn"] == "arn:aws:iam::123456789012:role/plain_role" + assert plain_role_config["resourceType"] == "AWS::IAM::Role" + assert plain_role_config["resourceId"] == "plain_role" + assert plain_role_config["resourceName"] == "plain_role" + assert plain_role_config["awsRegion"] == "global" + assert plain_role_config["availabilityZone"] == "Not Applicable" + assert plain_role_config["resourceCreationTime"] is not None + assert plain_role_config["tags"] == {"foo": {"Key": "foo", "Value": "bar"}} + assert plain_role_config["configuration"]["path"] == "/" + assert plain_role_config["configuration"]["roleName"] == "plain_role" + assert plain_role_config["configuration"]["roleId"] == plain_role["id"] + assert plain_role_config["configuration"]["arn"] == plain_role_config["arn"] + assert plain_role_config["configuration"]["assumeRolePolicyDocument"] is None + assert plain_role_config["configuration"]["instanceProfileList"] == [] + assert plain_role_config["configuration"]["rolePolicyList"] == [] + assert plain_role_config["configuration"]["attachedManagedPolicies"] == [] + assert plain_role_config["configuration"]["permissionsBoundary"] is None + assert plain_role_config["configuration"]["tags"] == [ + {"key": "foo", "value": "bar"} + ] + assert plain_role_config["supplementaryConfiguration"] == {} + + # assume_role + assume_role_config = ( + role_config_query.backends["global"].roles[assume_role["id"]].to_config_dict() + ) + assert assume_role_config["arn"] == "arn:aws:iam::123456789012:role/assume_role" + assert assume_role_config["resourceId"] == "assume_role" + assert assume_role_config["resourceName"] == "assume_role" + assert assume_role_config["configuration"][ + "assumeRolePolicyDocument" + ] == parse.quote(json.dumps(basic_assume_role)) + + # assume_and_permission_boundary_role + assume_and_permission_boundary_role_config = ( + role_config_query.backends["global"] + .roles[assume_and_permission_boundary_role["id"]] + .to_config_dict() + ) + assert ( + assume_and_permission_boundary_role_config["arn"] + == "arn:aws:iam::123456789012:role/assume_and_permission_boundary_role" + ) + assert ( + assume_and_permission_boundary_role_config["resourceId"] + == "assume_and_permission_boundary_role" + ) + assert ( + assume_and_permission_boundary_role_config["resourceName"] + == "assume_and_permission_boundary_role" + ) + assert assume_and_permission_boundary_role_config["configuration"][ + "assumeRolePolicyDocument" + ] == parse.quote(json.dumps(basic_assume_role)) + assert ( + assume_and_permission_boundary_role_config["configuration"][ + "permissionsBoundary" + ] + == policy_arn + ) + + # role_with_attached_policy + role_with_attached_policy_config = ( + role_config_query.backends["global"] + .roles[role_with_attached_policy["id"]] + .to_config_dict() + ) + assert ( + role_with_attached_policy_config["arn"] + == "arn:aws:iam::123456789012:role/role_with_attached_policy" + ) + assert role_with_attached_policy_config["configuration"][ + "attachedManagedPolicies" + ] == [{"policyArn": policy_arn, "policyName": "basic_policy"}] + + # role_with_inline_policy + role_with_inline_policy_config = ( + role_config_query.backends["global"] + .roles[role_with_inline_policy["id"]] + .to_config_dict() + ) + assert ( + role_with_inline_policy_config["arn"] + == "arn:aws:iam::123456789012:role/role_with_inline_policy" + ) + assert role_with_inline_policy_config["configuration"]["rolePolicyList"] == [ + { + "policyName": "inline_policy", + "policyDocument": parse.quote(json.dumps(basic_policy)), + } + ] + + +@mock_iam +def test_policy_config_dict(): + from moto.iam.config import role_config_query, policy_config_query + from moto.iam.utils import random_policy_id + + # Without any roles + assert not policy_config_query.get_config_resource( + "arn:aws:iam::123456789012:policy/basic_policy" + ) + assert policy_config_query.list_config_service_resources(None, None, 100, None) == ( + [], + None, + ) + + basic_policy = { + "Version": "2012-10-17", + "Statement": [{"Action": ["ec2:*"], "Effect": "Allow", "Resource": "*"}], + } + + basic_policy_v2 = { + "Version": "2012-10-17", + "Statement": [ + {"Action": ["ec2:*", "s3:*"], "Effect": "Allow", "Resource": "*"} + ], + } + + policy_config_query.backends["global"].create_policy( + description="basic_policy", + path="/", + policy_document=json.dumps(basic_policy), + policy_name="basic_policy", + ) + + policy_arn = policy_config_query.list_config_service_resources( + None, None, 100, None + )[0][0]["id"] + assert policy_arn == "arn:aws:iam::123456789012:policy/basic_policy" + assert ( + policy_config_query.get_config_resource( + "arn:aws:iam::123456789012:policy/basic_policy" + ) + is not None + ) + + # Create a new version + policy_config_query.backends["global"].create_policy_version( + policy_arn, json.dumps(basic_policy_v2), "true" + ) + + # Create role to trigger attachment + role_config_query.backends["global"].create_role( + role_name="role_with_attached_policy", + assume_role_policy_document=None, + path="/", + permissions_boundary=None, + description="role_with_attached_policy", + tags=[], + max_session_duration=3600, + ) + role_config_query.backends["global"].attach_role_policy( + policy_arn, "role_with_attached_policy" + ) + + policy = ( + role_config_query.backends["global"] + .managed_policies["arn:aws:iam::123456789012:policy/basic_policy"] + .to_config_dict() + ) + assert policy["version"] == "1.3" + assert policy["configurationItemCaptureTime"] is not None + assert policy["configurationItemStatus"] == "OK" + assert policy["configurationStateId"] is not None + assert policy["arn"] == "arn:aws:iam::123456789012:policy/basic_policy" + assert policy["resourceType"] == "AWS::IAM::Policy" + assert len(policy["resourceId"]) == len(random_policy_id()) + assert policy["resourceName"] == "basic_policy" + assert policy["awsRegion"] == "global" + assert policy["availabilityZone"] == "Not Applicable" + assert policy["resourceCreationTime"] is not None + assert policy["configuration"]["policyName"] == policy["resourceName"] + assert policy["configuration"]["policyId"] == policy["resourceId"] + assert policy["configuration"]["arn"] == policy["arn"] + assert policy["configuration"]["path"] == "/" + assert policy["configuration"]["defaultVersionId"] == "v2" + assert policy["configuration"]["attachmentCount"] == 1 + assert policy["configuration"]["permissionsBoundaryUsageCount"] == 0 + assert policy["configuration"]["isAttachable"] == True + assert policy["configuration"]["description"] == "basic_policy" + assert policy["configuration"]["createDate"] is not None + assert policy["configuration"]["updateDate"] is not None + assert policy["configuration"]["policyVersionList"] == [ + { + "document": str(parse.quote(json.dumps(basic_policy))), + "versionId": "v1", + "isDefaultVersion": False, + "createDate": policy["configuration"]["policyVersionList"][0]["createDate"], + }, + { + "document": str(parse.quote(json.dumps(basic_policy_v2))), + "versionId": "v2", + "isDefaultVersion": True, + "createDate": policy["configuration"]["policyVersionList"][1]["createDate"], + }, + ] + assert policy["supplementaryConfiguration"] == {} From ff84b634845d67ed504d30db0906fea80248501b Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Sun, 2 Aug 2020 21:16:44 -0600 Subject: [PATCH 02/11] address PR comments --- CONFIG_README.md | 15 ++- moto/core/models.py | 26 ++++- moto/iam/config.py | 61 ++++++----- tests/test_iam/test_iam.py | 218 ++++++++++++++++++++++++++++--------- 4 files changed, 231 insertions(+), 89 deletions(-) diff --git a/CONFIG_README.md b/CONFIG_README.md index e223c8457..b0ae42181 100644 --- a/CONFIG_README.md +++ b/CONFIG_README.md @@ -53,15 +53,14 @@ An example of the above is implemented for S3. You can see that by looking at: 1. `moto/s3/config.py` 1. `moto/config/models.py` -As well as the corresponding unit tests in: +### Testing +For each resource type, you will need to test write tests for a few separate areas: -1. `tests/s3/test_s3.py` -1. `tests/config/test_config.py` +- Test the backend queries to ensure discovered resources come back (ie for `IAM::Policy`, write `tests.tests_iam.test_policy_list_config_discovered_resources`). For writing these tests, you must not make use of `boto` to create resources. You will need to use the backend model methods to provision the resources. This is to make tests compatible with the moto server. You must make tests for the resource type to test listing and object fetching. -Note for unit testing, you will want to add a test to ensure that you can query all the resources effectively. For testing this feature, -the unit tests for the `ConfigQueryModel` will not make use of `boto` to create resources, such as S3 buckets. You will need to use the -backend model methods to provision the resources. This is to make tests compatible with the moto server. You should absolutely make tests -in the resource type to test listing and object fetching. +- Test the config dict for all scenarios (ie for `IAM::Policy`, write `tests.tests_iam.test_policy_config_dict`). For writing this test, you'll need to create resources in the same way as the first test (without using `boto`), in every meaningful configuration that would produce a different config dict. Then, query the backend and ensure each of the dicts are as you expect. + +- Test that everything works end to end with the `boto` clients. (ie for `IAM::Policy`, write `tests.tests_iam.test_policy_config_client`). The main two items to test will be the `boto.client('config').list_discovered_resources()`, `boto.client('config').list_aggregate_discovered_resources()`, `moto.client('config').batch_get_resource_config()`, and `moto.client('config').batch_aggregate_get_resource_config()`. This test doesn't have to be super thorough, but it basically tests that the front end and backend logic all works together and returns correct resources. Beware the aggregate methods all have capital first letters (ie `Limit`), while non-aggregate methods have lowercase first letters (ie `limit`) ### Listing S3 is currently the model implementation, but it also odd in that S3 is a global resource type with regional resource residency. @@ -117,4 +116,4 @@ return for it. When implementing resource config fetching, you will need to return at a minimum `None` if the resource is not found, or a `dict` that looks like what AWS Config would return. -It's recommended to read the comment for the `ConfigQueryModel` 's `get_config_resource` function in [base class here](moto/core/models.py). +It's recommended to read the comment for the `ConfigQueryModel` 's `get_config_resource` function in [base class here](moto/core/models.py). \ No newline at end of file diff --git a/moto/core/models.py b/moto/core/models.py index bc7d282fd..422a9dd3d 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -27,8 +27,8 @@ from .utils import ( convert_flask_to_responses_response, ) - ACCOUNT_ID = os.environ.get("MOTO_ACCOUNT_ID", "123456789012") +CONFIG_BACKEND_DELIM = "\x1e" # Record Seperator "RS" ASCII Character class BaseMockAWS(object): @@ -768,15 +768,29 @@ class ConfigQueryModel(object): def aggregate_regions(self, path, backend_region, resource_region): """ - Returns a list of "region\1eresourcename" strings + This method will is called for both aggregated and non-aggregated calls for config resources. + It will figure out how to return the full list of resources for a given regional backend and append them to a final list. + It produces a list of both the region and the resource name with a delimiter character (CONFIG_BACKEND_DELIM, ASCII Record separator, \x1e). + IE: "us-east-1\x1ei-1234567800" + + Each config-enabled resource has a method named `list_config_service_resources` which has to parse the delimiter + ... + :param path: - A dict accessor string applied to the backend that locates the resource. + :param backend_region: + :param resource_region: + :return: - Returns a list of "region\x1eresourcename" strings """ filter_region = backend_region or resource_region if filter_region: filter_resources = list(self.backends[filter_region].__dict__[path].keys()) - return map( - lambda resource: "{}\1e{}".format(filter_region, resource), - filter_resources, + return list( + map( + lambda resource: "{}{}{}".format( + filter_region, CONFIG_BACKEND_DELIM, resource + ), + filter_resources, + ) ) # If we don't have a filter region @@ -784,7 +798,7 @@ class ConfigQueryModel(object): for region in self.backends: this_region_resources = list(self.backends[region].__dict__[path].keys()) for resource in this_region_resources: - ret.append("{}\1e{}".format(region, resource)) + ret.append("{}{}{}".format(region, CONFIG_BACKEND_DELIM, resource)) return ret diff --git a/moto/iam/config.py b/moto/iam/config.py index 4bb381248..7074569ec 100644 --- a/moto/iam/config.py +++ b/moto/iam/config.py @@ -4,6 +4,8 @@ from moto.core.exceptions import InvalidNextTokenException from moto.core.models import ConfigQueryModel from moto.iam import iam_backends +CONFIG_BACKEND_DELIM = "\x1e" # Record Seperator "RS" ASCII Character + class RoleConfigQuery(ConfigQueryModel): def list_config_service_resources( @@ -15,18 +17,17 @@ class RoleConfigQuery(ConfigQueryModel): backend_region=None, resource_region=None, ): - # For aggregation -- did we get both a resource ID and a resource name? - if resource_ids and resource_name: - # If the values are different, then return an empty list: - if resource_name not in resource_ids: - return [], None + # IAM roles are "global" and aren't assigned into any availability zone + # The resource ID is a AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ" + # The resource name is a user-assigned string like "MyDevelopmentAdminRole" - role_list = self.aggregate_regions("roles", backend_region, resource_region) + # Grab roles from backend + role_list = self.aggregate_regions("roles", "global", None) if not role_list: return [], None - # Pagination logic: + # Pagination logic sorted_roles = sorted(role_list) new_token = None @@ -34,7 +35,7 @@ class RoleConfigQuery(ConfigQueryModel): if not next_token: start = 0 else: - # "Tokens" are region + \00 + resource ID. + # "Tokens" are region + \x1e + resource ID. if next_token not in sorted_roles: raise InvalidNextTokenException() @@ -46,13 +47,16 @@ class RoleConfigQuery(ConfigQueryModel): if len(sorted_roles) > (start + limit): new_token = sorted_roles[start + limit] + # Each element is a string of "region\x1eresource_id" return ( [ { "type": "AWS::IAM::Role", - "id": role.split("\1e")[1], - "name": role.split("\1e")[1], - "region": role.split("\1e")[0], + "id": role.split(CONFIG_BACKEND_DELIM)[1], + "name": self.backends["global"] + .roles[role.split(CONFIG_BACKEND_DELIM)[1]] + .name, + "region": role.split(CONFIG_BACKEND_DELIM)[0], } for role in role_list ], @@ -71,7 +75,7 @@ class RoleConfigQuery(ConfigQueryModel): if resource_name and role.name != resource_name: return - # Format the bucket to the AWS Config format: + # Format the role to the AWS Config format: config_data = role.to_config_dict() # The 'configuration' field is also a JSON string: @@ -95,16 +99,19 @@ class PolicyConfigQuery(ConfigQueryModel): backend_region=None, resource_region=None, ): - # For aggregation -- did we get both a resource ID and a resource name? - if resource_ids and resource_name: - # If the values are different, then return an empty list: - if resource_name not in resource_ids: - return [], None + # IAM policies are "global" and aren't assigned into any availability zone + # The resource ID is a AWS-assigned random string like "ANPA0BSVNSZK00SJSPVUJ" + # The resource name is a user-assigned string like "my-development-policy" - # We don't want to include AWS Managed Policies + # We don't want to include AWS Managed Policies. This technically needs to + # respect the configuration recorder's 'includeGlobalResourceTypes' setting, + # but it's default set be default, and moto's config doesn't yet support + # custom configuration recorders, we'll just behave as default. policy_list = filter( - lambda policy: not policy.split("\1e")[1].startswith("arn:aws:iam::aws"), - self.aggregate_regions("managed_policies", backend_region, resource_region), + lambda policy: not policy.split(CONFIG_BACKEND_DELIM)[1].startswith( + "arn:aws:iam::aws" + ), + self.aggregate_regions("managed_policies", "global", None), ) if not policy_list: @@ -118,7 +125,7 @@ class PolicyConfigQuery(ConfigQueryModel): if not next_token: start = 0 else: - # "Tokens" are region + \00 + resource ID. + # "Tokens" are region + \x1e + resource ID. if next_token not in sorted_policies: raise InvalidNextTokenException() @@ -134,9 +141,13 @@ class PolicyConfigQuery(ConfigQueryModel): [ { "type": "AWS::IAM::Policy", - "id": policy.split("\1e")[1], - "name": policy.split("\1e")[1], - "region": policy.split("\1e")[0], + "id": self.backends["global"] + .managed_policies[policy.split(CONFIG_BACKEND_DELIM)[1]] + .id, + "name": self.backends["global"] + .managed_policies[policy.split(CONFIG_BACKEND_DELIM)[1]] + .name, + "region": policy.split(CONFIG_BACKEND_DELIM)[0], } for policy in policy_list ], @@ -155,7 +166,7 @@ class PolicyConfigQuery(ConfigQueryModel): if resource_name and policy.name != resource_name: return - # Format the bucket to the AWS Config format: + # Format the policy to the AWS Config format: config_data = policy.to_config_dict() # The 'configuration' field is also a JSON string: diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index b42f3d76f..c56a9260f 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -9,7 +9,7 @@ import sure # noqa from boto.exception import BotoServerError from botocore.exceptions import ClientError -from moto import mock_iam, mock_iam_deprecated, settings +from moto import mock_config, mock_iam, mock_iam_deprecated, settings from moto.core import ACCOUNT_ID from moto.iam.models import aws_managed_policies from moto.backends import get_backend @@ -2923,48 +2923,14 @@ def test_role_list_config_discovered_resources(): role = result[0] assert role["type"] == "AWS::IAM::Role" assert len(role["id"]) == len(random_resource_id()) - assert role["id"] == role["name"] + assert role["name"] == "something" assert role["region"] == "global" -@mock_iam -def test_policy_list_config_discovered_resources(): - from moto.iam.config import policy_config_query - - # Without any policies - assert policy_config_query.list_config_service_resources(None, None, 100, None) == ( - [], - None, - ) - - basic_policy = { - "Version": "2012-10-17", - "Statement": [ - {"Action": ["ec2:DeleteKeyPair"], "Effect": "Deny", "Resource": "*"} - ], - } - - # Create a role - policy_config_query.backends["global"].create_policy( - description="mypolicy", - path="", - policy_document=json.dumps(basic_policy), - policy_name="mypolicy", - ) - - result = policy_config_query.list_config_service_resources(None, None, 100, None)[0] - assert len(result) == 1 - - policy = result[0] - assert policy["type"] == "AWS::IAM::Policy" - assert policy["id"] == policy["name"] == "arn:aws:iam::123456789012:policy/mypolicy" - assert policy["region"] == "global" - - @mock_iam def test_role_config_dict(): from moto.iam.config import role_config_query, policy_config_query - from moto.iam.utils import random_resource_id + from moto.iam.utils import random_resource_id, random_policy_id # Without any roles assert not role_config_query.get_config_resource("something") @@ -2986,17 +2952,21 @@ def test_role_config_dict(): } # Create a policy for use in role permissions boundary - policy_config_query.backends["global"].create_policy( - description="basic_policy", - path="/", - policy_document=json.dumps(basic_policy), - policy_name="basic_policy", + policy_arn = ( + policy_config_query.backends["global"] + .create_policy( + description="basic_policy", + path="/", + policy_document=json.dumps(basic_policy), + policy_name="basic_policy", + ) + .arn ) - policy_arn = policy_config_query.list_config_service_resources( + policy_id = policy_config_query.list_config_service_resources( None, None, 100, None )[0][0]["id"] - assert policy_arn is not None + assert len(policy_id) == len(random_policy_id()) # Create some roles (and grab them repeatedly since they create with random names) role_config_query.backends["global"].create_role( @@ -3225,6 +3195,141 @@ def test_role_config_dict(): ] +@mock_iam +@mock_config +def test_role_config_client(): + from moto.iam.models import ACCOUNT_ID + from moto.iam.utils import random_resource_id + + iam_client = boto3.client("iam", region_name="us-west-2") + config_client = boto3.client("config", region_name="us-west-2") + + account_aggregation_source = { + "AccountIds": [ACCOUNT_ID], + "AllAwsRegions": True, + } + + config_client.put_configuration_aggregator( + ConfigurationAggregatorName="test_aggregator", + AccountAggregationSources=[account_aggregation_source], + ) + + result = config_client.list_discovered_resources(resourceType="AWS::IAM::Role") + assert not result["resourceIdentifiers"] + + role_id = iam_client.create_role( + Path="/", + RoleName="mytestrole", + Description="mytestrole", + AssumeRolePolicyDocument=json.dumps("{ }"), + )["Role"]["RoleId"] + + iam_client.create_role( + Path="/", + RoleName="mytestrole2", + Description="zmytestrole", + AssumeRolePolicyDocument=json.dumps("{ }"), + ) + + # Test non-aggregated query: (everything is getting a random id, so we can't test names by ordering) + result = config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", limit=1 + ) + first_result = result["resourceIdentifiers"][0]["resourceId"] + assert result["resourceIdentifiers"][0]["resourceType"] == "AWS::IAM::Role" + assert len(first_result) == len(random_resource_id()) + + # Test non-aggregated pagination + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", limit=1, nextToken=result["nextToken"] + )["resourceIdentifiers"][0]["resourceId"] + ) != first_result + + # Test aggregated query: (everything is getting a random id, so we can't test names by ordering) + agg_result = config_client.list_aggregate_discovered_resources( + ResourceType="AWS::IAM::Role", + ConfigurationAggregatorName="test_aggregator", + Limit=1, + ) + first_agg_result = agg_result["ResourceIdentifiers"][0]["ResourceId"] + assert agg_result["ResourceIdentifiers"][0]["ResourceType"] == "AWS::IAM::Role" + assert len(first_agg_result) == len(random_resource_id()) + assert agg_result["ResourceIdentifiers"][0]["SourceAccountId"] == ACCOUNT_ID + assert agg_result["ResourceIdentifiers"][0]["SourceRegion"] == "global" + + # Test aggregated pagination + assert ( + config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Role", + Limit=1, + NextToken=agg_result["NextToken"], + )["ResourceIdentifiers"][0]["ResourceId"] + != first_agg_result + ) + + # Test non-aggregated batch get + assert ( + config_client.batch_get_resource_config( + resourceKeys=[{"resourceType": "AWS::IAM::Role", "resourceId": role_id}] + )["baseConfigurationItems"][0]["resourceName"] + == "mytestrole" + ) + + # Test aggregated batch get + assert ( + config_client.batch_get_aggregate_resource_config( + ConfigurationAggregatorName="test_aggregator", + ResourceIdentifiers=[ + { + "SourceAccountId": ACCOUNT_ID, + "SourceRegion": "global", + "ResourceId": role_id, + "ResourceType": "AWS::IAM::Role", + } + ], + )["BaseConfigurationItems"][0]["resourceName"] + == "mytestrole" + ) + + +@mock_iam +def test_policy_list_config_discovered_resources(): + from moto.iam.config import policy_config_query + from moto.iam.utils import random_policy_id + + # Without any policies + assert policy_config_query.list_config_service_resources(None, None, 100, None) == ( + [], + None, + ) + + basic_policy = { + "Version": "2012-10-17", + "Statement": [ + {"Action": ["ec2:DeleteKeyPair"], "Effect": "Deny", "Resource": "*"} + ], + } + + # Create a role + policy_config_query.backends["global"].create_policy( + description="mypolicy", + path="", + policy_document=json.dumps(basic_policy), + policy_name="mypolicy", + ) + + result = policy_config_query.list_config_service_resources(None, None, 100, None)[0] + assert len(result) == 1 + + policy = result[0] + assert policy["type"] == "AWS::IAM::Policy" + assert len(policy["id"]) == len(random_policy_id()) + assert policy["name"] == "mypolicy" + assert policy["region"] == "global" + + @mock_iam def test_policy_config_dict(): from moto.iam.config import role_config_query, policy_config_query @@ -3251,17 +3356,24 @@ def test_policy_config_dict(): ], } - policy_config_query.backends["global"].create_policy( - description="basic_policy", - path="/", - policy_document=json.dumps(basic_policy), - policy_name="basic_policy", + policy_arn = ( + policy_config_query.backends["global"] + .create_policy( + description="basic_policy", + path="/", + policy_document=json.dumps(basic_policy), + policy_name="basic_policy", + ) + .arn ) - policy_arn = policy_config_query.list_config_service_resources( + policy_id = policy_config_query.list_config_service_resources( None, None, 100, None )[0][0]["id"] + assert len(policy_id) == len(random_policy_id()) + assert policy_arn == "arn:aws:iam::123456789012:policy/basic_policy" + assert ( policy_config_query.get_config_resource( "arn:aws:iam::123456789012:policy/basic_policy" @@ -3330,3 +3442,9 @@ def test_policy_config_dict(): }, ] assert policy["supplementaryConfiguration"] == {} + + +@mock_iam +@mock_config +def test_policy_config_client(): + assert 1 == 1 From ceefe970bc5de4e7c35af96978abfcfb13a0405f Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Sun, 2 Aug 2020 21:21:59 -0600 Subject: [PATCH 03/11] small flake issue --- moto/core/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index 422a9dd3d..bd5ae6634 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -768,12 +768,12 @@ class ConfigQueryModel(object): def aggregate_regions(self, path, backend_region, resource_region): """ - This method will is called for both aggregated and non-aggregated calls for config resources. + This method will is called for both aggregated and non-aggregated calls for config resources. It will figure out how to return the full list of resources for a given regional backend and append them to a final list. It produces a list of both the region and the resource name with a delimiter character (CONFIG_BACKEND_DELIM, ASCII Record separator, \x1e). - IE: "us-east-1\x1ei-1234567800" - - Each config-enabled resource has a method named `list_config_service_resources` which has to parse the delimiter + IE: "us-east-1\x1ei-1234567800" + + Each config-enabled resource has a method named `list_config_service_resources` which has to parse the delimiter ... :param path: - A dict accessor string applied to the backend that locates the resource. :param backend_region: From 8dd90db83cc73ec68e842ccd78842a9d3f2a20a1 Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Tue, 4 Aug 2020 09:11:26 -0600 Subject: [PATCH 04/11] add missing test for policies --- moto/iam/config.py | 10 +++- tests/test_iam/test_iam.py | 109 ++++++++++++++++++++++++++++++++++--- 2 files changed, 109 insertions(+), 10 deletions(-) diff --git a/moto/iam/config.py b/moto/iam/config.py index 7074569ec..4cd18bedc 100644 --- a/moto/iam/config.py +++ b/moto/iam/config.py @@ -157,8 +157,14 @@ class PolicyConfigQuery(ConfigQueryModel): def get_config_resource( self, resource_id, resource_name=None, backend_region=None, resource_region=None ): - - policy = self.backends["global"].managed_policies.get(resource_id, {}) + # policies are listed in the backend as arns, but we have to accept the PolicyID as the resource_id + # we'll make a really crude search for it + policy = None + for arn in self.backends["global"].managed_policies.keys(): + policy_candidate = self.backends["global"].managed_policies[arn] + if policy_candidate.id == resource_id: + policy = policy_candidate + break if not policy: return diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index c56a9260f..944b14acd 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -3373,13 +3373,7 @@ def test_policy_config_dict(): assert len(policy_id) == len(random_policy_id()) assert policy_arn == "arn:aws:iam::123456789012:policy/basic_policy" - - assert ( - policy_config_query.get_config_resource( - "arn:aws:iam::123456789012:policy/basic_policy" - ) - is not None - ) + assert policy_config_query.get_config_resource(policy_id) is not None # Create a new version policy_config_query.backends["global"].create_policy_version( @@ -3447,4 +3441,103 @@ def test_policy_config_dict(): @mock_iam @mock_config def test_policy_config_client(): - assert 1 == 1 + from moto.iam.models import ACCOUNT_ID + from moto.iam.utils import random_policy_id + + basic_policy = { + "Version": "2012-10-17", + "Statement": [{"Action": ["ec2:*"], "Effect": "Allow", "Resource": "*"}], + } + + iam_client = boto3.client("iam", region_name="us-west-2") + config_client = boto3.client("config", region_name="us-west-2") + + account_aggregation_source = { + "AccountIds": [ACCOUNT_ID], + "AllAwsRegions": True, + } + + config_client.put_configuration_aggregator( + ConfigurationAggregatorName="test_aggregator", + AccountAggregationSources=[account_aggregation_source], + ) + + result = config_client.list_discovered_resources(resourceType="AWS::IAM::Policy") + assert not result["resourceIdentifiers"] + + policy_id = iam_client.create_policy( + PolicyName="mypolicy", + Path="/", + PolicyDocument=json.dumps(basic_policy), + Description="mypolicy", + )["Policy"]["PolicyId"] + + # second policy + iam_client.create_policy( + PolicyName="zmypolicy", + Path="/", + PolicyDocument=json.dumps(basic_policy), + Description="zmypolicy", + ) + + # Test non-aggregated query: (everything is getting a random id, so we can't test names by ordering) + result = config_client.list_discovered_resources( + resourceType="AWS::IAM::Policy", limit=1 + ) + first_result = result["resourceIdentifiers"][0]["resourceId"] + assert result["resourceIdentifiers"][0]["resourceType"] == "AWS::IAM::Policy" + assert len(first_result) == len(random_policy_id()) + + # Test non-aggregated pagination + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Policy", limit=1, nextToken=result["nextToken"] + )["resourceIdentifiers"][0]["resourceId"] + ) != first_result + + # Test aggregated query: (everything is getting a random id, so we can't test names by ordering) + agg_result = config_client.list_aggregate_discovered_resources( + ResourceType="AWS::IAM::Policy", + ConfigurationAggregatorName="test_aggregator", + Limit=1, + ) + first_agg_result = agg_result["ResourceIdentifiers"][0]["ResourceId"] + assert agg_result["ResourceIdentifiers"][0]["ResourceType"] == "AWS::IAM::Policy" + assert len(first_agg_result) == len(random_policy_id()) + assert agg_result["ResourceIdentifiers"][0]["SourceAccountId"] == ACCOUNT_ID + assert agg_result["ResourceIdentifiers"][0]["SourceRegion"] == "global" + + # Test aggregated pagination + assert ( + config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Policy", + Limit=1, + NextToken=agg_result["NextToken"], + )["ResourceIdentifiers"][0]["ResourceId"] + != first_agg_result + ) + + # Test non-aggregated batch get + assert ( + config_client.batch_get_resource_config( + resourceKeys=[{"resourceType": "AWS::IAM::Policy", "resourceId": policy_id}] + )["baseConfigurationItems"][0]["resourceName"] + == "mypolicy" + ) + + # Test aggregated batch get + assert ( + config_client.batch_get_aggregate_resource_config( + ConfigurationAggregatorName="test_aggregator", + ResourceIdentifiers=[ + { + "SourceAccountId": ACCOUNT_ID, + "SourceRegion": "global", + "ResourceId": policy_id, + "ResourceType": "AWS::IAM::Policy", + } + ], + )["BaseConfigurationItems"][0]["resourceName"] + == "mypolicy" + ) From d8cea0213d6f98068b80d97d34c35c9fe46f2230 Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Thu, 6 Aug 2020 17:18:57 -0600 Subject: [PATCH 05/11] straighten out filter logic --- moto/core/models.py | 16 ++- moto/iam/config.py | 93 ++++++++++------ tests/test_iam/test_iam.py | 213 +++++++++++++++++++++++++++++++------ 3 files changed, 252 insertions(+), 70 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index bd5ae6634..b8b4322be 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -768,16 +768,22 @@ class ConfigQueryModel(object): def aggregate_regions(self, path, backend_region, resource_region): """ - This method will is called for both aggregated and non-aggregated calls for config resources. + This method is called for both aggregated and non-aggregated calls for config resources. + It will figure out how to return the full list of resources for a given regional backend and append them to a final list. It produces a list of both the region and the resource name with a delimiter character (CONFIG_BACKEND_DELIM, ASCII Record separator, \x1e). IE: "us-east-1\x1ei-1234567800" - Each config-enabled resource has a method named `list_config_service_resources` which has to parse the delimiter + You should only use this method if you need to aggregate resources over more than one region. + If your region is global, just query the global backend directly in the `list_config_service_resources` method + + If you use this method, your config-enabled resource must parse the delimited string in it's `list_config_service_resources` method. ... - :param path: - A dict accessor string applied to the backend that locates the resource. - :param backend_region: - :param resource_region: + :param path: - A dict accessor string applied to the backend that locates resources inside that backend. +            For example, if you passed path="keypairs", and you were working with an ec2 moto backend, it would yield the contents from + ec2_moto_backend[region].keypairs + :param backend_region: - Only used for filtering; A string representing the region IE: us-east-1 + :param resource_region: - Only used for filtering; A string representing the region IE: us-east-1 :return: - Returns a list of "region\x1eresourcename" strings """ diff --git a/moto/iam/config.py b/moto/iam/config.py index 4cd18bedc..fdf31b576 100644 --- a/moto/iam/config.py +++ b/moto/iam/config.py @@ -20,43 +20,59 @@ class RoleConfigQuery(ConfigQueryModel): # IAM roles are "global" and aren't assigned into any availability zone # The resource ID is a AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ" # The resource name is a user-assigned string like "MyDevelopmentAdminRole" + # Stored in moto backend with the AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ" - # Grab roles from backend - role_list = self.aggregate_regions("roles", "global", None) + # Grab roles from backend; need the full values since names and id's are different + role_list = list(self.backends["global"].roles.values()) if not role_list: return [], None - # Pagination logic - sorted_roles = sorted(role_list) + # Filter by resource name or ids + if resource_name or resource_ids: + filtered_roles = [] + # resource_name takes precendence over resource_ids + if resource_name: + for role in role_list: + if role.name == resource_name: + filtered_roles = [role] + break + else: + for role in role_list: + if role.id in resource_ids: + filtered_roles.append(role) + + # Filtered roles are now the subject for the listing + role_list = filtered_roles + + # Pagination logic, sort by role id + sorted_roles = sorted(role_list, key=lambda role: role.id) + # sorted_role_ids matches indicies of sorted_roles + sorted_role_ids = list(map(lambda role: role.id, sorted_roles)) new_token = None # Get the start: if not next_token: start = 0 else: - # "Tokens" are region + \x1e + resource ID. - if next_token not in sorted_roles: + if next_token not in sorted_role_ids: raise InvalidNextTokenException() - start = sorted_roles.index(next_token) + start = sorted_role_ids.index(next_token) # Get the list of items to collect: role_list = sorted_roles[start : (start + limit)] if len(sorted_roles) > (start + limit): - new_token = sorted_roles[start + limit] + new_token = sorted_role_ids[start + limit] - # Each element is a string of "region\x1eresource_id" return ( [ { "type": "AWS::IAM::Role", - "id": role.split(CONFIG_BACKEND_DELIM)[1], - "name": self.backends["global"] - .roles[role.split(CONFIG_BACKEND_DELIM)[1]] - .name, - "region": role.split(CONFIG_BACKEND_DELIM)[0], + "id": role.id, + "name": role.name, + "region": "global", } for role in role_list ], @@ -102,52 +118,67 @@ class PolicyConfigQuery(ConfigQueryModel): # IAM policies are "global" and aren't assigned into any availability zone # The resource ID is a AWS-assigned random string like "ANPA0BSVNSZK00SJSPVUJ" # The resource name is a user-assigned string like "my-development-policy" + # Stored in moto backend with the arn like "arn:aws:iam::123456789012:policy/my-development-policy" + + policy_list = list(self.backends["global"].managed_policies.values()) # We don't want to include AWS Managed Policies. This technically needs to # respect the configuration recorder's 'includeGlobalResourceTypes' setting, # but it's default set be default, and moto's config doesn't yet support # custom configuration recorders, we'll just behave as default. policy_list = filter( - lambda policy: not policy.split(CONFIG_BACKEND_DELIM)[1].startswith( - "arn:aws:iam::aws" - ), - self.aggregate_regions("managed_policies", "global", None), + lambda policy: not policy.arn.startswith("arn:aws:iam::aws"), policy_list, ) if not policy_list: return [], None - # Pagination logic: - sorted_policies = sorted(policy_list) + # Filter by resource name or ids + if resource_name or resource_ids: + filtered_policies = [] + # resource_name takes precendence over resource_ids + if resource_name: + for policy in policy_list: + if policy.name == resource_name: + filtered_policies = [policy] + break + else: + for policy in policy_list: + if policy.id in resource_ids: + filtered_policies.append(policy) + + # Filtered roles are now the subject for the listing + policy_list = filtered_policies + + # Pagination logic, sort by role id + sorted_policies = sorted(policy_list, key=lambda role: role.id) + # sorted_policy_ids matches indicies of sorted_policies + sorted_policy_ids = list(map(lambda policy: policy.id, sorted_policies)) + new_token = None # Get the start: if not next_token: start = 0 else: - # "Tokens" are region + \x1e + resource ID. - if next_token not in sorted_policies: + if next_token not in sorted_policy_ids: raise InvalidNextTokenException() - start = sorted_policies.index(next_token) + start = sorted_policy_ids.index(next_token) # Get the list of items to collect: policy_list = sorted_policies[start : (start + limit)] if len(sorted_policies) > (start + limit): - new_token = sorted_policies[start + limit] + new_token = sorted_policy_ids[start + limit] return ( [ { "type": "AWS::IAM::Policy", - "id": self.backends["global"] - .managed_policies[policy.split(CONFIG_BACKEND_DELIM)[1]] - .id, - "name": self.backends["global"] - .managed_policies[policy.split(CONFIG_BACKEND_DELIM)[1]] - .name, - "region": policy.split(CONFIG_BACKEND_DELIM)[0], + "id": policy.id, + "name": policy.name, + "region": "global", } for policy in policy_list ], diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 944b14acd..f71b96925 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -3217,19 +3217,24 @@ def test_role_config_client(): result = config_client.list_discovered_resources(resourceType="AWS::IAM::Role") assert not result["resourceIdentifiers"] - role_id = iam_client.create_role( - Path="/", - RoleName="mytestrole", - Description="mytestrole", - AssumeRolePolicyDocument=json.dumps("{ }"), - )["Role"]["RoleId"] + # Make 10 policies + roles = [] + num_roles = 10 + for ix in range(1, num_roles + 1): + this_policy = iam_client.create_role( + RoleName="role{}".format(ix), + Path="/", + Description="role{}".format(ix), + AssumeRolePolicyDocument=json.dumps("{ }"), + ) + roles.append( + { + "id": this_policy["Role"]["RoleId"], + "name": this_policy["Role"]["RoleName"], + } + ) - iam_client.create_role( - Path="/", - RoleName="mytestrole2", - Description="zmytestrole", - AssumeRolePolicyDocument=json.dumps("{ }"), - ) + assert len(roles) == num_roles # Test non-aggregated query: (everything is getting a random id, so we can't test names by ordering) result = config_client.list_discovered_resources( @@ -3269,12 +3274,77 @@ def test_role_config_client(): != first_agg_result ) + # Test non-aggregated resource name/id filter + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", resourceName=roles[1]["name"], limit=1, + )["resourceIdentifiers"][0]["resourceName"] + == roles[1]["name"] + ) + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", resourceIds=[roles[0]["id"]], limit=1, + )["resourceIdentifiers"][0]["resourceName"] + == roles[0]["name"] + ) + + # Test aggregated resource name/id filter + assert ( + config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Role", + Filters={"ResourceName": roles[5]["name"]}, + Limit=1, + )["ResourceIdentifiers"][0]["ResourceName"] + == roles[5]["name"] + ) + + assert ( + config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Role", + Filters={"ResourceId": roles[4]["id"]}, + Limit=1, + )["ResourceIdentifiers"][0]["ResourceName"] + == roles[4]["name"] + ) + + # Test name/id filter with pagination + first_call = config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", + resourceIds=[roles[1]["id"], roles[2]["id"]], + limit=1, + ) + + assert first_call["nextToken"] in [roles[1]["id"], roles[2]["id"]] + assert first_call["resourceIdentifiers"][0]["resourceName"] in [ + roles[1]["name"], + roles[2]["name"], + ] + second_call = config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", + resourceIds=[roles[1]["id"], roles[2]["id"]], + limit=1, + nextToken=first_call["nextToken"], + ) + assert "nextToken" not in second_call + assert first_call["resourceIdentifiers"][0]["resourceName"] in [ + roles[1]["name"], + roles[2]["name"], + ] + assert ( + first_call["resourceIdentifiers"][0]["resourceName"] + != second_call["resourceIdentifiers"][0]["resourceName"] + ) + # Test non-aggregated batch get assert ( config_client.batch_get_resource_config( - resourceKeys=[{"resourceType": "AWS::IAM::Role", "resourceId": role_id}] + resourceKeys=[ + {"resourceType": "AWS::IAM::Role", "resourceId": roles[0]["id"]} + ] )["baseConfigurationItems"][0]["resourceName"] - == "mytestrole" + == roles[0]["name"] ) # Test aggregated batch get @@ -3285,12 +3355,12 @@ def test_role_config_client(): { "SourceAccountId": ACCOUNT_ID, "SourceRegion": "global", - "ResourceId": role_id, + "ResourceId": roles[1]["id"], "ResourceType": "AWS::IAM::Role", } ], )["BaseConfigurationItems"][0]["resourceName"] - == "mytestrole" + == roles[1]["name"] ) @@ -3312,7 +3382,7 @@ def test_policy_list_config_discovered_resources(): ], } - # Create a role + # Create a policy policy_config_query.backends["global"].create_policy( description="mypolicy", path="", @@ -3320,6 +3390,12 @@ def test_policy_list_config_discovered_resources(): policy_name="mypolicy", ) + # We expect the backend to have arns as their keys + for backend_key in list( + policy_config_query.backends["global"].managed_policies.keys() + ): + assert backend_key.startswith("arn:aws:iam::") + result = policy_config_query.list_config_service_resources(None, None, 100, None)[0] assert len(result) == 1 @@ -3465,20 +3541,24 @@ def test_policy_config_client(): result = config_client.list_discovered_resources(resourceType="AWS::IAM::Policy") assert not result["resourceIdentifiers"] - policy_id = iam_client.create_policy( - PolicyName="mypolicy", - Path="/", - PolicyDocument=json.dumps(basic_policy), - Description="mypolicy", - )["Policy"]["PolicyId"] + # Make 10 policies + policies = [] + num_policies = 10 + for ix in range(1, num_policies + 1): + this_policy = iam_client.create_policy( + PolicyName="policy{}".format(ix), + Path="/", + PolicyDocument=json.dumps(basic_policy), + Description="policy{}".format(ix), + ) + policies.append( + { + "id": this_policy["Policy"]["PolicyId"], + "name": this_policy["Policy"]["PolicyName"], + } + ) - # second policy - iam_client.create_policy( - PolicyName="zmypolicy", - Path="/", - PolicyDocument=json.dumps(basic_policy), - Description="zmypolicy", - ) + assert len(policies) == num_policies # Test non-aggregated query: (everything is getting a random id, so we can't test names by ordering) result = config_client.list_discovered_resources( @@ -3518,12 +3598,77 @@ def test_policy_config_client(): != first_agg_result ) + # Test non-aggregated resource name/id filter + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Policy", resourceName=policies[1]["name"], limit=1, + )["resourceIdentifiers"][0]["resourceName"] + == policies[1]["name"] + ) + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Policy", resourceIds=[policies[0]["id"]], limit=1, + )["resourceIdentifiers"][0]["resourceName"] + == policies[0]["name"] + ) + + # Test aggregated resource name/id filter + assert ( + config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Policy", + Filters={"ResourceName": policies[5]["name"]}, + Limit=1, + )["ResourceIdentifiers"][0]["ResourceName"] + == policies[5]["name"] + ) + + assert ( + config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Policy", + Filters={"ResourceId": policies[4]["id"]}, + Limit=1, + )["ResourceIdentifiers"][0]["ResourceName"] + == policies[4]["name"] + ) + + # Test name/id filter with pagination + first_call = config_client.list_discovered_resources( + resourceType="AWS::IAM::Policy", + resourceIds=[policies[1]["id"], policies[2]["id"]], + limit=1, + ) + + assert first_call["nextToken"] in [policies[1]["id"], policies[2]["id"]] + assert first_call["resourceIdentifiers"][0]["resourceName"] in [ + policies[1]["name"], + policies[2]["name"], + ] + second_call = config_client.list_discovered_resources( + resourceType="AWS::IAM::Policy", + resourceIds=[policies[1]["id"], policies[2]["id"]], + limit=1, + nextToken=first_call["nextToken"], + ) + assert "nextToken" not in second_call + assert first_call["resourceIdentifiers"][0]["resourceName"] in [ + policies[1]["name"], + policies[2]["name"], + ] + assert ( + first_call["resourceIdentifiers"][0]["resourceName"] + != second_call["resourceIdentifiers"][0]["resourceName"] + ) + # Test non-aggregated batch get assert ( config_client.batch_get_resource_config( - resourceKeys=[{"resourceType": "AWS::IAM::Policy", "resourceId": policy_id}] + resourceKeys=[ + {"resourceType": "AWS::IAM::Policy", "resourceId": policies[7]["id"]} + ] )["baseConfigurationItems"][0]["resourceName"] - == "mypolicy" + == policies[7]["name"] ) # Test aggregated batch get @@ -3534,10 +3679,10 @@ def test_policy_config_client(): { "SourceAccountId": ACCOUNT_ID, "SourceRegion": "global", - "ResourceId": policy_id, + "ResourceId": policies[8]["id"], "ResourceType": "AWS::IAM::Policy", } ], )["BaseConfigurationItems"][0]["resourceName"] - == "mypolicy" + == policies[8]["name"] ) From 8d5c70a9246e8eb24e8e7d2229b369ef048fea29 Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Fri, 7 Aug 2020 22:34:59 -0600 Subject: [PATCH 06/11] different aggregation strategy --- moto/config/models.py | 26 ++- moto/core/models.py | 44 +---- moto/iam/config.py | 141 +++++++++++++--- moto/s3/config.py | 2 + tests/test_iam/test_iam.py | 330 ++++++++++++++++++++++++++++--------- 5 files changed, 406 insertions(+), 137 deletions(-) diff --git a/moto/config/models.py b/moto/config/models.py index 77f46e644..b8f31aa8d 100644 --- a/moto/config/models.py +++ b/moto/config/models.py @@ -48,6 +48,7 @@ from moto.config.exceptions import ( from moto.core import BaseBackend, BaseModel from moto.s3.config import s3_account_public_access_block_query, s3_config_query from moto.core import ACCOUNT_ID as DEFAULT_ACCOUNT_ID + from moto.iam.config import role_config_query, policy_config_query POP_STRINGS = [ @@ -68,6 +69,29 @@ RESOURCE_MAP = { "AWS::IAM::Policy": policy_config_query, } +CONFIG_REGIONS = [ + "af-south-1", + "ap-east-1", + "ap-northeast-1", + "ap-northeast-2", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "ca-central-1", + "eu-central-1", + "eu-north-1", + "eu-south-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", +] + def datetime2int(date): return int(time.mktime(date.timetuple())) @@ -979,6 +1003,7 @@ class ConfigBackend(BaseBackend): limit, next_token, resource_region=resource_region, + aggregator=self.config_aggregators.get(aggregator_name).__dict__, ) resource_identifiers = [] @@ -989,7 +1014,6 @@ class ConfigBackend(BaseBackend): "ResourceType": identifier["type"], "ResourceId": identifier["id"], } - if identifier.get("name"): item["ResourceName"] = identifier["name"] diff --git a/moto/core/models.py b/moto/core/models.py index b8b4322be..a3f720658 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -28,7 +28,6 @@ from .utils import ( ) ACCOUNT_ID = os.environ.get("MOTO_ACCOUNT_ID", "123456789012") -CONFIG_BACKEND_DELIM = "\x1e" # Record Seperator "RS" ASCII Character class BaseMockAWS(object): @@ -723,6 +722,8 @@ class ConfigQueryModel(object): :param backend_region: The region for the backend to pull results from. Set to `None` if this is an aggregated query. :param resource_region: The region for where the resources reside to pull results from. Set to `None` if this is a non-aggregated query. + :param aggregator: If an aggregated query, this will be the `ConfigAggregator instance from the backend. Set to `None` + if a non-aggregated query. Useful if you need special logic based off the aggregator (ie IAM) :return: This should return a list of Dicts that have the following fields: [ { @@ -766,47 +767,6 @@ class ConfigQueryModel(object): """ raise NotImplementedError() - def aggregate_regions(self, path, backend_region, resource_region): - """ - This method is called for both aggregated and non-aggregated calls for config resources. - - It will figure out how to return the full list of resources for a given regional backend and append them to a final list. - It produces a list of both the region and the resource name with a delimiter character (CONFIG_BACKEND_DELIM, ASCII Record separator, \x1e). - IE: "us-east-1\x1ei-1234567800" - - You should only use this method if you need to aggregate resources over more than one region. - If your region is global, just query the global backend directly in the `list_config_service_resources` method - - If you use this method, your config-enabled resource must parse the delimited string in it's `list_config_service_resources` method. - ... - :param path: - A dict accessor string applied to the backend that locates resources inside that backend. -            For example, if you passed path="keypairs", and you were working with an ec2 moto backend, it would yield the contents from - ec2_moto_backend[region].keypairs - :param backend_region: - Only used for filtering; A string representing the region IE: us-east-1 - :param resource_region: - Only used for filtering; A string representing the region IE: us-east-1 - :return: - Returns a list of "region\x1eresourcename" strings - """ - - filter_region = backend_region or resource_region - if filter_region: - filter_resources = list(self.backends[filter_region].__dict__[path].keys()) - return list( - map( - lambda resource: "{}{}{}".format( - filter_region, CONFIG_BACKEND_DELIM, resource - ), - filter_resources, - ) - ) - - # If we don't have a filter region - ret = [] - for region in self.backends: - this_region_resources = list(self.backends[region].__dict__[path].keys()) - for resource in this_region_resources: - ret.append("{}{}{}".format(region, CONFIG_BACKEND_DELIM, resource)) - return ret - class base_decorator(object): mock_backend = MockAWS diff --git a/moto/iam/config.py b/moto/iam/config.py index fdf31b576..484153217 100644 --- a/moto/iam/config.py +++ b/moto/iam/config.py @@ -4,8 +4,6 @@ from moto.core.exceptions import InvalidNextTokenException from moto.core.models import ConfigQueryModel from moto.iam import iam_backends -CONFIG_BACKEND_DELIM = "\x1e" # Record Seperator "RS" ASCII Character - class RoleConfigQuery(ConfigQueryModel): def list_config_service_resources( @@ -16,6 +14,7 @@ class RoleConfigQuery(ConfigQueryModel): next_token, backend_region=None, resource_region=None, + aggregator=None, ): # IAM roles are "global" and aren't assigned into any availability zone # The resource ID is a AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ" @@ -31,12 +30,16 @@ class RoleConfigQuery(ConfigQueryModel): # Filter by resource name or ids if resource_name or resource_ids: filtered_roles = [] - # resource_name takes precendence over resource_ids + # resource_name takes precedence over resource_ids if resource_name: for role in role_list: if role.name == resource_name: filtered_roles = [role] break + # but if both are passed, it must be a subset + if filtered_roles and resource_ids: + if filtered_roles[0].id not in resource_ids: + return [], None else: for role in role_list: if role.id in resource_ids: @@ -45,10 +48,54 @@ class RoleConfigQuery(ConfigQueryModel): # Filtered roles are now the subject for the listing role_list = filtered_roles - # Pagination logic, sort by role id - sorted_roles = sorted(role_list, key=lambda role: role.id) - # sorted_role_ids matches indicies of sorted_roles - sorted_role_ids = list(map(lambda role: role.id, sorted_roles)) + if aggregator: + # IAM is a little special; Roles are created in us-east-1 (which AWS calls the "global" region) + # However, the resource will return in the aggregator (in duplicate) for each region in the aggregator + # Therefore, we'll need to find out the regions where the aggregators are running, and then duplicate the resource there + + # In practice, it looks like AWS will only duplicate these resources if you've "used" any roles in the region, but since + # we can't really tell if this has happened in moto, we'll just bind this to the regions in your aggregator + from moto.config.models import CONFIG_REGIONS + + aggregated_regions = [] + aggregator_sources = aggregator.get( + "account_aggregation_sources" + ) or aggregator.get("organization_aggregation_source") + for source in aggregator_sources: + source_dict = source.__dict__ + if source_dict["all_aws_regions"]: + aggregated_regions = CONFIG_REGIONS + break + for region in source_dict["aws_regions"]: + aggregated_regions.append(region) + + duplicate_role_list = [] + for region in list(set(aggregated_regions)): + for role in role_list: + duplicate_role_list.append( + { + "_id": "{}{}".format( + role.id, region + ), # this is only for sorting, isn't returned outside of this functin + "type": "AWS::IAM::Role", + "id": role.id, + "name": role.name, + "region": region, + } + ) + + # Pagination logic, sort by role id + sorted_roles = sorted(duplicate_role_list, key=lambda role: role["_id"]) + + # sorted_role_ids matches indicies of sorted_roles + sorted_role_ids = list(map(lambda role: role["_id"], sorted_roles)) + else: + # Non-aggregated queries are in the else block, and we can treat these like a normal config resource + # Pagination logic, sort by role id + sorted_roles = sorted(role_list, key=lambda role: role.id) + # sorted_role_ids matches indicies of sorted_roles + sorted_role_ids = list(map(lambda role: role.id, sorted_roles)) + new_token = None # Get the start: @@ -70,9 +117,9 @@ class RoleConfigQuery(ConfigQueryModel): [ { "type": "AWS::IAM::Role", - "id": role.id, - "name": role.name, - "region": "global", + "id": role["id"] if aggregator else role.id, + "name": role["name"] if aggregator else role.name, + "region": role["region"] if aggregator else "global", } for role in role_list ], @@ -114,6 +161,7 @@ class PolicyConfigQuery(ConfigQueryModel): next_token, backend_region=None, resource_region=None, + aggregator=None, ): # IAM policies are "global" and aren't assigned into any availability zone # The resource ID is a AWS-assigned random string like "ANPA0BSVNSZK00SJSPVUJ" @@ -126,8 +174,11 @@ class PolicyConfigQuery(ConfigQueryModel): # respect the configuration recorder's 'includeGlobalResourceTypes' setting, # but it's default set be default, and moto's config doesn't yet support # custom configuration recorders, we'll just behave as default. - policy_list = filter( - lambda policy: not policy.arn.startswith("arn:aws:iam::aws"), policy_list, + policy_list = list( + filter( + lambda policy: not policy.arn.startswith("arn:aws:iam::aws"), + policy_list, + ) ) if not policy_list: @@ -136,12 +187,17 @@ class PolicyConfigQuery(ConfigQueryModel): # Filter by resource name or ids if resource_name or resource_ids: filtered_policies = [] - # resource_name takes precendence over resource_ids + # resource_name takes precedence over resource_ids if resource_name: for policy in policy_list: if policy.name == resource_name: filtered_policies = [policy] break + # but if both are passed, it must be a subset + if filtered_policies and resource_ids: + if filtered_policies[0].id not in resource_ids: + return [], None + else: for policy in policy_list: if policy.id in resource_ids: @@ -150,10 +206,55 @@ class PolicyConfigQuery(ConfigQueryModel): # Filtered roles are now the subject for the listing policy_list = filtered_policies - # Pagination logic, sort by role id - sorted_policies = sorted(policy_list, key=lambda role: role.id) - # sorted_policy_ids matches indicies of sorted_policies - sorted_policy_ids = list(map(lambda policy: policy.id, sorted_policies)) + if aggregator: + # IAM is a little special; Policies are created in us-east-1 (which AWS calls the "global" region) + # However, the resource will return in the aggregator (in duplicate) for each region in the aggregator + # Therefore, we'll need to find out the regions where the aggregators are running, and then duplicate the resource there + + # In practice, it looks like AWS will only duplicate these resources if you've "used" any policies in the region, but since + # we can't really tell if this has happened in moto, we'll just bind this to the regions in your aggregator + from moto.config.models import CONFIG_REGIONS + + aggregated_regions = [] + aggregator_sources = aggregator.get( + "account_aggregation_sources" + ) or aggregator.get("organization_aggregation_source") + for source in aggregator_sources: + source_dict = source.__dict__ + if source_dict["all_aws_regions"]: + aggregated_regions = CONFIG_REGIONS + break + for region in source_dict["aws_regions"]: + aggregated_regions.append(region) + + duplicate_policy_list = [] + for region in list(set(aggregated_regions)): + for policy in policy_list: + duplicate_policy_list.append( + { + "_id": "{}{}".format( + policy.id, region + ), # this is only for sorting, isn't returned outside of this functin + "type": "AWS::IAM::Policy", + "id": policy.id, + "name": policy.name, + "region": region, + } + ) + + # Pagination logic, sort by role id + sorted_policies = sorted( + duplicate_policy_list, key=lambda policy: policy["_id"] + ) + + # sorted_policy_ids matches indicies of sorted_policies + sorted_policy_ids = list(map(lambda policy: policy["_id"], sorted_policies)) + else: + # Non-aggregated queries are in the else block, and we can treat these like a normal config resource + # Pagination logic, sort by role id + sorted_policies = sorted(policy_list, key=lambda role: role.id) + # sorted_policy_ids matches indicies of sorted_policies + sorted_policy_ids = list(map(lambda policy: policy.id, sorted_policies)) new_token = None @@ -176,9 +277,9 @@ class PolicyConfigQuery(ConfigQueryModel): [ { "type": "AWS::IAM::Policy", - "id": policy.id, - "name": policy.name, - "region": "global", + "id": policy["id"] if aggregator else policy.id, + "name": policy["name"] if aggregator else policy.name, + "region": policy["region"] if aggregator else "global", } for policy in policy_list ], diff --git a/moto/s3/config.py b/moto/s3/config.py index 04b4315f3..932ebc3be 100644 --- a/moto/s3/config.py +++ b/moto/s3/config.py @@ -19,6 +19,7 @@ class S3ConfigQuery(ConfigQueryModel): next_token, backend_region=None, resource_region=None, + aggregator=None, ): # The resource_region only matters for aggregated queries as you can filter on bucket regions for them. # For other resource types, you would need to iterate appropriately for the backend_region. @@ -132,6 +133,7 @@ class S3AccountPublicAccessBlockConfigQuery(ConfigQueryModel): next_token, backend_region=None, resource_region=None, + aggregator=None, ): # For the Account Public Access Block, they are the same for all regions. The resource ID is the AWS account ID # There is no resource name -- it should be a blank string "" if provided. diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index f71b96925..b662cc527 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -2905,27 +2905,62 @@ def test_role_list_config_discovered_resources(): None, ) - # Create a role - role_config_query.backends["global"].create_role( - role_name="something", - assume_role_policy_document=None, - path="/", - permissions_boundary=None, - description="something", - tags=[], - max_session_duration=3600, - ) + # Make 3 roles + roles = [] + num_roles = 3 + for ix in range(1, num_roles + 1): + this_role = role_config_query.backends["global"].create_role( + role_name="role{}".format(ix), + assume_role_policy_document=None, + path="/", + permissions_boundary=None, + description="role{}".format(ix), + tags=[{"Key": "foo", "Value": "bar"}], + max_session_duration=3600, + ) + roles.append( + {"id": this_role.id, "name": this_role.name,} + ) + + assert len(roles) == num_roles result = role_config_query.list_config_service_resources(None, None, 100, None)[0] - assert len(result) == 1 + assert len(result) == num_roles - # The role gets a random ID, so we have to grab it + # The roles gets a random ID, so we can't directly test it role = result[0] assert role["type"] == "AWS::IAM::Role" - assert len(role["id"]) == len(random_resource_id()) - assert role["name"] == "something" + assert role["id"] in list(map(lambda p: p["id"], roles)) + assert role["name"] in list(map(lambda p: p["name"], roles)) assert role["region"] == "global" + # test passing list of resource ids + resource_ids = role_config_query.list_config_service_resources( + [roles[0]["id"], roles[1]["id"]], None, 100, None + )[0] + assert len(resource_ids) == 2 + + # test passing a single resource name + resource_name = role_config_query.list_config_service_resources( + None, roles[0]["name"], 100, None + )[0] + assert len(resource_name) == 1 + assert resource_name[0]["id"] == roles[0]["id"] + assert resource_name[0]["name"] == roles[0]["name"] + + # test passing a single resource name AND some resource id's + both_filter_good = role_config_query.list_config_service_resources( + [roles[0]["id"], roles[1]["id"]], roles[0]["name"], 100, None + )[0] + assert len(both_filter_good) == 1 + assert both_filter_good[0]["id"] == roles[0]["id"] + assert both_filter_good[0]["name"] == roles[0]["name"] + + both_filter_bad = role_config_query.list_config_service_resources( + [roles[0]["id"], roles[1]["id"]], roles[2]["name"], 100, None + )[0] + assert len(both_filter_bad) == 0 + @mock_iam def test_role_config_dict(): @@ -3200,18 +3235,29 @@ def test_role_config_dict(): def test_role_config_client(): from moto.iam.models import ACCOUNT_ID from moto.iam.utils import random_resource_id + from moto.config.models import CONFIG_REGIONS iam_client = boto3.client("iam", region_name="us-west-2") config_client = boto3.client("config", region_name="us-west-2") - account_aggregation_source = { + all_account_aggregation_source = { "AccountIds": [ACCOUNT_ID], "AllAwsRegions": True, } + two_region_account_aggregation_source = { + "AccountIds": [ACCOUNT_ID], + "AwsRegions": ["us-east-1", "us-west-2"], + } + config_client.put_configuration_aggregator( ConfigurationAggregatorName="test_aggregator", - AccountAggregationSources=[account_aggregation_source], + AccountAggregationSources=[all_account_aggregation_source], + ) + + config_client.put_configuration_aggregator( + ConfigurationAggregatorName="test_aggregator_two_regions", + AccountAggregationSources=[two_region_account_aggregation_source], ) result = config_client.list_discovered_resources(resourceType="AWS::IAM::Role") @@ -3251,29 +3297,88 @@ def test_role_config_client(): )["resourceIdentifiers"][0]["resourceId"] ) != first_result - # Test aggregated query: (everything is getting a random id, so we can't test names by ordering) + # Test aggregated query - by `Limit=len(CONFIG_REGIONS)`, we should get a single policy duplicated across all regions agg_result = config_client.list_aggregate_discovered_resources( ResourceType="AWS::IAM::Role", ConfigurationAggregatorName="test_aggregator", - Limit=1, + Limit=len(CONFIG_REGIONS), ) - first_agg_result = agg_result["ResourceIdentifiers"][0]["ResourceId"] - assert agg_result["ResourceIdentifiers"][0]["ResourceType"] == "AWS::IAM::Role" - assert len(first_agg_result) == len(random_resource_id()) - assert agg_result["ResourceIdentifiers"][0]["SourceAccountId"] == ACCOUNT_ID - assert agg_result["ResourceIdentifiers"][0]["SourceRegion"] == "global" + assert len(agg_result["ResourceIdentifiers"]) == len(CONFIG_REGIONS) + + agg_name = None + agg_id = None + for resource in agg_result["ResourceIdentifiers"]: + assert resource["ResourceType"] == "AWS::IAM::Role" + assert resource["SourceRegion"] in CONFIG_REGIONS + assert resource["SourceAccountId"] == ACCOUNT_ID + if agg_id: + assert resource["ResourceId"] == agg_id + if agg_name: + assert resource["ResourceName"] == agg_name + agg_name = resource["ResourceName"] + agg_id = resource["ResourceId"] # Test aggregated pagination + for resource in config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Role", + NextToken=agg_result["NextToken"], + )["ResourceIdentifiers"]: + assert resource["ResourceId"] != agg_id + + # Test non-aggregated resource name/id filter assert ( - config_client.list_aggregate_discovered_resources( - ConfigurationAggregatorName="test_aggregator", - ResourceType="AWS::IAM::Role", - Limit=1, - NextToken=agg_result["NextToken"], - )["ResourceIdentifiers"][0]["ResourceId"] - != first_agg_result + config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", resourceName=roles[1]["name"], limit=1, + )["resourceIdentifiers"][0]["resourceName"] + == roles[1]["name"] ) + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", resourceIds=[roles[0]["id"]], limit=1, + )["resourceIdentifiers"][0]["resourceName"] + == roles[0]["name"] + ) + + # Test aggregated resource name/id filter + agg_name_filter = config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Role", + Filters={"ResourceName": roles[5]["name"]}, + ) + assert len(agg_name_filter["ResourceIdentifiers"]) == len(CONFIG_REGIONS) + assert agg_name_filter["ResourceIdentifiers"][0]["ResourceId"] == roles[5]["id"] + + agg_name_filter = config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator_two_regions", + ResourceType="AWS::IAM::Role", + Filters={"ResourceName": roles[5]["name"]}, + ) + assert len(agg_name_filter["ResourceIdentifiers"]) == len( + two_region_account_aggregation_source["AwsRegions"] + ) + assert agg_name_filter["ResourceIdentifiers"][0]["ResourceId"] == roles[5]["id"] + + agg_id_filter = config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Role", + Filters={"ResourceId": roles[4]["id"]}, + ) + + assert len(agg_id_filter["ResourceIdentifiers"]) == len(CONFIG_REGIONS) + assert agg_id_filter["ResourceIdentifiers"][0]["ResourceName"] == roles[4]["name"] + + agg_name_filter = config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator_two_regions", + ResourceType="AWS::IAM::Role", + Filters={"ResourceId": roles[5]["id"]}, + ) + assert len(agg_name_filter["ResourceIdentifiers"]) == len( + two_region_account_aggregation_source["AwsRegions"] + ) + assert agg_name_filter["ResourceIdentifiers"][0]["ResourceName"] == roles[5]["name"] + # Test non-aggregated resource name/id filter assert ( config_client.list_discovered_resources( @@ -3354,7 +3459,7 @@ def test_role_config_client(): ResourceIdentifiers=[ { "SourceAccountId": ACCOUNT_ID, - "SourceRegion": "global", + "SourceRegion": "us-east-1", "ResourceId": roles[1]["id"], "ResourceType": "AWS::IAM::Role", } @@ -3382,13 +3487,21 @@ def test_policy_list_config_discovered_resources(): ], } - # Create a policy - policy_config_query.backends["global"].create_policy( - description="mypolicy", - path="", - policy_document=json.dumps(basic_policy), - policy_name="mypolicy", - ) + # Make 3 policies + policies = [] + num_policies = 3 + for ix in range(1, num_policies + 1): + this_policy = policy_config_query.backends["global"].create_policy( + description="policy{}".format(ix), + path="", + policy_document=json.dumps(basic_policy), + policy_name="policy{}".format(ix), + ) + policies.append( + {"id": this_policy.id, "name": this_policy.name,} + ) + + assert len(policies) == num_policies # We expect the backend to have arns as their keys for backend_key in list( @@ -3397,14 +3510,41 @@ def test_policy_list_config_discovered_resources(): assert backend_key.startswith("arn:aws:iam::") result = policy_config_query.list_config_service_resources(None, None, 100, None)[0] - assert len(result) == 1 + assert len(result) == num_policies policy = result[0] assert policy["type"] == "AWS::IAM::Policy" - assert len(policy["id"]) == len(random_policy_id()) - assert policy["name"] == "mypolicy" + assert policy["id"] in list(map(lambda p: p["id"], policies)) + assert policy["name"] in list(map(lambda p: p["name"], policies)) assert policy["region"] == "global" + # test passing list of resource ids + resource_ids = policy_config_query.list_config_service_resources( + [policies[0]["id"], policies[1]["id"]], None, 100, None + )[0] + assert len(resource_ids) == 2 + + # test passing a single resource name + resource_name = policy_config_query.list_config_service_resources( + None, policies[0]["name"], 100, None + )[0] + assert len(resource_name) == 1 + assert resource_name[0]["id"] == policies[0]["id"] + assert resource_name[0]["name"] == policies[0]["name"] + + # test passing a single resource name AND some resource id's + both_filter_good = policy_config_query.list_config_service_resources( + [policies[0]["id"], policies[1]["id"]], policies[0]["name"], 100, None + )[0] + assert len(both_filter_good) == 1 + assert both_filter_good[0]["id"] == policies[0]["id"] + assert both_filter_good[0]["name"] == policies[0]["name"] + + both_filter_bad = policy_config_query.list_config_service_resources( + [policies[0]["id"], policies[1]["id"]], policies[2]["name"], 100, None + )[0] + assert len(both_filter_bad) == 0 + @mock_iam def test_policy_config_dict(): @@ -3519,6 +3659,7 @@ def test_policy_config_dict(): def test_policy_config_client(): from moto.iam.models import ACCOUNT_ID from moto.iam.utils import random_policy_id + from moto.config.models import CONFIG_REGIONS basic_policy = { "Version": "2012-10-17", @@ -3528,14 +3669,24 @@ def test_policy_config_client(): iam_client = boto3.client("iam", region_name="us-west-2") config_client = boto3.client("config", region_name="us-west-2") - account_aggregation_source = { + all_account_aggregation_source = { "AccountIds": [ACCOUNT_ID], "AllAwsRegions": True, } + two_region_account_aggregation_source = { + "AccountIds": [ACCOUNT_ID], + "AwsRegions": ["us-east-1", "us-west-2"], + } + config_client.put_configuration_aggregator( ConfigurationAggregatorName="test_aggregator", - AccountAggregationSources=[account_aggregation_source], + AccountAggregationSources=[all_account_aggregation_source], + ) + + config_client.put_configuration_aggregator( + ConfigurationAggregatorName="test_aggregator_two_regions", + AccountAggregationSources=[two_region_account_aggregation_source], ) result = config_client.list_discovered_resources(resourceType="AWS::IAM::Policy") @@ -3575,28 +3726,35 @@ def test_policy_config_client(): )["resourceIdentifiers"][0]["resourceId"] ) != first_result - # Test aggregated query: (everything is getting a random id, so we can't test names by ordering) + # Test aggregated query - by `Limit=len(CONFIG_REGIONS)`, we should get a single policy duplicated across all regions agg_result = config_client.list_aggregate_discovered_resources( ResourceType="AWS::IAM::Policy", ConfigurationAggregatorName="test_aggregator", - Limit=1, + Limit=len(CONFIG_REGIONS), ) - first_agg_result = agg_result["ResourceIdentifiers"][0]["ResourceId"] - assert agg_result["ResourceIdentifiers"][0]["ResourceType"] == "AWS::IAM::Policy" - assert len(first_agg_result) == len(random_policy_id()) - assert agg_result["ResourceIdentifiers"][0]["SourceAccountId"] == ACCOUNT_ID - assert agg_result["ResourceIdentifiers"][0]["SourceRegion"] == "global" + assert len(agg_result["ResourceIdentifiers"]) == len(CONFIG_REGIONS) + + agg_name = None + agg_id = None + for resource in agg_result["ResourceIdentifiers"]: + assert resource["ResourceType"] == "AWS::IAM::Policy" + assert resource["SourceRegion"] in CONFIG_REGIONS + assert resource["SourceAccountId"] == ACCOUNT_ID + if agg_id: + assert resource["ResourceId"] == agg_id + if agg_name: + assert resource["ResourceName"] == agg_name + agg_name = resource["ResourceName"] + agg_id = resource["ResourceId"] # Test aggregated pagination - assert ( - config_client.list_aggregate_discovered_resources( - ConfigurationAggregatorName="test_aggregator", - ResourceType="AWS::IAM::Policy", - Limit=1, - NextToken=agg_result["NextToken"], - )["ResourceIdentifiers"][0]["ResourceId"] - != first_agg_result - ) + for resource in config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Policy", + Limit=1, + NextToken=agg_result["NextToken"], + )["ResourceIdentifiers"]: + assert resource["ResourceId"] != agg_id # Test non-aggregated resource name/id filter assert ( @@ -3605,6 +3763,7 @@ def test_policy_config_client(): )["resourceIdentifiers"][0]["resourceName"] == policies[1]["name"] ) + assert ( config_client.list_discovered_resources( resourceType="AWS::IAM::Policy", resourceIds=[policies[0]["id"]], limit=1, @@ -3613,24 +3772,47 @@ def test_policy_config_client(): ) # Test aggregated resource name/id filter + agg_name_filter = config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Policy", + Filters={"ResourceName": policies[5]["name"]}, + ) + assert len(agg_name_filter["ResourceIdentifiers"]) == len(CONFIG_REGIONS) assert ( - config_client.list_aggregate_discovered_resources( - ConfigurationAggregatorName="test_aggregator", - ResourceType="AWS::IAM::Policy", - Filters={"ResourceName": policies[5]["name"]}, - Limit=1, - )["ResourceIdentifiers"][0]["ResourceName"] - == policies[5]["name"] + agg_name_filter["ResourceIdentifiers"][0]["ResourceName"] == policies[5]["name"] ) + agg_name_filter = config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator_two_regions", + ResourceType="AWS::IAM::Policy", + Filters={"ResourceName": policies[5]["name"]}, + ) + assert len(agg_name_filter["ResourceIdentifiers"]) == len( + two_region_account_aggregation_source["AwsRegions"] + ) + assert agg_name_filter["ResourceIdentifiers"][0]["ResourceId"] == policies[5]["id"] + + agg_id_filter = config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Policy", + Filters={"ResourceId": policies[4]["id"]}, + ) + + assert len(agg_id_filter["ResourceIdentifiers"]) == len(CONFIG_REGIONS) assert ( - config_client.list_aggregate_discovered_resources( - ConfigurationAggregatorName="test_aggregator", - ResourceType="AWS::IAM::Policy", - Filters={"ResourceId": policies[4]["id"]}, - Limit=1, - )["ResourceIdentifiers"][0]["ResourceName"] - == policies[4]["name"] + agg_id_filter["ResourceIdentifiers"][0]["ResourceName"] == policies[4]["name"] + ) + + agg_name_filter = config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator_two_regions", + ResourceType="AWS::IAM::Policy", + Filters={"ResourceId": policies[5]["id"]}, + ) + assert len(agg_name_filter["ResourceIdentifiers"]) == len( + two_region_account_aggregation_source["AwsRegions"] + ) + assert ( + agg_name_filter["ResourceIdentifiers"][0]["ResourceName"] == policies[5]["name"] ) # Test name/id filter with pagination @@ -3678,7 +3860,7 @@ def test_policy_config_client(): ResourceIdentifiers=[ { "SourceAccountId": ACCOUNT_ID, - "SourceRegion": "global", + "SourceRegion": "us-east-2", "ResourceId": policies[8]["id"], "ResourceType": "AWS::IAM::Policy", } From fd69c93a09edd2ee3675c6e39063b524e62a16d2 Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Wed, 12 Aug 2020 17:16:47 -0600 Subject: [PATCH 07/11] use botocore regions and refactor sorting --- moto/config/models.py | 23 ------------------- moto/iam/config.py | 47 +++++++++++++++++++------------------- tests/test_iam/test_iam.py | 6 +++-- 3 files changed, 27 insertions(+), 49 deletions(-) diff --git a/moto/config/models.py b/moto/config/models.py index b8f31aa8d..db2556343 100644 --- a/moto/config/models.py +++ b/moto/config/models.py @@ -69,29 +69,6 @@ RESOURCE_MAP = { "AWS::IAM::Policy": policy_config_query, } -CONFIG_REGIONS = [ - "af-south-1", - "ap-east-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-south-1", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "eu-central-1", - "eu-north-1", - "eu-south-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "me-south-1", - "sa-east-1", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", -] - def datetime2int(date): return int(time.mktime(date.timetuple())) diff --git a/moto/iam/config.py b/moto/iam/config.py index 484153217..2f2cafa5f 100644 --- a/moto/iam/config.py +++ b/moto/iam/config.py @@ -1,5 +1,5 @@ import json - +import boto3 from moto.core.exceptions import InvalidNextTokenException from moto.core.models import ConfigQueryModel from moto.iam import iam_backends @@ -55,8 +55,6 @@ class RoleConfigQuery(ConfigQueryModel): # In practice, it looks like AWS will only duplicate these resources if you've "used" any roles in the region, but since # we can't really tell if this has happened in moto, we'll just bind this to the regions in your aggregator - from moto.config.models import CONFIG_REGIONS - aggregated_regions = [] aggregator_sources = aggregator.get( "account_aggregation_sources" @@ -64,7 +62,7 @@ class RoleConfigQuery(ConfigQueryModel): for source in aggregator_sources: source_dict = source.__dict__ if source_dict["all_aws_regions"]: - aggregated_regions = CONFIG_REGIONS + aggregated_regions = boto3.Session().get_available_regions("config") break for region in source_dict["aws_regions"]: aggregated_regions.append(region) @@ -86,15 +84,10 @@ class RoleConfigQuery(ConfigQueryModel): # Pagination logic, sort by role id sorted_roles = sorted(duplicate_role_list, key=lambda role: role["_id"]) - - # sorted_role_ids matches indicies of sorted_roles - sorted_role_ids = list(map(lambda role: role["_id"], sorted_roles)) else: # Non-aggregated queries are in the else block, and we can treat these like a normal config resource # Pagination logic, sort by role id sorted_roles = sorted(role_list, key=lambda role: role.id) - # sorted_role_ids matches indicies of sorted_roles - sorted_role_ids = list(map(lambda role: role.id, sorted_roles)) new_token = None @@ -102,16 +95,22 @@ class RoleConfigQuery(ConfigQueryModel): if not next_token: start = 0 else: - if next_token not in sorted_role_ids: + try: + # Find the index of the next + start = next( + index + for (index, r) in enumerate(sorted_roles) + if next_token == (r["_id"] if aggregator else r.id) + ) + except StopIteration: raise InvalidNextTokenException() - start = sorted_role_ids.index(next_token) - # Get the list of items to collect: role_list = sorted_roles[start : (start + limit)] if len(sorted_roles) > (start + limit): - new_token = sorted_role_ids[start + limit] + record = sorted_roles[start + limit] + new_token = record["_id"] if aggregator else record.id return ( [ @@ -213,8 +212,6 @@ class PolicyConfigQuery(ConfigQueryModel): # In practice, it looks like AWS will only duplicate these resources if you've "used" any policies in the region, but since # we can't really tell if this has happened in moto, we'll just bind this to the regions in your aggregator - from moto.config.models import CONFIG_REGIONS - aggregated_regions = [] aggregator_sources = aggregator.get( "account_aggregation_sources" @@ -222,7 +219,7 @@ class PolicyConfigQuery(ConfigQueryModel): for source in aggregator_sources: source_dict = source.__dict__ if source_dict["all_aws_regions"]: - aggregated_regions = CONFIG_REGIONS + aggregated_regions = boto3.Session().get_available_regions("config") break for region in source_dict["aws_regions"]: aggregated_regions.append(region) @@ -247,14 +244,10 @@ class PolicyConfigQuery(ConfigQueryModel): duplicate_policy_list, key=lambda policy: policy["_id"] ) - # sorted_policy_ids matches indicies of sorted_policies - sorted_policy_ids = list(map(lambda policy: policy["_id"], sorted_policies)) else: # Non-aggregated queries are in the else block, and we can treat these like a normal config resource # Pagination logic, sort by role id sorted_policies = sorted(policy_list, key=lambda role: role.id) - # sorted_policy_ids matches indicies of sorted_policies - sorted_policy_ids = list(map(lambda policy: policy.id, sorted_policies)) new_token = None @@ -262,16 +255,22 @@ class PolicyConfigQuery(ConfigQueryModel): if not next_token: start = 0 else: - if next_token not in sorted_policy_ids: + try: + # Find the index of the next + start = next( + index + for (index, p) in enumerate(sorted_policies) + if next_token == (p["_id"] if aggregator else p.id) + ) + except StopIteration: raise InvalidNextTokenException() - start = sorted_policy_ids.index(next_token) - # Get the list of items to collect: policy_list = sorted_policies[start : (start + limit)] if len(sorted_policies) > (start + limit): - new_token = sorted_policy_ids[start + limit] + record = sorted_policies[start + limit] + new_token = record["_id"] if aggregator else record.id return ( [ diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index b662cc527..e1bc93d57 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -3235,7 +3235,8 @@ def test_role_config_dict(): def test_role_config_client(): from moto.iam.models import ACCOUNT_ID from moto.iam.utils import random_resource_id - from moto.config.models import CONFIG_REGIONS + + CONFIG_REGIONS = boto3.Session().get_available_regions("config") iam_client = boto3.client("iam", region_name="us-west-2") config_client = boto3.client("config", region_name="us-west-2") @@ -3659,7 +3660,8 @@ def test_policy_config_dict(): def test_policy_config_client(): from moto.iam.models import ACCOUNT_ID from moto.iam.utils import random_policy_id - from moto.config.models import CONFIG_REGIONS + + CONFIG_REGIONS = boto3.Session().get_available_regions("config") basic_policy = { "Version": "2012-10-17", From 4354cb06d1252cc08733cc6c0b6b615d02bd1dcf Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Wed, 26 Aug 2020 19:02:14 -0600 Subject: [PATCH 08/11] remove comment --- moto/core/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/moto/core/models.py b/moto/core/models.py index a3f720658..96535f500 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -722,8 +722,6 @@ class ConfigQueryModel(object): :param backend_region: The region for the backend to pull results from. Set to `None` if this is an aggregated query. :param resource_region: The region for where the resources reside to pull results from. Set to `None` if this is a non-aggregated query. - :param aggregator: If an aggregated query, this will be the `ConfigAggregator instance from the backend. Set to `None` - if a non-aggregated query. Useful if you need special logic based off the aggregator (ie IAM) :return: This should return a list of Dicts that have the following fields: [ { From fc7f3fecb69c2c39f223aa6366ef056dfccff858 Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Thu, 17 Sep 2020 17:43:19 -0600 Subject: [PATCH 09/11] clean up and bring up to master --- moto/core/models.py | 5 +++++ moto/iam/models.py | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/moto/core/models.py b/moto/core/models.py index 96535f500..d8de6b29f 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -691,6 +691,7 @@ class ConfigQueryModel(object): next_token, backend_region=None, resource_region=None, + aggregator=None, ): """For AWS Config. This will list all of the resources of the given type and optional resource name and region. @@ -722,6 +723,10 @@ class ConfigQueryModel(object): :param backend_region: The region for the backend to pull results from. Set to `None` if this is an aggregated query. :param resource_region: The region for where the resources reside to pull results from. Set to `None` if this is a non-aggregated query. + :param aggregator: If the query is an aggregated query, *AND* the resource has "non-standard" aggregation logic (mainly, IAM), + you'll need to pass aggregator used. In most cases, this should be omitted/set to `None`. See the + conditional logic under `if aggregator` in the moto/iam/config.py for the IAM example. + :return: This should return a list of Dicts that have the following fields: [ { diff --git a/moto/iam/models.py b/moto/iam/models.py index 9ae1ddcdf..617da69b0 100755 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -12,7 +12,6 @@ import time from cryptography import x509 from cryptography.hazmat.backends import default_backend -from six.moves.urllib.parse import urlparse from six.moves.urllib import parse from moto.core.exceptions import RESTError From 56c78ee39f8dc82f60ce2dd76c0a0637e930ce9b Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Mon, 21 Sep 2020 17:42:22 -0600 Subject: [PATCH 10/11] use get instead of direct dict access --- moto/iam/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/iam/config.py b/moto/iam/config.py index 2f2cafa5f..018709346 100644 --- a/moto/iam/config.py +++ b/moto/iam/config.py @@ -61,10 +61,10 @@ class RoleConfigQuery(ConfigQueryModel): ) or aggregator.get("organization_aggregation_source") for source in aggregator_sources: source_dict = source.__dict__ - if source_dict["all_aws_regions"]: + if source_dict.get("all_aws_regions", False): aggregated_regions = boto3.Session().get_available_regions("config") break - for region in source_dict["aws_regions"]: + for region in source_dict.get("aws_regions", []): aggregated_regions.append(region) duplicate_role_list = [] From e2fe33bf07786d2119024ed200c6c8ed23b543ae Mon Sep 17 00:00:00 2001 From: Nick Stocchero Date: Mon, 21 Sep 2020 17:56:04 -0600 Subject: [PATCH 11/11] duplicate dict.get logic --- moto/iam/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/iam/config.py b/moto/iam/config.py index 018709346..cf116f945 100644 --- a/moto/iam/config.py +++ b/moto/iam/config.py @@ -218,10 +218,10 @@ class PolicyConfigQuery(ConfigQueryModel): ) or aggregator.get("organization_aggregation_source") for source in aggregator_sources: source_dict = source.__dict__ - if source_dict["all_aws_regions"]: + if source_dict.get("all_aws_regions", False): aggregated_regions = boto3.Session().get_available_regions("config") break - for region in source_dict["aws_regions"]: + for region in source_dict.get("aws_regions", []): aggregated_regions.append(region) duplicate_policy_list = []