diff --git a/moto/core/models.py b/moto/core/models.py index bd5ae6634..b8b4322be 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -768,16 +768,22 @@ class ConfigQueryModel(object): def aggregate_regions(self, path, backend_region, resource_region): """ - This method will is called for both aggregated and non-aggregated calls for config resources. + This method is called for both aggregated and non-aggregated calls for config resources. + It will figure out how to return the full list of resources for a given regional backend and append them to a final list. It produces a list of both the region and the resource name with a delimiter character (CONFIG_BACKEND_DELIM, ASCII Record separator, \x1e). IE: "us-east-1\x1ei-1234567800" - Each config-enabled resource has a method named `list_config_service_resources` which has to parse the delimiter + You should only use this method if you need to aggregate resources over more than one region. + If your region is global, just query the global backend directly in the `list_config_service_resources` method + + If you use this method, your config-enabled resource must parse the delimited string in it's `list_config_service_resources` method. ... - :param path: - A dict accessor string applied to the backend that locates the resource. - :param backend_region: - :param resource_region: + :param path: - A dict accessor string applied to the backend that locates resources inside that backend. +            For example, if you passed path="keypairs", and you were working with an ec2 moto backend, it would yield the contents from + ec2_moto_backend[region].keypairs + :param backend_region: - Only used for filtering; A string representing the region IE: us-east-1 + :param resource_region: - Only used for filtering; A string representing the region IE: us-east-1 :return: - Returns a list of "region\x1eresourcename" strings """ diff --git a/moto/iam/config.py b/moto/iam/config.py index 4cd18bedc..fdf31b576 100644 --- a/moto/iam/config.py +++ b/moto/iam/config.py @@ -20,43 +20,59 @@ class RoleConfigQuery(ConfigQueryModel): # IAM roles are "global" and aren't assigned into any availability zone # The resource ID is a AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ" # The resource name is a user-assigned string like "MyDevelopmentAdminRole" + # Stored in moto backend with the AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ" - # Grab roles from backend - role_list = self.aggregate_regions("roles", "global", None) + # Grab roles from backend; need the full values since names and id's are different + role_list = list(self.backends["global"].roles.values()) if not role_list: return [], None - # Pagination logic - sorted_roles = sorted(role_list) + # Filter by resource name or ids + if resource_name or resource_ids: + filtered_roles = [] + # resource_name takes precendence over resource_ids + if resource_name: + for role in role_list: + if role.name == resource_name: + filtered_roles = [role] + break + else: + for role in role_list: + if role.id in resource_ids: + filtered_roles.append(role) + + # Filtered roles are now the subject for the listing + role_list = filtered_roles + + # Pagination logic, sort by role id + sorted_roles = sorted(role_list, key=lambda role: role.id) + # sorted_role_ids matches indicies of sorted_roles + sorted_role_ids = list(map(lambda role: role.id, sorted_roles)) new_token = None # Get the start: if not next_token: start = 0 else: - # "Tokens" are region + \x1e + resource ID. - if next_token not in sorted_roles: + if next_token not in sorted_role_ids: raise InvalidNextTokenException() - start = sorted_roles.index(next_token) + start = sorted_role_ids.index(next_token) # Get the list of items to collect: role_list = sorted_roles[start : (start + limit)] if len(sorted_roles) > (start + limit): - new_token = sorted_roles[start + limit] + new_token = sorted_role_ids[start + limit] - # Each element is a string of "region\x1eresource_id" return ( [ { "type": "AWS::IAM::Role", - "id": role.split(CONFIG_BACKEND_DELIM)[1], - "name": self.backends["global"] - .roles[role.split(CONFIG_BACKEND_DELIM)[1]] - .name, - "region": role.split(CONFIG_BACKEND_DELIM)[0], + "id": role.id, + "name": role.name, + "region": "global", } for role in role_list ], @@ -102,52 +118,67 @@ class PolicyConfigQuery(ConfigQueryModel): # IAM policies are "global" and aren't assigned into any availability zone # The resource ID is a AWS-assigned random string like "ANPA0BSVNSZK00SJSPVUJ" # The resource name is a user-assigned string like "my-development-policy" + # Stored in moto backend with the arn like "arn:aws:iam::123456789012:policy/my-development-policy" + + policy_list = list(self.backends["global"].managed_policies.values()) # We don't want to include AWS Managed Policies. This technically needs to # respect the configuration recorder's 'includeGlobalResourceTypes' setting, # but it's default set be default, and moto's config doesn't yet support # custom configuration recorders, we'll just behave as default. policy_list = filter( - lambda policy: not policy.split(CONFIG_BACKEND_DELIM)[1].startswith( - "arn:aws:iam::aws" - ), - self.aggregate_regions("managed_policies", "global", None), + lambda policy: not policy.arn.startswith("arn:aws:iam::aws"), policy_list, ) if not policy_list: return [], None - # Pagination logic: - sorted_policies = sorted(policy_list) + # Filter by resource name or ids + if resource_name or resource_ids: + filtered_policies = [] + # resource_name takes precendence over resource_ids + if resource_name: + for policy in policy_list: + if policy.name == resource_name: + filtered_policies = [policy] + break + else: + for policy in policy_list: + if policy.id in resource_ids: + filtered_policies.append(policy) + + # Filtered roles are now the subject for the listing + policy_list = filtered_policies + + # Pagination logic, sort by role id + sorted_policies = sorted(policy_list, key=lambda role: role.id) + # sorted_policy_ids matches indicies of sorted_policies + sorted_policy_ids = list(map(lambda policy: policy.id, sorted_policies)) + new_token = None # Get the start: if not next_token: start = 0 else: - # "Tokens" are region + \x1e + resource ID. - if next_token not in sorted_policies: + if next_token not in sorted_policy_ids: raise InvalidNextTokenException() - start = sorted_policies.index(next_token) + start = sorted_policy_ids.index(next_token) # Get the list of items to collect: policy_list = sorted_policies[start : (start + limit)] if len(sorted_policies) > (start + limit): - new_token = sorted_policies[start + limit] + new_token = sorted_policy_ids[start + limit] return ( [ { "type": "AWS::IAM::Policy", - "id": self.backends["global"] - .managed_policies[policy.split(CONFIG_BACKEND_DELIM)[1]] - .id, - "name": self.backends["global"] - .managed_policies[policy.split(CONFIG_BACKEND_DELIM)[1]] - .name, - "region": policy.split(CONFIG_BACKEND_DELIM)[0], + "id": policy.id, + "name": policy.name, + "region": "global", } for policy in policy_list ], diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 944b14acd..f71b96925 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -3217,19 +3217,24 @@ def test_role_config_client(): result = config_client.list_discovered_resources(resourceType="AWS::IAM::Role") assert not result["resourceIdentifiers"] - role_id = iam_client.create_role( - Path="/", - RoleName="mytestrole", - Description="mytestrole", - AssumeRolePolicyDocument=json.dumps("{ }"), - )["Role"]["RoleId"] + # Make 10 policies + roles = [] + num_roles = 10 + for ix in range(1, num_roles + 1): + this_policy = iam_client.create_role( + RoleName="role{}".format(ix), + Path="/", + Description="role{}".format(ix), + AssumeRolePolicyDocument=json.dumps("{ }"), + ) + roles.append( + { + "id": this_policy["Role"]["RoleId"], + "name": this_policy["Role"]["RoleName"], + } + ) - iam_client.create_role( - Path="/", - RoleName="mytestrole2", - Description="zmytestrole", - AssumeRolePolicyDocument=json.dumps("{ }"), - ) + assert len(roles) == num_roles # Test non-aggregated query: (everything is getting a random id, so we can't test names by ordering) result = config_client.list_discovered_resources( @@ -3269,12 +3274,77 @@ def test_role_config_client(): != first_agg_result ) + # Test non-aggregated resource name/id filter + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", resourceName=roles[1]["name"], limit=1, + )["resourceIdentifiers"][0]["resourceName"] + == roles[1]["name"] + ) + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", resourceIds=[roles[0]["id"]], limit=1, + )["resourceIdentifiers"][0]["resourceName"] + == roles[0]["name"] + ) + + # Test aggregated resource name/id filter + assert ( + config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Role", + Filters={"ResourceName": roles[5]["name"]}, + Limit=1, + )["ResourceIdentifiers"][0]["ResourceName"] + == roles[5]["name"] + ) + + assert ( + config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Role", + Filters={"ResourceId": roles[4]["id"]}, + Limit=1, + )["ResourceIdentifiers"][0]["ResourceName"] + == roles[4]["name"] + ) + + # Test name/id filter with pagination + first_call = config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", + resourceIds=[roles[1]["id"], roles[2]["id"]], + limit=1, + ) + + assert first_call["nextToken"] in [roles[1]["id"], roles[2]["id"]] + assert first_call["resourceIdentifiers"][0]["resourceName"] in [ + roles[1]["name"], + roles[2]["name"], + ] + second_call = config_client.list_discovered_resources( + resourceType="AWS::IAM::Role", + resourceIds=[roles[1]["id"], roles[2]["id"]], + limit=1, + nextToken=first_call["nextToken"], + ) + assert "nextToken" not in second_call + assert first_call["resourceIdentifiers"][0]["resourceName"] in [ + roles[1]["name"], + roles[2]["name"], + ] + assert ( + first_call["resourceIdentifiers"][0]["resourceName"] + != second_call["resourceIdentifiers"][0]["resourceName"] + ) + # Test non-aggregated batch get assert ( config_client.batch_get_resource_config( - resourceKeys=[{"resourceType": "AWS::IAM::Role", "resourceId": role_id}] + resourceKeys=[ + {"resourceType": "AWS::IAM::Role", "resourceId": roles[0]["id"]} + ] )["baseConfigurationItems"][0]["resourceName"] - == "mytestrole" + == roles[0]["name"] ) # Test aggregated batch get @@ -3285,12 +3355,12 @@ def test_role_config_client(): { "SourceAccountId": ACCOUNT_ID, "SourceRegion": "global", - "ResourceId": role_id, + "ResourceId": roles[1]["id"], "ResourceType": "AWS::IAM::Role", } ], )["BaseConfigurationItems"][0]["resourceName"] - == "mytestrole" + == roles[1]["name"] ) @@ -3312,7 +3382,7 @@ def test_policy_list_config_discovered_resources(): ], } - # Create a role + # Create a policy policy_config_query.backends["global"].create_policy( description="mypolicy", path="", @@ -3320,6 +3390,12 @@ def test_policy_list_config_discovered_resources(): policy_name="mypolicy", ) + # We expect the backend to have arns as their keys + for backend_key in list( + policy_config_query.backends["global"].managed_policies.keys() + ): + assert backend_key.startswith("arn:aws:iam::") + result = policy_config_query.list_config_service_resources(None, None, 100, None)[0] assert len(result) == 1 @@ -3465,20 +3541,24 @@ def test_policy_config_client(): result = config_client.list_discovered_resources(resourceType="AWS::IAM::Policy") assert not result["resourceIdentifiers"] - policy_id = iam_client.create_policy( - PolicyName="mypolicy", - Path="/", - PolicyDocument=json.dumps(basic_policy), - Description="mypolicy", - )["Policy"]["PolicyId"] + # Make 10 policies + policies = [] + num_policies = 10 + for ix in range(1, num_policies + 1): + this_policy = iam_client.create_policy( + PolicyName="policy{}".format(ix), + Path="/", + PolicyDocument=json.dumps(basic_policy), + Description="policy{}".format(ix), + ) + policies.append( + { + "id": this_policy["Policy"]["PolicyId"], + "name": this_policy["Policy"]["PolicyName"], + } + ) - # second policy - iam_client.create_policy( - PolicyName="zmypolicy", - Path="/", - PolicyDocument=json.dumps(basic_policy), - Description="zmypolicy", - ) + assert len(policies) == num_policies # Test non-aggregated query: (everything is getting a random id, so we can't test names by ordering) result = config_client.list_discovered_resources( @@ -3518,12 +3598,77 @@ def test_policy_config_client(): != first_agg_result ) + # Test non-aggregated resource name/id filter + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Policy", resourceName=policies[1]["name"], limit=1, + )["resourceIdentifiers"][0]["resourceName"] + == policies[1]["name"] + ) + assert ( + config_client.list_discovered_resources( + resourceType="AWS::IAM::Policy", resourceIds=[policies[0]["id"]], limit=1, + )["resourceIdentifiers"][0]["resourceName"] + == policies[0]["name"] + ) + + # Test aggregated resource name/id filter + assert ( + config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Policy", + Filters={"ResourceName": policies[5]["name"]}, + Limit=1, + )["ResourceIdentifiers"][0]["ResourceName"] + == policies[5]["name"] + ) + + assert ( + config_client.list_aggregate_discovered_resources( + ConfigurationAggregatorName="test_aggregator", + ResourceType="AWS::IAM::Policy", + Filters={"ResourceId": policies[4]["id"]}, + Limit=1, + )["ResourceIdentifiers"][0]["ResourceName"] + == policies[4]["name"] + ) + + # Test name/id filter with pagination + first_call = config_client.list_discovered_resources( + resourceType="AWS::IAM::Policy", + resourceIds=[policies[1]["id"], policies[2]["id"]], + limit=1, + ) + + assert first_call["nextToken"] in [policies[1]["id"], policies[2]["id"]] + assert first_call["resourceIdentifiers"][0]["resourceName"] in [ + policies[1]["name"], + policies[2]["name"], + ] + second_call = config_client.list_discovered_resources( + resourceType="AWS::IAM::Policy", + resourceIds=[policies[1]["id"], policies[2]["id"]], + limit=1, + nextToken=first_call["nextToken"], + ) + assert "nextToken" not in second_call + assert first_call["resourceIdentifiers"][0]["resourceName"] in [ + policies[1]["name"], + policies[2]["name"], + ] + assert ( + first_call["resourceIdentifiers"][0]["resourceName"] + != second_call["resourceIdentifiers"][0]["resourceName"] + ) + # Test non-aggregated batch get assert ( config_client.batch_get_resource_config( - resourceKeys=[{"resourceType": "AWS::IAM::Policy", "resourceId": policy_id}] + resourceKeys=[ + {"resourceType": "AWS::IAM::Policy", "resourceId": policies[7]["id"]} + ] )["baseConfigurationItems"][0]["resourceName"] - == "mypolicy" + == policies[7]["name"] ) # Test aggregated batch get @@ -3534,10 +3679,10 @@ def test_policy_config_client(): { "SourceAccountId": ACCOUNT_ID, "SourceRegion": "global", - "ResourceId": policy_id, + "ResourceId": policies[8]["id"], "ResourceType": "AWS::IAM::Policy", } ], )["BaseConfigurationItems"][0]["resourceName"] - == "mypolicy" + == policies[8]["name"] )