straighten out filter logic

moto/core/models.py

@@ -768,16 +768,22 @@ class ConfigQueryModel(object):
 
     def aggregate_regions(self, path, backend_region, resource_region):
         """
-        This method will is called for both aggregated and non-aggregated calls for config resources.
+        This method is called for both aggregated and non-aggregated calls for config resources.
+
         It will figure out how to return the full list of resources for a given regional backend and append them to a final list.
         It produces a list of both the region and the resource name with a delimiter character (CONFIG_BACKEND_DELIM, ASCII Record separator, \x1e).
         IE: "us-east-1\x1ei-1234567800"
 
-        Each config-enabled resource has a method named `list_config_service_resources` which has to parse the delimiter
+        You should only use this method if you need to aggregate resources over more than one region.
+        If your region is global, just query the global backend directly in the `list_config_service_resources` method
+
+        If you use this method, your config-enabled resource must parse the delimited string in it's `list_config_service_resources` method.
         ...
-        :param path: - A dict accessor string applied to the backend that locates the resource.
+        :param path: - A dict accessor string applied to the backend that locates resources inside that backend.
-        :param backend_region:
+                       For example, if you passed path="keypairs", and you were working with an ec2 moto backend, it would yield the contents from
-        :param resource_region:
+                       ec2_moto_backend[region].keypairs
+        :param backend_region: - Only used for filtering; A string representing the region IE: us-east-1
+        :param resource_region: - Only used for filtering; A string representing the region IE: us-east-1
         :return: - Returns a list of "region\x1eresourcename" strings
         """
 

moto/iam/config.py

@@ -20,43 +20,59 @@ class RoleConfigQuery(ConfigQueryModel):
         # IAM roles are "global" and aren't assigned into any availability zone
         # The resource ID is a AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ"
         # The resource name is a user-assigned string like "MyDevelopmentAdminRole"
+        # Stored in moto backend with the AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ"
 
-        # Grab roles from backend
+        # Grab roles from backend; need the full values since names and id's are different
-        role_list = self.aggregate_regions("roles", "global", None)
+        role_list = list(self.backends["global"].roles.values())
 
         if not role_list:
             return [], None
 
-        # Pagination logic
+        # Filter by resource name or ids
-        sorted_roles = sorted(role_list)
+        if resource_name or resource_ids:
+            filtered_roles = []
+            # resource_name takes precendence over resource_ids
+            if resource_name:
+                for role in role_list:
+                    if role.name == resource_name:
+                        filtered_roles = [role]
+                        break
+            else:
+                for role in role_list:
+                    if role.id in resource_ids:
+                        filtered_roles.append(role)
+
+            # Filtered roles are now the subject for the listing
+            role_list = filtered_roles
+
+        # Pagination logic, sort by role id
+        sorted_roles = sorted(role_list, key=lambda role: role.id)
+        # sorted_role_ids matches indicies of sorted_roles
+        sorted_role_ids = list(map(lambda role: role.id, sorted_roles))
         new_token = None
 
         # Get the start:
         if not next_token:
             start = 0
         else:
-            # "Tokens" are region + \x1e + resource ID.
+            if next_token not in sorted_role_ids:
-            if next_token not in sorted_roles:
                 raise InvalidNextTokenException()
 
-            start = sorted_roles.index(next_token)
+            start = sorted_role_ids.index(next_token)
 
         # Get the list of items to collect:
         role_list = sorted_roles[start : (start + limit)]
 
         if len(sorted_roles) > (start + limit):
-            new_token = sorted_roles[start + limit]
+            new_token = sorted_role_ids[start + limit]
 
-        # Each element is a string of "region\x1eresource_id"
         return (
             [
                 {
                     "type": "AWS::IAM::Role",
-                    "id": role.split(CONFIG_BACKEND_DELIM)[1],
+                    "id": role.id,
-                    "name": self.backends["global"]
+                    "name": role.name,
-                    .roles[role.split(CONFIG_BACKEND_DELIM)[1]]
+                    "region": "global",
-                    .name,
-                    "region": role.split(CONFIG_BACKEND_DELIM)[0],
                 }
                 for role in role_list
             ],
@@ -102,52 +118,67 @@ class PolicyConfigQuery(ConfigQueryModel):
         # IAM policies are "global" and aren't assigned into any availability zone
         # The resource ID is a AWS-assigned random string like "ANPA0BSVNSZK00SJSPVUJ"
         # The resource name is a user-assigned string like "my-development-policy"
+        # Stored in moto backend with the arn like "arn:aws:iam::123456789012:policy/my-development-policy"
+
+        policy_list = list(self.backends["global"].managed_policies.values())
 
         # We don't want to include AWS Managed Policies. This technically needs to
         # respect the configuration recorder's 'includeGlobalResourceTypes' setting,
         # but it's default set be default, and moto's config doesn't yet support
         # custom configuration recorders, we'll just behave as default.
         policy_list = filter(
-            lambda policy: not policy.split(CONFIG_BACKEND_DELIM)[1].startswith(
+            lambda policy: not policy.arn.startswith("arn:aws:iam::aws"), policy_list,
-                "arn:aws:iam::aws"
-            ),
-            self.aggregate_regions("managed_policies", "global", None),
         )
 
         if not policy_list:
             return [], None
 
-        # Pagination logic:
+        # Filter by resource name or ids
-        sorted_policies = sorted(policy_list)
+        if resource_name or resource_ids:
+            filtered_policies = []
+            # resource_name takes precendence over resource_ids
+            if resource_name:
+                for policy in policy_list:
+                    if policy.name == resource_name:
+                        filtered_policies = [policy]
+                        break
+            else:
+                for policy in policy_list:
+                    if policy.id in resource_ids:
+                        filtered_policies.append(policy)
+
+            # Filtered roles are now the subject for the listing
+            policy_list = filtered_policies
+
+        # Pagination logic, sort by role id
+        sorted_policies = sorted(policy_list, key=lambda role: role.id)
+        # sorted_policy_ids matches indicies of sorted_policies
+        sorted_policy_ids = list(map(lambda policy: policy.id, sorted_policies))
+
         new_token = None
 
         # Get the start:
         if not next_token:
             start = 0
         else:
-            # "Tokens" are region + \x1e + resource ID.
+            if next_token not in sorted_policy_ids:
-            if next_token not in sorted_policies:
                 raise InvalidNextTokenException()
 
-            start = sorted_policies.index(next_token)
+            start = sorted_policy_ids.index(next_token)
 
         # Get the list of items to collect:
         policy_list = sorted_policies[start : (start + limit)]
 
         if len(sorted_policies) > (start + limit):
-            new_token = sorted_policies[start + limit]
+            new_token = sorted_policy_ids[start + limit]
 
         return (
             [
                 {
                     "type": "AWS::IAM::Policy",
-                    "id": self.backends["global"]
+                    "id": policy.id,
-                    .managed_policies[policy.split(CONFIG_BACKEND_DELIM)[1]]
+                    "name": policy.name,
-                    .id,
+                    "region": "global",
-                    "name": self.backends["global"]
-                    .managed_policies[policy.split(CONFIG_BACKEND_DELIM)[1]]
-                    .name,
-                    "region": policy.split(CONFIG_BACKEND_DELIM)[0],
                 }
                 for policy in policy_list
             ],

tests/test_iam/test_iam.py

@@ -3217,19 +3217,24 @@ def test_role_config_client():
     result = config_client.list_discovered_resources(resourceType="AWS::IAM::Role")
     assert not result["resourceIdentifiers"]
 
-    role_id = iam_client.create_role(
+    # Make 10 policies
-        Path="/",
+    roles = []
-        RoleName="mytestrole",
+    num_roles = 10
-        Description="mytestrole",
+    for ix in range(1, num_roles + 1):
-        AssumeRolePolicyDocument=json.dumps("{ }"),
+        this_policy = iam_client.create_role(
-    )["Role"]["RoleId"]
+            RoleName="role{}".format(ix),
+            Path="/",
+            Description="role{}".format(ix),
+            AssumeRolePolicyDocument=json.dumps("{ }"),
+        )
+        roles.append(
+            {
+                "id": this_policy["Role"]["RoleId"],
+                "name": this_policy["Role"]["RoleName"],
+            }
+        )
 
-    iam_client.create_role(
+    assert len(roles) == num_roles
-        Path="/",
-        RoleName="mytestrole2",
-        Description="zmytestrole",
-        AssumeRolePolicyDocument=json.dumps("{ }"),
-    )
 
     # Test non-aggregated query: (everything is getting a random id, so we can't test names by ordering)
     result = config_client.list_discovered_resources(
@@ -3269,12 +3274,77 @@ def test_role_config_client():
         != first_agg_result
     )
 
+    # Test non-aggregated resource name/id filter
+    assert (
+        config_client.list_discovered_resources(
+            resourceType="AWS::IAM::Role", resourceName=roles[1]["name"], limit=1,
+        )["resourceIdentifiers"][0]["resourceName"]
+        == roles[1]["name"]
+    )
+    assert (
+        config_client.list_discovered_resources(
+            resourceType="AWS::IAM::Role", resourceIds=[roles[0]["id"]], limit=1,
+        )["resourceIdentifiers"][0]["resourceName"]
+        == roles[0]["name"]
+    )
+
+    # Test aggregated resource name/id filter
+    assert (
+        config_client.list_aggregate_discovered_resources(
+            ConfigurationAggregatorName="test_aggregator",
+            ResourceType="AWS::IAM::Role",
+            Filters={"ResourceName": roles[5]["name"]},
+            Limit=1,
+        )["ResourceIdentifiers"][0]["ResourceName"]
+        == roles[5]["name"]
+    )
+
+    assert (
+        config_client.list_aggregate_discovered_resources(
+            ConfigurationAggregatorName="test_aggregator",
+            ResourceType="AWS::IAM::Role",
+            Filters={"ResourceId": roles[4]["id"]},
+            Limit=1,
+        )["ResourceIdentifiers"][0]["ResourceName"]
+        == roles[4]["name"]
+    )
+
+    # Test name/id filter with pagination
+    first_call = config_client.list_discovered_resources(
+        resourceType="AWS::IAM::Role",
+        resourceIds=[roles[1]["id"], roles[2]["id"]],
+        limit=1,
+    )
+
+    assert first_call["nextToken"] in [roles[1]["id"], roles[2]["id"]]
+    assert first_call["resourceIdentifiers"][0]["resourceName"] in [
+        roles[1]["name"],
+        roles[2]["name"],
+    ]
+    second_call = config_client.list_discovered_resources(
+        resourceType="AWS::IAM::Role",
+        resourceIds=[roles[1]["id"], roles[2]["id"]],
+        limit=1,
+        nextToken=first_call["nextToken"],
+    )
+    assert "nextToken" not in second_call
+    assert first_call["resourceIdentifiers"][0]["resourceName"] in [
+        roles[1]["name"],
+        roles[2]["name"],
+    ]
+    assert (
+        first_call["resourceIdentifiers"][0]["resourceName"]
+        != second_call["resourceIdentifiers"][0]["resourceName"]
+    )
+
     # Test non-aggregated batch get
     assert (
         config_client.batch_get_resource_config(
-            resourceKeys=[{"resourceType": "AWS::IAM::Role", "resourceId": role_id}]
+            resourceKeys=[
+                {"resourceType": "AWS::IAM::Role", "resourceId": roles[0]["id"]}
+            ]
         )["baseConfigurationItems"][0]["resourceName"]
-        == "mytestrole"
+        == roles[0]["name"]
     )
 
     # Test aggregated batch get
@@ -3285,12 +3355,12 @@ def test_role_config_client():
                 {
                     "SourceAccountId": ACCOUNT_ID,
                     "SourceRegion": "global",
-                    "ResourceId": role_id,
+                    "ResourceId": roles[1]["id"],
                     "ResourceType": "AWS::IAM::Role",
                 }
             ],
         )["BaseConfigurationItems"][0]["resourceName"]
-        == "mytestrole"
+        == roles[1]["name"]
     )
 
 
@@ -3312,7 +3382,7 @@ def test_policy_list_config_discovered_resources():
         ],
     }
 
-    # Create a role
+    # Create a policy
     policy_config_query.backends["global"].create_policy(
         description="mypolicy",
         path="",
@@ -3320,6 +3390,12 @@ def test_policy_list_config_discovered_resources():
         policy_name="mypolicy",
     )
 
+    # We expect the backend to have arns as their keys
+    for backend_key in list(
+        policy_config_query.backends["global"].managed_policies.keys()
+    ):
+        assert backend_key.startswith("arn:aws:iam::")
+
     result = policy_config_query.list_config_service_resources(None, None, 100, None)[0]
     assert len(result) == 1
 
@@ -3465,20 +3541,24 @@ def test_policy_config_client():
     result = config_client.list_discovered_resources(resourceType="AWS::IAM::Policy")
     assert not result["resourceIdentifiers"]
 
-    policy_id = iam_client.create_policy(
+    # Make 10 policies
-        PolicyName="mypolicy",
+    policies = []
-        Path="/",
+    num_policies = 10
-        PolicyDocument=json.dumps(basic_policy),
+    for ix in range(1, num_policies + 1):
-        Description="mypolicy",
+        this_policy = iam_client.create_policy(
-    )["Policy"]["PolicyId"]
+            PolicyName="policy{}".format(ix),
+            Path="/",
+            PolicyDocument=json.dumps(basic_policy),
+            Description="policy{}".format(ix),
+        )
+        policies.append(
+            {
+                "id": this_policy["Policy"]["PolicyId"],
+                "name": this_policy["Policy"]["PolicyName"],
+            }
+        )
 
-    # second policy
+    assert len(policies) == num_policies
-    iam_client.create_policy(
-        PolicyName="zmypolicy",
-        Path="/",
-        PolicyDocument=json.dumps(basic_policy),
-        Description="zmypolicy",
-    )
 
     # Test non-aggregated query: (everything is getting a random id, so we can't test names by ordering)
     result = config_client.list_discovered_resources(
@@ -3518,12 +3598,77 @@ def test_policy_config_client():
         != first_agg_result
     )
 
+    # Test non-aggregated resource name/id filter
+    assert (
+        config_client.list_discovered_resources(
+            resourceType="AWS::IAM::Policy", resourceName=policies[1]["name"], limit=1,
+        )["resourceIdentifiers"][0]["resourceName"]
+        == policies[1]["name"]
+    )
+    assert (
+        config_client.list_discovered_resources(
+            resourceType="AWS::IAM::Policy", resourceIds=[policies[0]["id"]], limit=1,
+        )["resourceIdentifiers"][0]["resourceName"]
+        == policies[0]["name"]
+    )
+
+    # Test aggregated resource name/id filter
+    assert (
+        config_client.list_aggregate_discovered_resources(
+            ConfigurationAggregatorName="test_aggregator",
+            ResourceType="AWS::IAM::Policy",
+            Filters={"ResourceName": policies[5]["name"]},
+            Limit=1,
+        )["ResourceIdentifiers"][0]["ResourceName"]
+        == policies[5]["name"]
+    )
+
+    assert (
+        config_client.list_aggregate_discovered_resources(
+            ConfigurationAggregatorName="test_aggregator",
+            ResourceType="AWS::IAM::Policy",
+            Filters={"ResourceId": policies[4]["id"]},
+            Limit=1,
+        )["ResourceIdentifiers"][0]["ResourceName"]
+        == policies[4]["name"]
+    )
+
+    # Test name/id filter with pagination
+    first_call = config_client.list_discovered_resources(
+        resourceType="AWS::IAM::Policy",
+        resourceIds=[policies[1]["id"], policies[2]["id"]],
+        limit=1,
+    )
+
+    assert first_call["nextToken"] in [policies[1]["id"], policies[2]["id"]]
+    assert first_call["resourceIdentifiers"][0]["resourceName"] in [
+        policies[1]["name"],
+        policies[2]["name"],
+    ]
+    second_call = config_client.list_discovered_resources(
+        resourceType="AWS::IAM::Policy",
+        resourceIds=[policies[1]["id"], policies[2]["id"]],
+        limit=1,
+        nextToken=first_call["nextToken"],
+    )
+    assert "nextToken" not in second_call
+    assert first_call["resourceIdentifiers"][0]["resourceName"] in [
+        policies[1]["name"],
+        policies[2]["name"],
+    ]
+    assert (
+        first_call["resourceIdentifiers"][0]["resourceName"]
+        != second_call["resourceIdentifiers"][0]["resourceName"]
+    )
+
     # Test non-aggregated batch get
     assert (
         config_client.batch_get_resource_config(
-            resourceKeys=[{"resourceType": "AWS::IAM::Policy", "resourceId": policy_id}]
+            resourceKeys=[
+                {"resourceType": "AWS::IAM::Policy", "resourceId": policies[7]["id"]}
+            ]
         )["baseConfigurationItems"][0]["resourceName"]
-        == "mypolicy"
+        == policies[7]["name"]
     )
 
     # Test aggregated batch get
@@ -3534,10 +3679,10 @@ def test_policy_config_client():
                 {
                     "SourceAccountId": ACCOUNT_ID,
                     "SourceRegion": "global",
-                    "ResourceId": policy_id,
+                    "ResourceId": policies[8]["id"],
                     "ResourceType": "AWS::IAM::Policy",
                 }
             ],
         )["BaseConfigurationItems"][0]["resourceName"]
-        == "mypolicy"
+        == policies[8]["name"]
     )