Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

straighten out filter logic

Browse Source
This commit is contained in:
Nick Stocchero 2020-08-06 17:18:57 -06:00
parent 8dd90db83c
commit d8cea0213d
3 changed files with 252 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
moto/core/models.py
View File

@ -768,16 +768,22 @@ class ConfigQueryModel(object):
def aggregate_regions(self, path, backend_region, resource_region):
"""
This method will is called for both aggregated and non-aggregated calls for config resources.
This method is called for both aggregated and non-aggregated calls for config resources.
It will figure out how to return the full list of resources for a given regional backend and append them to a final list.
It produces a list of both the region and the resource name with a delimiter character (CONFIG_BACKEND_DELIM, ASCII Record separator, \x1e).
IE: "us-east-1\x1ei-1234567800"
Each config-enabled resource has a method named `list_config_service_resources` which has to parse the delimiter
You should only use this method if you need to aggregate resources over more than one region.
If your region is global, just query the global backend directly in the `list_config_service_resources` method
If you use this method, your config-enabled resource must parse the delimited string in it's `list_config_service_resources` method.
...
:param path: - A dict accessor string applied to the backend that locates the resource.
:param backend_region:
:param resource_region:
:param path: - A dict accessor string applied to the backend that locates resources inside that backend.
           For example, if you passed path="keypairs", and you were working with an ec2 moto backend, it would yield the contents from
ec2_moto_backend[region].keypairs
:param backend_region: - Only used for filtering; A string representing the region IE: us-east-1
:param resource_region: - Only used for filtering; A string representing the region IE: us-east-1
:return: - Returns a list of "region\x1eresourcename" strings
"""

93
moto/iam/config.py
View File

@ -20,43 +20,59 @@ class RoleConfigQuery(ConfigQueryModel):
# IAM roles are "global" and aren't assigned into any availability zone
# The resource ID is a AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ"
# The resource name is a user-assigned string like "MyDevelopmentAdminRole"
# Stored in moto backend with the AWS-assigned random string like "AROA0BSVNSZKXVHS00SBJ"
# Grab roles from backend
role_list = self.aggregate_regions("roles", "global", None)
# Grab roles from backend; need the full values since names and id's are different
role_list = list(self.backends["global"].roles.values())
if not role_list:
return [], None
# Pagination logic
sorted_roles = sorted(role_list)
# Filter by resource name or ids
if resource_name or resource_ids:
filtered_roles = []
# resource_name takes precendence over resource_ids
if resource_name:
for role in role_list:
if role.name == resource_name:
filtered_roles = [role]
break
else:
for role in role_list:
if role.id in resource_ids:
filtered_roles.append(role)
# Filtered roles are now the subject for the listing
role_list = filtered_roles
# Pagination logic, sort by role id
sorted_roles = sorted(role_list, key=lambda role: role.id)
# sorted_role_ids matches indicies of sorted_roles
sorted_role_ids = list(map(lambda role: role.id, sorted_roles))
new_token = None
# Get the start:
if not next_token:
start = 0
else:
# "Tokens" are region + \x1e + resource ID.
if next_token not in sorted_roles:
if next_token not in sorted_role_ids:
raise InvalidNextTokenException()
start = sorted_roles.index(next_token)
start = sorted_role_ids.index(next_token)
# Get the list of items to collect:
role_list = sorted_roles[start : (start + limit)]
if len(sorted_roles) > (start + limit):
new_token = sorted_roles[start + limit]
new_token = sorted_role_ids[start + limit]
# Each element is a string of "region\x1eresource_id"
return (
[
{
"type": "AWS::IAM::Role",
"id": role.split(CONFIG_BACKEND_DELIM)[1],
"name": self.backends["global"]
.roles[role.split(CONFIG_BACKEND_DELIM)[1]]
.name,
"region": role.split(CONFIG_BACKEND_DELIM)[0],
"id": role.id,
"name": role.name,
"region": "global",
}
for role in role_list
],
@ -102,52 +118,67 @@ class PolicyConfigQuery(ConfigQueryModel):
# IAM policies are "global" and aren't assigned into any availability zone
# The resource ID is a AWS-assigned random string like "ANPA0BSVNSZK00SJSPVUJ"
# The resource name is a user-assigned string like "my-development-policy"
# Stored in moto backend with the arn like "arn:aws:iam::123456789012:policy/my-development-policy"
policy_list = list(self.backends["global"].managed_policies.values())
# We don't want to include AWS Managed Policies. This technically needs to
# respect the configuration recorder's 'includeGlobalResourceTypes' setting,
# but it's default set be default, and moto's config doesn't yet support
# custom configuration recorders, we'll just behave as default.
policy_list = filter(
lambda policy: not policy.split(CONFIG_BACKEND_DELIM)[1].startswith(
"arn:aws:iam::aws"
),
self.aggregate_regions("managed_policies", "global", None),
lambda policy: not policy.arn.startswith("arn:aws:iam::aws"), policy_list,
)
if not policy_list:
return [], None
# Pagination logic:
sorted_policies = sorted(policy_list)
# Filter by resource name or ids
if resource_name or resource_ids:
filtered_policies = []
# resource_name takes precendence over resource_ids
if resource_name:
for policy in policy_list:
if policy.name == resource_name:
filtered_policies = [policy]
break
else:
for policy in policy_list:
if policy.id in resource_ids:
filtered_policies.append(policy)
# Filtered roles are now the subject for the listing
policy_list = filtered_policies
# Pagination logic, sort by role id
sorted_policies = sorted(policy_list, key=lambda role: role.id)
# sorted_policy_ids matches indicies of sorted_policies
sorted_policy_ids = list(map(lambda policy: policy.id, sorted_policies))
new_token = None
# Get the start:
if not next_token:
start = 0
else:
# "Tokens" are region + \x1e + resource ID.
if next_token not in sorted_policies:
if next_token not in sorted_policy_ids:
raise InvalidNextTokenException()
start = sorted_policies.index(next_token)
start = sorted_policy_ids.index(next_token)
# Get the list of items to collect:
policy_list = sorted_policies[start : (start + limit)]
if len(sorted_policies) > (start + limit):
new_token = sorted_policies[start + limit]
new_token = sorted_policy_ids[start + limit]
return (
[
{
"type": "AWS::IAM::Policy",
"id": self.backends["global"]
.managed_policies[policy.split(CONFIG_BACKEND_DELIM)[1]]
.id,
"name": self.backends["global"]
.managed_policies[policy.split(CONFIG_BACKEND_DELIM)[1]]
.name,
"region": policy.split(CONFIG_BACKEND_DELIM)[0],
"id": policy.id,
"name": policy.name,
"region": "global",
}
for policy in policy_list
],

213
tests/test_iam/test_iam.py
View File

@ -3217,19 +3217,24 @@ def test_role_config_client():
result = config_client.list_discovered_resources(resourceType="AWS::IAM::Role")
assert not result["resourceIdentifiers"]
role_id = iam_client.create_role(
Path="/",
RoleName="mytestrole",
Description="mytestrole",
AssumeRolePolicyDocument=json.dumps("{ }"),
)["Role"]["RoleId"]
# Make 10 policies
roles = []
num_roles = 10
for ix in range(1, num_roles + 1):
this_policy = iam_client.create_role(
RoleName="role{}".format(ix),
Path="/",
Description="role{}".format(ix),
AssumeRolePolicyDocument=json.dumps("{ }"),
)
roles.append(
{
"id": this_policy["Role"]["RoleId"],
"name": this_policy["Role"]["RoleName"],
}
)
iam_client.create_role(
Path="/",
RoleName="mytestrole2",
Description="zmytestrole",
AssumeRolePolicyDocument=json.dumps("{ }"),
)
assert len(roles) == num_roles
# Test non-aggregated query: (everything is getting a random id, so we can't test names by ordering)
result = config_client.list_discovered_resources(
@ -3269,12 +3274,77 @@ def test_role_config_client():
!= first_agg_result
)
# Test non-aggregated resource name/id filter
assert (
config_client.list_discovered_resources(
resourceType="AWS::IAM::Role", resourceName=roles[1]["name"], limit=1,
)["resourceIdentifiers"][0]["resourceName"]
== roles[1]["name"]
)
assert (
config_client.list_discovered_resources(
resourceType="AWS::IAM::Role", resourceIds=[roles[0]["id"]], limit=1,
)["resourceIdentifiers"][0]["resourceName"]
== roles[0]["name"]
)
# Test aggregated resource name/id filter
assert (
config_client.list_aggregate_discovered_resources(
ConfigurationAggregatorName="test_aggregator",
ResourceType="AWS::IAM::Role",
Filters={"ResourceName": roles[5]["name"]},
Limit=1,
)["ResourceIdentifiers"][0]["ResourceName"]
== roles[5]["name"]
)
assert (
config_client.list_aggregate_discovered_resources(
ConfigurationAggregatorName="test_aggregator",
ResourceType="AWS::IAM::Role",
Filters={"ResourceId": roles[4]["id"]},
Limit=1,
)["ResourceIdentifiers"][0]["ResourceName"]
== roles[4]["name"]
)
# Test name/id filter with pagination
first_call = config_client.list_discovered_resources(
resourceType="AWS::IAM::Role",
resourceIds=[roles[1]["id"], roles[2]["id"]],
limit=1,
)
assert first_call["nextToken"] in [roles[1]["id"], roles[2]["id"]]
assert first_call["resourceIdentifiers"][0]["resourceName"] in [
roles[1]["name"],
roles[2]["name"],
]
second_call = config_client.list_discovered_resources(
resourceType="AWS::IAM::Role",
resourceIds=[roles[1]["id"], roles[2]["id"]],
limit=1,
nextToken=first_call["nextToken"],
)
assert "nextToken" not in second_call
assert first_call["resourceIdentifiers"][0]["resourceName"] in [
roles[1]["name"],
roles[2]["name"],
]
assert (
first_call["resourceIdentifiers"][0]["resourceName"]
!= second_call["resourceIdentifiers"][0]["resourceName"]
)
# Test non-aggregated batch get
assert (
config_client.batch_get_resource_config(
resourceKeys=[{"resourceType": "AWS::IAM::Role", "resourceId": role_id}]
resourceKeys=[
{"resourceType": "AWS::IAM::Role", "resourceId": roles[0]["id"]}
]
)["baseConfigurationItems"][0]["resourceName"]
== "mytestrole"
== roles[0]["name"]
)
# Test aggregated batch get
@ -3285,12 +3355,12 @@ def test_role_config_client():
{
"SourceAccountId": ACCOUNT_ID,
"SourceRegion": "global",
"ResourceId": role_id,
"ResourceId": roles[1]["id"],
"ResourceType": "AWS::IAM::Role",
}
],
)["BaseConfigurationItems"][0]["resourceName"]
== "mytestrole"
== roles[1]["name"]
)
@ -3312,7 +3382,7 @@ def test_policy_list_config_discovered_resources():
],
}
# Create a role
# Create a policy
policy_config_query.backends["global"].create_policy(
description="mypolicy",
path="",
@ -3320,6 +3390,12 @@ def test_policy_list_config_discovered_resources():
policy_name="mypolicy",
)
# We expect the backend to have arns as their keys
for backend_key in list(
policy_config_query.backends["global"].managed_policies.keys()
):
assert backend_key.startswith("arn:aws:iam::")
result = policy_config_query.list_config_service_resources(None, None, 100, None)[0]
assert len(result) == 1
@ -3465,20 +3541,24 @@ def test_policy_config_client():
result = config_client.list_discovered_resources(resourceType="AWS::IAM::Policy")
assert not result["resourceIdentifiers"]
policy_id = iam_client.create_policy(
PolicyName="mypolicy",
Path="/",
PolicyDocument=json.dumps(basic_policy),
Description="mypolicy",
)["Policy"]["PolicyId"]
# Make 10 policies
policies = []
num_policies = 10
for ix in range(1, num_policies + 1):
this_policy = iam_client.create_policy(
PolicyName="policy{}".format(ix),
Path="/",
PolicyDocument=json.dumps(basic_policy),
Description="policy{}".format(ix),
)
policies.append(
{
"id": this_policy["Policy"]["PolicyId"],
"name": this_policy["Policy"]["PolicyName"],
}
)
# second policy
iam_client.create_policy(
PolicyName="zmypolicy",
Path="/",
PolicyDocument=json.dumps(basic_policy),
Description="zmypolicy",
)
assert len(policies) == num_policies
# Test non-aggregated query: (everything is getting a random id, so we can't test names by ordering)
result = config_client.list_discovered_resources(
@ -3518,12 +3598,77 @@ def test_policy_config_client():
!= first_agg_result
)
# Test non-aggregated resource name/id filter
assert (
config_client.list_discovered_resources(
resourceType="AWS::IAM::Policy", resourceName=policies[1]["name"], limit=1,
)["resourceIdentifiers"][0]["resourceName"]
== policies[1]["name"]
)
assert (
config_client.list_discovered_resources(
resourceType="AWS::IAM::Policy", resourceIds=[policies[0]["id"]], limit=1,
)["resourceIdentifiers"][0]["resourceName"]
== policies[0]["name"]
)
# Test aggregated resource name/id filter
assert (
config_client.list_aggregate_discovered_resources(
ConfigurationAggregatorName="test_aggregator",
ResourceType="AWS::IAM::Policy",
Filters={"ResourceName": policies[5]["name"]},
Limit=1,
)["ResourceIdentifiers"][0]["ResourceName"]
== policies[5]["name"]
)
assert (
config_client.list_aggregate_discovered_resources(
ConfigurationAggregatorName="test_aggregator",
ResourceType="AWS::IAM::Policy",
Filters={"ResourceId": policies[4]["id"]},
Limit=1,
)["ResourceIdentifiers"][0]["ResourceName"]
== policies[4]["name"]
)
# Test name/id filter with pagination
first_call = config_client.list_discovered_resources(
resourceType="AWS::IAM::Policy",
resourceIds=[policies[1]["id"], policies[2]["id"]],
limit=1,
)
assert first_call["nextToken"] in [policies[1]["id"], policies[2]["id"]]
assert first_call["resourceIdentifiers"][0]["resourceName"] in [
policies[1]["name"],
policies[2]["name"],
]
second_call = config_client.list_discovered_resources(
resourceType="AWS::IAM::Policy",
resourceIds=[policies[1]["id"], policies[2]["id"]],
limit=1,
nextToken=first_call["nextToken"],
)
assert "nextToken" not in second_call
assert first_call["resourceIdentifiers"][0]["resourceName"] in [
policies[1]["name"],
policies[2]["name"],
]
assert (
first_call["resourceIdentifiers"][0]["resourceName"]
!= second_call["resourceIdentifiers"][0]["resourceName"]
)
# Test non-aggregated batch get
assert (
config_client.batch_get_resource_config(
resourceKeys=[{"resourceType": "AWS::IAM::Policy", "resourceId": policy_id}]
resourceKeys=[
{"resourceType": "AWS::IAM::Policy", "resourceId": policies[7]["id"]}
]
)["baseConfigurationItems"][0]["resourceName"]
== "mypolicy"
== policies[7]["name"]
)
# Test aggregated batch get
@ -3534,10 +3679,10 @@ def test_policy_config_client():
{
"SourceAccountId": ACCOUNT_ID,
"SourceRegion": "global",
"ResourceId": policy_id,
"ResourceId": policies[8]["id"],
"ResourceType": "AWS::IAM::Policy",
}
],
)["BaseConfigurationItems"][0]["resourceName"]
== "mypolicy"
== policies[8]["name"]
)