Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Config: Add checks for valid html when pulling managed rules (#6849)

Browse Source
This commit is contained in:
kbalk 2023-09-26 04:44:46 -04:00 committed by GitHub
parent c1bbae3604
commit 178f2b8c03
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 143 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
moto/config/resources/aws_managed_rules.json
View File

@ -194,6 +194,18 @@
"Resource Types": "AWS::AppSync::GraphQLApi",
"Trigger type": "Periodic"
},
"APPSYNC_AUTHORIZATION_CHECK": {
"AWS Region": "All supported AWS regions except Asia Pacific (Jakarta), Africa (Cape Town), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [
{
"Name": "AllowedAuthorizationTypes",
"Optional": false,
"Type": "CSV"
}
],
"Resource Types": "AWS::AppSync::GraphQLApi",
"Trigger type": "Configuration changes"
},
"APPSYNC_CACHE_ENCRYPTION_AT_REST": {
"AWS Region": "All supported AWS regions except China (Beijing), Asia Pacific (Jakarta), Africa (Cape Town), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), China (Ningxia), Europe (Zurich) Region",
"Parameters": [],
@ -874,6 +886,12 @@
"Resource Types": "AWS::CodePipeline::Pipeline",
"Trigger type": "Configuration changes"
},
"CUSTOM_EVENTBUS_POLICY_ATTACHED": {
"AWS Region": "All supported AWS regions except Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::Events::EventBus",
"Trigger type": "Configuration changes"
},
"CUSTOM_SCHEMA_REGISTRY_POLICY_ATTACHED": {
"AWS Region": "All supported AWS regions except Middle East (Bahrain), China (Beijing), Asia Pacific (Jakarta), Africa (Cape Town), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Osaka), Asia Pacific (Melbourne), Europe (Milan), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), China (Ningxia), Europe (Zurich) Region",
"Parameters": [],
@ -962,11 +980,41 @@
"Resource Types": "AWS::EC2::Instance",
"Trigger type": "Configuration changes"
},
"DMS_AUTO_MINOR_VERSION_UPGRADE_CHECK": {
"AWS Region": "All supported AWS regions except Asia Pacific (Jakarta), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::DMS::ReplicationInstance",
"Trigger type": "Configuration changes"
},
"DMS_ENDPOINT_SSL_CONFIGURED": {
"AWS Region": "All supported AWS regions except Asia Pacific (Jakarta), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::DMS::Endpoint",
"Trigger type": "Configuration changes"
},
"DMS_REPLICATION_NOT_PUBLIC": {
"AWS Region": "All supported AWS regions except Asia Pacific (Jakarta), Africa (Cape Town), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), Europe (Milan), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [],
"Trigger type": "Periodic"
},
"DMS_REPLICATION_TASK_SOURCEDB_LOGGING": {
"AWS Region": "All supported AWS regions except Asia Pacific (Jakarta), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Osaka), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::DMS::ReplicationTask",
"Trigger type": "Configuration changes"
},
"DMS_REPLICATION_TASK_TARGETDB_LOGGING": {
"AWS Region": "All supported AWS regions except Asia Pacific (Jakarta), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Osaka), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::DMS::ReplicationTask",
"Trigger type": "Configuration changes"
},
"DOCDB_CLUSTER_AUDIT_LOGGING_ENABLED": {
"AWS Region": "Only available in Asia Pacific (Mumbai), Europe (Paris), US East (Ohio), Europe (Ireland), Europe (Frankfurt), South America (Sao Paulo), US East (N. Virginia), Asia Pacific (Seoul), Europe (London), Europe (Milan), Asia Pacific (Tokyo), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Canada (Central), China (Ningxia) Region",
"Parameters": [],
"Resource Types": "AWS::RDS::DBCluster",
"Trigger type": "Configuration changes"
},
"DOCDB_CLUSTER_BACKUP_RETENTION_CHECK": {
"AWS Region": "Only available in Asia Pacific (Mumbai), Europe (Paris), US East (Ohio), Europe (Ireland), Europe (Frankfurt), South America (Sao Paulo), US East (N. Virginia), Asia Pacific (Seoul), Europe (London), Europe (Milan), Asia Pacific (Tokyo), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Canada (Central), China (Ningxia) Region",
"Parameters": [
@ -979,6 +1027,12 @@
"Resource Types": "AWS::RDS::DBCluster",
"Trigger type": "Configuration changes"
},
"DOCDB_CLUSTER_DELETION_PROTECTION_ENABLED": {
"AWS Region": "Only available in Asia Pacific (Mumbai), Europe (Paris), US East (Ohio), Europe (Ireland), Europe (Frankfurt), South America (Sao Paulo), US East (N. Virginia), Asia Pacific (Seoul), Europe (London), Europe (Milan), Asia Pacific (Tokyo), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Canada (Central), China (Ningxia) Region",
"Parameters": [],
"Resource Types": "AWS::RDS::DBCluster",
"Trigger type": "Configuration changes"
},
"DOCDB_CLUSTER_ENCRYPTED": {
"AWS Region": "Only available in Asia Pacific (Mumbai), Europe (Paris), US East (Ohio), Europe (Ireland), Europe (Frankfurt), South America (Sao Paulo), US East (N. Virginia), Asia Pacific (Seoul), Europe (London), Europe (Milan), Asia Pacific (Tokyo), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Canada (Central), China (Ningxia) Region",
"Parameters": [
@ -991,6 +1045,12 @@
"Resource Types": "AWS::RDS::DBCluster",
"Trigger type": "Configuration changes"
},
"DOCDB_CLUSTER_SNAPSHOT_PUBLIC_PROHIBITED": {
"AWS Region": "Only available in Asia Pacific (Mumbai), Europe (Paris), US East (Ohio), Europe (Ireland), Europe (Frankfurt), South America (Sao Paulo), US East (N. Virginia), Asia Pacific (Seoul), Europe (London), Europe (Milan), Asia Pacific (Tokyo), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Canada (Central) Region",
"Parameters": [],
"Resource Types": "AWS::RDS::DBClusterSnapshot",
"Trigger type": "Configuration changes"
},
"DYNAMODB_AUTOSCALING_ENABLED": {
"AWS Region": "All supported AWS regions",
"Parameters": [
@ -2242,6 +2302,12 @@
"Resource Types": "AWS::FSx::FileSystem",
"Trigger type": "Periodic"
},
"GLOBAL_ENDPOINT_EVENT_REPLICATION_ENABLED": {
"AWS Region": "All supported AWS regions except Middle East (Bahrain), China (Beijing), Asia Pacific (Jakarta), Africa (Cape Town), Middle East (UAE), Asia Pacific (Hong Kong), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), Europe (Milan), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), China (Ningxia), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::Events::Endpoint",
"Trigger type": "Configuration changes"
},
"GUARDDUTY_ENABLED_CENTRALIZED": {
"AWS Region": "All supported AWS regions except Middle East (Bahrain), China (Beijing), Africa (Cape Town), Middle East (UAE), Asia Pacific (Osaka), Europe (Milan), Israel (Tel Aviv), Europe (Spain), China (Ningxia), Europe (Zurich) Region",
"Parameters": [
@ -2628,6 +2694,12 @@
"Parameters": [],
"Trigger type": "Periodic"
},
"MQ_ACTIVE_DEPLOYMENT_MODE": {
"AWS Region": "All supported AWS regions except Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::AmazonMQ::Broker",
"Trigger type": "Configuration changes"
},
"MQ_AUTOMATIC_MINOR_VERSION_UPGRADE_ENABLED": {
"AWS Region": "All supported AWS regions except China (Beijing), Asia Pacific (Jakarta), Africa (Cape Town), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), China (Ningxia), Europe (Zurich) Region",
"Parameters": [],
@ -2646,6 +2718,18 @@
"Resource Types": "AWS::AmazonMQ::Broker",
"Trigger type": "Periodic"
},
"MQ_RABBIT_DEPLOYMENT_MODE": {
"AWS Region": "All supported AWS regions except Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::AmazonMQ::Broker",
"Trigger type": "Configuration changes"
},
"MSK_IN_CLUSTER_NODE_REQUIRE_TLS": {
"AWS Region": "All supported AWS regions except Asia Pacific (Jakarta), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::MSK::Cluster",
"Trigger type": "Configuration changes"
},
"MULTI_REGION_CLOUD_TRAIL_ENABLED": {
"AWS Region": "All supported AWS regions except Middle East (UAE) Region",
"Parameters": [
@ -2743,6 +2827,12 @@
"Resource Types": "AWS::RDS::DBClusterSnapshot",
"Trigger type": "Configuration changes"
},
"NETFW_DELETION_PROTECTION_ENABLED": {
"AWS Region": "All supported AWS regions except China (Beijing), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), China (Ningxia), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::NetworkFirewall::Firewall",
"Trigger type": "Configuration changes"
},
"NETFW_LOGGING_ENABLED": {
"AWS Region": "All supported AWS regions except China (Beijing), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), China (Ningxia), Europe (Zurich) Region",
"Parameters": [
@ -2887,12 +2977,24 @@
"Resource Types": "AWS::OpenSearch::Domain",
"Trigger type": "Configuration changes"
},
"RDS_AURORA_MYSQL_AUDIT_LOGGING_ENABLED": {
"AWS Region": "All supported AWS regions except China (Beijing), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain) Region",
"Parameters": [],
"Resource Types": "AWS::RDS::DBCluster",
"Trigger type": "Configuration changes"
},
"RDS_AUTOMATIC_MINOR_VERSION_UPGRADE_ENABLED": {
"AWS Region": "All supported AWS regions except Asia Pacific (Osaka), Europe (Spain) Region",
"Parameters": [],
"Resource Types": "AWS::RDS::DBInstance",
"Trigger type": "Configuration changes"
},
"RDS_CLUSTER_AUTO_MINOR_VERSION_UPGRADE_ENABLE": {
"AWS Region": "All supported AWS regions except China (Beijing), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Osaka), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), US West (N. California), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [],
"Resource Types": "AWS::RDS::DBCluster",
"Trigger type": "Configuration changes"
},
"RDS_CLUSTER_DEFAULT_ADMIN_CHECK": {
"AWS Region": "All supported AWS regions except Middle East (Bahrain), China (Beijing), Asia Pacific (Jakarta), Middle East (UAE), South America (Sao Paulo), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), China (Ningxia), Europe (Zurich) Region",
"Parameters": [
@ -3353,6 +3455,12 @@
"Parameters": [],
"Trigger type": "Periodic"
},
"ROUTE53_QUERY_LOGGING_ENABLED": {
"AWS Region": "Only available in US East (N. Virginia) Region",
"Parameters": [],
"Resource Types": "AWS::Route53::HostedZone",
"Trigger type": "Configuration changes"
},
"S3_ACCOUNT_LEVEL_PUBLIC_ACCESS_BLOCKS": {
"AWS Region": "All supported AWS regions except Middle East (Bahrain), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Osaka), Asia Pacific (Melbourne), Europe (Milan), Israel (Tel Aviv), Europe (Spain), Europe (Zurich) Region",
"Parameters": [
@ -4103,6 +4211,12 @@
],
"Trigger type": "Periodic"
},
"WAFV2_RULEGROUP_LOGGING_ENABLED": {
"AWS Region": "All supported AWS regions except AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv) Region",
"Parameters": [],
"Resource Types": "AWS::WAFv2::RuleGroup",
"Trigger type": "Configuration changes"
},
"WAFV2_RULEGROUP_NOT_EMPTY": {
"AWS Region": "All supported AWS regions except China (Beijing), Asia Pacific (Jakarta), Middle East (UAE), Asia Pacific (Hyderabad), Asia Pacific (Melbourne), AWS GovCloud (US-East), AWS GovCloud (US-West), Israel (Tel Aviv), Europe (Spain), China (Ningxia), Europe (Zurich) Region",
"Parameters": [],

37
scripts/pull_down_aws_managed_rules.py
View File

@ -58,7 +58,7 @@ AWS_CONFIG_MANAGED_RULES_URL_START = (
LIST_OF_RULES_URL = "managed-rules-by-aws-config.html"
def extract_param_info(page_content):
def extract_param_info(page_content, rule_name):
"""Return dict containing parameter info extracted from page.
The info for all (not each) parameters is contained within a "dl" tag,
@ -70,12 +70,16 @@ def extract_param_info(page_content):
dl_tags = page_content.xpath('//div[@class="variablelist"]//dl')
if len(dl_tags) > 1:
print(
f"ERROR: Found {len(dl_tags)} 'dl' tags for parameters; "
"only expecting one. Ignoring extra 'dl' tag.",
file=sys.stderr
f"ERROR: '{rule_name}' has {len(dl_tags)} 'dl' tags for "
"rule parameters; only expecting one.",
file=sys.stderr,
)
sys.exit(1)
dt_tags = dl_tags[0].xpath(".//dt")
if len(dt_tags) < 1:
print(f"ERROR: '{rule_name}' has no rule parameters.", file=sys.stderr)
sys.exit(1)
all_params = []
param_details = {}
@ -110,7 +114,7 @@ def extract_param_info(page_content):
return all_params
def extract_managed_rule_info(page_content):
def extract_managed_rule_info(page_content, rule_name):
"""Return dict of qualifiers/rules extracted from web page.
An example of the html that's being processed:
@ -141,6 +145,13 @@ def extract_managed_rule_info(page_content):
"""
rule_info = {}
paragraphs = page_content.xpath('//div[@id="main-content"]/descendant::p')
if len(paragraphs) < 1:
print(
f"ERROR: '{rule_name}' has no managed rule details (i.e., "
"Identifier, AWS Region).",
file=sys.stderr,
)
sys.exit(1)
for paragraph in paragraphs:
text = paragraph.text_content()
@ -155,7 +166,7 @@ def extract_managed_rule_info(page_content):
rule_info[parts[0]] = parts[1]
# The parameters are in their own "div", so handle them separately.
rule_info["Parameters"] = extract_param_info(page_content)
rule_info["Parameters"] = extract_param_info(page_content, rule_name)
return rule_info
@ -180,7 +191,16 @@ def main():
# Get the list of links for all the services.
page = requests.get(AWS_CONFIG_MANAGED_RULES_URL_START + LIST_OF_RULES_URL)
tree = html.fromstring(page.content)
links = [x.lstrip("./") for x in tree.xpath('//div[@class="highlights"]//ul//a/@href')]
links = [
x.lstrip("./") for x in tree.xpath('//div[@class="highlights"]//ul//a/@href')
]
if len(links) < 300:
print(
f"ERROR: Found {len(links)} links to managed rules, expected "
"over 300 links",
file=sys.stderr,
)
sys.exit(1)
# From each linked page, extract the id, region, trigger type and parameter
# information.
@ -189,7 +209,8 @@ def main():
if args.verbose:
print(f"Extracting from {link} ...")
page = requests.get(AWS_CONFIG_MANAGED_RULES_URL_START + link)
rules = extract_managed_rule_info(html.fromstring(page.content))
rule_name = link.rstrip(".html")
rules = extract_managed_rule_info(html.fromstring(page.content), rule_name)
rule_id = rules.pop("Identifier")
managed_rules["ManagedRules"][rule_id] = rules