Add url_bases index for faster backend lookup (#4209)

This commit is contained in:
Thomas Rausch 2021-09-08 16:06:43 +02:00 committed by GitHub
parent e6c6ce5942
commit e30169cfcb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 236 additions and 114 deletions

View File

@ -80,6 +80,9 @@ Select Operation: create_deployment
You will still need to add the mock into "__init__.py"
```
### URL Indexing
In order to speed up the performance of MotoServer, all AWS URL's that need intercepting are indexed.
When adding/replacing any URLs in `{service}/urls.py`, please run `python scripts/update_backend_index.py` to update this index.
## Maintainers

View File

@ -3,11 +3,15 @@ from __future__ import unicode_literals
import importlib
def lazy_load(module_name, element):
def lazy_load(module_name, element, boto3_name=None, backend=None):
def f(*args, **kwargs):
module = importlib.import_module(module_name, "moto")
return getattr(module, element)(*args, **kwargs)
setattr(f, "name", module_name.replace(".", ""))
setattr(f, "element", element)
setattr(f, "boto3_name", boto3_name or f.name)
setattr(f, "backend", backend or f"{f.name}_backends")
return f
@ -20,7 +24,9 @@ mock_applicationautoscaling = lazy_load(
)
mock_autoscaling = lazy_load(".autoscaling", "mock_autoscaling")
mock_autoscaling_deprecated = lazy_load(".autoscaling", "mock_autoscaling_deprecated")
mock_lambda = lazy_load(".awslambda", "mock_lambda")
mock_lambda = lazy_load(
".awslambda", "mock_lambda", boto3_name="lambda", backend="lambda_backends"
)
mock_lambda_deprecated = lazy_load(".awslambda", "mock_lambda_deprecated")
mock_batch = lazy_load(".batch", "mock_batch")
mock_batch = lazy_load(".batch", "mock_batch")
@ -32,11 +38,13 @@ mock_cloudwatch = lazy_load(".cloudwatch", "mock_cloudwatch")
mock_cloudwatch_deprecated = lazy_load(".cloudwatch", "mock_cloudwatch_deprecated")
mock_codecommit = lazy_load(".codecommit", "mock_codecommit")
mock_codepipeline = lazy_load(".codepipeline", "mock_codepipeline")
mock_cognitoidentity = lazy_load(".cognitoidentity", "mock_cognitoidentity")
mock_cognitoidentity = lazy_load(
".cognitoidentity", "mock_cognitoidentity", boto3_name="cognito-identity"
)
mock_cognitoidentity_deprecated = lazy_load(
".cognitoidentity", "mock_cognitoidentity_deprecated"
)
mock_cognitoidp = lazy_load(".cognitoidp", "mock_cognitoidp")
mock_cognitoidp = lazy_load(".cognitoidp", "mock_cognitoidp", boto3_name="cognito-idp")
mock_cognitoidp_deprecated = lazy_load(".cognitoidp", "mock_cognitoidp_deprecated")
mock_config = lazy_load(".config", "mock_config")
mock_datapipeline = lazy_load(".datapipeline", "mock_datapipeline")
@ -47,10 +55,12 @@ mock_datasync = lazy_load(".datasync", "mock_datasync")
mock_dms = lazy_load(".dms", "mock_dms")
mock_dynamodb = lazy_load(".dynamodb", "mock_dynamodb")
mock_dynamodb_deprecated = lazy_load(".dynamodb", "mock_dynamodb_deprecated")
mock_dynamodb2 = lazy_load(".dynamodb2", "mock_dynamodb2")
mock_dynamodb2 = lazy_load(".dynamodb2", "mock_dynamodb2", backend="dynamodb_backends2")
mock_dynamodb2_deprecated = lazy_load(".dynamodb2", "mock_dynamodb2_deprecated")
mock_dynamodbstreams = lazy_load(".dynamodbstreams", "mock_dynamodbstreams")
mock_elasticbeanstalk = lazy_load(".elasticbeanstalk", "mock_elasticbeanstalk")
mock_elasticbeanstalk = lazy_load(
".elasticbeanstalk", "mock_elasticbeanstalk", backend="eb_backends"
)
mock_ec2 = lazy_load(".ec2", "mock_ec2")
mock_ec2_deprecated = lazy_load(".ec2", "mock_ec2_deprecated")
mock_ec2instanceconnect = lazy_load(".ec2instanceconnect", "mock_ec2instanceconnect")
@ -72,7 +82,7 @@ mock_glue = lazy_load(".glue", "mock_glue")
mock_iam = lazy_load(".iam", "mock_iam")
mock_iam_deprecated = lazy_load(".iam", "mock_iam_deprecated")
mock_iot = lazy_load(".iot", "mock_iot")
mock_iotdata = lazy_load(".iotdata", "mock_iotdata")
mock_iotdata = lazy_load(".iotdata", "mock_iotdata", boto3_name="iot-data")
mock_kinesis = lazy_load(".kinesis", "mock_kinesis")
mock_kinesis_deprecated = lazy_load(".kinesis", "mock_kinesis_deprecated")
mock_kms = lazy_load(".kms", "mock_kms")
@ -87,11 +97,13 @@ mock_polly = lazy_load(".polly", "mock_polly")
mock_ram = lazy_load(".ram", "mock_ram")
mock_rds = lazy_load(".rds", "mock_rds")
mock_rds_deprecated = lazy_load(".rds", "mock_rds_deprecated")
mock_rds2 = lazy_load(".rds2", "mock_rds2")
mock_rds2 = lazy_load(".rds2", "mock_rds2", boto3_name="rds")
mock_rds2_deprecated = lazy_load(".rds2", "mock_rds2_deprecated")
mock_redshift = lazy_load(".redshift", "mock_redshift")
mock_redshift_deprecated = lazy_load(".redshift", "mock_redshift_deprecated")
mock_resourcegroups = lazy_load(".resourcegroups", "mock_resourcegroups")
mock_resourcegroups = lazy_load(
".resourcegroups", "mock_resourcegroups", boto3_name="resource-groups"
)
mock_resourcegroupstaggingapi = lazy_load(
".resourcegroupstaggingapi", "mock_resourcegroupstaggingapi"
)
@ -108,7 +120,9 @@ mock_sns_deprecated = lazy_load(".sns", "mock_sns_deprecated")
mock_sqs = lazy_load(".sqs", "mock_sqs")
mock_sqs_deprecated = lazy_load(".sqs", "mock_sqs_deprecated")
mock_ssm = lazy_load(".ssm", "mock_ssm")
mock_stepfunctions = lazy_load(".stepfunctions", "mock_stepfunctions")
mock_stepfunctions = lazy_load(
".stepfunctions", "mock_stepfunctions", backend="stepfunction_backends"
)
mock_sts = lazy_load(".sts", "mock_sts")
mock_sts_deprecated = lazy_load(".sts", "mock_sts_deprecated")
mock_swf = lazy_load(".swf", "mock_swf")
@ -119,7 +133,9 @@ mock_xray = lazy_load(".xray", "mock_xray")
mock_xray_client = lazy_load(".xray", "mock_xray_client")
mock_kinesisvideo = lazy_load(".kinesisvideo", "mock_kinesisvideo")
mock_kinesisvideoarchivedmedia = lazy_load(
".kinesisvideoarchivedmedia", "mock_kinesisvideoarchivedmedia"
".kinesisvideoarchivedmedia",
"mock_kinesisvideoarchivedmedia",
boto3_name="kinesis-video-archived-media",
)
mock_medialive = lazy_load(".medialive", "mock_medialive")
mock_support = lazy_load(".support", "mock_support")
@ -127,7 +143,9 @@ mock_mediaconnect = lazy_load(".mediaconnect", "mock_mediaconnect")
mock_mediapackage = lazy_load(".mediapackage", "mock_mediapackage")
mock_mediastore = lazy_load(".mediastore", "mock_mediastore")
mock_eks = lazy_load(".eks", "mock_eks")
mock_mediastoredata = lazy_load(".mediastoredata", "mock_mediastoredata")
mock_mediastoredata = lazy_load(
".mediastoredata", "mock_mediastoredata", boto3_name="mediastore-data"
)
mock_efs = lazy_load(".efs", "mock_efs")
mock_wafv2 = lazy_load(".wafv2", "mock_wafv2")

115
moto/backend_index.py Normal file
View File

@ -0,0 +1,115 @@
# autogenerated by ./scripts/update_backend_index.py
import re
backend_url_patterns = [
("acm", re.compile("https?://acm.(.+).amazonaws.com")),
("apigateway", re.compile("https?://apigateway.(.+).amazonaws.com")),
("athena", re.compile("https?://athena.(.+).amazonaws.com")),
(
"applicationautoscaling",
re.compile("https?://application-autoscaling.(.+).amazonaws.com"),
),
("autoscaling", re.compile("https?://autoscaling.(.+).amazonaws.com")),
("batch", re.compile("https?://batch.(.+).amazonaws.com")),
("cloudformation", re.compile("https?://cloudformation.(.+).amazonaws.com")),
("cloudwatch", re.compile("https?://monitoring.(.+).amazonaws.com")),
("codecommit", re.compile("https?://codecommit.(.+).amazonaws.com")),
("codepipeline", re.compile("https?://codepipeline.(.+).amazonaws.com")),
("cognito-identity", re.compile("https?://cognito-identity.(.+).amazonaws.com")),
("cognito-idp", re.compile("https?://cognito-idp.(.+).amazonaws.com")),
("config", re.compile("https?://config.(.+).amazonaws.com")),
("datapipeline", re.compile("https?://datapipeline.(.+).amazonaws.com")),
("datasync", re.compile("https?://(.*?)(datasync)(.*?).amazonaws.com")),
("dms", re.compile("https?://dms.(.+).amazonaws.com")),
("dynamodb", re.compile("https?://dynamodb.(.+).amazonaws.com")),
("dynamodb2", re.compile("https?://dynamodb.(.+).amazonaws.com")),
("dynamodbstreams", re.compile("https?://streams.dynamodb.(.+).amazonaws.com")),
("ec2", re.compile("https?://ec2\\.(.+)\\.amazonaws\\.com(|\\.cn)")),
(
"ec2instanceconnect",
re.compile("https?://ec2-instance-connect\\.(.+)\\.amazonaws\\.com"),
),
("ecr", re.compile("https?://ecr.(.+).amazonaws.com")),
("ecr", re.compile("https?://api.ecr.(.+).amazonaws.com")),
("ecs", re.compile("https?://ecs.(.+).amazonaws.com")),
(
"elasticbeanstalk",
re.compile(
"https?://elasticbeanstalk.(?P<region>[a-zA-Z0-9\\-_]+).amazonaws.com"
),
),
("elb", re.compile("https?://elasticloadbalancing.(.+).amazonaws.com")),
("elbv2", re.compile("https?://elasticloadbalancing.(.+).amazonaws.com")),
("emr", re.compile("https?://(.+).elasticmapreduce.amazonaws.com")),
("emr", re.compile("https?://elasticmapreduce.(.+).amazonaws.com")),
("events", re.compile("https?://events.(.+).amazonaws.com")),
("glacier", re.compile("https?://glacier.(.+).amazonaws.com")),
("glue", re.compile("https?://glue(.*).amazonaws.com")),
("iam", re.compile("https?://iam(.*).amazonaws.com")),
("instance_metadata", re.compile("http://169.254.169.254")),
("iot", re.compile("https?://iot.(.+).amazonaws.com")),
("iot-data", re.compile("https?://data.iot.(.+).amazonaws.com")),
("kinesis", re.compile("https?://kinesis\\.(.+).amazonaws.com")),
("kinesis", re.compile("https?://firehose.(.+).amazonaws.com")),
("kms", re.compile("https?://kms.(.+).amazonaws.com")),
("lambda", re.compile("https?://lambda.(.+).amazonaws.com(|.cn)")),
("logs", re.compile("https?://logs.(.+).amazonaws.com")),
("managedblockchain", re.compile("https?://managedblockchain.(.+).amazonaws.com")),
("moto_api", re.compile("https?://motoapi.amazonaws.com")),
("opsworks", re.compile("https?://opsworks.us-east-1.amazonaws.com")),
("organizations", re.compile("https?://organizations.(.+).amazonaws.com")),
("polly", re.compile("https?://polly.(.+).amazonaws.com")),
("ram", re.compile("https?://ram.(.+).amazonaws.com")),
("rds", re.compile("https?://rds.(.+).amazonaws.com")),
("rds", re.compile("https?://rds.amazonaws.com")),
("redshift", re.compile("https?://redshift.(.+).amazonaws.com")),
(
"resource-groups",
re.compile("https?://resource-groups(-fips)?.(.+).amazonaws.com"),
),
("resourcegroupstaggingapi", re.compile("https?://tagging.(.+).amazonaws.com")),
("route53", re.compile("https?://route53(.*).amazonaws.com")),
("s3", re.compile("https?://s3(.*).amazonaws.com")),
(
"s3",
re.compile(
"https?://(?P<bucket_name>[a-zA-Z0-9\\-_.]*)\\.?s3(.*).amazonaws.com"
),
),
("s3bucket_path", re.compile("https?://s3(.*).amazonaws.com")),
(
"s3bucket_path",
re.compile(
"https?://(?P<bucket_name>[a-zA-Z0-9\\-_.]*)\\.?s3(.*).amazonaws.com"
),
),
("sagemaker", re.compile("https?://api.sagemaker.(.+).amazonaws.com")),
("secretsmanager", re.compile("https?://secretsmanager.(.+).amazonaws.com")),
("ses", re.compile("https?://email.(.+).amazonaws.com")),
("ses", re.compile("https?://ses.(.+).amazonaws.com")),
("sns", re.compile("https?://sns.(.+).amazonaws.com")),
("sqs", re.compile("https?://(.*?)(queue|sqs)(.*?).amazonaws.com")),
("ssm", re.compile("https?://ssm.(.+).amazonaws.com")),
("ssm", re.compile("https?://ssm.(.+).amazonaws.com.cn")),
("stepfunctions", re.compile("https?://states.(.+).amazonaws.com")),
("sts", re.compile("https?://sts(.*).amazonaws.com(|.cn)")),
("swf", re.compile("https?://swf.(.+).amazonaws.com")),
("transcribe", re.compile("https?://transcribe.(.+).amazonaws.com")),
("xray", re.compile("https?://xray.(.+).amazonaws.com")),
("kinesisvideo", re.compile("https?://kinesisvideo.(.+).amazonaws.com")),
("medialive", re.compile("https?://medialive.(.+).amazonaws.com")),
(
"kinesis-video-archived-media",
re.compile("https?://.*\\.kinesisvideo.(.+).amazonaws.com"),
),
("forecast", re.compile("https?://forecast.(.+).amazonaws.com")),
("support", re.compile("https?://support.(.+).amazonaws.com")),
("mediaconnect", re.compile("https?://mediaconnect.(.+).amazonaws.com")),
("mediapackage", re.compile("https?://mediapackage.(.+).amazonaws.com")),
("mediastore", re.compile("https?://mediastore.(.+).amazonaws.com")),
("mediastore-data", re.compile("https?://data.mediastore.(.+).amazonaws.com")),
("eks", re.compile("https?://eks.(.+).amazonaws.com")),
("efs", re.compile("https?://elasticfilesystem.(.+).amazonaws.com")),
("efs", re.compile("https?://elasticfilesystem.amazonaws.com")),
("wafv2", re.compile("https?://wafv2.(.+).amazonaws.com")),
]

View File

@ -1,93 +1,19 @@
from __future__ import unicode_literals
import importlib
import moto
BACKENDS = {
"acm": ("acm", "acm_backends"),
"apigateway": ("apigateway", "apigateway_backends"),
"athena": ("athena", "athena_backends"),
"applicationautoscaling": (
"applicationautoscaling",
"applicationautoscaling_backends",
),
"autoscaling": ("autoscaling", "autoscaling_backends"),
"batch": ("batch", "batch_backends"),
"cloudformation": ("cloudformation", "cloudformation_backends"),
"cloudwatch": ("cloudwatch", "cloudwatch_backends"),
"codecommit": ("codecommit", "codecommit_backends"),
"codepipeline": ("codepipeline", "codepipeline_backends"),
"cognito-identity": ("cognitoidentity", "cognitoidentity_backends"),
"cognito-idp": ("cognitoidp", "cognitoidp_backends"),
"config": ("config", "config_backends"),
"datapipeline": ("datapipeline", "datapipeline_backends"),
"datasync": ("datasync", "datasync_backends"),
"dms": ("dms", "dms_backends"),
"dynamodb": ("dynamodb", "dynamodb_backends"),
"dynamodb2": ("dynamodb2", "dynamodb_backends2"),
"dynamodbstreams": ("dynamodbstreams", "dynamodbstreams_backends"),
"ec2": ("ec2", "ec2_backends"),
"ec2instanceconnect": ("ec2instanceconnect", "ec2instanceconnect_backends"),
"ecr": ("ecr", "ecr_backends"),
"ecs": ("ecs", "ecs_backends"),
"elasticbeanstalk": ("elasticbeanstalk", "eb_backends"),
"elastictranscoder": ("elastictranscoder", "elastictranscoder_backends"),
"elb": ("elb", "elb_backends"),
"elbv2": ("elbv2", "elbv2_backends"),
"emr": ("emr", "emr_backends"),
"events": ("events", "events_backends"),
"glacier": ("glacier", "glacier_backends"),
"glue": ("glue", "glue_backends"),
"iam": ("iam", "iam_backends"),
"instance_metadata": ("instance_metadata", "instance_metadata_backends"),
"iot": ("iot", "iot_backends"),
"iot-data": ("iotdata", "iotdata_backends"),
"kinesis": ("kinesis", "kinesis_backends"),
"kms": ("kms", "kms_backends"),
"lambda": ("awslambda", "lambda_backends"),
"logs": ("logs", "logs_backends"),
"managedblockchain": ("managedblockchain", "managedblockchain_backends"),
"moto_api": ("core", "moto_api_backends"),
"opsworks": ("opsworks", "opsworks_backends"),
"organizations": ("organizations", "organizations_backends"),
"polly": ("polly", "polly_backends"),
"ram": ("ram", "ram_backends"),
"rds": ("rds2", "rds2_backends"),
"redshift": ("redshift", "redshift_backends"),
"resource-groups": ("resourcegroups", "resourcegroups_backends"),
"resourcegroupstaggingapi": (
"resourcegroupstaggingapi",
"resourcegroupstaggingapi_backends",
),
"route53": ("route53", "route53_backends"),
"s3": ("s3", "s3_backends"),
"s3bucket_path": ("s3", "s3_backends"),
"sagemaker": ("sagemaker", "sagemaker_backends"),
"secretsmanager": ("secretsmanager", "secretsmanager_backends"),
"ses": ("ses", "ses_backends"),
"sns": ("sns", "sns_backends"),
"sqs": ("sqs", "sqs_backends"),
"ssm": ("ssm", "ssm_backends"),
"stepfunctions": ("stepfunctions", "stepfunction_backends"),
"sts": ("sts", "sts_backends"),
"swf": ("swf", "swf_backends"),
"transcribe": ("transcribe", "transcribe_backends"),
"xray": ("xray", "xray_backends"),
"kinesisvideo": ("kinesisvideo", "kinesisvideo_backends"),
"medialive": ("medialive", "medialive_backends"),
"kinesis-video-archived-media": (
"kinesisvideoarchivedmedia",
"kinesisvideoarchivedmedia_backends",
),
"forecast": ("forecast", "forecast_backends"),
"support": ("support", "support_backends"),
"mediaconnect": ("mediaconnect", "mediaconnect_backends"),
"mediapackage": ("mediapackage", "mediapackage_backends"),
"mediastore": ("mediastore", "mediastore_backends"),
"mediastore-data": ("mediastoredata", "mediastoredata_backends"),
"eks": ("eks", "eks_backends"),
"efs": ("efs", "efs_backends"),
"wafv2": ("wafv2", "wafv2_backends"),
}
decorators = [
d
for d in dir(moto)
if d.startswith("mock_") and not d.endswith("_deprecated") and not d == "mock_all"
]
decorator_functions = [getattr(moto, f) for f in decorators]
BACKENDS = {f.boto3_name: (f.name, f.backend) for f in decorator_functions}
BACKENDS["moto_api"] = ("core", "moto_api_backends")
BACKENDS["instance_metadata"] = ("instance_metadata", "instance_metadata_backends")
BACKENDS["s3bucket_path"] = ("s3", "s3_backends")
def _import_backend(module_name, backends_name):
@ -110,12 +36,6 @@ def get_backend(name):
return _import_backend(module_name, backends_name)
def search_backend(predicate):
for name, backend in named_backends():
if predicate(backend):
return name
def get_model(name, region_name):
for backends_ in backends():
for region, backend in backends_.items():

View File

@ -4,7 +4,6 @@ import argparse
import io
import json
import os
import re
import signal
import sys
from threading import Lock
@ -18,9 +17,9 @@ from werkzeug.routing import BaseConverter
from werkzeug.serving import run_simple
import moto.backends as backends
import moto.backend_index as backend_index
from moto.core.utils import convert_flask_to_httpretty_response
HTTP_METHODS = ["GET", "POST", "PUT", "DELETE", "HEAD", "PATCH"]
@ -53,6 +52,7 @@ class DomainDispatcherApplication(object):
self.lock = Lock()
self.app_instances = {}
self.service = service
self.backend_url_patterns = backend_index.backend_url_patterns
def get_backend_for_host(self, host):
if host == "moto_api":
@ -64,11 +64,13 @@ class DomainDispatcherApplication(object):
if host in backends.BACKENDS:
return host
return backends.search_backend(
lambda backend: any(
re.match(url_base, "http://%s" % host)
for url_base in list(backend.values())[0].url_bases
)
for backend, pattern in self.backend_url_patterns:
if pattern.match("http://%s" % host):
return backend
print(
"Unable to find appropriate URL for {}."
"Remember to add the URL to urls.py, and run script/update_backend_index.py to index it."
)
def infer_service_region_host(self, environ):

View File

@ -128,10 +128,11 @@ def append_mock_to_init_py(service):
filtered_lines = [_ for _ in lines if re.match("^mock_.*lazy_load(.*)$", _)]
last_import_line_index = lines.index(filtered_lines[-1])
new_line = 'mock_{} = lazy_load(".{}", "mock_{}")'.format(
new_line = 'mock_{} = lazy_load(".{}", "mock_{}", boto3_name="{}")'.format(
get_escaped_service(service),
get_escaped_service(service),
get_escaped_service(service),
service
)
lines.insert(last_import_line_index + 1, new_line)
@ -163,7 +164,7 @@ def append_mock_dict_to_backends_py(service):
fhandle.write(body)
def initialize_service(service, api_protocol):
def initialize_service(service, operation, api_protocol):
"""create lib and test dirs if not exist"""
lib_dir = get_lib_dir(service)
test_dir = get_test_dir(service)
@ -210,7 +211,6 @@ def initialize_service(service, api_protocol):
# append mock to init files
append_mock_to_init_py(service)
append_mock_dict_to_backends_py(service)
def to_upper_camel_case(string):

64
scripts/update_backend_index.py Executable file
View File

@ -0,0 +1,64 @@
#!/usr/bin/env python
# This updates the cache used by the dispatcher to find backends.
import importlib
import os
import re
from pathlib import Path
import black
import pprint
import moto.backends as backends
output_file = "moto/backend_index.py"
script_dir = os.path.dirname(os.path.abspath(__file__))
output_path = os.path.join(script_dir, "..", output_file)
def iter_backend_url_patterns():
for backend, (module_name, _) in backends.BACKENDS.items():
# otherwise we need to import the module
url_module_name = f"moto.{module_name}.urls"
module = importlib.import_module(url_module_name)
for pattern in getattr(module, "url_bases"):
yield backend, pattern
def build_backend_url_pattern_index():
"""
Builds an index between an url pattern and the associated backend.
:rtype: List[Tuple[str, pattern]]
"""
index = list()
for backend, url_pattern in iter_backend_url_patterns():
index.append((backend, re.compile(url_pattern)))
return index
def main():
with open(output_path, "w") as fd:
fd.write("# autogenerated by %s\n" % __file__)
fd.write("import re\n")
print("build backend_url_patterns")
index = build_backend_url_pattern_index()
with open(output_path, "a") as fd:
fd.write("backend_url_patterns = ")
pprint.pprint(index, fd)
fd.write(os.linesep)
print("format with black")
black.format_file_in_place(
Path(output_path),
fast=False,
mode=black.FileMode(),
write_back=black.WriteBack.YES,
)
if __name__ == "__main__":
main()