Add url_bases index for faster backend lookup (#4209)

CONTRIBUTING.md

@@ -80,6 +80,9 @@ Select Operation: create_deployment
 You will still need to add the mock into "__init__.py"
 ```
 
+### URL Indexing
+In order to speed up the performance of MotoServer, all AWS URL's that need intercepting are indexed.  
+When adding/replacing any URLs in `{service}/urls.py`, please run `python scripts/update_backend_index.py` to update this index.
 
 ## Maintainers
 

moto/__init__.py

@@ -3,11 +3,15 @@ from __future__ import unicode_literals
 import importlib
 
 
-def lazy_load(module_name, element):
+def lazy_load(module_name, element, boto3_name=None, backend=None):
     def f(*args, **kwargs):
         module = importlib.import_module(module_name, "moto")
         return getattr(module, element)(*args, **kwargs)
 
+    setattr(f, "name", module_name.replace(".", ""))
+    setattr(f, "element", element)
+    setattr(f, "boto3_name", boto3_name or f.name)
+    setattr(f, "backend", backend or f"{f.name}_backends")
     return f
 
 
@@ -20,7 +24,9 @@ mock_applicationautoscaling = lazy_load(
 )
 mock_autoscaling = lazy_load(".autoscaling", "mock_autoscaling")
 mock_autoscaling_deprecated = lazy_load(".autoscaling", "mock_autoscaling_deprecated")
-mock_lambda = lazy_load(".awslambda", "mock_lambda")
+mock_lambda = lazy_load(
+    ".awslambda", "mock_lambda", boto3_name="lambda", backend="lambda_backends"
+)
 mock_lambda_deprecated = lazy_load(".awslambda", "mock_lambda_deprecated")
 mock_batch = lazy_load(".batch", "mock_batch")
 mock_batch = lazy_load(".batch", "mock_batch")
@@ -32,11 +38,13 @@ mock_cloudwatch = lazy_load(".cloudwatch", "mock_cloudwatch")
 mock_cloudwatch_deprecated = lazy_load(".cloudwatch", "mock_cloudwatch_deprecated")
 mock_codecommit = lazy_load(".codecommit", "mock_codecommit")
 mock_codepipeline = lazy_load(".codepipeline", "mock_codepipeline")
-mock_cognitoidentity = lazy_load(".cognitoidentity", "mock_cognitoidentity")
+mock_cognitoidentity = lazy_load(
+    ".cognitoidentity", "mock_cognitoidentity", boto3_name="cognito-identity"
+)
 mock_cognitoidentity_deprecated = lazy_load(
     ".cognitoidentity", "mock_cognitoidentity_deprecated"
 )
-mock_cognitoidp = lazy_load(".cognitoidp", "mock_cognitoidp")
+mock_cognitoidp = lazy_load(".cognitoidp", "mock_cognitoidp", boto3_name="cognito-idp")
 mock_cognitoidp_deprecated = lazy_load(".cognitoidp", "mock_cognitoidp_deprecated")
 mock_config = lazy_load(".config", "mock_config")
 mock_datapipeline = lazy_load(".datapipeline", "mock_datapipeline")
@@ -47,10 +55,12 @@ mock_datasync = lazy_load(".datasync", "mock_datasync")
 mock_dms = lazy_load(".dms", "mock_dms")
 mock_dynamodb = lazy_load(".dynamodb", "mock_dynamodb")
 mock_dynamodb_deprecated = lazy_load(".dynamodb", "mock_dynamodb_deprecated")
-mock_dynamodb2 = lazy_load(".dynamodb2", "mock_dynamodb2")
+mock_dynamodb2 = lazy_load(".dynamodb2", "mock_dynamodb2", backend="dynamodb_backends2")
 mock_dynamodb2_deprecated = lazy_load(".dynamodb2", "mock_dynamodb2_deprecated")
 mock_dynamodbstreams = lazy_load(".dynamodbstreams", "mock_dynamodbstreams")
-mock_elasticbeanstalk = lazy_load(".elasticbeanstalk", "mock_elasticbeanstalk")
+mock_elasticbeanstalk = lazy_load(
+    ".elasticbeanstalk", "mock_elasticbeanstalk", backend="eb_backends"
+)
 mock_ec2 = lazy_load(".ec2", "mock_ec2")
 mock_ec2_deprecated = lazy_load(".ec2", "mock_ec2_deprecated")
 mock_ec2instanceconnect = lazy_load(".ec2instanceconnect", "mock_ec2instanceconnect")
@@ -72,7 +82,7 @@ mock_glue = lazy_load(".glue", "mock_glue")
 mock_iam = lazy_load(".iam", "mock_iam")
 mock_iam_deprecated = lazy_load(".iam", "mock_iam_deprecated")
 mock_iot = lazy_load(".iot", "mock_iot")
-mock_iotdata = lazy_load(".iotdata", "mock_iotdata")
+mock_iotdata = lazy_load(".iotdata", "mock_iotdata", boto3_name="iot-data")
 mock_kinesis = lazy_load(".kinesis", "mock_kinesis")
 mock_kinesis_deprecated = lazy_load(".kinesis", "mock_kinesis_deprecated")
 mock_kms = lazy_load(".kms", "mock_kms")
@@ -87,11 +97,13 @@ mock_polly = lazy_load(".polly", "mock_polly")
 mock_ram = lazy_load(".ram", "mock_ram")
 mock_rds = lazy_load(".rds", "mock_rds")
 mock_rds_deprecated = lazy_load(".rds", "mock_rds_deprecated")
-mock_rds2 = lazy_load(".rds2", "mock_rds2")
+mock_rds2 = lazy_load(".rds2", "mock_rds2", boto3_name="rds")
 mock_rds2_deprecated = lazy_load(".rds2", "mock_rds2_deprecated")
 mock_redshift = lazy_load(".redshift", "mock_redshift")
 mock_redshift_deprecated = lazy_load(".redshift", "mock_redshift_deprecated")
-mock_resourcegroups = lazy_load(".resourcegroups", "mock_resourcegroups")
+mock_resourcegroups = lazy_load(
+    ".resourcegroups", "mock_resourcegroups", boto3_name="resource-groups"
+)
 mock_resourcegroupstaggingapi = lazy_load(
     ".resourcegroupstaggingapi", "mock_resourcegroupstaggingapi"
 )
@@ -108,7 +120,9 @@ mock_sns_deprecated = lazy_load(".sns", "mock_sns_deprecated")
 mock_sqs = lazy_load(".sqs", "mock_sqs")
 mock_sqs_deprecated = lazy_load(".sqs", "mock_sqs_deprecated")
 mock_ssm = lazy_load(".ssm", "mock_ssm")
-mock_stepfunctions = lazy_load(".stepfunctions", "mock_stepfunctions")
+mock_stepfunctions = lazy_load(
+    ".stepfunctions", "mock_stepfunctions", backend="stepfunction_backends"
+)
 mock_sts = lazy_load(".sts", "mock_sts")
 mock_sts_deprecated = lazy_load(".sts", "mock_sts_deprecated")
 mock_swf = lazy_load(".swf", "mock_swf")
@@ -119,7 +133,9 @@ mock_xray = lazy_load(".xray", "mock_xray")
 mock_xray_client = lazy_load(".xray", "mock_xray_client")
 mock_kinesisvideo = lazy_load(".kinesisvideo", "mock_kinesisvideo")
 mock_kinesisvideoarchivedmedia = lazy_load(
-    ".kinesisvideoarchivedmedia", "mock_kinesisvideoarchivedmedia"
+    ".kinesisvideoarchivedmedia",
+    "mock_kinesisvideoarchivedmedia",
+    boto3_name="kinesis-video-archived-media",
 )
 mock_medialive = lazy_load(".medialive", "mock_medialive")
 mock_support = lazy_load(".support", "mock_support")
@@ -127,7 +143,9 @@ mock_mediaconnect = lazy_load(".mediaconnect", "mock_mediaconnect")
 mock_mediapackage = lazy_load(".mediapackage", "mock_mediapackage")
 mock_mediastore = lazy_load(".mediastore", "mock_mediastore")
 mock_eks = lazy_load(".eks", "mock_eks")
-mock_mediastoredata = lazy_load(".mediastoredata", "mock_mediastoredata")
+mock_mediastoredata = lazy_load(
+    ".mediastoredata", "mock_mediastoredata", boto3_name="mediastore-data"
+)
 mock_efs = lazy_load(".efs", "mock_efs")
 mock_wafv2 = lazy_load(".wafv2", "mock_wafv2")
 

moto/backend_index.py

@@ -0,0 +1,115 @@
+# autogenerated by ./scripts/update_backend_index.py
+import re
+
+backend_url_patterns = [
+    ("acm", re.compile("https?://acm.(.+).amazonaws.com")),
+    ("apigateway", re.compile("https?://apigateway.(.+).amazonaws.com")),
+    ("athena", re.compile("https?://athena.(.+).amazonaws.com")),
+    (
+        "applicationautoscaling",
+        re.compile("https?://application-autoscaling.(.+).amazonaws.com"),
+    ),
+    ("autoscaling", re.compile("https?://autoscaling.(.+).amazonaws.com")),
+    ("batch", re.compile("https?://batch.(.+).amazonaws.com")),
+    ("cloudformation", re.compile("https?://cloudformation.(.+).amazonaws.com")),
+    ("cloudwatch", re.compile("https?://monitoring.(.+).amazonaws.com")),
+    ("codecommit", re.compile("https?://codecommit.(.+).amazonaws.com")),
+    ("codepipeline", re.compile("https?://codepipeline.(.+).amazonaws.com")),
+    ("cognito-identity", re.compile("https?://cognito-identity.(.+).amazonaws.com")),
+    ("cognito-idp", re.compile("https?://cognito-idp.(.+).amazonaws.com")),
+    ("config", re.compile("https?://config.(.+).amazonaws.com")),
+    ("datapipeline", re.compile("https?://datapipeline.(.+).amazonaws.com")),
+    ("datasync", re.compile("https?://(.*?)(datasync)(.*?).amazonaws.com")),
+    ("dms", re.compile("https?://dms.(.+).amazonaws.com")),
+    ("dynamodb", re.compile("https?://dynamodb.(.+).amazonaws.com")),
+    ("dynamodb2", re.compile("https?://dynamodb.(.+).amazonaws.com")),
+    ("dynamodbstreams", re.compile("https?://streams.dynamodb.(.+).amazonaws.com")),
+    ("ec2", re.compile("https?://ec2\\.(.+)\\.amazonaws\\.com(|\\.cn)")),
+    (
+        "ec2instanceconnect",
+        re.compile("https?://ec2-instance-connect\\.(.+)\\.amazonaws\\.com"),
+    ),
+    ("ecr", re.compile("https?://ecr.(.+).amazonaws.com")),
+    ("ecr", re.compile("https?://api.ecr.(.+).amazonaws.com")),
+    ("ecs", re.compile("https?://ecs.(.+).amazonaws.com")),
+    (
+        "elasticbeanstalk",
+        re.compile(
+            "https?://elasticbeanstalk.(?P<region>[a-zA-Z0-9\\-_]+).amazonaws.com"
+        ),
+    ),
+    ("elb", re.compile("https?://elasticloadbalancing.(.+).amazonaws.com")),
+    ("elbv2", re.compile("https?://elasticloadbalancing.(.+).amazonaws.com")),
+    ("emr", re.compile("https?://(.+).elasticmapreduce.amazonaws.com")),
+    ("emr", re.compile("https?://elasticmapreduce.(.+).amazonaws.com")),
+    ("events", re.compile("https?://events.(.+).amazonaws.com")),
+    ("glacier", re.compile("https?://glacier.(.+).amazonaws.com")),
+    ("glue", re.compile("https?://glue(.*).amazonaws.com")),
+    ("iam", re.compile("https?://iam(.*).amazonaws.com")),
+    ("instance_metadata", re.compile("http://169.254.169.254")),
+    ("iot", re.compile("https?://iot.(.+).amazonaws.com")),
+    ("iot-data", re.compile("https?://data.iot.(.+).amazonaws.com")),
+    ("kinesis", re.compile("https?://kinesis\\.(.+).amazonaws.com")),
+    ("kinesis", re.compile("https?://firehose.(.+).amazonaws.com")),
+    ("kms", re.compile("https?://kms.(.+).amazonaws.com")),
+    ("lambda", re.compile("https?://lambda.(.+).amazonaws.com(|.cn)")),
+    ("logs", re.compile("https?://logs.(.+).amazonaws.com")),
+    ("managedblockchain", re.compile("https?://managedblockchain.(.+).amazonaws.com")),
+    ("moto_api", re.compile("https?://motoapi.amazonaws.com")),
+    ("opsworks", re.compile("https?://opsworks.us-east-1.amazonaws.com")),
+    ("organizations", re.compile("https?://organizations.(.+).amazonaws.com")),
+    ("polly", re.compile("https?://polly.(.+).amazonaws.com")),
+    ("ram", re.compile("https?://ram.(.+).amazonaws.com")),
+    ("rds", re.compile("https?://rds.(.+).amazonaws.com")),
+    ("rds", re.compile("https?://rds.amazonaws.com")),
+    ("redshift", re.compile("https?://redshift.(.+).amazonaws.com")),
+    (
+        "resource-groups",
+        re.compile("https?://resource-groups(-fips)?.(.+).amazonaws.com"),
+    ),
+    ("resourcegroupstaggingapi", re.compile("https?://tagging.(.+).amazonaws.com")),
+    ("route53", re.compile("https?://route53(.*).amazonaws.com")),
+    ("s3", re.compile("https?://s3(.*).amazonaws.com")),
+    (
+        "s3",
+        re.compile(
+            "https?://(?P<bucket_name>[a-zA-Z0-9\\-_.]*)\\.?s3(.*).amazonaws.com"
+        ),
+    ),
+    ("s3bucket_path", re.compile("https?://s3(.*).amazonaws.com")),
+    (
+        "s3bucket_path",
+        re.compile(
+            "https?://(?P<bucket_name>[a-zA-Z0-9\\-_.]*)\\.?s3(.*).amazonaws.com"
+        ),
+    ),
+    ("sagemaker", re.compile("https?://api.sagemaker.(.+).amazonaws.com")),
+    ("secretsmanager", re.compile("https?://secretsmanager.(.+).amazonaws.com")),
+    ("ses", re.compile("https?://email.(.+).amazonaws.com")),
+    ("ses", re.compile("https?://ses.(.+).amazonaws.com")),
+    ("sns", re.compile("https?://sns.(.+).amazonaws.com")),
+    ("sqs", re.compile("https?://(.*?)(queue|sqs)(.*?).amazonaws.com")),
+    ("ssm", re.compile("https?://ssm.(.+).amazonaws.com")),
+    ("ssm", re.compile("https?://ssm.(.+).amazonaws.com.cn")),
+    ("stepfunctions", re.compile("https?://states.(.+).amazonaws.com")),
+    ("sts", re.compile("https?://sts(.*).amazonaws.com(|.cn)")),
+    ("swf", re.compile("https?://swf.(.+).amazonaws.com")),
+    ("transcribe", re.compile("https?://transcribe.(.+).amazonaws.com")),
+    ("xray", re.compile("https?://xray.(.+).amazonaws.com")),
+    ("kinesisvideo", re.compile("https?://kinesisvideo.(.+).amazonaws.com")),
+    ("medialive", re.compile("https?://medialive.(.+).amazonaws.com")),
+    (
+        "kinesis-video-archived-media",
+        re.compile("https?://.*\\.kinesisvideo.(.+).amazonaws.com"),
+    ),
+    ("forecast", re.compile("https?://forecast.(.+).amazonaws.com")),
+    ("support", re.compile("https?://support.(.+).amazonaws.com")),
+    ("mediaconnect", re.compile("https?://mediaconnect.(.+).amazonaws.com")),
+    ("mediapackage", re.compile("https?://mediapackage.(.+).amazonaws.com")),
+    ("mediastore", re.compile("https?://mediastore.(.+).amazonaws.com")),
+    ("mediastore-data", re.compile("https?://data.mediastore.(.+).amazonaws.com")),
+    ("eks", re.compile("https?://eks.(.+).amazonaws.com")),
+    ("efs", re.compile("https?://elasticfilesystem.(.+).amazonaws.com")),
+    ("efs", re.compile("https?://elasticfilesystem.amazonaws.com")),
+    ("wafv2", re.compile("https?://wafv2.(.+).amazonaws.com")),
+]

moto/backends.py

@@ -1,93 +1,19 @@
 from __future__ import unicode_literals
 
 import importlib
+import moto
 
-BACKENDS = {
+
-    "acm": ("acm", "acm_backends"),
+decorators = [
-    "apigateway": ("apigateway", "apigateway_backends"),
+    d
-    "athena": ("athena", "athena_backends"),
+    for d in dir(moto)
-    "applicationautoscaling": (
+    if d.startswith("mock_") and not d.endswith("_deprecated") and not d == "mock_all"
-        "applicationautoscaling",
+]
-        "applicationautoscaling_backends",
+decorator_functions = [getattr(moto, f) for f in decorators]
-    ),
+BACKENDS = {f.boto3_name: (f.name, f.backend) for f in decorator_functions}
-    "autoscaling": ("autoscaling", "autoscaling_backends"),
+BACKENDS["moto_api"] = ("core", "moto_api_backends")
-    "batch": ("batch", "batch_backends"),
+BACKENDS["instance_metadata"] = ("instance_metadata", "instance_metadata_backends")
-    "cloudformation": ("cloudformation", "cloudformation_backends"),
+BACKENDS["s3bucket_path"] = ("s3", "s3_backends")
-    "cloudwatch": ("cloudwatch", "cloudwatch_backends"),
-    "codecommit": ("codecommit", "codecommit_backends"),
-    "codepipeline": ("codepipeline", "codepipeline_backends"),
-    "cognito-identity": ("cognitoidentity", "cognitoidentity_backends"),
-    "cognito-idp": ("cognitoidp", "cognitoidp_backends"),
-    "config": ("config", "config_backends"),
-    "datapipeline": ("datapipeline", "datapipeline_backends"),
-    "datasync": ("datasync", "datasync_backends"),
-    "dms": ("dms", "dms_backends"),
-    "dynamodb": ("dynamodb", "dynamodb_backends"),
-    "dynamodb2": ("dynamodb2", "dynamodb_backends2"),
-    "dynamodbstreams": ("dynamodbstreams", "dynamodbstreams_backends"),
-    "ec2": ("ec2", "ec2_backends"),
-    "ec2instanceconnect": ("ec2instanceconnect", "ec2instanceconnect_backends"),
-    "ecr": ("ecr", "ecr_backends"),
-    "ecs": ("ecs", "ecs_backends"),
-    "elasticbeanstalk": ("elasticbeanstalk", "eb_backends"),
-    "elastictranscoder": ("elastictranscoder", "elastictranscoder_backends"),
-    "elb": ("elb", "elb_backends"),
-    "elbv2": ("elbv2", "elbv2_backends"),
-    "emr": ("emr", "emr_backends"),
-    "events": ("events", "events_backends"),
-    "glacier": ("glacier", "glacier_backends"),
-    "glue": ("glue", "glue_backends"),
-    "iam": ("iam", "iam_backends"),
-    "instance_metadata": ("instance_metadata", "instance_metadata_backends"),
-    "iot": ("iot", "iot_backends"),
-    "iot-data": ("iotdata", "iotdata_backends"),
-    "kinesis": ("kinesis", "kinesis_backends"),
-    "kms": ("kms", "kms_backends"),
-    "lambda": ("awslambda", "lambda_backends"),
-    "logs": ("logs", "logs_backends"),
-    "managedblockchain": ("managedblockchain", "managedblockchain_backends"),
-    "moto_api": ("core", "moto_api_backends"),
-    "opsworks": ("opsworks", "opsworks_backends"),
-    "organizations": ("organizations", "organizations_backends"),
-    "polly": ("polly", "polly_backends"),
-    "ram": ("ram", "ram_backends"),
-    "rds": ("rds2", "rds2_backends"),
-    "redshift": ("redshift", "redshift_backends"),
-    "resource-groups": ("resourcegroups", "resourcegroups_backends"),
-    "resourcegroupstaggingapi": (
-        "resourcegroupstaggingapi",
-        "resourcegroupstaggingapi_backends",
-    ),
-    "route53": ("route53", "route53_backends"),
-    "s3": ("s3", "s3_backends"),
-    "s3bucket_path": ("s3", "s3_backends"),
-    "sagemaker": ("sagemaker", "sagemaker_backends"),
-    "secretsmanager": ("secretsmanager", "secretsmanager_backends"),
-    "ses": ("ses", "ses_backends"),
-    "sns": ("sns", "sns_backends"),
-    "sqs": ("sqs", "sqs_backends"),
-    "ssm": ("ssm", "ssm_backends"),
-    "stepfunctions": ("stepfunctions", "stepfunction_backends"),
-    "sts": ("sts", "sts_backends"),
-    "swf": ("swf", "swf_backends"),
-    "transcribe": ("transcribe", "transcribe_backends"),
-    "xray": ("xray", "xray_backends"),
-    "kinesisvideo": ("kinesisvideo", "kinesisvideo_backends"),
-    "medialive": ("medialive", "medialive_backends"),
-    "kinesis-video-archived-media": (
-        "kinesisvideoarchivedmedia",
-        "kinesisvideoarchivedmedia_backends",
-    ),
-    "forecast": ("forecast", "forecast_backends"),
-    "support": ("support", "support_backends"),
-    "mediaconnect": ("mediaconnect", "mediaconnect_backends"),
-    "mediapackage": ("mediapackage", "mediapackage_backends"),
-    "mediastore": ("mediastore", "mediastore_backends"),
-    "mediastore-data": ("mediastoredata", "mediastoredata_backends"),
-    "eks": ("eks", "eks_backends"),
-    "efs": ("efs", "efs_backends"),
-    "wafv2": ("wafv2", "wafv2_backends"),
-}
 
 
 def _import_backend(module_name, backends_name):
@@ -110,12 +36,6 @@ def get_backend(name):
     return _import_backend(module_name, backends_name)
 
 
-def search_backend(predicate):
-    for name, backend in named_backends():
-        if predicate(backend):
-            return name
-
-
 def get_model(name, region_name):
     for backends_ in backends():
         for region, backend in backends_.items():

moto/server.py

@@ -4,7 +4,6 @@ import argparse
 import io
 import json
 import os
-import re
 import signal
 import sys
 from threading import Lock
@@ -18,9 +17,9 @@ from werkzeug.routing import BaseConverter
 from werkzeug.serving import run_simple
 
 import moto.backends as backends
+import moto.backend_index as backend_index
 from moto.core.utils import convert_flask_to_httpretty_response
 
-
 HTTP_METHODS = ["GET", "POST", "PUT", "DELETE", "HEAD", "PATCH"]
 
 
@@ -53,6 +52,7 @@ class DomainDispatcherApplication(object):
         self.lock = Lock()
         self.app_instances = {}
         self.service = service
+        self.backend_url_patterns = backend_index.backend_url_patterns
 
     def get_backend_for_host(self, host):
         if host == "moto_api":
@@ -64,11 +64,13 @@ class DomainDispatcherApplication(object):
         if host in backends.BACKENDS:
             return host
 
-        return backends.search_backend(
+        for backend, pattern in self.backend_url_patterns:
-            lambda backend: any(
+            if pattern.match("http://%s" % host):
-                re.match(url_base, "http://%s" % host)
+                return backend
-                for url_base in list(backend.values())[0].url_bases
+
-            )
+        print(
+            "Unable to find appropriate URL for {}."
+            "Remember to add the URL to urls.py, and run script/update_backend_index.py to index it."
         )
 
     def infer_service_region_host(self, environ):

scripts/scaffold.py

@@ -128,10 +128,11 @@ def append_mock_to_init_py(service):
     filtered_lines = [_ for _ in lines if re.match("^mock_.*lazy_load(.*)$", _)]
     last_import_line_index = lines.index(filtered_lines[-1])
 
-    new_line = 'mock_{} = lazy_load(".{}", "mock_{}")'.format(
+    new_line = 'mock_{} = lazy_load(".{}", "mock_{}", boto3_name="{}")'.format(
         get_escaped_service(service),
         get_escaped_service(service),
         get_escaped_service(service),
+        service
     )
     lines.insert(last_import_line_index + 1, new_line)
 
@@ -163,7 +164,7 @@ def append_mock_dict_to_backends_py(service):
         fhandle.write(body)
 
 
-def initialize_service(service, api_protocol):
+def initialize_service(service, operation, api_protocol):
     """create lib and test dirs if not exist"""
     lib_dir = get_lib_dir(service)
     test_dir = get_test_dir(service)
@@ -210,7 +211,6 @@ def initialize_service(service, api_protocol):
 
     # append mock to init files
     append_mock_to_init_py(service)
-    append_mock_dict_to_backends_py(service)
 
 
 def to_upper_camel_case(string):

scripts/update_backend_index.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# This updates the cache used by the dispatcher to find backends.
+import importlib
+import os
+import re
+from pathlib import Path
+
+import black
+import pprint
+
+import moto.backends as backends
+
+output_file = "moto/backend_index.py"
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+output_path = os.path.join(script_dir, "..", output_file)
+
+
+def iter_backend_url_patterns():
+    for backend, (module_name, _) in backends.BACKENDS.items():
+        # otherwise we need to import the module
+        url_module_name = f"moto.{module_name}.urls"
+        module = importlib.import_module(url_module_name)
+        for pattern in getattr(module, "url_bases"):
+            yield backend, pattern
+
+
+def build_backend_url_pattern_index():
+    """
+    Builds an index between an url pattern and the associated backend.
+
+    :rtype: List[Tuple[str, pattern]]
+    """
+    index = list()
+
+    for backend, url_pattern in iter_backend_url_patterns():
+        index.append((backend, re.compile(url_pattern)))
+
+    return index
+
+
+def main():
+    with open(output_path, "w") as fd:
+        fd.write("# autogenerated by %s\n" % __file__)
+        fd.write("import re\n")
+
+    print("build backend_url_patterns")
+    index = build_backend_url_pattern_index()
+    with open(output_path, "a") as fd:
+        fd.write("backend_url_patterns = ")
+        pprint.pprint(index, fd)
+        fd.write(os.linesep)
+
+    print("format with black")
+    black.format_file_in_place(
+        Path(output_path),
+        fast=False,
+        mode=black.FileMode(),
+        write_back=black.WriteBack.YES,
+    )
+
+
+if __name__ == "__main__":
+    main()