Techdebt: Update scaffolding to use mock_aws (#7221)

Makefile

@@ -4,9 +4,8 @@ SERVICE_NAME = "default"
 TEST_NAMES = "*"
 
 ifeq ($(TEST_SERVER_MODE), true)
-	# exclude test_kinesisvideoarchivedmedia
+	# Exclude parallel tests
-	# because testing with moto_server is difficult with data-endpoint
+	TEST_EXCLUDE := --ignore tests/test_acm --ignore tests/test_amp --ignore tests/test_awslambda --ignore tests/test_batch --ignore tests/test_ec2 --ignore tests/test_sqs
-	TEST_EXCLUDE := --ignore tests/test_kinesisvideoarchivedmedia --ignore tests/test_acm --ignore tests/test_amp --ignore tests/test_awslambda --ignore tests/test_batch --ignore tests/test_ec2 --ignore tests/test_sqs
 	# Parallel tests will be run separate
 	PARALLEL_TESTS := ./tests/test_acm/ ./tests/test_acmpca/ ./tests/test_amp/ ./tests/test_awslambda ./tests/test_batch ./tests/test_ec2 ./tests/test_sqs
 else

scripts/scaffold.py

@@ -24,6 +24,7 @@ import random
 import re
 import inspect
 import importlib
+import subprocess
 from lxml import etree
 
 import click
@@ -47,6 +48,7 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "./template")
 INPUT_IGNORED_IN_BACKEND = ["Marker", "PageSize"]
 OUTPUT_IGNORED_IN_BACKEND = ["NextMarker"]
 
+root_dir = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode().strip()
 
 def print_progress(title, body, color):
     """Prints a color-code message describing current state of progress."""
@@ -143,29 +145,6 @@ def render_template(tmpl_dir, tmpl_filename, context, service, alt_filename=None
             fhandle.write(rendered)
 
 
-def append_mock_to_init_py(service):
-    """Update __init_.py to add line to load the mock service."""
-    path = os.path.join(os.path.dirname(__file__), "..", "moto", "__init__.py")
-    with open(path, encoding="utf-8") as fhandle:
-        lines = [_.replace("\n", "") for _ in fhandle.readlines()]
-
-    escaped_service = get_escaped_service(service)
-    if any(_ for _ in lines if _.startswith(f"^mock_{escaped_service} = lazy_load")):
-        return
-    filtered_lines = [_ for _ in lines if re.match("^mock_.*lazy_load(.*)$", _)]
-    last_import_line_index = lines.index(filtered_lines[-1])
-
-    new_line = (
-        f"mock_{escaped_service} = lazy_load("
-        f'".{escaped_service}", "mock_{escaped_service}", boto3_name="{service}")'
-    )
-    lines.insert(last_import_line_index + 1, new_line)
-
-    body = "\n".join(lines) + "\n"
-    with open(path, "w", encoding="utf-8") as fhandle:
-        fhandle.write(body)
-
-
 def initialize_service(service, api_protocol):
     """Create lib and test dirs if they don't exist."""
     lib_dir = get_lib_dir(service)
@@ -213,8 +192,6 @@ def initialize_service(service, api_protocol):
             else None
         )
         render_template(tmpl_dir, tmpl_filename, tmpl_context, service, alt_filename)
-    # append mock to initi files
-    append_mock_to_init_py(service)
 
 
 def to_upper_camel_case(string):
@@ -340,7 +317,7 @@ def get_func_in_tests(service, operation):
     escaped_service = get_escaped_service(service)
     random_region = random.choice(["us-east-2", "eu-west-1", "ap-southeast-1"])
     body = "\n\n"
-    body += f"@mock_{escaped_service}\n"
+    body += f"@mock_aws\n"
     body += f"def test_{operation}():\n"
     body += f"    client = boto3.client(\"{service}\", region_name=\"{random_region}\")\n"
     body += f"    resp = client.{operation}()\n"
@@ -593,7 +570,7 @@ def insert_url(service, operation, api_protocol):  # pylint: disable=too-many-lo
 
     # generate url pattern
     if api_protocol == "rest-json":
-        new_line = '    "{0}/.*$": response.dispatch,'
+        new_line = f'    "{0}/.*$": {service_class}Response.dispatch,'
     elif api_protocol == "rest-xml":
         new_line = f'    "{{0}}{uri}$": {service_class}Response.{operation},'
     else:
@@ -674,13 +651,15 @@ def main():
                 "yellow",
             )
 
+        click.echo("Updating backend index...")
+        subprocess.check_output([f"{root_dir}/scripts/update_backend_index.py"]).decode().strip()
+
         click.echo(
             "\n"
             "Please select another operation, or Ctrl-X/Ctrl-C to cancel."
             "\n\n"
             "Remaining steps after development is complete:\n"
             '- Run scripts/implementation_coverage.py,\n'
-            "- Run scripts/update_backend_index.py."
             "\n"
         )
 

scripts/template/lib/__init__.py.j2

@@ -1,5 +1 @@
-"""{{ escaped_service }} module initialization; sets value for base decorator."""
+from .models import {{ escaped_service }}_backends  #noqa: F401
-from .models import {{ escaped_service }}_backends
-from ..core.models import base_decorator
-
-mock_{{ escaped_service }} = base_decorator({{ escaped_service }}_backends)

scripts/template/lib/urls.py.j2

@@ -5,9 +5,5 @@ url_bases = [
     r"https?://{{ endpoint_prefix }}\.(.+)\.amazonaws\.com",
 ]
 
-{% if api_protocol == 'rest-json' %}
-response = {{ service_class }}Response()
-{% endif %}
-
 url_paths = {
 }

scripts/template/test/test_service.py.j2

@@ -1,7 +1,7 @@
 """Unit tests for {{ escaped_service }}-supported APIs."""
 import boto3
 
-from moto import mock_{{ escaped_service }}
+from moto import mock_aws
 
 # See our Development Tips on writing tests for hints on how to write good tests:
 # http://docs.getmoto.org/en/latest/docs/contributing/development_tips/tests.html

scripts/update_backend_index.py

@@ -9,8 +9,6 @@ from pathlib import Path
 import black
 import pprint
 
-import moto
-
 output_file = "moto/backend_index.py"
 
 script_dir = os.path.dirname(os.path.abspath(__file__))
@@ -25,7 +23,6 @@ IGNORE_BACKENDS = ["moto_server", "moto_proxy", "apigatewayv2", "awslambda_simpl
 
 
 def iter_backend_url_patterns():
-    path = os.path.dirname(moto.__file__)
     for backend in list_of_moto_modules():
         # Special case
         if backend == "moto_api":
@@ -55,7 +52,7 @@ def build_backend_url_pattern_index():
 
 def main():
     with open(output_path, "w") as fd:
-        fd.write("# autogenerated by %s\n" % __file__)
+        fd.write("# autogenerated by moto/scripts/update_backend_index.py\n")
         fd.write("import re\n")
 
     print("build backend_url_patterns")

tests/test_kinesisvideoarchivedmedia/test_kinesisvideoarchivedmedia.py

@@ -1,13 +1,16 @@
 from datetime import timedelta
+from unittest import SkipTest
 
 import boto3
 
-from moto import mock_aws
+from moto import mock_aws, settings
 from moto.core.utils import utcnow
 
 
 @mock_aws
 def test_get_hls_streaming_session_url():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't mock KinesisVideo, as DataEndpoint is set to real URL")
     region_name = "ap-northeast-1"
     kvs_client = boto3.client("kinesisvideo", region_name=region_name)
     stream_name = "my-stream"
@@ -30,6 +33,8 @@ def test_get_hls_streaming_session_url():
 
 @mock_aws
 def test_get_dash_streaming_session_url():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't mock KinesisVideo, as DataEndpoint is set to real URL")
     region_name = "ap-northeast-1"
     kvs_client = boto3.client("kinesisvideo", region_name=region_name)
     stream_name = "my-stream"
@@ -52,6 +57,8 @@ def test_get_dash_streaming_session_url():
 
 @mock_aws
 def test_get_clip():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't mock KinesisVideo, as DataEndpoint is set to real URL")
     region_name = "ap-northeast-1"
     kvs_client = boto3.client("kinesisvideo", region_name=region_name)
     stream_name = "my-stream"