From 8199a884468f5c2369040b8a3e50b57df3ab23fa Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 17 Jan 2024 10:05:50 +0000 Subject: [PATCH] Techdebt: Update scaffolding to use mock_aws (#7221) --- Makefile | 5 ++- scripts/scaffold.py | 35 ++++--------------- scripts/template/lib/__init__.py.j2 | 6 +--- scripts/template/lib/urls.py.j2 | 4 --- scripts/template/test/test_service.py.j2 | 2 +- scripts/update_backend_index.py | 5 +-- .../test_kinesisvideoarchivedmedia.py | 9 ++++- 7 files changed, 20 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index 9c80b4c46..a5c5e34ce 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,8 @@ SERVICE_NAME = "default" TEST_NAMES = "*" ifeq ($(TEST_SERVER_MODE), true) - # exclude test_kinesisvideoarchivedmedia - # because testing with moto_server is difficult with data-endpoint - TEST_EXCLUDE := --ignore tests/test_kinesisvideoarchivedmedia --ignore tests/test_acm --ignore tests/test_amp --ignore tests/test_awslambda --ignore tests/test_batch --ignore tests/test_ec2 --ignore tests/test_sqs + # Exclude parallel tests + TEST_EXCLUDE := --ignore tests/test_acm --ignore tests/test_amp --ignore tests/test_awslambda --ignore tests/test_batch --ignore tests/test_ec2 --ignore tests/test_sqs # Parallel tests will be run separate PARALLEL_TESTS := ./tests/test_acm/ ./tests/test_acmpca/ ./tests/test_amp/ ./tests/test_awslambda ./tests/test_batch ./tests/test_ec2 ./tests/test_sqs else diff --git a/scripts/scaffold.py b/scripts/scaffold.py index ce8df0c3b..eff34751d 100755 --- a/scripts/scaffold.py +++ b/scripts/scaffold.py @@ -24,6 +24,7 @@ import random import re import inspect import importlib +import subprocess from lxml import etree import click @@ -47,6 +48,7 @@ TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "./template") INPUT_IGNORED_IN_BACKEND = ["Marker", "PageSize"] OUTPUT_IGNORED_IN_BACKEND = ["NextMarker"] +root_dir = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode().strip() def print_progress(title, body, color): """Prints a color-code message describing current state of progress.""" @@ -143,29 +145,6 @@ def render_template(tmpl_dir, tmpl_filename, context, service, alt_filename=None fhandle.write(rendered) -def append_mock_to_init_py(service): - """Update __init_.py to add line to load the mock service.""" - path = os.path.join(os.path.dirname(__file__), "..", "moto", "__init__.py") - with open(path, encoding="utf-8") as fhandle: - lines = [_.replace("\n", "") for _ in fhandle.readlines()] - - escaped_service = get_escaped_service(service) - if any(_ for _ in lines if _.startswith(f"^mock_{escaped_service} = lazy_load")): - return - filtered_lines = [_ for _ in lines if re.match("^mock_.*lazy_load(.*)$", _)] - last_import_line_index = lines.index(filtered_lines[-1]) - - new_line = ( - f"mock_{escaped_service} = lazy_load(" - f'".{escaped_service}", "mock_{escaped_service}", boto3_name="{service}")' - ) - lines.insert(last_import_line_index + 1, new_line) - - body = "\n".join(lines) + "\n" - with open(path, "w", encoding="utf-8") as fhandle: - fhandle.write(body) - - def initialize_service(service, api_protocol): """Create lib and test dirs if they don't exist.""" lib_dir = get_lib_dir(service) @@ -213,8 +192,6 @@ def initialize_service(service, api_protocol): else None ) render_template(tmpl_dir, tmpl_filename, tmpl_context, service, alt_filename) - # append mock to initi files - append_mock_to_init_py(service) def to_upper_camel_case(string): @@ -340,7 +317,7 @@ def get_func_in_tests(service, operation): escaped_service = get_escaped_service(service) random_region = random.choice(["us-east-2", "eu-west-1", "ap-southeast-1"]) body = "\n\n" - body += f"@mock_{escaped_service}\n" + body += f"@mock_aws\n" body += f"def test_{operation}():\n" body += f" client = boto3.client(\"{service}\", region_name=\"{random_region}\")\n" body += f" resp = client.{operation}()\n" @@ -593,7 +570,7 @@ def insert_url(service, operation, api_protocol): # pylint: disable=too-many-lo # generate url pattern if api_protocol == "rest-json": - new_line = ' "{0}/.*$": response.dispatch,' + new_line = f' "{0}/.*$": {service_class}Response.dispatch,' elif api_protocol == "rest-xml": new_line = f' "{{0}}{uri}$": {service_class}Response.{operation},' else: @@ -674,13 +651,15 @@ def main(): "yellow", ) + click.echo("Updating backend index...") + subprocess.check_output([f"{root_dir}/scripts/update_backend_index.py"]).decode().strip() + click.echo( "\n" "Please select another operation, or Ctrl-X/Ctrl-C to cancel." "\n\n" "Remaining steps after development is complete:\n" '- Run scripts/implementation_coverage.py,\n' - "- Run scripts/update_backend_index.py." "\n" ) diff --git a/scripts/template/lib/__init__.py.j2 b/scripts/template/lib/__init__.py.j2 index c182490a4..f31e376cb 100644 --- a/scripts/template/lib/__init__.py.j2 +++ b/scripts/template/lib/__init__.py.j2 @@ -1,5 +1 @@ -"""{{ escaped_service }} module initialization; sets value for base decorator.""" -from .models import {{ escaped_service }}_backends -from ..core.models import base_decorator - -mock_{{ escaped_service }} = base_decorator({{ escaped_service }}_backends) +from .models import {{ escaped_service }}_backends #noqa: F401 diff --git a/scripts/template/lib/urls.py.j2 b/scripts/template/lib/urls.py.j2 index 1a189bfe7..9bfb7b80a 100644 --- a/scripts/template/lib/urls.py.j2 +++ b/scripts/template/lib/urls.py.j2 @@ -5,9 +5,5 @@ url_bases = [ r"https?://{{ endpoint_prefix }}\.(.+)\.amazonaws\.com", ] -{% if api_protocol == 'rest-json' %} -response = {{ service_class }}Response() -{% endif %} - url_paths = { } diff --git a/scripts/template/test/test_service.py.j2 b/scripts/template/test/test_service.py.j2 index b5c4a87d4..bb958a84d 100644 --- a/scripts/template/test/test_service.py.j2 +++ b/scripts/template/test/test_service.py.j2 @@ -1,7 +1,7 @@ """Unit tests for {{ escaped_service }}-supported APIs.""" import boto3 -from moto import mock_{{ escaped_service }} +from moto import mock_aws # See our Development Tips on writing tests for hints on how to write good tests: # http://docs.getmoto.org/en/latest/docs/contributing/development_tips/tests.html diff --git a/scripts/update_backend_index.py b/scripts/update_backend_index.py index 40050bdd6..e46ea9dc6 100755 --- a/scripts/update_backend_index.py +++ b/scripts/update_backend_index.py @@ -9,8 +9,6 @@ from pathlib import Path import black import pprint -import moto - output_file = "moto/backend_index.py" script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -25,7 +23,6 @@ IGNORE_BACKENDS = ["moto_server", "moto_proxy", "apigatewayv2", "awslambda_simpl def iter_backend_url_patterns(): - path = os.path.dirname(moto.__file__) for backend in list_of_moto_modules(): # Special case if backend == "moto_api": @@ -55,7 +52,7 @@ def build_backend_url_pattern_index(): def main(): with open(output_path, "w") as fd: - fd.write("# autogenerated by %s\n" % __file__) + fd.write("# autogenerated by moto/scripts/update_backend_index.py\n") fd.write("import re\n") print("build backend_url_patterns") diff --git a/tests/test_kinesisvideoarchivedmedia/test_kinesisvideoarchivedmedia.py b/tests/test_kinesisvideoarchivedmedia/test_kinesisvideoarchivedmedia.py index ce8ba9e1a..01cc0ffe5 100644 --- a/tests/test_kinesisvideoarchivedmedia/test_kinesisvideoarchivedmedia.py +++ b/tests/test_kinesisvideoarchivedmedia/test_kinesisvideoarchivedmedia.py @@ -1,13 +1,16 @@ from datetime import timedelta +from unittest import SkipTest import boto3 -from moto import mock_aws +from moto import mock_aws, settings from moto.core.utils import utcnow @mock_aws def test_get_hls_streaming_session_url(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't mock KinesisVideo, as DataEndpoint is set to real URL") region_name = "ap-northeast-1" kvs_client = boto3.client("kinesisvideo", region_name=region_name) stream_name = "my-stream" @@ -30,6 +33,8 @@ def test_get_hls_streaming_session_url(): @mock_aws def test_get_dash_streaming_session_url(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't mock KinesisVideo, as DataEndpoint is set to real URL") region_name = "ap-northeast-1" kvs_client = boto3.client("kinesisvideo", region_name=region_name) stream_name = "my-stream" @@ -52,6 +57,8 @@ def test_get_dash_streaming_session_url(): @mock_aws def test_get_clip(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't mock KinesisVideo, as DataEndpoint is set to real URL") region_name = "ap-northeast-1" kvs_client = boto3.client("kinesisvideo", region_name=region_name) stream_name = "my-stream"