From 264723d981d6cadfa7ec67288e204acb15caff6e Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 3 Sep 2023 14:59:09 +0000 Subject: [PATCH] Techdebt: Do not load EC2 images unless necessary, to speed things up (#6761) --- .github/workflows/tests_servermode.yml | 2 +- moto/ec2/models/amis.py | 4 ++ moto/settings.py | 5 +++ tests/__init__.py | 4 ++ .../test_launch_configurations.py | 15 +++++-- tests/test_ec2/test_amis.py | 45 +++++++++++++++++-- tests/test_ec2/test_elastic_block_store.py | 10 ++++- tests/test_ec2/test_instances.py | 6 +++ tests/test_ec2/test_windows.py | 9 +++- tests/test_special_cases/test_custom_amis.py | 5 ++- tests/test_ssm/test_ssm_ec2_integration.py | 9 +++- tests/test_ssm/test_ssm_ecs_images.py | 9 +++- 12 files changed, 110 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests_servermode.yml b/.github/workflows/tests_servermode.yml index 38d8c11b0..c526900f3 100644 --- a/.github/workflows/tests_servermode.yml +++ b/.github/workflows/tests_servermode.yml @@ -21,7 +21,7 @@ jobs: run: | pip install build python -m build - docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${{ matrix.python-version }}-buster /moto/scripts/ci_moto_server.sh & + docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e MOTO_EC2_LOAD_DEFAULT_AMIS=false -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${{ matrix.python-version }}-buster /moto/scripts/ci_moto_server.sh & python scripts/ci_wait_for_server.py - name: Get pip cache dir id: pip-cache diff --git a/moto/ec2/models/amis.py b/moto/ec2/models/amis.py index c1b041b72..74884ad6a 100644 --- a/moto/ec2/models/amis.py +++ b/moto/ec2/models/amis.py @@ -1,7 +1,9 @@ import json +import os import re from os import environ from typing import Any, Dict, List, Optional, Set, cast +from moto import settings from moto.utilities.utils import load_resource from ..exceptions import ( InvalidAMIIdError, @@ -156,6 +158,8 @@ class AmiBackend: self._load_amis() def _load_amis(self) -> None: + if "MOTO_AMIS_PATH" not in os.environ and not settings.ec2_load_default_amis(): + return for ami in AMIS: ami_id = ami["ami_id"] # we are assuming the default loaded amis are owned by amazon diff --git a/moto/settings.py b/moto/settings.py index e332db7f9..89c421372 100644 --- a/moto/settings.py +++ b/moto/settings.py @@ -69,6 +69,11 @@ def s3_allow_crossdomain_access() -> bool: return os.environ.get("MOTO_S3_ALLOW_CROSSACCOUNT_ACCESS", "true").lower() == "true" +def ec2_load_default_amis() -> bool: + # True by default - only the value 'false' will return false + return os.environ.get("MOTO_EC2_LOAD_DEFAULT_AMIS", "true").lower() != "false" + + def ecs_new_arn_format() -> bool: # True by default - only the value 'false' will return false return os.environ.get("MOTO_ECS_NEW_ARN", "true").lower() != "false" diff --git a/tests/__init__.py b/tests/__init__.py index 82e3d494d..db2939e1d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,5 @@ import logging +import os # Disable extra logging for tests @@ -14,3 +15,6 @@ EXAMPLE_AMI_PARAVIRTUAL = "ami-fa7cdd89" EXAMPLE_AMI_WINDOWS = "ami-f4cf1d8d" DEFAULT_ACCOUNT_ID = "123456789012" + +# For the majority of tests we don't need the default AMI's +os.environ["MOTO_EC2_LOAD_DEFAULT_AMIS"] = "false" diff --git a/tests/test_autoscaling/test_launch_configurations.py b/tests/test_autoscaling/test_launch_configurations.py index 71384ad2e..4e7b7d931 100644 --- a/tests/test_autoscaling/test_launch_configurations.py +++ b/tests/test_autoscaling/test_launch_configurations.py @@ -1,12 +1,13 @@ import base64 import boto3 -from botocore.exceptions import ClientError - +import os import pytest -from moto import mock_autoscaling, mock_ec2 +from botocore.exceptions import ClientError +from moto import mock_autoscaling, mock_ec2, settings from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID from tests import EXAMPLE_AMI_ID +from unittest import mock, SkipTest @mock_autoscaling @@ -130,9 +131,14 @@ def test_create_launch_configuration_additional_parameters(): } +# The default AMIs are not loaded for our test case, to speed things up +# But we do need it for this specific test (and others in this file..) +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_autoscaling @mock_ec2 def test_create_launch_configuration_without_public_ip(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") ec2 = boto3.resource("ec2", "us-east-1") vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16") subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/27") @@ -288,9 +294,12 @@ def test_invalid_launch_configuration_request_raises_error(request_params): assert "Valid requests must contain" in ex.value.response["Error"]["Message"] +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_autoscaling @mock_ec2 def test_launch_config_with_block_device_mappings__volumes_are_created(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") as_client = boto3.client("autoscaling", "us-east-2") ec2_client = boto3.client("ec2", "us-east-2") random_image_id = ec2_client.describe_images()["Images"][0]["ImageId"] diff --git a/tests/test_ec2/test_amis.py b/tests/test_ec2/test_amis.py index ce4060203..47dcd6c2e 100644 --- a/tests/test_ec2/test_amis.py +++ b/tests/test_ec2/test_amis.py @@ -1,18 +1,24 @@ import boto3 -from botocore.exceptions import ClientError - +import os import pytest import random -from moto import mock_ec2 +from moto import mock_ec2, settings from moto.ec2.models.amis import AMIS from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID +from botocore.exceptions import ClientError from tests import EXAMPLE_AMI_ID, EXAMPLE_AMI_PARAVIRTUAL +from unittest import mock, SkipTest from uuid import uuid4 +# The default AMIs are not loaded for our test case, to speed things up +# But we do need it for this specific test (and others in this file..) +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_snapshots_for_initial_amis(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") ec2 = boto3.client("ec2", region_name="us-east-1") snapshots = ec2.describe_snapshots()["Snapshots"] @@ -29,8 +35,11 @@ def test_snapshots_for_initial_amis(): assert expected_description in snapshot_descs +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_ami_create_and_delete(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") ec2 = boto3.client("ec2", region_name="us-east-1") reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) @@ -182,8 +191,11 @@ def test_ami_copy_dryrun(): ) +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_ami_copy(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") ec2 = boto3.client("ec2", region_name="us-west-1") reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1) @@ -271,8 +283,11 @@ def test_ami_copy_nonexisting_source_region(): assert ex.value.response["Error"]["Code"] == "InvalidAMIID.NotFound" +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_copy_image_changes_owner_id(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") conn = boto3.client("ec2", region_name="us-east-1") # this source AMI ID is from moto/ec2/resources/amis.json @@ -377,8 +392,11 @@ def test_ami_uses_account_id_if_valid_access_key_is_supplied(): ] +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_ami_filters(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") image_name_A = f"test-ami-{str(uuid4())[0:6]}" kernel_value_A = f"k-{str(uuid4())[0:6]}" kernel_value_B = f"k-{str(uuid4())[0:6]}" @@ -834,8 +852,11 @@ def test_ami_attribute_user_and_group_permissions(): assert image.public is False +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_filter_description(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") # https://github.com/getmoto/moto/issues/4460 client = boto3.client("ec2", region_name="us-west-2") @@ -1006,8 +1027,11 @@ def test_ami_filter_wildcard(): assert len(my_images) == 1 +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_ami_filter_by_owner_id(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") client = boto3.client("ec2", region_name="us-east-1") ubuntu_id = "099720109477" @@ -1164,8 +1188,11 @@ def test_ami_filter_by_empty_tag(): assert len(client.describe_images(Filters=images_filter)["Images"]) == 3 +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_ami_filter_by_ownerid(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") ec2_connection = boto3.client("ec2", region_name="us-east-1") images = ec2_connection.describe_images( @@ -1237,8 +1264,11 @@ def test_delete_snapshot_from_create_image(): assert exc.value.response["Error"]["Code"] == "InvalidSnapshot.NotFound" +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_ami_describe_image_attribute_product_codes(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") # Setup conn = boto3.client("ec2", region_name="us-east-1") @@ -1262,8 +1292,11 @@ def test_ami_describe_image_attribute_product_codes(): assert attributes["ProductCodes"] == expected_codes +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_ami_describe_image_attribute(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") # Setup conn = boto3.client("ec2", region_name="us-east-1") @@ -1293,8 +1326,11 @@ def test_ami_describe_image_attribute(): assert sriov["SriovNetSupport"]["Value"] == "simple" +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_ami_describe_image_attribute_block_device_fail(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") # Setup conn = boto3.client("ec2", region_name="us-east-1") test_image = conn.describe_images() @@ -1314,8 +1350,11 @@ def test_ami_describe_image_attribute_block_device_fail(): ) +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_ami_describe_image_attribute_invalid_param(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") # Setup conn = boto3.client("ec2", region_name="us-east-1") test_image = conn.describe_images() diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index d7741597c..0dbfccfab 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -1,12 +1,13 @@ import boto3 - +import os import pytest from botocore.exceptions import ClientError -from moto import mock_ec2 +from moto import mock_ec2, settings from moto.core import DEFAULT_ACCOUNT_ID as OWNER_ID from moto.ec2.models.elastic_block_store import IOPS_REQUIRED_VOLUME_TYPES from moto.kms import mock_kms from tests import EXAMPLE_AMI_ID +from unittest import mock, SkipTest from uuid import uuid4 @@ -1067,8 +1068,13 @@ def test_create_snapshots_multiple_volumes(): assert snapshot2["VolumeSize"] == 100 +# The default AMIs are not loaded for our test case, to speed things up +# But we do need it for this specific test +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_create_snapshots_multiple_volumes_without_boot(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") client = boto3.client("ec2", region_name="us-east-1") ec2 = boto3.resource("ec2", region_name="us-east-1") diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index b45d52d89..ca07b812f 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -1,6 +1,7 @@ import base64 import ipaddress import json +import os import warnings from unittest import SkipTest, mock from uuid import uuid4 @@ -2339,6 +2340,9 @@ def test_instance_lifecycle(): assert instance.instance_lifecycle is None +# The default AMIs are not loaded for our test case, to speed things up +# But we do need it for this specific test +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 @pytest.mark.parametrize( "launch_template_kind", ("LaunchTemplateId", "LaunchTemplateName") @@ -2346,6 +2350,8 @@ def test_instance_lifecycle(): def test_create_instance_with_launch_template_id_produces_no_warning( launch_template_kind, ): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") client, resource = ( boto3.client("ec2", region_name="us-west-1"), boto3.resource("ec2", region_name="us-west-1"), diff --git a/tests/test_ec2/test_windows.py b/tests/test_ec2/test_windows.py index a31add55b..c9bd9516c 100644 --- a/tests/test_ec2/test_windows.py +++ b/tests/test_ec2/test_windows.py @@ -1,11 +1,18 @@ import boto3 +import os -from moto import mock_ec2 +from moto import mock_ec2, settings from tests import EXAMPLE_AMI_WINDOWS, EXAMPLE_AMI_PARAVIRTUAL +from unittest import mock, SkipTest +# The default AMIs are not loaded for our test case, to speed things up +# But we do need it for this specific test (and others in this file..) +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 def test_get_password_data(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") client = boto3.client("ec2", region_name="us-east-1") # Ensure non-windows instances return empty password data diff --git a/tests/test_special_cases/test_custom_amis.py b/tests/test_special_cases/test_custom_amis.py index 60def2ecb..ccdfaffb6 100644 --- a/tests/test_special_cases/test_custom_amis.py +++ b/tests/test_special_cases/test_custom_amis.py @@ -6,7 +6,7 @@ import boto3 import json import os import importlib -from unittest import SkipTest, TestCase +from unittest import SkipTest, TestCase, mock from pathlib import Path import moto @@ -15,6 +15,9 @@ from moto.core import DEFAULT_ACCOUNT_ID from moto.ec2.models import ec2_backends +# The default AMIs are not loaded for our test case, to speed things up +# But we do need it for this specific test (and others in this file..) +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 class TestEC2CustomAMIs(TestCase): def setup_amis(self): diff --git a/tests/test_ssm/test_ssm_ec2_integration.py b/tests/test_ssm/test_ssm_ec2_integration.py index c7a89c6ea..a9b23e15a 100644 --- a/tests/test_ssm/test_ssm_ec2_integration.py +++ b/tests/test_ssm/test_ssm_ec2_integration.py @@ -1,14 +1,21 @@ import boto3 +import os -from moto import mock_ec2, mock_ssm +from moto import mock_ec2, mock_ssm, settings +from unittest import mock, SkipTest test_ami = "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64" +# The default AMIs are not loaded for our test case, to speed things up +# But we do need it for this specific test +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 @mock_ssm def test_ssm_get_latest_ami_by_path(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") ssm = boto3.client("ssm", region_name="us-east-1") ami = ssm.get_parameter(Name=test_ami)["Parameter"]["Value"] diff --git a/tests/test_ssm/test_ssm_ecs_images.py b/tests/test_ssm/test_ssm_ecs_images.py index efb8fda52..0e6f038ce 100644 --- a/tests/test_ssm/test_ssm_ecs_images.py +++ b/tests/test_ssm/test_ssm_ecs_images.py @@ -1,11 +1,18 @@ import boto3 +import os -from moto import mock_ec2, mock_ssm +from moto import mock_ec2, mock_ssm, settings +from unittest import mock, SkipTest +# The default AMIs are not loaded for our test case, to speed things up +# But we do need it for this specific test +@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"}) @mock_ec2 @mock_ssm def test_ssm_get_latest_ami_by_path(): + if settings.TEST_SERVER_MODE: + raise SkipTest("Can't set environment variables in ServerMode") ssm = boto3.client("ssm", region_name="us-east-1") path = "/aws/service/ecs/optimized-ami" params = ssm.get_parameters_by_path(Path=path, Recursive=True)["Parameters"]