Techdebt: Do not load EC2 images unless necessary, to speed things up (#6761)

.github/workflows/tests_servermode.yml

@@ -21,7 +21,7 @@ jobs:
       run: |
         pip install build
         python -m build
-        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${{ matrix.python-version }}-buster /moto/scripts/ci_moto_server.sh &
+        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e MOTO_EC2_LOAD_DEFAULT_AMIS=false -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${{ matrix.python-version }}-buster /moto/scripts/ci_moto_server.sh &
         python scripts/ci_wait_for_server.py
     - name: Get pip cache dir
       id: pip-cache

moto/ec2/models/amis.py

@@ -1,7 +1,9 @@
 import json
+import os
 import re
 from os import environ
 from typing import Any, Dict, List, Optional, Set, cast
+from moto import settings
 from moto.utilities.utils import load_resource
 from ..exceptions import (
     InvalidAMIIdError,
@@ -156,6 +158,8 @@ class AmiBackend:
         self._load_amis()
 
     def _load_amis(self) -> None:
+        if "MOTO_AMIS_PATH" not in os.environ and not settings.ec2_load_default_amis():
+            return
         for ami in AMIS:
             ami_id = ami["ami_id"]
             # we are assuming the default loaded amis are owned by amazon

moto/settings.py

@@ -69,6 +69,11 @@ def s3_allow_crossdomain_access() -> bool:
     return os.environ.get("MOTO_S3_ALLOW_CROSSACCOUNT_ACCESS", "true").lower() == "true"
 
 
+def ec2_load_default_amis() -> bool:
+    # True by default - only the value 'false' will return false
+    return os.environ.get("MOTO_EC2_LOAD_DEFAULT_AMIS", "true").lower() != "false"
+
+
 def ecs_new_arn_format() -> bool:
     # True by default - only the value 'false' will return false
     return os.environ.get("MOTO_ECS_NEW_ARN", "true").lower() != "false"

tests/__init__.py

@@ -1,4 +1,5 @@
 import logging
+import os
 
 
 # Disable extra logging for tests
@@ -14,3 +15,6 @@ EXAMPLE_AMI_PARAVIRTUAL = "ami-fa7cdd89"
 EXAMPLE_AMI_WINDOWS = "ami-f4cf1d8d"
 
 DEFAULT_ACCOUNT_ID = "123456789012"
+
+# For the majority of tests we don't need the default AMI's
+os.environ["MOTO_EC2_LOAD_DEFAULT_AMIS"] = "false"

tests/test_autoscaling/test_launch_configurations.py

@@ -1,12 +1,13 @@
 import base64
 import boto3
-from botocore.exceptions import ClientError
+import os
-
 import pytest
 
-from moto import mock_autoscaling, mock_ec2
+from botocore.exceptions import ClientError
+from moto import mock_autoscaling, mock_ec2, settings
 from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
 from tests import EXAMPLE_AMI_ID
+from unittest import mock, SkipTest
 
 
 @mock_autoscaling
@@ -130,9 +131,14 @@ def test_create_launch_configuration_additional_parameters():
     }
 
 
+# The default AMIs are not loaded for our test case, to speed things up
+# But we do need it for this specific test (and others in this file..)
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_autoscaling
 @mock_ec2
 def test_create_launch_configuration_without_public_ip():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     ec2 = boto3.resource("ec2", "us-east-1")
     vpc = ec2.create_vpc(CidrBlock="10.0.0.0/16")
     subnet = ec2.create_subnet(VpcId=vpc.id, CidrBlock="10.0.0.0/27")
@@ -288,9 +294,12 @@ def test_invalid_launch_configuration_request_raises_error(request_params):
     assert "Valid requests must contain" in ex.value.response["Error"]["Message"]
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_autoscaling
 @mock_ec2
 def test_launch_config_with_block_device_mappings__volumes_are_created():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     as_client = boto3.client("autoscaling", "us-east-2")
     ec2_client = boto3.client("ec2", "us-east-2")
     random_image_id = ec2_client.describe_images()["Images"][0]["ImageId"]

tests/test_ec2/test_amis.py

@@ -1,18 +1,24 @@
 import boto3
-from botocore.exceptions import ClientError
+import os
-
 import pytest
 import random
 
-from moto import mock_ec2
+from moto import mock_ec2, settings
 from moto.ec2.models.amis import AMIS
 from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
+from botocore.exceptions import ClientError
 from tests import EXAMPLE_AMI_ID, EXAMPLE_AMI_PARAVIRTUAL
+from unittest import mock, SkipTest
 from uuid import uuid4
 
 
+# The default AMIs are not loaded for our test case, to speed things up
+# But we do need it for this specific test (and others in this file..)
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_snapshots_for_initial_amis():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     ec2 = boto3.client("ec2", region_name="us-east-1")
 
     snapshots = ec2.describe_snapshots()["Snapshots"]
@@ -29,8 +35,11 @@ def test_snapshots_for_initial_amis():
         assert expected_description in snapshot_descs
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_ami_create_and_delete():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     ec2 = boto3.client("ec2", region_name="us-east-1")
 
     reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)
@@ -182,8 +191,11 @@ def test_ami_copy_dryrun():
     )
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_ami_copy():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     ec2 = boto3.client("ec2", region_name="us-west-1")
 
     reservation = ec2.run_instances(ImageId=EXAMPLE_AMI_ID, MinCount=1, MaxCount=1)
@@ -271,8 +283,11 @@ def test_ami_copy_nonexisting_source_region():
     assert ex.value.response["Error"]["Code"] == "InvalidAMIID.NotFound"
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_copy_image_changes_owner_id():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     conn = boto3.client("ec2", region_name="us-east-1")
 
     # this source AMI ID is from moto/ec2/resources/amis.json
@@ -377,8 +392,11 @@ def test_ami_uses_account_id_if_valid_access_key_is_supplied():
     ]
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_ami_filters():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     image_name_A = f"test-ami-{str(uuid4())[0:6]}"
     kernel_value_A = f"k-{str(uuid4())[0:6]}"
     kernel_value_B = f"k-{str(uuid4())[0:6]}"
@@ -834,8 +852,11 @@ def test_ami_attribute_user_and_group_permissions():
     assert image.public is False
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_filter_description():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     # https://github.com/getmoto/moto/issues/4460
     client = boto3.client("ec2", region_name="us-west-2")
 
@@ -1006,8 +1027,11 @@ def test_ami_filter_wildcard():
     assert len(my_images) == 1
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_ami_filter_by_owner_id():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     client = boto3.client("ec2", region_name="us-east-1")
 
     ubuntu_id = "099720109477"
@@ -1164,8 +1188,11 @@ def test_ami_filter_by_empty_tag():
     assert len(client.describe_images(Filters=images_filter)["Images"]) == 3
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_ami_filter_by_ownerid():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     ec2_connection = boto3.client("ec2", region_name="us-east-1")
 
     images = ec2_connection.describe_images(
@@ -1237,8 +1264,11 @@ def test_delete_snapshot_from_create_image():
     assert exc.value.response["Error"]["Code"] == "InvalidSnapshot.NotFound"
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_ami_describe_image_attribute_product_codes():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     # Setup
     conn = boto3.client("ec2", region_name="us-east-1")
 
@@ -1262,8 +1292,11 @@ def test_ami_describe_image_attribute_product_codes():
     assert attributes["ProductCodes"] == expected_codes
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_ami_describe_image_attribute():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     # Setup
     conn = boto3.client("ec2", region_name="us-east-1")
 
@@ -1293,8 +1326,11 @@ def test_ami_describe_image_attribute():
     assert sriov["SriovNetSupport"]["Value"] == "simple"
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_ami_describe_image_attribute_block_device_fail():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     # Setup
     conn = boto3.client("ec2", region_name="us-east-1")
     test_image = conn.describe_images()
@@ -1314,8 +1350,11 @@ def test_ami_describe_image_attribute_block_device_fail():
     )
 
 
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_ami_describe_image_attribute_invalid_param():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     # Setup
     conn = boto3.client("ec2", region_name="us-east-1")
     test_image = conn.describe_images()

tests/test_ec2/test_elastic_block_store.py

@@ -1,12 +1,13 @@
 import boto3
-
+import os
 import pytest
 from botocore.exceptions import ClientError
-from moto import mock_ec2
+from moto import mock_ec2, settings
 from moto.core import DEFAULT_ACCOUNT_ID as OWNER_ID
 from moto.ec2.models.elastic_block_store import IOPS_REQUIRED_VOLUME_TYPES
 from moto.kms import mock_kms
 from tests import EXAMPLE_AMI_ID
+from unittest import mock, SkipTest
 from uuid import uuid4
 
 
@@ -1067,8 +1068,13 @@ def test_create_snapshots_multiple_volumes():
     assert snapshot2["VolumeSize"] == 100
 
 
+# The default AMIs are not loaded for our test case, to speed things up
+# But we do need it for this specific test
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_create_snapshots_multiple_volumes_without_boot():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     client = boto3.client("ec2", region_name="us-east-1")
     ec2 = boto3.resource("ec2", region_name="us-east-1")
 

tests/test_ec2/test_instances.py

@@ -1,6 +1,7 @@
 import base64
 import ipaddress
 import json
+import os
 import warnings
 from unittest import SkipTest, mock
 from uuid import uuid4
@@ -2339,6 +2340,9 @@ def test_instance_lifecycle():
     assert instance.instance_lifecycle is None
 
 
+# The default AMIs are not loaded for our test case, to speed things up
+# But we do need it for this specific test
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 @pytest.mark.parametrize(
     "launch_template_kind", ("LaunchTemplateId", "LaunchTemplateName")
@@ -2346,6 +2350,8 @@ def test_instance_lifecycle():
 def test_create_instance_with_launch_template_id_produces_no_warning(
     launch_template_kind,
 ):
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     client, resource = (
         boto3.client("ec2", region_name="us-west-1"),
         boto3.resource("ec2", region_name="us-west-1"),

tests/test_ec2/test_windows.py

@@ -1,11 +1,18 @@
 import boto3
+import os
 
-from moto import mock_ec2
+from moto import mock_ec2, settings
 from tests import EXAMPLE_AMI_WINDOWS, EXAMPLE_AMI_PARAVIRTUAL
+from unittest import mock, SkipTest
 
 
+# The default AMIs are not loaded for our test case, to speed things up
+# But we do need it for this specific test (and others in this file..)
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 def test_get_password_data():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     client = boto3.client("ec2", region_name="us-east-1")
 
     # Ensure non-windows instances return empty password data

tests/test_special_cases/test_custom_amis.py

@@ -6,7 +6,7 @@ import boto3
 import json
 import os
 import importlib
-from unittest import SkipTest, TestCase
+from unittest import SkipTest, TestCase, mock
 from pathlib import Path
 
 import moto
@@ -15,6 +15,9 @@ from moto.core import DEFAULT_ACCOUNT_ID
 from moto.ec2.models import ec2_backends
 
 
+# The default AMIs are not loaded for our test case, to speed things up
+# But we do need it for this specific test (and others in this file..)
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 class TestEC2CustomAMIs(TestCase):
     def setup_amis(self):

tests/test_ssm/test_ssm_ec2_integration.py

@@ -1,14 +1,21 @@
 import boto3
+import os
 
-from moto import mock_ec2, mock_ssm
+from moto import mock_ec2, mock_ssm, settings
+from unittest import mock, SkipTest
 
 
 test_ami = "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64"
 
 
+# The default AMIs are not loaded for our test case, to speed things up
+# But we do need it for this specific test
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 @mock_ssm
 def test_ssm_get_latest_ami_by_path():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     ssm = boto3.client("ssm", region_name="us-east-1")
     ami = ssm.get_parameter(Name=test_ami)["Parameter"]["Value"]
 

tests/test_ssm/test_ssm_ecs_images.py

@@ -1,11 +1,18 @@
 import boto3
+import os
 
-from moto import mock_ec2, mock_ssm
+from moto import mock_ec2, mock_ssm, settings
+from unittest import mock, SkipTest
 
 
+# The default AMIs are not loaded for our test case, to speed things up
+# But we do need it for this specific test
+@mock.patch.dict(os.environ, {"MOTO_EC2_LOAD_DEFAULT_AMIS": "true"})
 @mock_ec2
 @mock_ssm
 def test_ssm_get_latest_ami_by_path():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Can't set environment variables in ServerMode")
     ssm = boto3.client("ssm", region_name="us-east-1")
     path = "/aws/service/ecs/optimized-ami"
     params = ssm.get_parameters_by_path(Path=path, Recursive=True)["Parameters"]