From 40a3a529f91571f322ac827cec1f37c791a3f5d6 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 12 Feb 2023 16:58:30 -0100 Subject: [PATCH] Core: Rework model_data to only clear on exit (#5920) --- moto/core/base_backend.py | 12 +----------- moto/core/model_instances.py | 15 +++++++++++++++ moto/core/models.py | 2 ++ moto/iam/models.py | 1 - moto/moto_api/_internal/models.py | 2 ++ moto/moto_api/_internal/responses.py | 16 ++++++++-------- tests/test_s3/test_s3_file_handles.py | 8 ++++++++ 7 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 moto/core/model_instances.py diff --git a/moto/core/base_backend.py b/moto/core/base_backend.py index 840bfceef..87f7e5909 100644 --- a/moto/core/base_backend.py +++ b/moto/core/base_backend.py @@ -1,18 +1,15 @@ from boto3 import Session import re import string -from collections import defaultdict from functools import lru_cache from threading import RLock from typing import Any, List, Dict, Optional, ClassVar, TypeVar, Iterator from uuid import uuid4 from moto.settings import allow_unknown_region +from .model_instances import model_data from .utils import convert_regex_to_flask_path -model_data: Dict[str, Dict[str, object]] = defaultdict(dict) - - class InstanceTrackerMeta(type): def __new__(meta, name: str, bases: Any, dct: Dict[str, Any]) -> type: cls = super(InstanceTrackerMeta, meta).__new__(meta, name, bases, dct) @@ -31,16 +28,9 @@ class BaseBackend: self.region_name = region_name self.account_id = account_id - def _reset_model_refs(self) -> None: - # Remove all references to the models stored - for models in model_data.values(): - for model in models.values(): - model.instances = [] # type: ignore[attr-defined] - def reset(self) -> None: region_name = self.region_name account_id = self.account_id - self._reset_model_refs() self.__dict__ = {} self.__init__(region_name, account_id) # type: ignore[misc] diff --git a/moto/core/model_instances.py b/moto/core/model_instances.py new file mode 100644 index 000000000..a73f333dd --- /dev/null +++ b/moto/core/model_instances.py @@ -0,0 +1,15 @@ +from collections import defaultdict +from typing import Dict + +""" +Storage of all instances that extend BaseModel +This allows us to easily expose all internal state using the MotoServer dashboard +""" +model_data: Dict[str, Dict[str, object]] = defaultdict(dict) + + +def reset_model_data() -> None: + # Remove all references to the models stored + for models in model_data.values(): + for model in models.values(): + model.instances = [] # type: ignore[attr-defined] diff --git a/moto/core/models.py b/moto/core/models.py index 8a84c24fa..68497694c 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -23,6 +23,7 @@ from .custom_responses_mock import ( not_implemented_callback, reset_responses_mock, ) +from .model_instances import reset_model_data DEFAULT_ACCOUNT_ID = "123456789012" CALLABLE_RETURN = TypeVar("CALLABLE_RETURN") @@ -104,6 +105,7 @@ class BaseMockAWS: pass self.unmock_env_variables() self.__class__.mocks_active = False + reset_model_data() self.disable_patching() # type: ignore[attr-defined] def decorate_callable( diff --git a/moto/iam/models.py b/moto/iam/models.py index f60a00470..602d5ae8b 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -1693,7 +1693,6 @@ class IAMBackend(BaseBackend): account_id = self.account_id # Do not reset these policies, as they take a long time to load aws_policies = self.aws_managed_policies - self._reset_model_refs() self.__dict__ = {} self.__init__(region_name, account_id, aws_policies) diff --git a/moto/moto_api/_internal/models.py b/moto/moto_api/_internal/models.py index a16b96df6..3738b3fc7 100644 --- a/moto/moto_api/_internal/models.py +++ b/moto/moto_api/_internal/models.py @@ -1,4 +1,5 @@ from moto.core import BaseBackend, DEFAULT_ACCOUNT_ID +from moto.core.model_instances import reset_model_data from typing import Any, Dict @@ -14,6 +15,7 @@ class MotoAPIBackend(BaseBackend): continue for backend in backends_.values(): backend.reset() + reset_model_data() self.__init__(region_name, account_id) # type: ignore[misc] def get_transition(self, model_name: str) -> Dict[str, Any]: diff --git a/moto/moto_api/_internal/responses.py b/moto/moto_api/_internal/responses.py index d3a4f40b1..ae52c6e47 100644 --- a/moto/moto_api/_internal/responses.py +++ b/moto/moto_api/_internal/responses.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List class MotoAPIResponse(BaseResponse): def reset_response( self, - request: Any, # pylint: disable=unused-argument + request: Any, full_url: str, # pylint: disable=unused-argument headers: Any, # pylint: disable=unused-argument ) -> TYPE_RESPONSE: @@ -22,7 +22,7 @@ class MotoAPIResponse(BaseResponse): def reset_auth_response( self, - request: Any, # pylint: disable=unused-argument + request: Any, full_url: str, # pylint: disable=unused-argument headers: Any, # pylint: disable=unused-argument ) -> TYPE_RESPONSE: @@ -52,7 +52,7 @@ class MotoAPIResponse(BaseResponse): full_url: str, # pylint: disable=unused-argument headers: Any, # pylint: disable=unused-argument ) -> TYPE_RESPONSE: - from moto.core.base_backend import model_data + from moto.core.model_instances import model_data results: Dict[str, Dict[str, List[Any]]] = {} for service in sorted(model_data): @@ -86,7 +86,7 @@ class MotoAPIResponse(BaseResponse): def get_transition( self, - request: Any, # pylint: disable=unused-argument + request: Any, full_url: str, # pylint: disable=unused-argument headers: Any, # pylint: disable=unused-argument ) -> TYPE_RESPONSE: @@ -103,9 +103,9 @@ class MotoAPIResponse(BaseResponse): def set_transition( self, - request: Any, # pylint: disable=unused-argument + request: Any, full_url: str, # pylint: disable=unused-argument - headers: Any, # pylint: disable=unused-argument + headers: Any, ) -> TYPE_RESPONSE: from .models import moto_api_backend @@ -120,9 +120,9 @@ class MotoAPIResponse(BaseResponse): def unset_transition( self, - request: Any, # pylint: disable=unused-argument + request: Any, full_url: str, # pylint: disable=unused-argument - headers: Any, # pylint: disable=unused-argument + headers: Any, ) -> TYPE_RESPONSE: from .models import moto_api_backend diff --git a/tests/test_s3/test_s3_file_handles.py b/tests/test_s3/test_s3_file_handles.py index 868717dfd..216f578cb 100644 --- a/tests/test_s3/test_s3_file_handles.py +++ b/tests/test_s3/test_s3_file_handles.py @@ -3,6 +3,7 @@ import gc import warnings from functools import wraps from moto import settings +from moto.dynamodb.models import DynamoDBBackend from moto.s3 import models as s3model from moto.s3.responses import S3ResponseInstance from unittest import SkipTest, TestCase @@ -188,6 +189,13 @@ class TestS3FileHandleClosures(TestCase): bucket_name="my-bucket", key_name="my-key", version_id=key._version_id ) + @verify_zero_warnings + def test_reset_other_backend(self): + db = DynamoDBBackend("us-west-1", "1234") + # This used to empty the entire list of `model_instances`, which can contain FakeKey-references + # Verify that we can reset an unrelated backend, without throwing away FakeKey-references that still need to be disposed + db.reset() + def test_verify_key_can_be_copied_after_disposing(): # https://github.com/getmoto/moto/issues/5588