Core: Rework model_data to only clear on exit (#5920)

moto/core/base_backend.py

@@ -1,18 +1,15 @@
 from boto3 import Session
 import re
 import string
-from collections import defaultdict
 from functools import lru_cache
 from threading import RLock
 from typing import Any, List, Dict, Optional, ClassVar, TypeVar, Iterator
 from uuid import uuid4
 from moto.settings import allow_unknown_region
+from .model_instances import model_data
 from .utils import convert_regex_to_flask_path
 
 
-model_data: Dict[str, Dict[str, object]] = defaultdict(dict)
-
-
 class InstanceTrackerMeta(type):
     def __new__(meta, name: str, bases: Any, dct: Dict[str, Any]) -> type:
         cls = super(InstanceTrackerMeta, meta).__new__(meta, name, bases, dct)
@@ -31,16 +28,9 @@ class BaseBackend:
         self.region_name = region_name
         self.account_id = account_id
 
-    def _reset_model_refs(self) -> None:
-        # Remove all references to the models stored
-        for models in model_data.values():
-            for model in models.values():
-                model.instances = []  # type: ignore[attr-defined]
-
     def reset(self) -> None:
         region_name = self.region_name
         account_id = self.account_id
-        self._reset_model_refs()
         self.__dict__ = {}
         self.__init__(region_name, account_id)  # type: ignore[misc]
 

moto/core/model_instances.py

@@ -0,0 +1,15 @@
+from collections import defaultdict
+from typing import Dict
+
+"""
+Storage of all instances that extend BaseModel
+This allows us to easily expose all internal state using the MotoServer dashboard
+"""
+model_data: Dict[str, Dict[str, object]] = defaultdict(dict)
+
+
+def reset_model_data() -> None:
+    # Remove all references to the models stored
+    for models in model_data.values():
+        for model in models.values():
+            model.instances = []  # type: ignore[attr-defined]

moto/core/models.py

@@ -23,6 +23,7 @@ from .custom_responses_mock import (
     not_implemented_callback,
     reset_responses_mock,
 )
+from .model_instances import reset_model_data
 
 DEFAULT_ACCOUNT_ID = "123456789012"
 CALLABLE_RETURN = TypeVar("CALLABLE_RETURN")
@@ -104,6 +105,7 @@ class BaseMockAWS:
                     pass
                 self.unmock_env_variables()
                 self.__class__.mocks_active = False
+                reset_model_data()
             self.disable_patching()  # type: ignore[attr-defined]
 
     def decorate_callable(

moto/iam/models.py

@@ -1693,7 +1693,6 @@ class IAMBackend(BaseBackend):
         account_id = self.account_id
         # Do not reset these policies, as they take a long time to load
         aws_policies = self.aws_managed_policies
-        self._reset_model_refs()
         self.__dict__ = {}
         self.__init__(region_name, account_id, aws_policies)
 

moto/moto_api/_internal/models.py

@@ -1,4 +1,5 @@
 from moto.core import BaseBackend, DEFAULT_ACCOUNT_ID
+from moto.core.model_instances import reset_model_data
 from typing import Any, Dict
 
 
@@ -14,6 +15,7 @@ class MotoAPIBackend(BaseBackend):
                 continue
             for backend in backends_.values():
                 backend.reset()
+        reset_model_data()
         self.__init__(region_name, account_id)  # type: ignore[misc]
 
     def get_transition(self, model_name: str) -> Dict[str, Any]:

moto/moto_api/_internal/responses.py

@@ -9,7 +9,7 @@ from typing import Any, Dict, List
 class MotoAPIResponse(BaseResponse):
     def reset_response(
         self,
-        request: Any,  # pylint: disable=unused-argument
+        request: Any,
         full_url: str,  # pylint: disable=unused-argument
         headers: Any,  # pylint: disable=unused-argument
     ) -> TYPE_RESPONSE:
@@ -22,7 +22,7 @@ class MotoAPIResponse(BaseResponse):
 
     def reset_auth_response(
         self,
-        request: Any,  # pylint: disable=unused-argument
+        request: Any,
         full_url: str,  # pylint: disable=unused-argument
         headers: Any,  # pylint: disable=unused-argument
     ) -> TYPE_RESPONSE:
@@ -52,7 +52,7 @@ class MotoAPIResponse(BaseResponse):
         full_url: str,  # pylint: disable=unused-argument
         headers: Any,  # pylint: disable=unused-argument
     ) -> TYPE_RESPONSE:
-        from moto.core.base_backend import model_data
+        from moto.core.model_instances import model_data
 
         results: Dict[str, Dict[str, List[Any]]] = {}
         for service in sorted(model_data):
@@ -86,7 +86,7 @@ class MotoAPIResponse(BaseResponse):
 
     def get_transition(
         self,
-        request: Any,  # pylint: disable=unused-argument
+        request: Any,
         full_url: str,  # pylint: disable=unused-argument
         headers: Any,  # pylint: disable=unused-argument
     ) -> TYPE_RESPONSE:
@@ -103,9 +103,9 @@ class MotoAPIResponse(BaseResponse):
 
     def set_transition(
         self,
-        request: Any,  # pylint: disable=unused-argument
+        request: Any,
         full_url: str,  # pylint: disable=unused-argument
-        headers: Any,  # pylint: disable=unused-argument
+        headers: Any,
     ) -> TYPE_RESPONSE:
         from .models import moto_api_backend
 
@@ -120,9 +120,9 @@ class MotoAPIResponse(BaseResponse):
 
     def unset_transition(
         self,
-        request: Any,  # pylint: disable=unused-argument
+        request: Any,
         full_url: str,  # pylint: disable=unused-argument
-        headers: Any,  # pylint: disable=unused-argument
+        headers: Any,
     ) -> TYPE_RESPONSE:
         from .models import moto_api_backend
 

tests/test_s3/test_s3_file_handles.py

@@ -3,6 +3,7 @@ import gc
 import warnings
 from functools import wraps
 from moto import settings
+from moto.dynamodb.models import DynamoDBBackend
 from moto.s3 import models as s3model
 from moto.s3.responses import S3ResponseInstance
 from unittest import SkipTest, TestCase
@@ -188,6 +189,13 @@ class TestS3FileHandleClosures(TestCase):
             bucket_name="my-bucket", key_name="my-key", version_id=key._version_id
         )
 
+    @verify_zero_warnings
+    def test_reset_other_backend(self):
+        db = DynamoDBBackend("us-west-1", "1234")
+        # This used to empty the entire list of `model_instances`, which can contain FakeKey-references
+        # Verify that we can reset an unrelated backend, without throwing away FakeKey-references that still need to be disposed
+        db.reset()
+
 
 def test_verify_key_can_be_copied_after_disposing():
     # https://github.com/getmoto/moto/issues/5588