Work-around for import order (#4103)

docs/docs/contributing/development_tips/urls.rst

@@ -1,5 +1,8 @@
 .. _contributing urls:
 
+.. role:: raw-html(raw)
+    :format: html
+
 ***********************
 Intercepting URL's
 ***********************
@@ -8,8 +11,7 @@ Intercepting URL's
 Determining which URLs to intercept
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In order for Moto to know which requests to intercept, Moto needs to know which URLs to intercept.
+In order for Moto to know which requests to intercept, Moto needs to know which URLs to intercept. :raw-html:`<br />` But how do we know which URL's should be intercepted? There are a few ways of doing it:
-| But how do we know which URL's should be intercepted? There are a few ways of doing it:
 
  - For an existing service, copy/paste the url-path for an existing feature and cross your fingers and toes
  - Use the service model that is used by botocore: https://github.com/boto/botocore/tree/develop/botocore/data

docs/docs/getting_started.rst

@@ -200,18 +200,18 @@ How do I avoid tests from mutating my real infrastructure
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 You need to ensure that the mocks are actually in place.
 
-#. Ensure that your tests have dummy environment variables set up:
+ #. Ensure that your tests have dummy environment variables set up:
 
-.. sourcecode:: bash
+    .. sourcecode:: bash
 
-    export AWS_ACCESS_KEY_ID='testing'
+        export AWS_ACCESS_KEY_ID='testing'
-    export AWS_SECRET_ACCESS_KEY='testing'
+        export AWS_SECRET_ACCESS_KEY='testing'
-    export AWS_SECURITY_TOKEN='testing'
+        export AWS_SECURITY_TOKEN='testing'
-    export AWS_SESSION_TOKEN='testing'
+        export AWS_SESSION_TOKEN='testing'
 
-#. **VERY IMPORTANT**: ensure that you have your mocks set up *BEFORE* your `boto3` client is established.
+ #. **VERY IMPORTANT**: ensure that you have your mocks set up *BEFORE* your `boto3` client is established.
-   This can typically happen if you import a module that has a `boto3` client instantiated outside of a function.
+    This can typically happen if you import a module that has a `boto3` client instantiated outside of a function.
-   See the pesky imports section below on how to work around this.
+    See the pesky imports section below on how to work around this.
 
 Example on usage
 ~~~~~~~~~~~~~~~~
@@ -269,6 +269,29 @@ Example:
         some_func()  # The mock has been established from the "s3" pytest fixture, so this function that uses
                      # a package-level S3 client will properly use the mock and not reach out to AWS.
 
+Patching the client or resource
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If it is not possible to rearrange imports, we can patch the boto3-client or resource after the mock has started. See the following code sample:
+
+.. sourcecode:: python
+
+    # The client can come from an import, an __init__-file, wherever..
+    client = boto3.client("s3")
+    s3 = boto3.resource("s3")
+
+    @mock_s3
+    def test_mock_works_with_client_or_resource_created_outside():
+        from moto.core import patch_client, patch_resource
+        patch_client(outside_client)
+        patch_resource(s3)
+
+        assert client.list_buckets()["Buckets"] == []
+
+        assert list(s3.buckets.all()) == []
+
+This will ensure that the boto3 requests are still mocked.
+
 Other caveats
 ~~~~~~~~~~~~~
 For Tox, Travis CI, and other build systems, you might need to also perform a `touch ~/.aws/credentials`

moto/core/__init__.py

@@ -1,5 +1,6 @@
 from .models import BaseModel, BaseBackend, moto_api_backend, ACCOUNT_ID  # noqa
 from .models import CloudFormationModel  # noqa
+from .models import patch_client, patch_resource  # noqa
 from .responses import ActionAuthenticatorMixin
 
 moto_api_backends = {"global": moto_api_backend}

moto/core/models.py

@@ -1,3 +1,5 @@
+import botocore
+import boto3
 import functools
 import inspect
 import os
@@ -407,6 +409,40 @@ botocore_stubber = BotocoreStubber()
 BUILTIN_HANDLERS.append(("before-send", botocore_stubber))
 
 
+def patch_client(client):
+    """
+    Explicitly patch a boto3-client
+    """
+    """
+    Adding the botocore_stubber to the BUILTIN_HANDLERS, as above, will mock everything as long as the import ordering is correct
+     - user:   start mock_service decorator
+     - system: imports core.model
+     - system: adds the stubber to the BUILTIN_HANDLERS
+     - user:   create a boto3 client - which will use the BUILTIN_HANDLERS
+
+    But, if for whatever reason the imports are wrong and the client is created first, it doesn't know about our stub yet
+    This method can be used to tell a client that it needs to be mocked, and append the botocore_stubber after creation
+    :param client:
+    :return:
+    """
+    if isinstance(client, botocore.client.BaseClient):
+        client.meta.events.register("before-send", botocore_stubber)
+    else:
+        raise Exception(f"Argument {client} should be of type boto3.client")
+
+
+def patch_resource(resource):
+    """
+    Explicitly patch a boto3-resource
+    """
+    if hasattr(resource, "meta") and isinstance(
+        resource.meta, boto3.resources.factory.ResourceMeta
+    ):
+        patch_client(resource.meta.client)
+    else:
+        raise Exception(f"Argument {resource} should be of type boto3.resource")
+
+
 def not_implemented_callback(request):
     status = 400
     headers = {}

tests/test_core/test_importorder.py

@@ -0,0 +1,109 @@
+import boto3
+import pytest
+import sure  # noqa # pylint: disable=unused-import
+from moto import mock_s3
+from moto import settings
+from os import environ
+from unittest import SkipTest
+
+
+@pytest.fixture(scope="function")
+def aws_credentials():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("No point in testing this in ServerMode.")
+    """Mocked AWS Credentials for moto."""
+    environ["AWS_ACCESS_KEY_ID"] = "testing"
+    environ["AWS_SECRET_ACCESS_KEY"] = "testing"
+    environ["AWS_SECURITY_TOKEN"] = "testing"
+    environ["AWS_SESSION_TOKEN"] = "testing"
+
+
+def test_mock_works_with_client_created_inside(aws_credentials):
+    m = mock_s3()
+    m.start()
+    client = boto3.client("s3", region_name="us-east-1")
+
+    b = client.list_buckets()
+    b["Buckets"].should.be.empty
+    m.stop()
+
+
+def test_mock_works_with_client_created_outside(aws_credentials):
+    # Create the boto3 client first
+    outside_client = boto3.client("s3", region_name="us-east-1")
+
+    # Start the mock afterwards - which does not mock an already created client
+    m = mock_s3()
+    m.start()
+
+    # So remind us to mock this client
+    from moto.core import patch_client
+
+    patch_client(outside_client)
+
+    b = outside_client.list_buckets()
+    b["Buckets"].should.be.empty
+    m.stop()
+
+
+def test_mock_works_with_resource_created_outside(aws_credentials):
+    # Create the boto3 client first
+    outside_resource = boto3.resource("s3", region_name="us-east-1")
+
+    # Start the mock afterwards - which does not mock an already created resource
+    m = mock_s3()
+    m.start()
+
+    # So remind us to mock this client
+    from moto.core import patch_resource
+
+    patch_resource(outside_resource)
+
+    b = list(outside_resource.buckets.all())
+    b.should.be.empty
+    m.stop()
+
+
+def test_patch_client_does_not_work_for_random_parameters():
+    from moto.core import patch_client
+
+    with pytest.raises(Exception, match="Argument sth should be of type boto3.client"):
+        patch_client("sth")
+
+
+def test_patch_resource_does_not_work_for_random_parameters():
+    from moto.core import patch_resource
+
+    with pytest.raises(
+        Exception, match="Argument sth should be of type boto3.resource"
+    ):
+        patch_resource("sth")
+
+
+class ImportantBusinessLogic:
+    def __init__(self):
+        self._s3 = boto3.client("s3", region_name="us-east-1")
+
+    def do_important_things(self):
+        return self._s3.list_buckets()["Buckets"]
+
+
+def test_mock_works_when_replacing_client(aws_credentials):
+
+    logic = ImportantBusinessLogic()
+
+    m = mock_s3()
+    m.start()
+
+    # This will fail, as the S3 client was created before the mock was initialized
+    try:
+        logic.do_important_things()
+    except Exception as e:
+        str(e).should.contain("InvalidAccessKeyId")
+
+    client_initialized_after_mock = boto3.client("s3", region_name="us-east-1")
+    logic._s3 = client_initialized_after_mock
+    # This will work, as we now use a properly mocked client
+    logic.do_important_things().should.equal([])
+
+    m.stop()