diff --git a/docs/docs/contributing/development_tips/urls.rst b/docs/docs/contributing/development_tips/urls.rst index f4fb4418a..3549a2d1e 100644 --- a/docs/docs/contributing/development_tips/urls.rst +++ b/docs/docs/contributing/development_tips/urls.rst @@ -1,5 +1,8 @@ .. _contributing urls: +.. role:: raw-html(raw) + :format: html + *********************** Intercepting URL's *********************** @@ -8,8 +11,7 @@ Intercepting URL's Determining which URLs to intercept ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In order for Moto to know which requests to intercept, Moto needs to know which URLs to intercept. -| But how do we know which URL's should be intercepted? There are a few ways of doing it: +In order for Moto to know which requests to intercept, Moto needs to know which URLs to intercept. :raw-html:`
` But how do we know which URL's should be intercepted? There are a few ways of doing it: - For an existing service, copy/paste the url-path for an existing feature and cross your fingers and toes - Use the service model that is used by botocore: https://github.com/boto/botocore/tree/develop/botocore/data diff --git a/docs/docs/getting_started.rst b/docs/docs/getting_started.rst index 29ae23b8e..f8ed52879 100644 --- a/docs/docs/getting_started.rst +++ b/docs/docs/getting_started.rst @@ -200,18 +200,18 @@ How do I avoid tests from mutating my real infrastructure ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You need to ensure that the mocks are actually in place. -#. Ensure that your tests have dummy environment variables set up: + #. Ensure that your tests have dummy environment variables set up: -.. sourcecode:: bash + .. sourcecode:: bash - export AWS_ACCESS_KEY_ID='testing' - export AWS_SECRET_ACCESS_KEY='testing' - export AWS_SECURITY_TOKEN='testing' - export AWS_SESSION_TOKEN='testing' + export AWS_ACCESS_KEY_ID='testing' + export AWS_SECRET_ACCESS_KEY='testing' + export AWS_SECURITY_TOKEN='testing' + export AWS_SESSION_TOKEN='testing' -#. **VERY IMPORTANT**: ensure that you have your mocks set up *BEFORE* your `boto3` client is established. - This can typically happen if you import a module that has a `boto3` client instantiated outside of a function. - See the pesky imports section below on how to work around this. + #. **VERY IMPORTANT**: ensure that you have your mocks set up *BEFORE* your `boto3` client is established. + This can typically happen if you import a module that has a `boto3` client instantiated outside of a function. + See the pesky imports section below on how to work around this. Example on usage ~~~~~~~~~~~~~~~~ @@ -269,6 +269,29 @@ Example: some_func() # The mock has been established from the "s3" pytest fixture, so this function that uses # a package-level S3 client will properly use the mock and not reach out to AWS. +Patching the client or resource +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If it is not possible to rearrange imports, we can patch the boto3-client or resource after the mock has started. See the following code sample: + +.. sourcecode:: python + + # The client can come from an import, an __init__-file, wherever.. + client = boto3.client("s3") + s3 = boto3.resource("s3") + + @mock_s3 + def test_mock_works_with_client_or_resource_created_outside(): + from moto.core import patch_client, patch_resource + patch_client(outside_client) + patch_resource(s3) + + assert client.list_buckets()["Buckets"] == [] + + assert list(s3.buckets.all()) == [] + +This will ensure that the boto3 requests are still mocked. + Other caveats ~~~~~~~~~~~~~ For Tox, Travis CI, and other build systems, you might need to also perform a `touch ~/.aws/credentials` diff --git a/moto/core/__init__.py b/moto/core/__init__.py index 3511ca9d3..710540885 100644 --- a/moto/core/__init__.py +++ b/moto/core/__init__.py @@ -1,5 +1,6 @@ from .models import BaseModel, BaseBackend, moto_api_backend, ACCOUNT_ID # noqa from .models import CloudFormationModel # noqa +from .models import patch_client, patch_resource # noqa from .responses import ActionAuthenticatorMixin moto_api_backends = {"global": moto_api_backend} diff --git a/moto/core/models.py b/moto/core/models.py index 70ac02b81..c5e8fec94 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -1,3 +1,5 @@ +import botocore +import boto3 import functools import inspect import os @@ -407,6 +409,40 @@ botocore_stubber = BotocoreStubber() BUILTIN_HANDLERS.append(("before-send", botocore_stubber)) +def patch_client(client): + """ + Explicitly patch a boto3-client + """ + """ + Adding the botocore_stubber to the BUILTIN_HANDLERS, as above, will mock everything as long as the import ordering is correct + - user: start mock_service decorator + - system: imports core.model + - system: adds the stubber to the BUILTIN_HANDLERS + - user: create a boto3 client - which will use the BUILTIN_HANDLERS + + But, if for whatever reason the imports are wrong and the client is created first, it doesn't know about our stub yet + This method can be used to tell a client that it needs to be mocked, and append the botocore_stubber after creation + :param client: + :return: + """ + if isinstance(client, botocore.client.BaseClient): + client.meta.events.register("before-send", botocore_stubber) + else: + raise Exception(f"Argument {client} should be of type boto3.client") + + +def patch_resource(resource): + """ + Explicitly patch a boto3-resource + """ + if hasattr(resource, "meta") and isinstance( + resource.meta, boto3.resources.factory.ResourceMeta + ): + patch_client(resource.meta.client) + else: + raise Exception(f"Argument {resource} should be of type boto3.resource") + + def not_implemented_callback(request): status = 400 headers = {} diff --git a/tests/test_core/test_importorder.py b/tests/test_core/test_importorder.py new file mode 100644 index 000000000..6c0a8567b --- /dev/null +++ b/tests/test_core/test_importorder.py @@ -0,0 +1,109 @@ +import boto3 +import pytest +import sure # noqa # pylint: disable=unused-import +from moto import mock_s3 +from moto import settings +from os import environ +from unittest import SkipTest + + +@pytest.fixture(scope="function") +def aws_credentials(): + if settings.TEST_SERVER_MODE: + raise SkipTest("No point in testing this in ServerMode.") + """Mocked AWS Credentials for moto.""" + environ["AWS_ACCESS_KEY_ID"] = "testing" + environ["AWS_SECRET_ACCESS_KEY"] = "testing" + environ["AWS_SECURITY_TOKEN"] = "testing" + environ["AWS_SESSION_TOKEN"] = "testing" + + +def test_mock_works_with_client_created_inside(aws_credentials): + m = mock_s3() + m.start() + client = boto3.client("s3", region_name="us-east-1") + + b = client.list_buckets() + b["Buckets"].should.be.empty + m.stop() + + +def test_mock_works_with_client_created_outside(aws_credentials): + # Create the boto3 client first + outside_client = boto3.client("s3", region_name="us-east-1") + + # Start the mock afterwards - which does not mock an already created client + m = mock_s3() + m.start() + + # So remind us to mock this client + from moto.core import patch_client + + patch_client(outside_client) + + b = outside_client.list_buckets() + b["Buckets"].should.be.empty + m.stop() + + +def test_mock_works_with_resource_created_outside(aws_credentials): + # Create the boto3 client first + outside_resource = boto3.resource("s3", region_name="us-east-1") + + # Start the mock afterwards - which does not mock an already created resource + m = mock_s3() + m.start() + + # So remind us to mock this client + from moto.core import patch_resource + + patch_resource(outside_resource) + + b = list(outside_resource.buckets.all()) + b.should.be.empty + m.stop() + + +def test_patch_client_does_not_work_for_random_parameters(): + from moto.core import patch_client + + with pytest.raises(Exception, match="Argument sth should be of type boto3.client"): + patch_client("sth") + + +def test_patch_resource_does_not_work_for_random_parameters(): + from moto.core import patch_resource + + with pytest.raises( + Exception, match="Argument sth should be of type boto3.resource" + ): + patch_resource("sth") + + +class ImportantBusinessLogic: + def __init__(self): + self._s3 = boto3.client("s3", region_name="us-east-1") + + def do_important_things(self): + return self._s3.list_buckets()["Buckets"] + + +def test_mock_works_when_replacing_client(aws_credentials): + + logic = ImportantBusinessLogic() + + m = mock_s3() + m.start() + + # This will fail, as the S3 client was created before the mock was initialized + try: + logic.do_important_things() + except Exception as e: + str(e).should.contain("InvalidAccessKeyId") + + client_initialized_after_mock = boto3.client("s3", region_name="us-east-1") + logic._s3 = client_initialized_after_mock + # This will work, as we now use a properly mocked client + logic.do_important_things().should.equal([]) + + m.stop()