From 8e909f580a13bc87e6281cea8d44c8ccf32cc2ac Mon Sep 17 00:00:00 2001 From: Jordan Guymon Date: Thu, 6 Sep 2018 15:15:27 -0700 Subject: [PATCH 1/7] MockAWS implementation using botocore event hooks --- moto/apigateway/models.py | 4 +- moto/awslambda/responses.py | 6 +-- moto/core/models.py | 86 ++++++++++++++++++++++++++++++++++++- moto/core/utils.py | 11 +++++ moto/s3/responses.py | 8 +++- requirements-dev.txt | 2 +- setup.py | 4 +- 7 files changed, 110 insertions(+), 11 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index db4746a0e..41a49e361 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -10,6 +10,7 @@ from boto3.session import Session import responses from moto.core import BaseBackend, BaseModel from .utils import create_id +from moto.core.utils import path_url from .exceptions import StageNotFoundException, ApiKeyNotFoundException STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}" @@ -372,7 +373,8 @@ class RestAPI(BaseModel): # TODO deal with no matching resource def resource_callback(self, request): - path_after_stage_name = '/'.join(request.path_url.split("/")[2:]) + path = path_url(request.url) + path_after_stage_name = '/'.join(path.split("/")[2:]) if not path_after_stage_name: path_after_stage_name = '/' diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index 2c8a54523..1a9a4df83 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -7,7 +7,7 @@ try: except ImportError: from urllib.parse import unquote -from moto.core.utils import amz_crc32, amzn_request_id +from moto.core.utils import amz_crc32, amzn_request_id, path_url from moto.core.responses import BaseResponse from .models import lambda_backends @@ -94,7 +94,7 @@ class LambdaResponse(BaseResponse): return self._add_policy(request, full_url, headers) def _add_policy(self, request, full_url, headers): - path = request.path if hasattr(request, 'path') else request.path_url + path = request.path if hasattr(request, 'path') else path_url(request.url) function_name = path.split('/')[-2] if self.lambda_backend.get_function(function_name): policy = request.body.decode('utf8') @@ -104,7 +104,7 @@ class LambdaResponse(BaseResponse): return 404, {}, "{}" def _get_policy(self, request, full_url, headers): - path = request.path if hasattr(request, 'path') else request.path_url + path = request.path if hasattr(request, 'path') else path_url(request.url) function_name = path.split('/')[-2] if self.lambda_backend.get_function(function_name): lambda_function = self.lambda_backend.get_function(function_name) diff --git a/moto/core/models.py b/moto/core/models.py index adc06a9c0..b12374fdd 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -2,11 +2,14 @@ from __future__ import unicode_literals from __future__ import absolute_import -from collections import defaultdict import functools import inspect import re import six +from io import BytesIO +from collections import defaultdict +from botocore.handlers import BUILTIN_HANDLERS +from botocore.awsrequest import AWSResponse from moto import settings import responses @@ -233,7 +236,86 @@ class ResponsesMockAWS(BaseMockAWS): pass -MockAWS = ResponsesMockAWS +BOTOCORE_HTTP_METHODS = [ + 'GET', 'DELETE', 'HEAD', 'OPTIONS', 'PATCH', 'POST', 'PUT' +] + + +class MockRawResponse(BytesIO): + def __init__(self, input): + if isinstance(input, six.text_type): + input = input.encode('utf-8') + super(MockRawResponse, self).__init__(input) + + def stream(self, **kwargs): + contents = self.read() + while contents: + yield contents + contents = self.read() + + +class BotocoreStubber(object): + def __init__(self): + self.enabled = False + self.methods = defaultdict(list) + + def reset(self): + self.methods.clear() + + def register_response(self, method, pattern, response): + matchers = self.methods[method] + matchers.append((pattern, response)) + + def __call__(self, event_name, request, **kwargs): + if not self.enabled: + return None + + response = None + response_callback = None + found_index = None + matchers = self.methods.get(request.method) + + base_url = request.url.split('?', 1)[0] + for i, (pattern, callback) in enumerate(matchers): + if pattern.match(base_url): + if found_index is None: + found_index = i + response_callback = callback + else: + matchers.pop(found_index) + break + + if response_callback is not None: + for header, value in request.headers.items(): + if isinstance(value, six.binary_type): + request.headers[header] = value.decode('utf-8') + status, headers, body = response_callback(request, request.url, request.headers) + body = MockRawResponse(body) + response = AWSResponse(request.url, status, headers, body) + + return response + +botocore_stubber = BotocoreStubber() +BUILTIN_HANDLERS.append(('before-send', botocore_stubber)) + +class BotocoreEventMockAWS(BaseMockAWS): + def reset(self): + botocore_stubber.reset() + + def enable_patching(self): + botocore_stubber.enabled = True + for method in BOTOCORE_HTTP_METHODS: + for backend in self.backends_for_urls.values(): + for key, value in backend.urls.items(): + pattern = re.compile(key) + botocore_stubber.register_response(method, pattern, value) + + def disable_patching(self): + botocore_stubber.enabled = False + self.reset() + + +MockAWS = BotocoreEventMockAWS class ServerModeMockAWS(BaseMockAWS): diff --git a/moto/core/utils.py b/moto/core/utils.py index 86e7632b0..777a03752 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -8,6 +8,7 @@ import random import re import six import string +from six.moves.urllib.parse import urlparse REQUEST_ID_LONG = string.digits + string.ascii_uppercase @@ -286,3 +287,13 @@ def amzn_request_id(f): return status, headers, body return _wrapper + + +def path_url(url): + parsed_url = urlparse(url) + path = parsed_url.path + if not path: + path = '/' + if parsed_url.query: + path = path + '?' + parsed_url.query + return path diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 962025cb1..13e5f87d9 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -10,6 +10,7 @@ import xmltodict from moto.packages.httpretty.core import HTTPrettyRequest from moto.core.responses import _TemplateEnvironmentMixin +from moto.core.utils import path_url from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, \ parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys @@ -487,7 +488,7 @@ class ResponseObject(_TemplateEnvironmentMixin): if isinstance(request, HTTPrettyRequest): path = request.path else: - path = request.full_path if hasattr(request, 'full_path') else request.path_url + path = request.full_path if hasattr(request, 'full_path') else path_url(request.url) if self.is_delete_keys(request, path, bucket_name): return self._bucket_response_delete_keys(request, body, bucket_name, headers) @@ -708,7 +709,10 @@ class ResponseObject(_TemplateEnvironmentMixin): # Copy key # you can have a quoted ?version=abc with a version Id, so work on # we need to parse the unquoted string first - src_key_parsed = urlparse(request.headers.get("x-amz-copy-source")) + src_key = request.headers.get("x-amz-copy-source") + if isinstance(src_key, six.binary_type): + src_key = src_key.decode('utf-8') + src_key_parsed = urlparse(src_key) src_bucket, src_key = unquote(src_key_parsed.path).\ lstrip("/").split("/", 1) src_version_id = parse_qs(src_key_parsed.query).get( diff --git a/requirements-dev.txt b/requirements-dev.txt index 111cd5f3f..f87ab3db6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ freezegun flask boto>=2.45.0 boto3>=1.4.4 -botocore>=1.8.36 +botocore>=1.12.13 six>=1.9 prompt-toolkit==1.0.14 click==6.7 diff --git a/setup.py b/setup.py index 98780dd5a..66dba0f2f 100755 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ import sys install_requires = [ "Jinja2>=2.7.3", "boto>=2.36.0", - "boto3>=1.6.16,<1.8", - "botocore>=1.9.16,<1.11", + "boto3>=1.6.16", + "botocore>=1.12.13", "cryptography>=2.3.0", "requests>=2.5", "xmltodict", From fd4e5248559545a9c559cc42b263a9b61970c9c1 Mon Sep 17 00:00:00 2001 From: Jordan Guymon Date: Mon, 1 Oct 2018 09:45:12 -0700 Subject: [PATCH 2/7] Use env credentials for all tests --- .travis.yml | 4 ++-- moto/core/models.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index de22818b8..3a5de0fa2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: sudo: true before_install: - export BOTO_CONFIG=/dev/null + - export AWS_SECRET_ACCESS_KEY=foobar_secret + - export AWS_ACCESS_KEY_ID=foobar_key install: # We build moto first so the docker container doesn't try to compile it as well, also note we don't use # -d for docker run so the logs show up in travis @@ -32,8 +34,6 @@ install: if [ "$TEST_SERVER_MODE" = "true" ]; then docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${TRAVIS_PYTHON_VERSION}-stretch /moto/travis_moto_server.sh & - export AWS_SECRET_ACCESS_KEY=foobar_secret - export AWS_ACCESS_KEY_ID=foobar_key fi travis_retry pip install boto==2.45.0 travis_retry pip install boto3 diff --git a/moto/core/models.py b/moto/core/models.py index b12374fdd..d5f7b8427 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -295,9 +295,11 @@ class BotocoreStubber(object): return response + botocore_stubber = BotocoreStubber() BUILTIN_HANDLERS.append(('before-send', botocore_stubber)) + class BotocoreEventMockAWS(BaseMockAWS): def reset(self): botocore_stubber.reset() From b20e190995ed469244da3a7e556d76aeaf0cdfa5 Mon Sep 17 00:00:00 2001 From: Lorenz Hufnagel Date: Sun, 14 Oct 2018 19:58:56 +0200 Subject: [PATCH 3/7] Try to get tests running --- moto/core/models.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/moto/core/models.py b/moto/core/models.py index d5f7b8427..19267ca08 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -303,6 +303,7 @@ BUILTIN_HANDLERS.append(('before-send', botocore_stubber)) class BotocoreEventMockAWS(BaseMockAWS): def reset(self): botocore_stubber.reset() + responses_mock.reset() def enable_patching(self): botocore_stubber.enabled = True @@ -312,10 +313,32 @@ class BotocoreEventMockAWS(BaseMockAWS): pattern = re.compile(key) botocore_stubber.register_response(method, pattern, value) + if not hasattr(responses_mock, '_patcher') or not hasattr(responses_mock._patcher, 'target'): + responses_mock.start() + + for method in RESPONSES_METHODS: + # for backend in default_backends.values(): + for backend in self.backends_for_urls.values(): + for key, value in backend.urls.items(): + responses_mock.add( + CallbackResponse( + method=method, + url=re.compile(key), + callback=convert_flask_to_responses_response(value), + stream=True, + match_querystring=False, + ) + ) + def disable_patching(self): botocore_stubber.enabled = False self.reset() + try: + responses_mock.stop() + except RuntimeError: + pass + MockAWS = BotocoreEventMockAWS From 75f2c56a3643b053fbddf458e6e48e5a8e5e6214 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 30 Oct 2018 22:03:09 -0400 Subject: [PATCH 4/7] Fix ecs error response to be json. --- moto/ecs/exceptions.py | 4 +++- tests/test_ecs/test_ecs_boto3.py | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/moto/ecs/exceptions.py b/moto/ecs/exceptions.py index c23d6fd1d..a780dc7c2 100644 --- a/moto/ecs/exceptions.py +++ b/moto/ecs/exceptions.py @@ -8,4 +8,6 @@ class ServiceNotFoundException(RESTError): def __init__(self, service_name): super(ServiceNotFoundException, self).__init__( error_type="ServiceNotFoundException", - message="The service {0} does not exist".format(service_name)) + message="The service {0} does not exist".format(service_name), + template='error_json', + ) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index 70c1463ee..a0e8318da 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -631,7 +631,22 @@ def test_delete_service(): response['service']['schedulingStrategy'].should.equal('REPLICA') response['service']['taskDefinition'].should.equal( 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') - + + +@mock_ecs +def test_update_non_existant_service(): + client = boto3.client('ecs', region_name='us-east-1') + try: + client.update_service( + cluster="my-clustet", + service="my-service", + desiredCount=0, + ) + except ClientError as exc: + error_code = exc.response['Error']['Code'] + error_code.should.equal('ServiceNotFoundException') + else: + raise Exception("Didn't raise ClientError") @mock_ec2 From a8bc7a608e7b7de47b1f8ad1f9dec8ceecf2dd1c Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 30 Oct 2018 22:09:47 -0400 Subject: [PATCH 5/7] Lint. --- moto/ecs/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/ecs/exceptions.py b/moto/ecs/exceptions.py index a780dc7c2..bb7e685c8 100644 --- a/moto/ecs/exceptions.py +++ b/moto/ecs/exceptions.py @@ -10,4 +10,4 @@ class ServiceNotFoundException(RESTError): error_type="ServiceNotFoundException", message="The service {0} does not exist".format(service_name), template='error_json', - ) + ) From 90a62b56400251620684a094d85e458bc3abb17e Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 4 Nov 2018 17:30:44 -0500 Subject: [PATCH 6/7] 1.3.7 --- CHANGELOG.md | 5 +++++ moto/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f7ee4448..f42619b33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ Moto Changelog =================== +1.3.7 +----- + + * Switch from mocking requests to using before-send for AWS calls + 1.3.6 ----- diff --git a/moto/__init__.py b/moto/__init__.py index 6992c535e..dd3593d5d 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.6' +__version__ = '1.3.7' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/setup.py b/setup.py index 66dba0f2f..046ecf6e0 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ else: setup( name='moto', - version='1.3.6', + version='1.3.7', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From ed861ecae1039a048a6350a4ff832ef094cdf2c2 Mon Sep 17 00:00:00 2001 From: Niko Eckerskorn Date: Thu, 15 Nov 2018 18:29:05 +1100 Subject: [PATCH 7/7] Loosen aws-xray-sdk requirements (#1948) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 046ecf6e0..a1b8c5dae 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ install_requires = [ "mock", "docker>=2.5.1", "jsondiff==1.1.1", - "aws-xray-sdk<0.96,>=0.93", + "aws-xray-sdk!=0.96,>=0.93", "responses>=0.9.0", ]