Merge remote-tracking branch 'upstream/master'

.travis.yml

@@ -1,50 +1,50 @@
-pyhlanguage: python
-sudo: false
-services:
-  - docker
-python:
-  - 2.7
-  - 3.6
-env:
-    - TEST_SERVER_MODE=false
-    - TEST_SERVER_MODE=true
-# Due to incomplete Python 3.7 support on Travis CI (
-# https://github.com/travis-ci/travis-ci/issues/9815),
-# using a matrix is necessary
-matrix:
-  include:
-    - python: 3.7
-      env: TEST_SERVER_MODE=false
-      dist: xenial
-      sudo: true
-    - python: 3.7
-      env: TEST_SERVER_MODE=true
-      dist: xenial
-      sudo: true
-before_install:
-  - export BOTO_CONFIG=/dev/null
-install:
-  # We build moto first so the docker container doesn't try to compile it as well, also note we don't use
-  # -d for docker run so the logs show up in travis
-  # Python images come from here: https://hub.docker.com/_/python/
-  - |
-    python setup.py sdist
-
-    if [ "$TEST_SERVER_MODE" = "true" ]; then
-      docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${TRAVIS_PYTHON_VERSION}-stretch /moto/travis_moto_server.sh &
-      export AWS_SECRET_ACCESS_KEY=foobar_secret
-      export AWS_ACCESS_KEY_ID=foobar_key
-    fi
-    travis_retry pip install boto==2.45.0
-    travis_retry pip install boto3
-    travis_retry pip install dist/moto*.gz
-    travis_retry pip install coveralls==1.1
-    travis_retry pip install -r requirements-dev.txt
-
-    if [ "$TEST_SERVER_MODE" = "true" ]; then
-      python wait_for.py
-    fi
-script:
-  - make test
-after_success:
-  - coveralls
+language: python
+sudo: false
+services:
+  - docker
+python:
+  - 2.7
+  - 3.6
+env:
+    - TEST_SERVER_MODE=false
+    - TEST_SERVER_MODE=true
+# Due to incomplete Python 3.7 support on Travis CI (
+# https://github.com/travis-ci/travis-ci/issues/9815),
+# using a matrix is necessary
+matrix:
+  include:
+    - python: 3.7
+      env: TEST_SERVER_MODE=false
+      dist: xenial
+      sudo: true
+    - python: 3.7
+      env: TEST_SERVER_MODE=true
+      dist: xenial
+      sudo: true
+before_install:
+  - export BOTO_CONFIG=/dev/null
+  - export AWS_SECRET_ACCESS_KEY=foobar_secret
+  - export AWS_ACCESS_KEY_ID=foobar_key
+install:
+  # We build moto first so the docker container doesn't try to compile it as well, also note we don't use
+  # -d for docker run so the logs show up in travis
+  # Python images come from here: https://hub.docker.com/_/python/
+  - |
+    python setup.py sdist
+
+    if [ "$TEST_SERVER_MODE" = "true" ]; then
+      docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${TRAVIS_PYTHON_VERSION}-stretch /moto/travis_moto_server.sh &
+    fi
+    travis_retry pip install boto==2.45.0
+    travis_retry pip install boto3
+    travis_retry pip install dist/moto*.gz
+    travis_retry pip install coveralls==1.1
+    travis_retry pip install -r requirements-dev.txt
+
+    if [ "$TEST_SERVER_MODE" = "true" ]; then
+      python wait_for.py
+    fi
+script:
+  - make test
+after_success:
+  - coveralls

CHANGELOG.md

@@ -1,6 +1,11 @@
 Moto Changelog
 ===================
 
+1.3.7
+-----
+
+    * Switch from mocking requests to using before-send for AWS calls
+
 1.3.6
 -----
 

moto/__init__.py

@@ -3,7 +3,7 @@ import logging
 # logging.getLogger('boto').setLevel(logging.CRITICAL)
 
 __title__ = 'moto'
-__version__ = '1.3.6'
+__version__ = '1.3.7'
 
 from .acm import mock_acm  # flake8: noqa
 from .apigateway import mock_apigateway, mock_apigateway_deprecated  # flake8: noqa

moto/apigateway/models.py

@@ -10,6 +10,7 @@ from boto3.session import Session
 import responses
 from moto.core import BaseBackend, BaseModel
 from .utils import create_id
+from moto.core.utils import path_url
 from .exceptions import StageNotFoundException, ApiKeyNotFoundException
 
 STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}"
@@ -372,7 +373,8 @@ class RestAPI(BaseModel):
         # TODO deal with no matching resource
 
     def resource_callback(self, request):
-        path_after_stage_name = '/'.join(request.path_url.split("/")[2:])
+        path = path_url(request.url)
+        path_after_stage_name = '/'.join(path.split("/")[2:])
         if not path_after_stage_name:
             path_after_stage_name = '/'
 

moto/awslambda/responses.py

@@ -7,7 +7,7 @@ try:
 except ImportError:
     from urllib.parse import unquote
 
-from moto.core.utils import amz_crc32, amzn_request_id
+from moto.core.utils import amz_crc32, amzn_request_id, path_url
 from moto.core.responses import BaseResponse
 from .models import lambda_backends
 
@@ -94,7 +94,7 @@ class LambdaResponse(BaseResponse):
             return self._add_policy(request, full_url, headers)
 
     def _add_policy(self, request, full_url, headers):
-        path = request.path if hasattr(request, 'path') else request.path_url
+        path = request.path if hasattr(request, 'path') else path_url(request.url)
         function_name = path.split('/')[-2]
         if self.lambda_backend.get_function(function_name):
             policy = request.body.decode('utf8')
@@ -104,7 +104,7 @@ class LambdaResponse(BaseResponse):
             return 404, {}, "{}"
 
     def _get_policy(self, request, full_url, headers):
-        path = request.path if hasattr(request, 'path') else request.path_url
+        path = request.path if hasattr(request, 'path') else path_url(request.url)
         function_name = path.split('/')[-2]
         if self.lambda_backend.get_function(function_name):
             lambda_function = self.lambda_backend.get_function(function_name)

moto/core/models.py

@@ -2,11 +2,14 @@
 from __future__ import unicode_literals
 from __future__ import absolute_import
 
-from collections import defaultdict
 import functools
 import inspect
 import re
 import six
+from io import BytesIO
+from collections import defaultdict
+from botocore.handlers import BUILTIN_HANDLERS
+from botocore.awsrequest import AWSResponse
 
 from moto import settings
 import responses
@@ -233,7 +236,111 @@ class ResponsesMockAWS(BaseMockAWS):
             pass
 
 
-MockAWS = ResponsesMockAWS
+BOTOCORE_HTTP_METHODS = [
+    'GET', 'DELETE', 'HEAD', 'OPTIONS', 'PATCH', 'POST', 'PUT'
+]
+
+
+class MockRawResponse(BytesIO):
+    def __init__(self, input):
+        if isinstance(input, six.text_type):
+            input = input.encode('utf-8')
+        super(MockRawResponse, self).__init__(input)
+
+    def stream(self, **kwargs):
+        contents = self.read()
+        while contents:
+            yield contents
+            contents = self.read()
+
+
+class BotocoreStubber(object):
+    def __init__(self):
+        self.enabled = False
+        self.methods = defaultdict(list)
+
+    def reset(self):
+        self.methods.clear()
+
+    def register_response(self, method, pattern, response):
+        matchers = self.methods[method]
+        matchers.append((pattern, response))
+
+    def __call__(self, event_name, request, **kwargs):
+        if not self.enabled:
+            return None
+
+        response = None
+        response_callback = None
+        found_index = None
+        matchers = self.methods.get(request.method)
+
+        base_url = request.url.split('?', 1)[0]
+        for i, (pattern, callback) in enumerate(matchers):
+            if pattern.match(base_url):
+                if found_index is None:
+                    found_index = i
+                    response_callback = callback
+                else:
+                    matchers.pop(found_index)
+                    break
+
+        if response_callback is not None:
+            for header, value in request.headers.items():
+                if isinstance(value, six.binary_type):
+                    request.headers[header] = value.decode('utf-8')
+            status, headers, body = response_callback(request, request.url, request.headers)
+            body = MockRawResponse(body)
+            response = AWSResponse(request.url, status, headers, body)
+
+        return response
+
+
+botocore_stubber = BotocoreStubber()
+BUILTIN_HANDLERS.append(('before-send', botocore_stubber))
+
+
+class BotocoreEventMockAWS(BaseMockAWS):
+    def reset(self):
+        botocore_stubber.reset()
+        responses_mock.reset()
+
+    def enable_patching(self):
+        botocore_stubber.enabled = True
+        for method in BOTOCORE_HTTP_METHODS:
+            for backend in self.backends_for_urls.values():
+                for key, value in backend.urls.items():
+                    pattern = re.compile(key)
+                    botocore_stubber.register_response(method, pattern, value)
+
+        if not hasattr(responses_mock, '_patcher') or not hasattr(responses_mock._patcher, 'target'):
+            responses_mock.start()
+
+        for method in RESPONSES_METHODS:
+            # for backend in default_backends.values():
+            for backend in self.backends_for_urls.values():
+                for key, value in backend.urls.items():
+                    responses_mock.add(
+                        CallbackResponse(
+                            method=method,
+                            url=re.compile(key),
+                            callback=convert_flask_to_responses_response(value),
+                            stream=True,
+                            match_querystring=False,
+                        )
+                    )
+
+    def disable_patching(self):
+        botocore_stubber.enabled = False
+        self.reset()
+
+        try:
+            responses_mock.stop()
+        except RuntimeError:
+            pass
+
+
+MockAWS = BotocoreEventMockAWS
 
 
 class ServerModeMockAWS(BaseMockAWS):

moto/core/utils.py

@@ -8,6 +8,7 @@ import random
 import re
 import six
 import string
+from six.moves.urllib.parse import urlparse
 
 
 REQUEST_ID_LONG = string.digits + string.ascii_uppercase
@@ -286,3 +287,13 @@ def amzn_request_id(f):
         return status, headers, body
 
     return _wrapper
+
+
+def path_url(url):
+    parsed_url = urlparse(url)
+    path = parsed_url.path
+    if not path:
+        path = '/'
+    if parsed_url.query:
+        path = path + '?' + parsed_url.query
+    return path

moto/ecs/exceptions.py

@@ -8,4 +8,6 @@ class ServiceNotFoundException(RESTError):
     def __init__(self, service_name):
         super(ServiceNotFoundException, self).__init__(
             error_type="ServiceNotFoundException",
-            message="The service {0} does not exist".format(service_name))
+            message="The service {0} does not exist".format(service_name),
+            template='error_json',
+        )

moto/s3/responses.py

@@ -10,6 +10,7 @@ import xmltodict
 
 from moto.packages.httpretty.core import HTTPrettyRequest
 from moto.core.responses import _TemplateEnvironmentMixin
+from moto.core.utils import path_url
 
 from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, \
     parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys
@@ -487,7 +488,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
         if isinstance(request, HTTPrettyRequest):
             path = request.path
         else:
-            path = request.full_path if hasattr(request, 'full_path') else request.path_url
+            path = request.full_path if hasattr(request, 'full_path') else path_url(request.url)
 
         if self.is_delete_keys(request, path, bucket_name):
             return self._bucket_response_delete_keys(request, body, bucket_name, headers)
@@ -708,7 +709,10 @@ class ResponseObject(_TemplateEnvironmentMixin):
             # Copy key
             # you can have a quoted ?version=abc with a version Id, so work on
             # we need to parse the unquoted string first
-            src_key_parsed = urlparse(request.headers.get("x-amz-copy-source"))
+            src_key = request.headers.get("x-amz-copy-source")
+            if isinstance(src_key, six.binary_type):
+                src_key = src_key.decode('utf-8')
+            src_key_parsed = urlparse(src_key)
             src_bucket, src_key = unquote(src_key_parsed.path).\
                 lstrip("/").split("/", 1)
             src_version_id = parse_qs(src_key_parsed.query).get(

requirements-dev.txt

@@ -1,17 +1,17 @@
--r requirements.txt
-mock
-nose
-sure==1.4.11
-coverage
-flake8==3.5.0
-freezegun
-flask
-boto>=2.45.0
-boto3>=1.4.4
-botocore>=1.12.13
-six>=1.9
-prompt-toolkit==1.0.14
-click==6.7
-inflection==0.3.1
-lxml==4.2.3
-beautifulsoup4==4.6.0
+-r requirements.txt
+mock
+nose
+sure==1.4.11
+coverage
+flake8==3.5.0
+freezegun
+flask
+boto>=2.45.0
+boto3>=1.4.4
+botocore>=1.12.13
+six>=1.9
+prompt-toolkit==1.0.14
+click==6.7
+inflection==0.3.1
+lxml==4.2.3
+beautifulsoup4==4.6.0

setup.py

@@ -1,71 +1,71 @@
-#!/usr/bin/env python
-from __future__ import unicode_literals
-import setuptools
-from setuptools import setup, find_packages
-import sys
-
-
-install_requires = [
-    "Jinja2>=2.7.3",
-    "boto>=2.36.0",
-    "boto3>=1.6.16",
-    "botocore>=1.12.13",
-    "cryptography>=2.3.0",
-    "requests>=2.5",
-    "xmltodict",
-    "six>1.9",
-    "werkzeug",
-    "pyaml",
-    "pytz",
-    "python-dateutil<3.0.0,>=2.1",
-    "python-jose<3.0.0",
-    "mock",
-    "docker>=2.5.1",
-    "jsondiff==1.1.1",
-    "aws-xray-sdk!=0.96,>=0.93",
-    "responses>=0.9.0",
-]
-
-extras_require = {
-    'server': ['flask'],
-}
-
-# https://hynek.me/articles/conditional-python-dependencies/
-if int(setuptools.__version__.split(".", 1)[0]) < 18:
-    if sys.version_info[0:2] < (3, 3):
-        install_requires.append("backports.tempfile")
-else:
-    extras_require[":python_version<'3.3'"] = ["backports.tempfile"]
-
-
-setup(
-    name='moto',
-    version='1.3.7',
-    description='A library that allows your python tests to easily'
-                ' mock out the boto library',
-    author='Steve Pulec',
-    author_email='spulec@gmail.com',
-    url='https://github.com/spulec/moto',
-    entry_points={
-        'console_scripts': [
-            'moto_server = moto.server:main',
-        ],
-    },
-    packages=find_packages(exclude=("tests", "tests.*")),
-    install_requires=install_requires,
-    extras_require=extras_require,
-    include_package_data=True,
-    license="Apache",
-    test_suite="tests",
-    classifiers=[
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.3",
-        "Programming Language :: Python :: 3.4",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "License :: OSI Approved :: Apache Software License",
-        "Topic :: Software Development :: Testing",
-    ],
+#!/usr/bin/env python
+from __future__ import unicode_literals
+import setuptools
+from setuptools import setup, find_packages
+import sys
+
+
+install_requires = [
+    "Jinja2>=2.7.3",
+    "boto>=2.36.0",
+    "boto3>=1.6.16",
+    "botocore>=1.12.13",
+    "cryptography>=2.3.0",
+    "requests>=2.5",
+    "xmltodict",
+    "six>1.9",
+    "werkzeug",
+    "pyaml",
+    "pytz",
+    "python-dateutil<3.0.0,>=2.1",
+    "python-jose<3.0.0",
+    "mock",
+    "docker>=2.5.1",
+    "jsondiff==1.1.1",
+    "aws-xray-sdk!=0.96,>=0.93",
+    "responses>=0.9.0",
+]
+
+extras_require = {
+    'server': ['flask'],
+}
+
+# https://hynek.me/articles/conditional-python-dependencies/
+if int(setuptools.__version__.split(".", 1)[0]) < 18:
+    if sys.version_info[0:2] < (3, 3):
+        install_requires.append("backports.tempfile")
+else:
+    extras_require[":python_version<'3.3'"] = ["backports.tempfile"]
+
+
+setup(
+    name='moto',
+    version='1.3.7',
+    description='A library that allows your python tests to easily'
+                ' mock out the boto library',
+    author='Steve Pulec',
+    author_email='spulec@gmail.com',
+    url='https://github.com/spulec/moto',
+    entry_points={
+        'console_scripts': [
+            'moto_server = moto.server:main',
+        ],
+    },
+    packages=find_packages(exclude=("tests", "tests.*")),
+    install_requires=install_requires,
+    extras_require=extras_require,
+    include_package_data=True,
+    license="Apache",
+    test_suite="tests",
+    classifiers=[
+        "Programming Language :: Python :: 2",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "License :: OSI Approved :: Apache Software License",
+        "Topic :: Software Development :: Testing",
+    ],
 )