Cleanup responses mocking. Closes #1567

This unblocks requests to other websites with requests while Moto
is activated. It also adds a wildcard for AWS services to still
ensure no accidental requests are made for unmocked services

moto/apigateway/models.py

@@ -388,10 +388,16 @@ class RestAPI(BaseModel):
         stage_url_upper = STAGE_URL.format(api_id=self.id.upper(),
             region_name=self.region_name, stage_name=stage_name)
 
-        responses.add_callback(responses.GET, stage_url_lower,
+        for url in [stage_url_lower, stage_url_upper]:
-                               callback=self.resource_callback)
+            responses._default_mock._matches.insert(0,
-        responses.add_callback(responses.GET, stage_url_upper,
+                responses.CallbackResponse(
-                               callback=self.resource_callback)
+                    url=url,
+                    method=responses.GET,
+                    callback=self.resource_callback,
+                    content_type="text/plain",
+                    match_querystring=False,
+                )
+            )
 
     def create_stage(self, name, deployment_id, variables=None, description='', cacheClusterEnabled=None, cacheClusterSize=None):
         if variables is None:

moto/core/models.py

@@ -193,53 +193,8 @@ class CallbackResponse(responses.CallbackResponse):
 
 botocore_mock = responses.RequestsMock(assert_all_requests_are_fired=False, target='botocore.vendored.requests.adapters.HTTPAdapter.send')
 responses_mock = responses._default_mock
-
+# Add passthrough to allow any other requests to work
-
+responses_mock.add_passthru("http")
-class ResponsesMockAWS(BaseMockAWS):
-    def reset(self):
-        botocore_mock.reset()
-        responses_mock.reset()
-
-    def enable_patching(self):
-        if not hasattr(botocore_mock, '_patcher') or not hasattr(botocore_mock._patcher, 'target'):
-            # Check for unactivated patcher
-            botocore_mock.start()
-
-        if not hasattr(responses_mock, '_patcher') or not hasattr(responses_mock._patcher, 'target'):
-            responses_mock.start()
-
-        for method in RESPONSES_METHODS:
-            for backend in self.backends_for_urls.values():
-                for key, value in backend.urls.items():
-                    responses_mock.add(
-                        CallbackResponse(
-                            method=method,
-                            url=re.compile(key),
-                            callback=convert_flask_to_responses_response(value),
-                            stream=True,
-                            match_querystring=False,
-                        )
-                    )
-                    botocore_mock.add(
-                        CallbackResponse(
-                            method=method,
-                            url=re.compile(key),
-                            callback=convert_flask_to_responses_response(value),
-                            stream=True,
-                            match_querystring=False,
-                        )
-                    )
-
-    def disable_patching(self):
-        try:
-            botocore_mock.stop()
-        except RuntimeError:
-            pass
-
-        try:
-            responses_mock.stop()
-        except RuntimeError:
-            pass
 
 
 BOTOCORE_HTTP_METHODS = [
@@ -306,6 +261,14 @@ botocore_stubber = BotocoreStubber()
 BUILTIN_HANDLERS.append(('before-send', botocore_stubber))
 
 
+def not_implemented_callback(request):
+    status = 400
+    headers = {}
+    response = "The method is not implemented"
+
+    return status, headers, response
+
+
 class BotocoreEventMockAWS(BaseMockAWS):
     def reset(self):
         botocore_stubber.reset()
@@ -335,6 +298,24 @@ class BotocoreEventMockAWS(BaseMockAWS):
                             match_querystring=False,
                         )
                     )
+            responses_mock.add(
+                CallbackResponse(
+                    method=method,
+                    url=re.compile("https?://.+.amazonaws.com/.*"),
+                    callback=not_implemented_callback,
+                    stream=True,
+                    match_querystring=False,
+                )
+            )
+            botocore_mock.add(
+                CallbackResponse(
+                    method=method,
+                    url=re.compile("https?://.+.amazonaws.com/.*"),
+                    callback=not_implemented_callback,
+                    stream=True,
+                    match_querystring=False,
+                )
+            )
 
     def disable_patching(self):
         botocore_stubber.enabled = False

tests/test_core/test_request_mocking.py

@@ -0,0 +1,21 @@
+import requests
+import sure  # noqa
+
+import boto3
+from moto import mock_sqs
+
+
+@mock_sqs
+def test_passthrough_requests():
+    conn = boto3.client("sqs", region_name='us-west-1')
+    conn.create_queue(QueueName="queue1")
+
+    res = requests.get("https://httpbin.org/ip")
+    assert res.status_code == 200
+
+
+@mock_sqs
+def test_requests_to_amazon_subdomains_dont_work():
+    res = requests.get("https://fakeservice.amazonaws.com/foo/bar")
+    assert res.content == b"The method is not implemented"
+    assert res.status_code == 400