API Gateway - improve mocking of public API (#4613)

IMPLEMENTATION_COVERAGE.md

@@ -22,7 +22,7 @@
 
 ## apigateway
 <details>
-<summary>51% implemented</summary>
+<summary>55% implemented</summary>
 
 - [X] create_api_key
 - [X] create_authorizer
@@ -113,10 +113,10 @@
 - [ ] import_documentation_parts
 - [ ] import_rest_api
 - [ ] put_gateway_response
-- [ ] put_integration
+- [X] put_integration
-- [ ] put_integration_response
+- [X] put_integration_response
-- [ ] put_method
+- [X] put_method
-- [ ] put_method_response
+- [X] put_method_response
 - [ ] put_rest_api
 - [ ] tag_resource
 - [ ] test_invoke_authorizer

docs/docs/services/apigateway.rst

@@ -12,6 +12,8 @@
 apigateway
 ==========
 
+.. autoclass:: moto.apigateway.models.APIGatewayBackend
+
 |start-h3| Example usage |end-h3|
 
 .. sourcecode:: python
@@ -114,10 +116,10 @@ apigateway
 - [ ] import_documentation_parts
 - [ ] import_rest_api
 - [ ] put_gateway_response
-- [ ] put_integration
+- [X] put_integration
-- [ ] put_integration_response
+- [X] put_integration_response
-- [ ] put_method
+- [X] put_method
-- [ ] put_method_response
+- [X] put_method_response
 - [ ] put_rest_api
 - [ ] tag_resource
 - [ ] test_invoke_authorizer

moto/apigateway/integration_parsers/aws_parser.py

@@ -0,0 +1,24 @@
+import requests
+
+
+class TypeAwsParser:
+    def invoke(self, request, integration):
+        # integration.uri = arn:aws:apigateway:{region}:{subdomain.service|service}:path|action/{service_api}
+        # example value = 'arn:aws:apigateway:us-west-2:dynamodb:action/PutItem'
+        try:
+            # We need a better way to support services automatically
+            # This is how AWS does it though - sending a new HTTP request to the target service
+            arn, action = integration["uri"].split("/")
+            _, _, _, region, service, path_or_action = arn.split(":")
+            if service == "dynamodb" and path_or_action == "action":
+                target_url = f"https://dynamodb.{region}.amazonaws.com/"
+                headers = {"X-Amz-Target": f"DynamoDB_20120810.{action}"}
+                res = requests.post(target_url, request.body, headers=headers)
+                return res.status_code, res.content
+            else:
+                return (
+                    400,
+                    f"Integration for service {service} / {path_or_action} is not yet supported",
+                )
+        except Exception as e:
+            return 400, str(e)

moto/apigateway/integration_parsers/http_parser.py

@@ -0,0 +1,13 @@
+import requests
+
+
+class TypeHttpParser:
+    """
+    Parse invocations to a APIGateway resource with integration type HTTP
+    """
+
+    def invoke(self, request, integration):
+        uri = integration["uri"]
+        requests_func = getattr(requests, integration["httpMethod"].lower())
+        response = requests_func(uri)
+        return response.status_code, response.text

moto/apigateway/integration_parsers/unknown_parser.py

@@ -0,0 +1,8 @@
+class TypeUnknownParser:
+    """
+    Parse invocations to a APIGateway resource with an unknown integration type
+    """
+
+    def invoke(self, request, integration):
+        _type = integration["type"]
+        raise NotImplementedError("The {0} type has not been implemented".format(_type))

moto/apigateway/models.py

@@ -3,9 +3,9 @@ from __future__ import absolute_import
 import random
 import string
 import re
+from collections import defaultdict
 from copy import copy
 
-import requests
 import time
 
 from boto3.session import Session
@@ -18,6 +18,9 @@ import responses
 from moto.core import ACCOUNT_ID, BaseBackend, BaseModel, CloudFormationModel
 from .utils import create_id, to_path
 from moto.core.utils import path_url
+from .integration_parsers.aws_parser import TypeAwsParser
+from .integration_parsers.http_parser import TypeHttpParser
+from .integration_parsers.unknown_parser import TypeUnknownParser
 from .exceptions import (
     ApiKeyNotFoundException,
     UsagePlanNotFoundException,
@@ -199,7 +202,7 @@ class Method(CloudFormationModel, dict):
         auth_type = properties["AuthorizationType"]
         key_req = properties["ApiKeyRequired"]
         backend = apigateway_backends[region_name]
-        m = backend.create_method(
+        m = backend.put_method(
             function_id=rest_api_id,
             resource_id=resource_id,
             method_type=method_type,
@@ -209,7 +212,7 @@ class Method(CloudFormationModel, dict):
         int_method = properties["Integration"]["IntegrationHttpMethod"]
         int_type = properties["Integration"]["Type"]
         int_uri = properties["Integration"]["Uri"]
-        backend.create_integration(
+        backend.put_integration(
             function_id=rest_api_id,
             resource_id=resource_id,
             method_type=method_type,
@@ -242,6 +245,9 @@ class Resource(CloudFormationModel):
         self.path_part = path_part
         self.parent_id = parent_id
         self.resource_methods = {}
+        self.integration_parsers = defaultdict(TypeUnknownParser)
+        self.integration_parsers["HTTP"] = TypeHttpParser()
+        self.integration_parsers["AWS"] = TypeAwsParser()
 
     def to_dict(self):
         response = {
@@ -306,15 +312,11 @@ class Resource(CloudFormationModel):
         integration = self.get_integration(request.method)
         integration_type = integration["type"]
 
-        if integration_type == "HTTP":
+        status, result = self.integration_parsers[integration_type].invoke(
-            uri = integration["uri"]
+            request, integration
-            requests_func = getattr(requests, integration["httpMethod"].lower())
-            response = requests_func(uri)
-        else:
-            raise NotImplementedError(
-                "The {0} type has not been implemented".format(integration_type)
         )
-        return response.status_code, response.text
+
+        return status, result
 
     def add_method(
         self,
@@ -893,9 +895,7 @@ class RestAPI(CloudFormationModel):
 
     def resource_callback(self, request):
         path = path_url(request.url)
-        path_after_stage_name = "/".join(path.split("/")[2:])
+        path_after_stage_name = "/" + "/".join(path.split("/")[2:])
-        if not path_after_stage_name:
-            path_after_stage_name = "/"
 
         resource = self.get_resource_for_path(path_after_stage_name)
         status_code, response = resource.get_response(request)
@@ -909,17 +909,20 @@ class RestAPI(CloudFormationModel):
             api_id=self.id.upper(), region_name=self.region_name, stage_name=stage_name
         )
 
+        for resource_id, resource in self.resources.items():
+            path = resource.get_path()
+            path = "" if path == "/" else path
+
+            for http_method, method in resource.resource_methods.items():
                 for url in [stage_url_lower, stage_url_upper]:
-            responses_mock._matches.insert(
+                    callback_response = responses.CallbackResponse(
-                0,
+                        url=url + path,
-                responses.CallbackResponse(
+                        method=http_method,
-                    url=url,
-                    method=responses.GET,
                         callback=self.resource_callback,
                         content_type="text/plain",
                         match_querystring=False,
-                ),
                     )
+                    responses_mock._matches.insert(0, callback_response)
 
     def create_authorizer(
         self,
@@ -1105,6 +1108,32 @@ class BasePathMapping(BaseModel, dict):
 
 
 class APIGatewayBackend(BaseBackend):
+    """
+    API Gateway mock.
+
+    The public URLs of an API integration are mocked as well, i.e. the following would be supported in Moto:
+
+    .. sourcecode:: python
+
+        client.put_integration(
+            restApiId=api_id,
+            ...,
+            uri="http://httpbin.org/robots.txt",
+            integrationHttpMethod="GET",
+        )
+        deploy_url = f"https://{api_id}.execute-api.us-east-1.amazonaws.com/dev"
+        requests.get(deploy_url).content.should.equal(b"a fake response")
+
+    Limitations:
+     - Integrations of type HTTP are supported
+     - Integrations of type AWS with service DynamoDB are supported
+     - Other types (AWS_PROXY, MOCK, etc) are ignored
+     - Other services are not yet supported
+     - The BasePath of an API is ignored
+     - TemplateMapping is not yet supported for requests/responses
+     - This only works when using the decorators, not in ServerMode
+    """
+
     def __init__(self, region_name):
         super(APIGatewayBackend, self).__init__()
         self.apis = {}
@@ -1191,7 +1220,7 @@ class APIGatewayBackend(BaseBackend):
         resource = self.get_resource(function_id, resource_id)
         return resource.get_method(method_type)
 
-    def create_method(
+    def put_method(
         self,
         function_id,
         resource_id,
@@ -1322,7 +1351,7 @@ class APIGatewayBackend(BaseBackend):
         method_response = method.get_response(response_code)
         return method_response
 
-    def create_method_response(
+    def put_method_response(
         self,
         function_id,
         resource_id,
@@ -1352,7 +1381,7 @@ class APIGatewayBackend(BaseBackend):
         method_response = method.delete_response(response_code)
         return method_response
 
-    def create_integration(
+    def put_integration(
         self,
         function_id,
         resource_id,
@@ -1414,7 +1443,7 @@ class APIGatewayBackend(BaseBackend):
         resource = self.get_resource(function_id, resource_id)
         return resource.delete_integration(method_type)
 
-    def create_integration_response(
+    def put_integration_response(
         self,
         function_id,
         resource_id,
@@ -1458,8 +1487,7 @@ class APIGatewayBackend(BaseBackend):
         if not any(methods):
             raise NoMethodDefined()
         method_integrations = [
-            method["methodIntegration"] if "methodIntegration" in method else None
+            method.get("methodIntegration", None) for method in methods
-            for method in methods
         ]
         if not any(method_integrations):
             raise NoIntegrationDefined()

moto/apigateway/responses.py

@@ -193,7 +193,7 @@ class APIGatewayResponse(BaseResponse):
             authorizer_id = self._get_param("authorizerId")
             authorization_scopes = self._get_param("authorizationScopes")
             request_validator_id = self._get_param("requestValidatorId")
-            method = self.backend.create_method(
+            method = self.backend.put_method(
                 function_id,
                 resource_id,
                 method_type,
@@ -234,7 +234,7 @@ class APIGatewayResponse(BaseResponse):
         elif self.method == "PUT":
             response_models = self._get_param("responseModels")
             response_parameters = self._get_param("responseParameters")
-            method_response = self.backend.create_method_response(
+            method_response = self.backend.put_method_response(
                 function_id,
                 resource_id,
                 method_type,
@@ -482,7 +482,7 @@ class APIGatewayResponse(BaseResponse):
                     "httpMethod"
                 )  # default removed because it's a required parameter
 
-                integration_response = self.backend.create_integration(
+                integration_response = self.backend.put_integration(
                     function_id,
                     resource_id,
                     method_type,
@@ -526,7 +526,7 @@ class APIGatewayResponse(BaseResponse):
                 selection_pattern = self._get_param("selectionPattern")
                 response_templates = self._get_param("responseTemplates")
                 content_handling = self._get_param("contentHandling")
-                integration_response = self.backend.create_integration_response(
+                integration_response = self.backend.put_integration_response(
                     function_id,
                     resource_id,
                     method_type,

tests/test_apigateway/test_apigateway.py

@@ -2,13 +2,11 @@ import json
 
 import boto3
 from freezegun import freeze_time
-import requests
 import sure  # noqa # pylint: disable=unused-import
 from botocore.exceptions import ClientError
 
 from moto import mock_apigateway, mock_cognitoidp, settings
 from moto.core import ACCOUNT_ID
-from moto.core.models import responses_mock
 import pytest
 
 
@@ -1811,50 +1809,6 @@ def test_get_model_with_invalid_name():
     ex.value.response["Error"]["Code"].should.equal("NotFoundException")
 
 
-@mock_apigateway
-def test_http_proxying_integration():
-    responses_mock.add(
-        responses_mock.GET, "http://httpbin.org/robots.txt", body="a fake response"
-    )
-
-    region_name = "us-west-2"
-    client = boto3.client("apigateway", region_name=region_name)
-    response = client.create_rest_api(name="my_api", description="this is my api")
-    api_id = response["id"]
-
-    resources = client.get_resources(restApiId=api_id)
-    root_id = [resource for resource in resources["items"] if resource["path"] == "/"][
-        0
-    ]["id"]
-
-    client.put_method(
-        restApiId=api_id, resourceId=root_id, httpMethod="GET", authorizationType="none"
-    )
-
-    client.put_method_response(
-        restApiId=api_id, resourceId=root_id, httpMethod="GET", statusCode="200"
-    )
-
-    response = client.put_integration(
-        restApiId=api_id,
-        resourceId=root_id,
-        httpMethod="GET",
-        type="HTTP",
-        uri="http://httpbin.org/robots.txt",
-        integrationHttpMethod="GET",
-    )
-
-    stage_name = "staging"
-    client.create_deployment(restApiId=api_id, stageName=stage_name)
-
-    deploy_url = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}".format(
-        api_id=api_id, region_name=region_name, stage_name=stage_name
-    )
-
-    if not settings.TEST_SERVER_MODE:
-        requests.get(deploy_url).content.should.equal(b"a fake response")
-
-
 @mock_apigateway
 def test_api_key_value_min_length():
     region_name = "us-east-1"

tests/test_apigateway/test_apigateway_integration.py

@@ -0,0 +1,219 @@
+import boto3
+import json
+import requests
+
+from moto import mock_apigateway, mock_dynamodb2
+from moto import settings
+from moto.core.models import responses_mock
+from unittest import SkipTest
+
+
+@mock_apigateway
+def test_http_integration():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Cannot test mock of execute-api.apigateway in ServerMode")
+    responses_mock.add(
+        responses_mock.GET, "http://httpbin.org/robots.txt", body="a fake response"
+    )
+
+    region_name = "us-west-2"
+    client = boto3.client("apigateway", region_name=region_name)
+    response = client.create_rest_api(name="my_api", description="this is my api")
+    api_id = response["id"]
+
+    resources = client.get_resources(restApiId=api_id)
+    root_id = [resource for resource in resources["items"] if resource["path"] == "/"][
+        0
+    ]["id"]
+
+    client.put_method(
+        restApiId=api_id, resourceId=root_id, httpMethod="GET", authorizationType="none"
+    )
+
+    client.put_method_response(
+        restApiId=api_id, resourceId=root_id, httpMethod="GET", statusCode="200"
+    )
+
+    response = client.put_integration(
+        restApiId=api_id,
+        resourceId=root_id,
+        httpMethod="GET",
+        type="HTTP",
+        uri="http://httpbin.org/robots.txt",
+        integrationHttpMethod="GET",
+    )
+
+    stage_name = "staging"
+    client.create_deployment(restApiId=api_id, stageName=stage_name)
+
+    deploy_url = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}".format(
+        api_id=api_id, region_name=region_name, stage_name=stage_name
+    )
+
+    requests.get(deploy_url).content.should.equal(b"a fake response")
+
+
+@mock_apigateway
+@mock_dynamodb2
+def test_aws_integration_dynamodb():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Cannot test mock of execute-api.apigateway in ServerMode")
+
+    client = boto3.client("apigateway", region_name="us-west-2")
+    dynamodb = boto3.client("dynamodb", region_name="us-west-2")
+    table_name = "test_1"
+    integration_action = "arn:aws:apigateway:us-west-2:dynamodb:action/PutItem"
+    stage_name = "staging"
+
+    create_table(dynamodb, table_name)
+    api_id, _ = create_integration_test_api(client, integration_action)
+
+    client.create_deployment(restApiId=api_id, stageName=stage_name)
+
+    res = requests.put(
+        f"https://{api_id}.execute-api.us-west-2.amazonaws.com/{stage_name}",
+        json={"TableName": table_name, "Item": {"name": {"S": "the-key"}}},
+    )
+    res.status_code.should.equal(200)
+    res.content.should.equal(b"{}")
+
+
+@mock_apigateway
+@mock_dynamodb2
+def test_aws_integration_dynamodb_multiple_stages():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Cannot test mock of execute-api.apigateway in ServerMode")
+
+    client = boto3.client("apigateway", region_name="us-west-2")
+    dynamodb = boto3.client("dynamodb", region_name="us-west-2")
+    table_name = "test_1"
+    integration_action = "arn:aws:apigateway:us-west-2:dynamodb:action/PutItem"
+
+    create_table(dynamodb, table_name)
+    api_id, _ = create_integration_test_api(client, integration_action)
+
+    client.create_deployment(restApiId=api_id, stageName="dev")
+    client.create_deployment(restApiId=api_id, stageName="staging")
+
+    res = requests.put(
+        f"https://{api_id}.execute-api.us-west-2.amazonaws.com/dev",
+        json={"TableName": table_name, "Item": {"name": {"S": "the-key"}}},
+    )
+    res.status_code.should.equal(200)
+
+    res = requests.put(
+        f"https://{api_id}.execute-api.us-west-2.amazonaws.com/staging",
+        json={"TableName": table_name, "Item": {"name": {"S": "the-key"}}},
+    )
+    res.status_code.should.equal(200)
+
+    # We haven't pushed to prod yet
+    res = requests.put(
+        f"https://{api_id}.execute-api.us-west-2.amazonaws.com/prod",
+        json={"TableName": table_name, "Item": {"name": {"S": "the-key"}}},
+    )
+    res.status_code.should.equal(400)
+
+
+@mock_apigateway
+@mock_dynamodb2
+def test_aws_integration_dynamodb_multiple_resources():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Cannot test mock of execute-api.apigateway in ServerMode")
+
+    client = boto3.client("apigateway", region_name="us-west-2")
+    dynamodb = boto3.client("dynamodb", region_name="us-west-2")
+    table_name = "test_1"
+    create_table(dynamodb, table_name)
+
+    # Create API integration to PutItem
+    integration_action = "arn:aws:apigateway:us-west-2:dynamodb:action/PutItem"
+    api_id, root_id = create_integration_test_api(client, integration_action)
+
+    # Create API integration to GetItem
+    res = client.create_resource(restApiId=api_id, parentId=root_id, pathPart="item")
+    parent_id = res["id"]
+    integration_action = "arn:aws:apigateway:us-west-2:dynamodb:action/GetItem"
+    api_id, root_id = create_integration_test_api(
+        client,
+        integration_action,
+        api_id=api_id,
+        parent_id=parent_id,
+        http_method="GET",
+    )
+
+    client.create_deployment(restApiId=api_id, stageName="dev")
+
+    # Put item at the root resource
+    res = requests.put(
+        f"https://{api_id}.execute-api.us-west-2.amazonaws.com/dev",
+        json={
+            "TableName": table_name,
+            "Item": {"name": {"S": "the-key"}, "attr2": {"S": "sth"}},
+        },
+    )
+    res.status_code.should.equal(200)
+
+    # Get item from child resource
+    res = requests.get(
+        f"https://{api_id}.execute-api.us-west-2.amazonaws.com/dev/item",
+        json={"TableName": table_name, "Key": {"name": {"S": "the-key"}}},
+    )
+    res.status_code.should.equal(200)
+    json.loads(res.content).should.equal(
+        {"Item": {"name": {"S": "the-key"}, "attr2": {"S": "sth"}}}
+    )
+
+
+def create_table(dynamodb, table_name):
+    # Create DynamoDB table
+    dynamodb.create_table(
+        TableName=table_name,
+        KeySchema=[{"AttributeName": "name", "KeyType": "HASH"}],
+        AttributeDefinitions=[{"AttributeName": "name", "AttributeType": "S"}],
+        BillingMode="PAY_PER_REQUEST",
+    )
+
+
+def create_integration_test_api(
+    client, integration_action, api_id=None, parent_id=None, http_method="PUT"
+):
+    if not api_id:
+        # We do not have a root yet - create the API first
+        response = client.create_rest_api(name="my_api", description="this is my api")
+        api_id = response["id"]
+    if not parent_id:
+        resources = client.get_resources(restApiId=api_id)
+        parent_id = [
+            resource for resource in resources["items"] if resource["path"] == "/"
+        ][0]["id"]
+
+    client.put_method(
+        restApiId=api_id,
+        resourceId=parent_id,
+        httpMethod=http_method,
+        authorizationType="NONE",
+    )
+    client.put_method_response(
+        restApiId=api_id,
+        resourceId=parent_id,
+        httpMethod=http_method,
+        statusCode="200",
+    )
+    client.put_integration(
+        restApiId=api_id,
+        resourceId=parent_id,
+        httpMethod=http_method,
+        type="AWS",
+        uri=integration_action,
+        integrationHttpMethod=http_method,
+    )
+    client.put_integration_response(
+        restApiId=api_id,
+        resourceId=parent_id,
+        httpMethod=http_method,
+        statusCode="200",
+        selectionPattern="",
+        responseTemplates={"application/json": "{}"},
+    )
+    return api_id, parent_id