Propagate MotoHost via env vars to Lambda (#4658)

.github/workflows/dockertests.yml

@@ -0,0 +1,209 @@
+name: DockerTests
+
+on: [push, pull_request]
+
+jobs:
+  cache:
+    name: Caching
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ 3.9 ]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Get pip cache dir
+        id: pip-cache-dir
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+      - name: pip cache
+        id: pip-cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache-dir.outputs.dir }}
+          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
+      - name: Update pip
+        if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
+        run: |
+          python -m pip install --upgrade pip
+      - name: Install project dependencies
+        if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
+        run: |
+          pip install -r requirements-dev.txt
+
+  test_custom_port:
+    name: Test Custom Port
+    runs-on: ubuntu-latest
+    needs: cache
+    strategy:
+      matrix:
+        python-version: [3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Start MotoServer on an unusual port
+      run: |
+        python setup.py sdist
+        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e MOTO_PORT=4555 -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4555:4555 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
+        MOTO_PORT=4555 python scripts/ci_wait_for_server.py
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+    - name: pip cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
+    - name: Update pip
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install project dependencies
+      run: |
+        pip install -r requirements-dev.txt
+    - name: Test
+      env:
+        TEST_SERVER_MODE: ${{ true }}
+        MOTO_PORT: 4555
+      run: |
+        pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_environment_port tests/test_cloudformation/test_cloudformation_custom_resources.py
+    - name: Collect Logs
+      if: always()
+      run: |
+        mkdir serverlogs1
+        pwd
+        ls -la
+        cp server_output.log serverlogs1/server_output.log
+        docker stop motoserver
+    - name: Archive Logs
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: motoserver-${{ matrix.python-version }}
+        path: |
+          serverlogs1/*
+
+  test_custom_name:
+    name: Test Custom Network Name
+    runs-on: ubuntu-latest
+    needs: cache
+    strategy:
+      matrix:
+        python-version: [3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Start MotoServer on a custom Docker network bridge
+      run: |
+        python setup.py sdist
+        docker network create -d bridge my-custom-network
+        docker run --rm -t -e TEST_SERVER_MODE=true -e MOTO_DOCKER_NETWORK_NAME=my-custom-network -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 --network my-custom-network -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
+        python scripts/ci_wait_for_server.py
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+    - name: pip cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
+    - name: Update pip
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install project dependencies
+      run: |
+        pip install -r requirements-dev.txt
+    - name: Test
+      env:
+        TEST_SERVER_MODE: ${{ true }}
+      run: |
+        pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_environment_port tests/test_cloudformation/test_cloudformation_custom_resources.py
+    - name: Collect Logs
+      if: always()
+      run: |
+        mkdir serverlogs2
+        pwd
+        ls -la
+        cp server_output.log serverlogs2/server_output.log
+    - name: Archive logs
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: motoserver-${{ matrix.python-version }}
+        path: |
+          serverlogs2/*
+
+  test_custom_networkmode:
+    name: Pass NetworkMode to AWSLambda
+    runs-on: ubuntu-latest
+    needs: cache
+    strategy:
+      matrix:
+        python-version: [3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Start MotoServer on an unusual port
+      run: |
+        python setup.py sdist
+        docker run --rm -t -e MOTO_DOCKER_NETWORK_MODE=host -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e MOTO_PORT=4555 -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4555:4555 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
+        MOTO_PORT=4555 python scripts/ci_wait_for_server.py
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+    - name: pip cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
+    - name: Update pip
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install project dependencies
+      run: |
+        pip install -r requirements-dev.txt
+    - name: Test
+      env:
+        TEST_SERVER_MODE: ${{ true }}
+        MOTO_PORT: 4555
+        MOTO_DOCKER_NETWORK_MODE: host
+      run: |
+        pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_networkmode tests/test_cloudformation/test_cloudformation_custom_resources.py
+    - name: Collect Logs
+      if: always()
+      run: |
+        mkdir serverlogs3
+        pwd
+        ls -la
+        cp server_output.log serverlogs3/server_output.log
+    - name: Archive Logs
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: motoserver-${{ matrix.python-version }}
+        path: |
+          serverlogs3/*

docs/docs/services/lambda.rst

@@ -12,6 +12,8 @@
 lambda
 ======
 
+.. autoclass:: moto.awslambda.models.LambdaBackend
+
 |start-h3| Example usage |end-h3|
 
 .. sourcecode:: python

moto/awslambda/models.py

@@ -566,17 +566,30 @@ class LambdaFunction(CloudFormationModel, DockerModel):
             }
 
             env_vars.update(self.environment_vars)
+            env_vars["MOTO_HOST"] = settings.moto_server_host()
+            env_vars["MOTO_PORT"] = settings.moto_server_port()
+            env_vars[
+                "MOTO_HTTP_ENDPOINT"
+            ] = f'{env_vars["MOTO_HOST"]}:{env_vars["MOTO_PORT"]}'
 
             container = exit_code = None
             log_config = docker.types.LogConfig(type=docker.types.LogConfig.types.JSON)
 
             with _DockerDataVolumeContext(self) as data_vol:
                 try:
-                    run_kwargs = (
+                    run_kwargs = dict()
-                        dict(links={"motoserver": "motoserver"})
+                    network_name = settings.moto_network_name()
-                        if settings.TEST_SERVER_MODE
+                    network_mode = settings.moto_network_mode()
-                        else {}
+                    if network_name:
-                    )
+                        run_kwargs["network"] = network_name
+                    elif network_mode:
+                        run_kwargs["network_mode"] = network_mode
+                    elif settings.TEST_SERVER_MODE:
+                        # AWSLambda can make HTTP requests to a Docker container called 'motoserver'
+                        # Only works if our Docker-container is named 'motoserver'
+                        # TODO: should remove this and rely on 'network_mode' instead, as this is too tightly coupled with our own test setup
+                        run_kwargs["links"] = {"motoserver": "motoserver"}
+
                     # add host.docker.internal host on linux to emulate Mac + Windows behavior
                     #   for communication with other mock AWS services running on localhost
                     if platform == "linux" or platform == "linux2":
@@ -595,7 +608,7 @@ class LambdaFunction(CloudFormationModel, DockerModel):
                         environment=env_vars,
                         detach=True,
                         log_config=log_config,
-                        **run_kwargs
+                        **run_kwargs,
                     )
                 finally:
                     if container:
@@ -1104,6 +1117,51 @@ class LayerStorage(object):
 
 
 class LambdaBackend(BaseBackend):
+    """
+Implementation of the AWS Lambda endpoint.
+Invoking functions is supported - they will run inside a Docker container, emulating the real AWS behaviour as closely as possible.
+
+It is possible to connect from AWS Lambdas to other services, as long as you are running Moto in ServerMode.
+The Lambda has access to environment variables `MOTO_HOST` and `MOTO_PORT`, which can be used to build the url that MotoServer runs on:
+
+.. sourcecode:: python
+
+    def lambda_handler(event, context):
+        host = os.environ.get("MOTO_HOST")
+        port = os.environ.get("MOTO_PORT")
+        url = host + ":" + port
+        ec2 = boto3.client('ec2', region_name='us-west-2', endpoint_url=url)
+
+        # Or even simpler:
+        full_url = os.environ.get("MOTO_HTTP_ENDPOINT")
+        ec2 = boto3.client("ec2", region_name="eu-west-1", endpoint_url=full_url)
+
+        ec2.do_whatever_inside_the_existing_moto_server()
+
+Moto will run on port 5000 by default. This can be overwritten by setting an environment variable when starting Moto:
+
+.. sourcecode:: bash
+
+    # This env var will be propagated to the Docker containers
+    MOTO_PORT=5000 moto_server
+
+The Docker container uses the default network mode, `bridge`.
+The following environment variables are available for fine-grained control over the Docker connection options:
+
+.. sourcecode:: bash
+
+    # Provide the name of a custom network to connect to
+    MOTO_DOCKER_NETWORK_NAME=mycustomnetwork moto_server
+
+    # Override the network mode
+    # For example, network_mode=host would use the network of the host machine
+    # Note that this option will be ignored if MOTO_DOCKER_NETWORK_NAME is also set
+    MOTO_DOCKER_NETWORK_MODE=host moto_server
+
+
+.. note:: When using the decorators, a Docker container cannot reach Moto, as it does not run as a server. Any boto3-invocations used within your Lambda will try to connect to AWS.
+    """
+
     def __init__(self, region_name):
         self._lambdas = LambdaStorage()
         self._event_source_mappings = {}

moto/cloudformation/custom_model.py

@@ -55,16 +55,21 @@ class CustomModel(CloudFormationModel):
         stack = cloudformation_backends[region_name].get_stack(stack_id)
         stack.add_custom_resource(custom_resource)
 
-        event = {
-            "RequestType": "Create",
-            "ServiceToken": service_token,
         # A request will be send to this URL to indicate success/failure
         # This request will be coming from inside a Docker container
         # Note that, in order to reach the Moto host, the Moto-server should be listening on 0.0.0.0
         #
         # Alternative: Maybe we should let the user pass in a container-name where Moto is running?
         # Similar to how we know for sure that the container in our CI is called 'motoserver'
-            "ResponseURL": f"{settings.moto_server_host()}/cloudformation_{region_name}/cfnresponse?stack={stack_id}",
+        host = f"{settings.moto_server_host()}:{settings.moto_server_port()}"
+        response_url = (
+            f"{host}/cloudformation_{region_name}/cfnresponse?stack={stack_id}"
+        )
+
+        event = {
+            "RequestType": "Create",
+            "ServiceToken": service_token,
+            "ResponseURL": response_url,
             "StackId": stack_id,
             "RequestId": request_id,
             "LogicalResourceId": logical_id,

moto/core/models.py

@@ -387,12 +387,16 @@ MockAWS = BotocoreEventMockAWS
 
 
 class ServerModeMockAWS(BaseMockAWS):
+    def __init__(self, *args, **kwargs):
+        self.port = settings.moto_server_port()
+        super().__init__(*args, **kwargs)
+
     def reset(self):
         call_reset_api = os.environ.get("MOTO_CALL_RESET_API")
         if not call_reset_api or call_reset_api.lower() != "false":
             import requests
 
-            requests.post("http://localhost:5000/moto-api/reset")
+            requests.post(f"http://localhost:{self.port}/moto-api/reset")
 
     def enable_patching(self, reset=True):
         if self.__class__.nested_count == 1 and reset:
@@ -410,12 +414,12 @@ class ServerModeMockAWS(BaseMockAWS):
                     config = Config(user_agent_extra="region/" + region)
                     kwargs["config"] = config
             if "endpoint_url" not in kwargs:
-                kwargs["endpoint_url"] = "http://localhost:5000"
+                kwargs["endpoint_url"] = f"http://localhost:{self.port}"
             return real_boto3_client(*args, **kwargs)
 
         def fake_boto3_resource(*args, **kwargs):
             if "endpoint_url" not in kwargs:
-                kwargs["endpoint_url"] = "http://localhost:5000"
+                kwargs["endpoint_url"] = f"http://localhost:{self.port}"
             return real_boto3_resource(*args, **kwargs)
 
         self._client_patcher = patch("boto3.client", fake_boto3_client)

moto/core/responses.py

@@ -141,11 +141,13 @@ class ActionAuthenticatorMixin(object):
 
     @staticmethod
     def set_initial_no_auth_action_count(initial_no_auth_action_count):
+        _port = settings.moto_server_port()
+
         def decorator(function):
             def wrapper(*args, **kwargs):
                 if settings.TEST_SERVER_MODE:
                     response = requests.post(
-                        "http://localhost:5000/moto-api/reset-auth",
+                        f"http://localhost:{_port}/moto-api/reset-auth",
                         data=str(initial_no_auth_action_count).encode("utf-8"),
                     )
                     original_initial_no_auth_action_count = response.json()[
@@ -163,7 +165,7 @@ class ActionAuthenticatorMixin(object):
                 finally:
                     if settings.TEST_SERVER_MODE:
                         requests.post(
-                            "http://localhost:5000/moto-api/reset-auth",
+                            f"http://localhost:{_port}/moto-api/reset-auth",
                             data=str(original_initial_no_auth_action_count).encode(
                                 "utf-8"
                             ),

moto/server.py

@@ -4,7 +4,6 @@ import json
 import os
 import signal
 import sys
-from functools import partial
 from threading import Lock
 
 from flask import Flask
@@ -321,9 +320,7 @@ def create_backend_app(service):
     return backend_app
 
 
-def signal_handler(reset_server_port, signum, frame):
+def signal_handler(signum, frame):
-    if reset_server_port:
-        del os.environ["MOTO_PORT"]
     sys.exit(0)
 
 
@@ -371,14 +368,12 @@ def main(argv=None):
 
     args = parser.parse_args(argv)
 
-    reset_server_port = False
     if "MOTO_PORT" not in os.environ:
-        reset_server_port = True
         os.environ["MOTO_PORT"] = f"{args.port}"
 
     try:
-        signal.signal(signal.SIGINT, partial(signal_handler, reset_server_port))
+        signal.signal(signal.SIGINT, signal_handler)
-        signal.signal(signal.SIGTERM, partial(signal_handler, reset_server_port))
+        signal.signal(signal.SIGTERM, signal_handler)
     except Exception:
         pass  # ignore "ValueError: signal only works in main thread"
 

moto/settings.py

@@ -56,12 +56,18 @@ def moto_server_port():
 
 
 def moto_server_host():
-    _port = moto_server_port()
     if is_docker():
-        host = get_docker_host()
+        return get_docker_host()
     else:
-        host = "http://host.docker.internal"
+        return "http://host.docker.internal"
-    return f"{host}:{_port}"
+
+
+def moto_network_name():
+    return os.environ.get("MOTO_DOCKER_NETWORK_NAME")
+
+
+def moto_network_mode():
+    return os.environ.get("MOTO_DOCKER_NETWORK_MODE")
 
 
 def is_docker():
@@ -77,7 +83,20 @@ def get_docker_host():
     try:
         cmd = "curl -s --unix-socket /run/docker.sock http://docker/containers/$HOSTNAME/json"
         container_info = os.popen(cmd).read()
-        _ip = json.loads(container_info)["NetworkSettings"]["IPAddress"]
+        network_settings = json.loads(container_info)["NetworkSettings"]
+        network_name = moto_network_name()
+        if network_name and network_name in network_settings["Networks"]:
+            _ip = network_settings["Networks"][network_name]["IPAddress"]
+        else:
+            _ip = network_settings["IPAddress"]
+            if network_name:
+                print(
+                    f"WARNING - Moto couldn't find network '{network_name}' - defaulting to {_ip}"
+                )
         return f"http://{_ip}"
-    except:  # noqa
+    except Exception as e:  # noqa
+        print(
+            "WARNING - Unable to parse Docker API response. Defaulting to 'host.docker.internal'"
+        )
+        print(f"{type(e)}::{e}")
         return "http://host.docker.internal"

tests/test_awslambda/test_lambda_invoke.py

@@ -13,12 +13,15 @@ from moto import (
     settings,
 )
 from uuid import uuid4
+from unittest import SkipTest
 from .utilities import (
     get_role_name,
     get_test_zip_file_error,
     get_test_zip_file1,
     get_zip_with_multiple_files,
     get_test_zip_file2,
+    get_lambda_using_environment_port,
+    get_lambda_using_network_mode,
     get_test_zip_largeresponse,
 )
 
@@ -146,6 +149,70 @@ def test_invoke_event_function():
     json.loads(success_result["Payload"].read().decode("utf-8")).should.equal(in_data)
 
 
+@pytest.mark.network
+@mock_lambda
+def test_invoke_lambda_using_environment_port():
+    if not settings.TEST_SERVER_MODE:
+        raise SkipTest("Can only test environment variables in server mode")
+    conn = boto3.client("lambda", _lambda_region)
+    function_name = str(uuid4())[0:6]
+    conn.create_function(
+        FunctionName=function_name,
+        Runtime="python3.7",
+        Role=get_role_name(),
+        Handler="lambda_function.lambda_handler",
+        Code={"ZipFile": get_lambda_using_environment_port()},
+    )
+
+    success_result = conn.invoke(
+        FunctionName=function_name, InvocationType="Event", Payload="{}"
+    )
+
+    success_result["StatusCode"].should.equal(202)
+    response = success_result["Payload"].read()
+    response = json.loads(response.decode("utf-8"))
+
+    functions = response["functions"]
+    function_names = [f["FunctionName"] for f in functions]
+    function_names.should.contain(function_name)
+
+    # Host matches the full URL, so one of:
+    # http://host.docker.internal:5000
+    # http://172.0.2.1:5000
+    # http://172.0.1.1:4555
+    response["host"].should.match("http://.+:[0-9]{4}")
+
+
+@pytest.mark.network
+@mock_lambda
+def test_invoke_lambda_using_networkmode():
+    """
+    Special use case - verify that Lambda can send a request to 'http://localhost'
+    This is only possible when the `network_mode` is set to host in the Docker args
+    Test is only run in our CI (for now)
+    """
+    if not settings.moto_network_mode():
+        raise SkipTest("Can only test this when NETWORK_MODE is specified")
+    conn = boto3.client("lambda", _lambda_region)
+    function_name = str(uuid4())[0:6]
+    conn.create_function(
+        FunctionName=function_name,
+        Runtime="python3.7",
+        Role=get_role_name(),
+        Handler="lambda_function.lambda_handler",
+        Code={"ZipFile": get_lambda_using_network_mode()},
+    )
+
+    success_result = conn.invoke(
+        FunctionName=function_name, InvocationType="Event", Payload="{}"
+    )
+
+    response = success_result["Payload"].read()
+    functions = json.loads(response.decode("utf-8"))["response"]
+    function_names = [f["FunctionName"] for f in functions]
+    function_names.should.contain(function_name)
+
+
 @pytest.mark.network
 @mock_lambda
 def test_invoke_function_with_multiple_files_in_zip():

tests/test_awslambda/utilities.py

@@ -48,6 +48,42 @@ def lambda_handler(event, context):
     return _process_lambda(func_str)
 
 
+def get_lambda_using_environment_port():
+    func_str = """
+import boto3
+import os
+
+def lambda_handler(event, context):
+    base_url = os.environ.get("MOTO_HOST")
+    port = os.environ.get("MOTO_PORT")
+    url = base_url + ":" + port
+    conn = boto3.client('lambda', region_name='us-west-2', endpoint_url=url)
+
+    full_url = os.environ["MOTO_HTTP_ENDPOINT"]
+
+    functions = conn.list_functions()["Functions"]
+
+    return {'functions': functions, 'host': full_url}
+"""
+    return _process_lambda(func_str)
+
+
+def get_lambda_using_network_mode():
+    func_str = """
+import boto3
+import os
+
+def lambda_handler(event, context):
+    port = os.environ.get("MOTO_PORT")
+    url = "http://localhost:" + port
+    conn = boto3.client('lambda', region_name='us-west-2', endpoint_url=url)
+
+    functions = conn.list_functions()["Functions"]
+    return {'response': functions}
+"""
+    return _process_lambda(func_str)
+
+
 def get_test_zip_file3():
     pfunc = """
 def lambda_handler(event, context):