From 3ba3f1460f07833bf96f62f7e8e47eddc292306b Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 27 Jan 2022 11:04:03 -0100 Subject: [PATCH] Propagate MotoHost via env vars to Lambda (#4658) --- .github/workflows/dockertests.yml | 209 +++++++++++++++++++++ docs/docs/services/lambda.rst | 2 + moto/awslambda/models.py | 70 ++++++- moto/cloudformation/custom_model.py | 19 +- moto/core/models.py | 10 +- moto/core/responses.py | 6 +- moto/server.py | 11 +- moto/settings.py | 31 ++- tests/test_awslambda/test_lambda_invoke.py | 67 +++++++ tests/test_awslambda/utilities.py | 36 ++++ 10 files changed, 429 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/dockertests.yml diff --git a/.github/workflows/dockertests.yml b/.github/workflows/dockertests.yml new file mode 100644 index 000000000..0245c47b4 --- /dev/null +++ b/.github/workflows/dockertests.yml @@ -0,0 +1,209 @@ +name: DockerTests + +on: [push, pull_request] + +jobs: + cache: + name: Caching + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ 3.9 ] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Get pip cache dir + id: pip-cache-dir + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: pip cache + id: pip-cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }} + - name: Update pip + if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }} + run: | + python -m pip install --upgrade pip + - name: Install project dependencies + if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }} + run: | + pip install -r requirements-dev.txt + + test_custom_port: + name: Test Custom Port + runs-on: ubuntu-latest + needs: cache + strategy: + matrix: + python-version: [3.9] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Start MotoServer on an unusual port + run: | + python setup.py sdist + docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e MOTO_PORT=4555 -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4555:4555 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh & + MOTO_PORT=4555 python scripts/ci_wait_for_server.py + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }} + - name: Update pip + run: | + python -m pip install --upgrade pip + - name: Install project dependencies + run: | + pip install -r requirements-dev.txt + - name: Test + env: + TEST_SERVER_MODE: ${{ true }} + MOTO_PORT: 4555 + run: | + pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_environment_port tests/test_cloudformation/test_cloudformation_custom_resources.py + - name: Collect Logs + if: always() + run: | + mkdir serverlogs1 + pwd + ls -la + cp server_output.log serverlogs1/server_output.log + docker stop motoserver + - name: Archive Logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: motoserver-${{ matrix.python-version }} + path: | + serverlogs1/* + + test_custom_name: + name: Test Custom Network Name + runs-on: ubuntu-latest + needs: cache + strategy: + matrix: + python-version: [3.9] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Start MotoServer on a custom Docker network bridge + run: | + python setup.py sdist + docker network create -d bridge my-custom-network + docker run --rm -t -e TEST_SERVER_MODE=true -e MOTO_DOCKER_NETWORK_NAME=my-custom-network -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 --network my-custom-network -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh & + python scripts/ci_wait_for_server.py + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }} + - name: Update pip + run: | + python -m pip install --upgrade pip + - name: Install project dependencies + run: | + pip install -r requirements-dev.txt + - name: Test + env: + TEST_SERVER_MODE: ${{ true }} + run: | + pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_environment_port tests/test_cloudformation/test_cloudformation_custom_resources.py + - name: Collect Logs + if: always() + run: | + mkdir serverlogs2 + pwd + ls -la + cp server_output.log serverlogs2/server_output.log + - name: Archive logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: motoserver-${{ matrix.python-version }} + path: | + serverlogs2/* + + test_custom_networkmode: + name: Pass NetworkMode to AWSLambda + runs-on: ubuntu-latest + needs: cache + strategy: + matrix: + python-version: [3.9] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Start MotoServer on an unusual port + run: | + python setup.py sdist + docker run --rm -t -e MOTO_DOCKER_NETWORK_MODE=host -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e MOTO_PORT=4555 -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4555:4555 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh & + MOTO_PORT=4555 python scripts/ci_wait_for_server.py + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }} + - name: Update pip + run: | + python -m pip install --upgrade pip + - name: Install project dependencies + run: | + pip install -r requirements-dev.txt + - name: Test + env: + TEST_SERVER_MODE: ${{ true }} + MOTO_PORT: 4555 + MOTO_DOCKER_NETWORK_MODE: host + run: | + pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_networkmode tests/test_cloudformation/test_cloudformation_custom_resources.py + - name: Collect Logs + if: always() + run: | + mkdir serverlogs3 + pwd + ls -la + cp server_output.log serverlogs3/server_output.log + - name: Archive Logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: motoserver-${{ matrix.python-version }} + path: | + serverlogs3/* diff --git a/docs/docs/services/lambda.rst b/docs/docs/services/lambda.rst index bf8c329bc..614f38a10 100644 --- a/docs/docs/services/lambda.rst +++ b/docs/docs/services/lambda.rst @@ -12,6 +12,8 @@ lambda ====== +.. autoclass:: moto.awslambda.models.LambdaBackend + |start-h3| Example usage |end-h3| .. sourcecode:: python diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index cd3f4af8b..8dbf14b58 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -566,17 +566,30 @@ class LambdaFunction(CloudFormationModel, DockerModel): } env_vars.update(self.environment_vars) + env_vars["MOTO_HOST"] = settings.moto_server_host() + env_vars["MOTO_PORT"] = settings.moto_server_port() + env_vars[ + "MOTO_HTTP_ENDPOINT" + ] = f'{env_vars["MOTO_HOST"]}:{env_vars["MOTO_PORT"]}' container = exit_code = None log_config = docker.types.LogConfig(type=docker.types.LogConfig.types.JSON) with _DockerDataVolumeContext(self) as data_vol: try: - run_kwargs = ( - dict(links={"motoserver": "motoserver"}) - if settings.TEST_SERVER_MODE - else {} - ) + run_kwargs = dict() + network_name = settings.moto_network_name() + network_mode = settings.moto_network_mode() + if network_name: + run_kwargs["network"] = network_name + elif network_mode: + run_kwargs["network_mode"] = network_mode + elif settings.TEST_SERVER_MODE: + # AWSLambda can make HTTP requests to a Docker container called 'motoserver' + # Only works if our Docker-container is named 'motoserver' + # TODO: should remove this and rely on 'network_mode' instead, as this is too tightly coupled with our own test setup + run_kwargs["links"] = {"motoserver": "motoserver"} + # add host.docker.internal host on linux to emulate Mac + Windows behavior # for communication with other mock AWS services running on localhost if platform == "linux" or platform == "linux2": @@ -595,7 +608,7 @@ class LambdaFunction(CloudFormationModel, DockerModel): environment=env_vars, detach=True, log_config=log_config, - **run_kwargs + **run_kwargs, ) finally: if container: @@ -1104,6 +1117,51 @@ class LayerStorage(object): class LambdaBackend(BaseBackend): + """ +Implementation of the AWS Lambda endpoint. +Invoking functions is supported - they will run inside a Docker container, emulating the real AWS behaviour as closely as possible. + +It is possible to connect from AWS Lambdas to other services, as long as you are running Moto in ServerMode. +The Lambda has access to environment variables `MOTO_HOST` and `MOTO_PORT`, which can be used to build the url that MotoServer runs on: + +.. sourcecode:: python + + def lambda_handler(event, context): + host = os.environ.get("MOTO_HOST") + port = os.environ.get("MOTO_PORT") + url = host + ":" + port + ec2 = boto3.client('ec2', region_name='us-west-2', endpoint_url=url) + + # Or even simpler: + full_url = os.environ.get("MOTO_HTTP_ENDPOINT") + ec2 = boto3.client("ec2", region_name="eu-west-1", endpoint_url=full_url) + + ec2.do_whatever_inside_the_existing_moto_server() + +Moto will run on port 5000 by default. This can be overwritten by setting an environment variable when starting Moto: + +.. sourcecode:: bash + + # This env var will be propagated to the Docker containers + MOTO_PORT=5000 moto_server + +The Docker container uses the default network mode, `bridge`. +The following environment variables are available for fine-grained control over the Docker connection options: + +.. sourcecode:: bash + + # Provide the name of a custom network to connect to + MOTO_DOCKER_NETWORK_NAME=mycustomnetwork moto_server + + # Override the network mode + # For example, network_mode=host would use the network of the host machine + # Note that this option will be ignored if MOTO_DOCKER_NETWORK_NAME is also set + MOTO_DOCKER_NETWORK_MODE=host moto_server + + +.. note:: When using the decorators, a Docker container cannot reach Moto, as it does not run as a server. Any boto3-invocations used within your Lambda will try to connect to AWS. + """ + def __init__(self, region_name): self._lambdas = LambdaStorage() self._event_source_mappings = {} diff --git a/moto/cloudformation/custom_model.py b/moto/cloudformation/custom_model.py index b6f2bfd4c..841d7172c 100644 --- a/moto/cloudformation/custom_model.py +++ b/moto/cloudformation/custom_model.py @@ -55,16 +55,21 @@ class CustomModel(CloudFormationModel): stack = cloudformation_backends[region_name].get_stack(stack_id) stack.add_custom_resource(custom_resource) + # A request will be send to this URL to indicate success/failure + # This request will be coming from inside a Docker container + # Note that, in order to reach the Moto host, the Moto-server should be listening on 0.0.0.0 + # + # Alternative: Maybe we should let the user pass in a container-name where Moto is running? + # Similar to how we know for sure that the container in our CI is called 'motoserver' + host = f"{settings.moto_server_host()}:{settings.moto_server_port()}" + response_url = ( + f"{host}/cloudformation_{region_name}/cfnresponse?stack={stack_id}" + ) + event = { "RequestType": "Create", "ServiceToken": service_token, - # A request will be send to this URL to indicate success/failure - # This request will be coming from inside a Docker container - # Note that, in order to reach the Moto host, the Moto-server should be listening on 0.0.0.0 - # - # Alternative: Maybe we should let the user pass in a container-name where Moto is running? - # Similar to how we know for sure that the container in our CI is called 'motoserver' - "ResponseURL": f"{settings.moto_server_host()}/cloudformation_{region_name}/cfnresponse?stack={stack_id}", + "ResponseURL": response_url, "StackId": stack_id, "RequestId": request_id, "LogicalResourceId": logical_id, diff --git a/moto/core/models.py b/moto/core/models.py index 8cb0f65fb..b69d9c824 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -387,12 +387,16 @@ MockAWS = BotocoreEventMockAWS class ServerModeMockAWS(BaseMockAWS): + def __init__(self, *args, **kwargs): + self.port = settings.moto_server_port() + super().__init__(*args, **kwargs) + def reset(self): call_reset_api = os.environ.get("MOTO_CALL_RESET_API") if not call_reset_api or call_reset_api.lower() != "false": import requests - requests.post("http://localhost:5000/moto-api/reset") + requests.post(f"http://localhost:{self.port}/moto-api/reset") def enable_patching(self, reset=True): if self.__class__.nested_count == 1 and reset: @@ -410,12 +414,12 @@ class ServerModeMockAWS(BaseMockAWS): config = Config(user_agent_extra="region/" + region) kwargs["config"] = config if "endpoint_url" not in kwargs: - kwargs["endpoint_url"] = "http://localhost:5000" + kwargs["endpoint_url"] = f"http://localhost:{self.port}" return real_boto3_client(*args, **kwargs) def fake_boto3_resource(*args, **kwargs): if "endpoint_url" not in kwargs: - kwargs["endpoint_url"] = "http://localhost:5000" + kwargs["endpoint_url"] = f"http://localhost:{self.port}" return real_boto3_resource(*args, **kwargs) self._client_patcher = patch("boto3.client", fake_boto3_client) diff --git a/moto/core/responses.py b/moto/core/responses.py index 9dd5dfd99..6f3f9c3f2 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -141,11 +141,13 @@ class ActionAuthenticatorMixin(object): @staticmethod def set_initial_no_auth_action_count(initial_no_auth_action_count): + _port = settings.moto_server_port() + def decorator(function): def wrapper(*args, **kwargs): if settings.TEST_SERVER_MODE: response = requests.post( - "http://localhost:5000/moto-api/reset-auth", + f"http://localhost:{_port}/moto-api/reset-auth", data=str(initial_no_auth_action_count).encode("utf-8"), ) original_initial_no_auth_action_count = response.json()[ @@ -163,7 +165,7 @@ class ActionAuthenticatorMixin(object): finally: if settings.TEST_SERVER_MODE: requests.post( - "http://localhost:5000/moto-api/reset-auth", + f"http://localhost:{_port}/moto-api/reset-auth", data=str(original_initial_no_auth_action_count).encode( "utf-8" ), diff --git a/moto/server.py b/moto/server.py index 0b4ab32fa..e8c3da3ef 100644 --- a/moto/server.py +++ b/moto/server.py @@ -4,7 +4,6 @@ import json import os import signal import sys -from functools import partial from threading import Lock from flask import Flask @@ -321,9 +320,7 @@ def create_backend_app(service): return backend_app -def signal_handler(reset_server_port, signum, frame): - if reset_server_port: - del os.environ["MOTO_PORT"] +def signal_handler(signum, frame): sys.exit(0) @@ -371,14 +368,12 @@ def main(argv=None): args = parser.parse_args(argv) - reset_server_port = False if "MOTO_PORT" not in os.environ: - reset_server_port = True os.environ["MOTO_PORT"] = f"{args.port}" try: - signal.signal(signal.SIGINT, partial(signal_handler, reset_server_port)) - signal.signal(signal.SIGTERM, partial(signal_handler, reset_server_port)) + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) except Exception: pass # ignore "ValueError: signal only works in main thread" diff --git a/moto/settings.py b/moto/settings.py index ce21ac665..c5d77591e 100644 --- a/moto/settings.py +++ b/moto/settings.py @@ -56,12 +56,18 @@ def moto_server_port(): def moto_server_host(): - _port = moto_server_port() if is_docker(): - host = get_docker_host() + return get_docker_host() else: - host = "http://host.docker.internal" - return f"{host}:{_port}" + return "http://host.docker.internal" + + +def moto_network_name(): + return os.environ.get("MOTO_DOCKER_NETWORK_NAME") + + +def moto_network_mode(): + return os.environ.get("MOTO_DOCKER_NETWORK_MODE") def is_docker(): @@ -77,7 +83,20 @@ def get_docker_host(): try: cmd = "curl -s --unix-socket /run/docker.sock http://docker/containers/$HOSTNAME/json" container_info = os.popen(cmd).read() - _ip = json.loads(container_info)["NetworkSettings"]["IPAddress"] + network_settings = json.loads(container_info)["NetworkSettings"] + network_name = moto_network_name() + if network_name and network_name in network_settings["Networks"]: + _ip = network_settings["Networks"][network_name]["IPAddress"] + else: + _ip = network_settings["IPAddress"] + if network_name: + print( + f"WARNING - Moto couldn't find network '{network_name}' - defaulting to {_ip}" + ) return f"http://{_ip}" - except: # noqa + except Exception as e: # noqa + print( + "WARNING - Unable to parse Docker API response. Defaulting to 'host.docker.internal'" + ) + print(f"{type(e)}::{e}") return "http://host.docker.internal" diff --git a/tests/test_awslambda/test_lambda_invoke.py b/tests/test_awslambda/test_lambda_invoke.py index 8db0e9ef2..e52a4c593 100644 --- a/tests/test_awslambda/test_lambda_invoke.py +++ b/tests/test_awslambda/test_lambda_invoke.py @@ -13,12 +13,15 @@ from moto import ( settings, ) from uuid import uuid4 +from unittest import SkipTest from .utilities import ( get_role_name, get_test_zip_file_error, get_test_zip_file1, get_zip_with_multiple_files, get_test_zip_file2, + get_lambda_using_environment_port, + get_lambda_using_network_mode, get_test_zip_largeresponse, ) @@ -146,6 +149,70 @@ def test_invoke_event_function(): json.loads(success_result["Payload"].read().decode("utf-8")).should.equal(in_data) +@pytest.mark.network +@mock_lambda +def test_invoke_lambda_using_environment_port(): + if not settings.TEST_SERVER_MODE: + raise SkipTest("Can only test environment variables in server mode") + conn = boto3.client("lambda", _lambda_region) + function_name = str(uuid4())[0:6] + conn.create_function( + FunctionName=function_name, + Runtime="python3.7", + Role=get_role_name(), + Handler="lambda_function.lambda_handler", + Code={"ZipFile": get_lambda_using_environment_port()}, + ) + + success_result = conn.invoke( + FunctionName=function_name, InvocationType="Event", Payload="{}" + ) + + success_result["StatusCode"].should.equal(202) + response = success_result["Payload"].read() + response = json.loads(response.decode("utf-8")) + + functions = response["functions"] + function_names = [f["FunctionName"] for f in functions] + function_names.should.contain(function_name) + + # Host matches the full URL, so one of: + # http://host.docker.internal:5000 + # http://172.0.2.1:5000 + # http://172.0.1.1:4555 + response["host"].should.match("http://.+:[0-9]{4}") + + +@pytest.mark.network +@mock_lambda +def test_invoke_lambda_using_networkmode(): + """ + Special use case - verify that Lambda can send a request to 'http://localhost' + This is only possible when the `network_mode` is set to host in the Docker args + Test is only run in our CI (for now) + """ + if not settings.moto_network_mode(): + raise SkipTest("Can only test this when NETWORK_MODE is specified") + conn = boto3.client("lambda", _lambda_region) + function_name = str(uuid4())[0:6] + conn.create_function( + FunctionName=function_name, + Runtime="python3.7", + Role=get_role_name(), + Handler="lambda_function.lambda_handler", + Code={"ZipFile": get_lambda_using_network_mode()}, + ) + + success_result = conn.invoke( + FunctionName=function_name, InvocationType="Event", Payload="{}" + ) + + response = success_result["Payload"].read() + functions = json.loads(response.decode("utf-8"))["response"] + function_names = [f["FunctionName"] for f in functions] + function_names.should.contain(function_name) + + @pytest.mark.network @mock_lambda def test_invoke_function_with_multiple_files_in_zip(): diff --git a/tests/test_awslambda/utilities.py b/tests/test_awslambda/utilities.py index 827c63082..7b8db46e0 100644 --- a/tests/test_awslambda/utilities.py +++ b/tests/test_awslambda/utilities.py @@ -48,6 +48,42 @@ def lambda_handler(event, context): return _process_lambda(func_str) +def get_lambda_using_environment_port(): + func_str = """ +import boto3 +import os + +def lambda_handler(event, context): + base_url = os.environ.get("MOTO_HOST") + port = os.environ.get("MOTO_PORT") + url = base_url + ":" + port + conn = boto3.client('lambda', region_name='us-west-2', endpoint_url=url) + + full_url = os.environ["MOTO_HTTP_ENDPOINT"] + + functions = conn.list_functions()["Functions"] + + return {'functions': functions, 'host': full_url} +""" + return _process_lambda(func_str) + + +def get_lambda_using_network_mode(): + func_str = """ +import boto3 +import os + +def lambda_handler(event, context): + port = os.environ.get("MOTO_PORT") + url = "http://localhost:" + port + conn = boto3.client('lambda', region_name='us-west-2', endpoint_url=url) + + functions = conn.list_functions()["Functions"] + return {'response': functions} +""" + return _process_lambda(func_str) + + def get_test_zip_file3(): pfunc = """ def lambda_handler(event, context):