Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Propagate MotoHost via env vars to Lambda (#4658)

Browse Source
This commit is contained in:
Bert Blommers 2022-01-27 11:04:03 -01:00 committed by GitHub
parent cd2d7a9c7a
commit 3ba3f1460f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 429 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

209
.github/workflows/dockertests.yml vendored Normal file
View File

@ -0,0 +1,209 @@
name: DockerTests
on: [push, pull_request]
jobs:
cache:
name: Caching
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ 3.9 ]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache dir
id: pip-cache-dir
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: pip cache
id: pip-cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache-dir.outputs.dir }}
key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
- name: Update pip
if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
run: |
python -m pip install --upgrade pip
- name: Install project dependencies
if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
run: |
pip install -r requirements-dev.txt
test_custom_port:
name: Test Custom Port
runs-on: ubuntu-latest
needs: cache
strategy:
matrix:
python-version: [3.9]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Start MotoServer on an unusual port
run: |
python setup.py sdist
docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e MOTO_PORT=4555 -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4555:4555 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
MOTO_PORT=4555 python scripts/ci_wait_for_server.py
- name: Get pip cache dir
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
- name: Update pip
run: |
python -m pip install --upgrade pip
- name: Install project dependencies
run: |
pip install -r requirements-dev.txt
- name: Test
env:
TEST_SERVER_MODE: ${{ true }}
MOTO_PORT: 4555
run: |
pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_environment_port tests/test_cloudformation/test_cloudformation_custom_resources.py
- name: Collect Logs
if: always()
run: |
mkdir serverlogs1
pwd
ls -la
cp server_output.log serverlogs1/server_output.log
docker stop motoserver
- name: Archive Logs
if: always()
uses: actions/upload-artifact@v2
with:
name: motoserver-${{ matrix.python-version }}
path: |
serverlogs1/*
test_custom_name:
name: Test Custom Network Name
runs-on: ubuntu-latest
needs: cache
strategy:
matrix:
python-version: [3.9]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Start MotoServer on a custom Docker network bridge
run: |
python setup.py sdist
docker network create -d bridge my-custom-network
docker run --rm -t -e TEST_SERVER_MODE=true -e MOTO_DOCKER_NETWORK_NAME=my-custom-network -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 --network my-custom-network -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
python scripts/ci_wait_for_server.py
- name: Get pip cache dir
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
- name: Update pip
run: |
python -m pip install --upgrade pip
- name: Install project dependencies
run: |
pip install -r requirements-dev.txt
- name: Test
env:
TEST_SERVER_MODE: ${{ true }}
run: |
pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_environment_port tests/test_cloudformation/test_cloudformation_custom_resources.py
- name: Collect Logs
if: always()
run: |
mkdir serverlogs2
pwd
ls -la
cp server_output.log serverlogs2/server_output.log
- name: Archive logs
if: always()
uses: actions/upload-artifact@v2
with:
name: motoserver-${{ matrix.python-version }}
path: |
serverlogs2/*
test_custom_networkmode:
name: Pass NetworkMode to AWSLambda
runs-on: ubuntu-latest
needs: cache
strategy:
matrix:
python-version: [3.9]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Start MotoServer on an unusual port
run: |
python setup.py sdist
docker run --rm -t -e MOTO_DOCKER_NETWORK_MODE=host -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e MOTO_PORT=4555 -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4555:4555 -v /var/run/docker.sock:/var/run/docker.sock python:3.7-buster /moto/scripts/ci_moto_server.sh &
MOTO_PORT=4555 python scripts/ci_wait_for_server.py
- name: Get pip cache dir
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.py') }}
- name: Update pip
run: |
python -m pip install --upgrade pip
- name: Install project dependencies
run: |
pip install -r requirements-dev.txt
- name: Test
env:
TEST_SERVER_MODE: ${{ true }}
MOTO_PORT: 4555
MOTO_DOCKER_NETWORK_MODE: host
run: |
pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_networkmode tests/test_cloudformation/test_cloudformation_custom_resources.py
- name: Collect Logs
if: always()
run: |
mkdir serverlogs3
pwd
ls -la
cp server_output.log serverlogs3/server_output.log
- name: Archive Logs
if: always()
uses: actions/upload-artifact@v2
with:
name: motoserver-${{ matrix.python-version }}
path: |
serverlogs3/*

2
docs/docs/services/lambda.rst
View File

@ -12,6 +12,8 @@
lambda
======
.. autoclass:: moto.awslambda.models.LambdaBackend
|start-h3| Example usage |end-h3|
.. sourcecode:: python

70
moto/awslambda/models.py
View File

@ -566,17 +566,30 @@ class LambdaFunction(CloudFormationModel, DockerModel):
}
env_vars.update(self.environment_vars)
env_vars["MOTO_HOST"] = settings.moto_server_host()
env_vars["MOTO_PORT"] = settings.moto_server_port()
env_vars[
"MOTO_HTTP_ENDPOINT"
] = f'{env_vars["MOTO_HOST"]}:{env_vars["MOTO_PORT"]}'
container = exit_code = None
log_config = docker.types.LogConfig(type=docker.types.LogConfig.types.JSON)
with _DockerDataVolumeContext(self) as data_vol:
try:
run_kwargs = (
dict(links={"motoserver": "motoserver"})
if settings.TEST_SERVER_MODE
else {}
)
run_kwargs = dict()
network_name = settings.moto_network_name()
network_mode = settings.moto_network_mode()
if network_name:
run_kwargs["network"] = network_name
elif network_mode:
run_kwargs["network_mode"] = network_mode
elif settings.TEST_SERVER_MODE:
# AWSLambda can make HTTP requests to a Docker container called 'motoserver'
# Only works if our Docker-container is named 'motoserver'
# TODO: should remove this and rely on 'network_mode' instead, as this is too tightly coupled with our own test setup
run_kwargs["links"] = {"motoserver": "motoserver"}
# add host.docker.internal host on linux to emulate Mac + Windows behavior
# for communication with other mock AWS services running on localhost
if platform == "linux" or platform == "linux2":
@ -595,7 +608,7 @@ class LambdaFunction(CloudFormationModel, DockerModel):
environment=env_vars,
detach=True,
log_config=log_config,
**run_kwargs
**run_kwargs,
)
finally:
if container:
@ -1104,6 +1117,51 @@ class LayerStorage(object):
class LambdaBackend(BaseBackend):
"""
Implementation of the AWS Lambda endpoint.
Invoking functions is supported - they will run inside a Docker container, emulating the real AWS behaviour as closely as possible.
It is possible to connect from AWS Lambdas to other services, as long as you are running Moto in ServerMode.
The Lambda has access to environment variables `MOTO_HOST` and `MOTO_PORT`, which can be used to build the url that MotoServer runs on:
.. sourcecode:: python
def lambda_handler(event, context):
host = os.environ.get("MOTO_HOST")
port = os.environ.get("MOTO_PORT")
url = host + ":" + port
ec2 = boto3.client('ec2', region_name='us-west-2', endpoint_url=url)
# Or even simpler:
full_url = os.environ.get("MOTO_HTTP_ENDPOINT")
ec2 = boto3.client("ec2", region_name="eu-west-1", endpoint_url=full_url)
ec2.do_whatever_inside_the_existing_moto_server()
Moto will run on port 5000 by default. This can be overwritten by setting an environment variable when starting Moto:
.. sourcecode:: bash
# This env var will be propagated to the Docker containers
MOTO_PORT=5000 moto_server
The Docker container uses the default network mode, `bridge`.
The following environment variables are available for fine-grained control over the Docker connection options:
.. sourcecode:: bash
# Provide the name of a custom network to connect to
MOTO_DOCKER_NETWORK_NAME=mycustomnetwork moto_server
# Override the network mode
# For example, network_mode=host would use the network of the host machine
# Note that this option will be ignored if MOTO_DOCKER_NETWORK_NAME is also set
MOTO_DOCKER_NETWORK_MODE=host moto_server
.. note:: When using the decorators, a Docker container cannot reach Moto, as it does not run as a server. Any boto3-invocations used within your Lambda will try to connect to AWS.
"""
def __init__(self, region_name):
self._lambdas = LambdaStorage()
self._event_source_mappings = {}

19
moto/cloudformation/custom_model.py
View File

@ -55,16 +55,21 @@ class CustomModel(CloudFormationModel):
stack = cloudformation_backends[region_name].get_stack(stack_id)
stack.add_custom_resource(custom_resource)
# A request will be send to this URL to indicate success/failure
# This request will be coming from inside a Docker container
# Note that, in order to reach the Moto host, the Moto-server should be listening on 0.0.0.0
#
# Alternative: Maybe we should let the user pass in a container-name where Moto is running?
# Similar to how we know for sure that the container in our CI is called 'motoserver'
host = f"{settings.moto_server_host()}:{settings.moto_server_port()}"
response_url = (
f"{host}/cloudformation_{region_name}/cfnresponse?stack={stack_id}"
)
event = {
"RequestType": "Create",
"ServiceToken": service_token,
# A request will be send to this URL to indicate success/failure
# This request will be coming from inside a Docker container
# Note that, in order to reach the Moto host, the Moto-server should be listening on 0.0.0.0
#
# Alternative: Maybe we should let the user pass in a container-name where Moto is running?
# Similar to how we know for sure that the container in our CI is called 'motoserver'
"ResponseURL": f"{settings.moto_server_host()}/cloudformation_{region_name}/cfnresponse?stack={stack_id}",
"ResponseURL": response_url,
"StackId": stack_id,
"RequestId": request_id,
"LogicalResourceId": logical_id,

10
moto/core/models.py
View File

@ -387,12 +387,16 @@ MockAWS = BotocoreEventMockAWS
class ServerModeMockAWS(BaseMockAWS):
def __init__(self, *args, **kwargs):
self.port = settings.moto_server_port()
super().__init__(*args, **kwargs)
def reset(self):
call_reset_api = os.environ.get("MOTO_CALL_RESET_API")
if not call_reset_api or call_reset_api.lower() != "false":
import requests
requests.post("http://localhost:5000/moto-api/reset")
requests.post(f"http://localhost:{self.port}/moto-api/reset")
def enable_patching(self, reset=True):
if self.__class__.nested_count == 1 and reset:
@ -410,12 +414,12 @@ class ServerModeMockAWS(BaseMockAWS):
config = Config(user_agent_extra="region/" + region)
kwargs["config"] = config
if "endpoint_url" not in kwargs:
kwargs["endpoint_url"] = "http://localhost:5000"
kwargs["endpoint_url"] = f"http://localhost:{self.port}"
return real_boto3_client(*args, **kwargs)
def fake_boto3_resource(*args, **kwargs):
if "endpoint_url" not in kwargs:
kwargs["endpoint_url"] = "http://localhost:5000"
kwargs["endpoint_url"] = f"http://localhost:{self.port}"
return real_boto3_resource(*args, **kwargs)
self._client_patcher = patch("boto3.client", fake_boto3_client)

6
moto/core/responses.py
View File

@ -141,11 +141,13 @@ class ActionAuthenticatorMixin(object):
@staticmethod
def set_initial_no_auth_action_count(initial_no_auth_action_count):
_port = settings.moto_server_port()
def decorator(function):
def wrapper(*args, **kwargs):
if settings.TEST_SERVER_MODE:
response = requests.post(
"http://localhost:5000/moto-api/reset-auth",
f"http://localhost:{_port}/moto-api/reset-auth",
data=str(initial_no_auth_action_count).encode("utf-8"),
)
original_initial_no_auth_action_count = response.json()[
@ -163,7 +165,7 @@ class ActionAuthenticatorMixin(object):
finally:
if settings.TEST_SERVER_MODE:
requests.post(
"http://localhost:5000/moto-api/reset-auth",
f"http://localhost:{_port}/moto-api/reset-auth",
data=str(original_initial_no_auth_action_count).encode(
"utf-8"
),

11
moto/server.py
View File

@ -4,7 +4,6 @@ import json
import os
import signal
import sys
from functools import partial
from threading import Lock
from flask import Flask
@ -321,9 +320,7 @@ def create_backend_app(service):
return backend_app
def signal_handler(reset_server_port, signum, frame):
if reset_server_port:
del os.environ["MOTO_PORT"]
def signal_handler(signum, frame):
sys.exit(0)
@ -371,14 +368,12 @@ def main(argv=None):
args = parser.parse_args(argv)
reset_server_port = False
if "MOTO_PORT" not in os.environ:
reset_server_port = True
os.environ["MOTO_PORT"] = f"{args.port}"
try:
signal.signal(signal.SIGINT, partial(signal_handler, reset_server_port))
signal.signal(signal.SIGTERM, partial(signal_handler, reset_server_port))
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
except Exception:
pass # ignore "ValueError: signal only works in main thread"

31
moto/settings.py
View File

@ -56,12 +56,18 @@ def moto_server_port():
def moto_server_host():
_port = moto_server_port()
if is_docker():
host = get_docker_host()
return get_docker_host()
else:
host = "http://host.docker.internal"
return f"{host}:{_port}"
return "http://host.docker.internal"
def moto_network_name():
return os.environ.get("MOTO_DOCKER_NETWORK_NAME")
def moto_network_mode():
return os.environ.get("MOTO_DOCKER_NETWORK_MODE")
def is_docker():
@ -77,7 +83,20 @@ def get_docker_host():
try:
cmd = "curl -s --unix-socket /run/docker.sock http://docker/containers/$HOSTNAME/json"
container_info = os.popen(cmd).read()
_ip = json.loads(container_info)["NetworkSettings"]["IPAddress"]
network_settings = json.loads(container_info)["NetworkSettings"]
network_name = moto_network_name()
if network_name and network_name in network_settings["Networks"]:
_ip = network_settings["Networks"][network_name]["IPAddress"]
else:
_ip = network_settings["IPAddress"]
if network_name:
print(
f"WARNING - Moto couldn't find network '{network_name}' - defaulting to {_ip}"
)
return f"http://{_ip}"
except: # noqa
except Exception as e: # noqa
print(
"WARNING - Unable to parse Docker API response. Defaulting to 'host.docker.internal'"
)
print(f"{type(e)}::{e}")
return "http://host.docker.internal"

67
tests/test_awslambda/test_lambda_invoke.py
View File

@ -13,12 +13,15 @@ from moto import (
settings,
)
from uuid import uuid4
from unittest import SkipTest
from .utilities import (
get_role_name,
get_test_zip_file_error,
get_test_zip_file1,
get_zip_with_multiple_files,
get_test_zip_file2,
get_lambda_using_environment_port,
get_lambda_using_network_mode,
get_test_zip_largeresponse,
)
@ -146,6 +149,70 @@ def test_invoke_event_function():
json.loads(success_result["Payload"].read().decode("utf-8")).should.equal(in_data)
@pytest.mark.network
@mock_lambda
def test_invoke_lambda_using_environment_port():
if not settings.TEST_SERVER_MODE:
raise SkipTest("Can only test environment variables in server mode")
conn = boto3.client("lambda", _lambda_region)
function_name = str(uuid4())[0:6]
conn.create_function(
FunctionName=function_name,
Runtime="python3.7",
Role=get_role_name(),
Handler="lambda_function.lambda_handler",
Code={"ZipFile": get_lambda_using_environment_port()},
)
success_result = conn.invoke(
FunctionName=function_name, InvocationType="Event", Payload="{}"
)
success_result["StatusCode"].should.equal(202)
response = success_result["Payload"].read()
response = json.loads(response.decode("utf-8"))
functions = response["functions"]
function_names = [f["FunctionName"] for f in functions]
function_names.should.contain(function_name)
# Host matches the full URL, so one of:
# http://host.docker.internal:5000
# http://172.0.2.1:5000
# http://172.0.1.1:4555
response["host"].should.match("http://.+:[0-9]{4}")
@pytest.mark.network
@mock_lambda
def test_invoke_lambda_using_networkmode():
"""
Special use case - verify that Lambda can send a request to 'http://localhost'
This is only possible when the `network_mode` is set to host in the Docker args
Test is only run in our CI (for now)
"""
if not settings.moto_network_mode():
raise SkipTest("Can only test this when NETWORK_MODE is specified")
conn = boto3.client("lambda", _lambda_region)
function_name = str(uuid4())[0:6]
conn.create_function(
FunctionName=function_name,
Runtime="python3.7",
Role=get_role_name(),
Handler="lambda_function.lambda_handler",
Code={"ZipFile": get_lambda_using_network_mode()},
)
success_result = conn.invoke(
FunctionName=function_name, InvocationType="Event", Payload="{}"
)
response = success_result["Payload"].read()
functions = json.loads(response.decode("utf-8"))["response"]
function_names = [f["FunctionName"] for f in functions]
function_names.should.contain(function_name)
@pytest.mark.network
@mock_lambda
def test_invoke_function_with_multiple_files_in_zip():

36
tests/test_awslambda/utilities.py
View File

@ -48,6 +48,42 @@ def lambda_handler(event, context):
return _process_lambda(func_str)
def get_lambda_using_environment_port():
func_str = """
import boto3
import os
def lambda_handler(event, context):
base_url = os.environ.get("MOTO_HOST")
port = os.environ.get("MOTO_PORT")
url = base_url + ":" + port
conn = boto3.client('lambda', region_name='us-west-2', endpoint_url=url)
full_url = os.environ["MOTO_HTTP_ENDPOINT"]
functions = conn.list_functions()["Functions"]
return {'functions': functions, 'host': full_url}
"""
return _process_lambda(func_str)
def get_lambda_using_network_mode():
func_str = """
import boto3
import os
def lambda_handler(event, context):
port = os.environ.get("MOTO_PORT")
url = "http://localhost:" + port
conn = boto3.client('lambda', region_name='us-west-2', endpoint_url=url)
functions = conn.list_functions()["Functions"]
return {'response': functions}
"""
return _process_lambda(func_str)
def get_test_zip_file3():
pfunc = """
def lambda_handler(event, context):