Support Podman for mocking Lambda (#3702)

* Support Podman for mocking Lambda

Podman supports all Docker APIs used in moto since version 3.0. Note
that Podman requires pulling the image before creating a container
using a fully-qualified image name (e.g., "docker.io/library/busybox"
instead of "busybox").

Test plan:
$ podman system service -t 0
$ DOCKER_HOST="unix://$XDG_RUNTIME_DIR/podman/podman.sock" pytest

Fixes https://github.com/spulec/moto/issues/3276

* Run black

* Python 2 compatibility

* Address review comments and improve parse_image_ref

moto/awslambda/models.py

@@ -51,7 +51,7 @@ from moto.sqs import sqs_backends
 from moto.dynamodb2 import dynamodb_backends2
 from moto.dynamodbstreams import dynamodbstreams_backends
 from moto.core import ACCOUNT_ID
-from moto.utilities.docker_utilities import DockerModel
+from moto.utilities.docker_utilities import DockerModel, parse_image_ref
 
 logger = logging.getLogger(__name__)
 
@@ -131,6 +131,9 @@ class _DockerDataVolumeContext:
                 volumes = {self.name: {"bind": "/tmp/data", "mode": "rw"}}
             else:
                 volumes = {self.name: "/tmp/data"}
+            self._lambda_func.docker_client.images.pull(
+                ":".join(parse_image_ref("alpine"))
+            )
             container = self._lambda_func.docker_client.containers.run(
                 "alpine", "sleep 100", volumes=volumes, detach=True
             )
@@ -574,8 +577,10 @@ class LambdaFunction(CloudFormationModel, DockerModel):
                         if settings.TEST_SERVER_MODE
                         else {}
                     )
+                    image_ref = "lambci/lambda:{}".format(self.run_time)
+                    self.docker_client.images.pull(":".join(parse_image_ref(image_ref)))
                     container = self.docker_client.containers.run(
-                        "lambci/lambda:{}".format(self.run_time),
+                        image_ref,
                         [self.handler, json.dumps(event)],
                         remove=False,
                         mem_limit="{}m".format(self.memory_size),

moto/batch/models.py

@@ -28,7 +28,7 @@ from moto.ec2.exceptions import InvalidSubnetIdError
 from moto.ec2.models import INSTANCE_TYPES as EC2_INSTANCE_TYPES
 from moto.iam.exceptions import IAMNotFoundException
 from moto.core import ACCOUNT_ID as DEFAULT_ACCOUNT_ID
-from moto.utilities.docker_utilities import DockerModel
+from moto.utilities.docker_utilities import DockerModel, parse_image_ref
 
 logger = logging.getLogger(__name__)
 COMPUTE_ENVIRONMENT_NAME_REGEX = re.compile(
@@ -428,6 +428,8 @@ class Job(threading.Thread, BaseModel, DockerModel):
             self.job_started_at = datetime.datetime.now()
             self.job_state = "STARTING"
             log_config = docker.types.LogConfig(type=docker.types.LogConfig.types.JSON)
+            image_repository, image_tag = parse_image_ref(image)
+            self.docker_client.images.pull(image_repository, image_tag)
             container = self.docker_client.containers.run(
                 image,
                 cmd,

moto/settings.py

@@ -4,6 +4,7 @@ TEST_SERVER_MODE = os.environ.get("TEST_SERVER_MODE", "0").lower() == "true"
 INITIAL_NO_AUTH_ACTION_COUNT = float(
     os.environ.get("INITIAL_NO_AUTH_ACTION_COUNT", float("inf"))
 )
+DEFAULT_CONTAINER_REGISTRY = os.environ.get("DEFAULT_CONTAINER_REGISTRY", "docker.io")
 
 
 def get_sf_execution_history_type():

moto/utilities/docker_utilities.py

@@ -2,6 +2,8 @@ import docker
 import functools
 import requests.adapters
 
+from moto import settings
+
 
 _orig_adapter_send = requests.adapters.HTTPAdapter.send
 
@@ -31,3 +33,27 @@ class DockerModel:
 
                 self.docker_client.api.get_adapter = replace_adapter_send
         return self.__docker_client
+
+
+def parse_image_ref(image_name):
+    # podman does not support short container image name out of box - try to make a full name
+    # See ParseDockerRef() in https://github.com/distribution/distribution/blob/main/reference/normalize.go
+    parts = image_name.split("/")
+    if len(parts) == 1 or (
+        "." not in parts[0] and ":" not in parts[0] and parts[0] != "localhost"
+    ):
+        domain = settings.DEFAULT_CONTAINER_REGISTRY
+        remainder = parts
+    else:
+        domain = parts[0]
+        remainder = parts[1:]
+    # Special handling for docker.io
+    # https://github.com/containers/image/blob/master/docs/containers-registries.conf.5.md#normalization-of-dockerio-references
+    if domain == "docker.io" and len(remainder) == 1:
+        remainder = ["library"] + remainder
+    if ":" in remainder[-1]:
+        remainder[-1], image_tag = remainder[-1].split(":", 1)
+    else:
+        image_tag = "latest"
+    image_repository = "/".join([domain] + remainder)
+    return image_repository, image_tag

tests/test_utilities/test_docker_utilities.py

@@ -0,0 +1,31 @@
+import sure
+import pytest
+
+from moto.utilities.docker_utilities import parse_image_ref
+
+
+@pytest.mark.parametrize(
+    "image_name,expected",
+    [
+        ("python", ("docker.io/library/python", "latest")),
+        ("python:3.9", ("docker.io/library/python", "3.9")),
+        ("docker.io/python", ("docker.io/library/python", "latest")),
+        ("localhost/foobar", ("localhost/foobar", "latest")),
+        ("lambci/lambda:python2.7", ("docker.io/lambci/lambda", "python2.7")),
+        (
+            "gcr.io/google.com/cloudsdktool/cloud-sdk",
+            ("gcr.io/google.com/cloudsdktool/cloud-sdk", "latest"),
+        ),
+    ],
+)
+def test_parse_image_ref(image_name, expected):
+    expected.should.be.equal(parse_image_ref(image_name))
+
+
+def test_parse_image_ref_default_container_registry(monkeypatch):
+    import moto.settings
+
+    monkeypatch.setattr(moto.settings, "DEFAULT_CONTAINER_REGISTRY", "quay.io")
+    ("quay.io/centos/centos", "latest").should.be.equal(
+        parse_image_ref("centos/centos")
+    )