From 2cf770f6974b716682d5430ac9f2ebd355467be8 Mon Sep 17 00:00:00 2001 From: Brendan Keane Date: Fri, 16 Dec 2022 10:22:43 -0800 Subject: [PATCH] ECR Manifest List Support (#5753) --- moto/awslambda/models.py | 11 +++- moto/ecr/models.py | 70 ++++++++++++++++++--- tests/test_awslambda/test_lambda.py | 42 +++++++++++-- tests/test_ecr/test_ecr_boto3.py | 95 +++++++++++++++++++++++++++-- tests/test_ecr/test_ecr_helpers.py | 42 ++++++++++++- 5 files changed, 234 insertions(+), 26 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index e4e5f9246..b96d3ba7a 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -489,9 +489,16 @@ class LambdaFunction(CloudFormationModel, DockerModel): ).hexdigest() self.code_size = 0 else: - uri, tag = self.code["ImageUri"].split(":") + if "@" in self.code["ImageUri"]: + # deploying via digest + uri, digest = self.code["ImageUri"].split("@") + image_id = {"imageDigest": digest} + else: + # deploying via tag + uri, tag = self.code["ImageUri"].split(":") + image_id = {"imageTag": tag} + repo_name = uri.split("/")[-1] - image_id = {"imageTag": tag} ecr_backend = ecr_backends[self.account_id][self.region] registry_id = ecr_backend.describe_registry()["registryId"] images = ecr_backend.batch_get_image( diff --git a/moto/ecr/models.py b/moto/ecr/models.py index 7ccb65a3d..c9ddcdf05 100644 --- a/moto/ecr/models.py +++ b/moto/ecr/models.py @@ -236,12 +236,19 @@ class Repository(BaseObject, CloudFormationModel): class Image(BaseObject): def __init__( - self, account_id, tag, manifest, repository, digest=None, registry_id=None + self, + account_id, + tag, + manifest, + repository, + image_manifest_mediatype=None, + digest=None, + registry_id=None, ): self.image_tag = tag self.image_tags = [tag] if tag is not None else [] self.image_manifest = manifest - self.image_size_in_bytes = 50 * 1024 * 1024 + self.image_manifest_mediatype = image_manifest_mediatype self.repository = repository self.registry_id = registry_id or account_id self.image_digest = digest @@ -250,17 +257,33 @@ class Image(BaseObject): def _create_digest(self): image_manifest = json.loads(self.image_manifest) - layer_digests = [layer["digest"] for layer in image_manifest["layers"]] - self.image_digest = ( - "sha256:" - + hashlib.sha256("".join(layer_digests).encode("utf-8")).hexdigest() - ) + if "layers" in image_manifest: + layer_digests = [layer["digest"] for layer in image_manifest["layers"]] + self.image_digest = ( + "sha256:" + + hashlib.sha256("".join(layer_digests).encode("utf-8")).hexdigest() + ) + else: + random_sha = hashlib.sha256( + f"{random.randint(0,100)}".encode("utf-8") + ).hexdigest() + self.image_digest = f"sha256:{random_sha}" def get_image_digest(self): if not self.image_digest: self._create_digest() return self.image_digest + def get_image_size_in_bytes(self): + image_manifest = json.loads(self.image_manifest) + if "layers" in image_manifest: + try: + return image_manifest["config"]["size"] + except KeyError: + return 50 * 1024 * 1024 + else: + return None + def get_image_manifest(self): return self.image_manifest @@ -303,9 +326,10 @@ class Image(BaseObject): response_object["imageTags"] = self.image_tags response_object["imageDigest"] = self.get_image_digest() response_object["imageManifest"] = self.image_manifest + response_object["imageManifestMediaType"] = self.image_manifest_mediatype response_object["repositoryName"] = self.repository response_object["registryId"] = self.registry_id - response_object["imageSizeInBytes"] = self.image_size_in_bytes + response_object["imageSizeInBytes"] = self.get_image_size_in_bytes() response_object["imagePushedAt"] = self.image_pushed_at return {k: v for k, v in response_object.items() if v is not None and v != []} @@ -486,12 +510,31 @@ class ECRBackend(BaseBackend): return response - def put_image(self, repository_name, image_manifest, image_tag): + def put_image( + self, repository_name, image_manifest, image_tag, image_manifest_mediatype=None + ): if repository_name in self.repositories: repository = self.repositories[repository_name] else: raise Exception(f"{repository_name} is not a repository") + try: + parsed_image_manifest = json.loads(image_manifest) + except json.JSONDecodeError: + raise Exception( + "Invalid parameter at 'ImageManifest' failed to satisfy constraint: 'Invalid JSON syntax'" + ) + + if image_manifest_mediatype: + parsed_image_manifest["imageManifest"] = image_manifest_mediatype + else: + if "mediaType" not in parsed_image_manifest: + raise Exception( + "image manifest mediatype not provided in manifest or parameter" + ) + else: + image_manifest_mediatype = parsed_image_manifest["mediaType"] + # Tags are unique, so delete any existing image with this tag first self.batch_delete_image( repository_name=repository_name, image_ids=[{"imageTag": image_tag}] @@ -503,9 +546,16 @@ class ECRBackend(BaseBackend): repository.images, ) ) + if not existing_images: # this image is not in ECR yet - image = Image(self.account_id, image_tag, image_manifest, repository_name) + image = Image( + self.account_id, + image_tag, + image_manifest, + repository_name, + image_manifest_mediatype, + ) repository.images.append(image) return image else: diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 802132728..569c21c3e 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -218,12 +218,12 @@ def ecr_repo_fixture(): repo_name = "testlambdaecr" ecr_client = ecr_client = boto3.client("ecr", "us-east-1") ecr_client.create_repository(repositoryName=repo_name) - ecr_client.put_image( + response = ecr_client.put_image( repositoryName=repo_name, imageManifest=json.dumps(_create_image_manifest()), imageTag="latest", ) - yield + yield response["image"]["imageId"] ecr_client.delete_repository(repositoryName=repo_name, force=True) os.environ["MOTO_LAMBDA_STUB_ECR"] = "TRUE" @@ -262,7 +262,7 @@ def test_create_function_from_stubbed_ecr(): @mock_lambda -def test_create_function_from_mocked_ecr_image( +def test_create_function_from_mocked_ecr_image_tag( with_ecr_mock, ): # pylint: disable=unused-argument if settings.TEST_SERVER_MODE: @@ -272,7 +272,8 @@ def test_create_function_from_mocked_ecr_image( lambda_client = boto3.client("lambda", "us-east-1") fn_name = str(uuid4())[0:6] - image_uri = f"{ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/testlambdaecr:latest" + image = with_ecr_mock + image_uri = f"{ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/testlambdaecr:{image['imageTag']}" dic = { "FunctionName": fn_name, @@ -291,7 +292,9 @@ def test_create_function_from_mocked_ecr_image( result = lambda_client.get_function(FunctionName=fn_name) result.should.have.key("Configuration") config = result["Configuration"] - config.should.have.key("CodeSha256").equals(resp["CodeSha256"]) + config.should.have.key("CodeSha256").equals( + image["imageDigest"].replace("sha256:", "") + ) config.should.have.key("CodeSize").equals(resp["CodeSize"]) result.should.have.key("Code") code = result["Code"] @@ -302,6 +305,35 @@ def test_create_function_from_mocked_ecr_image( code.should.have.key("ResolvedImageUri").equals(resolved_image_uri) +@mock_lambda +def test_create_function_from_mocked_ecr_image_digest( + with_ecr_mock, +): # pylint: disable=unused-argument + if settings.TEST_SERVER_MODE: + raise SkipTest( + "Envars not easily set in server mode, feature off by default, skipping..." + ) + lambda_client = boto3.client("lambda", "us-east-1") + fn_name = str(uuid4())[0:6] + image = with_ecr_mock + image_uri = f"{ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/testlambdaecr@{image['imageDigest']}" + + dic = { + "FunctionName": fn_name, + "Role": get_role_name(), + "Code": {"ImageUri": image_uri}, + "PackageType": "Image", + "Timeout": 100, + } + resp = lambda_client.create_function(**dic) + resp.should.have.key("FunctionName").equals(fn_name) + resp.should.have.key("CodeSize").greater_than(0) + resp.should.have.key("CodeSha256").equals( + image["imageDigest"].replace("sha256:", "") + ) + resp.should.have.key("PackageType").equals("Image") + + @mock_lambda def test_create_function_from_mocked_ecr_missing_image( with_ecr_mock, diff --git a/tests/test_ecr/test_ecr_boto3.py b/tests/test_ecr/test_ecr_boto3.py index 50b3a7fae..31d41e28c 100644 --- a/tests/test_ecr/test_ecr_boto3.py +++ b/tests/test_ecr/test_ecr_boto3.py @@ -17,7 +17,10 @@ from unittest import SkipTest from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID -from tests.test_ecr.test_ecr_helpers import _create_image_manifest +from tests.test_ecr.test_ecr_helpers import ( + _create_image_manifest, + _create_image_manifest_list, +) @mock_ecr @@ -347,6 +350,34 @@ def test_put_image(): response["image"]["registryId"].should.equal(ACCOUNT_ID) +@mock_ecr() +def test_put_manifest_list(): + client = boto3.client("ecr", region_name="us-east-1") + _ = client.create_repository(repositoryName="test_repository") + + manifest_list = _create_image_manifest_list() + for image_manifest in manifest_list["image_manifests"]: + _ = client.put_image( + repositoryName="test_repository", + imageManifest=json.dumps(image_manifest), + ) + + response = client.put_image( + repositoryName="test_repository", + imageManifest=json.dumps(manifest_list["manifest_list"]), + imageTag="multiArch", + ) + + response["image"]["imageId"]["imageTag"].should.equal("multiArch") + response["image"]["imageId"]["imageDigest"].should.contain("sha") + response["image"]["repositoryName"].should.equal("test_repository") + response["image"]["registryId"].should.equal(ACCOUNT_ID) + response["image"].should.have.key("imageManifest") + image_manifest = json.loads(response["image"]["imageManifest"]) + image_manifest.should.have.key("mediaType") + image_manifest.should.have.key("manifests") + + @mock_ecr def test_put_image_with_push_date(): if os.environ.get("TEST_SERVER_MODE", "false").lower() == "true": @@ -571,29 +602,77 @@ def test_describe_images(): imageTag="v2", ) + manifest_list = _create_image_manifest_list() + for image_manifest in manifest_list["image_manifests"]: + _ = client.put_image( + repositoryName="test_repository", + imageManifest=json.dumps(image_manifest), + ) + + _ = client.put_image( + repositoryName="test_repository", + imageManifest=json.dumps(manifest_list["manifest_list"]), + imageTag="multiArch", + ) + response = client.describe_images(repositoryName="test_repository") type(response["imageDetails"]).should.be(list) - len(response["imageDetails"]).should.be(4) + len(response["imageDetails"]).should.be(7) + + response["imageDetails"][0]["imageManifestMediaType"].should.contain( + "distribution.manifest.v2+json" + ) + response["imageDetails"][1]["imageManifestMediaType"].should.contain( + "distribution.manifest.v2+json" + ) + response["imageDetails"][2]["imageManifestMediaType"].should.contain( + "distribution.manifest.v2+json" + ) + response["imageDetails"][3]["imageManifestMediaType"].should.contain( + "distribution.manifest.v2+json" + ) + response["imageDetails"][4]["imageManifestMediaType"].should.contain( + "distribution.manifest.v2+json" + ) + response["imageDetails"][5]["imageManifestMediaType"].should.contain( + "distribution.manifest.v2+json" + ) + response["imageDetails"][6]["imageManifestMediaType"].should.contain( + "distribution.manifest.list.v2+json" + ) response["imageDetails"][0]["imageDigest"].should.contain("sha") response["imageDetails"][1]["imageDigest"].should.contain("sha") response["imageDetails"][2]["imageDigest"].should.contain("sha") response["imageDetails"][3]["imageDigest"].should.contain("sha") + response["imageDetails"][4]["imageDigest"].should.contain("sha") + response["imageDetails"][5]["imageDigest"].should.contain("sha") + response["imageDetails"][6]["imageDigest"].should.contain("sha") response["imageDetails"][0]["registryId"].should.equal(ACCOUNT_ID) response["imageDetails"][1]["registryId"].should.equal(ACCOUNT_ID) response["imageDetails"][2]["registryId"].should.equal(ACCOUNT_ID) response["imageDetails"][3]["registryId"].should.equal(ACCOUNT_ID) + response["imageDetails"][4]["registryId"].should.equal(ACCOUNT_ID) + response["imageDetails"][5]["registryId"].should.equal(ACCOUNT_ID) + response["imageDetails"][6]["registryId"].should.equal(ACCOUNT_ID) response["imageDetails"][0]["repositoryName"].should.equal("test_repository") response["imageDetails"][1]["repositoryName"].should.equal("test_repository") response["imageDetails"][2]["repositoryName"].should.equal("test_repository") response["imageDetails"][3]["repositoryName"].should.equal("test_repository") + response["imageDetails"][4]["repositoryName"].should.equal("test_repository") + response["imageDetails"][5]["repositoryName"].should.equal("test_repository") + response["imageDetails"][6]["repositoryName"].should.equal("test_repository") response["imageDetails"][0].should_not.have.key("imageTags") + response["imageDetails"][4].should_not.have.key("imageTags") + response["imageDetails"][5].should_not.have.key("imageTags") + len(response["imageDetails"][1]["imageTags"]).should.be(1) len(response["imageDetails"][2]["imageTags"]).should.be(1) len(response["imageDetails"][3]["imageTags"]).should.be(1) + len(response["imageDetails"][6]["imageTags"]).should.be(1) image_tags = ["latest", "v1", "v2"] set( @@ -604,10 +683,14 @@ def test_describe_images(): ] ).should.equal(set(image_tags)) - response["imageDetails"][0]["imageSizeInBytes"].should.equal(52428800) - response["imageDetails"][1]["imageSizeInBytes"].should.equal(52428800) - response["imageDetails"][2]["imageSizeInBytes"].should.equal(52428800) - response["imageDetails"][3]["imageSizeInBytes"].should.equal(52428800) + response["imageDetails"][6].should_not.have.key("imageSizeInBytes") + + response["imageDetails"][0]["imageSizeInBytes"].should.be.greater_than(0) + response["imageDetails"][1]["imageSizeInBytes"].should.be.greater_than(0) + response["imageDetails"][2]["imageSizeInBytes"].should.be.greater_than(0) + response["imageDetails"][3]["imageSizeInBytes"].should.be.greater_than(0) + response["imageDetails"][4]["imageSizeInBytes"].should.be.greater_than(0) + response["imageDetails"][5]["imageSizeInBytes"].should.be.greater_than(0) @mock_ecr diff --git a/tests/test_ecr/test_ecr_helpers.py b/tests/test_ecr/test_ecr_helpers.py index 1b04159c9..f0f1a9cf0 100644 --- a/tests/test_ecr/test_ecr_helpers.py +++ b/tests/test_ecr/test_ecr_helpers.py @@ -1,9 +1,12 @@ import hashlib import random +# Manifests Spec: https://docs.docker.com/registry/spec/manifest-v2-2/ + def _generate_random_sha(): - return hashlib.sha256(f"{random.randint(0,100)}".encode("utf-8")).hexdigest() + random_sha = hashlib.sha256(f"{random.randint(0,100)}".encode("utf-8")).hexdigest() + return f"sha256:{random_sha}" def _create_image_layers(n): @@ -13,7 +16,7 @@ def _create_image_layers(n): { "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", "size": random.randint(100, 1000), - "digest": f"sha256:{_generate_random_sha()}", + "digest": _generate_random_sha(), } ) return layers @@ -21,7 +24,8 @@ def _create_image_layers(n): def _create_image_digest(layers): layer_digests = "".join([layer["digest"] for layer in layers]) - return hashlib.sha256(f"{layer_digests}".encode("utf-8")).hexdigest() + summed_digest = hashlib.sha256(f"{layer_digests}".encode("utf-8")).hexdigest() + return f"sha256:{summed_digest}" def _create_image_manifest(image_digest=None): @@ -38,3 +42,35 @@ def _create_image_manifest(image_digest=None): }, "layers": layers, } + + +def _create_manifest_list_distribution( + image_manifest: dict, architecture: str = "amd64", os: str = "linux" +): + return { + "mediaType": image_manifest["config"]["mediaType"], + "digest": image_manifest["config"]["digest"], + "size": image_manifest["config"]["size"], + "platform": {"architecture": architecture, "os": os}, + } + + +def _create_image_manifest_list(): + arm_image_manifest = _create_image_manifest() + amd_image_manifest = _create_image_manifest() + arm_distribution = _create_manifest_list_distribution( + arm_image_manifest, architecture="arm64" + ) + amd_distribution = _create_manifest_list_distribution( + amd_image_manifest, architecture="amd64" + ) + manifest_list = { + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "schemaVersion": 2, + "manifests": [arm_distribution, amd_distribution], + } + + return { + "image_manifests": [arm_image_manifest, amd_image_manifest], + "manifest_list": manifest_list, + }