Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ECR Manifest List Support (#5753)

Browse Source
This commit is contained in:
Brendan Keane 2022-12-16 10:22:43 -08:00 committed by GitHub
parent 16f9ff56a3
commit 2cf770f697
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 234 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
moto/awslambda/models.py
View File

@ -489,9 +489,16 @@ class LambdaFunction(CloudFormationModel, DockerModel):
).hexdigest()
self.code_size = 0
else:
uri, tag = self.code["ImageUri"].split(":")
if "@" in self.code["ImageUri"]:
# deploying via digest
uri, digest = self.code["ImageUri"].split("@")
image_id = {"imageDigest": digest}
else:
# deploying via tag
uri, tag = self.code["ImageUri"].split(":")
image_id = {"imageTag": tag}
repo_name = uri.split("/")[-1]
image_id = {"imageTag": tag}
ecr_backend = ecr_backends[self.account_id][self.region]
registry_id = ecr_backend.describe_registry()["registryId"]
images = ecr_backend.batch_get_image(

70
moto/ecr/models.py
View File

@ -236,12 +236,19 @@ class Repository(BaseObject, CloudFormationModel):
class Image(BaseObject):
def __init__(
self, account_id, tag, manifest, repository, digest=None, registry_id=None
self,
account_id,
tag,
manifest,
repository,
image_manifest_mediatype=None,
digest=None,
registry_id=None,
):
self.image_tag = tag
self.image_tags = [tag] if tag is not None else []
self.image_manifest = manifest
self.image_size_in_bytes = 50 * 1024 * 1024
self.image_manifest_mediatype = image_manifest_mediatype
self.repository = repository
self.registry_id = registry_id or account_id
self.image_digest = digest
@ -250,17 +257,33 @@ class Image(BaseObject):
def _create_digest(self):
image_manifest = json.loads(self.image_manifest)
layer_digests = [layer["digest"] for layer in image_manifest["layers"]]
self.image_digest = (
"sha256:"
+ hashlib.sha256("".join(layer_digests).encode("utf-8")).hexdigest()
)
if "layers" in image_manifest:
layer_digests = [layer["digest"] for layer in image_manifest["layers"]]
self.image_digest = (
"sha256:"
+ hashlib.sha256("".join(layer_digests).encode("utf-8")).hexdigest()
)
else:
random_sha = hashlib.sha256(
f"{random.randint(0,100)}".encode("utf-8")
).hexdigest()
self.image_digest = f"sha256:{random_sha}"
def get_image_digest(self):
if not self.image_digest:
self._create_digest()
return self.image_digest
def get_image_size_in_bytes(self):
image_manifest = json.loads(self.image_manifest)
if "layers" in image_manifest:
try:
return image_manifest["config"]["size"]
except KeyError:
return 50 * 1024 * 1024
else:
return None
def get_image_manifest(self):
return self.image_manifest
@ -303,9 +326,10 @@ class Image(BaseObject):
response_object["imageTags"] = self.image_tags
response_object["imageDigest"] = self.get_image_digest()
response_object["imageManifest"] = self.image_manifest
response_object["imageManifestMediaType"] = self.image_manifest_mediatype
response_object["repositoryName"] = self.repository
response_object["registryId"] = self.registry_id
response_object["imageSizeInBytes"] = self.image_size_in_bytes
response_object["imageSizeInBytes"] = self.get_image_size_in_bytes()
response_object["imagePushedAt"] = self.image_pushed_at
return {k: v for k, v in response_object.items() if v is not None and v != []}
@ -486,12 +510,31 @@ class ECRBackend(BaseBackend):
return response
def put_image(self, repository_name, image_manifest, image_tag):
def put_image(
self, repository_name, image_manifest, image_tag, image_manifest_mediatype=None
):
if repository_name in self.repositories:
repository = self.repositories[repository_name]
else:
raise Exception(f"{repository_name} is not a repository")
try:
parsed_image_manifest = json.loads(image_manifest)
except json.JSONDecodeError:
raise Exception(
"Invalid parameter at 'ImageManifest' failed to satisfy constraint: 'Invalid JSON syntax'"
)
if image_manifest_mediatype:
parsed_image_manifest["imageManifest"] = image_manifest_mediatype
else:
if "mediaType" not in parsed_image_manifest:
raise Exception(
"image manifest mediatype not provided in manifest or parameter"
)
else:
image_manifest_mediatype = parsed_image_manifest["mediaType"]
# Tags are unique, so delete any existing image with this tag first
self.batch_delete_image(
repository_name=repository_name, image_ids=[{"imageTag": image_tag}]
@ -503,9 +546,16 @@ class ECRBackend(BaseBackend):
repository.images,
)
)
if not existing_images:
# this image is not in ECR yet
image = Image(self.account_id, image_tag, image_manifest, repository_name)
image = Image(
self.account_id,
image_tag,
image_manifest,
repository_name,
image_manifest_mediatype,
)
repository.images.append(image)
return image
else:

42
tests/test_awslambda/test_lambda.py
View File

@ -218,12 +218,12 @@ def ecr_repo_fixture():
repo_name = "testlambdaecr"
ecr_client = ecr_client = boto3.client("ecr", "us-east-1")
ecr_client.create_repository(repositoryName=repo_name)
ecr_client.put_image(
response = ecr_client.put_image(
repositoryName=repo_name,
imageManifest=json.dumps(_create_image_manifest()),
imageTag="latest",
)
yield
yield response["image"]["imageId"]
ecr_client.delete_repository(repositoryName=repo_name, force=True)
os.environ["MOTO_LAMBDA_STUB_ECR"] = "TRUE"
@ -262,7 +262,7 @@ def test_create_function_from_stubbed_ecr():
@mock_lambda
def test_create_function_from_mocked_ecr_image(
def test_create_function_from_mocked_ecr_image_tag(
with_ecr_mock,
): # pylint: disable=unused-argument
if settings.TEST_SERVER_MODE:
@ -272,7 +272,8 @@ def test_create_function_from_mocked_ecr_image(
lambda_client = boto3.client("lambda", "us-east-1")
fn_name = str(uuid4())[0:6]
image_uri = f"{ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/testlambdaecr:latest"
image = with_ecr_mock
image_uri = f"{ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/testlambdaecr:{image['imageTag']}"
dic = {
"FunctionName": fn_name,
@ -291,7 +292,9 @@ def test_create_function_from_mocked_ecr_image(
result = lambda_client.get_function(FunctionName=fn_name)
result.should.have.key("Configuration")
config = result["Configuration"]
config.should.have.key("CodeSha256").equals(resp["CodeSha256"])
config.should.have.key("CodeSha256").equals(
image["imageDigest"].replace("sha256:", "")
)
config.should.have.key("CodeSize").equals(resp["CodeSize"])
result.should.have.key("Code")
code = result["Code"]
@ -302,6 +305,35 @@ def test_create_function_from_mocked_ecr_image(
code.should.have.key("ResolvedImageUri").equals(resolved_image_uri)
@mock_lambda
def test_create_function_from_mocked_ecr_image_digest(
with_ecr_mock,
): # pylint: disable=unused-argument
if settings.TEST_SERVER_MODE:
raise SkipTest(
"Envars not easily set in server mode, feature off by default, skipping..."
)
lambda_client = boto3.client("lambda", "us-east-1")
fn_name = str(uuid4())[0:6]
image = with_ecr_mock
image_uri = f"{ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/testlambdaecr@{image['imageDigest']}"
dic = {
"FunctionName": fn_name,
"Role": get_role_name(),
"Code": {"ImageUri": image_uri},
"PackageType": "Image",
"Timeout": 100,
}
resp = lambda_client.create_function(**dic)
resp.should.have.key("FunctionName").equals(fn_name)
resp.should.have.key("CodeSize").greater_than(0)
resp.should.have.key("CodeSha256").equals(
image["imageDigest"].replace("sha256:", "")
)
resp.should.have.key("PackageType").equals("Image")
@mock_lambda
def test_create_function_from_mocked_ecr_missing_image(
with_ecr_mock,

95
tests/test_ecr/test_ecr_boto3.py
View File

@ -17,7 +17,10 @@ from unittest import SkipTest
from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID
from tests.test_ecr.test_ecr_helpers import _create_image_manifest
from tests.test_ecr.test_ecr_helpers import (
_create_image_manifest,
_create_image_manifest_list,
)
@mock_ecr
@ -347,6 +350,34 @@ def test_put_image():
response["image"]["registryId"].should.equal(ACCOUNT_ID)
@mock_ecr()
def test_put_manifest_list():
client = boto3.client("ecr", region_name="us-east-1")
_ = client.create_repository(repositoryName="test_repository")
manifest_list = _create_image_manifest_list()
for image_manifest in manifest_list["image_manifests"]:
_ = client.put_image(
repositoryName="test_repository",
imageManifest=json.dumps(image_manifest),
)
response = client.put_image(
repositoryName="test_repository",
imageManifest=json.dumps(manifest_list["manifest_list"]),
imageTag="multiArch",
)
response["image"]["imageId"]["imageTag"].should.equal("multiArch")
response["image"]["imageId"]["imageDigest"].should.contain("sha")
response["image"]["repositoryName"].should.equal("test_repository")
response["image"]["registryId"].should.equal(ACCOUNT_ID)
response["image"].should.have.key("imageManifest")
image_manifest = json.loads(response["image"]["imageManifest"])
image_manifest.should.have.key("mediaType")
image_manifest.should.have.key("manifests")
@mock_ecr
def test_put_image_with_push_date():
if os.environ.get("TEST_SERVER_MODE", "false").lower() == "true":
@ -571,29 +602,77 @@ def test_describe_images():
imageTag="v2",
)
manifest_list = _create_image_manifest_list()
for image_manifest in manifest_list["image_manifests"]:
_ = client.put_image(
repositoryName="test_repository",
imageManifest=json.dumps(image_manifest),
)
_ = client.put_image(
repositoryName="test_repository",
imageManifest=json.dumps(manifest_list["manifest_list"]),
imageTag="multiArch",
)
response = client.describe_images(repositoryName="test_repository")
type(response["imageDetails"]).should.be(list)
len(response["imageDetails"]).should.be(4)
len(response["imageDetails"]).should.be(7)
response["imageDetails"][0]["imageManifestMediaType"].should.contain(
"distribution.manifest.v2+json"
)
response["imageDetails"][1]["imageManifestMediaType"].should.contain(
"distribution.manifest.v2+json"
)
response["imageDetails"][2]["imageManifestMediaType"].should.contain(
"distribution.manifest.v2+json"
)
response["imageDetails"][3]["imageManifestMediaType"].should.contain(
"distribution.manifest.v2+json"
)
response["imageDetails"][4]["imageManifestMediaType"].should.contain(
"distribution.manifest.v2+json"
)
response["imageDetails"][5]["imageManifestMediaType"].should.contain(
"distribution.manifest.v2+json"
)
response["imageDetails"][6]["imageManifestMediaType"].should.contain(
"distribution.manifest.list.v2+json"
)
response["imageDetails"][0]["imageDigest"].should.contain("sha")
response["imageDetails"][1]["imageDigest"].should.contain("sha")
response["imageDetails"][2]["imageDigest"].should.contain("sha")
response["imageDetails"][3]["imageDigest"].should.contain("sha")
response["imageDetails"][4]["imageDigest"].should.contain("sha")
response["imageDetails"][5]["imageDigest"].should.contain("sha")
response["imageDetails"][6]["imageDigest"].should.contain("sha")
response["imageDetails"][0]["registryId"].should.equal(ACCOUNT_ID)
response["imageDetails"][1]["registryId"].should.equal(ACCOUNT_ID)
response["imageDetails"][2]["registryId"].should.equal(ACCOUNT_ID)
response["imageDetails"][3]["registryId"].should.equal(ACCOUNT_ID)
response["imageDetails"][4]["registryId"].should.equal(ACCOUNT_ID)
response["imageDetails"][5]["registryId"].should.equal(ACCOUNT_ID)
response["imageDetails"][6]["registryId"].should.equal(ACCOUNT_ID)
response["imageDetails"][0]["repositoryName"].should.equal("test_repository")
response["imageDetails"][1]["repositoryName"].should.equal("test_repository")
response["imageDetails"][2]["repositoryName"].should.equal("test_repository")
response["imageDetails"][3]["repositoryName"].should.equal("test_repository")
response["imageDetails"][4]["repositoryName"].should.equal("test_repository")
response["imageDetails"][5]["repositoryName"].should.equal("test_repository")
response["imageDetails"][6]["repositoryName"].should.equal("test_repository")
response["imageDetails"][0].should_not.have.key("imageTags")
response["imageDetails"][4].should_not.have.key("imageTags")
response["imageDetails"][5].should_not.have.key("imageTags")
len(response["imageDetails"][1]["imageTags"]).should.be(1)
len(response["imageDetails"][2]["imageTags"]).should.be(1)
len(response["imageDetails"][3]["imageTags"]).should.be(1)
len(response["imageDetails"][6]["imageTags"]).should.be(1)
image_tags = ["latest", "v1", "v2"]
set(
@ -604,10 +683,14 @@ def test_describe_images():
]
).should.equal(set(image_tags))
response["imageDetails"][0]["imageSizeInBytes"].should.equal(52428800)
response["imageDetails"][1]["imageSizeInBytes"].should.equal(52428800)
response["imageDetails"][2]["imageSizeInBytes"].should.equal(52428800)
response["imageDetails"][3]["imageSizeInBytes"].should.equal(52428800)
response["imageDetails"][6].should_not.have.key("imageSizeInBytes")
response["imageDetails"][0]["imageSizeInBytes"].should.be.greater_than(0)
response["imageDetails"][1]["imageSizeInBytes"].should.be.greater_than(0)
response["imageDetails"][2]["imageSizeInBytes"].should.be.greater_than(0)
response["imageDetails"][3]["imageSizeInBytes"].should.be.greater_than(0)
response["imageDetails"][4]["imageSizeInBytes"].should.be.greater_than(0)
response["imageDetails"][5]["imageSizeInBytes"].should.be.greater_than(0)
@mock_ecr

42
tests/test_ecr/test_ecr_helpers.py
View File

@ -1,9 +1,12 @@
import hashlib
import random
# Manifests Spec: https://docs.docker.com/registry/spec/manifest-v2-2/
def _generate_random_sha():
return hashlib.sha256(f"{random.randint(0,100)}".encode("utf-8")).hexdigest()
random_sha = hashlib.sha256(f"{random.randint(0,100)}".encode("utf-8")).hexdigest()
return f"sha256:{random_sha}"
def _create_image_layers(n):
@ -13,7 +16,7 @@ def _create_image_layers(n):
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": random.randint(100, 1000),
"digest": f"sha256:{_generate_random_sha()}",
"digest": _generate_random_sha(),
}
)
return layers
@ -21,7 +24,8 @@ def _create_image_layers(n):
def _create_image_digest(layers):
layer_digests = "".join([layer["digest"] for layer in layers])
return hashlib.sha256(f"{layer_digests}".encode("utf-8")).hexdigest()
summed_digest = hashlib.sha256(f"{layer_digests}".encode("utf-8")).hexdigest()
return f"sha256:{summed_digest}"
def _create_image_manifest(image_digest=None):
@ -38,3 +42,35 @@ def _create_image_manifest(image_digest=None):
},
"layers": layers,
}
def _create_manifest_list_distribution(
image_manifest: dict, architecture: str = "amd64", os: str = "linux"
):
return {
"mediaType": image_manifest["config"]["mediaType"],
"digest": image_manifest["config"]["digest"],
"size": image_manifest["config"]["size"],
"platform": {"architecture": architecture, "os": os},
}
def _create_image_manifest_list():
arm_image_manifest = _create_image_manifest()
amd_image_manifest = _create_image_manifest()
arm_distribution = _create_manifest_list_distribution(
arm_image_manifest, architecture="arm64"
)
amd_distribution = _create_manifest_list_distribution(
amd_image_manifest, architecture="amd64"
)
manifest_list = {
"mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
"schemaVersion": 2,
"manifests": [arm_distribution, amd_distribution],
}
return {
"image_manifests": [arm_image_manifest, amd_image_manifest],
"manifest_list": manifest_list,
}