From d9177f382e32f40d7a16a7fd04806f0d31f9fcf5 Mon Sep 17 00:00:00 2001 From: Franck Ndame Date: Thu, 8 Apr 2021 16:51:50 +0100 Subject: [PATCH] Implementation of core AWS Mediastore endpoints (#3825) * write boilerplate code * generate boilerplate code with scaffold script * create mediapackage channel * remove duplicate mediapackage reference * remove Channel key from mediapackage response * describe_channel endpoint added * create_origin_endpoint-added * keys changed to camel case to fix issue * minor changes to clean up * minor clean up again * implement & test delete_channel * delete origin endpoint created; WIP-tests failing * fix delete_origin_endpoint issue * refactor function call * delete origin endpoint completed; test_server tests added * implement and test describe_origin_endpoint * update origin endpoint added * remove print statements * implement test_list_origin_endpoint_succeeds * create test name changed * create test name changed * changes after flake8 and black run * url assertion added to decribe origin endpoint test * region dependent url enabled * initial commit; WIP * create container added, still WIP * create_container working * test_create_channel_succeeds * test_describe_container_succeeds * get_lifecycle_policy added; error tests added * changes to pass linting * added exception for container but no policy * linting * put_container_policy added * put_metric_policy added * list_containers added * resolved linting * test_describe_container_raises_error_if_container_does_not_exist * added __init__ file * __init__ added to mediapackage as well * Mediastore (#20) * initial commit; WIP * create container added, still WIP * create_container working * test_create_channel_succeeds * test_describe_container_succeeds * get_lifecycle_policy added; error tests added * changes to pass linting * added exception for container but no policy * linting * put_container_policy added * put_metric_policy added * list_containers added * resolved linting * test_describe_container_raises_error_if_container_does_not_exist * added __init__ file * __init__ added to mediapackage as well Co-authored-by: FranckNdame * test_server fixed; resolved rebasing mix ups on tests * [FIX] Ensures MediaConnect create_flow sets a valid sourceArn * code clean up Co-authored-by: Anya Co-authored-by: AnyaChamp <71766808+AnyaChamp@users.noreply.github.com> --- moto/__init__.py | 1 + moto/backends.py | 1 + moto/mediaconnect/models.py | 8 + moto/mediastore/__init__.py | 6 + moto/mediastore/exceptions.py | 23 +++ moto/mediastore/models.py | 120 ++++++++++++ moto/mediastore/responses.py | 79 ++++++++ moto/mediastore/urls.py | 12 ++ tests/test_mediaconnect/test_mediaconnect.py | 3 + tests/test_mediapackage/__init__.py | 0 tests/test_mediastore/__init__.py | 0 tests/test_mediastore/test_mediastore.py | 194 +++++++++++++++++++ tests/test_mediastore/test_server.py | 23 +++ 13 files changed, 470 insertions(+) create mode 100644 moto/mediastore/__init__.py create mode 100644 moto/mediastore/exceptions.py create mode 100644 moto/mediastore/models.py create mode 100644 moto/mediastore/responses.py create mode 100644 moto/mediastore/urls.py create mode 100644 tests/test_mediapackage/__init__.py create mode 100644 tests/test_mediastore/__init__.py create mode 100644 tests/test_mediastore/test_mediastore.py create mode 100644 tests/test_mediastore/test_server.py diff --git a/moto/__init__.py b/moto/__init__.py index 72b2e68c6..9e76f5141 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -123,6 +123,7 @@ mock_medialive = lazy_load(".medialive", "mock_medialive") mock_support = lazy_load(".support", "mock_support") mock_mediaconnect = lazy_load(".mediaconnect", "mock_mediaconnect") mock_mediapackage = lazy_load(".mediapackage", "mock_mediapackage") +mock_mediastore = lazy_load(".mediastore", "mock_mediastore") # import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) diff --git a/moto/backends.py b/moto/backends.py index 15c894eb9..33f4b2973 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -80,6 +80,7 @@ BACKENDS = { "support": ("support", "support_backends"), "mediaconnect": ("mediaconnect", "mediaconnect_backends"), "mediapackage": ("mediapackage", "mediapackage_backends"), + "mediastore": ("mediastore", "mediastore_backends"), } diff --git a/moto/mediaconnect/models.py b/moto/mediaconnect/models.py index ead3dbf7e..ae4b16578 100644 --- a/moto/mediaconnect/models.py +++ b/moto/mediaconnect/models.py @@ -23,6 +23,10 @@ class Flow(BaseModel): self.description = None self.flow_arn = None self.egress_ip = None + if self.source and not self.sources: + self.sources = [ + self.source, + ] def to_dict(self, include=None): data = { @@ -92,6 +96,10 @@ class MediaConnectBackend(BaseBackend): sources, vpc_interfaces, ): + if isinstance(source, dict) and source.get("name"): + source["sourceArn"] = "arn:aws:mediaconnect:source:{}".format( + source["name"] + ) flow = Flow( availability_zone=availability_zone, entitlements=entitlements, diff --git a/moto/mediastore/__init__.py b/moto/mediastore/__init__.py new file mode 100644 index 000000000..35bf5778f --- /dev/null +++ b/moto/mediastore/__init__.py @@ -0,0 +1,6 @@ +from __future__ import unicode_literals +from .models import mediastore_backends +from ..core.models import base_decorator + +mediastore_backend = mediastore_backends["us-east-1"] +mock_mediastore = base_decorator(mediastore_backends) diff --git a/moto/mediastore/exceptions.py b/moto/mediastore/exceptions.py new file mode 100644 index 000000000..9ed31cb2c --- /dev/null +++ b/moto/mediastore/exceptions.py @@ -0,0 +1,23 @@ +from __future__ import unicode_literals +from moto.core.exceptions import JsonRESTError + + +class MediaStoreClientError(JsonRESTError): + code = 400 + + +class ResourceNotFoundException(MediaStoreClientError): + def __init__(self, msg=None): + self.code = 400 + super(ResourceNotFoundException, self).__init__( + "ResourceNotFoundException", msg or "The specified container does not exist" + ) + + +class PolicyNotFoundException(MediaStoreClientError): + def __init__(self, msg=None): + self.code = 400 + super(PolicyNotFoundException, self).__init__( + "PolicyNotFoundException", + msg or "The policy does not exist within the specfied container", + ) diff --git a/moto/mediastore/models.py b/moto/mediastore/models.py new file mode 100644 index 000000000..b26d606b0 --- /dev/null +++ b/moto/mediastore/models.py @@ -0,0 +1,120 @@ +from __future__ import unicode_literals +from boto3 import Session +from moto.core import BaseBackend, BaseModel +from collections import OrderedDict +from datetime import date +from .exceptions import ResourceNotFoundException, PolicyNotFoundException + + +class Container(BaseModel): + def __init__(self, *args, **kwargs): + self.arn = kwargs.get("arn") + self.name = kwargs.get("name") + self.endpoint = kwargs.get("endpoint") + self.status = kwargs.get("status") + self.creation_time = kwargs.get("creation_time") + self.lifecycle_policy = None + self.policy = None + self.metric_policy = None + + def to_dict(self, exclude=None): + data = { + "ARN": self.arn, + "Name": self.name, + "Endpoint": self.endpoint, + "Status": self.status, + "CreationTime": self.creation_time, + } + if exclude: + for key in exclude: + del data[key] + return data + + +class MediaStoreBackend(BaseBackend): + def __init__(self, region_name=None): + super(MediaStoreBackend, self).__init__() + self.region_name = region_name + self._containers = OrderedDict() + + def reset(self): + region_name = self.region_name + self.__dict__ = {} + self.__init__(region_name) + + def create_container(self, name, tags): + arn = "arn:aws:mediastore:container:{}".format(name) + container = Container( + arn=arn, + name=name, + endpoint="/{}".format(name), + status="CREATING", + creation_time=date.today().strftime("%m/%d/%Y, %H:%M:%S"), + ) + self._containers[name] = container + return container + + def describe_container(self, name): + if name not in self._containers: + raise ResourceNotFoundException() + container = self._containers[name] + container.status = "ACTIVE" + return container + + def list_containers(self, next_token, max_results): + containers = list(self._containers.values()) + response_containers = [c.to_dict() for c in containers] + return response_containers, None + + def put_lifecycle_policy(self, container_name, lifecycle_policy): + if container_name not in self._containers: + raise ResourceNotFoundException() + self._containers[container_name].lifecycle_policy = lifecycle_policy + return {} + + def get_lifecycle_policy(self, container_name): + if container_name not in self._containers: + raise ResourceNotFoundException() + lifecycle_policy = self._containers[container_name].lifecycle_policy + if not lifecycle_policy: + raise PolicyNotFoundException() + return lifecycle_policy + + def put_container_policy(self, container_name, policy): + if container_name not in self._containers: + raise ResourceNotFoundException() + self._containers[container_name].policy = policy + return {} + + def get_container_policy(self, container_name): + if container_name not in self._containers: + raise ResourceNotFoundException() + policy = self._containers[container_name].policy + if not policy: + raise PolicyNotFoundException() + return policy + + def put_metric_policy(self, container_name, metric_policy): + if container_name not in self._containers: + raise ResourceNotFoundException() + self._containers[container_name].metric_policy = metric_policy + return {} + + def get_metric_policy(self, container_name): + if container_name not in self._containers: + raise ResourceNotFoundException() + metric_policy = self._containers[container_name].metric_policy + if not metric_policy: + raise PolicyNotFoundException() + return metric_policy + + +mediastore_backends = {} +for region in Session().get_available_regions("mediastore"): + mediastore_backends[region] = MediaStoreBackend(region) +for region in Session().get_available_regions( + "mediastore", partition_name="aws-us-gov" +): + mediastore_backends[region] = MediaStoreBackend(region) +for region in Session().get_available_regions("mediastore", partition_name="aws-cn"): + mediastore_backends[region] = MediaStoreBackend(region) diff --git a/moto/mediastore/responses.py b/moto/mediastore/responses.py new file mode 100644 index 000000000..0ff1ec5dc --- /dev/null +++ b/moto/mediastore/responses.py @@ -0,0 +1,79 @@ +from __future__ import unicode_literals +from moto.core.responses import BaseResponse +from .models import mediastore_backends +import json + + +class MediaStoreResponse(BaseResponse): + SERVICE_NAME = "mediastore" + + @property + def mediastore_backend(self): + return mediastore_backends[self.region] + + def create_container(self): + name = self._get_param("ContainerName") + tags = self._get_param("Tags") + container = self.mediastore_backend.create_container(name=name, tags=tags) + return json.dumps(dict(Container=container.to_dict())) + + def describe_container(self): + name = self._get_param("ContainerName") + container = self.mediastore_backend.describe_container(name=name) + return json.dumps(dict(Container=container.to_dict())) + + def list_containers(self): + next_token = self._get_param("NextToken") + max_results = self._get_int_param("MaxResults") + containers, next_token = self.mediastore_backend.list_containers( + next_token=next_token, max_results=max_results, + ) + return json.dumps(dict(dict(Containers=containers), NextToken=next_token)) + + def put_lifecycle_policy(self): + container_name = self._get_param("ContainerName") + lifecycle_policy = self._get_param("LifecyclePolicy") + policy = self.mediastore_backend.put_lifecycle_policy( + container_name=container_name, lifecycle_policy=lifecycle_policy, + ) + return json.dumps(policy) + + def get_lifecycle_policy(self): + container_name = self._get_param("ContainerName") + lifecycle_policy = self.mediastore_backend.get_lifecycle_policy( + container_name=container_name, + ) + return json.dumps(dict(LifecyclePolicy=lifecycle_policy)) + + def put_container_policy(self): + container_name = self._get_param("ContainerName") + policy = self._get_param("Policy") + container_policy = self.mediastore_backend.put_container_policy( + container_name=container_name, policy=policy, + ) + return json.dumps(container_policy) + + def get_container_policy(self): + container_name = self._get_param("ContainerName") + policy = self.mediastore_backend.get_container_policy( + container_name=container_name, + ) + return json.dumps(dict(Policy=policy)) + + def put_metric_policy(self): + container_name = self._get_param("ContainerName") + metric_policy = self._get_param("MetricPolicy") + self.mediastore_backend.put_metric_policy( + container_name=container_name, metric_policy=metric_policy, + ) + return json.dumps(metric_policy) + + def get_metric_policy(self): + container_name = self._get_param("ContainerName") + metric_policy = self.mediastore_backend.get_metric_policy( + container_name=container_name, + ) + return json.dumps(dict(MetricPolicy=metric_policy)) + + +# add templates from here diff --git a/moto/mediastore/urls.py b/moto/mediastore/urls.py new file mode 100644 index 000000000..dc1507264 --- /dev/null +++ b/moto/mediastore/urls.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals +from .responses import MediaStoreResponse + +url_bases = [ + "https?://mediastore.(.+).amazonaws.com", +] + +response = MediaStoreResponse() + +url_paths = { + "{0}/$": response.dispatch, +} diff --git a/tests/test_mediaconnect/test_mediaconnect.py b/tests/test_mediaconnect/test_mediaconnect.py index 194ee9e24..0e7d985da 100644 --- a/tests/test_mediaconnect/test_mediaconnect.py +++ b/tests/test_mediaconnect/test_mediaconnect.py @@ -59,6 +59,9 @@ def test_create_flow_succeeds(): response["Flow"]["FlowArn"][:26].should.equal("arn:aws:mediaconnect:flow:") response["Flow"]["Name"].should.equal("test Flow 1") response["Flow"]["Status"].should.equal("STANDBY") + response["Flow"]["Sources"][0][ + "SourceArn" + ] == "arn:aws:mediaconnect:source:Source A" @mock_mediaconnect diff --git a/tests/test_mediapackage/__init__.py b/tests/test_mediapackage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_mediastore/__init__.py b/tests/test_mediastore/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_mediastore/test_mediastore.py b/tests/test_mediastore/test_mediastore.py new file mode 100644 index 000000000..baf16ff1c --- /dev/null +++ b/tests/test_mediastore/test_mediastore.py @@ -0,0 +1,194 @@ +from __future__ import unicode_literals + +import boto3 +import sure # noqa +import pytest +from moto import mock_mediastore +from botocore.exceptions import ClientError + +region = "eu-west-1" + + +@mock_mediastore +def test_create_container_succeeds(): + client = boto3.client("mediastore", region_name=region) + response = client.create_container( + ContainerName="Awesome container!", Tags=[{"Key": "customer"}] + ) + container = response["Container"] + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + container["ARN"].should.equal( + "arn:aws:mediastore:container:{}".format(container["Name"]) + ) + container["Name"].should.equal("Awesome container!") + container["Status"].should.equal("CREATING") + + +@mock_mediastore +def test_describe_container_succeeds(): + client = boto3.client("mediastore", region_name=region) + create_response = client.create_container( + ContainerName="Awesome container!", Tags=[{"Key": "customer"}] + ) + container_name = create_response["Container"]["Name"] + response = client.describe_container(ContainerName=container_name) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + container = response["Container"] + container["ARN"].should.equal( + "arn:aws:mediastore:container:{}".format(container["Name"]) + ) + container["Name"].should.equal("Awesome container!") + container["Status"].should.equal("ACTIVE") + + +@mock_mediastore +def test_list_containers_succeeds(): + client = boto3.client("mediastore", region_name=region) + client.create_container( + ContainerName="Awesome container!", Tags=[{"Key": "customer"}] + ) + list_response = client.list_containers(NextToken="next-token", MaxResults=123) + containers_list = list_response["Containers"] + len(containers_list).should.equal(1) + client.create_container( + ContainerName="Awesome container2!", Tags=[{"Key": "customer"}] + ) + list_response = client.list_containers(NextToken="next-token", MaxResults=123) + containers_list = list_response["Containers"] + len(containers_list).should.equal(2) + + +@mock_mediastore +def test_describe_container_raises_error_if_container_does_not_exist(): + client = boto3.client("mediastore", region_name=region) + with pytest.raises(ClientError) as ex: + client.describe_container(ContainerName="container-name") + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + + +@mock_mediastore +def test_put_lifecycle_policy_succeeds(): + client = boto3.client("mediastore", region_name=region) + container_response = client.create_container( + ContainerName="container-name", Tags=[{"Key": "customer"}] + ) + container = container_response["Container"] + client.put_lifecycle_policy( + ContainerName=container["Name"], LifecyclePolicy="lifecycle-policy" + ) + response = client.get_lifecycle_policy(ContainerName=container["Name"]) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + response["LifecyclePolicy"].should.equal("lifecycle-policy") + + +@mock_mediastore +def test_put_lifecycle_policy_raises_error_if_container_does_not_exist(): + client = boto3.client("mediastore", region_name=region) + with pytest.raises(ClientError) as ex: + client.put_lifecycle_policy( + ContainerName="container-name", LifecyclePolicy="lifecycle-policy" + ) + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + + +@mock_mediastore +def test_get_lifecycle_policy_raises_error_if_container_does_not_exist(): + client = boto3.client("mediastore", region_name=region) + with pytest.raises(ClientError) as ex: + client.get_lifecycle_policy(ContainerName="container-name") + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + + +@mock_mediastore +def test_get_lifecycle_policy_raises_error_if_container_does_not_have_lifecycle_policy(): + client = boto3.client("mediastore", region_name=region) + client.create_container(ContainerName="container-name", Tags=[{"Key": "customer"}]) + with pytest.raises(ClientError) as ex: + client.get_lifecycle_policy(ContainerName="container-name") + ex.value.response["Error"]["Code"].should.equal("PolicyNotFoundException") + + +@mock_mediastore +def test_put_container_policy_succeeds(): + client = boto3.client("mediastore", region_name=region) + container_response = client.create_container( + ContainerName="container-name", Tags=[{"Key": "customer"}] + ) + container = container_response["Container"] + response = client.put_container_policy( + ContainerName=container["Name"], Policy="container-policy" + ) + response = client.get_container_policy(ContainerName=container["Name"]) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + response["Policy"].should.equal("container-policy") + + +@mock_mediastore +def test_put_container_policy_raises_error_if_container_does_not_exist(): + client = boto3.client("mediastore", region_name=region) + with pytest.raises(ClientError) as ex: + client.put_container_policy( + ContainerName="container-name", Policy="container-policy" + ) + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + + +@mock_mediastore +def test_get_container_policy_raises_error_if_container_does_not_exist(): + client = boto3.client("mediastore", region_name=region) + with pytest.raises(ClientError) as ex: + client.get_container_policy(ContainerName="container-name") + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + + +@mock_mediastore +def test_get_container_policy_raises_error_if_container_does_not_have_container_policy(): + client = boto3.client("mediastore", region_name=region) + client.create_container(ContainerName="container-name", Tags=[{"Key": "customer"}]) + with pytest.raises(ClientError) as ex: + client.get_container_policy(ContainerName="container-name") + ex.value.response["Error"]["Code"].should.equal("PolicyNotFoundException") + + +@mock_mediastore +def test_put_metric_policy_succeeds(): + client = boto3.client("mediastore", region_name=region) + container_response = client.create_container( + ContainerName="container-name", Tags=[{"Key": "customer"}] + ) + container = container_response["Container"] + response = client.put_metric_policy( + ContainerName=container["Name"], + MetricPolicy={"ContainerLevelMetrics": "ENABLED"}, + ) + response = client.get_metric_policy(ContainerName=container["Name"]) + response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200) + response["MetricPolicy"].should.equal({"ContainerLevelMetrics": "ENABLED"}) + + +@mock_mediastore +def test_put_metric_policy_raises_error_if_container_does_not_exist(): + client = boto3.client("mediastore", region_name=region) + with pytest.raises(ClientError) as ex: + client.put_metric_policy( + ContainerName="container-name", + MetricPolicy={"ContainerLevelMetrics": "ENABLED"}, + ) + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + + +@mock_mediastore +def test_get_metric_policy_raises_error_if_container_does_not_exist(): + client = boto3.client("mediastore", region_name=region) + with pytest.raises(ClientError) as ex: + client.get_metric_policy(ContainerName="container-name") + ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException") + + +@mock_mediastore +def test_get_metric_policy_raises_error_if_container_does_not_have_metric_policy(): + client = boto3.client("mediastore", region_name=region) + client.create_container(ContainerName="container-name", Tags=[{"Key": "customer"}]) + with pytest.raises(ClientError) as ex: + client.get_metric_policy(ContainerName="container-name") + ex.value.response["Error"]["Code"].should.equal("PolicyNotFoundException") diff --git a/tests/test_mediastore/test_server.py b/tests/test_mediastore/test_server.py new file mode 100644 index 000000000..040721ae7 --- /dev/null +++ b/tests/test_mediastore/test_server.py @@ -0,0 +1,23 @@ +from __future__ import unicode_literals + +import sure # noqa + +import moto.server as server +from moto import mock_mediastore + +""" +Test the different server responses +""" + + +@mock_mediastore +def test_mediastore_lists_containers(): + backend = server.create_backend_app("mediastore") + test_client = backend.test_client() + + res = test_client.get( + "/", headers={"X-Amz-Target": "MediaStore_20170901.ListContainers"}, + ) + + result = res.data.decode("utf-8") + result.should.contain('"Containers": []')