Implementation of core AWS Mediastore endpoints (#3825)

* write boilerplate code

* generate boilerplate code with scaffold script

* create mediapackage channel

* remove duplicate mediapackage reference

* remove Channel key from mediapackage response

* describe_channel endpoint added

* create_origin_endpoint-added

* keys changed to camel case to fix issue

* minor changes to clean up

* minor clean up again

* implement & test delete_channel

* delete origin endpoint created; WIP-tests failing

* fix delete_origin_endpoint issue

* refactor function call

* delete origin endpoint completed; test_server tests added

* implement and test describe_origin_endpoint

* update origin endpoint added

* remove print statements

* implement test_list_origin_endpoint_succeeds

* create test name changed

* create test name changed

* changes after flake8 and black run

* url assertion added to decribe origin endpoint test

* region dependent url enabled

* initial commit; WIP

* create container added, still WIP

* create_container working

* test_create_channel_succeeds

* test_describe_container_succeeds

* get_lifecycle_policy added; error tests added

* changes to pass linting

* added exception for container but no policy

* linting

* put_container_policy added

* put_metric_policy added

* list_containers added

* resolved linting

* test_describe_container_raises_error_if_container_does_not_exist

* added __init__ file

* __init__ added to mediapackage as well

* Mediastore (#20)

* initial commit; WIP

* create container added, still WIP

* create_container working

* test_create_channel_succeeds

* test_describe_container_succeeds

* get_lifecycle_policy added; error tests added

* changes to pass linting

* added exception for container but no policy

* linting

* put_container_policy added

* put_metric_policy added

* list_containers added

* resolved linting

* test_describe_container_raises_error_if_container_does_not_exist

* added __init__ file

* __init__ added to mediapackage as well

Co-authored-by: FranckNdame <franck.mpouli@yahoo.com>

* test_server fixed; resolved rebasing mix ups on tests

* [FIX] Ensures MediaConnect create_flow sets a valid sourceArn

* code clean up

Co-authored-by: Anya <anya.champaneria@capablue.com>
Co-authored-by: AnyaChamp <71766808+AnyaChamp@users.noreply.github.com>

moto/__init__.py

@@ -123,6 +123,7 @@ mock_medialive = lazy_load(".medialive", "mock_medialive")
 mock_support = lazy_load(".support", "mock_support")
 mock_mediaconnect = lazy_load(".mediaconnect", "mock_mediaconnect")
 mock_mediapackage = lazy_load(".mediapackage", "mock_mediapackage")
+mock_mediastore = lazy_load(".mediastore", "mock_mediastore")
 
 # import logging
 # logging.getLogger('boto').setLevel(logging.CRITICAL)

moto/backends.py

@@ -80,6 +80,7 @@ BACKENDS = {
     "support": ("support", "support_backends"),
     "mediaconnect": ("mediaconnect", "mediaconnect_backends"),
     "mediapackage": ("mediapackage", "mediapackage_backends"),
+    "mediastore": ("mediastore", "mediastore_backends"),
 }
 
 

moto/mediaconnect/models.py

@@ -23,6 +23,10 @@ class Flow(BaseModel):
         self.description = None
         self.flow_arn = None
         self.egress_ip = None
+        if self.source and not self.sources:
+            self.sources = [
+                self.source,
+            ]
 
     def to_dict(self, include=None):
         data = {
@@ -92,6 +96,10 @@ class MediaConnectBackend(BaseBackend):
         sources,
         vpc_interfaces,
     ):
+        if isinstance(source, dict) and source.get("name"):
+            source["sourceArn"] = "arn:aws:mediaconnect:source:{}".format(
+                source["name"]
+            )
         flow = Flow(
             availability_zone=availability_zone,
             entitlements=entitlements,

moto/mediastore/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import unicode_literals
+from .models import mediastore_backends
+from ..core.models import base_decorator
+
+mediastore_backend = mediastore_backends["us-east-1"]
+mock_mediastore = base_decorator(mediastore_backends)

moto/mediastore/exceptions.py

@@ -0,0 +1,23 @@
+from __future__ import unicode_literals
+from moto.core.exceptions import JsonRESTError
+
+
+class MediaStoreClientError(JsonRESTError):
+    code = 400
+
+
+class ResourceNotFoundException(MediaStoreClientError):
+    def __init__(self, msg=None):
+        self.code = 400
+        super(ResourceNotFoundException, self).__init__(
+            "ResourceNotFoundException", msg or "The specified container does not exist"
+        )
+
+
+class PolicyNotFoundException(MediaStoreClientError):
+    def __init__(self, msg=None):
+        self.code = 400
+        super(PolicyNotFoundException, self).__init__(
+            "PolicyNotFoundException",
+            msg or "The policy does not exist within the specfied container",
+        )

moto/mediastore/models.py

@@ -0,0 +1,120 @@
+from __future__ import unicode_literals
+from boto3 import Session
+from moto.core import BaseBackend, BaseModel
+from collections import OrderedDict
+from datetime import date
+from .exceptions import ResourceNotFoundException, PolicyNotFoundException
+
+
+class Container(BaseModel):
+    def __init__(self, *args, **kwargs):
+        self.arn = kwargs.get("arn")
+        self.name = kwargs.get("name")
+        self.endpoint = kwargs.get("endpoint")
+        self.status = kwargs.get("status")
+        self.creation_time = kwargs.get("creation_time")
+        self.lifecycle_policy = None
+        self.policy = None
+        self.metric_policy = None
+
+    def to_dict(self, exclude=None):
+        data = {
+            "ARN": self.arn,
+            "Name": self.name,
+            "Endpoint": self.endpoint,
+            "Status": self.status,
+            "CreationTime": self.creation_time,
+        }
+        if exclude:
+            for key in exclude:
+                del data[key]
+        return data
+
+
+class MediaStoreBackend(BaseBackend):
+    def __init__(self, region_name=None):
+        super(MediaStoreBackend, self).__init__()
+        self.region_name = region_name
+        self._containers = OrderedDict()
+
+    def reset(self):
+        region_name = self.region_name
+        self.__dict__ = {}
+        self.__init__(region_name)
+
+    def create_container(self, name, tags):
+        arn = "arn:aws:mediastore:container:{}".format(name)
+        container = Container(
+            arn=arn,
+            name=name,
+            endpoint="/{}".format(name),
+            status="CREATING",
+            creation_time=date.today().strftime("%m/%d/%Y, %H:%M:%S"),
+        )
+        self._containers[name] = container
+        return container
+
+    def describe_container(self, name):
+        if name not in self._containers:
+            raise ResourceNotFoundException()
+        container = self._containers[name]
+        container.status = "ACTIVE"
+        return container
+
+    def list_containers(self, next_token, max_results):
+        containers = list(self._containers.values())
+        response_containers = [c.to_dict() for c in containers]
+        return response_containers, None
+
+    def put_lifecycle_policy(self, container_name, lifecycle_policy):
+        if container_name not in self._containers:
+            raise ResourceNotFoundException()
+        self._containers[container_name].lifecycle_policy = lifecycle_policy
+        return {}
+
+    def get_lifecycle_policy(self, container_name):
+        if container_name not in self._containers:
+            raise ResourceNotFoundException()
+        lifecycle_policy = self._containers[container_name].lifecycle_policy
+        if not lifecycle_policy:
+            raise PolicyNotFoundException()
+        return lifecycle_policy
+
+    def put_container_policy(self, container_name, policy):
+        if container_name not in self._containers:
+            raise ResourceNotFoundException()
+        self._containers[container_name].policy = policy
+        return {}
+
+    def get_container_policy(self, container_name):
+        if container_name not in self._containers:
+            raise ResourceNotFoundException()
+        policy = self._containers[container_name].policy
+        if not policy:
+            raise PolicyNotFoundException()
+        return policy
+
+    def put_metric_policy(self, container_name, metric_policy):
+        if container_name not in self._containers:
+            raise ResourceNotFoundException()
+        self._containers[container_name].metric_policy = metric_policy
+        return {}
+
+    def get_metric_policy(self, container_name):
+        if container_name not in self._containers:
+            raise ResourceNotFoundException()
+        metric_policy = self._containers[container_name].metric_policy
+        if not metric_policy:
+            raise PolicyNotFoundException()
+        return metric_policy
+
+
+mediastore_backends = {}
+for region in Session().get_available_regions("mediastore"):
+    mediastore_backends[region] = MediaStoreBackend(region)
+for region in Session().get_available_regions(
+    "mediastore", partition_name="aws-us-gov"
+):
+    mediastore_backends[region] = MediaStoreBackend(region)
+for region in Session().get_available_regions("mediastore", partition_name="aws-cn"):
+    mediastore_backends[region] = MediaStoreBackend(region)

moto/mediastore/responses.py

@@ -0,0 +1,79 @@
+from __future__ import unicode_literals
+from moto.core.responses import BaseResponse
+from .models import mediastore_backends
+import json
+
+
+class MediaStoreResponse(BaseResponse):
+    SERVICE_NAME = "mediastore"
+
+    @property
+    def mediastore_backend(self):
+        return mediastore_backends[self.region]
+
+    def create_container(self):
+        name = self._get_param("ContainerName")
+        tags = self._get_param("Tags")
+        container = self.mediastore_backend.create_container(name=name, tags=tags)
+        return json.dumps(dict(Container=container.to_dict()))
+
+    def describe_container(self):
+        name = self._get_param("ContainerName")
+        container = self.mediastore_backend.describe_container(name=name)
+        return json.dumps(dict(Container=container.to_dict()))
+
+    def list_containers(self):
+        next_token = self._get_param("NextToken")
+        max_results = self._get_int_param("MaxResults")
+        containers, next_token = self.mediastore_backend.list_containers(
+            next_token=next_token, max_results=max_results,
+        )
+        return json.dumps(dict(dict(Containers=containers), NextToken=next_token))
+
+    def put_lifecycle_policy(self):
+        container_name = self._get_param("ContainerName")
+        lifecycle_policy = self._get_param("LifecyclePolicy")
+        policy = self.mediastore_backend.put_lifecycle_policy(
+            container_name=container_name, lifecycle_policy=lifecycle_policy,
+        )
+        return json.dumps(policy)
+
+    def get_lifecycle_policy(self):
+        container_name = self._get_param("ContainerName")
+        lifecycle_policy = self.mediastore_backend.get_lifecycle_policy(
+            container_name=container_name,
+        )
+        return json.dumps(dict(LifecyclePolicy=lifecycle_policy))
+
+    def put_container_policy(self):
+        container_name = self._get_param("ContainerName")
+        policy = self._get_param("Policy")
+        container_policy = self.mediastore_backend.put_container_policy(
+            container_name=container_name, policy=policy,
+        )
+        return json.dumps(container_policy)
+
+    def get_container_policy(self):
+        container_name = self._get_param("ContainerName")
+        policy = self.mediastore_backend.get_container_policy(
+            container_name=container_name,
+        )
+        return json.dumps(dict(Policy=policy))
+
+    def put_metric_policy(self):
+        container_name = self._get_param("ContainerName")
+        metric_policy = self._get_param("MetricPolicy")
+        self.mediastore_backend.put_metric_policy(
+            container_name=container_name, metric_policy=metric_policy,
+        )
+        return json.dumps(metric_policy)
+
+    def get_metric_policy(self):
+        container_name = self._get_param("ContainerName")
+        metric_policy = self.mediastore_backend.get_metric_policy(
+            container_name=container_name,
+        )
+        return json.dumps(dict(MetricPolicy=metric_policy))
+
+
+# add templates from here

moto/mediastore/urls.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals
+from .responses import MediaStoreResponse
+
+url_bases = [
+    "https?://mediastore.(.+).amazonaws.com",
+]
+
+response = MediaStoreResponse()
+
+url_paths = {
+    "{0}/$": response.dispatch,
+}

tests/test_mediaconnect/test_mediaconnect.py

@@ -59,6 +59,9 @@ def test_create_flow_succeeds():
     response["Flow"]["FlowArn"][:26].should.equal("arn:aws:mediaconnect:flow:")
     response["Flow"]["Name"].should.equal("test Flow 1")
     response["Flow"]["Status"].should.equal("STANDBY")
+    response["Flow"]["Sources"][0][
+        "SourceArn"
+    ] == "arn:aws:mediaconnect:source:Source A"
 
 
 @mock_mediaconnect

tests/test_mediastore/test_mediastore.py

@@ -0,0 +1,194 @@
+from __future__ import unicode_literals
+
+import boto3
+import sure  # noqa
+import pytest
+from moto import mock_mediastore
+from botocore.exceptions import ClientError
+
+region = "eu-west-1"
+
+
+@mock_mediastore
+def test_create_container_succeeds():
+    client = boto3.client("mediastore", region_name=region)
+    response = client.create_container(
+        ContainerName="Awesome container!", Tags=[{"Key": "customer"}]
+    )
+    container = response["Container"]
+    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
+    container["ARN"].should.equal(
+        "arn:aws:mediastore:container:{}".format(container["Name"])
+    )
+    container["Name"].should.equal("Awesome container!")
+    container["Status"].should.equal("CREATING")
+
+
+@mock_mediastore
+def test_describe_container_succeeds():
+    client = boto3.client("mediastore", region_name=region)
+    create_response = client.create_container(
+        ContainerName="Awesome container!", Tags=[{"Key": "customer"}]
+    )
+    container_name = create_response["Container"]["Name"]
+    response = client.describe_container(ContainerName=container_name)
+    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
+    container = response["Container"]
+    container["ARN"].should.equal(
+        "arn:aws:mediastore:container:{}".format(container["Name"])
+    )
+    container["Name"].should.equal("Awesome container!")
+    container["Status"].should.equal("ACTIVE")
+
+
+@mock_mediastore
+def test_list_containers_succeeds():
+    client = boto3.client("mediastore", region_name=region)
+    client.create_container(
+        ContainerName="Awesome container!", Tags=[{"Key": "customer"}]
+    )
+    list_response = client.list_containers(NextToken="next-token", MaxResults=123)
+    containers_list = list_response["Containers"]
+    len(containers_list).should.equal(1)
+    client.create_container(
+        ContainerName="Awesome container2!", Tags=[{"Key": "customer"}]
+    )
+    list_response = client.list_containers(NextToken="next-token", MaxResults=123)
+    containers_list = list_response["Containers"]
+    len(containers_list).should.equal(2)
+
+
+@mock_mediastore
+def test_describe_container_raises_error_if_container_does_not_exist():
+    client = boto3.client("mediastore", region_name=region)
+    with pytest.raises(ClientError) as ex:
+        client.describe_container(ContainerName="container-name")
+    ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+
+
+@mock_mediastore
+def test_put_lifecycle_policy_succeeds():
+    client = boto3.client("mediastore", region_name=region)
+    container_response = client.create_container(
+        ContainerName="container-name", Tags=[{"Key": "customer"}]
+    )
+    container = container_response["Container"]
+    client.put_lifecycle_policy(
+        ContainerName=container["Name"], LifecyclePolicy="lifecycle-policy"
+    )
+    response = client.get_lifecycle_policy(ContainerName=container["Name"])
+    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
+    response["LifecyclePolicy"].should.equal("lifecycle-policy")
+
+
+@mock_mediastore
+def test_put_lifecycle_policy_raises_error_if_container_does_not_exist():
+    client = boto3.client("mediastore", region_name=region)
+    with pytest.raises(ClientError) as ex:
+        client.put_lifecycle_policy(
+            ContainerName="container-name", LifecyclePolicy="lifecycle-policy"
+        )
+    ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+
+
+@mock_mediastore
+def test_get_lifecycle_policy_raises_error_if_container_does_not_exist():
+    client = boto3.client("mediastore", region_name=region)
+    with pytest.raises(ClientError) as ex:
+        client.get_lifecycle_policy(ContainerName="container-name")
+    ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+
+
+@mock_mediastore
+def test_get_lifecycle_policy_raises_error_if_container_does_not_have_lifecycle_policy():
+    client = boto3.client("mediastore", region_name=region)
+    client.create_container(ContainerName="container-name", Tags=[{"Key": "customer"}])
+    with pytest.raises(ClientError) as ex:
+        client.get_lifecycle_policy(ContainerName="container-name")
+    ex.value.response["Error"]["Code"].should.equal("PolicyNotFoundException")
+
+
+@mock_mediastore
+def test_put_container_policy_succeeds():
+    client = boto3.client("mediastore", region_name=region)
+    container_response = client.create_container(
+        ContainerName="container-name", Tags=[{"Key": "customer"}]
+    )
+    container = container_response["Container"]
+    response = client.put_container_policy(
+        ContainerName=container["Name"], Policy="container-policy"
+    )
+    response = client.get_container_policy(ContainerName=container["Name"])
+    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
+    response["Policy"].should.equal("container-policy")
+
+
+@mock_mediastore
+def test_put_container_policy_raises_error_if_container_does_not_exist():
+    client = boto3.client("mediastore", region_name=region)
+    with pytest.raises(ClientError) as ex:
+        client.put_container_policy(
+            ContainerName="container-name", Policy="container-policy"
+        )
+    ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+
+
+@mock_mediastore
+def test_get_container_policy_raises_error_if_container_does_not_exist():
+    client = boto3.client("mediastore", region_name=region)
+    with pytest.raises(ClientError) as ex:
+        client.get_container_policy(ContainerName="container-name")
+    ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+
+
+@mock_mediastore
+def test_get_container_policy_raises_error_if_container_does_not_have_container_policy():
+    client = boto3.client("mediastore", region_name=region)
+    client.create_container(ContainerName="container-name", Tags=[{"Key": "customer"}])
+    with pytest.raises(ClientError) as ex:
+        client.get_container_policy(ContainerName="container-name")
+    ex.value.response["Error"]["Code"].should.equal("PolicyNotFoundException")
+
+
+@mock_mediastore
+def test_put_metric_policy_succeeds():
+    client = boto3.client("mediastore", region_name=region)
+    container_response = client.create_container(
+        ContainerName="container-name", Tags=[{"Key": "customer"}]
+    )
+    container = container_response["Container"]
+    response = client.put_metric_policy(
+        ContainerName=container["Name"],
+        MetricPolicy={"ContainerLevelMetrics": "ENABLED"},
+    )
+    response = client.get_metric_policy(ContainerName=container["Name"])
+    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
+    response["MetricPolicy"].should.equal({"ContainerLevelMetrics": "ENABLED"})
+
+
+@mock_mediastore
+def test_put_metric_policy_raises_error_if_container_does_not_exist():
+    client = boto3.client("mediastore", region_name=region)
+    with pytest.raises(ClientError) as ex:
+        client.put_metric_policy(
+            ContainerName="container-name",
+            MetricPolicy={"ContainerLevelMetrics": "ENABLED"},
+        )
+    ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+
+
+@mock_mediastore
+def test_get_metric_policy_raises_error_if_container_does_not_exist():
+    client = boto3.client("mediastore", region_name=region)
+    with pytest.raises(ClientError) as ex:
+        client.get_metric_policy(ContainerName="container-name")
+    ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+
+
+@mock_mediastore
+def test_get_metric_policy_raises_error_if_container_does_not_have_metric_policy():
+    client = boto3.client("mediastore", region_name=region)
+    client.create_container(ContainerName="container-name", Tags=[{"Key": "customer"}])
+    with pytest.raises(ClientError) as ex:
+        client.get_metric_policy(ContainerName="container-name")
+    ex.value.response["Error"]["Code"].should.equal("PolicyNotFoundException")

tests/test_mediastore/test_server.py

@@ -0,0 +1,23 @@
+from __future__ import unicode_literals
+
+import sure  # noqa
+
+import moto.server as server
+from moto import mock_mediastore
+
+"""
+Test the different server responses
+"""
+
+
+@mock_mediastore
+def test_mediastore_lists_containers():
+    backend = server.create_backend_app("mediastore")
+    test_client = backend.test_client()
+
+    res = test_client.get(
+        "/", headers={"X-Amz-Target": "MediaStore_20170901.ListContainers"},
+    )
+
+    result = res.data.decode("utf-8")
+    result.should.contain('"Containers": []')