Media store data Service (#3955)

* Add delete container and list tags endpoints to MediaStore

* Black reformat

* Fixed Lint problems

* Check if dictionary was deleted effectively

* lint fix

* MediaPackageClientError

* Lint Fix

* Test unknown channel describe

* Concatenation Fix

* MediaPackage - fix error message

* MediaPackage ClientError part2

* Mediastoredata not working

Base url

tests and renaming

typo

List Items not returning proper JSON and wrongly hitting get_object response

MediaStore2

Tests

* More implementation

* Fix tests and format

* Comments fix

* Comments 2

* MediastoreData - alternative logic to figure out appropriate host

Co-authored-by: av <arcovoltaico@gmail.com>
Co-authored-by: Bert Blommers <info@bertblommers.nl>

moto/__init__.py

@@ -126,6 +126,7 @@ mock_mediaconnect = lazy_load(".mediaconnect", "mock_mediaconnect")
 mock_mediapackage = lazy_load(".mediapackage", "mock_mediapackage")
 mock_mediastore = lazy_load(".mediastore", "mock_mediastore")
 mock_eks = lazy_load(".eks", "mock_eks")
+mock_mediastoredata = lazy_load(".mediastoredata", "mock_mediastoredata")
 
 # import logging
 # logging.getLogger('boto').setLevel(logging.CRITICAL)

moto/backends.py

@@ -82,6 +82,7 @@ BACKENDS = {
     "mediaconnect": ("mediaconnect", "mediaconnect_backends"),
     "mediapackage": ("mediapackage", "mediapackage_backends"),
     "mediastore": ("mediastore", "mediastore_backends"),
+    "mediastore-data": ("mediastoredata", "mediastoredata_backends"),
     "eks": ("eks", "eks_backends"),
 }
 

moto/mediastore/models.py

@@ -83,6 +83,8 @@ class MediaStoreBackend(BaseBackend):
         return response_containers, None
 
     def list_tags_for_resource(self, name):
+        if name not in self._containers:
+            raise ContainerNotFoundException()
         tags = self._containers[name].tags
         return tags
 

moto/mediastore/urls.py

@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+
 from .responses import MediaStoreResponse
 
 url_bases = [
@@ -7,6 +8,4 @@ url_bases = [
 
 response = MediaStoreResponse()
 
-url_paths = {
-    "{0}/$": response.dispatch,
-}
+url_paths = {"{0}/$": response.dispatch, "{0}/(?P<Path>[^/.]+)$": response.dispatch}

moto/mediastoredata/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import unicode_literals
+from .models import mediastoredata_backends
+from ..core.models import base_decorator
+
+mediastoredata_backend = mediastoredata_backends["us-east-1"]
+mock_mediastoredata = base_decorator(mediastoredata_backends)

moto/mediastoredata/exceptions.py

@@ -0,0 +1,13 @@
+from __future__ import unicode_literals
+
+from moto.core.exceptions import JsonRESTError
+
+
+class MediaStoreDataClientError(JsonRESTError):
+    code = 400
+
+
+# AWS service exceptions are caught with the underlying botocore exception, ClientError
+class ClientError(MediaStoreDataClientError):
+    def __init__(self, error, message):
+        super(ClientError, self).__init__(error, message)

moto/mediastoredata/models.py

@@ -0,0 +1,90 @@
+from __future__ import unicode_literals
+
+import hashlib
+from collections import OrderedDict
+
+from boto3 import Session
+
+from moto.core import BaseBackend, BaseModel
+from .exceptions import ClientError
+
+
+class Object(BaseModel):
+    def __init__(self, path, body, etag, storage_class="TEMPORAL"):
+        self.path = path
+        self.body = body
+        self.content_sha256 = hashlib.sha256(body.encode("utf-8")).hexdigest()
+        self.etag = etag
+        self.storage_class = storage_class
+
+    def to_dict(self):
+        data = {
+            "ETag": self.etag,
+            "Name": self.path,
+            "Type": "FILE",
+            "ContentLength": 123,
+            "StorageClass": self.storage_class,
+            "Path": self.path,
+            "ContentSHA256": self.content_sha256,
+        }
+
+        return data
+
+
+class MediaStoreDataBackend(BaseBackend):
+    def __init__(self, region_name=None):
+        super(MediaStoreDataBackend, self).__init__()
+        self.region_name = region_name
+        self._objects = OrderedDict()
+
+    def reset(self):
+        region_name = self.region_name
+        self.__dict__ = {}
+        self.__init__(region_name)
+
+    def put_object(
+        self,
+        body,
+        path,
+        content_type=None,
+        cache_control=None,
+        storage_class="TEMPORAL",
+        upload_availability="STANDARD",
+    ):
+        new_object = Object(
+            path=path, body=body, etag="etag", storage_class=storage_class
+        )
+        self._objects[path] = new_object
+        return new_object
+
+    def delete_object(self, path):
+        if path not in self._objects:
+            error = "ObjectNotFoundException"
+            raise ClientError(error, "Object with id={} not found".format(path))
+        del self._objects[path]
+        return {}
+
+    def get_object(self, path, range=None):
+        objects_found = [item for item in self._objects.values() if item.path == path]
+        if len(objects_found) == 0:
+            error = "ObjectNotFoundException"
+            raise ClientError(error, "Object with id={} not found".format(path))
+        return objects_found[0]
+
+    def list_items(self, path, max_results=1000, next_token=None):
+        items = self._objects.values()
+        response_items = [c.to_dict() for c in items]
+        return response_items
+
+
+mediastoredata_backends = {}
+for region in Session().get_available_regions("mediastore-data"):
+    mediastoredata_backends[region] = MediaStoreDataBackend(region)
+for region in Session().get_available_regions(
+    "mediastore-data", partition_name="aws-us-gov"
+):
+    mediastoredata_backends[region] = MediaStoreDataBackend(region)
+for region in Session().get_available_regions(
+    "mediastore-data", partition_name="aws-cn"
+):
+    mediastoredata_backends[region] = MediaStoreDataBackend(region)

moto/mediastoredata/responses.py

@@ -0,0 +1,43 @@
+from __future__ import unicode_literals
+
+import json
+
+from moto.core.responses import BaseResponse
+from .models import mediastoredata_backends
+
+
+class MediaStoreDataResponse(BaseResponse):
+    SERVICE_NAME = "mediastore-data"
+
+    @property
+    def mediastoredata_backend(self):
+        return mediastoredata_backends[self.region]
+
+    def get_object(self):
+        path = self._get_param("Path")
+        range = self._get_param("Range")
+        result = self.mediastoredata_backend.get_object(path=path, range=range)
+        headers = {"Path": result.path}
+        return result.body, headers
+
+    def put_object(self):
+        body = self.body
+        path = self._get_param("Path")
+        new_object = self.mediastoredata_backend.put_object(body, path)
+        object_dict = new_object.to_dict()
+        return json.dumps(object_dict)
+
+    def delete_object(self):
+        item_id = self._get_param("Path")
+        result = self.mediastoredata_backend.delete_object(path=item_id)
+        return json.dumps(result)
+
+    def list_items(self):
+        path = self._get_param("Path")
+        max_results = self._get_param("MaxResults")
+        next_token = self._get_param("NextToken")
+        items = self.mediastoredata_backend.list_items(
+            path=path, max_results=max_results, next_token=next_token
+        )
+        response_items = json.dumps(dict(Items=items))
+        return response_items

moto/mediastoredata/urls.py

@@ -0,0 +1,11 @@
+from __future__ import unicode_literals
+
+from .responses import MediaStoreDataResponse
+
+url_bases = [
+    "https?://data.mediastore.(.+).amazonaws.com",
+]
+
+response = MediaStoreDataResponse()
+
+url_paths = {"{0}/$": response.dispatch, "{0}/(?P<Path>[^/.]+)$": response.dispatch}

moto/server.py

@@ -74,6 +74,7 @@ class DomainDispatcherApplication(object):
 
     def infer_service_region_host(self, environ):
         auth = environ.get("HTTP_AUTHORIZATION")
+        target = environ.get("HTTP_X_AMZ_TARGET")
         if auth:
             # Signed request
             # Parse auth header to find service assuming a SigV4 request
@@ -91,7 +92,6 @@ class DomainDispatcherApplication(object):
                 service, region = DEFAULT_SERVICE_REGION
         else:
             # Unsigned request
-            target = environ.get("HTTP_X_AMZ_TARGET")
             action = self.get_action_from_body(environ)
             if target:
                 service, _ = target.split(".", 1)
@@ -103,7 +103,13 @@ class DomainDispatcherApplication(object):
                 # S3 is the last resort when the target is also unknown
                 service, region = DEFAULT_SERVICE_REGION
 
-        if service == "dynamodb":
+        if service == "mediastore" and not target:
+            # All MediaStore API calls have a target header
+            # If no target is set, assume we're trying to reach the mediastore-data service
+            host = "data.{service}.{region}.amazonaws.com".format(
+                service=service, region=region
+            )
+        elif service == "dynamodb":
             if environ["HTTP_X_AMZ_TARGET"].startswith("DynamoDBStreams"):
                 host = "dynamodbstreams"
             else:

tests/test_mediapackage/test_mediapackage.py

@@ -177,6 +177,18 @@ def test_describe_origin_endpoint_succeeds():
     )
 
 
+def test_describe_unknown_origin_endpoint_throws_error():
+    client = boto3.client("mediapackage", region_name=region)
+    channel_id = "unknown-channel"
+    with pytest.raises(ClientError) as err:
+        client.describe_origin_endpoint(Id=channel_id)
+    err = err.value.response["Error"]
+    err["Code"].should.equal("NotFoundException")
+    err["Message"].should.equal(
+        "originEndpoint with id={} not found".format(str(channel_id))
+    )
+
+
 @mock_mediapackage
 def test_describe_unknown_origin_endpoint_throws_error():
     client = boto3.client("mediapackage", region_name=region)
@@ -207,6 +219,18 @@ def test_delete_origin_endpoint_succeeds():
     )
 
 
+def test_delete_unknown_origin_endpoint_throws_error():
+    client = boto3.client("mediapackage", region_name=region)
+    channel_id = "unknown-channel"
+    with pytest.raises(ClientError) as err:
+        client.delete_origin_endpoint(Id=channel_id)
+    err = err.value.response["Error"]
+    err["Code"].should.equal("NotFoundException")
+    err["Message"].should.equal(
+        "originEndpoint with id={} not found".format(str(channel_id))
+    )
+
+
 @mock_mediapackage
 def test_delete_unknown_origin_endpoint_throws_error():
     client = boto3.client("mediapackage", region_name=region)
@@ -234,6 +258,22 @@ def test_update_origin_endpoint_succeeds():
     update_response["ManifestName"].should.equal("updated-manifest-name")
 
 
+def test_update_unknown_origin_endpoint_throws_error():
+    client = boto3.client("mediapackage", region_name=region)
+    channel_id = "unknown-channel"
+    with pytest.raises(ClientError) as err:
+        client.update_origin_endpoint(
+            Id=channel_id,
+            Description="updated-channel-description",
+            ManifestName="updated-manifest-name",
+        )
+    err = err.value.response["Error"]
+    err["Code"].should.equal("NotFoundException")
+    err["Message"].should.equal(
+        "originEndpoint with id={} not found".format(str(channel_id))
+    )
+
+
 @mock_mediapackage
 def test_update_unknown_origin_endpoint_throws_error():
     client = boto3.client("mediapackage", region_name=region)

tests/test_mediastore/test_mediastore.py

@@ -220,6 +220,14 @@ def test_list_tags_for_resource_return_none_if_no_tags():
     response.get("Tags").should.equal(None)
 
 
+@mock_mediastore
+def test_list_tags_for_resource_return_none_if_no_tags():
+    client = boto3.client("mediastore", region_name=region)
+    with pytest.raises(ClientError) as ex:
+        client.list_tags_for_resource(Resource="not_existing")
+    ex.value.response["Error"]["Code"].should.equal("ContainerNotFoundException")
+
+
 @mock_mediastore
 def test_delete_container():
     client = boto3.client("mediastore", region_name=region)

tests/test_mediastoredata/test_mediastoredata.py

@@ -0,0 +1,85 @@
+from __future__ import unicode_literals
+
+import boto3
+import pytest
+import sure  # noqa
+from botocore.exceptions import ClientError
+
+from moto import mock_mediastoredata
+
+region = "eu-west-1"
+
+
+@mock_mediastoredata
+def test_put_object():
+    client = boto3.client("mediastore-data", region_name=region)
+    response = client.put_object(Body="011001", Path="foo")
+    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
+    response["StorageClass"].should.equal("TEMPORAL")
+    items = client.list_items()["Items"]
+    object_exists = any(d["Name"] == "foo" for d in items)
+    object_exists.should.equal(True)
+
+
+@mock_mediastoredata
+def test_get_object_throws_not_found_error():
+    client = boto3.client("mediastore-data", region_name=region)
+    with pytest.raises(ClientError) as ex:
+        client.get_object(Path="foo")
+    ex.value.response["Error"]["Code"].should.equal("ObjectNotFoundException")
+
+
+@mock_mediastoredata
+def test_get_object():
+    client = boto3.client("mediastore-data", region_name=region)
+    client.put_object(Body="011001", Path="foo")
+    response = client.get_object(Path="foo")
+    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
+    response["ResponseMetadata"]["HTTPHeaders"]["path"].should.equal("foo")
+    data = response["Body"].read()
+    data.should.equal(b"011001")
+
+
+@mock_mediastoredata
+def test_delete_object_error():
+    client = boto3.client("mediastore-data", region_name=region)
+    with pytest.raises(ClientError) as ex:
+        client.delete_object(Path="foo")
+    ex.value.response["Error"]["Code"].should.equal("ObjectNotFoundException")
+
+
+@mock_mediastoredata
+def test_delete_object_succeeds():
+    client = boto3.client("mediastore-data", region_name=region)
+    object_path = "foo"
+    client.put_object(Body="011001", Path=object_path)
+    response = client.delete_object(Path=object_path)
+    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
+    items = client.list_items()["Items"]
+    len(items).should.equal(0)
+
+
+@mock_mediastoredata
+def test_list_items():
+    client = boto3.client("mediastore-data", region_name=region)
+    items = client.list_items()["Items"]
+    len(items).should.equal(0)
+    object_path = "foo"
+    client.put_object(Body="011001", Path=object_path)
+    items = client.list_items()["Items"]
+    len(items).should.equal(1)
+    object_exists = any(d["Name"] == object_path for d in items)
+    object_exists.should.equal(True)
+
+
+@mock_mediastoredata
+def test_list_items():
+    client = boto3.client("mediastore-data", region_name=region)
+    items = client.list_items()["Items"]
+    len(items).should.equal(0)
+    object_path = "foo"
+    client.put_object(Body="011001", Path=object_path)
+    items = client.list_items()["Items"]
+    len(items).should.equal(1)
+    object_exists = any(d["Name"] == object_path for d in items)
+    object_exists.should.equal(True)

tests/test_mediastoredata/test_server.py

@@ -0,0 +1,18 @@
+from __future__ import unicode_literals
+
+import sure  # noqa
+
+import moto.server as server
+from moto import mock_mediastoredata
+
+"""
+Test the different server responses
+"""
+
+
+@mock_mediastoredata
+def test_mediastore_lists_containers():
+    backend = server.create_backend_app("mediastore-data")
+    test_client = backend.test_client()
+    response = test_client.get("/").data
+    response.should.contain(b'"Items": []')