Merge pull request #359 from spulec/glacier

Glacier

moto/__init__.py

@@ -13,6 +13,7 @@ from .dynamodb2 import mock_dynamodb2  # flake8: noqa
 from .ec2 import mock_ec2  # flake8: noqa
 from .elb import mock_elb  # flake8: noqa
 from .emr import mock_emr  # flake8: noqa
+from .glacier import mock_glacier  # flake8: noqa
 from .iam import mock_iam  # flake8: noqa
 from .kinesis import mock_kinesis  # flake8: noqa
 from .rds import mock_rds  # flake8: noqa

moto/backends.py

@@ -6,6 +6,7 @@ from moto.dynamodb2 import dynamodb_backend2
 from moto.ec2 import ec2_backend
 from moto.elb import elb_backend
 from moto.emr import emr_backend
+from moto.glacier import glacier_backend
 from moto.kinesis import kinesis_backend
 from moto.rds import rds_backend
 from moto.redshift import redshift_backend
@@ -24,6 +25,7 @@ BACKENDS = {
     'ec2': ec2_backend,
     'elb': elb_backend,
     'emr': emr_backend,
+    'glacier': glacier_backend,
     'kinesis': kinesis_backend,
     'redshift': redshift_backend,
     'rds': rds_backend,

moto/glacier/__init__.py

@@ -0,0 +1,12 @@
+from __future__ import unicode_literals
+from .models import glacier_backends
+from ..core.models import MockAWS
+
+glacier_backend = glacier_backends['us-east-1']
+
+
+def mock_glacier(func=None):
+    if func:
+        return MockAWS(glacier_backends)(func)
+    else:
+        return MockAWS(glacier_backends)

moto/glacier/models.py

@@ -0,0 +1,123 @@
+from __future__ import unicode_literals
+
+import hashlib
+
+import boto.glacier
+from moto.core import BaseBackend
+
+from .utils import get_job_id
+
+
+class ArchiveJob(object):
+
+    def __init__(self, job_id, archive_id):
+        self.job_id = job_id
+        self.archive_id = archive_id
+
+    def to_dict(self):
+        return {
+            "Action": "InventoryRetrieval",
+            "ArchiveId": self.archive_id,
+            "ArchiveSizeInBytes": 0,
+            "ArchiveSHA256TreeHash": None,
+            "Completed": True,
+            "CompletionDate": "2013-03-20T17:03:43.221Z",
+            "CreationDate": "2013-03-20T17:03:43.221Z",
+            "InventorySizeInBytes": "0",
+            "JobDescription": None,
+            "JobId": self.job_id,
+            "RetrievalByteRange": None,
+            "SHA256TreeHash": None,
+            "SNSTopic": None,
+            "StatusCode": "Succeeded",
+            "StatusMessage": None,
+            "VaultARN": None,
+        }
+
+
+class Vault(object):
+    def __init__(self, vault_name, region):
+        self.vault_name = vault_name
+        self.region = region
+        self.archives = {}
+        self.jobs = {}
+
+    @property
+    def arn(self):
+        return "arn:aws:glacier:{0}:012345678901:vaults/{1}".format(self.region, self.vault_name)
+
+    def to_dict(self):
+        return {
+            "CreationDate": "2013-03-20T17:03:43.221Z",
+            "LastInventoryDate": "2013-03-20T17:03:43.221Z",
+            "NumberOfArchives": None,
+            "SizeInBytes": None,
+            "VaultARN": self.arn,
+            "VaultName": self.vault_name,
+        }
+
+    def create_archive(self, body):
+        archive_id = hashlib.sha256(body).hexdigest()
+        self.archives[archive_id] = body
+        return archive_id
+
+    def get_archive_body(self, archive_id):
+        return self.archives[archive_id]
+
+    def delete_archive(self, archive_id):
+        return self.archives.pop(archive_id)
+
+    def initiate_job(self, archive_id):
+        job_id = get_job_id()
+        job = ArchiveJob(job_id, archive_id)
+        self.jobs[job_id] = job
+        return job_id
+
+    def list_jobs(self):
+        return self.jobs.values()
+
+    def describe_job(self, job_id):
+        return self.jobs.get(job_id)
+
+    def get_job_output(self, job_id):
+        job = self.describe_job(job_id)
+        archive_body = self.get_archive_body(job.archive_id)
+        return archive_body
+
+
+class GlacierBackend(BaseBackend):
+
+    def __init__(self, region_name):
+        self.vaults = {}
+        self.region_name = region_name
+
+    def reset(self):
+        region_name = self.region_name
+        self.__dict__ = {}
+        self.__init__(region_name)
+
+    def get_vault(self, vault_name):
+        return self.vaults[vault_name]
+
+    def create_vault(self, vault_name):
+        self.vaults[vault_name] = Vault(vault_name, self.region_name)
+
+    def list_vaules(self):
+        return self.vaults.values()
+
+    def delete_vault(self, vault_name):
+        self.vaults.pop(vault_name)
+
+    def initiate_job(self, vault_name, archive_id):
+        vault = self.get_vault(vault_name)
+        job_id = vault.initiate_job(archive_id)
+        return job_id
+
+    def list_jobs(self, vault_name):
+        vault = self.get_vault(vault_name)
+        return vault.list_jobs()
+
+
+glacier_backends = {}
+for region in boto.glacier.regions():
+    glacier_backends[region.name] = GlacierBackend(region)

moto/glacier/responses.py

@@ -0,0 +1,160 @@
+from __future__ import unicode_literals
+
+import json
+from six.moves.urllib.parse import urlparse, parse_qs
+
+from moto.core.responses import _TemplateEnvironmentMixin
+from .models import glacier_backends
+from .utils import region_from_glacier_url, vault_from_glacier_url
+
+
+class GlacierResponse(_TemplateEnvironmentMixin):
+
+    def __init__(self, backend):
+        self.backend = backend
+
+    @classmethod
+    def all_vault_response(clazz, request, full_url, headers):
+        region_name = region_from_glacier_url(full_url)
+        response_instance = GlacierResponse(glacier_backends[region_name])
+        return response_instance._all_vault_response(request, full_url, headers)
+
+    def _all_vault_response(self, request, full_url, headers):
+        vaults = self.backend.list_vaules()
+        response = json.dumps({
+            "Marker": None,
+            "VaultList": [
+                vault.to_dict() for vault in vaults
+            ]
+        })
+
+        headers['content-type'] = 'application/json'
+        return 200, headers, response
+
+    @classmethod
+    def vault_response(clazz, request, full_url, headers):
+        region_name = region_from_glacier_url(full_url)
+        response_instance = GlacierResponse(glacier_backends[region_name])
+        return response_instance._vault_response(request, full_url, headers)
+
+    def _vault_response(self, request, full_url, headers):
+        method = request.method
+        parsed_url = urlparse(full_url)
+        querystring = parse_qs(parsed_url.query, keep_blank_values=True)
+        vault_name = vault_from_glacier_url(full_url)
+
+        if method == 'GET':
+            return self._vault_response_get(vault_name, querystring, headers)
+        elif method == 'PUT':
+            return self._vault_response_put(vault_name, querystring, headers)
+        elif method == 'DELETE':
+            return self._vault_response_delete(vault_name, querystring, headers)
+
+    def _vault_response_get(self, vault_name, querystring, headers):
+        vault = self.backend.get_vault(vault_name)
+        headers['content-type'] = 'application/json'
+        return 200, headers, json.dumps(vault.to_dict())
+
+    def _vault_response_put(self, vault_name, querystring, headers):
+        self.backend.create_vault(vault_name)
+        return 201, headers, ""
+
+    def _vault_response_delete(self, vault_name, querystring, headers):
+        self.backend.delete_vault(vault_name)
+        return 204, headers, ""
+
+    @classmethod
+    def vault_archive_response(clazz, request, full_url, headers):
+        region_name = region_from_glacier_url(full_url)
+        response_instance = GlacierResponse(glacier_backends[region_name])
+        return response_instance._vault_archive_response(request, full_url, headers)
+
+    def _vault_archive_response(self, request, full_url, headers):
+        method = request.method
+        body = request.body
+        parsed_url = urlparse(full_url)
+        querystring = parse_qs(parsed_url.query, keep_blank_values=True)
+        vault_name = full_url.split("/")[-2]
+
+        if method == 'POST':
+            return self._vault_archive_response_post(vault_name, body, querystring, headers)
+
+    def _vault_archive_response_post(self, vault_name, body, querystring, headers):
+        vault = self.backend.get_vault(vault_name)
+        vault_id = vault.create_archive(body)
+        headers['x-amz-archive-id'] = vault_id
+        return 201, headers, ""
+
+    @classmethod
+    def vault_archive_individual_response(clazz, request, full_url, headers):
+        region_name = region_from_glacier_url(full_url)
+        response_instance = GlacierResponse(glacier_backends[region_name])
+        return response_instance._vault_archive_individual_response(request, full_url, headers)
+
+    def _vault_archive_individual_response(self, request, full_url, headers):
+        method = request.method
+        vault_name = full_url.split("/")[-3]
+        archive_id = full_url.split("/")[-1]
+
+        if method == 'DELETE':
+            vault = self.backend.get_vault(vault_name)
+            vault.delete_archive(archive_id)
+            return 204, headers, ""
+
+    @classmethod
+    def vault_jobs_response(clazz, request, full_url, headers):
+        region_name = region_from_glacier_url(full_url)
+        response_instance = GlacierResponse(glacier_backends[region_name])
+        return response_instance._vault_jobs_response(request, full_url, headers)
+
+    def _vault_jobs_response(self, request, full_url, headers):
+        method = request.method
+        body = request.body
+        account_id = full_url.split("/")[1]
+        vault_name = full_url.split("/")[-2]
+
+        if method == 'GET':
+            jobs = self.backend.list_jobs(vault_name)
+            headers['content-type'] = 'application/json'
+            return 200, headers, json.dumps({
+                "JobList": [
+                    job.to_dict() for job in jobs
+                ],
+                "Marker": None,
+            })
+        elif method == 'POST':
+            json_body = json.loads(body.decode("utf-8"))
+            archive_id = json_body['ArchiveId']
+            job_id = self.backend.initiate_job(vault_name, archive_id)
+            headers['x-amz-job-id'] = job_id
+            headers['Location'] = "/{0}/vaults/{1}/jobs/{2}".format(account_id, vault_name, job_id)
+            return 202, headers, ""
+
+    @classmethod
+    def vault_jobs_individual_response(clazz, request, full_url, headers):
+        region_name = region_from_glacier_url(full_url)
+        response_instance = GlacierResponse(glacier_backends[region_name])
+        return response_instance._vault_jobs_individual_response(request, full_url, headers)
+
+    def _vault_jobs_individual_response(self, request, full_url, headers):
+        vault_name = full_url.split("/")[-3]
+        archive_id = full_url.split("/")[-1]
+
+        vault = self.backend.get_vault(vault_name)
+        job = vault.describe_job(archive_id)
+        return 200, headers, json.dumps(job.to_dict())
+
+    @classmethod
+    def vault_jobs_output_response(clazz, request, full_url, headers):
+        region_name = region_from_glacier_url(full_url)
+        response_instance = GlacierResponse(glacier_backends[region_name])
+        return response_instance._vault_jobs_output_response(request, full_url, headers)
+
+    def _vault_jobs_output_response(self, request, full_url, headers):
+        vault_name = full_url.split("/")[-4]
+        job_id = full_url.split("/")[-2]
+
+        vault = self.backend.get_vault(vault_name)
+        output = vault.get_job_output(job_id)
+        headers['content-type'] = 'application/octet-stream'
+        return 200, headers, output

moto/glacier/urls.py

@@ -0,0 +1,16 @@
+from __future__ import unicode_literals
+from .responses import GlacierResponse
+
+url_bases = [
+    "https?://glacier.(.+).amazonaws.com",
+]
+
+url_paths = {
+    '{0}/(?P<account_number>.+)/vaults$': GlacierResponse.all_vault_response,
+    '{0}/(?P<account_number>.+)/vaults/(?P<vault_name>[^/.]+)$': GlacierResponse.vault_response,
+    '{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/archives$': GlacierResponse.vault_archive_response,
+    '{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/archives/(?P<archive_id>.+)$': GlacierResponse.vault_archive_individual_response,
+    '{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs$': GlacierResponse.vault_jobs_response,
+    '{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs/(?P<job_id>[^/.]+)$': GlacierResponse.vault_jobs_individual_response,
+    '{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs/(?P<job_id>.+)/output$': GlacierResponse.vault_jobs_output_response,
+}

moto/glacier/utils.py

@@ -0,0 +1,21 @@
+import random
+import string
+
+from six.moves.urllib.parse import urlparse
+
+
+def region_from_glacier_url(url):
+    domain = urlparse(url).netloc
+
+    if '.' in domain:
+        return domain.split(".")[1]
+    else:
+        return 'us-east-1'
+
+
+def vault_from_glacier_url(full_url):
+    return full_url.split("/")[-1]
+
+
+def get_job_id():
+    return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(92))

tests/test_glacier/test_glacier_archives.py

@@ -0,0 +1,21 @@
+from __future__ import unicode_literals
+
+from tempfile import NamedTemporaryFile
+import boto.glacier
+import sure  # noqa
+
+from moto import mock_glacier
+
+
+@mock_glacier
+def test_create_and_delete_archive():
+    the_file = NamedTemporaryFile(delete=False)
+    the_file.write(b"some stuff")
+    the_file.close()
+
+    conn = boto.glacier.connect_to_region("us-west-2")
+    vault = conn.create_vault("my_vault")
+
+    archive_id = vault.upload_archive(the_file.name)
+
+    vault.delete_archive(archive_id)

tests/test_glacier/test_glacier_jobs.py

@@ -0,0 +1,94 @@
+from __future__ import unicode_literals
+
+import json
+
+from boto.glacier.layer1 import Layer1
+import sure  # noqa
+
+from moto import mock_glacier
+
+
+@mock_glacier
+def test_init_glacier_job():
+    conn = Layer1(region_name="us-west-2")
+    vault_name = "my_vault"
+    conn.create_vault(vault_name)
+    archive_id = conn.upload_archive(vault_name, "some stuff", "", "", "some description")
+
+    job_response = conn.initiate_job(vault_name, {
+        "ArchiveId": archive_id,
+        "Type": "archive-retrieval",
+    })
+    job_id = job_response['JobId']
+    job_response['Location'].should.equal("//vaults/my_vault/jobs/{0}".format(job_id))
+
+
+@mock_glacier
+def test_describe_job():
+    conn = Layer1(region_name="us-west-2")
+    vault_name = "my_vault"
+    conn.create_vault(vault_name)
+    archive_id = conn.upload_archive(vault_name, "some stuff", "", "", "some description")
+    job_response = conn.initiate_job(vault_name, {
+        "ArchiveId": archive_id,
+        "Type": "archive-retrieval",
+    })
+    job_id = job_response['JobId']
+
+    job = conn.describe_job(vault_name, job_id)
+    json.loads(job.read().decode("utf-8")).should.equal({
+        'CompletionDate': '2013-03-20T17:03:43.221Z',
+        'VaultARN': None,
+        'RetrievalByteRange': None,
+        'SHA256TreeHash': None,
+        'Completed': True,
+        'InventorySizeInBytes': '0',
+        'JobId': job_id,
+        'Action': 'InventoryRetrieval',
+        'JobDescription': None,
+        'SNSTopic': None,
+        'ArchiveSizeInBytes': 0,
+        'ArchiveId': archive_id,
+        'ArchiveSHA256TreeHash': None,
+        'CreationDate': '2013-03-20T17:03:43.221Z',
+        'StatusMessage': None,
+        'StatusCode': 'Succeeded',
+    })
+
+
+@mock_glacier
+def test_list_glacier_jobs():
+    conn = Layer1(region_name="us-west-2")
+    vault_name = "my_vault"
+    conn.create_vault(vault_name)
+    archive_id1 = conn.upload_archive(vault_name, "some stuff", "", "", "some description")['ArchiveId']
+    archive_id2 = conn.upload_archive(vault_name, "some other stuff", "", "", "some description")['ArchiveId']
+
+    conn.initiate_job(vault_name, {
+        "ArchiveId": archive_id1,
+        "Type": "archive-retrieval",
+    })
+    conn.initiate_job(vault_name, {
+        "ArchiveId": archive_id2,
+        "Type": "archive-retrieval",
+    })
+
+    jobs = conn.list_jobs(vault_name)
+    len(jobs['JobList']).should.equal(2)
+
+
+@mock_glacier
+def test_get_job_output():
+    conn = Layer1(region_name="us-west-2")
+    vault_name = "my_vault"
+    conn.create_vault(vault_name)
+    archive_response = conn.upload_archive(vault_name, "some stuff", "", "", "some description")
+    archive_id = archive_response['ArchiveId']
+    job_response = conn.initiate_job(vault_name, {
+        "ArchiveId": archive_id,
+        "Type": "archive-retrieval",
+    })
+    job_id = job_response['JobId']
+
+    output = conn.get_job_output(vault_name, job_id)
+    output.read().decode("utf-8").should.equal("some stuff")

tests/test_glacier/test_glacier_server.py

@@ -0,0 +1,21 @@
+from __future__ import unicode_literals
+
+import json
+import sure  # noqa
+
+import moto.server as server
+from moto import mock_glacier
+
+'''
+Test the different server responses
+'''
+
+
+@mock_glacier
+def test_list_vaults():
+    backend = server.create_backend_app("glacier")
+    test_client = backend.test_client()
+
+    res = test_client.get('/1234bcd/vaults')
+
+    json.loads(res.data.decode("utf-8")).should.equal({u'Marker': None, u'VaultList': []})

tests/test_glacier/test_glacier_vaults.py

@@ -0,0 +1,31 @@
+from __future__ import unicode_literals
+
+import boto.glacier
+import sure  # noqa
+
+from moto import mock_glacier
+
+
+@mock_glacier
+def test_create_vault():
+    conn = boto.glacier.connect_to_region("us-west-2")
+
+    conn.create_vault("my_vault")
+
+    vaults = conn.list_vaults()
+    vaults.should.have.length_of(1)
+    vaults[0].name.should.equal("my_vault")
+
+
+@mock_glacier
+def test_delete_vault():
+    conn = boto.glacier.connect_to_region("us-west-2")
+
+    conn.create_vault("my_vault")
+
+    vaults = conn.list_vaults()
+    vaults.should.have.length_of(1)
+
+    conn.delete_vault("my_vault")
+    vaults = conn.list_vaults()
+    vaults.should.have.length_of(0)