diff --git a/moto/__init__.py b/moto/__init__.py index 7e13cf6bf..851a84a0d 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -13,6 +13,7 @@ from .dynamodb2 import mock_dynamodb2 # flake8: noqa from .ec2 import mock_ec2 # flake8: noqa from .elb import mock_elb # flake8: noqa from .emr import mock_emr # flake8: noqa +from .glacier import mock_glacier # flake8: noqa from .iam import mock_iam # flake8: noqa from .kinesis import mock_kinesis # flake8: noqa from .rds import mock_rds # flake8: noqa diff --git a/moto/backends.py b/moto/backends.py index 460ac028f..aad7d403c 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -6,6 +6,7 @@ from moto.dynamodb2 import dynamodb_backend2 from moto.ec2 import ec2_backend from moto.elb import elb_backend from moto.emr import emr_backend +from moto.glacier import glacier_backend from moto.kinesis import kinesis_backend from moto.rds import rds_backend from moto.redshift import redshift_backend @@ -24,6 +25,7 @@ BACKENDS = { 'ec2': ec2_backend, 'elb': elb_backend, 'emr': emr_backend, + 'glacier': glacier_backend, 'kinesis': kinesis_backend, 'redshift': redshift_backend, 'rds': rds_backend, diff --git a/moto/glacier/__init__.py b/moto/glacier/__init__.py new file mode 100644 index 000000000..bc6fd1ff4 --- /dev/null +++ b/moto/glacier/__init__.py @@ -0,0 +1,12 @@ +from __future__ import unicode_literals +from .models import glacier_backends +from ..core.models import MockAWS + +glacier_backend = glacier_backends['us-east-1'] + + +def mock_glacier(func=None): + if func: + return MockAWS(glacier_backends)(func) + else: + return MockAWS(glacier_backends) diff --git a/moto/glacier/models.py b/moto/glacier/models.py new file mode 100644 index 000000000..836e84d37 --- /dev/null +++ b/moto/glacier/models.py @@ -0,0 +1,123 @@ +from __future__ import unicode_literals + +import hashlib + +import boto.glacier +from moto.core import BaseBackend + +from .utils import get_job_id + + +class ArchiveJob(object): + + def __init__(self, job_id, archive_id): + self.job_id = job_id + self.archive_id = archive_id + + def to_dict(self): + return { + "Action": "InventoryRetrieval", + "ArchiveId": self.archive_id, + "ArchiveSizeInBytes": 0, + "ArchiveSHA256TreeHash": None, + "Completed": True, + "CompletionDate": "2013-03-20T17:03:43.221Z", + "CreationDate": "2013-03-20T17:03:43.221Z", + "InventorySizeInBytes": "0", + "JobDescription": None, + "JobId": self.job_id, + "RetrievalByteRange": None, + "SHA256TreeHash": None, + "SNSTopic": None, + "StatusCode": "Succeeded", + "StatusMessage": None, + "VaultARN": None, + } + + +class Vault(object): + def __init__(self, vault_name, region): + self.vault_name = vault_name + self.region = region + self.archives = {} + self.jobs = {} + + @property + def arn(self): + return "arn:aws:glacier:{0}:012345678901:vaults/{1}".format(self.region, self.vault_name) + + def to_dict(self): + return { + "CreationDate": "2013-03-20T17:03:43.221Z", + "LastInventoryDate": "2013-03-20T17:03:43.221Z", + "NumberOfArchives": None, + "SizeInBytes": None, + "VaultARN": self.arn, + "VaultName": self.vault_name, + } + + def create_archive(self, body): + archive_id = hashlib.sha256(body).hexdigest() + self.archives[archive_id] = body + return archive_id + + def get_archive_body(self, archive_id): + return self.archives[archive_id] + + def delete_archive(self, archive_id): + return self.archives.pop(archive_id) + + def initiate_job(self, archive_id): + job_id = get_job_id() + job = ArchiveJob(job_id, archive_id) + self.jobs[job_id] = job + return job_id + + def list_jobs(self): + return self.jobs.values() + + def describe_job(self, job_id): + return self.jobs.get(job_id) + + def get_job_output(self, job_id): + job = self.describe_job(job_id) + archive_body = self.get_archive_body(job.archive_id) + return archive_body + + +class GlacierBackend(BaseBackend): + + def __init__(self, region_name): + self.vaults = {} + self.region_name = region_name + + def reset(self): + region_name = self.region_name + self.__dict__ = {} + self.__init__(region_name) + + def get_vault(self, vault_name): + return self.vaults[vault_name] + + def create_vault(self, vault_name): + self.vaults[vault_name] = Vault(vault_name, self.region_name) + + def list_vaules(self): + return self.vaults.values() + + def delete_vault(self, vault_name): + self.vaults.pop(vault_name) + + def initiate_job(self, vault_name, archive_id): + vault = self.get_vault(vault_name) + job_id = vault.initiate_job(archive_id) + return job_id + + def list_jobs(self, vault_name): + vault = self.get_vault(vault_name) + return vault.list_jobs() + + +glacier_backends = {} +for region in boto.glacier.regions(): + glacier_backends[region.name] = GlacierBackend(region) diff --git a/moto/glacier/responses.py b/moto/glacier/responses.py new file mode 100644 index 000000000..37cbdc4c9 --- /dev/null +++ b/moto/glacier/responses.py @@ -0,0 +1,160 @@ +from __future__ import unicode_literals + +import json +from six.moves.urllib.parse import urlparse, parse_qs + +from moto.core.responses import _TemplateEnvironmentMixin +from .models import glacier_backends +from .utils import region_from_glacier_url, vault_from_glacier_url + + +class GlacierResponse(_TemplateEnvironmentMixin): + + def __init__(self, backend): + self.backend = backend + + @classmethod + def all_vault_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._all_vault_response(request, full_url, headers) + + def _all_vault_response(self, request, full_url, headers): + vaults = self.backend.list_vaules() + response = json.dumps({ + "Marker": None, + "VaultList": [ + vault.to_dict() for vault in vaults + ] + }) + + headers['content-type'] = 'application/json' + return 200, headers, response + + @classmethod + def vault_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_response(request, full_url, headers) + + def _vault_response(self, request, full_url, headers): + method = request.method + parsed_url = urlparse(full_url) + querystring = parse_qs(parsed_url.query, keep_blank_values=True) + vault_name = vault_from_glacier_url(full_url) + + if method == 'GET': + return self._vault_response_get(vault_name, querystring, headers) + elif method == 'PUT': + return self._vault_response_put(vault_name, querystring, headers) + elif method == 'DELETE': + return self._vault_response_delete(vault_name, querystring, headers) + + def _vault_response_get(self, vault_name, querystring, headers): + vault = self.backend.get_vault(vault_name) + headers['content-type'] = 'application/json' + return 200, headers, json.dumps(vault.to_dict()) + + def _vault_response_put(self, vault_name, querystring, headers): + self.backend.create_vault(vault_name) + return 201, headers, "" + + def _vault_response_delete(self, vault_name, querystring, headers): + self.backend.delete_vault(vault_name) + return 204, headers, "" + + @classmethod + def vault_archive_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_archive_response(request, full_url, headers) + + def _vault_archive_response(self, request, full_url, headers): + method = request.method + body = request.body + parsed_url = urlparse(full_url) + querystring = parse_qs(parsed_url.query, keep_blank_values=True) + vault_name = full_url.split("/")[-2] + + if method == 'POST': + return self._vault_archive_response_post(vault_name, body, querystring, headers) + + def _vault_archive_response_post(self, vault_name, body, querystring, headers): + vault = self.backend.get_vault(vault_name) + vault_id = vault.create_archive(body) + headers['x-amz-archive-id'] = vault_id + return 201, headers, "" + + @classmethod + def vault_archive_individual_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_archive_individual_response(request, full_url, headers) + + def _vault_archive_individual_response(self, request, full_url, headers): + method = request.method + vault_name = full_url.split("/")[-3] + archive_id = full_url.split("/")[-1] + + if method == 'DELETE': + vault = self.backend.get_vault(vault_name) + vault.delete_archive(archive_id) + return 204, headers, "" + + @classmethod + def vault_jobs_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_jobs_response(request, full_url, headers) + + def _vault_jobs_response(self, request, full_url, headers): + method = request.method + body = request.body + account_id = full_url.split("/")[1] + vault_name = full_url.split("/")[-2] + + if method == 'GET': + jobs = self.backend.list_jobs(vault_name) + headers['content-type'] = 'application/json' + return 200, headers, json.dumps({ + "JobList": [ + job.to_dict() for job in jobs + ], + "Marker": None, + }) + elif method == 'POST': + json_body = json.loads(body.decode("utf-8")) + archive_id = json_body['ArchiveId'] + job_id = self.backend.initiate_job(vault_name, archive_id) + headers['x-amz-job-id'] = job_id + headers['Location'] = "/{0}/vaults/{1}/jobs/{2}".format(account_id, vault_name, job_id) + return 202, headers, "" + + @classmethod + def vault_jobs_individual_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_jobs_individual_response(request, full_url, headers) + + def _vault_jobs_individual_response(self, request, full_url, headers): + vault_name = full_url.split("/")[-3] + archive_id = full_url.split("/")[-1] + + vault = self.backend.get_vault(vault_name) + job = vault.describe_job(archive_id) + return 200, headers, json.dumps(job.to_dict()) + + @classmethod + def vault_jobs_output_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_jobs_output_response(request, full_url, headers) + + def _vault_jobs_output_response(self, request, full_url, headers): + vault_name = full_url.split("/")[-4] + job_id = full_url.split("/")[-2] + + vault = self.backend.get_vault(vault_name) + output = vault.get_job_output(job_id) + headers['content-type'] = 'application/octet-stream' + return 200, headers, output diff --git a/moto/glacier/urls.py b/moto/glacier/urls.py new file mode 100644 index 000000000..6038c2bb4 --- /dev/null +++ b/moto/glacier/urls.py @@ -0,0 +1,16 @@ +from __future__ import unicode_literals +from .responses import GlacierResponse + +url_bases = [ + "https?://glacier.(.+).amazonaws.com", +] + +url_paths = { + '{0}/(?P.+)/vaults$': GlacierResponse.all_vault_response, + '{0}/(?P.+)/vaults/(?P[^/.]+)$': GlacierResponse.vault_response, + '{0}/(?P.+)/vaults/(?P.+)/archives$': GlacierResponse.vault_archive_response, + '{0}/(?P.+)/vaults/(?P.+)/archives/(?P.+)$': GlacierResponse.vault_archive_individual_response, + '{0}/(?P.+)/vaults/(?P.+)/jobs$': GlacierResponse.vault_jobs_response, + '{0}/(?P.+)/vaults/(?P.+)/jobs/(?P[^/.]+)$': GlacierResponse.vault_jobs_individual_response, + '{0}/(?P.+)/vaults/(?P.+)/jobs/(?P.+)/output$': GlacierResponse.vault_jobs_output_response, +} diff --git a/moto/glacier/utils.py b/moto/glacier/utils.py new file mode 100644 index 000000000..f4a869bf3 --- /dev/null +++ b/moto/glacier/utils.py @@ -0,0 +1,21 @@ +import random +import string + +from six.moves.urllib.parse import urlparse + + +def region_from_glacier_url(url): + domain = urlparse(url).netloc + + if '.' in domain: + return domain.split(".")[1] + else: + return 'us-east-1' + + +def vault_from_glacier_url(full_url): + return full_url.split("/")[-1] + + +def get_job_id(): + return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(92)) diff --git a/tests/test_glacier/test_glacier_archives.py b/tests/test_glacier/test_glacier_archives.py new file mode 100644 index 000000000..6a139a91c --- /dev/null +++ b/tests/test_glacier/test_glacier_archives.py @@ -0,0 +1,21 @@ +from __future__ import unicode_literals + +from tempfile import NamedTemporaryFile +import boto.glacier +import sure # noqa + +from moto import mock_glacier + + +@mock_glacier +def test_create_and_delete_archive(): + the_file = NamedTemporaryFile(delete=False) + the_file.write(b"some stuff") + the_file.close() + + conn = boto.glacier.connect_to_region("us-west-2") + vault = conn.create_vault("my_vault") + + archive_id = vault.upload_archive(the_file.name) + + vault.delete_archive(archive_id) diff --git a/tests/test_glacier/test_glacier_jobs.py b/tests/test_glacier/test_glacier_jobs.py new file mode 100644 index 000000000..7eff3566e --- /dev/null +++ b/tests/test_glacier/test_glacier_jobs.py @@ -0,0 +1,94 @@ +from __future__ import unicode_literals + +import json + +from boto.glacier.layer1 import Layer1 +import sure # noqa + +from moto import mock_glacier + + +@mock_glacier +def test_init_glacier_job(): + conn = Layer1(region_name="us-west-2") + vault_name = "my_vault" + conn.create_vault(vault_name) + archive_id = conn.upload_archive(vault_name, "some stuff", "", "", "some description") + + job_response = conn.initiate_job(vault_name, { + "ArchiveId": archive_id, + "Type": "archive-retrieval", + }) + job_id = job_response['JobId'] + job_response['Location'].should.equal("//vaults/my_vault/jobs/{0}".format(job_id)) + + +@mock_glacier +def test_describe_job(): + conn = Layer1(region_name="us-west-2") + vault_name = "my_vault" + conn.create_vault(vault_name) + archive_id = conn.upload_archive(vault_name, "some stuff", "", "", "some description") + job_response = conn.initiate_job(vault_name, { + "ArchiveId": archive_id, + "Type": "archive-retrieval", + }) + job_id = job_response['JobId'] + + job = conn.describe_job(vault_name, job_id) + json.loads(job.read().decode("utf-8")).should.equal({ + 'CompletionDate': '2013-03-20T17:03:43.221Z', + 'VaultARN': None, + 'RetrievalByteRange': None, + 'SHA256TreeHash': None, + 'Completed': True, + 'InventorySizeInBytes': '0', + 'JobId': job_id, + 'Action': 'InventoryRetrieval', + 'JobDescription': None, + 'SNSTopic': None, + 'ArchiveSizeInBytes': 0, + 'ArchiveId': archive_id, + 'ArchiveSHA256TreeHash': None, + 'CreationDate': '2013-03-20T17:03:43.221Z', + 'StatusMessage': None, + 'StatusCode': 'Succeeded', + }) + + +@mock_glacier +def test_list_glacier_jobs(): + conn = Layer1(region_name="us-west-2") + vault_name = "my_vault" + conn.create_vault(vault_name) + archive_id1 = conn.upload_archive(vault_name, "some stuff", "", "", "some description")['ArchiveId'] + archive_id2 = conn.upload_archive(vault_name, "some other stuff", "", "", "some description")['ArchiveId'] + + conn.initiate_job(vault_name, { + "ArchiveId": archive_id1, + "Type": "archive-retrieval", + }) + conn.initiate_job(vault_name, { + "ArchiveId": archive_id2, + "Type": "archive-retrieval", + }) + + jobs = conn.list_jobs(vault_name) + len(jobs['JobList']).should.equal(2) + + +@mock_glacier +def test_get_job_output(): + conn = Layer1(region_name="us-west-2") + vault_name = "my_vault" + conn.create_vault(vault_name) + archive_response = conn.upload_archive(vault_name, "some stuff", "", "", "some description") + archive_id = archive_response['ArchiveId'] + job_response = conn.initiate_job(vault_name, { + "ArchiveId": archive_id, + "Type": "archive-retrieval", + }) + job_id = job_response['JobId'] + + output = conn.get_job_output(vault_name, job_id) + output.read().decode("utf-8").should.equal("some stuff") diff --git a/tests/test_glacier/test_glacier_server.py b/tests/test_glacier/test_glacier_server.py new file mode 100644 index 000000000..d3e09015f --- /dev/null +++ b/tests/test_glacier/test_glacier_server.py @@ -0,0 +1,21 @@ +from __future__ import unicode_literals + +import json +import sure # noqa + +import moto.server as server +from moto import mock_glacier + +''' +Test the different server responses +''' + + +@mock_glacier +def test_list_vaults(): + backend = server.create_backend_app("glacier") + test_client = backend.test_client() + + res = test_client.get('/1234bcd/vaults') + + json.loads(res.data.decode("utf-8")).should.equal({u'Marker': None, u'VaultList': []}) diff --git a/tests/test_glacier/test_glacier_vaults.py b/tests/test_glacier/test_glacier_vaults.py new file mode 100644 index 000000000..40f20e58e --- /dev/null +++ b/tests/test_glacier/test_glacier_vaults.py @@ -0,0 +1,31 @@ +from __future__ import unicode_literals + +import boto.glacier +import sure # noqa + +from moto import mock_glacier + + +@mock_glacier +def test_create_vault(): + conn = boto.glacier.connect_to_region("us-west-2") + + conn.create_vault("my_vault") + + vaults = conn.list_vaults() + vaults.should.have.length_of(1) + vaults[0].name.should.equal("my_vault") + + +@mock_glacier +def test_delete_vault(): + conn = boto.glacier.connect_to_region("us-west-2") + + conn.create_vault("my_vault") + + vaults = conn.list_vaults() + vaults.should.have.length_of(1) + + conn.delete_vault("my_vault") + vaults = conn.list_vaults() + vaults.should.have.length_of(0)