diff --git a/moto/glacier/models.py b/moto/glacier/models.py index b5b5e38d1..836e84d37 100644 --- a/moto/glacier/models.py +++ b/moto/glacier/models.py @@ -1,17 +1,50 @@ from __future__ import unicode_literals +import hashlib + import boto.glacier from moto.core import BaseBackend +from .utils import get_job_id + + +class ArchiveJob(object): + + def __init__(self, job_id, archive_id): + self.job_id = job_id + self.archive_id = archive_id + + def to_dict(self): + return { + "Action": "InventoryRetrieval", + "ArchiveId": self.archive_id, + "ArchiveSizeInBytes": 0, + "ArchiveSHA256TreeHash": None, + "Completed": True, + "CompletionDate": "2013-03-20T17:03:43.221Z", + "CreationDate": "2013-03-20T17:03:43.221Z", + "InventorySizeInBytes": "0", + "JobDescription": None, + "JobId": self.job_id, + "RetrievalByteRange": None, + "SHA256TreeHash": None, + "SNSTopic": None, + "StatusCode": "Succeeded", + "StatusMessage": None, + "VaultARN": None, + } + class Vault(object): def __init__(self, vault_name, region): self.vault_name = vault_name self.region = region + self.archives = {} + self.jobs = {} @property def arn(self): - return "arn:aws:glacier:{}:012345678901:vaults/{}".format(self.region, self.vault_name) + return "arn:aws:glacier:{0}:012345678901:vaults/{1}".format(self.region, self.vault_name) def to_dict(self): return { @@ -23,6 +56,34 @@ class Vault(object): "VaultName": self.vault_name, } + def create_archive(self, body): + archive_id = hashlib.sha256(body).hexdigest() + self.archives[archive_id] = body + return archive_id + + def get_archive_body(self, archive_id): + return self.archives[archive_id] + + def delete_archive(self, archive_id): + return self.archives.pop(archive_id) + + def initiate_job(self, archive_id): + job_id = get_job_id() + job = ArchiveJob(job_id, archive_id) + self.jobs[job_id] = job + return job_id + + def list_jobs(self): + return self.jobs.values() + + def describe_job(self, job_id): + return self.jobs.get(job_id) + + def get_job_output(self, job_id): + job = self.describe_job(job_id) + archive_body = self.get_archive_body(job.archive_id) + return archive_body + class GlacierBackend(BaseBackend): @@ -47,6 +108,16 @@ class GlacierBackend(BaseBackend): def delete_vault(self, vault_name): self.vaults.pop(vault_name) + def initiate_job(self, vault_name, archive_id): + vault = self.get_vault(vault_name) + job_id = vault.initiate_job(archive_id) + return job_id + + def list_jobs(self, vault_name): + vault = self.get_vault(vault_name) + return vault.list_jobs() + + glacier_backends = {} for region in boto.glacier.regions(): glacier_backends[region.name] = GlacierBackend(region) diff --git a/moto/glacier/responses.py b/moto/glacier/responses.py index a8194b317..fd3bf3525 100644 --- a/moto/glacier/responses.py +++ b/moto/glacier/responses.py @@ -62,3 +62,99 @@ class GlacierResponse(_TemplateEnvironmentMixin): def _vault_response_delete(self, vault_name, querystring, headers): self.backend.delete_vault(vault_name) return 204, headers, "" + + @classmethod + def vault_archive_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_archive_response(request, full_url, headers) + + def _vault_archive_response(self, request, full_url, headers): + method = request.method + body = request.body + parsed_url = urlparse(full_url) + querystring = parse_qs(parsed_url.query, keep_blank_values=True) + vault_name = full_url.split("/")[-2] + + if method == 'POST': + return self._vault_archive_response_post(vault_name, body, querystring, headers) + + def _vault_archive_response_post(self, vault_name, body, querystring, headers): + vault = self.backend.get_vault(vault_name) + vault_id = vault.create_archive(body) + headers['x-amz-archive-id'] = vault_id + return 201, headers, "" + + @classmethod + def vault_archive_individual_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_archive_individual_response(request, full_url, headers) + + def _vault_archive_individual_response(self, request, full_url, headers): + method = request.method + vault_name = full_url.split("/")[-3] + archive_id = full_url.split("/")[-1] + + if method == 'DELETE': + vault = self.backend.get_vault(vault_name) + vault.delete_archive(archive_id) + return 204, headers, "" + + @classmethod + def vault_jobs_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_jobs_response(request, full_url, headers) + + def _vault_jobs_response(self, request, full_url, headers): + method = request.method + body = request.body + account_id = full_url.split("/")[1] + vault_name = full_url.split("/")[-2] + + if method == 'GET': + jobs = self.backend.list_jobs(vault_name) + headers['content-type'] = 'application/json' + return 200, headers, json.dumps({ + "JobList": [ + job.to_dict() for job in jobs + ], + "Marker": None, + }) + elif method == 'POST': + json_body = json.loads(body) + archive_id = json_body['ArchiveId'] + job_id = self.backend.initiate_job(vault_name, archive_id) + headers['x-amz-job-id'] = job_id + headers['Location'] = "/{0}/vaults/{1}/jobs/{2}".format(account_id, vault_name, job_id) + return 202, headers, "" + + @classmethod + def vault_jobs_individual_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_jobs_individual_response(request, full_url, headers) + + def _vault_jobs_individual_response(self, request, full_url, headers): + vault_name = full_url.split("/")[-3] + archive_id = full_url.split("/")[-1] + + vault = self.backend.get_vault(vault_name) + job = vault.describe_job(archive_id) + return 200, headers, json.dumps(job.to_dict()) + + @classmethod + def vault_jobs_output_response(clazz, request, full_url, headers): + region_name = region_from_glacier_url(full_url) + response_instance = GlacierResponse(glacier_backends[region_name]) + return response_instance._vault_jobs_output_response(request, full_url, headers) + + def _vault_jobs_output_response(self, request, full_url, headers): + vault_name = full_url.split("/")[-4] + job_id = full_url.split("/")[-2] + + vault = self.backend.get_vault(vault_name) + output = vault.get_job_output(job_id) + headers['content-type'] = 'application/octet-stream' + return 200, headers, output diff --git a/moto/glacier/urls.py b/moto/glacier/urls.py index d5d812896..6038c2bb4 100644 --- a/moto/glacier/urls.py +++ b/moto/glacier/urls.py @@ -7,5 +7,10 @@ url_bases = [ url_paths = { '{0}/(?P.+)/vaults$': GlacierResponse.all_vault_response, - '{0}/(?P.+)/vaults/(?P.+)$': GlacierResponse.vault_response, + '{0}/(?P.+)/vaults/(?P[^/.]+)$': GlacierResponse.vault_response, + '{0}/(?P.+)/vaults/(?P.+)/archives$': GlacierResponse.vault_archive_response, + '{0}/(?P.+)/vaults/(?P.+)/archives/(?P.+)$': GlacierResponse.vault_archive_individual_response, + '{0}/(?P.+)/vaults/(?P.+)/jobs$': GlacierResponse.vault_jobs_response, + '{0}/(?P.+)/vaults/(?P.+)/jobs/(?P[^/.]+)$': GlacierResponse.vault_jobs_individual_response, + '{0}/(?P.+)/vaults/(?P.+)/jobs/(?P.+)/output$': GlacierResponse.vault_jobs_output_response, } diff --git a/moto/glacier/utils.py b/moto/glacier/utils.py index 5b53a089c..f4a869bf3 100644 --- a/moto/glacier/utils.py +++ b/moto/glacier/utils.py @@ -1,3 +1,6 @@ +import random +import string + from six.moves.urllib.parse import urlparse @@ -12,3 +15,7 @@ def region_from_glacier_url(url): def vault_from_glacier_url(full_url): return full_url.split("/")[-1] + + +def get_job_id(): + return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(92)) diff --git a/tests/test_glacier/test_glacier_archives.py b/tests/test_glacier/test_glacier_archives.py new file mode 100644 index 000000000..72b2afb16 --- /dev/null +++ b/tests/test_glacier/test_glacier_archives.py @@ -0,0 +1,21 @@ +from __future__ import unicode_literals + +from tempfile import NamedTemporaryFile +import boto.glacier +import sure # noqa + +from moto import mock_glacier + + +@mock_glacier +def test_create_and_delete_archive(): + the_file = NamedTemporaryFile(delete=False) + the_file.write("some stuff") + the_file.close() + + conn = boto.glacier.connect_to_region("us-west-2") + vault = conn.create_vault("my_vault") + + archive_id = vault.upload_archive(the_file.name) + + vault.delete_archive(archive_id) diff --git a/tests/test_glacier/test_glacier_jobs.py b/tests/test_glacier/test_glacier_jobs.py new file mode 100644 index 000000000..81bad597d --- /dev/null +++ b/tests/test_glacier/test_glacier_jobs.py @@ -0,0 +1,94 @@ +from __future__ import unicode_literals + +import json + +from boto.glacier.layer1 import Layer1 +import sure # noqa + +from moto import mock_glacier + + +@mock_glacier +def test_init_glacier_job(): + conn = Layer1(region_name="us-west-2") + vault_name = "my_vault" + conn.create_vault(vault_name) + archive_id = conn.upload_archive(vault_name, "some stuff", "", "", "some description") + + job_response = conn.initiate_job(vault_name, { + "ArchiveId": archive_id, + "Type": "archive-retrieval", + }) + job_id = job_response['JobId'] + job_response['Location'].should.equal("//vaults/my_vault/jobs/{0}".format(job_id)) + + +@mock_glacier +def test_describe_job(): + conn = Layer1(region_name="us-west-2") + vault_name = "my_vault" + conn.create_vault(vault_name) + archive_id = conn.upload_archive(vault_name, "some stuff", "", "", "some description") + job_response = conn.initiate_job(vault_name, { + "ArchiveId": archive_id, + "Type": "archive-retrieval", + }) + job_id = job_response['JobId'] + + job = conn.describe_job(vault_name, job_id) + json.loads(job.read()).should.equal({ + 'CompletionDate': '2013-03-20T17:03:43.221Z', + 'VaultARN': None, + 'RetrievalByteRange': None, + 'SHA256TreeHash': None, + 'Completed': True, + 'InventorySizeInBytes': '0', + 'JobId': job_id, + 'Action': 'InventoryRetrieval', + 'JobDescription': None, + 'SNSTopic': None, + 'ArchiveSizeInBytes': 0, + 'ArchiveId': archive_id, + 'ArchiveSHA256TreeHash': None, + 'CreationDate': '2013-03-20T17:03:43.221Z', + 'StatusMessage': None, + 'StatusCode': 'Succeeded', + }) + + +@mock_glacier +def test_list_glacier_jobs(): + conn = Layer1(region_name="us-west-2") + vault_name = "my_vault" + conn.create_vault(vault_name) + archive_id1 = conn.upload_archive(vault_name, "some stuff", "", "", "some description")['ArchiveId'] + archive_id2 = conn.upload_archive(vault_name, "some other stuff", "", "", "some description")['ArchiveId'] + + conn.initiate_job(vault_name, { + "ArchiveId": archive_id1, + "Type": "archive-retrieval", + }) + conn.initiate_job(vault_name, { + "ArchiveId": archive_id2, + "Type": "archive-retrieval", + }) + + jobs = conn.list_jobs(vault_name) + len(jobs['JobList']).should.equal(2) + + +@mock_glacier +def test_get_job_output(): + conn = Layer1(region_name="us-west-2") + vault_name = "my_vault" + conn.create_vault(vault_name) + archive_response = conn.upload_archive(vault_name, "some stuff", "", "", "some description") + archive_id = archive_response['ArchiveId'] + job_response = conn.initiate_job(vault_name, { + "ArchiveId": archive_id, + "Type": "archive-retrieval", + }) + job_id = job_response['JobId'] + + output = conn.get_job_output(vault_name, job_id) + output.read().should.equal("some stuff") diff --git a/tests/test_glacier/test_glacier_server.py b/tests/test_glacier/test_glacier_server.py index cbdd00b2e..d3e09015f 100644 --- a/tests/test_glacier/test_glacier_server.py +++ b/tests/test_glacier/test_glacier_server.py @@ -18,4 +18,4 @@ def test_list_vaults(): res = test_client.get('/1234bcd/vaults') - json.loads(res.data).should.equal({u'Marker': None, u'VaultList': []}) + json.loads(res.data.decode("utf-8")).should.equal({u'Marker': None, u'VaultList': []})