diff --git a/moto/glacier/models.py b/moto/glacier/models.py index 1afb1241a..2c16bc97d 100644 --- a/moto/glacier/models.py +++ b/moto/glacier/models.py @@ -2,42 +2,101 @@ from __future__ import unicode_literals import hashlib +import datetime + + import boto.glacier from moto.core import BaseBackend, BaseModel from .utils import get_job_id -class ArchiveJob(BaseModel): +class Job(BaseModel): + def __init__(self, tier): + self.st = datetime.datetime.now() - def __init__(self, job_id, archive_id): + if tier.lower() == "expedited": + self.et = self.st + datetime.timedelta(seconds=2) + elif tier.lower() == "bulk": + self.et = self.st + datetime.timedelta(seconds=10) + else: + # Standard + self.et = self.st + datetime.timedelta(seconds=5) + + +class ArchiveJob(Job): + + def __init__(self, job_id, tier, arn, archive_id): self.job_id = job_id + self.tier = tier + self.arn = arn self.archive_id = archive_id + Job.__init__(self, tier) def to_dict(self): - return { - "Action": "InventoryRetrieval", + d = { + "Action": "ArchiveRetrieval", "ArchiveId": self.archive_id, "ArchiveSizeInBytes": 0, "ArchiveSHA256TreeHash": None, - "Completed": True, - "CompletionDate": "2013-03-20T17:03:43.221Z", - "CreationDate": "2013-03-20T17:03:43.221Z", - "InventorySizeInBytes": "0", + "Completed": False, + "CreationDate": self.st.strftime("%Y-%m-%dT%H:%M:%S.000Z"), + "InventorySizeInBytes": 0, "JobDescription": None, "JobId": self.job_id, "RetrievalByteRange": None, "SHA256TreeHash": None, "SNSTopic": None, - "StatusCode": "Succeeded", + "StatusCode": "InProgress", "StatusMessage": None, - "VaultARN": None, + "VaultARN": self.arn, + "Tier": self.tier } + if datetime.datetime.now() > self.et: + d["Completed"] = True + d["CompletionDate"] = self.et.strftime("%Y-%m-%dT%H:%M:%S.000Z") + d["InventorySizeInBytes"] = 10000 + d["StatusCode"] = "Succeeded" + return d + + +class InventoryJob(Job): + + def __init__(self, job_id, tier, arn): + self.job_id = job_id + self.tier = tier + self.arn = arn + Job.__init__(self, tier) + + def to_dict(self): + d = { + "Action": "InventoryRetrieval", + "ArchiveSHA256TreeHash": None, + "Completed": False, + "CreationDate": self.st.strftime("%Y-%m-%dT%H:%M:%S.000Z"), + "InventorySizeInBytes": 0, + "JobDescription": None, + "JobId": self.job_id, + "RetrievalByteRange": None, + "SHA256TreeHash": None, + "SNSTopic": None, + "StatusCode": "InProgress", + "StatusMessage": None, + "VaultARN": self.arn, + "Tier": self.tier + } + if datetime.datetime.now() > self.et: + d["Completed"] = True + d["CompletionDate"] = self.et.strftime("%Y-%m-%dT%H:%M:%S.000Z") + d["InventorySizeInBytes"] = 10000 + d["StatusCode"] = "Succeeded" + return d class Vault(BaseModel): def __init__(self, vault_name, region): + self.st = datetime.datetime.now() self.vault_name = vault_name self.region = region self.archives = {} @@ -48,29 +107,57 @@ class Vault(BaseModel): return "arn:aws:glacier:{0}:012345678901:vaults/{1}".format(self.region, self.vault_name) def to_dict(self): - return { - "CreationDate": "2013-03-20T17:03:43.221Z", - "LastInventoryDate": "2013-03-20T17:03:43.221Z", - "NumberOfArchives": None, - "SizeInBytes": None, + archives_size = 0 + for k in self.archives: + archives_size += self.archives[k]["size"] + d = { + "CreationDate": self.st.strftime("%Y-%m-%dT%H:%M:%S.000Z"), + "LastInventoryDate": self.st.strftime("%Y-%m-%dT%H:%M:%S.000Z"), + "NumberOfArchives": len(self.archives), + "SizeInBytes": archives_size, "VaultARN": self.arn, "VaultName": self.vault_name, } + return d - def create_archive(self, body): - archive_id = hashlib.sha256(body).hexdigest() - self.archives[archive_id] = body + def create_archive(self, body, description): + archive_id = hashlib.md5(body).hexdigest() + self.archives[archive_id] = {} + self.archives[archive_id]["body"] = body + self.archives[archive_id]["size"] = len(body) + self.archives[archive_id]["sha256"] = hashlib.sha256(body).hexdigest() + self.archives[archive_id]["creation_date"] = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.000Z") + self.archives[archive_id]["description"] = description return archive_id def get_archive_body(self, archive_id): - return self.archives[archive_id] + return self.archives[archive_id]["body"] + + def get_archive_list(self): + archive_list = [] + for a in self.archives: + archive = self.archives[a] + aobj = { + "ArchiveId": a, + "ArchiveDescription": archive["description"], + "CreationDate": archive["creation_date"], + "Size": archive["size"], + "SHA256TreeHash": archive["sha256"] + } + archive_list.append(aobj) + return archive_list def delete_archive(self, archive_id): return self.archives.pop(archive_id) - def initiate_job(self, archive_id): + def initiate_job(self, job_type, tier, archive_id): job_id = get_job_id() - job = ArchiveJob(job_id, archive_id) + + if job_type == "inventory-retrieval": + job = InventoryJob(job_id, tier, self.arn) + elif job_type == "archive-retrieval": + job = ArchiveJob(job_id, tier, self.arn, archive_id) + self.jobs[job_id] = job return job_id @@ -80,10 +167,24 @@ class Vault(BaseModel): def describe_job(self, job_id): return self.jobs.get(job_id) + def job_ready(self, job_id): + job = self.describe_job(job_id) + jobj = job.to_dict() + return jobj["Completed"] + def get_job_output(self, job_id): job = self.describe_job(job_id) - archive_body = self.get_archive_body(job.archive_id) - return archive_body + jobj = job.to_dict() + if jobj["Action"] == "InventoryRetrieval": + archives = self.get_archive_list() + return { + "VaultARN": self.arn, + "InventoryDate": jobj["CompletionDate"], + "ArchiveList": archives + } + else: + archive_body = self.get_archive_body(job.archive_id) + return archive_body class GlacierBackend(BaseBackend): @@ -109,9 +210,9 @@ class GlacierBackend(BaseBackend): def delete_vault(self, vault_name): self.vaults.pop(vault_name) - def initiate_job(self, vault_name, archive_id): + def initiate_job(self, vault_name, job_type, tier, archive_id): vault = self.get_vault(vault_name) - job_id = vault.initiate_job(archive_id) + job_id = vault.initiate_job(job_type, tier, archive_id) return job_id def list_jobs(self, vault_name): diff --git a/moto/glacier/responses.py b/moto/glacier/responses.py index cda859b29..abdf83e4f 100644 --- a/moto/glacier/responses.py +++ b/moto/glacier/responses.py @@ -72,17 +72,25 @@ class GlacierResponse(_TemplateEnvironmentMixin): def _vault_archive_response(self, request, full_url, headers): method = request.method - body = request.body + if hasattr(request, 'body'): + body = request.body + else: + body = request.data + description = "" + if 'x-amz-archive-description' in request.headers: + description = request.headers['x-amz-archive-description'] parsed_url = urlparse(full_url) querystring = parse_qs(parsed_url.query, keep_blank_values=True) vault_name = full_url.split("/")[-2] if method == 'POST': - return self._vault_archive_response_post(vault_name, body, querystring, headers) + return self._vault_archive_response_post(vault_name, body, description, querystring, headers) + else: + return 400, headers, "400 Bad Request" - def _vault_archive_response_post(self, vault_name, body, querystring, headers): + def _vault_archive_response_post(self, vault_name, body, description, querystring, headers): vault = self.backend.get_vault(vault_name) - vault_id = vault.create_archive(body) + vault_id = vault.create_archive(body, description) headers['x-amz-archive-id'] = vault_id return 201, headers, "" @@ -110,7 +118,10 @@ class GlacierResponse(_TemplateEnvironmentMixin): def _vault_jobs_response(self, request, full_url, headers): method = request.method - body = request.body + if hasattr(request, 'body'): + body = request.body + else: + body = request.data account_id = full_url.split("/")[1] vault_name = full_url.split("/")[-2] @@ -125,11 +136,17 @@ class GlacierResponse(_TemplateEnvironmentMixin): }) elif method == 'POST': json_body = json.loads(body.decode("utf-8")) - archive_id = json_body['ArchiveId'] - job_id = self.backend.initiate_job(vault_name, archive_id) + job_type = json_body['Type'] + archive_id = None + if 'ArchiveId' in json_body: + archive_id = json_body['ArchiveId'] + if 'Tier' in json_body: + tier = json_body["Tier"] + else: + tier = "Standard" + job_id = self.backend.initiate_job(vault_name, job_type, tier, archive_id) headers['x-amz-job-id'] = job_id - headers[ - 'Location'] = "/{0}/vaults/{1}/jobs/{2}".format(account_id, vault_name, job_id) + headers['Location'] = "/{0}/vaults/{1}/jobs/{2}".format(account_id, vault_name, job_id) return 202, headers, "" @classmethod @@ -155,8 +172,14 @@ class GlacierResponse(_TemplateEnvironmentMixin): def _vault_jobs_output_response(self, request, full_url, headers): vault_name = full_url.split("/")[-4] job_id = full_url.split("/")[-2] - vault = self.backend.get_vault(vault_name) - output = vault.get_job_output(job_id) - headers['content-type'] = 'application/octet-stream' - return 200, headers, output + if vault.job_ready(job_id): + output = vault.get_job_output(job_id) + if isinstance(output, dict): + headers['content-type'] = 'application/json' + return 200, headers, json.dumps(output) + else: + headers['content-type'] = 'application/octet-stream' + return 200, headers, output + else: + return 404, headers, "404 Not Found" diff --git a/tests/test_glacier/test_glacier_jobs.py b/tests/test_glacier/test_glacier_jobs.py index 66780f681..152aa14c8 100644 --- a/tests/test_glacier/test_glacier_jobs.py +++ b/tests/test_glacier/test_glacier_jobs.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import json +import time from boto.glacier.layer1 import Layer1 import sure # noqa @@ -39,24 +40,11 @@ def test_describe_job(): job_id = job_response['JobId'] job = conn.describe_job(vault_name, job_id) - json.loads(job.read().decode("utf-8")).should.equal({ - 'CompletionDate': '2013-03-20T17:03:43.221Z', - 'VaultARN': None, - 'RetrievalByteRange': None, - 'SHA256TreeHash': None, - 'Completed': True, - 'InventorySizeInBytes': '0', - 'JobId': job_id, - 'Action': 'InventoryRetrieval', - 'JobDescription': None, - 'SNSTopic': None, - 'ArchiveSizeInBytes': 0, - 'ArchiveId': archive_id, - 'ArchiveSHA256TreeHash': None, - 'CreationDate': '2013-03-20T17:03:43.221Z', - 'StatusMessage': None, - 'StatusCode': 'Succeeded', - }) + joboutput = json.loads(job.read().decode("utf-8")) + + joboutput.should.have.key('Tier').which.should.equal('Standard') + joboutput.should.have.key('StatusCode').which.should.equal('InProgress') + joboutput.should.have.key('VaultARN').which.should.equal('arn:aws:glacier:RegionInfo:us-west-2:012345678901:vaults/my_vault') @mock_glacier_deprecated @@ -96,5 +84,7 @@ def test_get_job_output(): }) job_id = job_response['JobId'] + time.sleep(6) + output = conn.get_job_output(vault_name, job_id) output.read().decode("utf-8").should.equal("some stuff")