Add basics of Glacier.
This commit is contained in:
parent
7156df1a63
commit
625b1d2ac6
@ -1,17 +1,50 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
|
||||||
import boto.glacier
|
import boto.glacier
|
||||||
from moto.core import BaseBackend
|
from moto.core import BaseBackend
|
||||||
|
|
||||||
|
from .utils import get_job_id
|
||||||
|
|
||||||
|
|
||||||
|
class ArchiveJob(object):
|
||||||
|
|
||||||
|
def __init__(self, job_id, archive_id):
|
||||||
|
self.job_id = job_id
|
||||||
|
self.archive_id = archive_id
|
||||||
|
|
||||||
|
def to_dict(self):
|
||||||
|
return {
|
||||||
|
"Action": "InventoryRetrieval",
|
||||||
|
"ArchiveId": self.archive_id,
|
||||||
|
"ArchiveSizeInBytes": 0,
|
||||||
|
"ArchiveSHA256TreeHash": None,
|
||||||
|
"Completed": True,
|
||||||
|
"CompletionDate": "2013-03-20T17:03:43.221Z",
|
||||||
|
"CreationDate": "2013-03-20T17:03:43.221Z",
|
||||||
|
"InventorySizeInBytes": "0",
|
||||||
|
"JobDescription": None,
|
||||||
|
"JobId": self.job_id,
|
||||||
|
"RetrievalByteRange": None,
|
||||||
|
"SHA256TreeHash": None,
|
||||||
|
"SNSTopic": None,
|
||||||
|
"StatusCode": "Succeeded",
|
||||||
|
"StatusMessage": None,
|
||||||
|
"VaultARN": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class Vault(object):
|
class Vault(object):
|
||||||
def __init__(self, vault_name, region):
|
def __init__(self, vault_name, region):
|
||||||
self.vault_name = vault_name
|
self.vault_name = vault_name
|
||||||
self.region = region
|
self.region = region
|
||||||
|
self.archives = {}
|
||||||
|
self.jobs = {}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def arn(self):
|
def arn(self):
|
||||||
return "arn:aws:glacier:{}:012345678901:vaults/{}".format(self.region, self.vault_name)
|
return "arn:aws:glacier:{0}:012345678901:vaults/{1}".format(self.region, self.vault_name)
|
||||||
|
|
||||||
def to_dict(self):
|
def to_dict(self):
|
||||||
return {
|
return {
|
||||||
@ -23,6 +56,34 @@ class Vault(object):
|
|||||||
"VaultName": self.vault_name,
|
"VaultName": self.vault_name,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def create_archive(self, body):
|
||||||
|
archive_id = hashlib.sha256(body).hexdigest()
|
||||||
|
self.archives[archive_id] = body
|
||||||
|
return archive_id
|
||||||
|
|
||||||
|
def get_archive_body(self, archive_id):
|
||||||
|
return self.archives[archive_id]
|
||||||
|
|
||||||
|
def delete_archive(self, archive_id):
|
||||||
|
return self.archives.pop(archive_id)
|
||||||
|
|
||||||
|
def initiate_job(self, archive_id):
|
||||||
|
job_id = get_job_id()
|
||||||
|
job = ArchiveJob(job_id, archive_id)
|
||||||
|
self.jobs[job_id] = job
|
||||||
|
return job_id
|
||||||
|
|
||||||
|
def list_jobs(self):
|
||||||
|
return self.jobs.values()
|
||||||
|
|
||||||
|
def describe_job(self, job_id):
|
||||||
|
return self.jobs.get(job_id)
|
||||||
|
|
||||||
|
def get_job_output(self, job_id):
|
||||||
|
job = self.describe_job(job_id)
|
||||||
|
archive_body = self.get_archive_body(job.archive_id)
|
||||||
|
return archive_body
|
||||||
|
|
||||||
|
|
||||||
class GlacierBackend(BaseBackend):
|
class GlacierBackend(BaseBackend):
|
||||||
|
|
||||||
@ -47,6 +108,16 @@ class GlacierBackend(BaseBackend):
|
|||||||
def delete_vault(self, vault_name):
|
def delete_vault(self, vault_name):
|
||||||
self.vaults.pop(vault_name)
|
self.vaults.pop(vault_name)
|
||||||
|
|
||||||
|
def initiate_job(self, vault_name, archive_id):
|
||||||
|
vault = self.get_vault(vault_name)
|
||||||
|
job_id = vault.initiate_job(archive_id)
|
||||||
|
return job_id
|
||||||
|
|
||||||
|
def list_jobs(self, vault_name):
|
||||||
|
vault = self.get_vault(vault_name)
|
||||||
|
return vault.list_jobs()
|
||||||
|
|
||||||
|
|
||||||
glacier_backends = {}
|
glacier_backends = {}
|
||||||
for region in boto.glacier.regions():
|
for region in boto.glacier.regions():
|
||||||
glacier_backends[region.name] = GlacierBackend(region)
|
glacier_backends[region.name] = GlacierBackend(region)
|
||||||
|
@ -62,3 +62,99 @@ class GlacierResponse(_TemplateEnvironmentMixin):
|
|||||||
def _vault_response_delete(self, vault_name, querystring, headers):
|
def _vault_response_delete(self, vault_name, querystring, headers):
|
||||||
self.backend.delete_vault(vault_name)
|
self.backend.delete_vault(vault_name)
|
||||||
return 204, headers, ""
|
return 204, headers, ""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def vault_archive_response(clazz, request, full_url, headers):
|
||||||
|
region_name = region_from_glacier_url(full_url)
|
||||||
|
response_instance = GlacierResponse(glacier_backends[region_name])
|
||||||
|
return response_instance._vault_archive_response(request, full_url, headers)
|
||||||
|
|
||||||
|
def _vault_archive_response(self, request, full_url, headers):
|
||||||
|
method = request.method
|
||||||
|
body = request.body
|
||||||
|
parsed_url = urlparse(full_url)
|
||||||
|
querystring = parse_qs(parsed_url.query, keep_blank_values=True)
|
||||||
|
vault_name = full_url.split("/")[-2]
|
||||||
|
|
||||||
|
if method == 'POST':
|
||||||
|
return self._vault_archive_response_post(vault_name, body, querystring, headers)
|
||||||
|
|
||||||
|
def _vault_archive_response_post(self, vault_name, body, querystring, headers):
|
||||||
|
vault = self.backend.get_vault(vault_name)
|
||||||
|
vault_id = vault.create_archive(body)
|
||||||
|
headers['x-amz-archive-id'] = vault_id
|
||||||
|
return 201, headers, ""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def vault_archive_individual_response(clazz, request, full_url, headers):
|
||||||
|
region_name = region_from_glacier_url(full_url)
|
||||||
|
response_instance = GlacierResponse(glacier_backends[region_name])
|
||||||
|
return response_instance._vault_archive_individual_response(request, full_url, headers)
|
||||||
|
|
||||||
|
def _vault_archive_individual_response(self, request, full_url, headers):
|
||||||
|
method = request.method
|
||||||
|
vault_name = full_url.split("/")[-3]
|
||||||
|
archive_id = full_url.split("/")[-1]
|
||||||
|
|
||||||
|
if method == 'DELETE':
|
||||||
|
vault = self.backend.get_vault(vault_name)
|
||||||
|
vault.delete_archive(archive_id)
|
||||||
|
return 204, headers, ""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def vault_jobs_response(clazz, request, full_url, headers):
|
||||||
|
region_name = region_from_glacier_url(full_url)
|
||||||
|
response_instance = GlacierResponse(glacier_backends[region_name])
|
||||||
|
return response_instance._vault_jobs_response(request, full_url, headers)
|
||||||
|
|
||||||
|
def _vault_jobs_response(self, request, full_url, headers):
|
||||||
|
method = request.method
|
||||||
|
body = request.body
|
||||||
|
account_id = full_url.split("/")[1]
|
||||||
|
vault_name = full_url.split("/")[-2]
|
||||||
|
|
||||||
|
if method == 'GET':
|
||||||
|
jobs = self.backend.list_jobs(vault_name)
|
||||||
|
headers['content-type'] = 'application/json'
|
||||||
|
return 200, headers, json.dumps({
|
||||||
|
"JobList": [
|
||||||
|
job.to_dict() for job in jobs
|
||||||
|
],
|
||||||
|
"Marker": None,
|
||||||
|
})
|
||||||
|
elif method == 'POST':
|
||||||
|
json_body = json.loads(body)
|
||||||
|
archive_id = json_body['ArchiveId']
|
||||||
|
job_id = self.backend.initiate_job(vault_name, archive_id)
|
||||||
|
headers['x-amz-job-id'] = job_id
|
||||||
|
headers['Location'] = "/{0}/vaults/{1}/jobs/{2}".format(account_id, vault_name, job_id)
|
||||||
|
return 202, headers, ""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def vault_jobs_individual_response(clazz, request, full_url, headers):
|
||||||
|
region_name = region_from_glacier_url(full_url)
|
||||||
|
response_instance = GlacierResponse(glacier_backends[region_name])
|
||||||
|
return response_instance._vault_jobs_individual_response(request, full_url, headers)
|
||||||
|
|
||||||
|
def _vault_jobs_individual_response(self, request, full_url, headers):
|
||||||
|
vault_name = full_url.split("/")[-3]
|
||||||
|
archive_id = full_url.split("/")[-1]
|
||||||
|
|
||||||
|
vault = self.backend.get_vault(vault_name)
|
||||||
|
job = vault.describe_job(archive_id)
|
||||||
|
return 200, headers, json.dumps(job.to_dict())
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def vault_jobs_output_response(clazz, request, full_url, headers):
|
||||||
|
region_name = region_from_glacier_url(full_url)
|
||||||
|
response_instance = GlacierResponse(glacier_backends[region_name])
|
||||||
|
return response_instance._vault_jobs_output_response(request, full_url, headers)
|
||||||
|
|
||||||
|
def _vault_jobs_output_response(self, request, full_url, headers):
|
||||||
|
vault_name = full_url.split("/")[-4]
|
||||||
|
job_id = full_url.split("/")[-2]
|
||||||
|
|
||||||
|
vault = self.backend.get_vault(vault_name)
|
||||||
|
output = vault.get_job_output(job_id)
|
||||||
|
headers['content-type'] = 'application/octet-stream'
|
||||||
|
return 200, headers, output
|
||||||
|
@ -7,5 +7,10 @@ url_bases = [
|
|||||||
|
|
||||||
url_paths = {
|
url_paths = {
|
||||||
'{0}/(?P<account_number>.+)/vaults$': GlacierResponse.all_vault_response,
|
'{0}/(?P<account_number>.+)/vaults$': GlacierResponse.all_vault_response,
|
||||||
'{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)$': GlacierResponse.vault_response,
|
'{0}/(?P<account_number>.+)/vaults/(?P<vault_name>[^/.]+)$': GlacierResponse.vault_response,
|
||||||
|
'{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/archives$': GlacierResponse.vault_archive_response,
|
||||||
|
'{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/archives/(?P<archive_id>.+)$': GlacierResponse.vault_archive_individual_response,
|
||||||
|
'{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs$': GlacierResponse.vault_jobs_response,
|
||||||
|
'{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs/(?P<job_id>[^/.]+)$': GlacierResponse.vault_jobs_individual_response,
|
||||||
|
'{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs/(?P<job_id>.+)/output$': GlacierResponse.vault_jobs_output_response,
|
||||||
}
|
}
|
||||||
|
@ -1,3 +1,6 @@
|
|||||||
|
import random
|
||||||
|
import string
|
||||||
|
|
||||||
from six.moves.urllib.parse import urlparse
|
from six.moves.urllib.parse import urlparse
|
||||||
|
|
||||||
|
|
||||||
@ -12,3 +15,7 @@ def region_from_glacier_url(url):
|
|||||||
|
|
||||||
def vault_from_glacier_url(full_url):
|
def vault_from_glacier_url(full_url):
|
||||||
return full_url.split("/")[-1]
|
return full_url.split("/")[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def get_job_id():
|
||||||
|
return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(92))
|
||||||
|
21
tests/test_glacier/test_glacier_archives.py
Normal file
21
tests/test_glacier/test_glacier_archives.py
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from tempfile import NamedTemporaryFile
|
||||||
|
import boto.glacier
|
||||||
|
import sure # noqa
|
||||||
|
|
||||||
|
from moto import mock_glacier
|
||||||
|
|
||||||
|
|
||||||
|
@mock_glacier
|
||||||
|
def test_create_and_delete_archive():
|
||||||
|
the_file = NamedTemporaryFile(delete=False)
|
||||||
|
the_file.write("some stuff")
|
||||||
|
the_file.close()
|
||||||
|
|
||||||
|
conn = boto.glacier.connect_to_region("us-west-2")
|
||||||
|
vault = conn.create_vault("my_vault")
|
||||||
|
|
||||||
|
archive_id = vault.upload_archive(the_file.name)
|
||||||
|
|
||||||
|
vault.delete_archive(archive_id)
|
94
tests/test_glacier/test_glacier_jobs.py
Normal file
94
tests/test_glacier/test_glacier_jobs.py
Normal file
@ -0,0 +1,94 @@
|
|||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
from boto.glacier.layer1 import Layer1
|
||||||
|
import sure # noqa
|
||||||
|
|
||||||
|
from moto import mock_glacier
|
||||||
|
|
||||||
|
|
||||||
|
@mock_glacier
|
||||||
|
def test_init_glacier_job():
|
||||||
|
conn = Layer1(region_name="us-west-2")
|
||||||
|
vault_name = "my_vault"
|
||||||
|
conn.create_vault(vault_name)
|
||||||
|
archive_id = conn.upload_archive(vault_name, "some stuff", "", "", "some description")
|
||||||
|
|
||||||
|
job_response = conn.initiate_job(vault_name, {
|
||||||
|
"ArchiveId": archive_id,
|
||||||
|
"Type": "archive-retrieval",
|
||||||
|
})
|
||||||
|
job_id = job_response['JobId']
|
||||||
|
job_response['Location'].should.equal("//vaults/my_vault/jobs/{0}".format(job_id))
|
||||||
|
|
||||||
|
|
||||||
|
@mock_glacier
|
||||||
|
def test_describe_job():
|
||||||
|
conn = Layer1(region_name="us-west-2")
|
||||||
|
vault_name = "my_vault"
|
||||||
|
conn.create_vault(vault_name)
|
||||||
|
archive_id = conn.upload_archive(vault_name, "some stuff", "", "", "some description")
|
||||||
|
job_response = conn.initiate_job(vault_name, {
|
||||||
|
"ArchiveId": archive_id,
|
||||||
|
"Type": "archive-retrieval",
|
||||||
|
})
|
||||||
|
job_id = job_response['JobId']
|
||||||
|
|
||||||
|
job = conn.describe_job(vault_name, job_id)
|
||||||
|
json.loads(job.read()).should.equal({
|
||||||
|
'CompletionDate': '2013-03-20T17:03:43.221Z',
|
||||||
|
'VaultARN': None,
|
||||||
|
'RetrievalByteRange': None,
|
||||||
|
'SHA256TreeHash': None,
|
||||||
|
'Completed': True,
|
||||||
|
'InventorySizeInBytes': '0',
|
||||||
|
'JobId': job_id,
|
||||||
|
'Action': 'InventoryRetrieval',
|
||||||
|
'JobDescription': None,
|
||||||
|
'SNSTopic': None,
|
||||||
|
'ArchiveSizeInBytes': 0,
|
||||||
|
'ArchiveId': archive_id,
|
||||||
|
'ArchiveSHA256TreeHash': None,
|
||||||
|
'CreationDate': '2013-03-20T17:03:43.221Z',
|
||||||
|
'StatusMessage': None,
|
||||||
|
'StatusCode': 'Succeeded',
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@mock_glacier
|
||||||
|
def test_list_glacier_jobs():
|
||||||
|
conn = Layer1(region_name="us-west-2")
|
||||||
|
vault_name = "my_vault"
|
||||||
|
conn.create_vault(vault_name)
|
||||||
|
archive_id1 = conn.upload_archive(vault_name, "some stuff", "", "", "some description")['ArchiveId']
|
||||||
|
archive_id2 = conn.upload_archive(vault_name, "some other stuff", "", "", "some description")['ArchiveId']
|
||||||
|
|
||||||
|
conn.initiate_job(vault_name, {
|
||||||
|
"ArchiveId": archive_id1,
|
||||||
|
"Type": "archive-retrieval",
|
||||||
|
})
|
||||||
|
conn.initiate_job(vault_name, {
|
||||||
|
"ArchiveId": archive_id2,
|
||||||
|
"Type": "archive-retrieval",
|
||||||
|
})
|
||||||
|
|
||||||
|
jobs = conn.list_jobs(vault_name)
|
||||||
|
len(jobs['JobList']).should.equal(2)
|
||||||
|
|
||||||
|
|
||||||
|
@mock_glacier
|
||||||
|
def test_get_job_output():
|
||||||
|
conn = Layer1(region_name="us-west-2")
|
||||||
|
vault_name = "my_vault"
|
||||||
|
conn.create_vault(vault_name)
|
||||||
|
archive_response = conn.upload_archive(vault_name, "some stuff", "", "", "some description")
|
||||||
|
archive_id = archive_response['ArchiveId']
|
||||||
|
job_response = conn.initiate_job(vault_name, {
|
||||||
|
"ArchiveId": archive_id,
|
||||||
|
"Type": "archive-retrieval",
|
||||||
|
})
|
||||||
|
job_id = job_response['JobId']
|
||||||
|
|
||||||
|
output = conn.get_job_output(vault_name, job_id)
|
||||||
|
output.read().should.equal("some stuff")
|
@ -18,4 +18,4 @@ def test_list_vaults():
|
|||||||
|
|
||||||
res = test_client.get('/1234bcd/vaults')
|
res = test_client.get('/1234bcd/vaults')
|
||||||
|
|
||||||
json.loads(res.data).should.equal({u'Marker': None, u'VaultList': []})
|
json.loads(res.data.decode("utf-8")).should.equal({u'Marker': None, u'VaultList': []})
|
||||||
|
Loading…
Reference in New Issue
Block a user