diff --git a/moto/glacier/models.py b/moto/glacier/models.py index 9e91ea3a5..96f10b106 100644 --- a/moto/glacier/models.py +++ b/moto/glacier/models.py @@ -6,7 +6,7 @@ import datetime from boto3 import Session -from moto.core import BaseBackend, BaseModel +from moto.core import ACCOUNT_ID, BaseBackend, BaseModel from .utils import get_job_id @@ -101,8 +101,8 @@ class Vault(BaseModel): @property def arn(self): - return "arn:aws:glacier:{0}:012345678901:vaults/{1}".format( - self.region, self.vault_name + return "arn:aws:glacier:{0}:{1}:vaults/{2}".format( + self.region, ACCOUNT_ID, self.vault_name ) def to_dict(self): @@ -122,6 +122,7 @@ class Vault(BaseModel): def create_archive(self, body, description): archive_id = hashlib.md5(body).hexdigest() self.archives[archive_id] = {} + self.archives[archive_id]["archive_id"] = archive_id self.archives[archive_id]["body"] = body self.archives[archive_id]["size"] = len(body) self.archives[archive_id]["sha256"] = hashlib.sha256(body).hexdigest() @@ -129,7 +130,7 @@ class Vault(BaseModel): "%Y-%m-%dT%H:%M:%S.000Z" ) self.archives[archive_id]["description"] = description - return archive_id + return self.archives[archive_id] def get_archive_body(self, archive_id): return self.archives[archive_id]["body"] @@ -204,7 +205,7 @@ class GlacierBackend(BaseBackend): def create_vault(self, vault_name): self.vaults[vault_name] = Vault(vault_name, self.region_name) - def list_vaules(self): + def list_vaults(self): return self.vaults.values() def delete_vault(self, vault_name): @@ -215,10 +216,18 @@ class GlacierBackend(BaseBackend): job_id = vault.initiate_job(job_type, tier, archive_id) return job_id + def describe_job(self, vault_name, archive_id): + vault = self.get_vault(vault_name) + return vault.describe_job(archive_id) + def list_jobs(self, vault_name): vault = self.get_vault(vault_name) return vault.list_jobs() + def upload_archive(self, vault_name, body, description): + vault = self.get_vault(vault_name) + return vault.create_archive(body, description) + glacier_backends = {} for region in Session().get_available_regions("glacier"): diff --git a/moto/glacier/responses.py b/moto/glacier/responses.py index dc8e60df0..84cc4259f 100644 --- a/moto/glacier/responses.py +++ b/moto/glacier/responses.py @@ -3,24 +3,22 @@ from __future__ import unicode_literals import json from urllib.parse import urlparse, parse_qs -from moto.core.responses import _TemplateEnvironmentMixin +from moto.core.responses import BaseResponse from .models import glacier_backends -from .utils import region_from_glacier_url, vault_from_glacier_url +from .utils import vault_from_glacier_url -class GlacierResponse(_TemplateEnvironmentMixin): - def __init__(self, backend): - super(GlacierResponse, self).__init__() - self.backend = backend +class GlacierResponse(BaseResponse): + @property + def glacier_backend(self): + return glacier_backends[self.region] - @classmethod - def all_vault_response(clazz, request, full_url, headers): - region_name = region_from_glacier_url(full_url) - response_instance = GlacierResponse(glacier_backends[region_name]) - return response_instance._all_vault_response(request, full_url, headers) + def all_vault_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + return self._all_vault_response(request, full_url, headers) def _all_vault_response(self, request, full_url, headers): - vaults = self.backend.list_vaules() + vaults = self.glacier_backend.list_vaults() response = json.dumps( {"Marker": None, "VaultList": [vault.to_dict() for vault in vaults]} ) @@ -28,11 +26,9 @@ class GlacierResponse(_TemplateEnvironmentMixin): headers["content-type"] = "application/json" return 200, headers, response - @classmethod - def vault_response(clazz, request, full_url, headers): - region_name = region_from_glacier_url(full_url) - response_instance = GlacierResponse(glacier_backends[region_name]) - return response_instance._vault_response(request, full_url, headers) + def vault_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + return self._vault_response(request, full_url, headers) def _vault_response(self, request, full_url, headers): method = request.method @@ -48,23 +44,21 @@ class GlacierResponse(_TemplateEnvironmentMixin): return self._vault_response_delete(vault_name, querystring, headers) def _vault_response_get(self, vault_name, querystring, headers): - vault = self.backend.get_vault(vault_name) + vault = self.glacier_backend.get_vault(vault_name) headers["content-type"] = "application/json" return 200, headers, json.dumps(vault.to_dict()) def _vault_response_put(self, vault_name, querystring, headers): - self.backend.create_vault(vault_name) + self.glacier_backend.create_vault(vault_name) return 201, headers, "" def _vault_response_delete(self, vault_name, querystring, headers): - self.backend.delete_vault(vault_name) + self.glacier_backend.delete_vault(vault_name) return 204, headers, "" - @classmethod - def vault_archive_response(clazz, request, full_url, headers): - region_name = region_from_glacier_url(full_url) - response_instance = GlacierResponse(glacier_backends[region_name]) - return response_instance._vault_archive_response(request, full_url, headers) + def vault_archive_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + return self._vault_archive_response(request, full_url, headers) def _vault_archive_response(self, request, full_url, headers): method = request.method @@ -89,18 +83,14 @@ class GlacierResponse(_TemplateEnvironmentMixin): def _vault_archive_response_post( self, vault_name, body, description, querystring, headers ): - vault = self.backend.get_vault(vault_name) - vault_id = vault.create_archive(body, description) - headers["x-amz-archive-id"] = vault_id + vault = self.glacier_backend.upload_archive(vault_name, body, description) + headers["x-amz-archive-id"] = vault["archive_id"] + headers["x-amz-sha256-tree-hash"] = vault["sha256"] return 201, headers, "" - @classmethod - def vault_archive_individual_response(clazz, request, full_url, headers): - region_name = region_from_glacier_url(full_url) - response_instance = GlacierResponse(glacier_backends[region_name]) - return response_instance._vault_archive_individual_response( - request, full_url, headers - ) + def vault_archive_individual_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + return self._vault_archive_individual_response(request, full_url, headers) def _vault_archive_individual_response(self, request, full_url, headers): method = request.method @@ -108,15 +98,13 @@ class GlacierResponse(_TemplateEnvironmentMixin): archive_id = full_url.split("/")[-1] if method == "DELETE": - vault = self.backend.get_vault(vault_name) + vault = self.glacier_backend.get_vault(vault_name) vault.delete_archive(archive_id) return 204, headers, "" - @classmethod - def vault_jobs_response(clazz, request, full_url, headers): - region_name = region_from_glacier_url(full_url) - response_instance = GlacierResponse(glacier_backends[region_name]) - return response_instance._vault_jobs_response(request, full_url, headers) + def vault_jobs_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + return self._vault_jobs_response(request, full_url, headers) def _vault_jobs_response(self, request, full_url, headers): method = request.method @@ -128,7 +116,7 @@ class GlacierResponse(_TemplateEnvironmentMixin): vault_name = full_url.split("/")[-2] if method == "GET": - jobs = self.backend.list_jobs(vault_name) + jobs = self.glacier_backend.list_jobs(vault_name) headers["content-type"] = "application/json" return ( 200, @@ -147,39 +135,34 @@ class GlacierResponse(_TemplateEnvironmentMixin): tier = json_body["Tier"] else: tier = "Standard" - job_id = self.backend.initiate_job(vault_name, job_type, tier, archive_id) + job_id = self.glacier_backend.initiate_job( + vault_name, job_type, tier, archive_id + ) headers["x-amz-job-id"] = job_id headers["Location"] = "/{0}/vaults/{1}/jobs/{2}".format( account_id, vault_name, job_id ) return 202, headers, "" - @classmethod - def vault_jobs_individual_response(clazz, request, full_url, headers): - region_name = region_from_glacier_url(full_url) - response_instance = GlacierResponse(glacier_backends[region_name]) - return response_instance._vault_jobs_individual_response( - request, full_url, headers - ) + def vault_jobs_individual_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + return self._vault_jobs_individual_response(request, full_url, headers) def _vault_jobs_individual_response(self, request, full_url, headers): vault_name = full_url.split("/")[-3] archive_id = full_url.split("/")[-1] - vault = self.backend.get_vault(vault_name) - job = vault.describe_job(archive_id) + job = self.glacier_backend.describe_job(vault_name, archive_id) return 200, headers, json.dumps(job.to_dict()) - @classmethod - def vault_jobs_output_response(clazz, request, full_url, headers): - region_name = region_from_glacier_url(full_url) - response_instance = GlacierResponse(glacier_backends[region_name]) - return response_instance._vault_jobs_output_response(request, full_url, headers) + def vault_jobs_output_response(self, request, full_url, headers): + self.setup_class(request, full_url, headers) + return self._vault_jobs_output_response(request, full_url, headers) def _vault_jobs_output_response(self, request, full_url, headers): vault_name = full_url.split("/")[-4] job_id = full_url.split("/")[-2] - vault = self.backend.get_vault(vault_name) + vault = self.glacier_backend.get_vault(vault_name) if vault.job_ready(job_id): output = vault.get_job_output(job_id) if isinstance(output, dict): diff --git a/moto/glacier/urls.py b/moto/glacier/urls.py index 85a64881a..3c57a5940 100644 --- a/moto/glacier/urls.py +++ b/moto/glacier/urls.py @@ -3,12 +3,14 @@ from .responses import GlacierResponse url_bases = ["https?://glacier.(.+).amazonaws.com"] +response = GlacierResponse() + url_paths = { - "{0}/(?P.+)/vaults$": GlacierResponse.all_vault_response, - "{0}/(?P.+)/vaults/(?P[^/]+)$": GlacierResponse.vault_response, - "{0}/(?P.+)/vaults/(?P.+)/archives$": GlacierResponse.vault_archive_response, - "{0}/(?P.+)/vaults/(?P.+)/archives/(?P.+)$": GlacierResponse.vault_archive_individual_response, - "{0}/(?P.+)/vaults/(?P.+)/jobs$": GlacierResponse.vault_jobs_response, - "{0}/(?P.+)/vaults/(?P.+)/jobs/(?P[^/.]+)$": GlacierResponse.vault_jobs_individual_response, - "{0}/(?P.+)/vaults/(?P.+)/jobs/(?P.+)/output$": GlacierResponse.vault_jobs_output_response, + "{0}/(?P.+)/vaults$": response.all_vault_response, + "{0}/(?P.+)/vaults/(?P[^/]+)$": response.vault_response, + "{0}/(?P.+)/vaults/(?P.+)/archives$": response.vault_archive_response, + "{0}/(?P.+)/vaults/(?P.+)/archives/(?P.+)$": response.vault_archive_individual_response, + "{0}/(?P.+)/vaults/(?P.+)/jobs$": response.vault_jobs_response, + "{0}/(?P.+)/vaults/(?P.+)/jobs/(?P[^/.]+)$": response.vault_jobs_individual_response, + "{0}/(?P.+)/vaults/(?P.+)/jobs/(?P.+)/output$": response.vault_jobs_output_response, } diff --git a/moto/glacier/utils.py b/moto/glacier/utils.py index be7c38774..8c1ec9db3 100644 --- a/moto/glacier/utils.py +++ b/moto/glacier/utils.py @@ -1,17 +1,6 @@ import random import string -from urllib.parse import urlparse - - -def region_from_glacier_url(url): - domain = urlparse(url).netloc - - if "." in domain: - return domain.split(".")[1] - else: - return "us-east-1" - def vault_from_glacier_url(full_url): return full_url.split("/")[-1] diff --git a/tests/test_glacier/test_glacier_archives.py b/tests/test_glacier/test_glacier_archives.py index ec43e613c..e99d785de 100644 --- a/tests/test_glacier/test_glacier_archives.py +++ b/tests/test_glacier/test_glacier_archives.py @@ -1,10 +1,12 @@ from __future__ import unicode_literals from tempfile import NamedTemporaryFile +import boto3 import boto.glacier import sure # noqa +import pytest -from moto import mock_glacier_deprecated +from moto import mock_glacier_deprecated, mock_glacier @mock_glacier_deprecated @@ -19,3 +21,43 @@ def test_create_and_delete_archive(): archive_id = vault.upload_archive(the_file.name) vault.delete_archive(archive_id) + + +@mock_glacier +def test_upload_archive(): + client = boto3.client("glacier", region_name="us-west-2") + client.create_vault(vaultName="asdf") + + res = client.upload_archive( + vaultName="asdf", archiveDescription="my archive", body=b"body of archive" + ) + res["ResponseMetadata"]["HTTPStatusCode"].should.equal(201) + headers = res["ResponseMetadata"]["HTTPHeaders"] + + headers.should.have.key("x-amz-archive-id") + headers.should.have.key("x-amz-sha256-tree-hash") + + res.should.have.key("checksum") + res.should.have.key("archiveId") + + +@mock_glacier +def test_delete_archive(): + client = boto3.client("glacier", region_name="us-west-2") + client.create_vault(vaultName="asdf") + + archive = client.upload_archive(vaultName="asdf", body=b"body of archive") + + delete = client.delete_archive(vaultName="asdf", archiveId=archive["archiveId"]) + delete["ResponseMetadata"]["HTTPStatusCode"].should.equal(204) + + with pytest.raises(Exception): + # Not ideal - but this will throw an error if the archvie does not exist + # Which is a good indication that the deletion went through + client.initiate_job( + vaultName="myname", + jobParameters={ + "ArchiveId": archive["archiveId"], + "Type": "archive-retrieval", + }, + ) diff --git a/tests/test_glacier/test_glacier_jobs.py b/tests/test_glacier/test_glacier_jobs.py index cba2c1a27..9f08ac2ea 100644 --- a/tests/test_glacier/test_glacier_jobs.py +++ b/tests/test_glacier/test_glacier_jobs.py @@ -1,14 +1,18 @@ from __future__ import unicode_literals +import boto3 import json import time from boto.glacier.layer1 import Layer1 import sure # noqa +import time -from moto import mock_glacier_deprecated +from moto import mock_glacier_deprecated, mock_glacier +from moto.core import ACCOUNT_ID +# Has boto3 equivalent @mock_glacier_deprecated def test_init_glacier_job(): conn = Layer1(region_name="us-west-2") @@ -25,6 +29,33 @@ def test_init_glacier_job(): job_response["Location"].should.equal("//vaults/my_vault/jobs/{0}".format(job_id)) +@mock_glacier +def test_initiate_job(): + client = boto3.client("glacier", region_name="us-west-2") + client.create_vault(vaultName="myname") + + archive = client.upload_archive(vaultName="myname", body=b"body of archive") + + job = client.initiate_job( + vaultName="myname", + jobParameters={"ArchiveId": archive["archiveId"], "Type": "archive-retrieval"}, + ) + job["ResponseMetadata"]["HTTPStatusCode"].should.equal(202) + + headers = job["ResponseMetadata"]["HTTPHeaders"] + headers.should.have.key("x-amz-job-id") + # Should be an exact match, but Flask adds 'http' to the start of the Location-header + headers.should.have.key("location").match( + "//vaults/myname/jobs/" + headers["x-amz-job-id"] + ) + + # Don't think this is correct - the spec says no body is returned, only headers + # https://docs.aws.amazon.com/amazonglacier/latest/dev/api-initiate-job-post.html + job.should.have.key("jobId") + job.should.have.key("location") + + +# Has boto3 equivalent @mock_glacier_deprecated def test_describe_job(): conn = Layer1(region_name="us-west-2") @@ -44,10 +75,39 @@ def test_describe_job(): joboutput.should.have.key("Tier").which.should.equal("Standard") joboutput.should.have.key("StatusCode").which.should.equal("InProgress") joboutput.should.have.key("VaultARN").which.should.equal( - "arn:aws:glacier:us-west-2:012345678901:vaults/my_vault" + f"arn:aws:glacier:us-west-2:{ACCOUNT_ID}:vaults/my_vault" ) +@mock_glacier +def test_describe_job_boto3(): + client = boto3.client("glacier", region_name="us-west-2") + client.create_vault(vaultName="myname") + + archive = client.upload_archive(vaultName="myname", body=b"body of archive") + + job = client.initiate_job( + vaultName="myname", + jobParameters={"ArchiveId": archive["archiveId"], "Type": "archive-retrieval"}, + ) + job_id = job["jobId"] + + describe = client.describe_job(vaultName="myname", jobId=job_id) + describe.should.have.key("JobId").equal(job_id) + describe.should.have.key("Action").equal("ArchiveRetrieval") + describe.should.have.key("ArchiveId").equal(archive["archiveId"]) + describe.should.have.key("VaultARN").equal( + f"arn:aws:glacier:us-west-2:{ACCOUNT_ID}:vaults/myname" + ) + describe.should.have.key("CreationDate") + describe.should.have.key("Completed").equal(False) + describe.should.have.key("StatusCode").equal("InProgress") + describe.should.have.key("ArchiveSizeInBytes").equal(0) + describe.should.have.key("InventorySizeInBytes").equal(0) + describe.should.have.key("Tier").equal("Standard") + + +# Has boto3 equivalent @mock_glacier_deprecated def test_list_glacier_jobs(): conn = Layer1(region_name="us-west-2") @@ -71,6 +131,50 @@ def test_list_glacier_jobs(): len(jobs["JobList"]).should.equal(2) +@mock_glacier +def test_list_jobs(): + client = boto3.client("glacier", region_name="us-west-2") + client.create_vault(vaultName="myname") + + archive1 = client.upload_archive(vaultName="myname", body=b"first archive") + archive2 = client.upload_archive(vaultName="myname", body=b"second archive") + + job1 = client.initiate_job( + vaultName="myname", + jobParameters={"ArchiveId": archive1["archiveId"], "Type": "archive-retrieval"}, + ) + + job2 = client.initiate_job( + vaultName="myname", + jobParameters={"ArchiveId": archive2["archiveId"], "Type": "archive-retrieval"}, + ) + jobs = client.list_jobs(vaultName="myname")["JobList"] + + # Verify the created jobs are in this list + found_jobs = [j["JobId"] for j in jobs] + found_jobs.should.contain(job1["jobId"]) + found_jobs.should.contain(job2["jobId"]) + + found_job1 = [j for j in jobs if j["JobId"] == job1["jobId"]][0] + found_job1.should.have.key("ArchiveId").equal(archive1["archiveId"]) + found_job2 = [j for j in jobs if j["JobId"] == job2["jobId"]][0] + found_job2.should.have.key("ArchiveId").equal(archive2["archiveId"]) + + # Verify all jobs follow the correct format + for job in jobs: + job.should.have.key("JobId") + job.should.have.key("Action") + job.should.have.key("ArchiveId") + job.should.have.key("VaultARN") + job.should.have.key("CreationDate") + job.should.have.key("ArchiveSizeInBytes") + job.should.have.key("Completed") + job.should.have.key("StatusCode") + job.should.have.key("InventorySizeInBytes") + job.should.have.key("Tier") + + +# Has boto3 equivalent @mock_glacier_deprecated def test_get_job_output(): conn = Layer1(region_name="us-west-2") @@ -89,3 +193,33 @@ def test_get_job_output(): output = conn.get_job_output(vault_name, job_id) output.read().decode("utf-8").should.equal("some stuff") + + +@mock_glacier +def test_get_job_output_boto3(): + client = boto3.client("glacier", region_name="us-west-2") + client.create_vault(vaultName="myname") + + archive = client.upload_archive(vaultName="myname", body=b"contents of archive") + + job = client.initiate_job( + vaultName="myname", + jobParameters={"ArchiveId": archive["archiveId"], "Type": "archive-retrieval"}, + ) + + output = None + start = time.time() + while (time.time() - start) < 10: + try: + output = client.get_job_output(vaultName="myname", jobId=job["jobId"]) + break + except Exception: + time.sleep(1) + + output.shouldnt.be.none + output.should.have.key("status").equal(200) + output.should.have.key("contentType").equal("application/octet-stream") + output.should.have.key("body") + + body = output["body"].read().decode("utf-8") + body.should.equal("contents of archive") diff --git a/tests/test_glacier/test_glacier_vaults.py b/tests/test_glacier/test_glacier_vaults.py index 1964b52db..6c5e89227 100644 --- a/tests/test_glacier/test_glacier_vaults.py +++ b/tests/test_glacier/test_glacier_vaults.py @@ -2,11 +2,15 @@ from __future__ import unicode_literals import boto.glacier import boto3 +import pytest import sure # noqa from moto import mock_glacier_deprecated, mock_glacier +from moto.core import ACCOUNT_ID +from uuid import uuid4 +# Has boto3 equivalent @mock_glacier_deprecated def test_create_vault(): conn = boto.glacier.connect_to_region("us-west-2") @@ -18,6 +22,24 @@ def test_create_vault(): vaults[0].name.should.equal("my_vault") +@mock_glacier +def test_describe_vault(): + client = boto3.client("glacier", region_name="us-west-2") + + client.create_vault(vaultName="myvault") + + describe = client.describe_vault(vaultName="myvault") + describe.should.have.key("NumberOfArchives").equal(0) + describe.should.have.key("SizeInBytes").equal(0) + describe.should.have.key("LastInventoryDate") + describe.should.have.key("CreationDate") + describe.should.have.key("VaultName").equal("myvault") + describe.should.have.key("VaultARN").equal( + f"arn:aws:glacier:us-west-2:{ACCOUNT_ID}:vaults/myvault" + ) + + +# Has boto3 equivalent @mock_glacier_deprecated def test_delete_vault(): conn = boto.glacier.connect_to_region("us-west-2") @@ -32,6 +54,61 @@ def test_delete_vault(): vaults.should.have.length_of(0) +@mock_glacier +def test_delete_vault_boto3(): + client = boto3.client("glacier", region_name="us-west-2") + + client.create_vault(vaultName="myvault") + + client.delete_vault(vaultName="myvault") + + with pytest.raises(Exception): + client.describe_vault(vaultName="myvault") + + +@mock_glacier +def test_list_vaults(): + client = boto3.client("glacier", region_name="us-west-2") + + vault1_name = str(uuid4())[0:6] + vault2_name = str(uuid4())[0:6] + + # Verify we cannot find these vaults yet + vaults = client.list_vaults()["VaultList"] + found_vaults = [v["VaultName"] for v in vaults] + found_vaults.shouldnt.contain(vault1_name) + found_vaults.shouldnt.contain(vault2_name) + + client.create_vault(vaultName=vault1_name) + client.create_vault(vaultName=vault2_name) + + # Verify we can find the created vaults + vaults = client.list_vaults()["VaultList"] + found_vaults = [v["VaultName"] for v in vaults] + found_vaults.should.contain(vault1_name) + found_vaults.should.contain(vault2_name) + + # Verify all the vaults are in the correct format + for vault in vaults: + vault.should.have.key("NumberOfArchives").equal(0) + vault.should.have.key("SizeInBytes").equal(0) + vault.should.have.key("LastInventoryDate") + vault.should.have.key("CreationDate") + vault.should.have.key("VaultName") + vault_name = vault["VaultName"] + vault.should.have.key("VaultARN").equal( + f"arn:aws:glacier:us-west-2:{ACCOUNT_ID}:vaults/{vault_name}" + ) + + # Verify a deleted vault is no longer returned + client.delete_vault(vaultName=vault1_name) + + vaults = client.list_vaults()["VaultList"] + found_vaults = [v["VaultName"] for v in vaults] + found_vaults.shouldnt.contain(vault1_name) + found_vaults.should.contain(vault2_name) + + @mock_glacier def test_vault_name_with_special_characters(): vault_name = "Vault.name-with_Special.characters"