Glacier - rewrite deprecated tests (#4336)

This commit is contained in:
Bert Blommers 2021-09-23 17:22:16 +00:00 committed by GitHub
parent 953be7682b
commit 8c36da14c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 320 additions and 84 deletions

View File

@ -6,7 +6,7 @@ import datetime
from boto3 import Session
from moto.core import BaseBackend, BaseModel
from moto.core import ACCOUNT_ID, BaseBackend, BaseModel
from .utils import get_job_id
@ -101,8 +101,8 @@ class Vault(BaseModel):
@property
def arn(self):
return "arn:aws:glacier:{0}:012345678901:vaults/{1}".format(
self.region, self.vault_name
return "arn:aws:glacier:{0}:{1}:vaults/{2}".format(
self.region, ACCOUNT_ID, self.vault_name
)
def to_dict(self):
@ -122,6 +122,7 @@ class Vault(BaseModel):
def create_archive(self, body, description):
archive_id = hashlib.md5(body).hexdigest()
self.archives[archive_id] = {}
self.archives[archive_id]["archive_id"] = archive_id
self.archives[archive_id]["body"] = body
self.archives[archive_id]["size"] = len(body)
self.archives[archive_id]["sha256"] = hashlib.sha256(body).hexdigest()
@ -129,7 +130,7 @@ class Vault(BaseModel):
"%Y-%m-%dT%H:%M:%S.000Z"
)
self.archives[archive_id]["description"] = description
return archive_id
return self.archives[archive_id]
def get_archive_body(self, archive_id):
return self.archives[archive_id]["body"]
@ -204,7 +205,7 @@ class GlacierBackend(BaseBackend):
def create_vault(self, vault_name):
self.vaults[vault_name] = Vault(vault_name, self.region_name)
def list_vaules(self):
def list_vaults(self):
return self.vaults.values()
def delete_vault(self, vault_name):
@ -215,10 +216,18 @@ class GlacierBackend(BaseBackend):
job_id = vault.initiate_job(job_type, tier, archive_id)
return job_id
def describe_job(self, vault_name, archive_id):
vault = self.get_vault(vault_name)
return vault.describe_job(archive_id)
def list_jobs(self, vault_name):
vault = self.get_vault(vault_name)
return vault.list_jobs()
def upload_archive(self, vault_name, body, description):
vault = self.get_vault(vault_name)
return vault.create_archive(body, description)
glacier_backends = {}
for region in Session().get_available_regions("glacier"):

View File

@ -3,24 +3,22 @@ from __future__ import unicode_literals
import json
from urllib.parse import urlparse, parse_qs
from moto.core.responses import _TemplateEnvironmentMixin
from moto.core.responses import BaseResponse
from .models import glacier_backends
from .utils import region_from_glacier_url, vault_from_glacier_url
from .utils import vault_from_glacier_url
class GlacierResponse(_TemplateEnvironmentMixin):
def __init__(self, backend):
super(GlacierResponse, self).__init__()
self.backend = backend
class GlacierResponse(BaseResponse):
@property
def glacier_backend(self):
return glacier_backends[self.region]
@classmethod
def all_vault_response(clazz, request, full_url, headers):
region_name = region_from_glacier_url(full_url)
response_instance = GlacierResponse(glacier_backends[region_name])
return response_instance._all_vault_response(request, full_url, headers)
def all_vault_response(self, request, full_url, headers):
self.setup_class(request, full_url, headers)
return self._all_vault_response(request, full_url, headers)
def _all_vault_response(self, request, full_url, headers):
vaults = self.backend.list_vaules()
vaults = self.glacier_backend.list_vaults()
response = json.dumps(
{"Marker": None, "VaultList": [vault.to_dict() for vault in vaults]}
)
@ -28,11 +26,9 @@ class GlacierResponse(_TemplateEnvironmentMixin):
headers["content-type"] = "application/json"
return 200, headers, response
@classmethod
def vault_response(clazz, request, full_url, headers):
region_name = region_from_glacier_url(full_url)
response_instance = GlacierResponse(glacier_backends[region_name])
return response_instance._vault_response(request, full_url, headers)
def vault_response(self, request, full_url, headers):
self.setup_class(request, full_url, headers)
return self._vault_response(request, full_url, headers)
def _vault_response(self, request, full_url, headers):
method = request.method
@ -48,23 +44,21 @@ class GlacierResponse(_TemplateEnvironmentMixin):
return self._vault_response_delete(vault_name, querystring, headers)
def _vault_response_get(self, vault_name, querystring, headers):
vault = self.backend.get_vault(vault_name)
vault = self.glacier_backend.get_vault(vault_name)
headers["content-type"] = "application/json"
return 200, headers, json.dumps(vault.to_dict())
def _vault_response_put(self, vault_name, querystring, headers):
self.backend.create_vault(vault_name)
self.glacier_backend.create_vault(vault_name)
return 201, headers, ""
def _vault_response_delete(self, vault_name, querystring, headers):
self.backend.delete_vault(vault_name)
self.glacier_backend.delete_vault(vault_name)
return 204, headers, ""
@classmethod
def vault_archive_response(clazz, request, full_url, headers):
region_name = region_from_glacier_url(full_url)
response_instance = GlacierResponse(glacier_backends[region_name])
return response_instance._vault_archive_response(request, full_url, headers)
def vault_archive_response(self, request, full_url, headers):
self.setup_class(request, full_url, headers)
return self._vault_archive_response(request, full_url, headers)
def _vault_archive_response(self, request, full_url, headers):
method = request.method
@ -89,18 +83,14 @@ class GlacierResponse(_TemplateEnvironmentMixin):
def _vault_archive_response_post(
self, vault_name, body, description, querystring, headers
):
vault = self.backend.get_vault(vault_name)
vault_id = vault.create_archive(body, description)
headers["x-amz-archive-id"] = vault_id
vault = self.glacier_backend.upload_archive(vault_name, body, description)
headers["x-amz-archive-id"] = vault["archive_id"]
headers["x-amz-sha256-tree-hash"] = vault["sha256"]
return 201, headers, ""
@classmethod
def vault_archive_individual_response(clazz, request, full_url, headers):
region_name = region_from_glacier_url(full_url)
response_instance = GlacierResponse(glacier_backends[region_name])
return response_instance._vault_archive_individual_response(
request, full_url, headers
)
def vault_archive_individual_response(self, request, full_url, headers):
self.setup_class(request, full_url, headers)
return self._vault_archive_individual_response(request, full_url, headers)
def _vault_archive_individual_response(self, request, full_url, headers):
method = request.method
@ -108,15 +98,13 @@ class GlacierResponse(_TemplateEnvironmentMixin):
archive_id = full_url.split("/")[-1]
if method == "DELETE":
vault = self.backend.get_vault(vault_name)
vault = self.glacier_backend.get_vault(vault_name)
vault.delete_archive(archive_id)
return 204, headers, ""
@classmethod
def vault_jobs_response(clazz, request, full_url, headers):
region_name = region_from_glacier_url(full_url)
response_instance = GlacierResponse(glacier_backends[region_name])
return response_instance._vault_jobs_response(request, full_url, headers)
def vault_jobs_response(self, request, full_url, headers):
self.setup_class(request, full_url, headers)
return self._vault_jobs_response(request, full_url, headers)
def _vault_jobs_response(self, request, full_url, headers):
method = request.method
@ -128,7 +116,7 @@ class GlacierResponse(_TemplateEnvironmentMixin):
vault_name = full_url.split("/")[-2]
if method == "GET":
jobs = self.backend.list_jobs(vault_name)
jobs = self.glacier_backend.list_jobs(vault_name)
headers["content-type"] = "application/json"
return (
200,
@ -147,39 +135,34 @@ class GlacierResponse(_TemplateEnvironmentMixin):
tier = json_body["Tier"]
else:
tier = "Standard"
job_id = self.backend.initiate_job(vault_name, job_type, tier, archive_id)
job_id = self.glacier_backend.initiate_job(
vault_name, job_type, tier, archive_id
)
headers["x-amz-job-id"] = job_id
headers["Location"] = "/{0}/vaults/{1}/jobs/{2}".format(
account_id, vault_name, job_id
)
return 202, headers, ""
@classmethod
def vault_jobs_individual_response(clazz, request, full_url, headers):
region_name = region_from_glacier_url(full_url)
response_instance = GlacierResponse(glacier_backends[region_name])
return response_instance._vault_jobs_individual_response(
request, full_url, headers
)
def vault_jobs_individual_response(self, request, full_url, headers):
self.setup_class(request, full_url, headers)
return self._vault_jobs_individual_response(request, full_url, headers)
def _vault_jobs_individual_response(self, request, full_url, headers):
vault_name = full_url.split("/")[-3]
archive_id = full_url.split("/")[-1]
vault = self.backend.get_vault(vault_name)
job = vault.describe_job(archive_id)
job = self.glacier_backend.describe_job(vault_name, archive_id)
return 200, headers, json.dumps(job.to_dict())
@classmethod
def vault_jobs_output_response(clazz, request, full_url, headers):
region_name = region_from_glacier_url(full_url)
response_instance = GlacierResponse(glacier_backends[region_name])
return response_instance._vault_jobs_output_response(request, full_url, headers)
def vault_jobs_output_response(self, request, full_url, headers):
self.setup_class(request, full_url, headers)
return self._vault_jobs_output_response(request, full_url, headers)
def _vault_jobs_output_response(self, request, full_url, headers):
vault_name = full_url.split("/")[-4]
job_id = full_url.split("/")[-2]
vault = self.backend.get_vault(vault_name)
vault = self.glacier_backend.get_vault(vault_name)
if vault.job_ready(job_id):
output = vault.get_job_output(job_id)
if isinstance(output, dict):

View File

@ -3,12 +3,14 @@ from .responses import GlacierResponse
url_bases = ["https?://glacier.(.+).amazonaws.com"]
response = GlacierResponse()
url_paths = {
"{0}/(?P<account_number>.+)/vaults$": GlacierResponse.all_vault_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>[^/]+)$": GlacierResponse.vault_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/archives$": GlacierResponse.vault_archive_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/archives/(?P<archive_id>.+)$": GlacierResponse.vault_archive_individual_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs$": GlacierResponse.vault_jobs_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs/(?P<job_id>[^/.]+)$": GlacierResponse.vault_jobs_individual_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs/(?P<job_id>.+)/output$": GlacierResponse.vault_jobs_output_response,
"{0}/(?P<account_number>.+)/vaults$": response.all_vault_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>[^/]+)$": response.vault_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/archives$": response.vault_archive_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/archives/(?P<archive_id>.+)$": response.vault_archive_individual_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs$": response.vault_jobs_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs/(?P<job_id>[^/.]+)$": response.vault_jobs_individual_response,
"{0}/(?P<account_number>.+)/vaults/(?P<vault_name>.+)/jobs/(?P<job_id>.+)/output$": response.vault_jobs_output_response,
}

View File

@ -1,17 +1,6 @@
import random
import string
from urllib.parse import urlparse
def region_from_glacier_url(url):
domain = urlparse(url).netloc
if "." in domain:
return domain.split(".")[1]
else:
return "us-east-1"
def vault_from_glacier_url(full_url):
return full_url.split("/")[-1]

View File

@ -1,10 +1,12 @@
from __future__ import unicode_literals
from tempfile import NamedTemporaryFile
import boto3
import boto.glacier
import sure # noqa
import pytest
from moto import mock_glacier_deprecated
from moto import mock_glacier_deprecated, mock_glacier
@mock_glacier_deprecated
@ -19,3 +21,43 @@ def test_create_and_delete_archive():
archive_id = vault.upload_archive(the_file.name)
vault.delete_archive(archive_id)
@mock_glacier
def test_upload_archive():
client = boto3.client("glacier", region_name="us-west-2")
client.create_vault(vaultName="asdf")
res = client.upload_archive(
vaultName="asdf", archiveDescription="my archive", body=b"body of archive"
)
res["ResponseMetadata"]["HTTPStatusCode"].should.equal(201)
headers = res["ResponseMetadata"]["HTTPHeaders"]
headers.should.have.key("x-amz-archive-id")
headers.should.have.key("x-amz-sha256-tree-hash")
res.should.have.key("checksum")
res.should.have.key("archiveId")
@mock_glacier
def test_delete_archive():
client = boto3.client("glacier", region_name="us-west-2")
client.create_vault(vaultName="asdf")
archive = client.upload_archive(vaultName="asdf", body=b"body of archive")
delete = client.delete_archive(vaultName="asdf", archiveId=archive["archiveId"])
delete["ResponseMetadata"]["HTTPStatusCode"].should.equal(204)
with pytest.raises(Exception):
# Not ideal - but this will throw an error if the archvie does not exist
# Which is a good indication that the deletion went through
client.initiate_job(
vaultName="myname",
jobParameters={
"ArchiveId": archive["archiveId"],
"Type": "archive-retrieval",
},
)

View File

@ -1,14 +1,18 @@
from __future__ import unicode_literals
import boto3
import json
import time
from boto.glacier.layer1 import Layer1
import sure # noqa
import time
from moto import mock_glacier_deprecated
from moto import mock_glacier_deprecated, mock_glacier
from moto.core import ACCOUNT_ID
# Has boto3 equivalent
@mock_glacier_deprecated
def test_init_glacier_job():
conn = Layer1(region_name="us-west-2")
@ -25,6 +29,33 @@ def test_init_glacier_job():
job_response["Location"].should.equal("//vaults/my_vault/jobs/{0}".format(job_id))
@mock_glacier
def test_initiate_job():
client = boto3.client("glacier", region_name="us-west-2")
client.create_vault(vaultName="myname")
archive = client.upload_archive(vaultName="myname", body=b"body of archive")
job = client.initiate_job(
vaultName="myname",
jobParameters={"ArchiveId": archive["archiveId"], "Type": "archive-retrieval"},
)
job["ResponseMetadata"]["HTTPStatusCode"].should.equal(202)
headers = job["ResponseMetadata"]["HTTPHeaders"]
headers.should.have.key("x-amz-job-id")
# Should be an exact match, but Flask adds 'http' to the start of the Location-header
headers.should.have.key("location").match(
"//vaults/myname/jobs/" + headers["x-amz-job-id"]
)
# Don't think this is correct - the spec says no body is returned, only headers
# https://docs.aws.amazon.com/amazonglacier/latest/dev/api-initiate-job-post.html
job.should.have.key("jobId")
job.should.have.key("location")
# Has boto3 equivalent
@mock_glacier_deprecated
def test_describe_job():
conn = Layer1(region_name="us-west-2")
@ -44,10 +75,39 @@ def test_describe_job():
joboutput.should.have.key("Tier").which.should.equal("Standard")
joboutput.should.have.key("StatusCode").which.should.equal("InProgress")
joboutput.should.have.key("VaultARN").which.should.equal(
"arn:aws:glacier:us-west-2:012345678901:vaults/my_vault"
f"arn:aws:glacier:us-west-2:{ACCOUNT_ID}:vaults/my_vault"
)
@mock_glacier
def test_describe_job_boto3():
client = boto3.client("glacier", region_name="us-west-2")
client.create_vault(vaultName="myname")
archive = client.upload_archive(vaultName="myname", body=b"body of archive")
job = client.initiate_job(
vaultName="myname",
jobParameters={"ArchiveId": archive["archiveId"], "Type": "archive-retrieval"},
)
job_id = job["jobId"]
describe = client.describe_job(vaultName="myname", jobId=job_id)
describe.should.have.key("JobId").equal(job_id)
describe.should.have.key("Action").equal("ArchiveRetrieval")
describe.should.have.key("ArchiveId").equal(archive["archiveId"])
describe.should.have.key("VaultARN").equal(
f"arn:aws:glacier:us-west-2:{ACCOUNT_ID}:vaults/myname"
)
describe.should.have.key("CreationDate")
describe.should.have.key("Completed").equal(False)
describe.should.have.key("StatusCode").equal("InProgress")
describe.should.have.key("ArchiveSizeInBytes").equal(0)
describe.should.have.key("InventorySizeInBytes").equal(0)
describe.should.have.key("Tier").equal("Standard")
# Has boto3 equivalent
@mock_glacier_deprecated
def test_list_glacier_jobs():
conn = Layer1(region_name="us-west-2")
@ -71,6 +131,50 @@ def test_list_glacier_jobs():
len(jobs["JobList"]).should.equal(2)
@mock_glacier
def test_list_jobs():
client = boto3.client("glacier", region_name="us-west-2")
client.create_vault(vaultName="myname")
archive1 = client.upload_archive(vaultName="myname", body=b"first archive")
archive2 = client.upload_archive(vaultName="myname", body=b"second archive")
job1 = client.initiate_job(
vaultName="myname",
jobParameters={"ArchiveId": archive1["archiveId"], "Type": "archive-retrieval"},
)
job2 = client.initiate_job(
vaultName="myname",
jobParameters={"ArchiveId": archive2["archiveId"], "Type": "archive-retrieval"},
)
jobs = client.list_jobs(vaultName="myname")["JobList"]
# Verify the created jobs are in this list
found_jobs = [j["JobId"] for j in jobs]
found_jobs.should.contain(job1["jobId"])
found_jobs.should.contain(job2["jobId"])
found_job1 = [j for j in jobs if j["JobId"] == job1["jobId"]][0]
found_job1.should.have.key("ArchiveId").equal(archive1["archiveId"])
found_job2 = [j for j in jobs if j["JobId"] == job2["jobId"]][0]
found_job2.should.have.key("ArchiveId").equal(archive2["archiveId"])
# Verify all jobs follow the correct format
for job in jobs:
job.should.have.key("JobId")
job.should.have.key("Action")
job.should.have.key("ArchiveId")
job.should.have.key("VaultARN")
job.should.have.key("CreationDate")
job.should.have.key("ArchiveSizeInBytes")
job.should.have.key("Completed")
job.should.have.key("StatusCode")
job.should.have.key("InventorySizeInBytes")
job.should.have.key("Tier")
# Has boto3 equivalent
@mock_glacier_deprecated
def test_get_job_output():
conn = Layer1(region_name="us-west-2")
@ -89,3 +193,33 @@ def test_get_job_output():
output = conn.get_job_output(vault_name, job_id)
output.read().decode("utf-8").should.equal("some stuff")
@mock_glacier
def test_get_job_output_boto3():
client = boto3.client("glacier", region_name="us-west-2")
client.create_vault(vaultName="myname")
archive = client.upload_archive(vaultName="myname", body=b"contents of archive")
job = client.initiate_job(
vaultName="myname",
jobParameters={"ArchiveId": archive["archiveId"], "Type": "archive-retrieval"},
)
output = None
start = time.time()
while (time.time() - start) < 10:
try:
output = client.get_job_output(vaultName="myname", jobId=job["jobId"])
break
except Exception:
time.sleep(1)
output.shouldnt.be.none
output.should.have.key("status").equal(200)
output.should.have.key("contentType").equal("application/octet-stream")
output.should.have.key("body")
body = output["body"].read().decode("utf-8")
body.should.equal("contents of archive")

View File

@ -2,11 +2,15 @@ from __future__ import unicode_literals
import boto.glacier
import boto3
import pytest
import sure # noqa
from moto import mock_glacier_deprecated, mock_glacier
from moto.core import ACCOUNT_ID
from uuid import uuid4
# Has boto3 equivalent
@mock_glacier_deprecated
def test_create_vault():
conn = boto.glacier.connect_to_region("us-west-2")
@ -18,6 +22,24 @@ def test_create_vault():
vaults[0].name.should.equal("my_vault")
@mock_glacier
def test_describe_vault():
client = boto3.client("glacier", region_name="us-west-2")
client.create_vault(vaultName="myvault")
describe = client.describe_vault(vaultName="myvault")
describe.should.have.key("NumberOfArchives").equal(0)
describe.should.have.key("SizeInBytes").equal(0)
describe.should.have.key("LastInventoryDate")
describe.should.have.key("CreationDate")
describe.should.have.key("VaultName").equal("myvault")
describe.should.have.key("VaultARN").equal(
f"arn:aws:glacier:us-west-2:{ACCOUNT_ID}:vaults/myvault"
)
# Has boto3 equivalent
@mock_glacier_deprecated
def test_delete_vault():
conn = boto.glacier.connect_to_region("us-west-2")
@ -32,6 +54,61 @@ def test_delete_vault():
vaults.should.have.length_of(0)
@mock_glacier
def test_delete_vault_boto3():
client = boto3.client("glacier", region_name="us-west-2")
client.create_vault(vaultName="myvault")
client.delete_vault(vaultName="myvault")
with pytest.raises(Exception):
client.describe_vault(vaultName="myvault")
@mock_glacier
def test_list_vaults():
client = boto3.client("glacier", region_name="us-west-2")
vault1_name = str(uuid4())[0:6]
vault2_name = str(uuid4())[0:6]
# Verify we cannot find these vaults yet
vaults = client.list_vaults()["VaultList"]
found_vaults = [v["VaultName"] for v in vaults]
found_vaults.shouldnt.contain(vault1_name)
found_vaults.shouldnt.contain(vault2_name)
client.create_vault(vaultName=vault1_name)
client.create_vault(vaultName=vault2_name)
# Verify we can find the created vaults
vaults = client.list_vaults()["VaultList"]
found_vaults = [v["VaultName"] for v in vaults]
found_vaults.should.contain(vault1_name)
found_vaults.should.contain(vault2_name)
# Verify all the vaults are in the correct format
for vault in vaults:
vault.should.have.key("NumberOfArchives").equal(0)
vault.should.have.key("SizeInBytes").equal(0)
vault.should.have.key("LastInventoryDate")
vault.should.have.key("CreationDate")
vault.should.have.key("VaultName")
vault_name = vault["VaultName"]
vault.should.have.key("VaultARN").equal(
f"arn:aws:glacier:us-west-2:{ACCOUNT_ID}:vaults/{vault_name}"
)
# Verify a deleted vault is no longer returned
client.delete_vault(vaultName=vault1_name)
vaults = client.list_vaults()["VaultList"]
found_vaults = [v["VaultName"] for v in vaults]
found_vaults.shouldnt.contain(vault1_name)
found_vaults.should.contain(vault2_name)
@mock_glacier
def test_vault_name_with_special_characters():
vault_name = "Vault.name-with_Special.characters"