Implemented S3 Public Access Block

This commit is contained in:
Mike Grima 2019-12-09 17:38:26 -08:00
parent 4d5bf1c5c6
commit 84ccdbd1cd
7 changed files with 353 additions and 33 deletions

View File

@ -5589,63 +5589,63 @@
- [X] delete_bucket_cors
- [ ] delete_bucket_encryption
- [ ] delete_bucket_inventory_configuration
- [ ] delete_bucket_lifecycle
- [X] delete_bucket_lifecycle
- [ ] delete_bucket_metrics_configuration
- [X] delete_bucket_policy
- [ ] delete_bucket_replication
- [X] delete_bucket_tagging
- [ ] delete_bucket_website
- [ ] delete_object
- [X] delete_object
- [ ] delete_object_tagging
- [ ] delete_objects
- [ ] delete_public_access_block
- [X] delete_objects
- [X] delete_public_access_block
- [ ] get_bucket_accelerate_configuration
- [X] get_bucket_acl
- [ ] get_bucket_analytics_configuration
- [ ] get_bucket_cors
- [X] get_bucket_cors
- [ ] get_bucket_encryption
- [ ] get_bucket_inventory_configuration
- [ ] get_bucket_lifecycle
- [ ] get_bucket_lifecycle_configuration
- [ ] get_bucket_location
- [ ] get_bucket_logging
- [X] get_bucket_lifecycle
- [X] get_bucket_lifecycle_configuration
- [X] get_bucket_location
- [X] get_bucket_logging
- [ ] get_bucket_metrics_configuration
- [ ] get_bucket_notification
- [ ] get_bucket_notification_configuration
- [X] get_bucket_policy
- [ ] get_bucket_policy_status
- [X] get_bucket_policy_status
- [ ] get_bucket_replication
- [ ] get_bucket_request_payment
- [ ] get_bucket_tagging
- [X] get_bucket_tagging
- [X] get_bucket_versioning
- [ ] get_bucket_website
- [ ] get_object
- [ ] get_object_acl
- [X] get_object
- [X] get_object_acl
- [ ] get_object_legal_hold
- [ ] get_object_lock_configuration
- [ ] get_object_retention
- [ ] get_object_tagging
- [ ] get_object_torrent
- [ ] get_public_access_block
- [X] get_public_access_block
- [ ] head_bucket
- [ ] head_object
- [ ] list_bucket_analytics_configurations
- [ ] list_bucket_inventory_configurations
- [ ] list_bucket_metrics_configurations
- [ ] list_buckets
- [ ] list_multipart_uploads
- [X] list_buckets
- [X] list_multipart_uploads
- [ ] list_object_versions
- [ ] list_objects
- [ ] list_objects_v2
- [X] list_objects
- [X] list_objects_v2
- [ ] list_parts
- [X] put_bucket_accelerate_configuration
- [ ] put_bucket_acl
- [X] put_bucket_acl
- [ ] put_bucket_analytics_configuration
- [X] put_bucket_cors
- [ ] put_bucket_encryption
- [ ] put_bucket_inventory_configuration
- [ ] put_bucket_lifecycle
- [ ] put_bucket_lifecycle_configuration
- [X] put_bucket_lifecycle
- [X] put_bucket_lifecycle_configuration
- [X] put_bucket_logging
- [ ] put_bucket_metrics_configuration
- [ ] put_bucket_notification
@ -5654,15 +5654,15 @@
- [ ] put_bucket_replication
- [ ] put_bucket_request_payment
- [X] put_bucket_tagging
- [ ] put_bucket_versioning
- [X] put_bucket_versioning
- [ ] put_bucket_website
- [ ] put_object
- [X] put_object
- [ ] put_object_acl
- [ ] put_object_legal_hold
- [ ] put_object_lock_configuration
- [ ] put_object_retention
- [ ] put_object_tagging
- [ ] put_public_access_block
- [X] put_public_access_block
- [ ] restore_object
- [ ] select_object_content
- [ ] upload_part

View File

@ -304,3 +304,27 @@ def path_url(url):
if parsed_url.query:
path = path + "?" + parsed_url.query
return path
def py2_strip_unicode_keys(blob):
"""For Python 2 Only -- this will convert unicode keys in nested Dicts, Lists, and Sets to standard strings."""
if type(blob) == unicode: # noqa
return str(blob)
elif type(blob) == dict:
for key in list(blob.keys()):
value = blob.pop(key)
blob[str(key)] = py2_strip_unicode_keys(value)
elif type(blob) == list:
for i in range(0, len(blob)):
blob[i] = py2_strip_unicode_keys(blob[i])
elif type(blob) == set:
new_set = set()
for value in blob:
new_set.add(py2_strip_unicode_keys(value))
blob = new_set
return blob

View File

@ -323,3 +323,27 @@ class BucketSignatureDoesNotMatchError(S3ClientError):
*args,
**kwargs
)
class NoSuchPublicAccessBlockConfiguration(S3ClientError):
code = 404
def __init__(self, *args, **kwargs):
super(NoSuchPublicAccessBlockConfiguration, self).__init__(
"NoSuchPublicAccessBlockConfiguration",
"The public access block configuration was not found",
*args,
**kwargs
)
class InvalidPublicAccessBlockConfiguration(S3ClientError):
code = 400
def __init__(self, *args, **kwargs):
super(InvalidPublicAccessBlockConfiguration, self).__init__(
"InvalidRequest",
"Must specify at least one configuration.",
*args,
**kwargs
)

View File

@ -35,6 +35,8 @@ from .exceptions import (
InvalidTargetBucketForLogging,
DuplicateTagKeys,
CrossLocationLoggingProhibitted,
NoSuchPublicAccessBlockConfiguration,
InvalidPublicAccessBlockConfiguration,
)
from .utils import clean_key_name, _VersionedKeyStore
@ -659,11 +661,8 @@ class Notification(BaseModel):
else:
data["filter"] = None
data[
"objectPrefixes"
] = (
[]
) # Not sure why this is a thing since AWS just seems to return this as filters ¯\_(ツ)_/¯
# Not sure why this is a thing since AWS just seems to return this as filters ¯\_(ツ)_/¯
data["objectPrefixes"] = []
return data
@ -728,6 +727,38 @@ class NotificationConfiguration(BaseModel):
return data
def convert_str_to_bool(item):
"""Converts a boolean string to a boolean value"""
if isinstance(item, str):
return item.lower() == "true"
return False
class PublicAccessBlock(BaseModel):
def __init__(
self,
block_public_acls,
ignore_public_acls,
block_public_policy,
restrict_public_buckets,
):
# The boto XML appears to expect these values to exist as lowercase strings...
self.block_public_acls = block_public_acls or "false"
self.ignore_public_acls = ignore_public_acls or "false"
self.block_public_policy = block_public_policy or "false"
self.restrict_public_buckets = restrict_public_buckets or "false"
def to_config_dict(self):
# Need to make the string values booleans for Config:
return {
"blockPublicAcls": convert_str_to_bool(self.block_public_acls),
"ignorePublicAcls": convert_str_to_bool(self.ignore_public_acls),
"blockPublicPolicy": convert_str_to_bool(self.block_public_policy),
"restrictPublicBuckets": convert_str_to_bool(self.restrict_public_buckets),
}
class FakeBucket(BaseModel):
def __init__(self, name, region_name):
self.name = name
@ -746,6 +777,7 @@ class FakeBucket(BaseModel):
self.accelerate_configuration = None
self.payer = "BucketOwner"
self.creation_date = datetime.datetime.utcnow()
self.public_access_block = None
@property
def location(self):
@ -1079,13 +1111,16 @@ class FakeBucket(BaseModel):
}
# Make the supplementary configuration:
# TODO: Implement Public Access Block Support
# This is a dobule-wrapped JSON for some reason...
s_config = {
"AccessControlList": json.dumps(json.dumps(self.acl.to_config_dict()))
}
if self.public_access_block:
s_config["PublicAccessBlockConfiguration"] = json.dumps(
self.public_access_block.to_config_dict()
)
# Tagging is special:
if config_dict["tags"]:
s_config["BucketTaggingConfiguration"] = json.dumps(
@ -1221,6 +1256,14 @@ class S3Backend(BaseBackend):
bucket = self.get_bucket(bucket_name)
return bucket.website_configuration
def get_bucket_public_access_block(self, bucket_name):
bucket = self.get_bucket(bucket_name)
if not bucket.public_access_block:
raise NoSuchPublicAccessBlockConfiguration()
return bucket.public_access_block
def set_key(
self, bucket_name, key_name, value, storage=None, etag=None, multipart=None
):
@ -1309,6 +1352,10 @@ class S3Backend(BaseBackend):
bucket = self.get_bucket(bucket_name)
bucket.delete_cors()
def delete_bucket_public_access_block(self, bucket_name):
bucket = self.get_bucket(bucket_name)
bucket.public_access_block = None
def put_bucket_notification_configuration(self, bucket_name, notification_config):
bucket = self.get_bucket(bucket_name)
bucket.set_notification_configuration(notification_config)
@ -1324,6 +1371,19 @@ class S3Backend(BaseBackend):
raise InvalidRequest("PutBucketAccelerateConfiguration")
bucket.set_accelerate_configuration(accelerate_configuration)
def put_bucket_public_access_block(self, bucket_name, pub_block_config):
bucket = self.get_bucket(bucket_name)
if not pub_block_config:
raise InvalidPublicAccessBlockConfiguration()
bucket.public_access_block = PublicAccessBlock(
pub_block_config.get("BlockPublicAcls"),
pub_block_config.get("IgnorePublicAcls"),
pub_block_config.get("BlockPublicPolicy"),
pub_block_config.get("RestrictPublicBuckets"),
)
def initiate_multipart(self, bucket_name, key_name, metadata):
bucket = self.get_bucket(bucket_name)
new_multipart = FakeMultipart(key_name, metadata)

View File

@ -1,10 +1,11 @@
from __future__ import unicode_literals
import re
import sys
import six
from moto.core.utils import str_to_rfc_1123_datetime
from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys
from six.moves.urllib.parse import parse_qs, urlparse, unquote
import xmltodict
@ -70,6 +71,7 @@ ACTION_MAP = {
"notification": "GetBucketNotification",
"accelerate": "GetAccelerateConfiguration",
"versions": "ListBucketVersions",
"public_access_block": "GetPublicAccessBlock",
"DEFAULT": "ListBucket",
},
"PUT": {
@ -83,6 +85,7 @@ ACTION_MAP = {
"cors": "PutBucketCORS",
"notification": "PutBucketNotification",
"accelerate": "PutAccelerateConfiguration",
"public_access_block": "PutPublicAccessBlock",
"DEFAULT": "CreateBucket",
},
"DELETE": {
@ -90,6 +93,7 @@ ACTION_MAP = {
"policy": "DeleteBucketPolicy",
"tagging": "PutBucketTagging",
"cors": "PutBucketCORS",
"public_access_block": "DeletePublicAccessBlock",
"DEFAULT": "DeleteBucket",
},
},
@ -399,6 +403,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
return 200, {}, template.render()
template = self.response_template(S3_BUCKET_ACCELERATE)
return template.render(bucket=bucket)
elif "publicAccessBlock" in querystring:
public_block_config = self.backend.get_bucket_public_access_block(
bucket_name
)
template = self.response_template(S3_PUBLIC_ACCESS_BLOCK_CONFIGURATION)
return template.render(public_block_config=public_block_config)
elif "versions" in querystring:
delimiter = querystring.get("delimiter", [None])[0]
@ -651,6 +661,23 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
except Exception as e:
raise e
elif "publicAccessBlock" in querystring:
parsed_xml = xmltodict.parse(body)
parsed_xml["PublicAccessBlockConfiguration"].pop("@xmlns", None)
# If Python 2, fix the unicode strings:
if sys.version_info[0] < 3:
parsed_xml = {
"PublicAccessBlockConfiguration": py2_strip_unicode_keys(
dict(parsed_xml["PublicAccessBlockConfiguration"])
)
}
self.backend.put_bucket_public_access_block(
bucket_name, parsed_xml["PublicAccessBlockConfiguration"]
)
return ""
else:
if body:
# us-east-1, the default AWS region behaves a bit differently
@ -706,6 +733,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
bucket = self.backend.get_bucket(bucket_name)
bucket.delete_lifecycle()
return 204, {}, ""
elif "publicAccessBlock" in querystring:
self.backend.delete_bucket_public_access_block(bucket_name)
return 204, {}, ""
removed_bucket = self.backend.delete_bucket(bucket_name)
@ -2053,3 +2083,12 @@ S3_BUCKET_ACCELERATE = """
S3_BUCKET_ACCELERATE_NOT_SET = """
<AccelerateConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>
"""
S3_PUBLIC_ACCESS_BLOCK_CONFIGURATION = """
<PublicAccessBlockConfiguration>
<BlockPublicAcls>{{public_block_config.block_public_acls}}</BlockPublicAcls>
<IgnorePublicAcls>{{public_block_config.ignore_public_acls}}</IgnorePublicAcls>
<BlockPublicPolicy>{{public_block_config.block_public_policy}}</BlockPublicPolicy>
<RestrictPublicBuckets>{{public_block_config.restrict_public_buckets}}</RestrictPublicBuckets>
</PublicAccessBlockConfiguration>
"""

View File

@ -1,5 +1,8 @@
from __future__ import unicode_literals
import copy
import sys
import sure # noqa
from freezegun import freeze_time
@ -7,6 +10,7 @@ from moto.core.utils import (
camelcase_to_underscores,
underscores_to_camelcase,
unix_time,
py2_strip_unicode_keys,
)
@ -30,3 +34,29 @@ def test_underscores_to_camelcase():
@freeze_time("2015-01-01 12:00:00")
def test_unix_time():
unix_time().should.equal(1420113600.0)
if sys.version_info[0] < 3:
# Tests for unicode removals (Python 2 only)
def _verify_no_unicode(blob):
"""Verify that no unicode values exist"""
if type(blob) == dict:
for key, value in blob.items():
assert type(key) != unicode
_verify_no_unicode(value)
elif type(blob) in [list, set]:
for item in blob:
_verify_no_unicode(item)
assert blob != unicode
def test_py2_strip_unicode_keys():
bad_dict = {
"some": "value",
"a": {"nested": ["List", "of", {"unicode": "values"}]},
"and a": {"nested", "set", "of", 5, "values"},
}
result = py2_strip_unicode_keys(copy.deepcopy(bad_dict))
_verify_no_unicode(result)

View File

@ -32,9 +32,10 @@ from nose.tools import assert_raises
import sure # noqa
from moto import settings, mock_s3, mock_s3_deprecated
from moto import settings, mock_s3, mock_s3_deprecated, mock_config
import moto.s3.models as s3model
from moto.core.exceptions import InvalidNextTokenException
from moto.core.utils import py2_strip_unicode_keys
if settings.TEST_SERVER_MODE:
REDUCED_PART_SIZE = s3model.UPLOAD_PART_MIN_SIZE
@ -3278,6 +3279,148 @@ def test_delete_objects_with_url_encoded_key(key):
assert_deleted()
@mock_s3
@mock_config
def test_public_access_block():
client = boto3.client("s3")
client.create_bucket(Bucket="mybucket")
# Try to get the public access block (should not exist by default)
with assert_raises(ClientError) as ce:
client.get_public_access_block(Bucket="mybucket")
assert (
ce.exception.response["Error"]["Code"] == "NoSuchPublicAccessBlockConfiguration"
)
assert (
ce.exception.response["Error"]["Message"]
== "The public access block configuration was not found"
)
assert ce.exception.response["ResponseMetadata"]["HTTPStatusCode"] == 404
# Put a public block in place:
test_map = {
"BlockPublicAcls": False,
"IgnorePublicAcls": False,
"BlockPublicPolicy": False,
"RestrictPublicBuckets": False,
}
for field in test_map.keys():
# Toggle:
test_map[field] = True
client.put_public_access_block(
Bucket="mybucket", PublicAccessBlockConfiguration=test_map
)
# Test:
assert (
test_map
== client.get_public_access_block(Bucket="mybucket")[
"PublicAccessBlockConfiguration"
]
)
# Assume missing values are default False:
client.put_public_access_block(
Bucket="mybucket", PublicAccessBlockConfiguration={"BlockPublicAcls": True}
)
assert client.get_public_access_block(Bucket="mybucket")[
"PublicAccessBlockConfiguration"
] == {
"BlockPublicAcls": True,
"IgnorePublicAcls": False,
"BlockPublicPolicy": False,
"RestrictPublicBuckets": False,
}
# Test with a blank PublicAccessBlockConfiguration:
with assert_raises(ClientError) as ce:
client.put_public_access_block(
Bucket="mybucket", PublicAccessBlockConfiguration={}
)
assert ce.exception.response["Error"]["Code"] == "InvalidRequest"
assert (
ce.exception.response["Error"]["Message"]
== "Must specify at least one configuration."
)
assert ce.exception.response["ResponseMetadata"]["HTTPStatusCode"] == 400
# Test that things work with AWS Config:
config_client = boto3.client("config", region_name="us-east-1")
result = config_client.get_resource_config_history(
resourceType="AWS::S3::Bucket", resourceId="mybucket"
)
pub_block_config = json.loads(
result["configurationItems"][0]["supplementaryConfiguration"][
"PublicAccessBlockConfiguration"
]
)
assert pub_block_config == {
"blockPublicAcls": True,
"ignorePublicAcls": False,
"blockPublicPolicy": False,
"restrictPublicBuckets": False,
}
# Delete:
client.delete_public_access_block(Bucket="mybucket")
with assert_raises(ClientError) as ce:
client.get_public_access_block(Bucket="mybucket")
assert (
ce.exception.response["Error"]["Code"] == "NoSuchPublicAccessBlockConfiguration"
)
@mock_s3
def test_s3_public_access_block_to_config_dict():
from moto.s3.config import s3_config_query
# With 1 bucket in us-west-2:
s3_config_query.backends["global"].create_bucket("bucket1", "us-west-2")
public_access_block = {
"BlockPublicAcls": "True",
"IgnorePublicAcls": "False",
"BlockPublicPolicy": "True",
"RestrictPublicBuckets": "False",
}
# Python 2 unicode issues:
if sys.version_info[0] < 3:
public_access_block = py2_strip_unicode_keys(public_access_block)
# Add a public access block:
s3_config_query.backends["global"].put_bucket_public_access_block(
"bucket1", public_access_block
)
result = (
s3_config_query.backends["global"]
.buckets["bucket1"]
.public_access_block.to_config_dict()
)
convert_bool = lambda x: x == "True"
for key, value in public_access_block.items():
assert result[
"{lowercase}{rest}".format(lowercase=key[0].lower(), rest=key[1:])
] == convert_bool(value)
# Verify that this resides in the full bucket's to_config_dict:
full_result = s3_config_query.backends["global"].buckets["bucket1"].to_config_dict()
assert (
json.loads(
full_result["supplementaryConfiguration"]["PublicAccessBlockConfiguration"]
)
== result
)
@mock_s3
def test_list_config_discovered_resources():
from moto.s3.config import s3_config_query