diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index cd1d4d482..5d9f18ebf 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -5589,63 +5589,63 @@ - [X] delete_bucket_cors - [ ] delete_bucket_encryption - [ ] delete_bucket_inventory_configuration -- [ ] delete_bucket_lifecycle +- [X] delete_bucket_lifecycle - [ ] delete_bucket_metrics_configuration - [X] delete_bucket_policy - [ ] delete_bucket_replication - [X] delete_bucket_tagging - [ ] delete_bucket_website -- [ ] delete_object +- [X] delete_object - [ ] delete_object_tagging -- [ ] delete_objects -- [ ] delete_public_access_block +- [X] delete_objects +- [X] delete_public_access_block - [ ] get_bucket_accelerate_configuration - [X] get_bucket_acl - [ ] get_bucket_analytics_configuration -- [ ] get_bucket_cors +- [X] get_bucket_cors - [ ] get_bucket_encryption - [ ] get_bucket_inventory_configuration -- [ ] get_bucket_lifecycle -- [ ] get_bucket_lifecycle_configuration -- [ ] get_bucket_location -- [ ] get_bucket_logging +- [X] get_bucket_lifecycle +- [X] get_bucket_lifecycle_configuration +- [X] get_bucket_location +- [X] get_bucket_logging - [ ] get_bucket_metrics_configuration - [ ] get_bucket_notification - [ ] get_bucket_notification_configuration - [X] get_bucket_policy -- [ ] get_bucket_policy_status +- [X] get_bucket_policy_status - [ ] get_bucket_replication - [ ] get_bucket_request_payment -- [ ] get_bucket_tagging +- [X] get_bucket_tagging - [X] get_bucket_versioning - [ ] get_bucket_website -- [ ] get_object -- [ ] get_object_acl +- [X] get_object +- [X] get_object_acl - [ ] get_object_legal_hold - [ ] get_object_lock_configuration - [ ] get_object_retention - [ ] get_object_tagging - [ ] get_object_torrent -- [ ] get_public_access_block +- [X] get_public_access_block - [ ] head_bucket - [ ] head_object - [ ] list_bucket_analytics_configurations - [ ] list_bucket_inventory_configurations - [ ] list_bucket_metrics_configurations -- [ ] list_buckets -- [ ] list_multipart_uploads +- [X] list_buckets +- [X] list_multipart_uploads - [ ] list_object_versions -- [ ] list_objects -- [ ] list_objects_v2 +- [X] list_objects +- [X] list_objects_v2 - [ ] list_parts - [X] put_bucket_accelerate_configuration -- [ ] put_bucket_acl +- [X] put_bucket_acl - [ ] put_bucket_analytics_configuration - [X] put_bucket_cors - [ ] put_bucket_encryption - [ ] put_bucket_inventory_configuration -- [ ] put_bucket_lifecycle -- [ ] put_bucket_lifecycle_configuration +- [X] put_bucket_lifecycle +- [X] put_bucket_lifecycle_configuration - [X] put_bucket_logging - [ ] put_bucket_metrics_configuration - [ ] put_bucket_notification @@ -5654,15 +5654,15 @@ - [ ] put_bucket_replication - [ ] put_bucket_request_payment - [X] put_bucket_tagging -- [ ] put_bucket_versioning +- [X] put_bucket_versioning - [ ] put_bucket_website -- [ ] put_object +- [X] put_object - [ ] put_object_acl - [ ] put_object_legal_hold - [ ] put_object_lock_configuration - [ ] put_object_retention - [ ] put_object_tagging -- [ ] put_public_access_block +- [X] put_public_access_block - [ ] restore_object - [ ] select_object_content - [ ] upload_part diff --git a/moto/core/utils.py b/moto/core/utils.py index 57ff0f1b4..efad5679c 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -304,3 +304,27 @@ def path_url(url): if parsed_url.query: path = path + "?" + parsed_url.query return path + + +def py2_strip_unicode_keys(blob): + """For Python 2 Only -- this will convert unicode keys in nested Dicts, Lists, and Sets to standard strings.""" + if type(blob) == unicode: # noqa + return str(blob) + + elif type(blob) == dict: + for key in list(blob.keys()): + value = blob.pop(key) + blob[str(key)] = py2_strip_unicode_keys(value) + + elif type(blob) == list: + for i in range(0, len(blob)): + blob[i] = py2_strip_unicode_keys(blob[i]) + + elif type(blob) == set: + new_set = set() + for value in blob: + new_set.add(py2_strip_unicode_keys(value)) + + blob = new_set + + return blob diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index c8236398f..1f2ead639 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -323,3 +323,27 @@ class BucketSignatureDoesNotMatchError(S3ClientError): *args, **kwargs ) + + +class NoSuchPublicAccessBlockConfiguration(S3ClientError): + code = 404 + + def __init__(self, *args, **kwargs): + super(NoSuchPublicAccessBlockConfiguration, self).__init__( + "NoSuchPublicAccessBlockConfiguration", + "The public access block configuration was not found", + *args, + **kwargs + ) + + +class InvalidPublicAccessBlockConfiguration(S3ClientError): + code = 400 + + def __init__(self, *args, **kwargs): + super(InvalidPublicAccessBlockConfiguration, self).__init__( + "InvalidRequest", + "Must specify at least one configuration.", + *args, + **kwargs + ) diff --git a/moto/s3/models.py b/moto/s3/models.py index 9c8f64242..fe8e908ef 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -35,6 +35,8 @@ from .exceptions import ( InvalidTargetBucketForLogging, DuplicateTagKeys, CrossLocationLoggingProhibitted, + NoSuchPublicAccessBlockConfiguration, + InvalidPublicAccessBlockConfiguration, ) from .utils import clean_key_name, _VersionedKeyStore @@ -659,11 +661,8 @@ class Notification(BaseModel): else: data["filter"] = None - data[ - "objectPrefixes" - ] = ( - [] - ) # Not sure why this is a thing since AWS just seems to return this as filters ¯\_(ツ)_/¯ + # Not sure why this is a thing since AWS just seems to return this as filters ¯\_(ツ)_/¯ + data["objectPrefixes"] = [] return data @@ -728,6 +727,38 @@ class NotificationConfiguration(BaseModel): return data +def convert_str_to_bool(item): + """Converts a boolean string to a boolean value""" + if isinstance(item, str): + return item.lower() == "true" + + return False + + +class PublicAccessBlock(BaseModel): + def __init__( + self, + block_public_acls, + ignore_public_acls, + block_public_policy, + restrict_public_buckets, + ): + # The boto XML appears to expect these values to exist as lowercase strings... + self.block_public_acls = block_public_acls or "false" + self.ignore_public_acls = ignore_public_acls or "false" + self.block_public_policy = block_public_policy or "false" + self.restrict_public_buckets = restrict_public_buckets or "false" + + def to_config_dict(self): + # Need to make the string values booleans for Config: + return { + "blockPublicAcls": convert_str_to_bool(self.block_public_acls), + "ignorePublicAcls": convert_str_to_bool(self.ignore_public_acls), + "blockPublicPolicy": convert_str_to_bool(self.block_public_policy), + "restrictPublicBuckets": convert_str_to_bool(self.restrict_public_buckets), + } + + class FakeBucket(BaseModel): def __init__(self, name, region_name): self.name = name @@ -746,6 +777,7 @@ class FakeBucket(BaseModel): self.accelerate_configuration = None self.payer = "BucketOwner" self.creation_date = datetime.datetime.utcnow() + self.public_access_block = None @property def location(self): @@ -1079,13 +1111,16 @@ class FakeBucket(BaseModel): } # Make the supplementary configuration: - # TODO: Implement Public Access Block Support - # This is a dobule-wrapped JSON for some reason... s_config = { "AccessControlList": json.dumps(json.dumps(self.acl.to_config_dict())) } + if self.public_access_block: + s_config["PublicAccessBlockConfiguration"] = json.dumps( + self.public_access_block.to_config_dict() + ) + # Tagging is special: if config_dict["tags"]: s_config["BucketTaggingConfiguration"] = json.dumps( @@ -1221,6 +1256,14 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) return bucket.website_configuration + def get_bucket_public_access_block(self, bucket_name): + bucket = self.get_bucket(bucket_name) + + if not bucket.public_access_block: + raise NoSuchPublicAccessBlockConfiguration() + + return bucket.public_access_block + def set_key( self, bucket_name, key_name, value, storage=None, etag=None, multipart=None ): @@ -1309,6 +1352,10 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) bucket.delete_cors() + def delete_bucket_public_access_block(self, bucket_name): + bucket = self.get_bucket(bucket_name) + bucket.public_access_block = None + def put_bucket_notification_configuration(self, bucket_name, notification_config): bucket = self.get_bucket(bucket_name) bucket.set_notification_configuration(notification_config) @@ -1324,6 +1371,19 @@ class S3Backend(BaseBackend): raise InvalidRequest("PutBucketAccelerateConfiguration") bucket.set_accelerate_configuration(accelerate_configuration) + def put_bucket_public_access_block(self, bucket_name, pub_block_config): + bucket = self.get_bucket(bucket_name) + + if not pub_block_config: + raise InvalidPublicAccessBlockConfiguration() + + bucket.public_access_block = PublicAccessBlock( + pub_block_config.get("BlockPublicAcls"), + pub_block_config.get("IgnorePublicAcls"), + pub_block_config.get("BlockPublicPolicy"), + pub_block_config.get("RestrictPublicBuckets"), + ) + def initiate_multipart(self, bucket_name, key_name, metadata): bucket = self.get_bucket(bucket_name) new_multipart = FakeMultipart(key_name, metadata) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index fd3a7b2db..a9f3580a4 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1,10 +1,11 @@ from __future__ import unicode_literals import re +import sys import six -from moto.core.utils import str_to_rfc_1123_datetime +from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys from six.moves.urllib.parse import parse_qs, urlparse, unquote import xmltodict @@ -70,6 +71,7 @@ ACTION_MAP = { "notification": "GetBucketNotification", "accelerate": "GetAccelerateConfiguration", "versions": "ListBucketVersions", + "public_access_block": "GetPublicAccessBlock", "DEFAULT": "ListBucket", }, "PUT": { @@ -83,6 +85,7 @@ ACTION_MAP = { "cors": "PutBucketCORS", "notification": "PutBucketNotification", "accelerate": "PutAccelerateConfiguration", + "public_access_block": "PutPublicAccessBlock", "DEFAULT": "CreateBucket", }, "DELETE": { @@ -90,6 +93,7 @@ ACTION_MAP = { "policy": "DeleteBucketPolicy", "tagging": "PutBucketTagging", "cors": "PutBucketCORS", + "public_access_block": "DeletePublicAccessBlock", "DEFAULT": "DeleteBucket", }, }, @@ -399,6 +403,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return 200, {}, template.render() template = self.response_template(S3_BUCKET_ACCELERATE) return template.render(bucket=bucket) + elif "publicAccessBlock" in querystring: + public_block_config = self.backend.get_bucket_public_access_block( + bucket_name + ) + template = self.response_template(S3_PUBLIC_ACCESS_BLOCK_CONFIGURATION) + return template.render(public_block_config=public_block_config) elif "versions" in querystring: delimiter = querystring.get("delimiter", [None])[0] @@ -651,6 +661,23 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): except Exception as e: raise e + elif "publicAccessBlock" in querystring: + parsed_xml = xmltodict.parse(body) + parsed_xml["PublicAccessBlockConfiguration"].pop("@xmlns", None) + + # If Python 2, fix the unicode strings: + if sys.version_info[0] < 3: + parsed_xml = { + "PublicAccessBlockConfiguration": py2_strip_unicode_keys( + dict(parsed_xml["PublicAccessBlockConfiguration"]) + ) + } + + self.backend.put_bucket_public_access_block( + bucket_name, parsed_xml["PublicAccessBlockConfiguration"] + ) + return "" + else: if body: # us-east-1, the default AWS region behaves a bit differently @@ -706,6 +733,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): bucket = self.backend.get_bucket(bucket_name) bucket.delete_lifecycle() return 204, {}, "" + elif "publicAccessBlock" in querystring: + self.backend.delete_bucket_public_access_block(bucket_name) + return 204, {}, "" removed_bucket = self.backend.delete_bucket(bucket_name) @@ -2053,3 +2083,12 @@ S3_BUCKET_ACCELERATE = """ S3_BUCKET_ACCELERATE_NOT_SET = """ """ + +S3_PUBLIC_ACCESS_BLOCK_CONFIGURATION = """ + + {{public_block_config.block_public_acls}} + {{public_block_config.ignore_public_acls}} + {{public_block_config.block_public_policy}} + {{public_block_config.restrict_public_buckets}} + +""" diff --git a/tests/test_core/test_utils.py b/tests/test_core/test_utils.py index 7c72aaccd..d0dd97688 100644 --- a/tests/test_core/test_utils.py +++ b/tests/test_core/test_utils.py @@ -1,5 +1,8 @@ from __future__ import unicode_literals +import copy +import sys + import sure # noqa from freezegun import freeze_time @@ -7,6 +10,7 @@ from moto.core.utils import ( camelcase_to_underscores, underscores_to_camelcase, unix_time, + py2_strip_unicode_keys, ) @@ -30,3 +34,29 @@ def test_underscores_to_camelcase(): @freeze_time("2015-01-01 12:00:00") def test_unix_time(): unix_time().should.equal(1420113600.0) + + +if sys.version_info[0] < 3: + # Tests for unicode removals (Python 2 only) + def _verify_no_unicode(blob): + """Verify that no unicode values exist""" + if type(blob) == dict: + for key, value in blob.items(): + assert type(key) != unicode + _verify_no_unicode(value) + + elif type(blob) in [list, set]: + for item in blob: + _verify_no_unicode(item) + + assert blob != unicode + + def test_py2_strip_unicode_keys(): + bad_dict = { + "some": "value", + "a": {"nested": ["List", "of", {"unicode": "values"}]}, + "and a": {"nested", "set", "of", 5, "values"}, + } + + result = py2_strip_unicode_keys(copy.deepcopy(bad_dict)) + _verify_no_unicode(result) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 8f3c3538c..3cf3bc6f1 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -32,9 +32,10 @@ from nose.tools import assert_raises import sure # noqa -from moto import settings, mock_s3, mock_s3_deprecated +from moto import settings, mock_s3, mock_s3_deprecated, mock_config import moto.s3.models as s3model from moto.core.exceptions import InvalidNextTokenException +from moto.core.utils import py2_strip_unicode_keys if settings.TEST_SERVER_MODE: REDUCED_PART_SIZE = s3model.UPLOAD_PART_MIN_SIZE @@ -3278,6 +3279,148 @@ def test_delete_objects_with_url_encoded_key(key): assert_deleted() +@mock_s3 +@mock_config +def test_public_access_block(): + client = boto3.client("s3") + client.create_bucket(Bucket="mybucket") + + # Try to get the public access block (should not exist by default) + with assert_raises(ClientError) as ce: + client.get_public_access_block(Bucket="mybucket") + + assert ( + ce.exception.response["Error"]["Code"] == "NoSuchPublicAccessBlockConfiguration" + ) + assert ( + ce.exception.response["Error"]["Message"] + == "The public access block configuration was not found" + ) + assert ce.exception.response["ResponseMetadata"]["HTTPStatusCode"] == 404 + + # Put a public block in place: + test_map = { + "BlockPublicAcls": False, + "IgnorePublicAcls": False, + "BlockPublicPolicy": False, + "RestrictPublicBuckets": False, + } + + for field in test_map.keys(): + # Toggle: + test_map[field] = True + + client.put_public_access_block( + Bucket="mybucket", PublicAccessBlockConfiguration=test_map + ) + + # Test: + assert ( + test_map + == client.get_public_access_block(Bucket="mybucket")[ + "PublicAccessBlockConfiguration" + ] + ) + + # Assume missing values are default False: + client.put_public_access_block( + Bucket="mybucket", PublicAccessBlockConfiguration={"BlockPublicAcls": True} + ) + assert client.get_public_access_block(Bucket="mybucket")[ + "PublicAccessBlockConfiguration" + ] == { + "BlockPublicAcls": True, + "IgnorePublicAcls": False, + "BlockPublicPolicy": False, + "RestrictPublicBuckets": False, + } + + # Test with a blank PublicAccessBlockConfiguration: + with assert_raises(ClientError) as ce: + client.put_public_access_block( + Bucket="mybucket", PublicAccessBlockConfiguration={} + ) + + assert ce.exception.response["Error"]["Code"] == "InvalidRequest" + assert ( + ce.exception.response["Error"]["Message"] + == "Must specify at least one configuration." + ) + assert ce.exception.response["ResponseMetadata"]["HTTPStatusCode"] == 400 + + # Test that things work with AWS Config: + config_client = boto3.client("config", region_name="us-east-1") + result = config_client.get_resource_config_history( + resourceType="AWS::S3::Bucket", resourceId="mybucket" + ) + pub_block_config = json.loads( + result["configurationItems"][0]["supplementaryConfiguration"][ + "PublicAccessBlockConfiguration" + ] + ) + + assert pub_block_config == { + "blockPublicAcls": True, + "ignorePublicAcls": False, + "blockPublicPolicy": False, + "restrictPublicBuckets": False, + } + + # Delete: + client.delete_public_access_block(Bucket="mybucket") + + with assert_raises(ClientError) as ce: + client.get_public_access_block(Bucket="mybucket") + assert ( + ce.exception.response["Error"]["Code"] == "NoSuchPublicAccessBlockConfiguration" + ) + + +@mock_s3 +def test_s3_public_access_block_to_config_dict(): + from moto.s3.config import s3_config_query + + # With 1 bucket in us-west-2: + s3_config_query.backends["global"].create_bucket("bucket1", "us-west-2") + + public_access_block = { + "BlockPublicAcls": "True", + "IgnorePublicAcls": "False", + "BlockPublicPolicy": "True", + "RestrictPublicBuckets": "False", + } + + # Python 2 unicode issues: + if sys.version_info[0] < 3: + public_access_block = py2_strip_unicode_keys(public_access_block) + + # Add a public access block: + s3_config_query.backends["global"].put_bucket_public_access_block( + "bucket1", public_access_block + ) + + result = ( + s3_config_query.backends["global"] + .buckets["bucket1"] + .public_access_block.to_config_dict() + ) + + convert_bool = lambda x: x == "True" + for key, value in public_access_block.items(): + assert result[ + "{lowercase}{rest}".format(lowercase=key[0].lower(), rest=key[1:]) + ] == convert_bool(value) + + # Verify that this resides in the full bucket's to_config_dict: + full_result = s3_config_query.backends["global"].buckets["bucket1"].to_config_dict() + assert ( + json.loads( + full_result["supplementaryConfiguration"]["PublicAccessBlockConfiguration"] + ) + == result + ) + + @mock_s3 def test_list_config_discovered_resources(): from moto.s3.config import s3_config_query