Implemented S3 Public Access Block

IMPLEMENTATION_COVERAGE.md

@@ -5589,63 +5589,63 @@
 - [X] delete_bucket_cors
 - [ ] delete_bucket_encryption
 - [ ] delete_bucket_inventory_configuration
-- [ ] delete_bucket_lifecycle
+- [X] delete_bucket_lifecycle
 - [ ] delete_bucket_metrics_configuration
 - [X] delete_bucket_policy
 - [ ] delete_bucket_replication
 - [X] delete_bucket_tagging
 - [ ] delete_bucket_website
-- [ ] delete_object
+- [X] delete_object
 - [ ] delete_object_tagging
-- [ ] delete_objects
+- [X] delete_objects
-- [ ] delete_public_access_block
+- [X] delete_public_access_block
 - [ ] get_bucket_accelerate_configuration
 - [X] get_bucket_acl
 - [ ] get_bucket_analytics_configuration
-- [ ] get_bucket_cors
+- [X] get_bucket_cors
 - [ ] get_bucket_encryption
 - [ ] get_bucket_inventory_configuration
-- [ ] get_bucket_lifecycle
+- [X] get_bucket_lifecycle
-- [ ] get_bucket_lifecycle_configuration
+- [X] get_bucket_lifecycle_configuration
-- [ ] get_bucket_location
+- [X] get_bucket_location
-- [ ] get_bucket_logging
+- [X] get_bucket_logging
 - [ ] get_bucket_metrics_configuration
 - [ ] get_bucket_notification
 - [ ] get_bucket_notification_configuration
 - [X] get_bucket_policy
-- [ ] get_bucket_policy_status
+- [X] get_bucket_policy_status
 - [ ] get_bucket_replication
 - [ ] get_bucket_request_payment
-- [ ] get_bucket_tagging
+- [X] get_bucket_tagging
 - [X] get_bucket_versioning
 - [ ] get_bucket_website
-- [ ] get_object
+- [X] get_object
-- [ ] get_object_acl
+- [X] get_object_acl
 - [ ] get_object_legal_hold
 - [ ] get_object_lock_configuration
 - [ ] get_object_retention
 - [ ] get_object_tagging
 - [ ] get_object_torrent
-- [ ] get_public_access_block
+- [X] get_public_access_block
 - [ ] head_bucket
 - [ ] head_object
 - [ ] list_bucket_analytics_configurations
 - [ ] list_bucket_inventory_configurations
 - [ ] list_bucket_metrics_configurations
-- [ ] list_buckets
+- [X] list_buckets
-- [ ] list_multipart_uploads
+- [X] list_multipart_uploads
 - [ ] list_object_versions
-- [ ] list_objects
+- [X] list_objects
-- [ ] list_objects_v2
+- [X] list_objects_v2
 - [ ] list_parts
 - [X] put_bucket_accelerate_configuration
-- [ ] put_bucket_acl
+- [X] put_bucket_acl
 - [ ] put_bucket_analytics_configuration
 - [X] put_bucket_cors
 - [ ] put_bucket_encryption
 - [ ] put_bucket_inventory_configuration
-- [ ] put_bucket_lifecycle
+- [X] put_bucket_lifecycle
-- [ ] put_bucket_lifecycle_configuration
+- [X] put_bucket_lifecycle_configuration
 - [X] put_bucket_logging
 - [ ] put_bucket_metrics_configuration
 - [ ] put_bucket_notification
@@ -5654,15 +5654,15 @@
 - [ ] put_bucket_replication
 - [ ] put_bucket_request_payment
 - [X] put_bucket_tagging
-- [ ] put_bucket_versioning
+- [X] put_bucket_versioning
 - [ ] put_bucket_website
-- [ ] put_object
+- [X] put_object
 - [ ] put_object_acl
 - [ ] put_object_legal_hold
 - [ ] put_object_lock_configuration
 - [ ] put_object_retention
 - [ ] put_object_tagging
-- [ ] put_public_access_block
+- [X] put_public_access_block
 - [ ] restore_object
 - [ ] select_object_content
 - [ ] upload_part

moto/core/utils.py

@@ -304,3 +304,27 @@ def path_url(url):
     if parsed_url.query:
         path = path + "?" + parsed_url.query
     return path
+
+
+def py2_strip_unicode_keys(blob):
+    """For Python 2 Only -- this will convert unicode keys in nested Dicts, Lists, and Sets to standard strings."""
+    if type(blob) == unicode:  # noqa
+        return str(blob)
+
+    elif type(blob) == dict:
+        for key in list(blob.keys()):
+            value = blob.pop(key)
+            blob[str(key)] = py2_strip_unicode_keys(value)
+
+    elif type(blob) == list:
+        for i in range(0, len(blob)):
+            blob[i] = py2_strip_unicode_keys(blob[i])
+
+    elif type(blob) == set:
+        new_set = set()
+        for value in blob:
+            new_set.add(py2_strip_unicode_keys(value))
+
+        blob = new_set
+
+    return blob

moto/s3/exceptions.py

@@ -323,3 +323,27 @@ class BucketSignatureDoesNotMatchError(S3ClientError):
             *args,
             **kwargs
         )
+
+
+class NoSuchPublicAccessBlockConfiguration(S3ClientError):
+    code = 404
+
+    def __init__(self, *args, **kwargs):
+        super(NoSuchPublicAccessBlockConfiguration, self).__init__(
+            "NoSuchPublicAccessBlockConfiguration",
+            "The public access block configuration was not found",
+            *args,
+            **kwargs
+        )
+
+
+class InvalidPublicAccessBlockConfiguration(S3ClientError):
+    code = 400
+
+    def __init__(self, *args, **kwargs):
+        super(InvalidPublicAccessBlockConfiguration, self).__init__(
+            "InvalidRequest",
+            "Must specify at least one configuration.",
+            *args,
+            **kwargs
+        )

moto/s3/models.py

@@ -35,6 +35,8 @@ from .exceptions import (
     InvalidTargetBucketForLogging,
     DuplicateTagKeys,
     CrossLocationLoggingProhibitted,
+    NoSuchPublicAccessBlockConfiguration,
+    InvalidPublicAccessBlockConfiguration,
 )
 from .utils import clean_key_name, _VersionedKeyStore
 
@@ -659,11 +661,8 @@ class Notification(BaseModel):
         else:
             data["filter"] = None
 
-        data[
+        # Not sure why this is a thing since AWS just seems to return this as filters ¯\_(ツ)_/¯
-            "objectPrefixes"
+        data["objectPrefixes"] = []
-        ] = (
-            []
-        )  # Not sure why this is a thing since AWS just seems to return this as filters ¯\_(ツ)_/¯
 
         return data
 
@@ -728,6 +727,38 @@ class NotificationConfiguration(BaseModel):
         return data
 
 
+def convert_str_to_bool(item):
+    """Converts a boolean string to a boolean value"""
+    if isinstance(item, str):
+        return item.lower() == "true"
+
+    return False
+
+
+class PublicAccessBlock(BaseModel):
+    def __init__(
+        self,
+        block_public_acls,
+        ignore_public_acls,
+        block_public_policy,
+        restrict_public_buckets,
+    ):
+        # The boto XML appears to expect these values to exist as lowercase strings...
+        self.block_public_acls = block_public_acls or "false"
+        self.ignore_public_acls = ignore_public_acls or "false"
+        self.block_public_policy = block_public_policy or "false"
+        self.restrict_public_buckets = restrict_public_buckets or "false"
+
+    def to_config_dict(self):
+        # Need to make the string values booleans for Config:
+        return {
+            "blockPublicAcls": convert_str_to_bool(self.block_public_acls),
+            "ignorePublicAcls": convert_str_to_bool(self.ignore_public_acls),
+            "blockPublicPolicy": convert_str_to_bool(self.block_public_policy),
+            "restrictPublicBuckets": convert_str_to_bool(self.restrict_public_buckets),
+        }
+
+
 class FakeBucket(BaseModel):
     def __init__(self, name, region_name):
         self.name = name
@@ -746,6 +777,7 @@ class FakeBucket(BaseModel):
         self.accelerate_configuration = None
         self.payer = "BucketOwner"
         self.creation_date = datetime.datetime.utcnow()
+        self.public_access_block = None
 
     @property
     def location(self):
@@ -1079,13 +1111,16 @@ class FakeBucket(BaseModel):
         }
 
         # Make the supplementary configuration:
-        # TODO: Implement Public Access Block Support
-
         # This is a dobule-wrapped JSON for some reason...
         s_config = {
             "AccessControlList": json.dumps(json.dumps(self.acl.to_config_dict()))
         }
 
+        if self.public_access_block:
+            s_config["PublicAccessBlockConfiguration"] = json.dumps(
+                self.public_access_block.to_config_dict()
+            )
+
         # Tagging is special:
         if config_dict["tags"]:
             s_config["BucketTaggingConfiguration"] = json.dumps(
@@ -1221,6 +1256,14 @@ class S3Backend(BaseBackend):
         bucket = self.get_bucket(bucket_name)
         return bucket.website_configuration
 
+    def get_bucket_public_access_block(self, bucket_name):
+        bucket = self.get_bucket(bucket_name)
+
+        if not bucket.public_access_block:
+            raise NoSuchPublicAccessBlockConfiguration()
+
+        return bucket.public_access_block
+
     def set_key(
         self, bucket_name, key_name, value, storage=None, etag=None, multipart=None
     ):
@@ -1309,6 +1352,10 @@ class S3Backend(BaseBackend):
         bucket = self.get_bucket(bucket_name)
         bucket.delete_cors()
 
+    def delete_bucket_public_access_block(self, bucket_name):
+        bucket = self.get_bucket(bucket_name)
+        bucket.public_access_block = None
+
     def put_bucket_notification_configuration(self, bucket_name, notification_config):
         bucket = self.get_bucket(bucket_name)
         bucket.set_notification_configuration(notification_config)
@@ -1324,6 +1371,19 @@ class S3Backend(BaseBackend):
             raise InvalidRequest("PutBucketAccelerateConfiguration")
         bucket.set_accelerate_configuration(accelerate_configuration)
 
+    def put_bucket_public_access_block(self, bucket_name, pub_block_config):
+        bucket = self.get_bucket(bucket_name)
+
+        if not pub_block_config:
+            raise InvalidPublicAccessBlockConfiguration()
+
+        bucket.public_access_block = PublicAccessBlock(
+            pub_block_config.get("BlockPublicAcls"),
+            pub_block_config.get("IgnorePublicAcls"),
+            pub_block_config.get("BlockPublicPolicy"),
+            pub_block_config.get("RestrictPublicBuckets"),
+        )
+
     def initiate_multipart(self, bucket_name, key_name, metadata):
         bucket = self.get_bucket(bucket_name)
         new_multipart = FakeMultipart(key_name, metadata)

moto/s3/responses.py

@@ -1,10 +1,11 @@
 from __future__ import unicode_literals
 
 import re
+import sys
 
 import six
 
-from moto.core.utils import str_to_rfc_1123_datetime
+from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys
 from six.moves.urllib.parse import parse_qs, urlparse, unquote
 
 import xmltodict
@@ -70,6 +71,7 @@ ACTION_MAP = {
             "notification": "GetBucketNotification",
             "accelerate": "GetAccelerateConfiguration",
             "versions": "ListBucketVersions",
+            "public_access_block": "GetPublicAccessBlock",
             "DEFAULT": "ListBucket",
         },
         "PUT": {
@@ -83,6 +85,7 @@ ACTION_MAP = {
             "cors": "PutBucketCORS",
             "notification": "PutBucketNotification",
             "accelerate": "PutAccelerateConfiguration",
+            "public_access_block": "PutPublicAccessBlock",
             "DEFAULT": "CreateBucket",
         },
         "DELETE": {
@@ -90,6 +93,7 @@ ACTION_MAP = {
             "policy": "DeleteBucketPolicy",
             "tagging": "PutBucketTagging",
             "cors": "PutBucketCORS",
+            "public_access_block": "DeletePublicAccessBlock",
             "DEFAULT": "DeleteBucket",
         },
     },
@@ -399,6 +403,12 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
                 return 200, {}, template.render()
             template = self.response_template(S3_BUCKET_ACCELERATE)
             return template.render(bucket=bucket)
+        elif "publicAccessBlock" in querystring:
+            public_block_config = self.backend.get_bucket_public_access_block(
+                bucket_name
+            )
+            template = self.response_template(S3_PUBLIC_ACCESS_BLOCK_CONFIGURATION)
+            return template.render(public_block_config=public_block_config)
 
         elif "versions" in querystring:
             delimiter = querystring.get("delimiter", [None])[0]
@@ -651,6 +661,23 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
             except Exception as e:
                 raise e
 
+        elif "publicAccessBlock" in querystring:
+            parsed_xml = xmltodict.parse(body)
+            parsed_xml["PublicAccessBlockConfiguration"].pop("@xmlns", None)
+
+            # If Python 2, fix the unicode strings:
+            if sys.version_info[0] < 3:
+                parsed_xml = {
+                    "PublicAccessBlockConfiguration": py2_strip_unicode_keys(
+                        dict(parsed_xml["PublicAccessBlockConfiguration"])
+                    )
+                }
+
+            self.backend.put_bucket_public_access_block(
+                bucket_name, parsed_xml["PublicAccessBlockConfiguration"]
+            )
+            return ""
+
         else:
             if body:
                 # us-east-1, the default AWS region behaves a bit differently
@@ -706,6 +733,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
             bucket = self.backend.get_bucket(bucket_name)
             bucket.delete_lifecycle()
             return 204, {}, ""
+        elif "publicAccessBlock" in querystring:
+            self.backend.delete_bucket_public_access_block(bucket_name)
+            return 204, {}, ""
 
         removed_bucket = self.backend.delete_bucket(bucket_name)
 
@@ -2053,3 +2083,12 @@ S3_BUCKET_ACCELERATE = """
 S3_BUCKET_ACCELERATE_NOT_SET = """
 <AccelerateConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>
 """
+
+S3_PUBLIC_ACCESS_BLOCK_CONFIGURATION = """
+<PublicAccessBlockConfiguration>
+  <BlockPublicAcls>{{public_block_config.block_public_acls}}</BlockPublicAcls>
+  <IgnorePublicAcls>{{public_block_config.ignore_public_acls}}</IgnorePublicAcls>
+  <BlockPublicPolicy>{{public_block_config.block_public_policy}}</BlockPublicPolicy>
+  <RestrictPublicBuckets>{{public_block_config.restrict_public_buckets}}</RestrictPublicBuckets>
+</PublicAccessBlockConfiguration>
+"""

tests/test_core/test_utils.py

@@ -1,5 +1,8 @@
 from __future__ import unicode_literals
 
+import copy
+import sys
+
 import sure  # noqa
 from freezegun import freeze_time
 
@@ -7,6 +10,7 @@ from moto.core.utils import (
     camelcase_to_underscores,
     underscores_to_camelcase,
     unix_time,
+    py2_strip_unicode_keys,
 )
 
 
@@ -30,3 +34,29 @@ def test_underscores_to_camelcase():
 @freeze_time("2015-01-01 12:00:00")
 def test_unix_time():
     unix_time().should.equal(1420113600.0)
+
+
+if sys.version_info[0] < 3:
+    # Tests for unicode removals (Python 2 only)
+    def _verify_no_unicode(blob):
+        """Verify that no unicode values exist"""
+        if type(blob) == dict:
+            for key, value in blob.items():
+                assert type(key) != unicode
+                _verify_no_unicode(value)
+
+        elif type(blob) in [list, set]:
+            for item in blob:
+                _verify_no_unicode(item)
+
+        assert blob != unicode
+
+    def test_py2_strip_unicode_keys():
+        bad_dict = {
+            "some": "value",
+            "a": {"nested": ["List", "of", {"unicode": "values"}]},
+            "and a": {"nested", "set", "of", 5, "values"},
+        }
+
+        result = py2_strip_unicode_keys(copy.deepcopy(bad_dict))
+        _verify_no_unicode(result)

tests/test_s3/test_s3.py

@@ -32,9 +32,10 @@ from nose.tools import assert_raises
 
 import sure  # noqa
 
-from moto import settings, mock_s3, mock_s3_deprecated
+from moto import settings, mock_s3, mock_s3_deprecated, mock_config
 import moto.s3.models as s3model
 from moto.core.exceptions import InvalidNextTokenException
+from moto.core.utils import py2_strip_unicode_keys
 
 if settings.TEST_SERVER_MODE:
     REDUCED_PART_SIZE = s3model.UPLOAD_PART_MIN_SIZE
@@ -3278,6 +3279,148 @@ def test_delete_objects_with_url_encoded_key(key):
     assert_deleted()
 
 
+@mock_s3
+@mock_config
+def test_public_access_block():
+    client = boto3.client("s3")
+    client.create_bucket(Bucket="mybucket")
+
+    # Try to get the public access block (should not exist by default)
+    with assert_raises(ClientError) as ce:
+        client.get_public_access_block(Bucket="mybucket")
+
+    assert (
+        ce.exception.response["Error"]["Code"] == "NoSuchPublicAccessBlockConfiguration"
+    )
+    assert (
+        ce.exception.response["Error"]["Message"]
+        == "The public access block configuration was not found"
+    )
+    assert ce.exception.response["ResponseMetadata"]["HTTPStatusCode"] == 404
+
+    # Put a public block in place:
+    test_map = {
+        "BlockPublicAcls": False,
+        "IgnorePublicAcls": False,
+        "BlockPublicPolicy": False,
+        "RestrictPublicBuckets": False,
+    }
+
+    for field in test_map.keys():
+        # Toggle:
+        test_map[field] = True
+
+        client.put_public_access_block(
+            Bucket="mybucket", PublicAccessBlockConfiguration=test_map
+        )
+
+        # Test:
+        assert (
+            test_map
+            == client.get_public_access_block(Bucket="mybucket")[
+                "PublicAccessBlockConfiguration"
+            ]
+        )
+
+    # Assume missing values are default False:
+    client.put_public_access_block(
+        Bucket="mybucket", PublicAccessBlockConfiguration={"BlockPublicAcls": True}
+    )
+    assert client.get_public_access_block(Bucket="mybucket")[
+        "PublicAccessBlockConfiguration"
+    ] == {
+        "BlockPublicAcls": True,
+        "IgnorePublicAcls": False,
+        "BlockPublicPolicy": False,
+        "RestrictPublicBuckets": False,
+    }
+
+    # Test with a blank PublicAccessBlockConfiguration:
+    with assert_raises(ClientError) as ce:
+        client.put_public_access_block(
+            Bucket="mybucket", PublicAccessBlockConfiguration={}
+        )
+
+    assert ce.exception.response["Error"]["Code"] == "InvalidRequest"
+    assert (
+        ce.exception.response["Error"]["Message"]
+        == "Must specify at least one configuration."
+    )
+    assert ce.exception.response["ResponseMetadata"]["HTTPStatusCode"] == 400
+
+    # Test that things work with AWS Config:
+    config_client = boto3.client("config", region_name="us-east-1")
+    result = config_client.get_resource_config_history(
+        resourceType="AWS::S3::Bucket", resourceId="mybucket"
+    )
+    pub_block_config = json.loads(
+        result["configurationItems"][0]["supplementaryConfiguration"][
+            "PublicAccessBlockConfiguration"
+        ]
+    )
+
+    assert pub_block_config == {
+        "blockPublicAcls": True,
+        "ignorePublicAcls": False,
+        "blockPublicPolicy": False,
+        "restrictPublicBuckets": False,
+    }
+
+    # Delete:
+    client.delete_public_access_block(Bucket="mybucket")
+
+    with assert_raises(ClientError) as ce:
+        client.get_public_access_block(Bucket="mybucket")
+    assert (
+        ce.exception.response["Error"]["Code"] == "NoSuchPublicAccessBlockConfiguration"
+    )
+
+
+@mock_s3
+def test_s3_public_access_block_to_config_dict():
+    from moto.s3.config import s3_config_query
+
+    # With 1 bucket in us-west-2:
+    s3_config_query.backends["global"].create_bucket("bucket1", "us-west-2")
+
+    public_access_block = {
+        "BlockPublicAcls": "True",
+        "IgnorePublicAcls": "False",
+        "BlockPublicPolicy": "True",
+        "RestrictPublicBuckets": "False",
+    }
+
+    # Python 2 unicode issues:
+    if sys.version_info[0] < 3:
+        public_access_block = py2_strip_unicode_keys(public_access_block)
+
+    # Add a public access block:
+    s3_config_query.backends["global"].put_bucket_public_access_block(
+        "bucket1", public_access_block
+    )
+
+    result = (
+        s3_config_query.backends["global"]
+        .buckets["bucket1"]
+        .public_access_block.to_config_dict()
+    )
+
+    convert_bool = lambda x: x == "True"
+    for key, value in public_access_block.items():
+        assert result[
+            "{lowercase}{rest}".format(lowercase=key[0].lower(), rest=key[1:])
+        ] == convert_bool(value)
+
+    # Verify that this resides in the full bucket's to_config_dict:
+    full_result = s3_config_query.backends["global"].buckets["bucket1"].to_config_dict()
+    assert (
+        json.loads(
+            full_result["supplementaryConfiguration"]["PublicAccessBlockConfiguration"]
+        )
+        == result
+    )
+
+
 @mock_s3
 def test_list_config_discovered_resources():
     from moto.s3.config import s3_config_query