Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed bugs in AWS Config Querying

Browse Source 
- Fixed some log bugs in the Config querying APIs
- Fixed an issue with S3 ACLs when described from Config (it's actually
a double-wrapped JSON)
This commit is contained in:
Mike Grima 2019-10-23 18:37:35 -07:00
parent 577f81a18b
commit e9dc0c9a3a
5 changed files with 50 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
moto/config/models.py
View File

@ -688,7 +688,7 @@ class ConfigBackend(BaseBackend):
del self.delivery_channels[channel_name]
def list_discovered_resources(self, resource_type, backend_region, resource_ids, resource_name, limit, next_token):
"""This will query against the mocked AWS Config listing function that must exist for the resource backend.
"""This will query against the mocked AWS Config (non-aggregated) listing function that must exist for the resource backend.
:param resource_type:
:param backend_region:
@ -716,6 +716,7 @@ class ConfigBackend(BaseBackend):
# call upon the resource type's Config Query class to retrieve the list of resources that match the criteria:
if RESOURCE_MAP.get(resource_type, {}):
# Is this a global resource type? -- if so, re-write the region to 'global':
backend_query_region = backend_region # Always provide the backend this request arrived from.
if RESOURCE_MAP[resource_type].backends.get('global'):
backend_region = 'global'
@ -724,7 +725,8 @@ class ConfigBackend(BaseBackend):
if RESOURCE_MAP[resource_type].backends.get(backend_region):
# Fetch the resources for the backend's region:
identifiers, new_token = \
RESOURCE_MAP[resource_type].list_config_service_resources(resource_ids, resource_name, limit, next_token)
RESOURCE_MAP[resource_type].list_config_service_resources(resource_ids, resource_name, limit, next_token,
backend_region=backend_query_region)
result = {'resourceIdentifiers': [
{
@ -803,6 +805,7 @@ class ConfigBackend(BaseBackend):
raise ResourceNotDiscoveredException(resource_type, id)
# Is the resource type global?
backend_query_region = backend_region # Always provide the backend this request arrived from.
if RESOURCE_MAP[resource_type].backends.get('global'):
backend_region = 'global'
@ -811,7 +814,7 @@ class ConfigBackend(BaseBackend):
raise ResourceNotDiscoveredException(resource_type, id)
# Get the item:
item = RESOURCE_MAP[resource_type].get_config_resource(id, backend_region=backend_region)
item = RESOURCE_MAP[resource_type].get_config_resource(id, backend_region=backend_query_region)
if not item:
raise ResourceNotDiscoveredException(resource_type, id)
@ -837,15 +840,17 @@ class ConfigBackend(BaseBackend):
continue
# Is the resource type global?
config_backend_region = backend_region
backend_query_region = backend_region # Always provide the backend this request arrived from.
if RESOURCE_MAP[resource['resourceType']].backends.get('global'):
backend_region = 'global'
config_backend_region = 'global'
# If the backend region isn't implemented then we won't find the item:
if not RESOURCE_MAP[resource['resourceType']].backends.get(backend_region):
if not RESOURCE_MAP[resource['resourceType']].backends.get(config_backend_region):
continue
# Get the item:
item = RESOURCE_MAP[resource['resourceType']].get_config_resource(resource['resourceId'], backend_region=backend_region)
item = RESOURCE_MAP[resource['resourceType']].get_config_resource(resource['resourceId'], backend_region=backend_query_region)
if not item:
continue

16
moto/s3/config.py
View File

@ -8,8 +8,8 @@ from moto.s3 import s3_backends
class S3ConfigQuery(ConfigQueryModel):
def list_config_service_resources(self, resource_ids, resource_name, limit, next_token, backend_region=None, resource_region=None):
# S3 need not care about "backend_region" as S3 is global. The resource_region only matters for aggregated queries as you can
# filter on bucket regions for them. For other resource types, you would need to iterate appropriately for the backend_region.
# The resource_region only matters for aggregated queries as you can filter on bucket regions for them.
# For other resource types, you would need to iterate appropriately for the backend_region.
# Resource IDs are the same as S3 bucket names
# For aggregation -- did we get both a resource ID and a resource name?
@ -31,12 +31,13 @@ class S3ConfigQuery(ConfigQueryModel):
if bucket in filter_buckets:
bucket_list.append(bucket)
# If a resource_region was supplied (aggregated only), then filter on bucket region too:
if resource_region:
# Filter on the proper region if supplied:
region_filter = backend_region or resource_region
if region_filter:
region_buckets = []
for bucket in bucket_list:
if self.backends['global'].buckets[bucket].region_name == resource_region:
if self.backends['global'].buckets[bucket].region_name == region_filter:
region_buckets.append(bucket)
bucket_list = region_buckets
@ -69,8 +70,6 @@ class S3ConfigQuery(ConfigQueryModel):
for bucket in bucket_list], new_token
def get_config_resource(self, resource_id, resource_name=None, backend_region=None, resource_region=None):
# backend_region is ignored for S3 as the backend is 'global'
# Get the bucket:
bucket = self.backends['global'].buckets.get(resource_id, {})
@ -78,7 +77,8 @@ class S3ConfigQuery(ConfigQueryModel):
return
# Are we filtering based on region?
if resource_region and bucket.region_name != resource_region:
region_filter = backend_region or resource_region
if region_filter and bucket.region_name != region_filter:
return
# Are we also filtering on bucket name?

10
moto/s3/models.py
View File

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json
import os
import base64
import datetime
@ -520,7 +521,6 @@ class LifecycleRule(BaseModel):
Note: The following are missing that should be added in the future:
- transitions (returns None for now)
- noncurrentVersionTransitions (returns None for now)
- LifeCycle Filters that are NOT prefix
:param kwargs:
:return:
@ -530,9 +530,9 @@ class LifecycleRule(BaseModel):
'id': self.id,
'prefix': self.prefix,
'status': self.status,
'expirationInDays': self.expiration_days,
'expirationInDays': int(self.expiration_days) if self.expiration_days else None,
'expiredObjectDeleteMarker': self.expired_object_delete_marker,
'noncurrentVersionExpirationInDays': -1 or self.nve_noncurrent_days,
'noncurrentVersionExpirationInDays': -1 or int(self.nve_noncurrent_days),
'expirationDate': self.expiration_date,
'transitions': None, # Replace me with logic to fill in
'noncurrentVersionTransitions': None, # Replace me with logic to fill in
@ -930,7 +930,9 @@ class FakeBucket(BaseModel):
# Make the supplementary configuration:
# TODO: Implement Public Access Block Support
s_config = {'AccessControlList': self.acl.to_config_dict()}
# This is a dobule-wrapped JSON for some reason...
s_config = {'AccessControlList': json.dumps(json.dumps(self.acl.to_config_dict()))}
# TODO implement Accelerate Configuration:
s_config['BucketAccelerateConfiguration'] = {'status': None}

21
tests/test_config/test_config.py
View File

@ -1028,6 +1028,10 @@ def test_list_discovered_resource():
for x in range(0, 10):
s3_client.create_bucket(Bucket='bucket{}'.format(x), CreateBucketConfiguration={'LocationConstraint': 'us-west-2'})
# And with an EU bucket -- this should not show up for the us-west-2 config backend:
eu_client = boto3.client('s3', region_name='eu-west-1')
eu_client.create_bucket(Bucket='eu-bucket', CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
# Now try:
result = client.list_discovered_resources(resourceType='AWS::S3::Bucket')
assert len(result['resourceIdentifiers']) == 10
@ -1039,6 +1043,9 @@ def test_list_discovered_resource():
}
assert not result.get('nextToken')
result = client.list_discovered_resources(resourceType='AWS::S3::Bucket', resourceName='eu-bucket')
assert not result['resourceIdentifiers']
# Test that pagination places a proper nextToken in the response and also that the limit works:
result = client.list_discovered_resources(resourceType='AWS::S3::Bucket', limit=1, nextToken='bucket1')
assert len(result['resourceIdentifiers']) == 1
@ -1217,6 +1224,13 @@ def test_get_resource_config_history():
assert result[0]['resourceName'] == result[0]['resourceId'] == 'bucket1'
assert result[0]['arn'] == 'arn:aws:s3:::bucket1'
# Make a bucket in a different region and verify that it does not show up in the config backend:
s3_client = boto3.client('s3', region_name='eu-west-1')
s3_client.create_bucket(Bucket='eu-bucket', CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
with assert_raises(ClientError) as ce:
client.get_resource_config_history(resourceType='AWS::S3::Bucket', resourceId='eu-bucket')
assert ce.exception.response['Error']['Code'] == 'ResourceNotDiscoveredException'
@mock_config
@mock_s3
@ -1254,6 +1268,13 @@ def test_batch_get_resource_config():
assert not buckets_missing
# Make a bucket in a different region and verify that it does not show up in the config backend:
s3_client = boto3.client('s3', region_name='eu-west-1')
s3_client.create_bucket(Bucket='eu-bucket', CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
keys = [{'resourceType': 'AWS::S3::Bucket', 'resourceId': 'eu-bucket'}]
result = client.batch_get_resource_config(resourceKeys=keys)
assert not result['baseConfigurationItems']
@mock_config
@mock_s3

4
tests/test_s3/test_s3.py
View File

@ -3728,6 +3728,10 @@ def test_s3_config_dict():
assert json.loads(bucket1_result['supplementaryConfiguration']['BucketLoggingConfiguration']) == \
{'destinationBucketName': 'logbucket', 'logFilePrefix': ''}
# Verify that the AccessControlList is a double-wrapped JSON string:
assert json.loads(json.loads(bucket1_result['supplementaryConfiguration']['AccessControlList'])) == \
{'grantSet': None, 'owner': {'displayName': None, 'id': '75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a'}}
# Verify the policy:
assert json.loads(bucket1_result['supplementaryConfiguration']['BucketPolicy']) == {'policyText': policy}