Added AWS Config batching capabilities
- Support for aggregated and non-aggregated batching - Currently only implemented for S3
This commit is contained in:
Mike Grima
parent
893f0d4f83
commit
96f0666df9
@ -18,7 +18,7 @@ from moto.config.exceptions import InvalidResourceTypeException, InvalidDelivery
|
||||
NoSuchDeliveryChannelException, LastDeliveryChannelDeleteFailedException, TagKeyTooBig, \
|
||||
TooManyTags, TagValueTooBig, TooManyAccountSources, InvalidParameterValueException, InvalidNextTokenException, \
|
||||
NoSuchConfigurationAggregatorException, InvalidTagCharacters, DuplicateTags, InvalidLimit, InvalidResourceParameters, \
|
||||
TooManyResourceIds, ResourceNotDiscoveredException
|
||||
TooManyResourceIds, ResourceNotDiscoveredException, TooManyResourceKeys
|
||||
|
||||
from moto.core import BaseBackend, BaseModel
|
||||
from moto.s3.config import s3_config_query
|
||||
@ -791,7 +791,7 @@ class ConfigBackend(BaseBackend):
|
||||
|
||||
return result
|
||||
|
||||
def get_resource_config_history(self, type, id, backend_region):
|
||||
def get_resource_config_history(self, resource_type, id, backend_region):
|
||||
"""Returns the configuration of an item in the AWS Config format of the resource for the current regional backend.
|
||||
|
||||
NOTE: This is --NOT-- returning history as it is not supported in moto at this time. (PR's welcome!)
|
||||
@ -799,30 +799,102 @@ class ConfigBackend(BaseBackend):
|
||||
return 1 item. (If no items, it raises an exception)
|
||||
"""
|
||||
# If the type isn't implemented then we won't find the item:
|
||||
if type not in RESOURCE_MAP:
|
||||
raise ResourceNotDiscoveredException(type, id)
|
||||
if resource_type not in RESOURCE_MAP:
|
||||
raise ResourceNotDiscoveredException(resource_type, id)
|
||||
|
||||
# Is the resource type global?
|
||||
if RESOURCE_MAP[type].backends.get('global'):
|
||||
if RESOURCE_MAP[resource_type].backends.get('global'):
|
||||
backend_region = 'global'
|
||||
|
||||
# If the backend region isn't implemented then we won't find the item:
|
||||
if not RESOURCE_MAP[type].backends.get(backend_region):
|
||||
raise ResourceNotDiscoveredException(type, id)
|
||||
if not RESOURCE_MAP[resource_type].backends.get(backend_region):
|
||||
raise ResourceNotDiscoveredException(resource_type, id)
|
||||
|
||||
# Get the item:
|
||||
item = RESOURCE_MAP[type].get_config_resource(id, backend_region=backend_region)
|
||||
item = RESOURCE_MAP[resource_type].get_config_resource(id, backend_region=backend_region)
|
||||
if not item:
|
||||
raise ResourceNotDiscoveredException(type, id)
|
||||
raise ResourceNotDiscoveredException(resource_type, id)
|
||||
|
||||
item['accountId'] = DEFAULT_ACCOUNT_ID
|
||||
|
||||
return {'configurationItems': [item]}
|
||||
|
||||
def batch_get_resource_config(self, resource_keys, backend_region):
|
||||
"""Returns the configuration of an item in the AWS Config format of the resource for the current regional backend."""
|
||||
"""Returns the configuration of an item in the AWS Config format of the resource for the current regional backend.
|
||||
|
||||
:param resource_keys:
|
||||
:param backend_region:
|
||||
"""
|
||||
# Can't have more than 100 items
|
||||
pass
|
||||
if len(resource_keys) > 100:
|
||||
raise TooManyResourceKeys(['com.amazonaws.starling.dove.ResourceKey@12345'] * len(resource_keys))
|
||||
|
||||
results = []
|
||||
for resource in resource_keys:
|
||||
# Does the resource type exist?
|
||||
if not RESOURCE_MAP.get(resource['resourceType']):
|
||||
# Not found so skip.
|
||||
continue
|
||||
|
||||
# Is the resource type global?
|
||||
if RESOURCE_MAP[resource['resourceType']].backends.get('global'):
|
||||
backend_region = 'global'
|
||||
|
||||
# If the backend region isn't implemented then we won't find the item:
|
||||
if not RESOURCE_MAP[resource['resourceType']].backends.get(backend_region):
|
||||
continue
|
||||
|
||||
# Get the item:
|
||||
item = RESOURCE_MAP[resource['resourceType']].get_config_resource(resource['resourceId'], backend_region=backend_region)
|
||||
if not item:
|
||||
continue
|
||||
|
||||
item['accountId'] = DEFAULT_ACCOUNT_ID
|
||||
|
||||
results.append(item)
|
||||
|
||||
return {'baseConfigurationItems': results, 'unprocessedResourceKeys': []} # At this time, moto is not adding unprocessed items.
|
||||
|
||||
def batch_get_aggregate_resource_config(self, aggregator_name, resource_identifiers):
|
||||
"""Returns the configuration of an item in the AWS Config format of the resource for the current regional backend.
|
||||
|
||||
As far a moto goes -- the only real difference between this function and the `batch_get_resource_config` function is that
|
||||
this will require a Config Aggregator be set up a priori and can search based on resource regions.
|
||||
|
||||
Note: moto will IGNORE the resource account ID in the search query.
|
||||
"""
|
||||
if not self.config_aggregators.get(aggregator_name):
|
||||
raise NoSuchConfigurationAggregatorException()
|
||||
|
||||
# Can't have more than 100 items
|
||||
if len(resource_identifiers) > 100:
|
||||
raise TooManyResourceKeys(['com.amazonaws.starling.dove.AggregateResourceIdentifier@12345'] * len(resource_identifiers))
|
||||
|
||||
found = []
|
||||
not_found = []
|
||||
for identifier in resource_identifiers:
|
||||
resource_type = identifier['ResourceType']
|
||||
resource_region = identifier['SourceRegion']
|
||||
resource_id = identifier['ResourceId']
|
||||
resource_name = identifier.get('ResourceName', None)
|
||||
|
||||
# Does the resource type exist?
|
||||
if not RESOURCE_MAP.get(resource_type):
|
||||
not_found.append(identifier)
|
||||
continue
|
||||
|
||||
# Get the item:
|
||||
item = RESOURCE_MAP[resource_type].get_config_resource(resource_id, resource_name=resource_name,
|
||||
resource_region=resource_region)
|
||||
if not item:
|
||||
not_found.append(identifier)
|
||||
continue
|
||||
|
||||
item['accountId'] = DEFAULT_ACCOUNT_ID
|
||||
|
||||
found.append(item)
|
||||
|
||||
return {'BaseConfigurationItems': found, 'UnprocessedResourceIdentifiers': not_found}
|
||||
|
||||
|
||||
config_backends = {}
|
||||
|
||||
@ -108,12 +108,12 @@ class ConfigResponse(BaseResponse):
|
||||
self.region)
|
||||
return json.dumps(schema)
|
||||
|
||||
"""
|
||||
def batch_get_resource_config(self):
|
||||
# TODO implement me!
|
||||
return ""
|
||||
schema = self.config_backend.batch_get_resource_config(self._get_param('resourceKeys'),
|
||||
self.region)
|
||||
return json.dumps(schema)
|
||||
|
||||
def batch_get_aggregate_resource_config(self):
|
||||
# TODO implement me!
|
||||
return ""
|
||||
"""
|
||||
schema = self.config_backend.batch_get_aggregate_resource_config(self._get_param('ConfigurationAggregatorName'),
|
||||
self._get_param('ResourceIdentifiers'))
|
||||
return json.dumps(schema)
|
||||
|
||||
@ -1216,3 +1216,127 @@ def test_get_resource_config_history():
|
||||
assert len(result) == 1
|
||||
assert result[0]['resourceName'] == result[0]['resourceId'] == 'bucket1'
|
||||
assert result[0]['arn'] == 'arn:aws:s3:::bucket1'
|
||||
|
||||
|
||||
@mock_config
|
||||
@mock_s3
|
||||
def test_batch_get_resource_config():
|
||||
"""NOTE: We are only really testing the Config part. For each individual service, please add tests
|
||||
for that individual service's "get_config_resource" function.
|
||||
"""
|
||||
client = boto3.client('config', region_name='us-west-2')
|
||||
|
||||
# With more than 100 resourceKeys:
|
||||
with assert_raises(ClientError) as ce:
|
||||
client.batch_get_resource_config(resourceKeys=[{'resourceType': 'AWS::S3::Bucket', 'resourceId': 'someBucket'}] * 101)
|
||||
assert 'Member must have length less than or equal to 100' in ce.exception.response['Error']['Message']
|
||||
|
||||
# With invalid resource types and resources that don't exist:
|
||||
result = client.batch_get_resource_config(resourceKeys=[
|
||||
{'resourceType': 'NOT::A::RESOURCE', 'resourceId': 'NotAThing'}, {'resourceType': 'AWS::S3::Bucket', 'resourceId': 'NotAThing'},
|
||||
])
|
||||
|
||||
assert not result['baseConfigurationItems']
|
||||
assert not result['unprocessedResourceKeys']
|
||||
|
||||
# Create some S3 buckets:
|
||||
s3_client = boto3.client('s3', region_name='us-west-2')
|
||||
for x in range(0, 10):
|
||||
s3_client.create_bucket(Bucket='bucket{}'.format(x), CreateBucketConfiguration={'LocationConstraint': 'us-west-2'})
|
||||
|
||||
# Get them all:
|
||||
keys = [{'resourceType': 'AWS::S3::Bucket', 'resourceId': 'bucket{}'.format(x)} for x in range(0, 10)]
|
||||
result = client.batch_get_resource_config(resourceKeys=keys)
|
||||
assert len(result['baseConfigurationItems']) == 10
|
||||
buckets_missing = ['bucket{}'.format(x) for x in range(0, 10)]
|
||||
for r in result['baseConfigurationItems']:
|
||||
buckets_missing.remove(r['resourceName'])
|
||||
|
||||
assert not buckets_missing
|
||||
|
||||
|
||||
@mock_config
|
||||
@mock_s3
|
||||
def test_batch_get_aggregate_resource_config():
|
||||
"""NOTE: We are only really testing the Config part. For each individual service, please add tests
|
||||
for that individual service's "get_config_resource" function.
|
||||
"""
|
||||
from moto.config.models import DEFAULT_ACCOUNT_ID
|
||||
client = boto3.client('config', region_name='us-west-2')
|
||||
|
||||
# Without an aggregator:
|
||||
bad_ri = {'SourceAccountId': '000000000000', 'SourceRegion': 'not-a-region', 'ResourceType': 'NOT::A::RESOURCE', 'ResourceId': 'nope'}
|
||||
with assert_raises(ClientError) as ce:
|
||||
client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='lolno', ResourceIdentifiers=[bad_ri])
|
||||
assert 'The configuration aggregator does not exist' in ce.exception.response['Error']['Message']
|
||||
|
||||
# Create the aggregator:
|
||||
account_aggregation_source = {
|
||||
'AccountIds': [
|
||||
'012345678910',
|
||||
'111111111111',
|
||||
'222222222222'
|
||||
],
|
||||
'AllAwsRegions': True
|
||||
}
|
||||
client.put_configuration_aggregator(
|
||||
ConfigurationAggregatorName='testing',
|
||||
AccountAggregationSources=[account_aggregation_source]
|
||||
)
|
||||
|
||||
# With more than 100 items:
|
||||
with assert_raises(ClientError) as ce:
|
||||
client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='testing', ResourceIdentifiers=[bad_ri] * 101)
|
||||
assert 'Member must have length less than or equal to 100' in ce.exception.response['Error']['Message']
|
||||
|
||||
# Create some S3 buckets:
|
||||
s3_client = boto3.client('s3', region_name='us-west-2')
|
||||
for x in range(0, 10):
|
||||
s3_client.create_bucket(Bucket='bucket{}'.format(x), CreateBucketConfiguration={'LocationConstraint': 'us-west-2'})
|
||||
|
||||
s3_client_eu = boto3.client('s3', region_name='eu-west-1')
|
||||
for x in range(10, 12):
|
||||
s3_client_eu.create_bucket(Bucket='eu-bucket{}'.format(x), CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
|
||||
|
||||
# Now try with resources that exist and ones that don't:
|
||||
identifiers = [{'SourceAccountId': DEFAULT_ACCOUNT_ID, 'SourceRegion': 'us-west-2', 'ResourceType': 'AWS::S3::Bucket',
|
||||
'ResourceId': 'bucket{}'.format(x)} for x in range(0, 10)]
|
||||
identifiers += [{'SourceAccountId': DEFAULT_ACCOUNT_ID, 'SourceRegion': 'eu-west-1', 'ResourceType': 'AWS::S3::Bucket',
|
||||
'ResourceId': 'eu-bucket{}'.format(x)} for x in range(10, 12)]
|
||||
identifiers += [bad_ri]
|
||||
|
||||
result = client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='testing', ResourceIdentifiers=identifiers)
|
||||
assert len(result['UnprocessedResourceIdentifiers']) == 1
|
||||
assert result['UnprocessedResourceIdentifiers'][0] == bad_ri
|
||||
|
||||
# Verify all the buckets are there:
|
||||
assert len(result['BaseConfigurationItems']) == 12
|
||||
missing_buckets = ['bucket{}'.format(x) for x in range(0, 10)] + ['eu-bucket{}'.format(x) for x in range(10, 12)]
|
||||
|
||||
for r in result['BaseConfigurationItems']:
|
||||
missing_buckets.remove(r['resourceName'])
|
||||
|
||||
assert not missing_buckets
|
||||
|
||||
# Verify that if the resource name and ID are correct that things are good:
|
||||
identifiers = [{'SourceAccountId': DEFAULT_ACCOUNT_ID, 'SourceRegion': 'us-west-2', 'ResourceType': 'AWS::S3::Bucket',
|
||||
'ResourceId': 'bucket1', 'ResourceName': 'bucket1'}]
|
||||
result = client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='testing', ResourceIdentifiers=identifiers)
|
||||
assert not result['UnprocessedResourceIdentifiers']
|
||||
assert len(result['BaseConfigurationItems']) == 1 and result['BaseConfigurationItems'][0]['resourceName'] == 'bucket1'
|
||||
|
||||
# Verify that if the resource name and ID mismatch that we don't get a result:
|
||||
identifiers = [{'SourceAccountId': DEFAULT_ACCOUNT_ID, 'SourceRegion': 'us-west-2', 'ResourceType': 'AWS::S3::Bucket',
|
||||
'ResourceId': 'bucket1', 'ResourceName': 'bucket2'}]
|
||||
result = client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='testing', ResourceIdentifiers=identifiers)
|
||||
assert not result['BaseConfigurationItems']
|
||||
assert len(result['UnprocessedResourceIdentifiers']) == 1
|
||||
assert len(result['UnprocessedResourceIdentifiers']) == 1 and result['UnprocessedResourceIdentifiers'][0]['ResourceName'] == 'bucket2'
|
||||
|
||||
# Verify that if the region is incorrect that we don't get a result:
|
||||
identifiers = [{'SourceAccountId': DEFAULT_ACCOUNT_ID, 'SourceRegion': 'eu-west-1', 'ResourceType': 'AWS::S3::Bucket',
|
||||
'ResourceId': 'bucket1'}]
|
||||
result = client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='testing', ResourceIdentifiers=identifiers)
|
||||
assert not result['BaseConfigurationItems']
|
||||
assert len(result['UnprocessedResourceIdentifiers']) == 1
|
||||
assert len(result['UnprocessedResourceIdentifiers']) == 1 and result['UnprocessedResourceIdentifiers'][0]['SourceRegion'] == 'eu-west-1'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user