Added AWS Config batching capabilities

- Support for aggregated and non-aggregated batching
- Currently only implemented for S3

moto/config/models.py

@@ -18,7 +18,7 @@ from moto.config.exceptions import InvalidResourceTypeException, InvalidDelivery
     NoSuchDeliveryChannelException, LastDeliveryChannelDeleteFailedException, TagKeyTooBig, \
     TooManyTags, TagValueTooBig, TooManyAccountSources, InvalidParameterValueException, InvalidNextTokenException, \
     NoSuchConfigurationAggregatorException, InvalidTagCharacters, DuplicateTags, InvalidLimit, InvalidResourceParameters, \
-    TooManyResourceIds, ResourceNotDiscoveredException
+    TooManyResourceIds, ResourceNotDiscoveredException, TooManyResourceKeys
 
 from moto.core import BaseBackend, BaseModel
 from moto.s3.config import s3_config_query
@@ -791,7 +791,7 @@ class ConfigBackend(BaseBackend):
 
         return result
 
-    def get_resource_config_history(self, type, id, backend_region):
+    def get_resource_config_history(self, resource_type, id, backend_region):
         """Returns the configuration of an item in the AWS Config format of the resource for the current regional backend.
 
         NOTE: This is --NOT-- returning history as it is not supported in moto at this time. (PR's welcome!)
@@ -799,30 +799,102 @@ class ConfigBackend(BaseBackend):
               return 1 item. (If no items, it raises an exception)
         """
         # If the type isn't implemented then we won't find the item:
-        if type not in RESOURCE_MAP:
+        if resource_type not in RESOURCE_MAP:
-            raise ResourceNotDiscoveredException(type, id)
+            raise ResourceNotDiscoveredException(resource_type, id)
 
         # Is the resource type global?
-        if RESOURCE_MAP[type].backends.get('global'):
+        if RESOURCE_MAP[resource_type].backends.get('global'):
             backend_region = 'global'
 
         # If the backend region isn't implemented then we won't find the item:
-        if not RESOURCE_MAP[type].backends.get(backend_region):
+        if not RESOURCE_MAP[resource_type].backends.get(backend_region):
-            raise ResourceNotDiscoveredException(type, id)
+            raise ResourceNotDiscoveredException(resource_type, id)
 
         # Get the item:
-        item = RESOURCE_MAP[type].get_config_resource(id, backend_region=backend_region)
+        item = RESOURCE_MAP[resource_type].get_config_resource(id, backend_region=backend_region)
         if not item:
-            raise ResourceNotDiscoveredException(type, id)
+            raise ResourceNotDiscoveredException(resource_type, id)
 
         item['accountId'] = DEFAULT_ACCOUNT_ID
 
         return {'configurationItems': [item]}
 
     def batch_get_resource_config(self, resource_keys, backend_region):
-        """Returns the configuration of an item in the AWS Config format of the resource for the current regional backend."""
+        """Returns the configuration of an item in the AWS Config format of the resource for the current regional backend.
+
+        :param resource_keys:
+        :param backend_region:
+        """
         # Can't have more than 100 items
-        pass
+        if len(resource_keys) > 100:
+            raise TooManyResourceKeys(['com.amazonaws.starling.dove.ResourceKey@12345'] * len(resource_keys))
+
+        results = []
+        for resource in resource_keys:
+            # Does the resource type exist?
+            if not RESOURCE_MAP.get(resource['resourceType']):
+                # Not found so skip.
+                continue
+
+            # Is the resource type global?
+            if RESOURCE_MAP[resource['resourceType']].backends.get('global'):
+                backend_region = 'global'
+
+            # If the backend region isn't implemented then we won't find the item:
+            if not RESOURCE_MAP[resource['resourceType']].backends.get(backend_region):
+                continue
+
+            # Get the item:
+            item = RESOURCE_MAP[resource['resourceType']].get_config_resource(resource['resourceId'], backend_region=backend_region)
+            if not item:
+                continue
+
+            item['accountId'] = DEFAULT_ACCOUNT_ID
+
+            results.append(item)
+
+        return {'baseConfigurationItems': results, 'unprocessedResourceKeys': []}   # At this time, moto is not adding unprocessed items.
+
+    def batch_get_aggregate_resource_config(self, aggregator_name, resource_identifiers):
+        """Returns the configuration of an item in the AWS Config format of the resource for the current regional backend.
+
+            As far a moto goes -- the only real difference between this function and the `batch_get_resource_config` function is that
+            this will require a Config Aggregator be set up a priori and can search based on resource regions.
+
+            Note: moto will IGNORE the resource account ID in the search query.
+        """
+        if not self.config_aggregators.get(aggregator_name):
+            raise NoSuchConfigurationAggregatorException()
+
+        # Can't have more than 100 items
+        if len(resource_identifiers) > 100:
+            raise TooManyResourceKeys(['com.amazonaws.starling.dove.AggregateResourceIdentifier@12345'] * len(resource_identifiers))
+
+        found = []
+        not_found = []
+        for identifier in resource_identifiers:
+            resource_type = identifier['ResourceType']
+            resource_region = identifier['SourceRegion']
+            resource_id = identifier['ResourceId']
+            resource_name = identifier.get('ResourceName', None)
+
+            # Does the resource type exist?
+            if not RESOURCE_MAP.get(resource_type):
+                not_found.append(identifier)
+                continue
+
+            # Get the item:
+            item = RESOURCE_MAP[resource_type].get_config_resource(resource_id, resource_name=resource_name,
+                                                                   resource_region=resource_region)
+            if not item:
+                not_found.append(identifier)
+                continue
+
+            item['accountId'] = DEFAULT_ACCOUNT_ID
+
+            found.append(item)
+
+        return {'BaseConfigurationItems': found, 'UnprocessedResourceIdentifiers': not_found}
 
 
 config_backends = {}

moto/config/responses.py

@@ -108,12 +108,12 @@ class ConfigResponse(BaseResponse):
                                                                  self.region)
         return json.dumps(schema)
 
-    """
     def batch_get_resource_config(self):
-        # TODO implement me!
+        schema = self.config_backend.batch_get_resource_config(self._get_param('resourceKeys'),
-        return ""
+                                                               self.region)
+        return json.dumps(schema)
 
     def batch_get_aggregate_resource_config(self):
-        # TODO implement me!
+        schema = self.config_backend.batch_get_aggregate_resource_config(self._get_param('ConfigurationAggregatorName'),
-        return ""
+                                                                         self._get_param('ResourceIdentifiers'))
-    """
+        return json.dumps(schema)

tests/test_config/test_config.py

@@ -1216,3 +1216,127 @@ def test_get_resource_config_history():
     assert len(result) == 1
     assert result[0]['resourceName'] == result[0]['resourceId'] == 'bucket1'
     assert result[0]['arn'] == 'arn:aws:s3:::bucket1'
+
+
+@mock_config
+@mock_s3
+def test_batch_get_resource_config():
+    """NOTE: We are only really testing the Config part. For each individual service, please add tests
+             for that individual service's "get_config_resource" function.
+    """
+    client = boto3.client('config', region_name='us-west-2')
+
+    # With more than 100 resourceKeys:
+    with assert_raises(ClientError) as ce:
+        client.batch_get_resource_config(resourceKeys=[{'resourceType': 'AWS::S3::Bucket', 'resourceId': 'someBucket'}] * 101)
+    assert 'Member must have length less than or equal to 100' in ce.exception.response['Error']['Message']
+
+    # With invalid resource types and resources that don't exist:
+    result = client.batch_get_resource_config(resourceKeys=[
+        {'resourceType': 'NOT::A::RESOURCE', 'resourceId': 'NotAThing'}, {'resourceType': 'AWS::S3::Bucket', 'resourceId': 'NotAThing'},
+    ])
+
+    assert not result['baseConfigurationItems']
+    assert not result['unprocessedResourceKeys']
+
+    # Create some S3 buckets:
+    s3_client = boto3.client('s3', region_name='us-west-2')
+    for x in range(0, 10):
+        s3_client.create_bucket(Bucket='bucket{}'.format(x), CreateBucketConfiguration={'LocationConstraint': 'us-west-2'})
+
+    # Get them all:
+    keys = [{'resourceType': 'AWS::S3::Bucket', 'resourceId': 'bucket{}'.format(x)} for x in range(0, 10)]
+    result = client.batch_get_resource_config(resourceKeys=keys)
+    assert len(result['baseConfigurationItems']) == 10
+    buckets_missing = ['bucket{}'.format(x) for x in range(0, 10)]
+    for r in result['baseConfigurationItems']:
+        buckets_missing.remove(r['resourceName'])
+
+    assert not buckets_missing
+
+
+@mock_config
+@mock_s3
+def test_batch_get_aggregate_resource_config():
+    """NOTE: We are only really testing the Config part. For each individual service, please add tests
+             for that individual service's "get_config_resource" function.
+    """
+    from moto.config.models import DEFAULT_ACCOUNT_ID
+    client = boto3.client('config', region_name='us-west-2')
+
+    # Without an aggregator:
+    bad_ri = {'SourceAccountId': '000000000000', 'SourceRegion': 'not-a-region', 'ResourceType': 'NOT::A::RESOURCE', 'ResourceId': 'nope'}
+    with assert_raises(ClientError) as ce:
+        client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='lolno', ResourceIdentifiers=[bad_ri])
+    assert 'The configuration aggregator does not exist' in ce.exception.response['Error']['Message']
+
+    # Create the aggregator:
+    account_aggregation_source = {
+        'AccountIds': [
+            '012345678910',
+            '111111111111',
+            '222222222222'
+        ],
+        'AllAwsRegions': True
+    }
+    client.put_configuration_aggregator(
+        ConfigurationAggregatorName='testing',
+        AccountAggregationSources=[account_aggregation_source]
+    )
+
+    # With more than 100 items:
+    with assert_raises(ClientError) as ce:
+        client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='testing', ResourceIdentifiers=[bad_ri] * 101)
+    assert 'Member must have length less than or equal to 100' in ce.exception.response['Error']['Message']
+
+    # Create some S3 buckets:
+    s3_client = boto3.client('s3', region_name='us-west-2')
+    for x in range(0, 10):
+        s3_client.create_bucket(Bucket='bucket{}'.format(x), CreateBucketConfiguration={'LocationConstraint': 'us-west-2'})
+
+    s3_client_eu = boto3.client('s3', region_name='eu-west-1')
+    for x in range(10, 12):
+        s3_client_eu.create_bucket(Bucket='eu-bucket{}'.format(x), CreateBucketConfiguration={'LocationConstraint': 'eu-west-1'})
+
+    # Now try with resources that exist and ones that don't:
+    identifiers = [{'SourceAccountId': DEFAULT_ACCOUNT_ID, 'SourceRegion': 'us-west-2', 'ResourceType': 'AWS::S3::Bucket',
+                    'ResourceId': 'bucket{}'.format(x)} for x in range(0, 10)]
+    identifiers += [{'SourceAccountId': DEFAULT_ACCOUNT_ID, 'SourceRegion': 'eu-west-1', 'ResourceType': 'AWS::S3::Bucket',
+                    'ResourceId': 'eu-bucket{}'.format(x)} for x in range(10, 12)]
+    identifiers += [bad_ri]
+
+    result = client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='testing', ResourceIdentifiers=identifiers)
+    assert len(result['UnprocessedResourceIdentifiers']) == 1
+    assert result['UnprocessedResourceIdentifiers'][0] == bad_ri
+
+    # Verify all the buckets are there:
+    assert len(result['BaseConfigurationItems']) == 12
+    missing_buckets = ['bucket{}'.format(x) for x in range(0, 10)] + ['eu-bucket{}'.format(x) for x in range(10, 12)]
+
+    for r in result['BaseConfigurationItems']:
+        missing_buckets.remove(r['resourceName'])
+
+    assert not missing_buckets
+
+    # Verify that if the resource name and ID are correct that things are good:
+    identifiers = [{'SourceAccountId': DEFAULT_ACCOUNT_ID, 'SourceRegion': 'us-west-2', 'ResourceType': 'AWS::S3::Bucket',
+                    'ResourceId': 'bucket1', 'ResourceName': 'bucket1'}]
+    result = client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='testing', ResourceIdentifiers=identifiers)
+    assert not result['UnprocessedResourceIdentifiers']
+    assert len(result['BaseConfigurationItems']) == 1 and result['BaseConfigurationItems'][0]['resourceName'] == 'bucket1'
+
+    # Verify that if the resource name and ID mismatch that we don't get a result:
+    identifiers = [{'SourceAccountId': DEFAULT_ACCOUNT_ID, 'SourceRegion': 'us-west-2', 'ResourceType': 'AWS::S3::Bucket',
+                    'ResourceId': 'bucket1', 'ResourceName': 'bucket2'}]
+    result = client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='testing', ResourceIdentifiers=identifiers)
+    assert not result['BaseConfigurationItems']
+    assert len(result['UnprocessedResourceIdentifiers']) == 1
+    assert len(result['UnprocessedResourceIdentifiers']) == 1 and result['UnprocessedResourceIdentifiers'][0]['ResourceName'] == 'bucket2'
+
+    # Verify that if the region is incorrect that we don't get a result:
+    identifiers = [{'SourceAccountId': DEFAULT_ACCOUNT_ID, 'SourceRegion': 'eu-west-1', 'ResourceType': 'AWS::S3::Bucket',
+                    'ResourceId': 'bucket1'}]
+    result = client.batch_get_aggregate_resource_config(ConfigurationAggregatorName='testing', ResourceIdentifiers=identifiers)
+    assert not result['BaseConfigurationItems']
+    assert len(result['UnprocessedResourceIdentifiers']) == 1
+    assert len(result['UnprocessedResourceIdentifiers']) == 1 and result['UnprocessedResourceIdentifiers'][0]['SourceRegion'] == 'eu-west-1'