Merge pull request #2834 from bblommers/feature/cloudwatch-integrated-metrics

Cloudwatch  - Add default metrics for other services

moto/cloudwatch/models.py

@@ -22,6 +22,14 @@ class Dimension(object):
         self.name = name
         self.value = value
 
+    def __eq__(self, item):
+        if isinstance(item, Dimension):
+            return self.name == item.name and self.value == item.value
+        return False
+
+    def __ne__(self, item):  # Only needed on Py2; Py3 defines it implicitly
+        return self != item
+
 
 def daterange(start, stop, step=timedelta(days=1), inclusive=False):
     """
@@ -124,6 +132,17 @@ class MetricDatum(BaseModel):
             Dimension(dimension["Name"], dimension["Value"]) for dimension in dimensions
         ]
 
+    def filter(self, namespace, name, dimensions):
+        if namespace and namespace != self.namespace:
+            return False
+        if name and name != self.name:
+            return False
+        if dimensions and any(
+            Dimension(d["Name"], d["Value"]) not in self.dimensions for d in dimensions
+        ):
+            return False
+        return True
+
 
 class Dashboard(BaseModel):
     def __init__(self, name, body):
@@ -202,6 +221,15 @@ class CloudWatchBackend(BaseBackend):
         self.metric_data = []
         self.paged_metric_data = {}
 
+    @property
+    # Retrieve a list of all OOTB metrics that are provided by metrics providers
+    # Computed on the fly
+    def aws_metric_data(self):
+        md = []
+        for name, service in metric_providers.items():
+            md.extend(service.get_cloudwatch_metrics())
+        return md
+
     def put_metric_alarm(
         self,
         name,
@@ -334,7 +362,7 @@ class CloudWatchBackend(BaseBackend):
         return data
 
     def get_all_metrics(self):
-        return self.metric_data
+        return self.metric_data + self.aws_metric_data
 
     def put_dashboard(self, name, body):
         self.dashboards[name] = Dashboard(name, body)
@@ -386,7 +414,7 @@ class CloudWatchBackend(BaseBackend):
 
         self.alarms[alarm_name].update_state(reason, reason_data, state_value)
 
-    def list_metrics(self, next_token, namespace, metric_name):
+    def list_metrics(self, next_token, namespace, metric_name, dimensions):
         if next_token:
             if next_token not in self.paged_metric_data:
                 raise RESTError(
@@ -397,15 +425,16 @@ class CloudWatchBackend(BaseBackend):
                 del self.paged_metric_data[next_token]  # Cant reuse same token twice
                 return self._get_paginated(metrics)
         else:
-            metrics = self.get_filtered_metrics(metric_name, namespace)
+            metrics = self.get_filtered_metrics(metric_name, namespace, dimensions)
             return self._get_paginated(metrics)
 
-    def get_filtered_metrics(self, metric_name, namespace):
+    def get_filtered_metrics(self, metric_name, namespace, dimensions):
         metrics = self.get_all_metrics()
-        if namespace:
+        metrics = [
-            metrics = [md for md in metrics if md.namespace == namespace]
+            md
-        if metric_name:
+            for md in metrics
-            metrics = [md for md in metrics if md.name == metric_name]
+            if md.filter(namespace=namespace, name=metric_name, dimensions=dimensions)
+        ]
         return metrics
 
     def _get_paginated(self, metrics):
@@ -445,3 +474,8 @@ for region in Session().get_available_regions(
     cloudwatch_backends[region] = CloudWatchBackend()
 for region in Session().get_available_regions("cloudwatch", partition_name="aws-cn"):
     cloudwatch_backends[region] = CloudWatchBackend()
+
+# List of services that provide OOTB CW metrics
+# See the S3Backend constructor for an example
+# TODO: We might have to separate this out per region for non-global services
+metric_providers = {}

moto/cloudwatch/responses.py

@@ -124,9 +124,10 @@ class CloudWatchResponse(BaseResponse):
     def list_metrics(self):
         namespace = self._get_param("Namespace")
         metric_name = self._get_param("MetricName")
+        dimensions = self._get_multi_param("Dimensions.member")
         next_token = self._get_param("NextToken")
         next_token, metrics = self.cloudwatch_backend.list_metrics(
-            next_token, namespace, metric_name
+            next_token, namespace, metric_name, dimensions
         )
         template = self.response_template(LIST_METRICS_TEMPLATE)
         return template.render(metrics=metrics, next_token=next_token)
@@ -342,7 +343,7 @@ LIST_METRICS_TEMPLATE = """<ListMetricsResponse xmlns="http://monitoring.amazona
                     </member>
                     {% endfor %}
                 </Dimensions>
-                <MetricName>{{ metric.name }}</MetricName>
+                <MetricName>Metric:{{ metric.name }}</MetricName>
                 <Namespace>{{ metric.namespace }}</Namespace>
             </member>
             {% endfor %}

moto/s3/models.py

@@ -22,6 +22,7 @@ import six
 from bisect import insort
 from moto.core import ACCOUNT_ID, BaseBackend, BaseModel
 from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
+from moto.cloudwatch.models import metric_providers, MetricDatum
 from .exceptions import (
     BucketAlreadyExists,
     MissingBucket,
@@ -1181,6 +1182,38 @@ class S3Backend(BaseBackend):
     def __init__(self):
         self.buckets = {}
         self.account_public_access_block = None
+        # Register this class as a CloudWatch Metric Provider
+        # Must provide a method 'get_cloudwatch_metrics' that will return a list of metrics, based on the data available
+        metric_providers["S3"] = self
+
+    def get_cloudwatch_metrics(self):
+        metrics = []
+        for name, bucket in self.buckets.items():
+            metrics.append(
+                MetricDatum(
+                    namespace="AWS/S3",
+                    name="BucketSizeBytes",
+                    value=bucket.keys.item_size(),
+                    dimensions=[
+                        {"Name": "StorageType", "Value": "StandardStorage"},
+                        {"Name": "BucketName", "Value": name},
+                    ],
+                    timestamp=datetime.datetime.now(),
+                )
+            )
+            metrics.append(
+                MetricDatum(
+                    namespace="AWS/S3",
+                    name="NumberOfObjects",
+                    value=len(bucket.keys),
+                    dimensions=[
+                        {"Name": "StorageType", "Value": "AllStorageTypes"},
+                        {"Name": "BucketName", "Value": name},
+                    ],
+                    timestamp=datetime.datetime.now(),
+                )
+            )
+        return metrics
 
     def create_bucket(self, bucket_name, region_name):
         if bucket_name in self.buckets:

moto/s3/utils.py

@@ -146,6 +146,12 @@ class _VersionedKeyStore(dict):
         for key in self:
             yield key, self.getlist(key)
 
+    def item_size(self):
+        size = 0
+        for val in self.values():
+            size += sys.getsizeof(val)
+        return size
+
     items = iteritems = _iteritems
     lists = iterlists = _iterlists
     values = itervalues = _itervalues

tests/test_cloudwatch/test_cloudwatch.py

@@ -1,9 +1,10 @@
 import boto
 from boto.ec2.cloudwatch.alarm import MetricAlarm
+from boto.s3.key import Key
 from datetime import datetime
 import sure  # noqa
 
-from moto import mock_cloudwatch_deprecated
+from moto import mock_cloudwatch_deprecated, mock_s3_deprecated
 
 
 def alarm_fixture(name="tester", action=None):
@@ -83,10 +84,11 @@ def test_put_metric_data():
     )
 
     metrics = conn.list_metrics()
-    metrics.should.have.length_of(1)
+    metric_names = [m for m in metrics if m.name == "metric"]
+    metric_names.should.have(1)
     metric = metrics[0]
     metric.namespace.should.equal("tester")
-    metric.name.should.equal("metric")
+    metric.name.should.equal("Metric:metric")
     dict(metric.dimensions).should.equal({"InstanceId": ["i-0123456,i-0123457"]})
 
 
@@ -153,3 +155,35 @@ def test_get_metric_statistics():
     datapoint = datapoints[0]
     datapoint.should.have.key("Minimum").which.should.equal(1.5)
     datapoint.should.have.key("Timestamp").which.should.equal(metric_timestamp)
+
+
+@mock_s3_deprecated
+@mock_cloudwatch_deprecated
+def test_cloudwatch_return_s3_metrics():
+
+    region = "us-east-1"
+
+    cw = boto.ec2.cloudwatch.connect_to_region(region)
+    s3 = boto.s3.connect_to_region(region)
+
+    bucket_name_1 = "test-bucket-1"
+    bucket_name_2 = "test-bucket-2"
+
+    bucket1 = s3.create_bucket(bucket_name=bucket_name_1)
+    key = Key(bucket1)
+    key.key = "the-key"
+    key.set_contents_from_string("foobar" * 4)
+    s3.create_bucket(bucket_name=bucket_name_2)
+
+    metrics_s3_bucket_1 = cw.list_metrics(dimensions={"BucketName": bucket_name_1})
+    # Verify that the OOTB S3 metrics are available for the created buckets
+    len(metrics_s3_bucket_1).should.be(2)
+    metric_names = [m.name for m in metrics_s3_bucket_1]
+    sorted(metric_names).should.equal(
+        ["Metric:BucketSizeBytes", "Metric:NumberOfObjects"]
+    )
+
+    # Explicit clean up - the metrics for these buckets are messing with subsequent tests
+    key.delete()
+    s3.delete_bucket(bucket_name_1)
+    s3.delete_bucket(bucket_name_2)

tests/test_cloudwatch/test_cloudwatch_boto3.py

@@ -154,7 +154,7 @@ def test_put_metric_data_no_dimensions():
     metrics.should.have.length_of(1)
     metric = metrics[0]
     metric["Namespace"].should.equal("tester")
-    metric["MetricName"].should.equal("metric")
+    metric["MetricName"].should.equal("Metric:metric")
 
 
 @mock_cloudwatch
@@ -182,7 +182,7 @@ def test_put_metric_data_with_statistics():
     metrics.should.have.length_of(1)
     metric = metrics[0]
     metric["Namespace"].should.equal("tester")
-    metric["MetricName"].should.equal("statmetric")
+    metric["MetricName"].should.equal("Metric:statmetric")
     # TODO: test statistics - https://github.com/spulec/moto/issues/1615
 
 
@@ -233,8 +233,16 @@ def test_list_metrics():
     # Verify format
     res.should.equal(
         [
-            {u"Namespace": "list_test_1/", u"Dimensions": [], u"MetricName": "metric1"},
+            {
-            {u"Namespace": "list_test_1/", u"Dimensions": [], u"MetricName": "metric1"},
+                u"Namespace": "list_test_1/",
+                u"Dimensions": [],
+                u"MetricName": "Metric:metric1",
+            },
+            {
+                u"Namespace": "list_test_1/",
+                u"Dimensions": [],
+                u"MetricName": "Metric:metric1",
+            },
         ]
     )
     # Verify unknown namespace still has no results