Merge pull request #2834 from bblommers/feature/cloudwatch-integrated-metrics

Cloudwatch - Add default metrics for other services
2020-04-25 18:35:17 -05:00 · 2020-04-25 18:35:17 -05:00 · f4338876ff
commit f4338876ff
parent bd650d9b48 c3865532f9
6 changed files with 133 additions and 17 deletions
--- a/moto/cloudwatch/models.py
+++ b/moto/cloudwatch/models.py
@ -22,6 +22,14 @@ class Dimension(object):
        self.name = name
        self.value = value

+    def __eq__(self, item):
+        if isinstance(item, Dimension):
+            return self.name == item.name and self.value == item.value
+        return False
+
+    def __ne__(self, item):  # Only needed on Py2; Py3 defines it implicitly
+        return self != item
+

 def daterange(start, stop, step=timedelta(days=1), inclusive=False):
    """
@ -124,6 +132,17 @@ class MetricDatum(BaseModel):
            Dimension(dimension["Name"], dimension["Value"]) for dimension in dimensions
        ]

+    def filter(self, namespace, name, dimensions):
+        if namespace and namespace != self.namespace:
+            return False
+        if name and name != self.name:
+            return False
+        if dimensions and any(
+            Dimension(d["Name"], d["Value"]) not in self.dimensions for d in dimensions
+        ):
+            return False
+        return True
+

 class Dashboard(BaseModel):
    def __init__(self, name, body):
@ -202,6 +221,15 @@ class CloudWatchBackend(BaseBackend):
        self.metric_data = []
        self.paged_metric_data = {}

+    @property
+    # Retrieve a list of all OOTB metrics that are provided by metrics providers
+    # Computed on the fly
+    def aws_metric_data(self):
+        md = []
+        for name, service in metric_providers.items():
+            md.extend(service.get_cloudwatch_metrics())
+        return md
+
    def put_metric_alarm(
        self,
        name,
@ -334,7 +362,7 @@ class CloudWatchBackend(BaseBackend):
        return data

    def get_all_metrics(self):
-        return self.metric_data
+        return self.metric_data + self.aws_metric_data

    def put_dashboard(self, name, body):
        self.dashboards[name] = Dashboard(name, body)
@ -386,7 +414,7 @@ class CloudWatchBackend(BaseBackend):

        self.alarms[alarm_name].update_state(reason, reason_data, state_value)

-    def list_metrics(self, next_token, namespace, metric_name):
+    def list_metrics(self, next_token, namespace, metric_name, dimensions):
        if next_token:
            if next_token not in self.paged_metric_data:
                raise RESTError(
@ -397,15 +425,16 @@ class CloudWatchBackend(BaseBackend):
                del self.paged_metric_data[next_token]  # Cant reuse same token twice
                return self._get_paginated(metrics)
        else:
-            metrics = self.get_filtered_metrics(metric_name, namespace)
+            metrics = self.get_filtered_metrics(metric_name, namespace, dimensions)
            return self._get_paginated(metrics)

-    def get_filtered_metrics(self, metric_name, namespace):
+    def get_filtered_metrics(self, metric_name, namespace, dimensions):
        metrics = self.get_all_metrics()
-        if namespace:
-            metrics = [md for md in metrics if md.namespace == namespace]
-        if metric_name:
-            metrics = [md for md in metrics if md.name == metric_name]
+        metrics = [
+            md
+            for md in metrics
+            if md.filter(namespace=namespace, name=metric_name, dimensions=dimensions)
+        ]
        return metrics

    def _get_paginated(self, metrics):
@ -445,3 +474,8 @@ for region in Session().get_available_regions(
    cloudwatch_backends[region] = CloudWatchBackend()
 for region in Session().get_available_regions("cloudwatch", partition_name="aws-cn"):
    cloudwatch_backends[region] = CloudWatchBackend()
+
+# List of services that provide OOTB CW metrics
+# See the S3Backend constructor for an example
+# TODO: We might have to separate this out per region for non-global services
+metric_providers = {}
--- a/moto/cloudwatch/responses.py
+++ b/moto/cloudwatch/responses.py
@ -124,9 +124,10 @@ class CloudWatchResponse(BaseResponse):
    def list_metrics(self):
        namespace = self._get_param("Namespace")
        metric_name = self._get_param("MetricName")
+        dimensions = self._get_multi_param("Dimensions.member")
        next_token = self._get_param("NextToken")
        next_token, metrics = self.cloudwatch_backend.list_metrics(
-            next_token, namespace, metric_name
+            next_token, namespace, metric_name, dimensions
        )
        template = self.response_template(LIST_METRICS_TEMPLATE)
        return template.render(metrics=metrics, next_token=next_token)
@ -342,7 +343,7 @@ LIST_METRICS_TEMPLATE = """<ListMetricsResponse xmlns="http://monitoring.amazona
                    </member>
                    {% endfor %}
                </Dimensions>
-                <MetricName>{{ metric.name }}</MetricName>
+                <MetricName>Metric:{{ metric.name }}</MetricName>
                <Namespace>{{ metric.namespace }}</Namespace>
            </member>
            {% endfor %}
--- a/moto/s3/models.py
+++ b/moto/s3/models.py
@ -22,6 +22,7 @@ import six
 from bisect import insort
 from moto.core import ACCOUNT_ID, BaseBackend, BaseModel
 from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
+from moto.cloudwatch.models import metric_providers, MetricDatum
 from .exceptions import (
    BucketAlreadyExists,
    MissingBucket,
@ -1181,6 +1182,38 @@ class S3Backend(BaseBackend):
    def __init__(self):
        self.buckets = {}
        self.account_public_access_block = None
+        # Register this class as a CloudWatch Metric Provider
+        # Must provide a method 'get_cloudwatch_metrics' that will return a list of metrics, based on the data available
+        metric_providers["S3"] = self
+
+    def get_cloudwatch_metrics(self):
+        metrics = []
+        for name, bucket in self.buckets.items():
+            metrics.append(
+                MetricDatum(
+                    namespace="AWS/S3",
+                    name="BucketSizeBytes",
+                    value=bucket.keys.item_size(),
+                    dimensions=[
+                        {"Name": "StorageType", "Value": "StandardStorage"},
+                        {"Name": "BucketName", "Value": name},
+                    ],
+                    timestamp=datetime.datetime.now(),
+                )
+            )
+            metrics.append(
+                MetricDatum(
+                    namespace="AWS/S3",
+                    name="NumberOfObjects",
+                    value=len(bucket.keys),
+                    dimensions=[
+                        {"Name": "StorageType", "Value": "AllStorageTypes"},
+                        {"Name": "BucketName", "Value": name},
+                    ],
+                    timestamp=datetime.datetime.now(),
+                )
+            )
+        return metrics

    def create_bucket(self, bucket_name, region_name):
        if bucket_name in self.buckets:
--- a/moto/s3/utils.py
+++ b/moto/s3/utils.py
@ -146,6 +146,12 @@ class _VersionedKeyStore(dict):
        for key in self:
            yield key, self.getlist(key)

+    def item_size(self):
+        size = 0
+        for val in self.values():
+            size += sys.getsizeof(val)
+        return size
+
    items = iteritems = _iteritems
    lists = iterlists = _iterlists
    values = itervalues = _itervalues
--- a/tests/test_cloudwatch/test_cloudwatch.py
+++ b/tests/test_cloudwatch/test_cloudwatch.py
@ -1,9 +1,10 @@
 import boto
 from boto.ec2.cloudwatch.alarm import MetricAlarm
+from boto.s3.key import Key
 from datetime import datetime
 import sure  # noqa

-from moto import mock_cloudwatch_deprecated
+from moto import mock_cloudwatch_deprecated, mock_s3_deprecated


 def alarm_fixture(name="tester", action=None):
@ -83,10 +84,11 @@ def test_put_metric_data():
    )

    metrics = conn.list_metrics()
-    metrics.should.have.length_of(1)
+    metric_names = [m for m in metrics if m.name == "metric"]
+    metric_names.should.have(1)
    metric = metrics[0]
    metric.namespace.should.equal("tester")
-    metric.name.should.equal("metric")
+    metric.name.should.equal("Metric:metric")
    dict(metric.dimensions).should.equal({"InstanceId": ["i-0123456,i-0123457"]})


@ -153,3 +155,35 @@ def test_get_metric_statistics():
    datapoint = datapoints[0]
    datapoint.should.have.key("Minimum").which.should.equal(1.5)
    datapoint.should.have.key("Timestamp").which.should.equal(metric_timestamp)
+
+
+@mock_s3_deprecated
+@mock_cloudwatch_deprecated
+def test_cloudwatch_return_s3_metrics():
+
+    region = "us-east-1"
+
+    cw = boto.ec2.cloudwatch.connect_to_region(region)
+    s3 = boto.s3.connect_to_region(region)
+
+    bucket_name_1 = "test-bucket-1"
+    bucket_name_2 = "test-bucket-2"
+
+    bucket1 = s3.create_bucket(bucket_name=bucket_name_1)
+    key = Key(bucket1)
+    key.key = "the-key"
+    key.set_contents_from_string("foobar" * 4)
+    s3.create_bucket(bucket_name=bucket_name_2)
+
+    metrics_s3_bucket_1 = cw.list_metrics(dimensions={"BucketName": bucket_name_1})
+    # Verify that the OOTB S3 metrics are available for the created buckets
+    len(metrics_s3_bucket_1).should.be(2)
+    metric_names = [m.name for m in metrics_s3_bucket_1]
+    sorted(metric_names).should.equal(
+        ["Metric:BucketSizeBytes", "Metric:NumberOfObjects"]
+    )
+
+    # Explicit clean up - the metrics for these buckets are messing with subsequent tests
+    key.delete()
+    s3.delete_bucket(bucket_name_1)
+    s3.delete_bucket(bucket_name_2)
--- a/tests/test_cloudwatch/test_cloudwatch_boto3.py
+++ b/tests/test_cloudwatch/test_cloudwatch_boto3.py
@ -154,7 +154,7 @@ def test_put_metric_data_no_dimensions():
    metrics.should.have.length_of(1)
    metric = metrics[0]
    metric["Namespace"].should.equal("tester")
-    metric["MetricName"].should.equal("metric")
+    metric["MetricName"].should.equal("Metric:metric")


@mock_cloudwatch
@ -182,7 +182,7 @@ def test_put_metric_data_with_statistics():
    metrics.should.have.length_of(1)
    metric = metrics[0]
    metric["Namespace"].should.equal("tester")
-    metric["MetricName"].should.equal("statmetric")
+    metric["MetricName"].should.equal("Metric:statmetric")
    # TODO: test statistics - https://github.com/spulec/moto/issues/1615


@ -233,8 +233,16 @@ def test_list_metrics():
    # Verify format
    res.should.equal(
        [
-            {u"Namespace": "list_test_1/", u"Dimensions": [], u"MetricName": "metric1"},
-            {u"Namespace": "list_test_1/", u"Dimensions": [], u"MetricName": "metric1"},
+            {
+                u"Namespace": "list_test_1/",
+                u"Dimensions": [],
+                u"MetricName": "Metric:metric1",
+            },
+            {
+                u"Namespace": "list_test_1/",
+                u"Dimensions": [],
+                u"MetricName": "Metric:metric1",
+            },
        ]
    )
    # Verify unknown namespace still has no results