CloudWatch: get_metric_data() now support (simple) Expressions (#6570)

moto/cloudwatch/metric_data_expression_parser.py

@@ -0,0 +1,13 @@
+from typing import Any, Dict, List, Tuple, SupportsFloat
+
+
+def parse_expression(
+    expression: str, results: List[Dict[str, Any]]
+) -> Tuple[List[SupportsFloat], List[str]]:
+    values: List[SupportsFloat] = []
+    timestamps: List[str] = []
+    for result in results:
+        if result.get("id") == expression:
+            values.extend(result["vals"])
+            timestamps.extend(result["timestamps"])
+    return values, timestamps

moto/cloudwatch/models.py

@@ -17,6 +17,7 @@ from .exceptions import (
     ResourceNotFoundException,
     InvalidParameterCombination,
 )
+from .metric_data_expression_parser import parse_expression
 from .utils import make_arn_for_dashboard, make_arn_for_alarm
 from dateutil import parser
 from typing import Tuple, Optional, List, Iterable, Dict, Any, SupportsFloat
@@ -668,16 +669,23 @@ class CloudWatchBackend(BaseBackend):
         ]
 
         results = []
-        for query in queries:
+        results_to_return = []
+        metric_stat_queries = [q for q in queries if "MetricStat" in q]
+        expression_queries = [q for q in queries if "Expression" in q]
+        for query in metric_stat_queries:
             period_start_time = start_time
-            query_ns = query["metric_stat._metric._namespace"]
+            metric_stat = query["MetricStat"]
-            query_name = query["metric_stat._metric._metric_name"]
+            query_ns = metric_stat["Metric"]["Namespace"]
-            delta = timedelta(seconds=int(query["metric_stat._period"]))
+            query_name = metric_stat["Metric"]["MetricName"]
-            dimensions = self._extract_dimensions_from_get_metric_data_query(query)
+            delta = timedelta(seconds=int(metric_stat["Period"]))
-            unit = query.get("metric_stat._unit")
+            dimensions = [
+                Dimension(name=d["Name"], value=d["Value"])
+                for d in metric_stat["Metric"].get("Dimensions", [])
+            ]
+            unit = metric_stat.get("Unit")
             result_vals: List[SupportsFloat] = []
             timestamps: List[str] = []
-            stat = query["metric_stat._stat"]
+            stat = metric_stat["Stat"]
             while period_start_time <= end_time:
                 period_end_time = period_start_time + delta
                 period_md = [
@@ -715,21 +723,37 @@ class CloudWatchBackend(BaseBackend):
                 timestamps.reverse()
                 result_vals.reverse()
 
-            label = (
+            label = query.get("Label") or f"{query_name} {stat}"
-                query["label"]
-                if "label" in query
-                else query["metric_stat._metric._metric_name"] + " " + stat
-            )
 
             results.append(
                 {
-                    "id": query["id"],
+                    "id": query["Id"],
                     "label": label,
                     "vals": result_vals,
                     "timestamps": timestamps,
                 }
             )
-        return results
+            if query.get("ReturnData", "true") == "true":
+                results_to_return.append(
+                    {
+                        "id": query["Id"],
+                        "label": label,
+                        "vals": result_vals,
+                        "timestamps": timestamps,
+                    }
+                )
+        for query in expression_queries:
+            label = query.get("Label") or f"{query_name} {stat}"
+            result_vals, timestamps = parse_expression(query["Expression"], results)
+            results_to_return.append(
+                {
+                    "id": query["Id"],
+                    "label": label,
+                    "vals": result_vals,
+                    "timestamps": timestamps,
+                }
+            )
+        return results_to_return
 
     def get_metric_statistics(
         self,
@@ -904,23 +928,6 @@ class CloudWatchBackend(BaseBackend):
         else:
             return None, metrics
 
-    def _extract_dimensions_from_get_metric_data_query(
-        self, query: Dict[str, str]
-    ) -> List[Dimension]:
-        dimensions = []
-        prefix = "metric_stat._metric._dimensions.member."
-        suffix_name = "._name"
-        suffix_value = "._value"
-        counter = 1
-
-        while query.get(f"{prefix}{counter}{suffix_name}") and counter <= 10:
-            name = query.get(f"{prefix}{counter}{suffix_name}")
-            value = query.get(f"{prefix}{counter}{suffix_value}")
-            dimensions.append(Dimension(name=name, value=value))
-            counter = counter + 1
-
-        return dimensions
-
     def _validate_parameters_put_metric_data(
         self, metric: Dict[str, Any], query_num: int
     ) -> None:

moto/cloudwatch/responses.py

@@ -13,7 +13,7 @@ from .models import (
     Dimension,
     FakeAlarm,
 )
-from .exceptions import InvalidParameterCombination
+from .exceptions import InvalidParameterCombination, ValidationError
 
 
 ERROR_RESPONSE = Tuple[str, Dict[str, int]]
@@ -176,11 +176,18 @@ class CloudWatchResponse(BaseResponse):
 
     @amzn_request_id
     def get_metric_data(self) -> str:
-        start = dtparse(self._get_param("StartTime"))
+        params = self._get_params()
-        end = dtparse(self._get_param("EndTime"))
+        start = dtparse(params["StartTime"])
-        scan_by = self._get_param("ScanBy")
+        end = dtparse(params["EndTime"])
+        scan_by = params.get("ScanBy") or "TimestampDescending"
 
-        queries = self._get_list_prefix("MetricDataQueries.member")
+        queries = params.get("MetricDataQueries", [])
+        for query in queries:
+            if "MetricStat" not in query and "Expression" not in query:
+                # AWS also returns the empty line
+                raise ValidationError(
+                    "The parameter MetricDataQueries.member.1.MetricStat is required.\n"
+                )
         results = self.cloudwatch_backend.get_metric_data(
             start_time=start, end_time=end, queries=queries, scan_by=scan_by
         )

tests/test_cloudwatch/test_cloudwatch_expression_parser.py

@@ -0,0 +1,40 @@
+from moto.cloudwatch.metric_data_expression_parser import parse_expression
+
+
+def test_simple_expression():
+    result_from_previous_queries = [
+        {
+            "id": "totalBytes",
+            "label": "metric Sum",
+            "vals": [25.0],
+            "timestamps": ["timestamp1"],
+        }
+    ]
+    res = parse_expression("totalBytes", result_from_previous_queries)
+    assert res == ([25.0], ["timestamp1"])
+
+
+def test_missing_expression():
+    result_from_previous_queries = [
+        {
+            "id": "totalBytes",
+            "label": "metric Sum",
+            "vals": [25.0],
+            "timestamps": ["timestamp1"],
+        }
+    ]
+    res = parse_expression("unknown", result_from_previous_queries)
+    assert res == ([], [])
+
+
+def test_complex_expression():
+    result_from_previous_queries = [
+        {
+            "id": "totalBytes",
+            "label": "metric Sum",
+            "vals": [25.0],
+            "timestamps": ["timestamp1"],
+        }
+    ]
+    res = parse_expression("totalBytes/10", result_from_previous_queries)
+    assert res == ([], [])

tests/test_cloudwatch/test_cloudwatch_expressions.py

@@ -0,0 +1,70 @@
+import boto3
+import pytest
+
+from botocore.exceptions import ClientError
+from datetime import datetime, timedelta, timezone
+from moto import mock_cloudwatch
+
+
+@mock_cloudwatch
+def test_get_metric_data__no_metric_data_or_expression():
+    utc_now = datetime.now(tz=timezone.utc)
+    cloudwatch = boto3.client("cloudwatch", "eu-west-1")
+    with pytest.raises(ClientError) as exc:
+        cloudwatch.get_metric_data(
+            MetricDataQueries=[{"Id": "result1", "Label": "e1"}],
+            StartTime=utc_now - timedelta(minutes=5),
+            EndTime=utc_now,
+            ScanBy="TimestampDescending",
+        )
+    err = exc.value.response["Error"]
+    assert err["Code"] == "ValidationError"
+    assert (
+        err["Message"]
+        == "The parameter MetricDataQueries.member.1.MetricStat is required.\n"
+    )
+
+
+@mock_cloudwatch
+def test_get_metric_data_with_simple_expression():
+    utc_now = datetime.now(tz=timezone.utc)
+    cloudwatch = boto3.client("cloudwatch", "eu-west-1")
+    namespace = "my_namespace/"
+    cloudwatch.put_metric_data(
+        Namespace=namespace,
+        MetricData=[
+            {
+                "MetricName": "metric",
+                "Value": 25,
+                "Unit": "Bytes",
+            },
+        ],
+    )
+    # get_metric_data 1
+    results = cloudwatch.get_metric_data(
+        MetricDataQueries=[
+            {
+                "Id": "result1",
+                "Expression": "totalBytes",
+                "Label": "e1",
+            },
+            {
+                "Id": "totalBytes",
+                "MetricStat": {
+                    "Metric": {"Namespace": namespace, "MetricName": "metric"},
+                    "Period": 60,
+                    "Stat": "Sum",
+                    "Unit": "Bytes",
+                },
+                "ReturnData": False,
+            },
+        ],
+        StartTime=utc_now - timedelta(minutes=5),
+        EndTime=utc_now + timedelta(minutes=5),
+        ScanBy="TimestampDescending",
+    )["MetricDataResults"]
+    #
+    assert len(results) == 1
+    assert results[0]["Id"] == "result1"
+    assert results[0]["Label"] == "e1"
+    assert results[0]["Values"] == [25.0]