From 41c5b619b02015b3f3359872959c72732555914a Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 30 Jul 2023 10:34:05 +0000 Subject: [PATCH] CloudWatch: get_metric_data() now support (simple) Expressions (#6570) --- .../metric_data_expression_parser.py | 13 ++++ moto/cloudwatch/models.py | 69 ++++++++++-------- moto/cloudwatch/responses.py | 17 +++-- .../test_cloudwatch_expression_parser.py | 40 +++++++++++ .../test_cloudwatch_expressions.py | 70 +++++++++++++++++++ 5 files changed, 173 insertions(+), 36 deletions(-) create mode 100644 moto/cloudwatch/metric_data_expression_parser.py create mode 100644 tests/test_cloudwatch/test_cloudwatch_expression_parser.py create mode 100644 tests/test_cloudwatch/test_cloudwatch_expressions.py diff --git a/moto/cloudwatch/metric_data_expression_parser.py b/moto/cloudwatch/metric_data_expression_parser.py new file mode 100644 index 000000000..32b5fe131 --- /dev/null +++ b/moto/cloudwatch/metric_data_expression_parser.py @@ -0,0 +1,13 @@ +from typing import Any, Dict, List, Tuple, SupportsFloat + + +def parse_expression( + expression: str, results: List[Dict[str, Any]] +) -> Tuple[List[SupportsFloat], List[str]]: + values: List[SupportsFloat] = [] + timestamps: List[str] = [] + for result in results: + if result.get("id") == expression: + values.extend(result["vals"]) + timestamps.extend(result["timestamps"]) + return values, timestamps diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index 032bb4b30..e94513c43 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -17,6 +17,7 @@ from .exceptions import ( ResourceNotFoundException, InvalidParameterCombination, ) +from .metric_data_expression_parser import parse_expression from .utils import make_arn_for_dashboard, make_arn_for_alarm from dateutil import parser from typing import Tuple, Optional, List, Iterable, Dict, Any, SupportsFloat @@ -668,16 +669,23 @@ class CloudWatchBackend(BaseBackend): ] results = [] - for query in queries: + results_to_return = [] + metric_stat_queries = [q for q in queries if "MetricStat" in q] + expression_queries = [q for q in queries if "Expression" in q] + for query in metric_stat_queries: period_start_time = start_time - query_ns = query["metric_stat._metric._namespace"] - query_name = query["metric_stat._metric._metric_name"] - delta = timedelta(seconds=int(query["metric_stat._period"])) - dimensions = self._extract_dimensions_from_get_metric_data_query(query) - unit = query.get("metric_stat._unit") + metric_stat = query["MetricStat"] + query_ns = metric_stat["Metric"]["Namespace"] + query_name = metric_stat["Metric"]["MetricName"] + delta = timedelta(seconds=int(metric_stat["Period"])) + dimensions = [ + Dimension(name=d["Name"], value=d["Value"]) + for d in metric_stat["Metric"].get("Dimensions", []) + ] + unit = metric_stat.get("Unit") result_vals: List[SupportsFloat] = [] timestamps: List[str] = [] - stat = query["metric_stat._stat"] + stat = metric_stat["Stat"] while period_start_time <= end_time: period_end_time = period_start_time + delta period_md = [ @@ -715,21 +723,37 @@ class CloudWatchBackend(BaseBackend): timestamps.reverse() result_vals.reverse() - label = ( - query["label"] - if "label" in query - else query["metric_stat._metric._metric_name"] + " " + stat - ) + label = query.get("Label") or f"{query_name} {stat}" results.append( { - "id": query["id"], + "id": query["Id"], "label": label, "vals": result_vals, "timestamps": timestamps, } ) - return results + if query.get("ReturnData", "true") == "true": + results_to_return.append( + { + "id": query["Id"], + "label": label, + "vals": result_vals, + "timestamps": timestamps, + } + ) + for query in expression_queries: + label = query.get("Label") or f"{query_name} {stat}" + result_vals, timestamps = parse_expression(query["Expression"], results) + results_to_return.append( + { + "id": query["Id"], + "label": label, + "vals": result_vals, + "timestamps": timestamps, + } + ) + return results_to_return def get_metric_statistics( self, @@ -904,23 +928,6 @@ class CloudWatchBackend(BaseBackend): else: return None, metrics - def _extract_dimensions_from_get_metric_data_query( - self, query: Dict[str, str] - ) -> List[Dimension]: - dimensions = [] - prefix = "metric_stat._metric._dimensions.member." - suffix_name = "._name" - suffix_value = "._value" - counter = 1 - - while query.get(f"{prefix}{counter}{suffix_name}") and counter <= 10: - name = query.get(f"{prefix}{counter}{suffix_name}") - value = query.get(f"{prefix}{counter}{suffix_value}") - dimensions.append(Dimension(name=name, value=value)) - counter = counter + 1 - - return dimensions - def _validate_parameters_put_metric_data( self, metric: Dict[str, Any], query_num: int ) -> None: diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index ce728e869..0ebfe3932 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -13,7 +13,7 @@ from .models import ( Dimension, FakeAlarm, ) -from .exceptions import InvalidParameterCombination +from .exceptions import InvalidParameterCombination, ValidationError ERROR_RESPONSE = Tuple[str, Dict[str, int]] @@ -176,11 +176,18 @@ class CloudWatchResponse(BaseResponse): @amzn_request_id def get_metric_data(self) -> str: - start = dtparse(self._get_param("StartTime")) - end = dtparse(self._get_param("EndTime")) - scan_by = self._get_param("ScanBy") + params = self._get_params() + start = dtparse(params["StartTime"]) + end = dtparse(params["EndTime"]) + scan_by = params.get("ScanBy") or "TimestampDescending" - queries = self._get_list_prefix("MetricDataQueries.member") + queries = params.get("MetricDataQueries", []) + for query in queries: + if "MetricStat" not in query and "Expression" not in query: + # AWS also returns the empty line + raise ValidationError( + "The parameter MetricDataQueries.member.1.MetricStat is required.\n" + ) results = self.cloudwatch_backend.get_metric_data( start_time=start, end_time=end, queries=queries, scan_by=scan_by ) diff --git a/tests/test_cloudwatch/test_cloudwatch_expression_parser.py b/tests/test_cloudwatch/test_cloudwatch_expression_parser.py new file mode 100644 index 000000000..57afb1812 --- /dev/null +++ b/tests/test_cloudwatch/test_cloudwatch_expression_parser.py @@ -0,0 +1,40 @@ +from moto.cloudwatch.metric_data_expression_parser import parse_expression + + +def test_simple_expression(): + result_from_previous_queries = [ + { + "id": "totalBytes", + "label": "metric Sum", + "vals": [25.0], + "timestamps": ["timestamp1"], + } + ] + res = parse_expression("totalBytes", result_from_previous_queries) + assert res == ([25.0], ["timestamp1"]) + + +def test_missing_expression(): + result_from_previous_queries = [ + { + "id": "totalBytes", + "label": "metric Sum", + "vals": [25.0], + "timestamps": ["timestamp1"], + } + ] + res = parse_expression("unknown", result_from_previous_queries) + assert res == ([], []) + + +def test_complex_expression(): + result_from_previous_queries = [ + { + "id": "totalBytes", + "label": "metric Sum", + "vals": [25.0], + "timestamps": ["timestamp1"], + } + ] + res = parse_expression("totalBytes/10", result_from_previous_queries) + assert res == ([], []) diff --git a/tests/test_cloudwatch/test_cloudwatch_expressions.py b/tests/test_cloudwatch/test_cloudwatch_expressions.py new file mode 100644 index 000000000..663105b5f --- /dev/null +++ b/tests/test_cloudwatch/test_cloudwatch_expressions.py @@ -0,0 +1,70 @@ +import boto3 +import pytest + +from botocore.exceptions import ClientError +from datetime import datetime, timedelta, timezone +from moto import mock_cloudwatch + + +@mock_cloudwatch +def test_get_metric_data__no_metric_data_or_expression(): + utc_now = datetime.now(tz=timezone.utc) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + with pytest.raises(ClientError) as exc: + cloudwatch.get_metric_data( + MetricDataQueries=[{"Id": "result1", "Label": "e1"}], + StartTime=utc_now - timedelta(minutes=5), + EndTime=utc_now, + ScanBy="TimestampDescending", + ) + err = exc.value.response["Error"] + assert err["Code"] == "ValidationError" + assert ( + err["Message"] + == "The parameter MetricDataQueries.member.1.MetricStat is required.\n" + ) + + +@mock_cloudwatch +def test_get_metric_data_with_simple_expression(): + utc_now = datetime.now(tz=timezone.utc) + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + namespace = "my_namespace/" + cloudwatch.put_metric_data( + Namespace=namespace, + MetricData=[ + { + "MetricName": "metric", + "Value": 25, + "Unit": "Bytes", + }, + ], + ) + # get_metric_data 1 + results = cloudwatch.get_metric_data( + MetricDataQueries=[ + { + "Id": "result1", + "Expression": "totalBytes", + "Label": "e1", + }, + { + "Id": "totalBytes", + "MetricStat": { + "Metric": {"Namespace": namespace, "MetricName": "metric"}, + "Period": 60, + "Stat": "Sum", + "Unit": "Bytes", + }, + "ReturnData": False, + }, + ], + StartTime=utc_now - timedelta(minutes=5), + EndTime=utc_now + timedelta(minutes=5), + ScanBy="TimestampDescending", + )["MetricDataResults"] + # + assert len(results) == 1 + assert results[0]["Id"] == "result1" + assert results[0]["Label"] == "e1" + assert results[0]["Values"] == [25.0]