add metric filter support to cloudwatch logs (updated) (#4278)

Co-authored-by: Brady <brady.jubic@gsa.gov>

IMPLEMENTATION_COVERAGE.md

@@ -7068,7 +7068,7 @@
 
 ## logs
 <details>
-<summary>50% implemented</summary>
+<summary>57% implemented</summary>
 
 - [ ] associate_kms_key
 - [ ] cancel_export_task
@@ -7078,7 +7078,7 @@
 - [ ] delete_destination
 - [X] delete_log_group
 - [X] delete_log_stream
-- [ ] delete_metric_filter
+- [X] delete_metric_filter
 - [ ] delete_query_definition
 - [X] delete_resource_policy
 - [X] delete_retention_policy
@@ -7087,7 +7087,7 @@
 - [ ] describe_export_tasks
 - [X] describe_log_groups
 - [X] describe_log_streams
-- [ ] describe_metric_filters
+- [X] describe_metric_filters
 - [ ] describe_queries
 - [ ] describe_query_definitions
 - [X] describe_resource_policies
@@ -7102,7 +7102,7 @@
 - [ ] put_destination
 - [ ] put_destination_policy
 - [X] put_log_events
-- [ ] put_metric_filter
+- [X] put_metric_filter
 - [ ] put_query_definition
 - [X] put_resource_policy
 - [X] put_retention_policy

moto/logs/metric_filters.py

@@ -0,0 +1,65 @@
+def find_metric_transformation_by_name(metric_transformations, metric_name):
+    for metric in metric_transformations:
+        if metric["metricName"] == metric_name:
+            return metric
+
+
+def find_metric_transformation_by_namespace(metric_transformations, metric_namespace):
+    for metric in metric_transformations:
+        if metric["metricNamespace"] == metric_namespace:
+            return metric
+
+
+class MetricFilters:
+    def __init__(self):
+        self.metric_filters = []
+
+    def add_filter(
+        self, filter_name, filter_pattern, log_group_name, metric_transformations
+    ):
+        self.metric_filters.append(
+            {
+                "filterName": filter_name,
+                "filterPattern": filter_pattern,
+                "logGroupName": log_group_name,
+                "metricTransformations": metric_transformations,
+            }
+        )
+
+    def get_matching_filters(
+        self, prefix=None, log_group_name=None, metric_name=None, metric_namespace=None
+    ):
+        result = []
+        for f in self.metric_filters:
+            prefix_matches = prefix is None or f["filterName"].startswith(prefix)
+            log_group_matches = (
+                log_group_name is None or f["logGroupName"] == log_group_name
+            )
+            metric_name_matches = (
+                metric_name is None
+                or find_metric_transformation_by_name(
+                    f["metricTransformations"], metric_name
+                )
+            )
+            namespace_matches = (
+                metric_namespace is None
+                or find_metric_transformation_by_namespace(
+                    f["metricTransformations"], metric_namespace
+                )
+            )
+
+            if (
+                prefix_matches
+                and log_group_matches
+                and metric_name_matches
+                and namespace_matches
+            ):
+                result.append(f)
+
+        return result
+
+    def delete_filter(self, filter_name=None, log_group_name=None):
+        for f in self.metric_filters:
+            if f["filterName"] == filter_name and f["logGroupName"] == log_group_name:
+                self.metric_filters.remove(f)
+        return self.metric_filters

moto/logs/models.py

@@ -5,6 +5,7 @@ from boto3 import Session
 from moto import core as moto_core
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import unix_time_millis
+from moto.logs.metric_filters import MetricFilters
 from moto.logs.exceptions import (
     ResourceNotFoundException,
     ResourceAlreadyExistsException,
@@ -32,6 +33,7 @@ class LogEvent(BaseModel):
         self.message = log_event["message"]
         self.event_id = self.__class__._event_id
         self.__class__._event_id += 1
+        ""
 
     def to_filter_dict(self):
         return {
@@ -525,6 +527,7 @@ class LogsBackend(BaseBackend):
     def __init__(self, region_name):
         self.region_name = region_name
         self.groups = dict()  # { logGroupName: LogGroup}
+        self.filters = MetricFilters()
         self.queries = dict()
         self.resource_policies = dict()
 
@@ -796,6 +799,24 @@ class LogsBackend(BaseBackend):
         log_group = self.groups[log_group_name]
         log_group.untag(tags)
 
+    def put_metric_filter(
+        self, filter_name, filter_pattern, log_group_name, metric_transformations
+    ):
+        self.filters.add_filter(
+            filter_name, filter_pattern, log_group_name, metric_transformations
+        )
+
+    def describe_metric_filters(
+        self, prefix=None, log_group_name=None, metric_name=None, metric_namespace=None
+    ):
+        filters = self.filters.get_matching_filters(
+            prefix, log_group_name, metric_name, metric_namespace
+        )
+        return filters
+
+    def delete_metric_filter(self, filter_name=None, log_group_name=None):
+        self.filters.delete_filter(filter_name, log_group_name)
+
     def describe_subscription_filters(self, log_group_name):
         log_group = self.groups.get(log_group_name)
 

moto/logs/responses.py

@@ -1,4 +1,5 @@
 import json
+import re
 
 from .exceptions import InvalidParameterException
 
@@ -8,6 +9,26 @@ from .models import logs_backends
 # See http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/Welcome.html
 
 
+def validate_param(
+    param_name, param_value, constraint, constraint_expression, pattern=None
+):
+    try:
+        assert constraint_expression(param_value)
+    except (AssertionError, TypeError):
+        raise InvalidParameterException(
+            constraint=constraint, parameter=param_name, value=param_value,
+        )
+    if pattern and param_value:
+        try:
+            assert re.match(pattern, param_value)
+        except (AssertionError, TypeError):
+            raise InvalidParameterException(
+                constraint=f"Must match pattern: {pattern}",
+                parameter=param_name,
+                value=param_value,
+            )
+
+
 class LogsResponse(BaseResponse):
     @property
     def logs_backend(self):
@@ -23,6 +44,107 @@ class LogsResponse(BaseResponse):
     def _get_param(self, param, if_none=None):
         return self.request_params.get(param, if_none)
 
+    def _get_validated_param(
+        self, param, constraint, constraint_expression, pattern=None
+    ):
+        param_value = self._get_param(param)
+        validate_param(param, param_value, constraint, constraint_expression, pattern)
+        return param_value
+
+    def put_metric_filter(self):
+        filter_name = self._get_validated_param(
+            "filterName",
+            "Minimum length of 1. Maximum length of 512.",
+            lambda x: 1 <= len(x) <= 512,
+            pattern="[^:*]*",
+        )
+        filter_pattern = self._get_validated_param(
+            "filterPattern",
+            "Minimum length of 0. Maximum length of 1024.",
+            lambda x: 0 <= len(x) <= 1024,
+        )
+        log_group_name = self._get_validated_param(
+            "logGroupName",
+            "Minimum length of 1. Maximum length of 512.",
+            lambda x: 1 <= len(x) <= 512,
+            pattern="[.-_/#A-Za-z0-9]+",
+        )
+        metric_transformations = self._get_validated_param(
+            "metricTransformations", "Fixed number of 1 item.", lambda x: len(x) == 1
+        )
+
+        self.logs_backend.put_metric_filter(
+            filter_name, filter_pattern, log_group_name, metric_transformations
+        )
+
+        return ""
+
+    def describe_metric_filters(self):
+        filter_name_prefix = self._get_validated_param(
+            "filterNamePrefix",
+            "Minimum length of 1. Maximum length of 512.",
+            lambda x: x is None or 1 <= len(x) <= 512,
+            pattern="[^:*]*",
+        )
+        log_group_name = self._get_validated_param(
+            "logGroupName",
+            "Minimum length of 1. Maximum length of 512",
+            lambda x: x is None or 1 <= len(x) <= 512,
+            pattern="[.-_/#A-Za-z0-9]+",
+        )
+        metric_name = self._get_validated_param(
+            "metricName",
+            "Maximum length of 255.",
+            lambda x: x is None or len(x) <= 255,
+            pattern="[^:*$]*",
+        )
+        metric_namespace = self._get_validated_param(
+            "metricNamespace",
+            "Maximum length of 255.",
+            lambda x: x is None or len(x) <= 255,
+            pattern="[^:*$]*",
+        )
+        next_token = self._get_validated_param(
+            "nextToken", "Minimum length of 1.", lambda x: x is None or 1 <= len(x)
+        )
+
+        if metric_name and not metric_namespace:
+            raise InvalidParameterException(
+                constraint=f"If you include the metricName parameter in your request, "
+                f"you must also include the metricNamespace parameter.",
+                parameter="metricNamespace",
+                value=metric_namespace,
+            )
+        if metric_namespace and not metric_name:
+            raise InvalidParameterException(
+                constraint=f"If you include the metricNamespace parameter in your request, "
+                f"you must also include the metricName parameter.",
+                parameter="metricName",
+                value=metric_name,
+            )
+
+        filters = self.logs_backend.describe_metric_filters(
+            filter_name_prefix, log_group_name, metric_name, metric_namespace
+        )
+        return json.dumps({"metricFilters": filters, "nextToken": next_token})
+
+    def delete_metric_filter(self):
+        filter_name = self._get_validated_param(
+            "filterName",
+            "Minimum length of 1. Maximum length of 512.",
+            lambda x: 1 <= len(x) <= 512,
+            pattern="[^:*]*$",
+        )
+        log_group_name = self._get_validated_param(
+            "logGroupName",
+            "Minimum length of 1. Maximum length of 512.",
+            lambda x: 1 <= len(x) <= 512,
+            pattern="[.-_/#A-Za-z0-9]+$",
+        )
+
+        self.logs_backend.delete_metric_filter(filter_name, log_group_name)
+        return ""
+
     def create_log_group(self):
         log_group_name = self._get_param("logGroupName")
         tags = self._get_param("tags")

tests/test_logs/test_logs.py

@@ -37,6 +37,297 @@ def json_policy_doc():
     )
 
 
+@mock_logs
+def test_describe_metric_filters_happy_prefix():
+    conn = boto3.client("logs", "us-west-2")
+
+    response1 = put_metric_filter(conn, count=1)
+    assert response1["ResponseMetadata"]["HTTPStatusCode"] == 200
+    response2 = put_metric_filter(conn, count=2)
+    assert response2["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    response = conn.describe_metric_filters(filterNamePrefix="filter")
+
+    assert len(response["metricFilters"]) == 2
+    assert response["metricFilters"][0]["filterName"] == "filterName1"
+    assert response["metricFilters"][1]["filterName"] == "filterName2"
+
+
+@mock_logs
+def test_describe_metric_filters_happy_log_group_name():
+    conn = boto3.client("logs", "us-west-2")
+
+    response1 = put_metric_filter(conn, count=1)
+    assert response1["ResponseMetadata"]["HTTPStatusCode"] == 200
+    response2 = put_metric_filter(conn, count=2)
+    assert response2["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    response = conn.describe_metric_filters(logGroupName="logGroupName2")
+
+    assert len(response["metricFilters"]) == 1
+    assert response["metricFilters"][0]["logGroupName"] == "logGroupName2"
+
+
+@mock_logs
+def test_describe_metric_filters_happy_metric_name():
+    conn = boto3.client("logs", "us-west-2")
+
+    response1 = put_metric_filter(conn, count=1)
+    assert response1["ResponseMetadata"]["HTTPStatusCode"] == 200
+    response2 = put_metric_filter(conn, count=2)
+    assert response2["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    response = conn.describe_metric_filters(
+        metricName="metricName1", metricNamespace="metricNamespace1",
+    )
+
+    assert len(response["metricFilters"]) == 1
+    metrics = response["metricFilters"][0]["metricTransformations"]
+    assert metrics[0]["metricName"] == "metricName1"
+    assert metrics[0]["metricNamespace"] == "metricNamespace1"
+
+
+@mock_logs
+def test_put_metric_filters_validation():
+    conn = boto3.client("logs", "us-west-2")
+
+    invalid_filter_name = "X" * 513
+    invalid_filter_pattern = "X" * 1025
+    invalid_metric_transformations = [
+        {
+            "defaultValue": 1,
+            "metricName": "metricName",
+            "metricNamespace": "metricNamespace",
+            "metricValue": "metricValue",
+        },
+        {
+            "defaultValue": 1,
+            "metricName": "metricName",
+            "metricNamespace": "metricNamespace",
+            "metricValue": "metricValue",
+        },
+    ]
+
+    test_cases = [
+        build_put_case(name="Invalid filter name", filter_name=invalid_filter_name,),
+        build_put_case(
+            name="Invalid filter pattern", filter_pattern=invalid_filter_pattern,
+        ),
+        build_put_case(
+            name="Invalid filter metric transformations",
+            metric_transformations=invalid_metric_transformations,
+        ),
+    ]
+
+    for test_case in test_cases:
+        with pytest.raises(ClientError) as exc:
+            conn.put_metric_filter(**test_case["input"])
+        response = exc.value.response
+        response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+        response["Error"]["Code"].should.equal("InvalidParameterException")
+
+
+@mock_logs
+def test_describe_metric_filters_validation():
+    conn = boto3.client("logs", "us-west-2")
+
+    length_over_512 = "X" * 513
+    length_over_255 = "X" * 256
+
+    test_cases = [
+        build_describe_case(
+            name="Invalid filter name prefix", filter_name_prefix=length_over_512,
+        ),
+        build_describe_case(
+            name="Invalid log group name", log_group_name=length_over_512,
+        ),
+        build_describe_case(name="Invalid metric name", metric_name=length_over_255,),
+        build_describe_case(
+            name="Invalid metric namespace", metric_namespace=length_over_255,
+        ),
+    ]
+
+    for test_case in test_cases:
+        with pytest.raises(ClientError) as exc:
+            conn.describe_metric_filters(**test_case["input"])
+        response = exc.value.response
+        response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+        response["Error"]["Code"].should.equal("InvalidParameterException")
+
+
+@mock_logs
+def test_describe_metric_filters_multiple_happy():
+    conn = boto3.client("logs", "us-west-2")
+
+    response = put_metric_filter(conn, 1)
+    assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    response = put_metric_filter(conn, 2)
+    assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
+    response = conn.describe_metric_filters(
+        filterNamePrefix="filter", logGroupName="logGroupName1"
+    )
+    assert response["metricFilters"][0]["filterName"] == "filterName1"
+
+    response = conn.describe_metric_filters(filterNamePrefix="filter")
+    assert response["metricFilters"][0]["filterName"] == "filterName1"
+
+    response = conn.describe_metric_filters(logGroupName="logGroupName1")
+    assert response["metricFilters"][0]["filterName"] == "filterName1"
+
+
+@mock_logs
+def test_delete_metric_filter():
+    conn = boto3.client("logs", "us-west-2")
+
+    response = put_metric_filter(conn, 1)
+    assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    response = put_metric_filter(conn, 2)
+    assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    response = conn.delete_metric_filter(
+        filterName="filterName", logGroupName="logGroupName1"
+    )
+    assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
+
+    response = conn.describe_metric_filters(
+        filterNamePrefix="filter", logGroupName="logGroupName2"
+    )
+    assert response["metricFilters"][0]["filterName"] == "filterName2"
+
+    response = conn.describe_metric_filters(logGroupName="logGroupName2")
+    assert response["metricFilters"][0]["filterName"] == "filterName2"
+
+
+@mock_logs
+@pytest.mark.parametrize(
+    "filter_name, failing_constraint",
+    [
+        (
+            "X" * 513,
+            "Minimum length of 1. Maximum length of 512.",
+        ),  # filterName too long
+        ("x:x", "Must match pattern"),  # invalid filterName pattern
+    ],
+)
+def test_delete_metric_filter_invalid_filter_name(filter_name, failing_constraint):
+    conn = boto3.client("logs", "us-west-2")
+    with pytest.raises(ClientError) as exc:
+        conn.delete_metric_filter(filterName=filter_name, logGroupName="valid")
+    response = exc.value.response
+    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    response["Error"]["Code"].should.equal("InvalidParameterException")
+    response["Error"]["Message"].should.contain(
+        f"Value '{filter_name}' at 'filterName' failed to satisfy constraint"
+    )
+    response["Error"]["Message"].should.contain(failing_constraint)
+
+
+@mock_logs
+@pytest.mark.parametrize(
+    "log_group_name, failing_constraint",
+    [
+        (
+            "X" * 513,
+            "Minimum length of 1. Maximum length of 512.",
+        ),  # logGroupName too long
+        ("x!x", "Must match pattern"),  # invalid logGroupName pattern
+    ],
+)
+def test_delete_metric_filter_invalid_log_group_name(
+    log_group_name, failing_constraint
+):
+    conn = boto3.client("logs", "us-west-2")
+    with pytest.raises(ClientError) as exc:
+        conn.delete_metric_filter(filterName="valid", logGroupName=log_group_name)
+    response = exc.value.response
+    response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
+    response["Error"]["Code"].should.equal("InvalidParameterException")
+    response["Error"]["Message"].should.contain(
+        f"Value '{log_group_name}' at 'logGroupName' failed to satisfy constraint"
+    )
+    response["Error"]["Message"].should.contain(failing_constraint)
+
+
+def put_metric_filter(conn, count=1):
+    count = str(count)
+    return conn.put_metric_filter(
+        filterName="filterName" + count,
+        filterPattern="filterPattern" + count,
+        logGroupName="logGroupName" + count,
+        metricTransformations=[
+            {
+                "defaultValue": int(count),
+                "metricName": "metricName" + count,
+                "metricNamespace": "metricNamespace" + count,
+                "metricValue": "metricValue" + count,
+            },
+        ],
+    )
+
+
+def build_put_case(
+    name,
+    filter_name="filterName",
+    filter_pattern="filterPattern",
+    log_group_name="logGroupName",
+    metric_transformations=None,
+):
+    return {
+        "name": name,
+        "input": build_put_input(
+            filter_name, filter_pattern, log_group_name, metric_transformations
+        ),
+    }
+
+
+def build_put_input(
+    filter_name, filter_pattern, log_group_name, metric_transformations
+):
+    if metric_transformations is None:
+        metric_transformations = [
+            {
+                "defaultValue": 1,
+                "metricName": "metricName",
+                "metricNamespace": "metricNamespace",
+                "metricValue": "metricValue",
+            },
+        ]
+    return {
+        "filterName": filter_name,
+        "filterPattern": filter_pattern,
+        "logGroupName": log_group_name,
+        "metricTransformations": metric_transformations,
+    }
+
+
+def build_describe_input(
+    filter_name_prefix, log_group_name, metric_name, metric_namespace
+):
+    return {
+        "filterNamePrefix": filter_name_prefix,
+        "logGroupName": log_group_name,
+        "metricName": metric_name,
+        "metricNamespace": metric_namespace,
+    }
+
+
+def build_describe_case(
+    name,
+    filter_name_prefix="filterNamePrefix",
+    log_group_name="logGroupName",
+    metric_name="metricName",
+    metric_namespace="metricNamespace",
+):
+    return {
+        "name": name,
+        "input": build_describe_input(
+            filter_name_prefix, log_group_name, metric_name, metric_namespace
+        ),
+    }
+
+
 @mock_logs
 @pytest.mark.parametrize(
     "kms_key_id",