Yasuke/moto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add metric filter support to cloudwatch logs (updated) (#4278)

Browse Source 
Co-authored-by: Brady <brady.jubic@gsa.gov>
This commit is contained in:
Matt Ossman 2021-09-13 18:31:03 -04:00 committed by GitHub
parent d48cd31ebb
commit 9859d66ff8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 503 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
IMPLEMENTATION_COVERAGE.md
View File

@ -7068,7 +7068,7 @@
## logs
<details>
<summary>50% implemented</summary>
<summary>57% implemented</summary>
- [ ] associate_kms_key
- [ ] cancel_export_task
@ -7078,7 +7078,7 @@
- [ ] delete_destination
- [X] delete_log_group
- [X] delete_log_stream
- [ ] delete_metric_filter
- [X] delete_metric_filter
- [ ] delete_query_definition
- [X] delete_resource_policy
- [X] delete_retention_policy
@ -7087,7 +7087,7 @@
- [ ] describe_export_tasks
- [X] describe_log_groups
- [X] describe_log_streams
- [ ] describe_metric_filters
- [X] describe_metric_filters
- [ ] describe_queries
- [ ] describe_query_definitions
- [X] describe_resource_policies
@ -7102,7 +7102,7 @@
- [ ] put_destination
- [ ] put_destination_policy
- [X] put_log_events
- [ ] put_metric_filter
- [X] put_metric_filter
- [ ] put_query_definition
- [X] put_resource_policy
- [X] put_retention_policy

65
moto/logs/metric_filters.py Normal file
View File

@ -0,0 +1,65 @@
def find_metric_transformation_by_name(metric_transformations, metric_name):
for metric in metric_transformations:
if metric["metricName"] == metric_name:
return metric
def find_metric_transformation_by_namespace(metric_transformations, metric_namespace):
for metric in metric_transformations:
if metric["metricNamespace"] == metric_namespace:
return metric
class MetricFilters:
def __init__(self):
self.metric_filters = []
def add_filter(
self, filter_name, filter_pattern, log_group_name, metric_transformations
):
self.metric_filters.append(
{
"filterName": filter_name,
"filterPattern": filter_pattern,
"logGroupName": log_group_name,
"metricTransformations": metric_transformations,
}
)
def get_matching_filters(
self, prefix=None, log_group_name=None, metric_name=None, metric_namespace=None
):
result = []
for f in self.metric_filters:
prefix_matches = prefix is None or f["filterName"].startswith(prefix)
log_group_matches = (
log_group_name is None or f["logGroupName"] == log_group_name
)
metric_name_matches = (
metric_name is None
or find_metric_transformation_by_name(
f["metricTransformations"], metric_name
)
)
namespace_matches = (
metric_namespace is None
or find_metric_transformation_by_namespace(
f["metricTransformations"], metric_namespace
)
)
if (
prefix_matches
and log_group_matches
and metric_name_matches
and namespace_matches
):
result.append(f)
return result
def delete_filter(self, filter_name=None, log_group_name=None):
for f in self.metric_filters:
if f["filterName"] == filter_name and f["logGroupName"] == log_group_name:
self.metric_filters.remove(f)
return self.metric_filters

21
moto/logs/models.py
View File

@ -5,6 +5,7 @@ from boto3 import Session
from moto import core as moto_core
from moto.core import BaseBackend, BaseModel
from moto.core.utils import unix_time_millis
from moto.logs.metric_filters import MetricFilters
from moto.logs.exceptions import (
ResourceNotFoundException,
ResourceAlreadyExistsException,
@ -32,6 +33,7 @@ class LogEvent(BaseModel):
self.message = log_event["message"]
self.event_id = self.__class__._event_id
self.__class__._event_id += 1
""
def to_filter_dict(self):
return {
@ -525,6 +527,7 @@ class LogsBackend(BaseBackend):
def __init__(self, region_name):
self.region_name = region_name
self.groups = dict() # { logGroupName: LogGroup}
self.filters = MetricFilters()
self.queries = dict()
self.resource_policies = dict()
@ -796,6 +799,24 @@ class LogsBackend(BaseBackend):
log_group = self.groups[log_group_name]
log_group.untag(tags)
def put_metric_filter(
self, filter_name, filter_pattern, log_group_name, metric_transformations
):
self.filters.add_filter(
filter_name, filter_pattern, log_group_name, metric_transformations
)
def describe_metric_filters(
self, prefix=None, log_group_name=None, metric_name=None, metric_namespace=None
):
filters = self.filters.get_matching_filters(
prefix, log_group_name, metric_name, metric_namespace
)
return filters
def delete_metric_filter(self, filter_name=None, log_group_name=None):
self.filters.delete_filter(filter_name, log_group_name)
def describe_subscription_filters(self, log_group_name):
log_group = self.groups.get(log_group_name)

122
moto/logs/responses.py
View File

@ -1,4 +1,5 @@
import json
import re
from .exceptions import InvalidParameterException
@ -8,6 +9,26 @@ from .models import logs_backends
# See http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/Welcome.html
def validate_param(
param_name, param_value, constraint, constraint_expression, pattern=None
):
try:
assert constraint_expression(param_value)
except (AssertionError, TypeError):
raise InvalidParameterException(
constraint=constraint, parameter=param_name, value=param_value,
)
if pattern and param_value:
try:
assert re.match(pattern, param_value)
except (AssertionError, TypeError):
raise InvalidParameterException(
constraint=f"Must match pattern: {pattern}",
parameter=param_name,
value=param_value,
)
class LogsResponse(BaseResponse):
@property
def logs_backend(self):
@ -23,6 +44,107 @@ class LogsResponse(BaseResponse):
def _get_param(self, param, if_none=None):
return self.request_params.get(param, if_none)
def _get_validated_param(
self, param, constraint, constraint_expression, pattern=None
):
param_value = self._get_param(param)
validate_param(param, param_value, constraint, constraint_expression, pattern)
return param_value
def put_metric_filter(self):
filter_name = self._get_validated_param(
"filterName",
"Minimum length of 1. Maximum length of 512.",
lambda x: 1 <= len(x) <= 512,
pattern="[^:*]*",
)
filter_pattern = self._get_validated_param(
"filterPattern",
"Minimum length of 0. Maximum length of 1024.",
lambda x: 0 <= len(x) <= 1024,
)
log_group_name = self._get_validated_param(
"logGroupName",
"Minimum length of 1. Maximum length of 512.",
lambda x: 1 <= len(x) <= 512,
pattern="[.-_/#A-Za-z0-9]+",
)
metric_transformations = self._get_validated_param(
"metricTransformations", "Fixed number of 1 item.", lambda x: len(x) == 1
)
self.logs_backend.put_metric_filter(
filter_name, filter_pattern, log_group_name, metric_transformations
)
return ""
def describe_metric_filters(self):
filter_name_prefix = self._get_validated_param(
"filterNamePrefix",
"Minimum length of 1. Maximum length of 512.",
lambda x: x is None or 1 <= len(x) <= 512,
pattern="[^:*]*",
)
log_group_name = self._get_validated_param(
"logGroupName",
"Minimum length of 1. Maximum length of 512",
lambda x: x is None or 1 <= len(x) <= 512,
pattern="[.-_/#A-Za-z0-9]+",
)
metric_name = self._get_validated_param(
"metricName",
"Maximum length of 255.",
lambda x: x is None or len(x) <= 255,
pattern="[^:*$]*",
)
metric_namespace = self._get_validated_param(
"metricNamespace",
"Maximum length of 255.",
lambda x: x is None or len(x) <= 255,
pattern="[^:*$]*",
)
next_token = self._get_validated_param(
"nextToken", "Minimum length of 1.", lambda x: x is None or 1 <= len(x)
)
if metric_name and not metric_namespace:
raise InvalidParameterException(
constraint=f"If you include the metricName parameter in your request, "
f"you must also include the metricNamespace parameter.",
parameter="metricNamespace",
value=metric_namespace,
)
if metric_namespace and not metric_name:
raise InvalidParameterException(
constraint=f"If you include the metricNamespace parameter in your request, "
f"you must also include the metricName parameter.",
parameter="metricName",
value=metric_name,
)
filters = self.logs_backend.describe_metric_filters(
filter_name_prefix, log_group_name, metric_name, metric_namespace
)
return json.dumps({"metricFilters": filters, "nextToken": next_token})
def delete_metric_filter(self):
filter_name = self._get_validated_param(
"filterName",
"Minimum length of 1. Maximum length of 512.",
lambda x: 1 <= len(x) <= 512,
pattern="[^:*]*$",
)
log_group_name = self._get_validated_param(
"logGroupName",
"Minimum length of 1. Maximum length of 512.",
lambda x: 1 <= len(x) <= 512,
pattern="[.-_/#A-Za-z0-9]+$",
)
self.logs_backend.delete_metric_filter(filter_name, log_group_name)
return ""
def create_log_group(self):
log_group_name = self._get_param("logGroupName")
tags = self._get_param("tags")

291
tests/test_logs/test_logs.py
View File

@ -37,6 +37,297 @@ def json_policy_doc():
)
@mock_logs
def test_describe_metric_filters_happy_prefix():
conn = boto3.client("logs", "us-west-2")
response1 = put_metric_filter(conn, count=1)
assert response1["ResponseMetadata"]["HTTPStatusCode"] == 200
response2 = put_metric_filter(conn, count=2)
assert response2["ResponseMetadata"]["HTTPStatusCode"] == 200
response = conn.describe_metric_filters(filterNamePrefix="filter")
assert len(response["metricFilters"]) == 2
assert response["metricFilters"][0]["filterName"] == "filterName1"
assert response["metricFilters"][1]["filterName"] == "filterName2"
@mock_logs
def test_describe_metric_filters_happy_log_group_name():
conn = boto3.client("logs", "us-west-2")
response1 = put_metric_filter(conn, count=1)
assert response1["ResponseMetadata"]["HTTPStatusCode"] == 200
response2 = put_metric_filter(conn, count=2)
assert response2["ResponseMetadata"]["HTTPStatusCode"] == 200
response = conn.describe_metric_filters(logGroupName="logGroupName2")
assert len(response["metricFilters"]) == 1
assert response["metricFilters"][0]["logGroupName"] == "logGroupName2"
@mock_logs
def test_describe_metric_filters_happy_metric_name():
conn = boto3.client("logs", "us-west-2")
response1 = put_metric_filter(conn, count=1)
assert response1["ResponseMetadata"]["HTTPStatusCode"] == 200
response2 = put_metric_filter(conn, count=2)
assert response2["ResponseMetadata"]["HTTPStatusCode"] == 200
response = conn.describe_metric_filters(
metricName="metricName1", metricNamespace="metricNamespace1",
)
assert len(response["metricFilters"]) == 1
metrics = response["metricFilters"][0]["metricTransformations"]
assert metrics[0]["metricName"] == "metricName1"
assert metrics[0]["metricNamespace"] == "metricNamespace1"
@mock_logs
def test_put_metric_filters_validation():
conn = boto3.client("logs", "us-west-2")
invalid_filter_name = "X" * 513
invalid_filter_pattern = "X" * 1025
invalid_metric_transformations = [
{
"defaultValue": 1,
"metricName": "metricName",
"metricNamespace": "metricNamespace",
"metricValue": "metricValue",
},
{
"defaultValue": 1,
"metricName": "metricName",
"metricNamespace": "metricNamespace",
"metricValue": "metricValue",
},
]
test_cases = [
build_put_case(name="Invalid filter name", filter_name=invalid_filter_name,),
build_put_case(
name="Invalid filter pattern", filter_pattern=invalid_filter_pattern,
),
build_put_case(
name="Invalid filter metric transformations",
metric_transformations=invalid_metric_transformations,
),
]
for test_case in test_cases:
with pytest.raises(ClientError) as exc:
conn.put_metric_filter(**test_case["input"])
response = exc.value.response
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
response["Error"]["Code"].should.equal("InvalidParameterException")
@mock_logs
def test_describe_metric_filters_validation():
conn = boto3.client("logs", "us-west-2")
length_over_512 = "X" * 513
length_over_255 = "X" * 256
test_cases = [
build_describe_case(
name="Invalid filter name prefix", filter_name_prefix=length_over_512,
),
build_describe_case(
name="Invalid log group name", log_group_name=length_over_512,
),
build_describe_case(name="Invalid metric name", metric_name=length_over_255,),
build_describe_case(
name="Invalid metric namespace", metric_namespace=length_over_255,
),
]
for test_case in test_cases:
with pytest.raises(ClientError) as exc:
conn.describe_metric_filters(**test_case["input"])
response = exc.value.response
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
response["Error"]["Code"].should.equal("InvalidParameterException")
@mock_logs
def test_describe_metric_filters_multiple_happy():
conn = boto3.client("logs", "us-west-2")
response = put_metric_filter(conn, 1)
assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
response = put_metric_filter(conn, 2)
assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
response = conn.describe_metric_filters(
filterNamePrefix="filter", logGroupName="logGroupName1"
)
assert response["metricFilters"][0]["filterName"] == "filterName1"
response = conn.describe_metric_filters(filterNamePrefix="filter")
assert response["metricFilters"][0]["filterName"] == "filterName1"
response = conn.describe_metric_filters(logGroupName="logGroupName1")
assert response["metricFilters"][0]["filterName"] == "filterName1"
@mock_logs
def test_delete_metric_filter():
conn = boto3.client("logs", "us-west-2")
response = put_metric_filter(conn, 1)
assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
response = put_metric_filter(conn, 2)
assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
response = conn.delete_metric_filter(
filterName="filterName", logGroupName="logGroupName1"
)
assert response["ResponseMetadata"]["HTTPStatusCode"] == 200
response = conn.describe_metric_filters(
filterNamePrefix="filter", logGroupName="logGroupName2"
)
assert response["metricFilters"][0]["filterName"] == "filterName2"
response = conn.describe_metric_filters(logGroupName="logGroupName2")
assert response["metricFilters"][0]["filterName"] == "filterName2"
@mock_logs
@pytest.mark.parametrize(
"filter_name, failing_constraint",
[
(
"X" * 513,
"Minimum length of 1. Maximum length of 512.",
), # filterName too long
("x:x", "Must match pattern"), # invalid filterName pattern
],
)
def test_delete_metric_filter_invalid_filter_name(filter_name, failing_constraint):
conn = boto3.client("logs", "us-west-2")
with pytest.raises(ClientError) as exc:
conn.delete_metric_filter(filterName=filter_name, logGroupName="valid")
response = exc.value.response
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
response["Error"]["Code"].should.equal("InvalidParameterException")
response["Error"]["Message"].should.contain(
f"Value '{filter_name}' at 'filterName' failed to satisfy constraint"
)
response["Error"]["Message"].should.contain(failing_constraint)
@mock_logs
@pytest.mark.parametrize(
"log_group_name, failing_constraint",
[
(
"X" * 513,
"Minimum length of 1. Maximum length of 512.",
), # logGroupName too long
("x!x", "Must match pattern"), # invalid logGroupName pattern
],
)
def test_delete_metric_filter_invalid_log_group_name(
log_group_name, failing_constraint
):
conn = boto3.client("logs", "us-west-2")
with pytest.raises(ClientError) as exc:
conn.delete_metric_filter(filterName="valid", logGroupName=log_group_name)
response = exc.value.response
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
response["Error"]["Code"].should.equal("InvalidParameterException")
response["Error"]["Message"].should.contain(
f"Value '{log_group_name}' at 'logGroupName' failed to satisfy constraint"
)
response["Error"]["Message"].should.contain(failing_constraint)
def put_metric_filter(conn, count=1):
count = str(count)
return conn.put_metric_filter(
filterName="filterName" + count,
filterPattern="filterPattern" + count,
logGroupName="logGroupName" + count,
metricTransformations=[
{
"defaultValue": int(count),
"metricName": "metricName" + count,
"metricNamespace": "metricNamespace" + count,
"metricValue": "metricValue" + count,
},
],
)
def build_put_case(
name,
filter_name="filterName",
filter_pattern="filterPattern",
log_group_name="logGroupName",
metric_transformations=None,
):
return {
"name": name,
"input": build_put_input(
filter_name, filter_pattern, log_group_name, metric_transformations
),
}
def build_put_input(
filter_name, filter_pattern, log_group_name, metric_transformations
):
if metric_transformations is None:
metric_transformations = [
{
"defaultValue": 1,
"metricName": "metricName",
"metricNamespace": "metricNamespace",
"metricValue": "metricValue",
},
]
return {
"filterName": filter_name,
"filterPattern": filter_pattern,
"logGroupName": log_group_name,
"metricTransformations": metric_transformations,
}
def build_describe_input(
filter_name_prefix, log_group_name, metric_name, metric_namespace
):
return {
"filterNamePrefix": filter_name_prefix,
"logGroupName": log_group_name,
"metricName": metric_name,
"metricNamespace": metric_namespace,
}
def build_describe_case(
name,
filter_name_prefix="filterNamePrefix",
log_group_name="logGroupName",
metric_name="metricName",
metric_namespace="metricNamespace",
):
return {
"name": name,
"input": build_describe_input(
filter_name_prefix, log_group_name, metric_name, metric_namespace
),
}
@mock_logs
@pytest.mark.parametrize(
"kms_key_id",