diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index 4e8759b2f..d934f4bbd 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import json -import re try: from urllib import unquote @@ -198,7 +197,7 @@ class LambdaResponse(BaseResponse): return 404, {}, "{}" def _get_aws_region(self, full_url): - region = re.search(self.region_regex, full_url) + region = self.region_regex.search(full_url) if region: return region.group(1) else: diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index f9d571a23..74499f61e 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -1,13 +1,13 @@ -import json +import json +from moto.core.utils import iso_8601_datetime_with_milliseconds from moto.core import BaseBackend, BaseModel from moto.core.exceptions import RESTError import boto.ec2.cloudwatch -import datetime - +from datetime import datetime, timedelta +from dateutil.tz import tzutc from .utils import make_arn_for_dashboard - DEFAULT_ACCOUNT_ID = 123456789012 @@ -18,6 +18,34 @@ class Dimension(object): self.value = value +def daterange(start, stop, step=timedelta(days=1), inclusive=False): + """ + This method will iterate from `start` to `stop` datetimes with a timedelta step of `step` + (supports iteration forwards or backwards in time) + + :param start: start datetime + :param stop: end datetime + :param step: step size as a timedelta + :param inclusive: if True, last item returned will be as step closest to `end` (or `end` if no remainder). + """ + + # inclusive=False to behave like range by default + total_step_secs = step.total_seconds() + assert total_step_secs != 0 + + if total_step_secs > 0: + while start < stop: + yield start + start = start + step + else: + while stop < start: + yield start + start = start + step + + if inclusive and start == stop: + yield start + + class FakeAlarm(BaseModel): def __init__(self, name, namespace, metric_name, comparison_operator, evaluation_periods, @@ -38,14 +66,14 @@ class FakeAlarm(BaseModel): self.ok_actions = ok_actions self.insufficient_data_actions = insufficient_data_actions self.unit = unit - self.configuration_updated_timestamp = datetime.datetime.utcnow() + self.configuration_updated_timestamp = datetime.utcnow() self.history = [] self.state_reason = '' self.state_reason_data = '{}' self.state = 'OK' - self.state_updated_timestamp = datetime.datetime.utcnow() + self.state_updated_timestamp = datetime.utcnow() def update_state(self, reason, reason_data, state_value): # History type, that then decides what the rest of the items are, can be one of ConfigurationUpdate | StateUpdate | Action @@ -56,17 +84,18 @@ class FakeAlarm(BaseModel): self.state_reason = reason self.state_reason_data = reason_data self.state = state_value - self.state_updated_timestamp = datetime.datetime.utcnow() + self.state_updated_timestamp = datetime.utcnow() class MetricDatum(BaseModel): - def __init__(self, namespace, name, value, dimensions): + def __init__(self, namespace, name, value, dimensions, timestamp): self.namespace = namespace self.name = name self.value = value - self.dimensions = [Dimension(dimension['name'], dimension[ - 'value']) for dimension in dimensions] + self.timestamp = timestamp or datetime.utcnow().replace(tzinfo=tzutc()) + self.dimensions = [Dimension(dimension['Name'], dimension[ + 'Value']) for dimension in dimensions] class Dashboard(BaseModel): @@ -75,7 +104,7 @@ class Dashboard(BaseModel): self.arn = make_arn_for_dashboard(DEFAULT_ACCOUNT_ID, name) self.name = name self.body = body - self.last_modified = datetime.datetime.now() + self.last_modified = datetime.now() @property def last_modified_iso(self): @@ -92,6 +121,53 @@ class Dashboard(BaseModel): return ''.format(self.name) +class Statistics: + def __init__(self, stats, dt): + self.timestamp = iso_8601_datetime_with_milliseconds(dt) + self.values = [] + self.stats = stats + + @property + def sample_count(self): + if 'SampleCount' not in self.stats: + return None + + return len(self.values) + + @property + def unit(self): + return None + + @property + def sum(self): + if 'Sum' not in self.stats: + return None + + return sum(self.values) + + @property + def min(self): + if 'Minimum' not in self.stats: + return None + + return min(self.values) + + @property + def max(self): + if 'Maximum' not in self.stats: + return None + + return max(self.values) + + @property + def average(self): + if 'Average' not in self.stats: + return None + + # when moto is 3.4+ we can switch to the statistics module + return sum(self.values) / len(self.values) + + class CloudWatchBackend(BaseBackend): def __init__(self): @@ -150,9 +226,34 @@ class CloudWatchBackend(BaseBackend): self.alarms.pop(alarm_name, None) def put_metric_data(self, namespace, metric_data): - for name, value, dimensions in metric_data: + for metric_member in metric_data: self.metric_data.append(MetricDatum( - namespace, name, value, dimensions)) + namespace, metric_member['MetricName'], float(metric_member['Value']), metric_member['Dimensions.member'], metric_member.get('Timestamp'))) + + def get_metric_statistics(self, namespace, metric_name, start_time, end_time, period, stats): + period_delta = timedelta(seconds=period) + filtered_data = [md for md in self.metric_data if + md.namespace == namespace and md.name == metric_name and start_time <= md.timestamp <= end_time] + + # earliest to oldest + filtered_data = sorted(filtered_data, key=lambda x: x.timestamp) + if not filtered_data: + return [] + + idx = 0 + data = list() + for dt in daterange(filtered_data[0].timestamp, filtered_data[-1].timestamp + period_delta, period_delta): + s = Statistics(stats, dt) + while idx < len(filtered_data) and filtered_data[idx].timestamp < (dt + period_delta): + s.values.append(filtered_data[idx].value) + idx += 1 + + if not s.values: + continue + + data.append(s) + + return data def get_all_metrics(self): return self.metric_data diff --git a/moto/cloudwatch/responses.py b/moto/cloudwatch/responses.py index 7a5fa5ebd..ccb53337c 100644 --- a/moto/cloudwatch/responses.py +++ b/moto/cloudwatch/responses.py @@ -2,6 +2,7 @@ import json from moto.core.utils import amzn_request_id from moto.core.responses import BaseResponse from .models import cloudwatch_backends +from dateutil.parser import parse as dtparse class CloudWatchResponse(BaseResponse): @@ -75,35 +76,36 @@ class CloudWatchResponse(BaseResponse): @amzn_request_id def put_metric_data(self): namespace = self._get_param('Namespace') - metric_data = [] - metric_index = 1 - while True: - try: - metric_name = self.querystring[ - 'MetricData.member.{0}.MetricName'.format(metric_index)][0] - except KeyError: - break - value = self.querystring.get( - 'MetricData.member.{0}.Value'.format(metric_index), [None])[0] - dimensions = [] - dimension_index = 1 - while True: - try: - dimension_name = self.querystring[ - 'MetricData.member.{0}.Dimensions.member.{1}.Name'.format(metric_index, dimension_index)][0] - except KeyError: - break - dimension_value = self.querystring[ - 'MetricData.member.{0}.Dimensions.member.{1}.Value'.format(metric_index, dimension_index)][0] - dimensions.append( - {'name': dimension_name, 'value': dimension_value}) - dimension_index += 1 - metric_data.append([metric_name, value, dimensions]) - metric_index += 1 + metric_data = self._get_multi_param('MetricData.member') + self.cloudwatch_backend.put_metric_data(namespace, metric_data) template = self.response_template(PUT_METRIC_DATA_TEMPLATE) return template.render() + @amzn_request_id + def get_metric_statistics(self): + namespace = self._get_param('Namespace') + metric_name = self._get_param('MetricName') + start_time = dtparse(self._get_param('StartTime')) + end_time = dtparse(self._get_param('EndTime')) + period = int(self._get_param('Period')) + statistics = self._get_multi_param("Statistics.member") + + # Unsupported Parameters (To Be Implemented) + unit = self._get_param('Unit') + extended_statistics = self._get_param('ExtendedStatistics') + dimensions = self._get_param('Dimensions') + if unit or extended_statistics or dimensions: + raise NotImplemented() + + # TODO: this should instead throw InvalidParameterCombination + if not statistics: + raise NotImplemented("Must specify either Statistics or ExtendedStatistics") + + datapoints = self.cloudwatch_backend.get_metric_statistics(namespace, metric_name, start_time, end_time, period, statistics) + template = self.response_template(GET_METRIC_STATISTICS_TEMPLATE) + return template.render(label=metric_name, datapoints=datapoints) + @amzn_request_id def list_metrics(self): metrics = self.cloudwatch_backend.get_all_metrics() @@ -150,10 +152,6 @@ class CloudWatchResponse(BaseResponse): template = self.response_template(GET_DASHBOARD_TEMPLATE) return template.render(dashboard=dashboard) - @amzn_request_id - def get_metric_statistics(self): - raise NotImplementedError() - @amzn_request_id def list_dashboards(self): prefix = self._get_param('DashboardNamePrefix', '') @@ -266,6 +264,50 @@ PUT_METRIC_DATA_TEMPLATE = """ + + + {{ request_id }} + + + + + + + {% for datapoint in datapoints %} + + {% if datapoint.sum %} + {{ datapoint.sum }} + {% endif %} + + {% if datapoint.average %} + {{ datapoint.average }} + {% endif %} + + {% if datapoint.maximum %} + {{ datapoint.maximum }} + {% endif %} + + {% if datapoint.minimum %} + {{ datapoint.minimum }} + {% endif %} + + {% if datapoint.sample_count %} + {{ datapoint.sample_count }} + {% endif %} + + {% if datapoint.extended_statistics %} + {{ datapoint.extended_statistics }} + {% endif %} + + {{ datapoint.timestamp }} + {{ datapoint.unit }} + + {% endfor %} + + +""" + LIST_METRICS_TEMPLATE = """ diff --git a/moto/core/responses.py b/moto/core/responses.py index 5fce3bf9d..52be602f6 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -106,7 +106,8 @@ class BaseResponse(_TemplateEnvironmentMixin): default_region = 'us-east-1' # to extract region, use [^.] - region_regex = r'\.(?P[a-z]{2}-[a-z]+-\d{1})\.amazonaws\.com' + region_regex = re.compile(r'\.(?P[a-z]{2}-[a-z]+-\d{1})\.amazonaws\.com') + param_list_regex = re.compile(r'(.*)\.(\d+)\.') aws_service_spec = None @classmethod @@ -167,7 +168,7 @@ class BaseResponse(_TemplateEnvironmentMixin): self.response_headers = {"server": "amazon.com"} def get_region_from_url(self, request, full_url): - match = re.search(self.region_regex, full_url) + match = self.region_regex.search(full_url) if match: region = match.group(1) elif 'Authorization' in request.headers and 'AWS4' in request.headers['Authorization']: @@ -311,6 +312,41 @@ class BaseResponse(_TemplateEnvironmentMixin): return False return if_none + def _get_multi_param_helper(self, param_prefix): + value_dict = dict() + tracked_prefixes = set() # prefixes which have already been processed + + def is_tracked(name_param): + for prefix_loop in tracked_prefixes: + if name_param.startswith(prefix_loop): + return True + return False + + for name, value in self.querystring.items(): + if is_tracked(name) or not name.startswith(param_prefix): + continue + + match = self.param_list_regex.search(name[len(param_prefix):]) if len(name) > len(param_prefix) else None + if match: + prefix = param_prefix + match.group(1) + value = self._get_multi_param(prefix) + tracked_prefixes.add(prefix) + name = prefix + value_dict[name] = value + else: + value_dict[name] = value[0] + + if not value_dict: + return None + + if len(value_dict) > 1: + # strip off period prefix + value_dict = {name[len(param_prefix) + 1:]: value for name, value in value_dict.items()} + else: + value_dict = list(value_dict.values())[0] + + return value_dict + def _get_multi_param(self, param_prefix): """ Given a querystring of ?LaunchConfigurationNames.member.1=my-test-1&LaunchConfigurationNames.member.2=my-test-2 @@ -323,12 +359,13 @@ class BaseResponse(_TemplateEnvironmentMixin): values = [] index = 1 while True: - try: - values.append(self.querystring[prefix + str(index)][0]) - except KeyError: + value_dict = self._get_multi_param_helper(prefix + str(index)) + if not value_dict: break - else: - index += 1 + + values.append(value_dict) + index += 1 + return values def _get_dict_param(self, param_prefix): diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index bb21c1e2a..b31681f16 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -1,4 +1,6 @@ from __future__ import unicode_literals + +import re from six.moves.urllib.parse import urlparse from moto.core.responses import BaseResponse @@ -15,12 +17,11 @@ from .exceptions import ( MAXIMUM_VISIBILTY_TIMEOUT = 43200 MAXIMUM_MESSAGE_LENGTH = 262144 # 256 KiB DEFAULT_RECEIVED_MESSAGES = 1 -SQS_REGION_REGEX = r'://(.+?)\.queue\.amazonaws\.com' class SQSResponse(BaseResponse): - region_regex = SQS_REGION_REGEX + region_regex = re.compile(r'://(.+?)\.queue\.amazonaws\.com') @property def sqs_backend(self):