SSM - load global parameters (#3953)

MANIFEST.in

@@ -7,6 +7,7 @@ include moto/ec2/resources/instance_type_offerings/*/*.json
 include moto/ec2/resources/amis.json
 include moto/cognitoidp/resources/*.json
 include moto/dynamodb2/parsing/reserved_keywords.txt
+include moto/ssm/resources/*.json
 include moto/support/resources/*.json
 recursive-include moto/templates *
 recursive-include tests *

docs/docs/services/ssm.rst

@@ -12,6 +12,8 @@
 ssm
 ===
 
+.. autoclass:: moto.ssm.models.SimpleSystemManagerBackend
+
 |start-h3| Example usage |end-h3|
 
 .. sourcecode:: python

moto/ssm/models.py

@@ -8,6 +8,7 @@ from moto.core import ACCOUNT_ID, BaseBackend, BaseModel
 from moto.core.exceptions import RESTError
 from moto.core.utils import BackendDict
 from moto.ec2 import ec2_backends
+from moto.utilities.utils import load_resource
 
 import datetime
 import time
@@ -17,7 +18,7 @@ import yaml
 import hashlib
 import random
 
-from .utils import parameter_arn
+from .utils import parameter_arn, convert_to_params
 from .exceptions import (
     ValidationException,
     InvalidFilterValue,
@@ -42,6 +43,62 @@ from .exceptions import (
 )
 
 
+class ParameterDict(defaultdict):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.parameters_loaded = False
+
+    def _check_loading_status(self, key):
+        if not self.parameters_loaded and key and str(key).startswith("/aws"):
+            self._load_global_parameters()
+
+    def _load_global_parameters(self):
+        regions = load_resource(__name__, "resources/regions.json")
+        services = load_resource(__name__, "resources/services.json")
+        params = []
+        params.extend(convert_to_params(regions))
+        params.extend(convert_to_params(services))
+
+        for param in params:
+            last_modified_date = time.time()
+            name = param["Name"]
+            value = param["Value"]
+            # Following were lost in translation/conversion - using sensible defaults
+            parameter_type = "String"
+            version = 1
+            super().__getitem__(name).append(
+                Parameter(
+                    name=name,
+                    value=value,
+                    parameter_type=parameter_type,
+                    description=None,
+                    allowed_pattern=None,
+                    keyid=None,
+                    last_modified_date=last_modified_date,
+                    version=version,
+                    data_type="text",
+                )
+            )
+        self.parameters_loaded = True
+
+    def __getitem__(self, item):
+        self._check_loading_status(item)
+        return super().__getitem__(item)
+
+    def __contains__(self, k):
+        self._check_loading_status(k)
+        return super().__contains__(k)
+
+    def get_keys_beginning_with(self, path, recursive):
+        self._check_loading_status(path)
+        for param_name in self:
+            if path != "/" and not param_name.startswith(path):
+                continue
+            if "/" in param_name[len(path) + 1 :] and not recursive:
+                continue
+            yield param_name
+
+
 PARAMETER_VERSION_LIMIT = 100
 PARAMETER_HISTORY_MAX_RESULTS = 50
 
@@ -718,11 +775,20 @@ class FakeMaintenanceWindow:
 
 
 class SimpleSystemManagerBackend(BaseBackend):
+    """
+    Moto supports the following default parameters out of the box:
+
+     - /aws/service/global-infrastructure/regions
+     - /aws/service/global-infrastructure/services
+
+    Note that these are hardcoded, so they may be out of date for new services/regions.
+    """
+
     def __init__(self, region):
         super().__init__()
         # each value is a list of all of the versions for a parameter
         # to get the current value, grab the last item of the list
-        self._parameters = defaultdict(list)
+        self._parameters = ParameterDict(list)
 
         self._resource_tags = defaultdict(lambda: defaultdict(dict))
         self._commands = []
@@ -1359,16 +1425,11 @@ class SimpleSystemManagerBackend(BaseBackend):
         # path could be with or without a trailing /. we handle this
         # difference here.
         path = path.rstrip("/") + "/"
-        for param_name in self._parameters:
-            if path != "/" and not param_name.startswith(path):
+        for param_name in self._parameters.get_keys_beginning_with(path, recursive):
+            parameter = self.get_parameter(param_name, with_decryption)
+            if not self._match_filters(parameter, filters):
                 continue
-            if "/" in param_name[len(path) + 1 :] and not recursive:
-                continue
-            if not self._match_filters(
-                self.get_parameter(param_name, with_decryption), filters
-            ):
-                continue
-            result.append(self.get_parameter(param_name, with_decryption))
+            result.append(parameter)
 
         return self._get_values_nexttoken(result, max_results, next_token)
 

moto/ssm/utils.py

@@ -7,3 +7,39 @@ def parameter_arn(region, parameter_name):
     return "arn:aws:ssm:{0}:{1}:parameter/{2}".format(
         region, ACCOUNT_ID, parameter_name
     )
+
+
+def convert_to_tree(parameters):
+    """
+    Convert input into a smaller, less redundant data set in tree form
+    Input: [{"Name": "/a/b/c", "Value": "af-south-1", ...}, ..]
+    Output: {"a": {"b": {"c": {"Value": af-south-1}, ..}, ..}, ..}
+    """
+    tree_dict = {}
+    for p in parameters:
+        current_level = tree_dict
+        for path in p["Name"].split("/"):
+            if path == "":
+                continue
+            if path not in current_level:
+                current_level[path] = {}
+            current_level = current_level[path]
+        current_level["Value"] = p["Value"]
+    return tree_dict
+
+
+def convert_to_params(tree):
+    """
+    Inverse of 'convert_to_tree'
+    """
+
+    def m(tree, params, current_path=""):
+        for key, value in tree.items():
+            if key == "Value":
+                params.append({"Name": current_path, "Value": value})
+            else:
+                m(value, params, current_path + "/" + key)
+
+    params = []
+    m(tree, params)
+    return params

scripts/ssm_get_default_params.py

@@ -0,0 +1,61 @@
+import boto3
+import json
+import os
+import subprocess
+import sure  # noqa # pylint: disable=unused-import
+import time
+
+from moto.ssm.utils import convert_to_tree
+
+
+def retrieve_by_path(client, path):
+    print(f"Retrieving all parameters from {path}. "
+          f"AWS has around 14000 parameters, and we can only retrieve 10 at the time, so this may take a while.\n\n")
+    x = client.get_parameters_by_path(Path=path, Recursive=True)
+    parameters = x["Parameters"]
+    next_token = x["NextToken"]
+    while next_token:
+        x = client.get_parameters_by_path(Path=path, Recursive=True, NextToken=next_token)
+        parameters.extend(x["Parameters"])
+        next_token = x.get("NextToken")
+        if len(parameters) % 100 == 0:
+            print(f"Retrieved {len(parameters)} from {path}...")
+            time.sleep(0.5)
+
+    return parameters
+
+
+def main():
+    """
+    Retrieve global parameters from SSM
+     - Download from AWS
+     - Convert them to a more space-optimized data format
+     - Store this in the dedicated moto/ssm/resources-folder
+
+     Note:
+         There are around 20k parameters, and we can only retrieve 10 at a time.
+         So running this scripts takes a while.
+    """
+
+    client = boto3.client('ssm', region_name="us-west-1")
+
+    default_param_paths = ["/aws/service/global-infrastructure/regions",
+                           "/aws/service/global-infrastructure/services"]
+
+    for path in default_param_paths:
+        params = retrieve_by_path(client, path)
+        tree = convert_to_tree(params)
+        root_dir = (
+            subprocess.check_output(["git", "rev-parse", "--show-toplevel"])
+                .decode()
+                .strip()
+        )
+        filename = "{}.json".format(path.split("/")[-1])
+        dest = os.path.join(root_dir, "moto/ssm/resources/{}".format(filename))
+        print("Writing data to {0}".format(dest))
+        with open(dest, "w") as open_file:
+            json.dump(tree, open_file, sort_keys=True, indent=2)
+
+
+if __name__ == "__main__":
+    main()

tests/test_ssm/test_ssm_defaults.py

@@ -0,0 +1,35 @@
+import boto3
+import sure  # noqa # pylint: disable=unused-import
+
+from moto import mock_ssm
+from moto.core import ACCOUNT_ID
+
+
+@mock_ssm
+def test_ssm_get_by_path():
+    client = boto3.client("ssm", region_name="us-west-1")
+    path = "/aws/service/global-infrastructure/regions"
+    params = client.get_parameters_by_path(Path=path)["Parameters"]
+
+    pacific = [p for p in params if p["Value"] == "af-south-1"][0]
+    pacific["Name"].should.equal(
+        "/aws/service/global-infrastructure/regions/af-south-1"
+    )
+    pacific["Type"].should.equal("String")
+    pacific["Version"].should.equal(1)
+    pacific["ARN"].should.equal(
+        f"arn:aws:ssm:us-west-1:{ACCOUNT_ID}:parameter/aws/service/global-infrastructure/regions/af-south-1"
+    )
+    pacific.should.have.key("LastModifiedDate")
+
+
+@mock_ssm
+def test_ssm_region_query():
+    client = boto3.client("ssm", region_name="us-west-1")
+    param = client.get_parameter(
+        Name="/aws/service/global-infrastructure/regions/us-west-1/longName"
+    )
+
+    value = param["Parameter"]["Value"]
+
+    value.should.equal("US West (N. California)")

tests/test_ssm/test_ssm_parameterstore.py

@@ -0,0 +1,85 @@
+import sure  # noqa # pylint: disable=unused-import
+
+from moto.ssm.models import ParameterDict
+
+
+def test_simple_setget():
+    store = ParameterDict(list)
+    store["/a/b/c"] = "some object"
+
+    store.get("/a/b/c").should.equal("some object")
+
+
+def test_get_none():
+    store = ParameterDict(list)
+
+    store.get(None).should.equal(None)
+
+
+def test_get_aws_param():
+    store = ParameterDict(list)
+
+    p = store["/aws/service/global-infrastructure/regions/us-west-1/longName"]
+    p.should.have.length_of(1)
+    p[0].value.should.equal("US West (N. California)")
+
+
+def test_iter():
+    store = ParameterDict(list)
+    store["/a/b/c"] = "some object"
+
+    "/a/b/c".should.be.within(store)
+    "/a/b/d".shouldnt.be.within(store)
+
+
+def test_iter_none():
+    store = ParameterDict(list)
+    None.shouldnt.be.within(store)
+
+
+def test_iter_aws():
+    store = ParameterDict(list)
+
+    "/aws/service/global-infrastructure/regions/us-west-1/longName".should.be.within(
+        store
+    )
+
+
+def test_get_key_beginning_with():
+    store = ParameterDict(list)
+    store["/a/b/c"] = "some object"
+    store["/b/c/d"] = "some other object"
+    store["/a/c/d"] = "some third object"
+
+    l = list(store.get_keys_beginning_with("/a/b", recursive=False))
+    l.should.equal(["/a/b/c"])
+
+    l = list(store.get_keys_beginning_with("/a", recursive=False))
+    l.should.equal([])
+
+    l = list(store.get_keys_beginning_with("/a", recursive=True))
+    set(l).should.equal({"/a/b/c", "/a/c/d"})
+
+
+def test_get_key_beginning_with_aws():
+    """
+    ParameterDict should load the default parameters if we request a key starting with '/aws'
+    :return:
+    """
+    store = ParameterDict(list)
+
+    uswest_params = set(
+        store.get_keys_beginning_with(
+            "/aws/service/global-infrastructure/regions/us-west-1", recursive=False
+        )
+    )
+    uswest_params.should.equal(
+        {
+            "/aws/service/global-infrastructure/regions/us-west-1",
+            "/aws/service/global-infrastructure/regions/us-west-1/domain",
+            "/aws/service/global-infrastructure/regions/us-west-1/geolocationCountry",
+            "/aws/service/global-infrastructure/regions/us-west-1/geolocationRegion",
+            "/aws/service/global-infrastructure/regions/us-west-1/longName",
+            "/aws/service/global-infrastructure/regions/us-west-1/partition",
+        }
+    )

tests/test_ssm/test_ssm_utils.py

@@ -0,0 +1,101 @@
+import sure  # noqa # pylint: disable=unused-import
+
+
+from moto.ssm.utils import convert_to_tree, convert_to_params
+
+SOURCE_PARAMS = [
+    {
+        "ARN": "arn:aws:ssm:us-west-1::parameter/aws/service/global-infrastructure/regions/af-south-1",
+        "DataType": "text",
+        "Name": "/aws/service/global-infrastructure/regions/af-south-1",
+        "Type": "String",
+        "Value": "af-south-1",
+        "Version": 1,
+    },
+    {
+        "ARN": "arn:aws:ssm:us-west-1::parameter/aws/service/global-infrastructure/regions/ap-northeast-2",
+        "DataType": "text",
+        "Name": "/aws/service/global-infrastructure/regions/ap-northeast-2",
+        "Type": "String",
+        "Value": "ap-northeast-2",
+        "Version": 1,
+    },
+    {
+        "ARN": "arn:aws:ssm:us-west-1::parameter/aws/service/global-infrastructure/regions/cn-north-1",
+        "DataType": "text",
+        "Name": "/aws/service/global-infrastructure/regions/cn-north-1",
+        "Type": "String",
+        "Value": "cn-north-1",
+        "Version": 1,
+    },
+    {
+        "ARN": "arn:aws:ssm:us-west-1::parameter/aws/service/global-infrastructure/regions/ap-northeast-2/services/codestar-notifications",
+        "DataType": "text",
+        "Name": "/aws/service/global-infrastructure/regions/ap-northeast-2/services/codestar-notifications",
+        "Type": "String",
+        "Value": "codestar-notifications",
+        "Version": 1,
+    },
+]
+
+EXPECTED_TREE = {
+    "aws": {
+        "service": {
+            "global-infrastructure": {
+                "regions": {
+                    "af-south-1": {"Value": "af-south-1"},
+                    "cn-north-1": {"Value": "cn-north-1"},
+                    "ap-northeast-2": {
+                        "Value": "ap-northeast-2",
+                        "services": {
+                            "codestar-notifications": {
+                                "Value": "codestar-notifications"
+                            }
+                        },
+                    },
+                }
+            }
+        }
+    }
+}
+
+CONVERTED_PARAMS = [
+    {
+        "Name": "/aws/service/global-infrastructure/regions/af-south-1",
+        "Value": "af-south-1",
+    },
+    {
+        "Name": "/aws/service/global-infrastructure/regions/cn-north-1",
+        "Value": "cn-north-1",
+    },
+    {
+        "Name": "/aws/service/global-infrastructure/regions/ap-northeast-2",
+        "Value": "ap-northeast-2",
+    },
+    {
+        "Name": "/aws/service/global-infrastructure/regions/ap-northeast-2/services/codestar-notifications",
+        "Value": "codestar-notifications",
+    },
+]
+
+
+def test_convert_to_tree():
+    tree = convert_to_tree(SOURCE_PARAMS)
+
+    tree.should.equal(EXPECTED_TREE)
+
+
+def test_convert_to_params():
+    actual = convert_to_params(EXPECTED_TREE)
+    actual.should.have.length_of(len(CONVERTED_PARAMS))
+    for param in CONVERTED_PARAMS:
+        actual.should.contain(param)
+
+
+def test_input_is_correct():
+    """
+    Test input should match
+    """
+    for src in SOURCE_PARAMS:
+        minimized_src = {"Name": src["Name"], "Value": src["Value"]}
+        CONVERTED_PARAMS.should.contain(minimized_src)