Added ManagedPrefixList (#4169)

moto/ec2/exceptions.py

@@ -69,7 +69,7 @@ class InvalidSubnetIdError(EC2ClientError):
     def __init__(self, subnet_id):
         super(InvalidSubnetIdError, self).__init__(
             "InvalidSubnetID.NotFound",
-            "The subnet ID '{0}' does not exist".format(subnet_id),
+            "The subnet ID '{}' does not exist".format(subnet_id),
         )
 
 

moto/ec2/models.py

@@ -138,6 +138,7 @@ from .utils import (
     random_reservation_id,
     random_route_table_id,
     generate_route_id,
+    random_managed_prefix_list_id,
     create_dns_entries,
     split_route_id,
     random_security_group_id,
@@ -3263,6 +3264,7 @@ class VPCBackend(object):
         # validates if vpc is present or not.
         self.get_vpc(vpc_id)
 
+        service_destination_cidr = None
         if type and type.lower() == "interface":
 
             network_interface_ids = []
@@ -3297,12 +3299,29 @@ class VPCBackend(object):
             security_group_ids,
             tags,
             private_dns_enabled,
+            service_destination_cidr,
         )
 
         self.vpc_end_points[vpc_endpoint_id] = vpc_end_point
 
         return vpc_end_point
 
+    def delete_vpc_endpoints(self, vpce_ids=[]):
+        vpce_ids
+        for vpce_id in vpce_ids:
+            vpc_endpoint = self.vpc_end_points.get(vpce_id, None)
+            if vpc_endpoint:
+                if vpc_endpoint.type.lower() == "interface":
+                    for eni_id in vpc_endpoint.network_interface_ids:
+                        self.enis.pop(eni_id, None)
+                else:
+                    for route_table_id in vpc_endpoint.route_table_ids:
+                        self.delete_route(
+                            route_table_id, vpc_endpoint.service_destination_cidr
+                        )
+                vpc_endpoint.state = "deleted"
+        return True
+
     def get_vpc_end_point(self, vpc_end_point_ids, filters=None):
         vpc_end_points = self.vpc_end_points.values()
 
@@ -3759,7 +3778,7 @@ class SubnetBackend(object):
             matches = [sn for sn in matches if sn.id in subnet_ids]
             if len(subnet_ids) > len(matches):
                 unknown_ids = set(subnet_ids) - set(matches)
-                raise InvalidSubnetIdError(unknown_ids)
+                raise InvalidSubnetIdError(list(unknown_ids)[0])
         if filters:
             matches = generic_filter(filters, matches)
 
@@ -4358,12 +4377,14 @@ class VPCEndPoint(TaggedEC2Resource):
         security_group_ids=None,
         tags=None,
         private_dns_enabled=None,
+        service_destination_cidr=None,
     ):
         self.ec2_backend = ec2_backend
         self.id = id
         self.vpc_id = vpc_id
         self.service_name = service_name
         self.type = type
+        self.state = "available"
         self.policy_document = policy_document
         self.route_table_ids = route_table_ids
         self.network_interface_ids = network_interface_ids
@@ -4371,9 +4392,10 @@ class VPCEndPoint(TaggedEC2Resource):
         self.client_token = client_token
         self.security_group_ids = security_group_ids
         self.private_dns_enabled = private_dns_enabled
-        self._created_at = datetime.utcnow()
+        # self.created_at = utc_date_and_time()
         self.dns_entries = dns_entries
         self.add_tags(tags or {})
+        self.service_destination_cidr = service_destination_cidr
 
     @property
     def owner_id(self):
@@ -4381,7 +4403,170 @@ class VPCEndPoint(TaggedEC2Resource):
 
     @property
     def created_at(self):
-        return iso_8601_datetime_with_milliseconds(self._created_at)
+        return utc_date_and_time()
+
+
+class ManagedPrefixList(TaggedEC2Resource):
+    def __init__(
+        self,
+        backend,
+        address_family=None,
+        entry=[],
+        max_entries=None,
+        prefix_list_name=None,
+        region=None,
+        tags={},
+        owner_id=None,
+    ):
+        self.ec2_backend = backend
+        self.address_family = address_family
+        self.max_entries = max_entries
+        self.id = random_managed_prefix_list_id()
+        self.prefix_list_name = prefix_list_name
+        self.state = "create-complete"
+        self.state_message = "create complete"
+        self.add_tags(tags or {})
+        self.version = 1
+        self.entries = {self.version: entry} if entry else {}
+        self.resource_owner_id = owner_id if owner_id else None
+        self.prefix_list_arn = self.arn(region, self.owner_id)
+        self.delete_counter = 1
+
+    def arn(self, region, owner_id):
+        return "arn:aws:ec2:{region}:{owner_id}:prefix-list/{resource_id}".format(
+            region=region, resource_id=self.id, owner_id=owner_id
+        )
+
+    @property
+    def owner_id(self):
+        return ACCOUNT_ID if not self.resource_owner_id else self.resource_owner_id
+
+
+class ManagedPrefixListBackend(object):
+    def __init__(self):
+        self.managed_prefix_lists = {}
+        self.create_default_pls()
+        super(ManagedPrefixListBackend, self).__init__()
+
+    def create_managed_prefix_list(
+        self,
+        address_family=None,
+        entry=[],
+        max_entries=None,
+        prefix_list_name=None,
+        tags={},
+        owner_id=None,
+    ):
+        managed_prefix_list = ManagedPrefixList(
+            self,
+            address_family=address_family,
+            entry=entry,
+            max_entries=max_entries,
+            prefix_list_name=prefix_list_name,
+            region=self.region_name,
+            tags=tags,
+            owner_id=owner_id,
+        )
+        self.managed_prefix_lists[managed_prefix_list.id] = managed_prefix_list
+        return managed_prefix_list
+
+    def describe_managed_prefix_lists(self, prefix_list_ids=None, filters=None):
+        managed_prefix_lists = list(self.managed_prefix_lists.values())
+        attr_pairs = (
+            ("owner-id", "owner_id"),
+            ("prefix-list-id", "id"),
+            ("prefix-list-name", "prefix_list_name"),
+        )
+
+        if prefix_list_ids:
+            managed_prefix_lists = [
+                managed_prefix_list
+                for managed_prefix_list in managed_prefix_lists
+                if managed_prefix_list.id in prefix_list_ids
+            ]
+
+        result = managed_prefix_lists
+        if filters:
+            result = filter_resources(managed_prefix_lists, filters, attr_pairs)
+
+        for item in result.copy():
+            if not item.delete_counter:
+                self.managed_prefix_lists.pop(item.id, None)
+                result.remove(item)
+            if item.state == "delete-complete":
+                item.delete_counter -= 1
+        return result
+
+    def get_managed_prefix_list_entries(self, prefix_list_id=None):
+        managed_prefix_list = self.managed_prefix_lists.get(prefix_list_id)
+        return managed_prefix_list
+
+    def delete_managed_prefix_list(self, prefix_list_id):
+        managed_prefix_list = self.managed_prefix_lists.get(prefix_list_id)
+        managed_prefix_list.state = "delete-complete"
+        return managed_prefix_list
+
+    def modify_managed_prefix_list(
+        self,
+        add_entry=None,
+        prefix_list_id=None,
+        current_version=None,
+        prefix_list_name=None,
+        remove_entry=None,
+    ):
+        managed_pl = self.managed_prefix_lists.get(prefix_list_id)
+        managed_pl.prefix_list_name = prefix_list_name
+        if remove_entry or add_entry:
+            entries = (
+                managed_pl.entries.get(current_version, managed_pl.version).copy()
+                if managed_pl.entries
+                else []
+            )
+            for item in entries.copy():
+                if item.get("Cidr", "") in remove_entry:
+                    entries.remove(item)
+
+            for item in add_entry:
+                if item not in entries.copy():
+                    entries.append(item)
+            managed_pl.version += 1
+            managed_pl.entries[managed_pl.version] = entries
+        managed_pl.state = "modify-complete"
+        return managed_pl
+
+    def create_default_pls(self):
+        entry = [
+            {"Cidr": "52.216.0.0/15", "Description": "default"},
+            {"Cidr": "3.5.0.0/19", "Description": "default"},
+            {"Cidr": "54.231.0.0/16", "Description": "default"},
+        ]
+
+        managed_prefix_list = self.create_managed_prefix_list(
+            address_family="IPv4",
+            entry=entry,
+            prefix_list_name="com.amazonaws.{}.s3".format(self.region_name),
+            owner_id="aws",
+        )
+        managed_prefix_list.version = None
+        managed_prefix_list.max_entries = None
+        self.managed_prefix_lists[managed_prefix_list.id] = managed_prefix_list
+
+        entry = [
+            {"Cidr": "3.218.182.0/24", "Description": "default"},
+            {"Cidr": "3.218.180.0/23", "Description": "default"},
+            {"Cidr": "52.94.0.0/22", "Description": "default"},
+            {"Cidr": "52.119.224.0/20", "Description": "default"},
+        ]
+
+        managed_prefix_list = self.create_managed_prefix_list(
+            address_family="IPv4",
+            entry=entry,
+            prefix_list_name="com.amazonaws.{}.dynamodb".format(self.region_name),
+            owner_id="aws",
+        )
+        managed_prefix_list.version = None
+        managed_prefix_list.max_entries = None
+        self.managed_prefix_lists[managed_prefix_list.id] = managed_prefix_list
 
 
 class RouteBackend(object):
@@ -7080,6 +7265,7 @@ class EC2Backend(
     SecurityGroupBackend,
     AmiBackend,
     VPCBackend,
+    ManagedPrefixListBackend,
     SubnetBackend,
     SubnetRouteTableAssociationBackend,
     FlowLogsBackend,

moto/ec2/responses/vpcs.py

@@ -218,6 +218,97 @@ class VPCs(BaseResponse):
         template = self.response_template(DESCRIBE_VPC_ENDPOINT_RESPONSE)
         return template.render(vpc_end_points=vpc_end_points, account_id=ACCOUNT_ID)
 
+    def delete_vpc_endpoints(self):
+        vpc_end_points_ids = self._get_multi_param("VpcEndpointId")
+        response = self.ec2_backend.delete_vpc_endpoints(vpce_ids=vpc_end_points_ids,)
+        template = self.response_template(DELETE_VPC_ENDPOINT_RESPONSE)
+        return template.render(response=response)
+
+    def create_managed_prefix_list(self):
+        address_family = self._get_param("AddressFamily")
+        max_entries = self._get_param("MaxEntries")
+        prefix_list_name = self._get_param("PrefixListName")
+        entry = self._get_multi_param("Entry")
+
+        tags = self._get_multi_param("TagSpecification")
+        tags = tags[0] if isinstance(tags, list) and len(tags) == 1 else tags
+        tags = (tags or {}).get("Tag", [])
+        tags = {t["Key"]: t["Value"] for t in tags}
+
+        managed_prefix_list = self.ec2_backend.create_managed_prefix_list(
+            address_family=address_family,
+            entry=entry,
+            max_entries=max_entries,
+            prefix_list_name=prefix_list_name,
+            tags=tags,
+        )
+        template = self.response_template(CREATE_MANAGED_PREFIX_LIST)
+        return template.render(managed_prefix_list=managed_prefix_list)
+
+    def describe_managed_prefix_lists(self):
+        prefix_list_ids = self._get_multi_param("PrefixListId")
+        filters = filters_from_querystring(self.querystring)
+        managed_prefix_lists = self.ec2_backend.describe_managed_prefix_lists(
+            prefix_list_ids=prefix_list_ids, filters=filters
+        )
+        template = self.response_template(DESCRIBE_MANAGED_PREFIX_LIST)
+        return template.render(managed_prefix_lists=managed_prefix_lists)
+
+    def get_managed_prefix_list_entries(self):
+        prefix_list_id = self._get_param("PrefixListId")
+        target_version = self._get_param("TargetVersion")
+        managed_prefix_list = self.ec2_backend.get_managed_prefix_list_entries(
+            prefix_list_id=prefix_list_id,
+        )
+        entries = []
+        if managed_prefix_list:
+            entries = (
+                list(managed_prefix_list.entries.values())[-1]
+                if managed_prefix_list.entries.values()
+                else []
+            )
+            if target_version:
+                target_version = int(target_version)
+                entries = managed_prefix_list.entries.get(target_version)
+        template = self.response_template(GET_MANAGED_PREFIX_LIST_ENTRIES)
+        return template.render(entries=entries)
+
+    def delete_managed_prefix_list(self):
+        prefix_list_id = self._get_param("PrefixListId")
+        managed_prefix_list = self.ec2_backend.delete_managed_prefix_list(
+            prefix_list_id
+        )
+        template = self.response_template(DELETE_MANAGED_PREFIX_LIST)
+        return template.render(managed_prefix_list=managed_prefix_list)
+
+    def describe_prefix_lists(self):
+        prefix_list_ids = self._get_multi_param("PrefixListId")
+        filters = filters_from_querystring(self.querystring)
+        managed_pls = self.ec2_backend.describe_managed_prefix_lists(
+            prefix_list_ids=prefix_list_ids, filters=filters
+        )
+        template = self.response_template(DESCRIBE_PREFIX_LIST)
+        return template.render(managed_pls=managed_pls)
+
+    def modify_managed_prefix_list(self):
+        add_entry = self._get_multi_param("AddEntry")
+        prefix_list_id = self._get_param("PrefixListId")
+        current_version = self._get_param("CurrentVersion")
+        prefix_list_name = self._get_param("PrefixListName")
+        remove_entry = self._get_multi_param("RemoveEntry")
+
+        current_version = int(current_version) if current_version else None
+
+        managed_prefix_list = self.ec2_backend.modify_managed_prefix_list(
+            add_entry=add_entry,
+            prefix_list_id=prefix_list_id,
+            current_version=current_version,
+            prefix_list_name=prefix_list_name,
+            remove_entry=remove_entry,
+        )
+        template = self.response_template(MODIFY_PREFIX_LIST)
+        return template.render(managed_prefix_list=managed_prefix_list)
+
 
 CREATE_VPC_RESPONSE = """
 <CreateVpcResponse xmlns="http://ec2.amazonaws.com/doc/{{doc_date}}/">
@@ -452,7 +543,7 @@ IPV6_DISASSOCIATE_VPC_CIDR_BLOCK_RESPONSE = """
 CREATE_VPC_END_POINT = """ <CreateVpcEndpointResponse xmlns="http://monitoring.amazonaws.com/doc/2010-08-01/">
     <vpcEndpoint>
         <policyDocument>{{ vpc_end_point.policy_document }}</policyDocument>
-        <state> available </state>
+        <state>{{ vpc_end_point.state }}</state>
         <vpcEndpointPolicySupported> false </vpcEndpointPolicySupported>
         <serviceName>{{ vpc_end_point.service_name }}</serviceName>
         <vpcId>{{ vpc_end_point.vpc_id }}</vpcId>
@@ -535,7 +626,7 @@ DESCRIBE_VPC_ENDPOINT_RESPONSE = """<DescribeVpcEndpointsResponse xmlns="http://
                 {% if vpc_end_point.policy_document %}
                     <policyDocument>{{ vpc_end_point.policy_document }}</policyDocument>
                 {% endif %}
-                <state>available</state>
+                <state>{{ vpc_end_point.state }}</state>
                 <privateDnsEnabled>{{ 'true' if vpc_end_point.private_dns_enabled else 'false' }}</privateDnsEnabled>
                 <serviceName>{{ vpc_end_point.service_name }}</serviceName>
                 <vpcId>{{ vpc_end_point.vpc_id }}</vpcId>
@@ -593,3 +684,145 @@ DESCRIBE_VPC_ENDPOINT_RESPONSE = """<DescribeVpcEndpointsResponse xmlns="http://
         {% endfor %}
     </vpcEndpointSet>
 </DescribeVpcEndpointsResponse>"""
+
+
+DELETE_VPC_ENDPOINT_RESPONSE = """<DeleteVpcEndpointsResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
+    <requestId>19a9ff46-7df6-49b8-9726-3df27527089d</requestId>
+    <unsuccessful>{{ 'Error' if not response else '' }}</unsuccessful>
+</DeleteVpcEndpointsResponse>"""
+
+
+CREATE_MANAGED_PREFIX_LIST = """<CreateManagedPrefixListResponse xmlns="http://monitoring.amazonaws.com/doc/2010-08-01/">
+    <prefixList>
+        <addressFamily>{{ managed_prefix_list.address_family }}</addressFamily>
+        <maxEntries>{{ managed_prefix_list.max_entries }}</maxEntries>
+        <ownerId>{{ managed_prefix_list.owner_id }}</ownerId>
+        <prefixListArn>{{ managed_prefix_list.prefix_list_arn }}</prefixListArn>
+        <prefixListId>{{ managed_prefix_list.id }}</prefixListId>
+        <prefixListName>{{ managed_prefix_list.prefix_list_name }}</prefixListName>
+        <state>{{ managed_prefix_list.state }}</state>
+        <tagSet>
+        {% for tag in managed_prefix_list.get_tags() %}
+            <item>
+                <key>{{ tag.key }}</key>
+                <value>{{ tag.value }}</value>
+            </item>
+        {% endfor %}
+        </tagSet>
+        <version>{{ managed_prefix_list.version }}</version>
+    </prefixList>
+</CreateManagedPrefixListResponse>"""
+
+
+DESCRIBE_MANAGED_PREFIX_LIST = """<DescribeManagedPrefixListsResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
+        <requestId>934214d3-4501-4797-b896-13e8fc7ec256</requestId>
+        <prefixListSet>
+        {% for managed_prefix_list in managed_prefix_lists %}
+            <item>
+                <addressFamily>{{ managed_prefix_list.address_family }}</addressFamily>
+                {% if managed_prefix_list.max_entries %}
+                <maxEntries>{{ managed_prefix_list.max_entries }}</maxEntries>
+                {% endif %}
+                <ownerId>{{ managed_prefix_list.owner_id }}</ownerId>
+                <prefixListArn>{{ managed_prefix_list.prefix_list_arn }}</prefixListArn>
+                <prefixListId>{{ managed_prefix_list.id }}</prefixListId>
+                <prefixListName>{{ managed_prefix_list.prefix_list_name }}</prefixListName>
+                <state>{{ managed_prefix_list.state }}</state>
+                <tagSet>
+                {% for tag in managed_prefix_list.get_tags() %}
+                    <item>
+                        <key>{{ tag.key }}</key>
+                        <value>{{ tag.value }}</value>
+                    </item>
+                {% endfor %}
+                </tagSet>
+                {% if managed_prefix_list.version %}
+                <version>{{ managed_prefix_list.version }}</version>
+                {% endif %}
+            </item>
+        {% endfor %}
+    </prefixListSet>
+</DescribeManagedPrefixListsResponse>
+"""
+
+
+GET_MANAGED_PREFIX_LIST_ENTRIES = """<GetManagedPrefixListEntriesResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
+    <requestId>39a3c79f-846f-4382-a592-example</requestId>
+    <entrySet>
+    {% for entry in entries %}
+        <item>
+            <cidr>{{ entry.Cidr or ''}}</cidr>
+            <description>{{ entry.Description or ''}}</description>
+        </item>
+    {% endfor %}
+    </entrySet>
+</GetManagedPrefixListEntriesResponse>
+"""
+
+
+DELETE_MANAGED_PREFIX_LIST = """<DeleteManagedPrefixListResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
+    <requestId>39a3c79f-846f-4382-a592-example</requestId>
+    <prefixList>
+        <addressFamily>{{ managed_prefix_list.address_family }}</addressFamily>
+        <maxEntries>{{ managed_prefix_list.max_entries or '' }}</maxEntries>
+        <ownerId>{{ managed_prefix_list.owner_id }}</ownerId>
+        <prefixListArn>{{ managed_prefix_list.prefix_list_arn }}</prefixListArn>
+        <prefixListId>{{ managed_prefix_list.id }}</prefixListId>
+        <prefixListName>{{ managed_prefix_list.prefix_list_name }}</prefixListName>
+        <state>{{ managed_prefix_list.state }}</state>
+        <tagSet>
+        {% for tag in managed_prefix_list.get_tags() %}
+            <item>
+                <key>{{ tag.key }}</key>
+                <value>{{ tag.value }}</value>
+            </item>
+        {% endfor %}
+        </tagSet>
+        <version>{{ managed_prefix_list.version or ''}}</version>
+    </prefixList>
+</DeleteManagedPrefixListResponse>
+"""
+
+
+DESCRIBE_PREFIX_LIST = """<DescribePrefixListsResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
+    <requestId>8a2ec0e2-6918-4270-ae45-58e61971e97d</requestId>
+    <prefixListSet>
+    {% for pl in managed_pls %}
+    {% if pl.prefix_list_name.startswith("com.amazonaws.") %}
+        <item>
+            <cidrSet>
+                {% for entry in pl.entries.1 %}
+                <item>{{ entry.Cidr }}</item>
+                {% endfor %}
+            </cidrSet>
+            <prefixListId>{{ pl.id }}</prefixListId>
+            <prefixListName>{{ pl.prefix_list_name }}</prefixListName>
+        </item>
+    {% endif %}
+    {% endfor %}
+    </prefixListSet>
+</DescribePrefixListsResponse>
+"""
+
+MODIFY_PREFIX_LIST = """<ModifyManagedPrefixListResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
+    <requestId>602f3752-c348-4b14-81e2-example</requestId>
+    <prefixList>
+        <addressFamily>{{ managed_prefix_list.address_family }}</addressFamily>
+        <maxEntries>{{ managed_prefix_list.max_entries or '' }}</maxEntries>
+        <ownerId>{{ managed_prefix_list.owner_id }}</ownerId>
+        <prefixListArn>{{ managed_prefix_list.prefix_list_arn }}</prefixListArn>
+        <prefixListId>{{ managed_prefix_list.id }}</prefixListId>
+        <prefixListName>{{ managed_prefix_list.prefix_list_name }}</prefixListName>
+        <state>{{ managed_prefix_list.state }}</state>
+        <tagSet>
+        {% for tag in managed_prefix_list.get_tags() %}
+            <item>
+                <key>{{ tag.key }}</key>
+                <value>{{ tag.value }}</value>
+            </item>
+        {% endfor %}
+        </tagSet>
+        <version>{{ managed_prefix_list.version or ''}}</version>
+    </prefixList>
+</ModifyManagedPrefixListResponse>
+"""

moto/ec2/utils.py

@@ -41,6 +41,7 @@ EC2_RESOURCE_TO_PREFIX = {
     "volume": "vol",
     "vpc": "vpc",
     "vpc-endpoint": "vpce",
+    "managed-prefix-list": "pl",
     "vpc-cidr-association-id": "vpc-cidr-assoc",
     "vpc-elastic-ip": "eipalloc",
     "vpc-elastic-ip-association": "eipassoc",
@@ -230,6 +231,10 @@ def generate_route_id(route_table_id, cidr_block, ipv6_cidr_block=None):
     return "%s~%s" % (route_table_id, cidr_block)
 
 
+def random_managed_prefix_list_id():
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["managed-prefix-list"], size=8)
+
+
 def create_dns_entries(service_name, vpc_endpoint_id):
     dns_entries = {}
     dns_entries["dns_name"] = "{}-{}.{}".format(

tests/terraform-tests.success.txt

@@ -82,4 +82,5 @@ TestAccAWSRouteTable_vgwRoutePropagation
 TestAccAWSRouteTable_RequireRouteTarget
 TestAccAWSRouteTable_disappears_SubnetAssociation
 TestAccAWSRouteTable_disappears
-TestAccAWSRouteTable_basic
+TestAccAWSRouteTable_basic
+TestAccAwsEc2ManagedPrefixList