EC2: Support Dedicated Hosts (#5878)

IMPLEMENTATION_COVERAGE.md

@@ -1745,7 +1745,7 @@
 - [X] accept_vpc_peering_connection
 - [ ] advertise_byoip_cidr
 - [X] allocate_address
-- [ ] allocate_hosts
+- [X] allocate_hosts
 - [ ] allocate_ipam_pool_cidr
 - [ ] apply_security_groups_to_client_vpn_target_network
 - [X] assign_ipv6_addresses
@@ -1982,7 +1982,7 @@
 - [ ] describe_fpga_images
 - [ ] describe_host_reservation_offerings
 - [ ] describe_host_reservations
-- [ ] describe_hosts
+- [X] describe_hosts
 - [X] describe_iam_instance_profile_associations
 - [ ] describe_id_format
 - [ ] describe_identity_id_format
@@ -2193,7 +2193,7 @@
 - [ ] modify_ebs_default_kms_key_id
 - [ ] modify_fleet
 - [ ] modify_fpga_image_attribute
-- [ ] modify_hosts
+- [X] modify_hosts
 - [ ] modify_id_format
 - [ ] modify_identity_id_format
 - [ ] modify_image_attribute
@@ -2267,7 +2267,7 @@
 - [ ] reject_vpc_endpoint_connections
 - [X] reject_vpc_peering_connection
 - [X] release_address
-- [ ] release_hosts
+- [X] release_hosts
 - [ ] release_ipam_pool_allocation
 - [X] replace_iam_instance_profile_association
 - [X] replace_network_acl_association

docs/docs/services/ec2.rst

@@ -36,7 +36,7 @@ ec2
 - [X] accept_vpc_peering_connection
 - [ ] advertise_byoip_cidr
 - [X] allocate_address
-- [ ] allocate_hosts
+- [X] allocate_hosts
 - [ ] allocate_ipam_pool_cidr
 - [ ] apply_security_groups_to_client_vpn_target_network
 - [X] assign_ipv6_addresses
@@ -281,7 +281,11 @@ ec2
 - [ ] describe_fpga_images
 - [ ] describe_host_reservation_offerings
 - [ ] describe_host_reservations
-- [ ] describe_hosts
+- [X] describe_hosts
+  
+        Pagination is not yet implemented
+        
+
 - [X] describe_iam_instance_profile_associations
 - [ ] describe_id_format
 - [ ] describe_identity_id_format
@@ -511,7 +515,7 @@ ec2
 - [ ] modify_ebs_default_kms_key_id
 - [ ] modify_fleet
 - [ ] modify_fpga_image_attribute
-- [ ] modify_hosts
+- [X] modify_hosts
 - [ ] modify_id_format
 - [ ] modify_identity_id_format
 - [ ] modify_image_attribute
@@ -589,7 +593,7 @@ ec2
 - [ ] reject_vpc_endpoint_connections
 - [X] reject_vpc_peering_connection
 - [X] release_address
-- [ ] release_hosts
+- [X] release_hosts
 - [ ] release_ipam_pool_allocation
 - [X] replace_iam_instance_profile_association
 - [X] replace_network_acl_association

moto/ec2/models/__init__.py

@@ -12,6 +12,7 @@ from .dhcp_options import DHCPOptionsSetBackend
 from .elastic_block_store import EBSBackend
 from .elastic_ip_addresses import ElasticAddressBackend
 from .elastic_network_interfaces import NetworkInterfaceBackend
+from .hosts import HostsBackend
 from .fleets import FleetsBackend
 from .flow_logs import FlowLogsBackend
 from .key_pairs import KeyPairBackend
@@ -124,6 +125,7 @@ class EC2Backend(
     CarrierGatewayBackend,
     FleetsBackend,
     WindowsBackend,
+    HostsBackend,
 ):
     """
     moto includes a limited set of AMIs in `moto/ec2/resources/amis.json`.

moto/ec2/models/hosts.py

@@ -0,0 +1,102 @@
+from .core import TaggedEC2Resource
+from ..utils import generic_filter, random_dedicated_host_id
+from typing import Any, Dict, List
+
+
+class Host(TaggedEC2Resource):
+    def __init__(
+        self,
+        host_recovery: str,
+        zone: str,
+        instance_type: str,
+        instance_family: str,
+        auto_placement: str,
+        backend: Any,
+    ):
+        self.id = random_dedicated_host_id()
+        self.state = "available"
+        self.host_recovery = host_recovery or "off"
+        self.zone = zone
+        self.instance_type = instance_type
+        self.instance_family = instance_family
+        self.auto_placement = auto_placement or "on"
+        self.ec2_backend = backend
+
+    def release(self) -> None:
+        self.state = "released"
+
+    def get_filter_value(self, key):
+        if key == "availability-zone":
+            return self.zone
+        if key == "state":
+            return self.state
+        if key == "tag-key":
+            return [t["key"] for t in self.get_tags()]
+        if key == "instance-type":
+            return self.instance_type
+        return None
+
+
+class HostsBackend:
+    def __init__(self):
+        self.hosts = {}
+
+    def allocate_hosts(
+        self,
+        quantity: int,
+        host_recovery: str,
+        zone: str,
+        instance_type: str,
+        instance_family: str,
+        auto_placement: str,
+        tags: Dict[str, str],
+    ) -> List[str]:
+        hosts = [
+            Host(
+                host_recovery,
+                zone,
+                instance_type,
+                instance_family,
+                auto_placement,
+                self,
+            )
+            for _ in range(quantity)
+        ]
+        for host in hosts:
+            self.hosts[host.id] = host
+            if tags:
+                host.add_tags(tags)
+        return [host.id for host in hosts]
+
+    def describe_hosts(
+        self, host_ids: List[str], filters: Dict[str, Any]
+    ) -> List[Host]:
+        """
+        Pagination is not yet implemented
+        """
+        results = self.hosts.values()
+        if host_ids:
+            results = [r for r in results if r.id in host_ids]
+        if filters:
+            results = generic_filter(filters, results)
+        return results
+
+    def modify_hosts(
+        self, host_ids, auto_placement, host_recovery, instance_type, instance_family
+    ):
+        for _id in host_ids:
+            host = self.hosts[_id]
+            if auto_placement is not None:
+                host.auto_placement = auto_placement
+            if host_recovery is not None:
+                host.host_recovery = host_recovery
+            if instance_type is not None:
+                host.instance_type = instance_type
+                host.instance_family = None
+            if instance_family is not None:
+                host.instance_family = instance_family
+                host.instance_type = None
+
+    def release_hosts(self, host_ids: List[str]) -> None:
+        for host_id in host_ids:
+            self.hosts[host_id].release()

moto/ec2/responses/__init__.py

@@ -9,6 +9,7 @@ from .elastic_ip_addresses import ElasticIPAddresses
 from .elastic_network_interfaces import ElasticNetworkInterfaces
 from .fleets import Fleets
 from .general import General
+from .hosts import HostsResponse
 from .instances import InstanceResponse
 from .internet_gateways import InternetGateways
 from .egress_only_internet_gateways import EgressOnlyInternetGateway
@@ -55,6 +56,7 @@ class EC2Response(
     ElasticNetworkInterfaces,
     Fleets,
     General,
+    HostsResponse,
     InstanceResponse,
     InternetGateways,
     EgressOnlyInternetGateway,

moto/ec2/responses/hosts.py

@@ -0,0 +1,118 @@
+from ._base_response import EC2BaseResponse
+
+
+class HostsResponse(EC2BaseResponse):
+    def allocate_hosts(self):
+        params = self._get_params()
+        quantity = int(params.get("Quantity"))
+        host_recovery = params.get("HostRecovery")
+        zone = params.get("AvailabilityZone")
+        instance_type = params.get("InstanceType")
+        instance_family = params.get("InstanceFamily")
+        auto_placement = params.get("AutoPlacement")
+        tags = self._parse_tag_specification()
+        host_tags = tags.get("dedicated-host", {})
+        host_ids = self.ec2_backend.allocate_hosts(
+            quantity,
+            host_recovery,
+            zone,
+            instance_type,
+            instance_family,
+            auto_placement,
+            host_tags,
+        )
+        template = self.response_template(EC2_ALLOCATE_HOSTS)
+        return template.render(host_ids=host_ids)
+
+    def describe_hosts(self):
+        host_ids = list(self._get_params().get("HostId", {}).values())
+        filters = self._filters_from_querystring()
+        hosts = self.ec2_backend.describe_hosts(host_ids, filters)
+        template = self.response_template(EC2_DESCRIBE_HOSTS)
+        return template.render(account_id=self.current_account, hosts=hosts)
+
+    def modify_hosts(self):
+        params = self._get_params()
+        host_ids = list(self._get_params().get("HostId", {}).values())
+        auto_placement = params.get("AutoPlacement")
+        host_recovery = params.get("HostRecovery")
+        instance_type = params.get("InstanceType")
+        instance_family = params.get("InstanceFamily")
+        self.ec2_backend.modify_hosts(
+            host_ids, auto_placement, host_recovery, instance_type, instance_family
+        )
+        template = self.response_template(EC2_MODIFY_HOSTS)
+        return template.render(host_ids=host_ids)
+
+    def release_hosts(self):
+        host_ids = list(self._get_params().get("HostId", {}).values())
+        self.ec2_backend.release_hosts(host_ids)
+        template = self.response_template(EC2_RELEASE_HOSTS)
+        return template.render(host_ids=host_ids)
+
+
+EC2_ALLOCATE_HOSTS = """<AllocateHostsResult xmlns="http://ec2.amazonaws.com/doc/2013-10-15/">
+<requestId>fdcdcab1-ae5c-489e-9c33-4637c5dda355</requestId>
+    <hostIdSet>
+        {% for host_id in host_ids %}
+            <item>{{ host_id }}</item>
+        {% endfor %}
+    </hostIdSet>
+</AllocateHostsResult>"""
+
+
+EC2_DESCRIBE_HOSTS = """<DescribeHostsResult xmlns="http://ec2.amazonaws.com/doc/2013-10-15/">
+<requestId>fdcdcab1-ae5c-489e-9c33-4637c5dda355</requestId>
+    <hostSet>
+        {% for host in hosts %}
+            <item>
+                <autoPlacement>{{ host.auto_placement }}</autoPlacement>
+                <availabilityZone>{{ host.zone }}</availabilityZone>
+                <availableCapacity></availableCapacity>
+                <hostId>{{ host.id }}</hostId>
+                <state>{{ host.state }}</state>
+                <hostProperties>
+                  {% if host.instance_type %}
+                    <instanceType>{{ host.instance_type }}</instanceType>
+                  {% endif %}
+                  {% if host.instance_family %}
+                    <instanceFamily>{{ host.instance_family }}</instanceFamily>
+                  {% endif %}
+                </hostProperties>
+                <hostReservationId>reserv_id</hostReservationId>
+                <instances>
+                </instances>
+                <ownerId>{{ account_id }}</ownerId>
+                <hostRecovery>{{ host.host_recovery }}</hostRecovery>
+                <tagSet>
+                  {% for tag in host.get_tags() %}
+                    <item>
+                      <key>{{ tag.key }}</key>
+                      <value>{{ tag.value }}</value>
+                    </item>
+                  {% endfor %}
+                </tagSet>
+            </item>
+        {% endfor %}
+    </hostSet>
+</DescribeHostsResult>"""
+
+
+EC2_MODIFY_HOSTS = """<ModifyHostsResult xmlns="http://ec2.amazonaws.com/doc/2013-10-15/">
+<requestId>fdcdcab1-ae5c-489e-9c33-4637c5dda355</requestId>
+    <successful>
+        {% for host_id in host_ids %}
+            <item>{{ host_id }}</item>
+        {% endfor %}
+    </successful>
+</ModifyHostsResult>"""
+
+
+EC2_RELEASE_HOSTS = """<ReleaseHostsResult xmlns="http://ec2.amazonaws.com/doc/2013-10-15/">
+<requestId>fdcdcab1-ae5c-489e-9c33-4637c5dda355</requestId>
+    <successful>
+        {% for host_id in host_ids %}
+            <item>{{ host_id }}</item>
+        {% endfor %}
+    </successful>
+</ReleaseHostsResult>"""

moto/ec2/utils.py

@@ -17,6 +17,7 @@ EC2_RESOURCE_TO_PREFIX = {
     "transit-gateway": "tgw",
     "transit-gateway-route-table": "tgw-rtb",
     "transit-gateway-attachment": "tgw-attach",
+    "dedicated_host": "h",
     "dhcp-options": "dopt",
     "fleet": "fleet",
     "flow-logs": "fl",
@@ -238,6 +239,10 @@ def random_public_ip():
     return f"54.214.{random.choice(range(255))}.{random.choice(range(255))}"
 
 
+def random_dedicated_host_id():
+    return random_id(prefix=EC2_RESOURCE_TO_PREFIX["dedicated_host"], size=17)
+
+
 def random_private_ip(cidr=None, ipv6=False):
     # prefix - ula.prefixlen : get number of remaing length for the IP.
     #                          prefix will be 32 for IPv4 and 128 for IPv6.

tests/terraformtests/terraform-tests.success.txt

@@ -118,6 +118,11 @@ dynamodb:
 ec2:
   - TestAccEC2AvailabilityZonesDataSource_
   - TestAccEC2CarrierGateway_
+  - TestAccEC2HostDataSource_
+  - TestAccEC2Host_basic
+  - TestAccEC2Host_disappears
+  - TestAccEC2Host_instanceFamily
+  - TestAccEC2Host_tags
   - TestAccEC2InstanceTypeOfferingDataSource_
   - TestAccEC2InstanceTypeOfferingsDataSource_
   - TestAccEC2RouteTableAssociation_

tests/test_ec2/test_hosts.py

@@ -0,0 +1,168 @@
+import boto3
+
+from moto import mock_ec2
+from uuid import uuid4
+
+
+@mock_ec2
+def test_allocate_hosts():
+    client = boto3.client("ec2", "us-west-1")
+    resp = client.allocate_hosts(
+        AvailabilityZone="us-west-1a",
+        InstanceType="a1.small",
+        HostRecovery="off",
+        AutoPlacement="on",
+        Quantity=3,
+    )
+    resp["HostIds"].should.have.length_of(3)
+
+
+@mock_ec2
+def test_describe_hosts_with_instancefamily():
+    client = boto3.client("ec2", "us-west-1")
+    host_ids = client.allocate_hosts(
+        AvailabilityZone="us-west-1a", InstanceFamily="c5", Quantity=1
+    )["HostIds"]
+
+    host = client.describe_hosts(HostIds=host_ids)["Hosts"][0]
+
+    host.should.have.key("HostProperties").should.have.key("InstanceFamily").equals(
+        "c5"
+    )
+
+
+@mock_ec2
+def test_describe_hosts():
+    client = boto3.client("ec2", "us-west-1")
+    host_ids = client.allocate_hosts(
+        AvailabilityZone="us-west-1c",
+        InstanceType="a1.large",
+        HostRecovery="on",
+        AutoPlacement="off",
+        Quantity=2,
+    )["HostIds"]
+
+    hosts = client.describe_hosts(HostIds=host_ids)["Hosts"]
+    hosts.should.have.length_of(2)
+
+    hosts[0].should.have.key("State").equals("available")
+    hosts[0].should.have.key("AvailabilityZone").equals("us-west-1c")
+    hosts[0].should.have.key("HostRecovery").equals("on")
+    hosts[0].should.have.key("HostProperties").should.have.key("InstanceType").equals(
+        "a1.large"
+    )
+    hosts[0].should.have.key("AutoPlacement").equals("off")
+
+
+@mock_ec2
+def test_describe_hosts_with_tags():
+    client = boto3.client("ec2", "us-west-1")
+    tagkey = str(uuid4())
+    host_ids = client.allocate_hosts(
+        AvailabilityZone="us-west-1b",
+        InstanceType="b1.large",
+        Quantity=1,
+        TagSpecifications=[
+            {"ResourceType": "dedicated-host", "Tags": [{"Key": tagkey, "Value": "v1"}]}
+        ],
+    )["HostIds"]
+
+    host = client.describe_hosts(HostIds=host_ids)["Hosts"][0]
+    host.should.have.key("Tags").equals([{"Key": tagkey, "Value": "v1"}])
+
+    client.allocate_hosts(
+        AvailabilityZone="us-west-1a", InstanceType="b1.large", Quantity=1
+    )
+    hosts = client.describe_hosts(Filters=[{"Name": "tag-key", "Values": [tagkey]}])[
+        "Hosts"
+    ]
+    hosts.should.have.length_of(1)
+
+
+@mock_ec2
+def test_describe_hosts_using_filters():
+    client = boto3.client("ec2", "us-west-1")
+    host_id1 = client.allocate_hosts(
+        AvailabilityZone="us-west-1a", InstanceType="b1.large", Quantity=1
+    )["HostIds"][0]
+    host_id2 = client.allocate_hosts(
+        AvailabilityZone="us-west-1b", InstanceType="b1.large", Quantity=1
+    )["HostIds"][0]
+
+    hosts = client.describe_hosts(
+        Filters=[{"Name": "availability-zone", "Values": ["us-west-1b"]}]
+    )["Hosts"]
+    [h["HostId"] for h in hosts].should.contain(host_id2)
+
+    hosts = client.describe_hosts(
+        Filters=[{"Name": "availability-zone", "Values": ["us-west-1d"]}]
+    )["Hosts"]
+    hosts.should.have.length_of(0)
+
+    client.release_hosts(HostIds=[host_id1])
+    hosts = client.describe_hosts(Filters=[{"Name": "state", "Values": ["released"]}])[
+        "Hosts"
+    ]
+    [h["HostId"] for h in hosts].should.contain(host_id1)
+
+    hosts = client.describe_hosts(
+        Filters=[{"Name": "state", "Values": ["under-assessment"]}]
+    )["Hosts"]
+    hosts.should.have.length_of(0)
+
+
+@mock_ec2
+def test_modify_hosts():
+    client = boto3.client("ec2", "us-west-1")
+    host_ids = client.allocate_hosts(
+        AvailabilityZone="us-west-1a", InstanceFamily="c5", Quantity=1
+    )["HostIds"]
+
+    client.modify_hosts(
+        HostIds=host_ids,
+        AutoPlacement="off",
+        HostRecovery="on",
+        InstanceType="c5.medium",
+    )
+
+    host = client.describe_hosts(HostIds=host_ids)["Hosts"][0]
+
+    host.should.have.key("AutoPlacement").equals("off")
+    host.should.have.key("HostRecovery").equals("on")
+    host.should.have.key("HostProperties").shouldnt.have.key("InstanceFamily")
+    host.should.have.key("HostProperties").should.have.key("InstanceType").equals(
+        "c5.medium"
+    )
+
+
+@mock_ec2
+def test_release_hosts():
+    client = boto3.client("ec2", "us-west-1")
+    host_ids = client.allocate_hosts(
+        AvailabilityZone="us-west-1a",
+        InstanceType="a1.small",
+        HostRecovery="off",
+        AutoPlacement="on",
+        Quantity=2,
+    )["HostIds"]
+
+    resp = client.release_hosts(HostIds=[host_ids[0]])
+    resp.should.have.key("Successful").equals([host_ids[0]])
+
+    host = client.describe_hosts(HostIds=[host_ids[0]])["Hosts"][0]
+
+    host.should.have.key("State").equals("released")
+
+
+@mock_ec2
+def test_add_tags_to_dedicated_hosts():
+    client = boto3.client("ec2", "us-west-1")
+    resp = client.allocate_hosts(
+        AvailabilityZone="us-west-1a", InstanceType="a1.small", Quantity=1
+    )
+    host_id = resp["HostIds"][0]
+
+    client.create_tags(Resources=[host_id], Tags=[{"Key": "k1", "Value": "v1"}])
+
+    host = client.describe_hosts(HostIds=[host_id])["Hosts"][0]
+    host.should.have.key("Tags").equals([{"Key": "k1", "Value": "v1"}])