CloudFormation: Add coverage checklist (#6129)
This change introduces a checklist similar to IMPLEMENTATION_COVERAGE.md to document moto's adherence to the full CloudFormation specification. The script (invoked via `make cloudformation_coverage`) finds any moto models that implement `moto.core.common_models.CloudFormationModel` and checks to see if all CRUD methods are implemented. For `has_cfn_attr`, it also checks to see if all of the attributes exposed by `Fn::GetAtt` are implemented. It does not check to see if `physical_resource_id` is implemented because as far as I can tell, the published AWS spec doesn't include that information: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-resource-specification.html A more aggressive approach would be to remove default implementations from all the abstract methods and allow Python to throw for any unimplemented method. But that would obviously break a lot of things. Instead, I think the checklist represents good documentation for users about what they can and can't expect moto to handle when parsing their CloudFormation templates. It also serves as a nice to-do list of small improvements for contributors to add (and I'll likely add a few myself). Many of these would be particularly good "first issues" for first time contributors because in general, these methods just call existing methods.
This commit is contained in:
Jordan Sanders
GitHub
parent
fa2b6d28b0
commit
94d35af520
351
CLOUDFORMATION_COVERAGE.md
Normal file
351
CLOUDFORMATION_COVERAGE.md
Normal file
@ -0,0 +1,351 @@
|
||||
- AWS::ApiGateway::Deployment:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] DeploymentId
|
||||
- AWS::ApiGateway::Method:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::ApiGateway::Resource:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] ResourceId
|
||||
- AWS::AutoScaling::AutoScalingGroup:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] LaunchConfigurationName
|
||||
- [ ] LaunchTemplateSpecification
|
||||
- [ ] MixedInstancesPolicy
|
||||
- [ ] PlacementGroup
|
||||
- [ ] VPCZoneIdentifier
|
||||
- AWS::AutoScaling::LaunchConfiguration:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::AutoScaling::ScheduledAction:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] ScheduledActionName
|
||||
- AWS::Batch::ComputeEnvironment:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] ComputeEnvironmentArn
|
||||
- AWS::Batch::JobDefinition:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::Batch::JobQueue:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] JobQueueArn
|
||||
- AWS::CloudFormation::Stack:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::DataPipeline::Pipeline:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::DynamoDB::Table:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::EC2::Instance:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::EC2::InternetGateway:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] InternetGatewayId
|
||||
- AWS::EC2::LaunchTemplate:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] LatestVersionNumber
|
||||
- [ ] DefaultVersionNumber
|
||||
- AWS::EC2::NatGateway:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] NatGatewayId
|
||||
- AWS::EC2::NetworkInterface:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] Id
|
||||
- AWS::EC2::Route:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::EC2::RouteTable:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] RouteTableId
|
||||
- AWS::EC2::SecurityGroup:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] VpcId
|
||||
- AWS::EC2::SecurityGroupIngress:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::EC2::Subnet:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] VpcId
|
||||
- [ ] NetworkAclAssociationId
|
||||
- [ ] OutpostArn
|
||||
- [ ] SubnetId
|
||||
- [ ] Ipv6CidrBlocks
|
||||
- AWS::EC2::SubnetRouteTableAssociation:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] Id
|
||||
- AWS::EC2::TransitGateway:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] Id
|
||||
- AWS::EC2::VPC:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] VpcId
|
||||
- [ ] CidrBlockAssociations
|
||||
- [ ] CidrBlock
|
||||
- [ ] DefaultNetworkAcl
|
||||
- [ ] Ipv6CidrBlocks
|
||||
- [ ] DefaultSecurityGroup
|
||||
- AWS::EC2::VPCGatewayAttachment:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::EC2::VPCPeeringConnection:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] Id
|
||||
- AWS::EC2::Volume:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] VolumeId
|
||||
- AWS::EC2::VolumeAttachment:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::ECR::Repository:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::ECS::Cluster:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::ECS::Service:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] ServiceArn
|
||||
- AWS::ECS::TaskDefinition:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] TaskDefinitionArn
|
||||
- AWS::EFS::FileSystem:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] FileSystemId
|
||||
- [ ] Arn
|
||||
- AWS::EFS::MountTarget:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] IpAddress
|
||||
- [ ] Id
|
||||
- AWS::Events::Archive:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::Events::EventBus:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::Events::Rule:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::IAM::AccessKey:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::IAM::InstanceProfile:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::IAM::ManagedPolicy:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::IAM::Policy:
|
||||
- [ ] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::IAM::Role:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] RoleId
|
||||
- AWS::IAM::User:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::KMS::Key:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] KeyId
|
||||
- AWS::Kinesis::Stream:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::Logs::LogGroup:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] Arn
|
||||
- AWS::RDS::DBParameterGroup:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] DBParameterGroupName
|
||||
- AWS::Redshift::Cluster:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] Id
|
||||
- [ ] DeferMaintenanceIdentifier
|
||||
- AWS::Route53::HealthCheck:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] HealthCheckId
|
||||
- AWS::Route53::RecordSet:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::Route53::RecordSetGroup:
|
||||
- [x] create implemented
|
||||
- [ ] update implemented
|
||||
- [ ] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::S3::Bucket:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::SNS::Topic:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] TopicArn
|
||||
- AWS::SQS::Queue:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] QueueUrl
|
||||
- AWS::SSM::Parameter:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] Type
|
||||
- [ ] Value
|
||||
- AWS::SageMaker::Endpoint:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::SageMaker::EndpointConfig:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::SageMaker::Model:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [x] Fn::GetAtt implemented
|
||||
- AWS::StepFunctions::StateMachine:
|
||||
- [x] create implemented
|
||||
- [x] update implemented
|
||||
- [x] delete implemented
|
||||
- [ ] Fn::GetAtt implemented
|
||||
- [ ] StateMachineRevisionId
|
||||
- [ ] Arn
|
||||
6
Makefile
6
Makefile
@ -59,6 +59,12 @@ implementation_coverage:
|
||||
./scripts/implementation_coverage.py
|
||||
git commit IMPLEMENTATION_COVERAGE.md -m "Updating implementation coverage" || true
|
||||
|
||||
cloudformation_coverage:
|
||||
./scripts/cloudformation_coverage.py > CLOUDFORMATION_COVERAGE.md
|
||||
git commit CLOUDFORMATION_COVERAGE.md -m "Updating CloudFormation coverage" || true
|
||||
|
||||
coverage: implementation_coverage cloudformation_coverage
|
||||
|
||||
scaffold:
|
||||
@pip install -r requirements-dev.txt > /dev/null
|
||||
exec python scripts/scaffold.py
|
||||
|
||||
111
scripts/cloudformation_coverage.py
Executable file
111
scripts/cloudformation_coverage.py
Executable file
@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python
|
||||
import importlib
|
||||
import json
|
||||
import mock
|
||||
import requests
|
||||
|
||||
import moto
|
||||
|
||||
# Populate CloudFormationModel.__subclasses__()
|
||||
moto.mock_all()
|
||||
|
||||
|
||||
def check(condition):
|
||||
if bool(condition):
|
||||
return "x"
|
||||
else:
|
||||
return " "
|
||||
|
||||
|
||||
def is_implemented(model, method_name):
|
||||
# method_name in model.__dict__ will be True if the method
|
||||
# exists on the model and False if it's only inherited from
|
||||
# CloudFormationModel.
|
||||
return hasattr(model, method_name) and method_name in model.__dict__
|
||||
|
||||
|
||||
class CloudFormationChecklist:
|
||||
def __init__(self, resource_name, schema):
|
||||
self.resource_name = resource_name
|
||||
self.schema = schema
|
||||
|
||||
def __str__(self):
|
||||
missing_attrs_checklist = "\n".join(
|
||||
[f" - [ ] {attr}" for attr in self.missing_attrs]
|
||||
)
|
||||
report = (
|
||||
f"- {self.resource_name}:\n"
|
||||
f" - [{check(self.creatable)}] create implemented\n"
|
||||
f" - [{check(self.updatable)}] update implemented\n"
|
||||
f" - [{check(self.deletable)}] delete implemented\n"
|
||||
f" - [{check(not self.missing_attrs)}] Fn::GetAtt implemented\n"
|
||||
) + missing_attrs_checklist
|
||||
|
||||
return report.strip()
|
||||
|
||||
@property
|
||||
def service_name(self):
|
||||
return self.resource_name.split("::")[1].lower()
|
||||
|
||||
@property
|
||||
def model_name(self):
|
||||
return self.resource_name.split("::")[2]
|
||||
|
||||
@property
|
||||
def moto_model(self):
|
||||
for subclass in moto.core.common_models.CloudFormationModel.__subclasses__():
|
||||
subclass_service = subclass.__module__.split(".")[1]
|
||||
subclass_model = subclass.__name__
|
||||
|
||||
if subclass_service == self.service_name and subclass_model in (
|
||||
self.model_name,
|
||||
"Fake" + self.model_name,
|
||||
):
|
||||
return subclass
|
||||
|
||||
@property
|
||||
def expected_attrs(self):
|
||||
return list(self.schema.get("Attributes", {}).keys())
|
||||
|
||||
@property
|
||||
def missing_attrs(self):
|
||||
missing_attrs = []
|
||||
for attr in self.expected_attrs:
|
||||
try:
|
||||
# TODO: Change the actual abstract method to return False
|
||||
with mock.patch(
|
||||
"moto.core.common_models.CloudFormationModel.has_cfn_attr",
|
||||
return_value=False,
|
||||
):
|
||||
if not self.moto_model.has_cfn_attr(attr):
|
||||
missing_attrs.append(attr)
|
||||
except:
|
||||
missing_attrs.append(attr)
|
||||
return missing_attrs
|
||||
|
||||
@property
|
||||
def creatable(self):
|
||||
return is_implemented(self.moto_model, "create_from_cloudformation_json")
|
||||
|
||||
@property
|
||||
def updatable(self):
|
||||
return is_implemented(self.moto_model, "update_from_cloudformation_json")
|
||||
|
||||
@property
|
||||
def deletable(self):
|
||||
return is_implemented(self.moto_model, "delete_from_cloudformation_json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-resource-specification.html
|
||||
cfn_spec = requests.get(
|
||||
"https://dnwj8swjjbsbt.cloudfront.net/latest/gzip/CloudFormationResourceSpecification.json"
|
||||
).json()
|
||||
for resource_name, schema in sorted(cfn_spec["ResourceTypes"].items()):
|
||||
checklist = CloudFormationChecklist(resource_name, schema)
|
||||
# Only print checklists for models that implement CloudFormationModel;
|
||||
# otherwise the checklist is very long and mostly empty because there
|
||||
# are so many niche AWS services and resources that moto doesn't
|
||||
# implement yet.
|
||||
if checklist.moto_model:
|
||||
print(checklist)
|
||||
Loading…
x
Reference in New Issue
Block a user