[WIP] New code generator based on code policy

scripts/gen.py

@@ -1,199 +1,34 @@
-import argparse
-import json
-
-import sys
-
-
-# Python code generator to create new troposphere classes from the
-# AWS resource specification.
-#
-# Todo:
-# - Currently only handles the single files (not the all-in-one)
-#   (Note: but will deal with things like spec/GuardDuty*)
-# - Handle adding in validators
-# - Verify propery dependency/ordering in the file
-# - Needs better error checking
-# - Need to figure out the correct Timestamp type
-
-copyright_header = """\
-# Copyright (c) 2012-2018, Mark Peek <[email protected]>
-# All rights reserved.
-#
-# See LICENSE file for full license.
-
-from . import AWSObject, AWSProperty
-from .validators import boolean, integer
-"""
-
-
-def get_required(value):
-    return value['Required']
-
-
-map_type = {
-    'Boolean': 'boolean',
-    'Double': 'float',
-    'Integer': 'integer',
-    'Json': 'dict',
-    'Long': 'integer',
-    'String': 'basestring',
-    'Timestamp': 'basestring',
-}
-
-
-map_type3 = {
-    'Boolean': 'bool',
-    'Double': 'float',
-    'Integer': 'int',
-    'Json': 'dict',
-    'Long': 'int',
-    'String': 'str',
-    'Timestamp': 'str',
-}
-
-
-def get_type(value):
-    if 'PrimitiveType' in value:
-        return map_type.get(value['PrimitiveType'], value['PrimitiveType'])
-    if value['Type'] == 'List':
-        if 'ItemType' in value:
-            return "[%s]" % value['ItemType']
-        else:
-            return "[%s]" % map_type.get(value['PrimitiveItemType'])
-    elif value['Type'] == 'Map':
-        return 'dict'
-    else:
-        # Non-primitive (Property) name
-        return value['Type']
+from troposphere_gen.specification import Specification
+from troposphere_gen.generator import Generator
+from troposphere_gen.policy import *
+from troposphere_gen.policy import cc_to_sc
 
-    import pprint
-    pprint.pprint(value)
-    raise ValueError("get_type")
-
-
-def get_type3(value):
-    if 'PrimitiveType' in value:
-        return map_type3.get(value['PrimitiveType'], value['PrimitiveType'])
-    if value['Type'] == 'List':
-        if 'ItemType' in value:
-            return "[%s]" % value['ItemType']
-        else:
-            return "[%s]" % map_type3.get(value['PrimitiveItemType'])
-    elif value['Type'] == 'Map':
-        return 'dict'
-    else:
-        # Non-primitive (Property) name
-        return value['Type']
-
-    import pprint
-    pprint.pprint(value)
-    raise ValueError("get_type")
-
-
-def output_class(class_name, properties, resource_name=None):
-    print
-    print
-    if resource_name:
-        print 'class %s(AWSObject):' % class_name
-        print '    resource_type = "%s"' % resource_name
-        print
-    else:
-        print 'class %s(AWSProperty):' % class_name
-
-    # Output the props dict
-    print '    props = {'
-    for key, value in sorted(properties.iteritems()):
-        if key == 'Tags':
-            value_type = "Tags"
-        else:
-            value_type = get_type(value)
-
-        # Wrap long names for pycodestyle
-        if len(key) + len(value_type) < 55:
-            print "        '%s': (%s, %s)," % (
-                key, value_type, get_required(value))
-        else:
-            print "        '%s':\n            (%s, %s)," % (
-                key, value_type, get_required(value))
-    print '    }'
-
-
-def output_class_stub(class_name, properties, resource_name=None):
-    print
-    print
-    if resource_name:
-        print 'class %s(AWSObject):' % class_name
-        print '    resource_type: str'
-        print
-        sys.stdout.write('    def __init__(self, title')
-    else:
-        print 'class %s(AWSProperty):' % class_name
-        print
-        sys.stdout.write('    def __init__(self')
-
-    for key, value in sorted(properties.iteritems()):
-        if key == 'Tags':
-            value_type = "Tags"
-        else:
-            value_type = get_type3(value)
-
-        if value_type.startswith("["):  # Means that args are a list
-            sys.stdout.write(', %s:List%s=...' % (key, value_type))
-        else:
-            sys.stdout.write(', %s:%s=...' % (key, value_type))
-
-    print ') -> None: ...'
-    print
+import json
 
-    for key, value in sorted(properties.iteritems()):
-        if key == 'Tags':
-            value_type = "Tags"
-        else:
-            value_type = get_type3(value)
+from collections import OrderedDict
 
-        if value_type.startswith("["):  # Means that args are a list
-            print '    %s: List%s' % (key, value_type)
-        else:
-            print '    %s: %s' % (key, value_type)
 
+def generate(specificationfile: str, outdir: str, policy: Policy):
+    with open(specificationfile, "r") as f:
+        specdata = json.load(f, object_pairs_hook=OrderedDict)
 
-def process_file(filename, stub=False):
-    f = open(filename)
-    j = json.load(f)
+    spec = Specification(specdata)
+    gen = Generator(spec)
 
-    if 'PropertyTypes' in j:
-        for property_name, property_dict in j['PropertyTypes'].items():
-            if property_name == "Tag":
-                print "from troposphere import Tags"
-                print
-                continue
-            class_name = property_name.split('.')[1]
-            properties = property_dict['Properties']
-            if stub:
-                output_class_stub(class_name, properties)
-            else:
-                output_class(class_name, properties)
+    for name, module in gen.modules.items():
+        with open(outdir + cc_to_sc(name) + ".py", "w") as f:
+            f.write(policy.module_head_format(module, spec))
+            f.write(policy.after_import())
 
-    for resource_name, resource_dict in j['ResourceType'].items():
-        class_name = resource_name.split(':')[4]
-        properties = resource_dict['Properties']
-        if stub:
-            output_class_stub(class_name, properties, resource_name)
-        else:
-            output_class(class_name, properties, resource_name)
+            for name, cd in module.properties.items():
+                f.write(policy.class_format(cd))
+                f.write(policy.between_class())
 
+            for name, cd in module.resources.items():
+                f.write(policy.class_format(cd))
+                f.write(policy.between_class())
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--stub', action='store_true', default=False)
-    parser.add_argument('filename', nargs='+')
-    args = parser.parse_args()
 
-    if args.stub:
-        print copyright_header,
-        for f in args.filename:
-            process_file(f, stub=True)
-    else:
-        print copyright_header,
-        for f in args.filename:
-            process_file(f)
+if __name__ == "__main__":
+    generate("CloudFormationResourceSpecification.json", "build/2.7/", Policy_2_7())
+    generate("CloudFormationResourceSpecification.json", "build/3.7/", Policy_3_7())

tests/generator/specification_testdata.json

@@ -0,0 +1,170 @@
+{
+    "PropertyTypes": {
+        "AWS::Elasticsearch::Domain.EBSOptions": {
+            "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-ebsoptions.html",
+            "Properties": {
+                "EBSEnabled": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-ebsoptions.html#cfn-elasticsearch-domain-ebsoptions-ebsenabled",
+                    "PrimitiveType": "Boolean",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                },
+                "Iops": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-ebsoptions.html#cfn-elasticsearch-domain-ebsoptions-iops",
+                    "PrimitiveType": "Integer",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                },
+                "VolumeSize": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-ebsoptions.html#cfn-elasticsearch-domain-ebsoptions-volumesize",
+                    "PrimitiveType": "Integer",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                },
+                "VolumeType": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-ebsoptions.html#cfn-elasticsearch-domain-ebsoptions-volumetype",
+                    "PrimitiveType": "String",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                }
+            }
+        },
+        "AWS::Elasticsearch::Domain.ElasticsearchClusterConfig": {
+            "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-elasticsearchclusterconfig.html",
+            "Properties": {
+                "DedicatedMasterCount": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-elasticsearchclusterconfig.html#cfn-elasticsearch-domain-elasticseachclusterconfig-dedicatedmastercount",
+                    "PrimitiveType": "Integer",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                },
+                "DedicatedMasterEnabled": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-elasticsearchclusterconfig.html#cfn-elasticsearch-domain-elasticseachclusterconfig-dedicatedmasterenabled",
+                    "PrimitiveType": "Boolean",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                },
+                "DedicatedMasterType": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-elasticsearchclusterconfig.html#cfn-elasticsearch-domain-elasticseachclusterconfig-dedicatedmastertype",
+                    "PrimitiveType": "String",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                },
+                "InstanceCount": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-elasticsearchclusterconfig.html#cfn-elasticsearch-domain-elasticseachclusterconfig-instancecount",
+                    "PrimitiveType": "Integer",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                },
+                "InstanceType": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-elasticsearchclusterconfig.html#cfn-elasticsearch-domain-elasticseachclusterconfig-instnacetype",
+                    "PrimitiveType": "String",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                },
+                "ZoneAwarenessEnabled": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-elasticsearchclusterconfig.html#cfn-elasticsearch-domain-elasticseachclusterconfig-zoneawarenessenabled",
+                    "PrimitiveType": "Boolean",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                }
+            }
+        },
+        "AWS::Elasticsearch::Domain.SnapshotOptions": {
+            "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-snapshotoptions.html",
+            "Properties": {
+                "AutomatedSnapshotStartHour": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticsearch-domain-snapshotoptions.html#cfn-elasticsearch-domain-snapshotoptions-automatedsnapshotstarthour",
+                    "PrimitiveType": "Integer",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                }
+            }
+        },
+        "Tag": {
+            "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-resource-tags.html",
+            "Properties": {
+                "Key": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-resource-tags.html#cfn-resource-tags-key",
+                    "PrimitiveType": "String",
+                    "Required": true,
+                    "UpdateType": "Immutable"
+                },
+                "Value": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-resource-tags.html#cfn-resource-tags-value",
+                    "PrimitiveType": "String",
+                    "Required": true,
+                    "UpdateType": "Immutable"
+                }
+            }
+        }
+    },
+    "ResourceType": {
+        "AWS::Elasticsearch::Domain": {
+            "Attributes": {
+                "DomainArn": {
+                    "PrimitiveType": "String"
+                },
+                "DomainEndpoint": {
+                    "PrimitiveType": "String"
+                }
+            },
+            "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticsearch-domain.html",
+            "Properties": {
+                "AccessPolicies": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticsearch-domain.html#cfn-elasticsearch-domain-accesspolicies",
+                    "PrimitiveType": "Json",
+                    "Required": false,
+                    "UpdateType": "Mutable"
+                },
+                "AdvancedOptions": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticsearch-domain.html#cfn-elasticsearch-domain-advancedoptions",
+                    "DuplicatesAllowed": false,
+                    "PrimitiveItemType": "String",
+                    "Required": false,
+                    "Type": "Map",
+                    "UpdateType": "Mutable"
+                },
+                "DomainName": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticsearch-domain.html#cfn-elasticsearch-domain-domainname",
+                    "PrimitiveType": "String",
+                    "Required": false,
+                    "UpdateType": "Immutable"
+                },
+                "EBSOptions": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticsearch-domain.html#cfn-elasticsearch-domain-ebsoptions",
+                    "Required": false,
+                    "Type": "EBSOptions",
+                    "UpdateType": "Mutable"
+                },
+                "ElasticsearchClusterConfig": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticsearch-domain.html#cfn-elasticsearch-domain-elasticsearchclusterconfig",
+                    "Required": false,
+                    "Type": "ElasticsearchClusterConfig",
+                    "UpdateType": "Mutable"
+                },
+                "ElasticsearchVersion": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticsearch-domain.html#cfn-elasticsearch-domain-elasticsearchversion",
+                    "PrimitiveType": "String",
+                    "Required": false,
+                    "UpdateType": "Immutable"
+                },
+                "SnapshotOptions": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticsearch-domain.html#cfn-elasticsearch-domain-snapshotoptions",
+                    "Required": false,
+                    "Type": "SnapshotOptions",
+                    "UpdateType": "Mutable"
+                },
+                "Tags": {
+                    "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-elasticsearch-domain.html#cfn-elasticsearch-domain-tags",
+                    "DuplicatesAllowed": true,
+                    "ItemType": "Tag",
+                    "Required": false,
+                    "Type": "List",
+                    "UpdateType": "Mutable"
+                }
+            }
+        }
+    },
+    "ResourceSpecificationVersion": "1.4.1"
+}