Emr ranger 2.0

aws_emr_blog_v3/cloudformation/emr-template.template

@@ -244,13 +244,16 @@ Parameters:
     AllowedValues:
       - emr-5.32.0
       - emr-6.3.0
+      - emr-6.4.0
+      - emr-6.7.0
     Description: Release label for the EMR cluster
   AppsEMR:
     Description: 'Comma separated list of applications to install on the cluster e.g., '
     Type: String
     Default: Hadoop, Spark, Hive, Livy, Hue
     AllowedValues:
       - "Hadoop, Spark, Hive, Livy, Hue"
+      - "Hadoop, Spark, Hive, Livy, Hue, Trino"
       - "Hadoop, Spark, Hive, Livy"
   EnableKerberos:
     Description: Enable Kerberos on the Cluster. This is Required for Ranger EMR support
@@ -341,6 +344,11 @@ Parameters:
     Default: true
     Type: String
     AllowedValues: [true, false]
+  EnableSparkDDLAndIceberg:
+    Description: Installs Spark DDL's and Iceberg configuration
+    Default: false
+    Type: String
+    AllowedValues: [ true, false ]
   RangerAgentKeySecretName:
     Description: Name of Ranger Agent Cert Secrets mgr resource
     Type: String
@@ -533,10 +541,20 @@ Resources:
               - App: "EMRFS-S3"
                 ClientSecretARN: !Join ['', ['arn:aws:secretsmanager:', !Ref "AWS::Region", ':', !Ref "AWS::AccountId", ':secret:', !Ref RangerAgentKeySecretName]]
                 PolicyRepositoryName: "amazonemrs3"
+              - App: "Trino"
+                ClientSecretARN: !Join ['', ['arn:aws:secretsmanager:', !Ref "AWS::Region", ':', !Ref "AWS::AccountId", ':secret:', !Ref RangerAgentKeySecretName]]
+                PolicyRepositoryName: "amazonemrtrino"
             AuditConfiguration:
               Destinations:
                 AmazonCloudWatchLogs:
                   CloudWatchLogGroup: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:${RangerCloudWatchLogGroupName}'
+        EncryptionConfiguration:
+          EnableInTransitEncryption: true
+          EnableAtRestEncryption: false
+          InTransitEncryptionConfiguration:
+            TLSCertificateConfiguration:
+              CertificateProviderType: PEM
+              S3Object: !Join ['', ["s3://", !Ref S3ArtifactBucket, "/", !Ref S3ArtifactKey, "/", !Ref ProjectVersion, "/emr-tls/", "emr-certs-certs.zip"]]
   LaunchEMRClusterFunction:
     Type: AWS::Lambda::Function
     DependsOn: LambdaExecutionRole
@@ -603,9 +621,11 @@ Resources:
       DBHostName: !Ref DBHostName
       DBUserName: !Ref DBUserName
       DBRootPassword: !Ref DBRootPassword
+      ClientSecretARN: !Join ['', ['arn:aws:secretsmanager:', !Ref "AWS::Region", ':', !Ref "AWS::AccountId", ':secret:', !Ref RangerAgentKeySecretName]]
       CertLocationPath: !Join ['', ["s3://", !Ref S3ArtifactBucket, "/", !Ref S3ArtifactKey, "/", !Ref ProjectVersion]]
       RangerAdminPassword: !Ref RangerAdminPassword
       DefaultDomain: !If [ USEastRegion, 'EC2.INTERNAL', 'COMPUTE.INTERNAL' ]
+      EnableSparkDDLAndIceberg: !Ref EnableSparkDDLAndIceberg
 
   emrCreateWaitHandle:
     Type: AWS::CloudFormation::WaitConditionHandle

aws_emr_blog_v3/cloudformation/lambda-amilookup-win.template

@@ -68,6 +68,7 @@ Parameters:
       - Windows Server 2012 RTM English 64-bit
       - Windows Server 2012 R2 English 64-bit
       - Windows Server 2016 Base English 64-bit
+      - Windows Server 2019 Base English 64-bit
     ConstraintDescription: Must be a valid Windows version.
   ModuleName:
     Description: The name of the JavaScript file
@@ -87,6 +88,7 @@ Parameters:
     Type: String
     AllowedValues:
       - 3.0
+      - beta
 Resources:
 #  SampleInstance:
 #    Type: AWS::EC2::Instance

aws_emr_blog_v3/cloudformation/nestedstack.template

@@ -296,6 +296,7 @@ Parameters:
     AllowedValues:
       - emr-5.32.0
       - emr-6.3.0
+      - emr-6.4.0
   KeyPairName:
     Description: Name of an existing EC2 key pair to access the Amazon EMR cluster
     Type: AWS::EC2::KeyPair::KeyName

aws_emr_blog_v3/cloudformation/step1_vpc-ec2-ad.template

@@ -179,6 +179,7 @@ Resources:
     Properties:
       TemplateURL: !Join ['', ['https://s3.amazonaws.com/', !Ref 'S3ArtifactBucket', '/', !Ref 'S3ArtifactKey', '/', !Ref 'ProjectVersion', '/cloudformation/', 'lambda-amilookup-win.template']]
       Parameters:
+        ProjectVersion: !Ref 'ProjectVersion'
         S3Bucket: !Ref 'S3Bucket'
         S3Key: !Ref 'S3Key'
   STEP1VPC:

aws_emr_blog_v3/cloudformation/step2_ranger-rds-emr.template

@@ -216,7 +216,10 @@ Parameters:
     Description: 'Comma separated list of applications to install on the cluster e.g., '
     Type: String
     Default: Hadoop, Spark, Hive, Livy, Hue
-    AllowedValues: ["Hadoop, Spark, Hive, Livy, Hue"]
+    AllowedValues:
+      - "Hadoop, Spark, Hive, Livy, Hue"
+      - "Hadoop, Spark, Hive, Livy, Hue, Trino"
+      - "Hadoop, Spark, Hive, Livy"
   EnableKerberos:
     Description: Enable Kerberos on the Cluster. This is Required for Ranger EMR support
     Default: true
@@ -228,6 +231,8 @@ Parameters:
     AllowedValues:
       - emr-5.32.0
       - emr-6.3.0
+      - emr-6.4.0
+      - emr-6.7.0
   S3Bucket:
     Description: S3Bucket for the code [update this is you want to run this stack in a region other than US-EAST-1]
     Type: String
@@ -345,6 +350,7 @@ Resources:
       Parameters:
         S3Bucket: !Ref 'S3ArtifactBucket'
         S3Key: !Ref 'S3ArtifactKey'
+        ProjectVersion: !Ref 'ProjectVersion'
         VPC: !Ref VPC
         Subnet: !If [ InstallEMRRangerinPublicSubnet, !Ref PublicSubnet1AID, !Ref PrivateSubnet1AID ]
         DBHostName: !GetAtt 'RDSDatabase.Outputs.RDSInstanceAddress'
@@ -368,6 +374,7 @@ Resources:
     Properties:
       TemplateURL: !Join ['', ['https://s3.amazonaws.com/', !Ref 'S3ArtifactBucket', '/', !Ref 'S3ArtifactKey', '/', !Ref 'ProjectVersion', '/cloudformation/', 'emr-template.template']]
       Parameters:
+        ProjectVersion: !Ref 'ProjectVersion'
         S3Bucket: !Ref 'S3Bucket'
         S3Key: !Ref 'S3Key'
         S3ArtifactBucket: !Ref 'S3ArtifactBucket'

aws_emr_blog_v3/code/amilookup-win/amilookup-win.js

@@ -12,7 +12,8 @@ var osNameToPattern = {
     "Windows Server 2008 R2 English 64-bit": "Windows_Server-2008-R2_SP1-English-64Bit-Base-*",
 	"Windows Server 2012 RTM English 64-bit": "Windows_Server-2012-RTM-English-64Bit-Base-*",
 	"Windows Server 2012 R2 English 64-bit": "Windows_Server-2012-R2_RTM-English-64Bit-Base-*",
-	"Windows Server 2016 Base English 64-bit": "Windows_Server-2016-English-Full-Base-*"
+	"Windows Server 2016 Base English 64-bit": "Windows_Server-2016-English-Full-Base-*",
+	"Windows Server 2019 Base English 64-bit": "Windows_Server-2019-English-Full-Base-*"
 };
 
 var aws = require("aws-sdk");