From 9d05c29b286897ee41412af5bb164105bfdb1ecf Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Thu, 7 Jan 2016 14:08:08 -0800 Subject: [PATCH 1/7] Added new marketplace topology --- aerospike-cf-hvm-private.json | 2 +- aerospike-topology2.json | 413 ++++++++++++++++++++++++++++++ aerospike-topology3.json | 466 ++++++++++++++++++++++++++++++++++ custom_namespace.conf | 4 +- 4 files changed, 882 insertions(+), 3 deletions(-) create mode 100644 aerospike-topology2.json create mode 100644 aerospike-topology3.json diff --git a/aerospike-cf-hvm-private.json b/aerospike-cf-hvm-private.json index 7fce5d9..5ec732a 100644 --- a/aerospike-cf-hvm-private.json +++ b/aerospike-cf-hvm-private.json @@ -396,7 +396,7 @@ "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [ "#!/bin/bash -xe\n", "yum update -y aws-cfn-bootstrap\n", - + "yum install -y jq\n", "/opt/aws/bin/cfn-init -v ", " --stack ", { "Ref" : "AWS::StackName" }, " --resource LaunchConfig ", diff --git a/aerospike-topology2.json b/aerospike-topology2.json new file mode 100644 index 0000000..3821520 --- /dev/null +++ b/aerospike-topology2.json @@ -0,0 +1,413 @@ +{ + "AWSTemplateFormatVersion" : "2010-09-09", + "Description" : "Template to create an Aerospike cluster", + "Parameters" : { + "KeyPair" : { + "Description" : "Name of the KeyPair that would be used to ssh into the instances", + "Type" : "AWS::EC2::KeyPair::KeyName", + "ConstraintDescription" : "Please specify the name of the keypair that you use to login" + }, + "VPC" : { + "Description" : "The VPC to deploy into", + "Type" : "AWS::EC2::VPC::Id" + }, + "VPCSubnet" : { + "Description" : "Choose a subnet from the VPC selected above.", + "Type" : "AWS::EC2::Subnet::Id" + }, + "Tenancy" : { + "Description" : "The tenancy of your instance", + "Type" : "String", + "Default" : "default", + "AllowedValues" : [ "default", "dedicated"] + }, + "NumberOfInstances" : { + "Description" : "Number of instances in the cluster", + "Type" : "Number", + "Default" : "4", + "MinValue" : "1", + "MaxValue" : "15" + }, + "Cloudwatch" : { + "Description" : "Add basic Aerospike metrics to Cloundwatch. Will incur Cloudwatch expenses ~ $24/mo/instance", + "Type" : "String", + "Default" : "no", + "AllowedValues": [ "yes" , "no" ] + }, + "PermitSSH" : { + "Description" : "CIDR block that's permitted to SSH to the Aerospike Cluster", + "Type" : "String", + "AllowedPattern" : "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/([0-9]|[1-2][0-9]|3[0-2]))$", + "ConstraintDescription" : "Must be in CIDR notation. To specify one specifc IPv4 address, append /32. eg: 192.168.1.100/32" + }, + "InstanceType" : { + "Description" : "Type of EC2 instance to launch.", + "Type" : "String", + "Default" : "m3.large", + "AllowedValues" : [ "t2.micro", "t2.small", "t2.medium", "t2.large", + "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", + "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", + "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", + "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", + "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", + "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge" + ] + }, + "EBS" : { + "Description" : "Size of EBS SSD volume in GB. The volume will attach under /dev/sdg. Limit of 16000. Enter 0 to not use EBS.", + "Type" : "Number", + "Default" : "50", + "MinValue" : "0", + "MaxValue" : "16000" + }, + "NamespaceFile" : { + "Description" : "(Optional) Location of your namespace definition. Must be publically downloadable. Will append file directly to end of aerospike.conf", + "Type" : "String" + } + }, + + "Mappings" : { + "RegionMap" : { + "us-east-1" : {"name": "ami-9b3c79f1"}, + "us-west-2" : {"name": "ami-22abba43"}, + "us-west-1" : {"name": "ami-8286e8e2"}, + "eu-west-1" : {"name": "ami-27fc2654"}, + "eu-central-1" : {"name" : "ami-df3220b3"}, + "ap-southeast-1" : {"name": "ami-b58140d6"}, + "ap-southeast-2" : {"name": "ami-e9227b8a"}, + "ap-northeast-1" : {"name": "ami-fd321193"}, + "sa-east-1" : {"name": "ami-1c51eb70"} + } + }, + + "Conditions" : { + "NotUsingEBS" : { "Fn::Equals" : [ { "Ref" : "EBS" }, 0 ] } + }, + + "Resources" : { + "ClusterRole" : { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version" : "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Principal": { + "Service": [ "ec2.amazonaws.com","autoscaling.amazonaws.com" ] + }, + "Action": [ "sts:AssumeRole" ] + } ] + }, + "Path": "/", + "Policies": [ { + "PolicyName": "AerospikeClusterPolicy", + "PolicyDocument": { + "Version" : "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Action": "ec2:DescribeInstances", + "Resource": "*" + } ] + } + },{ + "PolicyName": "AerospikeCloudWatchPolicy", + "PolicyDocument" :{ + "Version" : "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Action": "cloudwatch:PutMetricData", + "Resource": "*" + } ] + } + },{ + "PolicyName": "AerospikeSQSPolicy", + "PolicyDocument" :{ + "Version" : "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Action": "sqs:*", + "Resource": { "Fn::GetAtt" : ["MigrationSQS", "Arn"]} + } ] + } + },{ + "PolicyName": "AerospikeAutoScalingPolicy", + "PolicyDocument" :{ + "Version" : "2012-10-17", + "Statement" : [ { + "Effect": "Allow", + "Action": "autoscaling:*", + "Resource": "*" + } ] + } + }] + } + }, + + + "ClusterInstanceProfile": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "Path": "/", + "Roles": [ { + "Ref": "ClusterRole" + } ] + } + }, + "MigrationSQS" : { + "Type": "AWS::SQS::Queue", + "Properties": { + "ReceiveMessageWaitTimeSeconds": 10 + } + }, + "MigrationHook": { + "Type": "AWS::AutoScaling::LifecycleHook", + "DependsOn" : "MigrationSQS", + "Properties": { + "AutoScalingGroupName": { "Ref": "ClusterGroup" }, + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING", + "NotificationTargetARN": { "Fn::GetAtt": [ "MigrationSQS", "Arn" ] }, + "RoleARN": { "Fn::GetAtt": [ "ClusterRole", "Arn" ] } + } + }, + + "ClusterGroup" : { + "Type" : "AWS::AutoScaling::AutoScalingGroup", + "Properties" : { + "LaunchConfigurationName" : { "Ref" : "LaunchConfig" }, + "DesiredCapacity" : { "Ref" : "NumberOfInstances"}, + "MinSize" : "1", + "MaxSize" : "15", + "VPCZoneIdentifier" : [{ "Ref" : "VPCSubnet" }], + "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"}, "PropagateAtLaunch" : "true" } ] + } + }, + "LaunchConfig" : { + "Type" : "AWS::AutoScaling::LaunchConfiguration", + "Metadata" : { + "AWS::CloudFormation::Init" : { + "config" : { + "files" : { + "/tmp/aerospike_cluster" : { + "content" : { "Fn::Join" : ["", [ + "#!/bin/bash\n", + "echo ClusterInstancesScriptStart > /var/log/awsuserdatascript\n", + " PUBLICIP=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)\n", + " CONF=/etc/aerospike/aerospike.conf\n", + " sed -i \"/port 3000/a \\\t\taccess-address $PUBLICIP virtual\" $CONF\n", + " ###Point to all instances using the mesh-address config option\n", + " PRIVATEIP=$(aws ec2 describe-instances --filter Name=tag-key,Values=StackID Name=tag-value,Values=", { "Ref" : "AWS::StackId" }," --output=text --region=",{ "Ref" : "AWS::Region" }," | grep PRIVATEIPADDRESSES | awk '{print $4}') \n", + " echo $PRIVATEIP >> /var/log/awsuserdatascript\n", + " sed -i '/.*mesh-seed-address-port/d' $CONF\n", + " for i in $PRIVATEIP; do ", + " sed -i \"/interval/i \\\t\tmesh-seed-address-port $i 3002\" $CONF\n", + " CODE=$(curl -Is",{ "Ref" :"NamespaceFile" }," | head -n 1 | cut -d$' ' -f2)\n", + " if [ \"$CODE\" != \"200\" ]; then echo 'Namespace File not found' >> /var/log/awsuserdatascript\n", + " else curl -s ",{ "Ref" :"NamespaceFile" }," >> $CONF; fi\n", + " /etc/init.d/aerospike start\n", + " /etc/init.d/amc start\n", + "echo OtherInstancesScriptFinish >> /var/log/awsuserdatascript\n", + "(crontab -l 2>/dev/null; echo '*/5 * * * * /opt/aerospike/poll_sqs') | crontab -\n", + " if [[ \"",{ "Ref" : "Cloudwatch" },"\" == \"yes\" ]]; then\n", + " (crontab -l 2>/dev/null; echo '*/5 * * * * /opt/aerospike/cloudwatch') | crontab -; fi\n" + ] ] }, + "mode" : "000744", + "owner" : "root", + "group" : "root" + }, + "/opt/aerospike/cloudwatch" : { + "content" : { "Fn::Join" : ["", [ + "#!/bin/bash\n", + "METRICS=$(asinfo -v stats -l)\n", + "NAMESPACE=aerospike\n", + "REGION=",{ "Ref" : "AWS::Region" },"\n", + "INSTANCE=$(curl 169.254.169.254/latest/meta-data/instance-id)\n", + "CLUSTER=",{ "Ref" : "AWS::StackId" },"\n", + "for L in $METRICS; do\n", + " LINE=(${L//=/ })\n", + " case ${LINE[0]} in\n", + " cluster_integrity)\n", + " if [[ ${LINE[1]} != \"true\" ]]; then\n", + " INTEGRITY_ERROR=1\n", + " else\n", + " INTEGRITY_ERROR=0\n", + " fi\n", + " ;;\n", + " total-bytes-memory)\n", + " TBM=${LINE[1]}\n", + " ;;\n", + " used-bytes-memory)\n", + " UBM=${LINE[1]}\n", + " ;;\n", + " total-bytes-disk)\n", + " TBD=${LINE[1]}\n", + " ;;\n", + " used-bytes-disk)\n", + " UBD=${LINE[1]}\n", + " ;;\n", + " objects)\n", + " OBJECTS=${LINE[1]}\n", + " ;;\n", + " *)\n", + " continue\n", + " ;;\n", + " esac\n", + "done\n", + "FM=$(expr $TBM - $UBM)\n", + "FD=$(expr $TBD - $UBD)\n", + "# Submit metrics\n", + "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $INTEGRITY_ERROR --metric-name 'Cluster Integrity'\n", + "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $FM --metric-name 'Free Memory' --unit 'Bytes'\n", + "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $FD --metric-name 'Free Disk' --unit 'Bytes'\n", + "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $OBJECTS --metric-name 'Number of Objects' --unit 'Count'\n" + ] ] }, + "mode" : "000744", + "owner" : "root", + "group" : "root" + }, + "/opt/aerospike/poll_sqs" : { + "content" : { "Fn::Join" : [ "" , [ + "#!/bin/bash\n", + "# This script will prevent autoscaling from terminating\n", + "# this instance until ASD migrations are completed\n", + "set -e\n", + "MYIP=$(curl 169.254.169.254/latest/meta-data/local-ipv4)\n", + "MYNODE=$(curl 169.254.169.254/latest/meta-data/instance-id)\n", + "REGION='",{ "Ref" : "AWS::Region"},"' # CFT\n", + "QUEUE='",{ "Ref" : "MigrationSQS"},"' # CFT\n", + "CLUSTER=$(aws ec2 describe-instances --filter Name=tag-key,Values=StackID Name=tag-value,Values=",{ "Ref":"AWS::StackId"}," --output=text --region=$REGION | grep PRIVATEIPADDRESSES | awk '{print $4}')\n", + "# Find SQS message with termination message\n", + "FOUND=false\n", + "MESSAGE=$(aws sqs receive-message --region $REGION --queue-url $QUEUE --region $REGION --wait-time-seconds 10 --visibility-timeout 2 )\n", "BODY=$(echo $MESSAGE | jq '.Messages[0] .Body')\n", + "RECEIPT=$(echo $MESSAGE | jq --raw-output '.Messages[0] .ReceiptHandle')\n", + "LIFECYCLE=$(eval echo $BODY | jq --raw-output '.LifecycleTransition')\n", "INSTANCE=$(eval echo $BODY | jq --raw-output '.EC2InstanceId')\n", + "if [[ $LIFECYCLE == 'autoscaling:EC2_INSTANCE_TERMINATING' ]] && [[ $INSTANCE == $MYNODE ]]; then\n", + " TOKEN=$(eval echo $BODY | jq --raw-output '.LifecycleActionToken')\n", + " HOOK=$(eval echo $BODY | jq --raw-output '.LifecycleHookName')\n", + " ASG=$(eval echo $BODY | jq --raw-output '.AutoScalingGroupName')\n", + " FOUND=true\n", + " aws sqs delete-message --region $REGION --queue-url $QUEUE --receipt-handle $RECEIPT\n", + "fi\n", + "# If not not found, exit\n", + "if [[ $FOUND == false ]]; then\n", + " exit 0\n", + "fi\n", + "# stop aerospike\n", + "/etc/init.d/aerospike stop\n", + "# Find first node that's not myself\n", + "for I in $CLUSTER; do\n", + " if [[ $I == $MYIP ]]; then\n", + " continue;\n", + " fi\n", + " NODE=$I\n", + " break\n", + "done\n", + "# Grab migration info\n", + "MIGRATIONS=$(asadm -h $NODE -e 'info service'| sed -n '4,/Number/ {/Number/!p}' | awk '{print $8}')\n", + "DONE=true\n", + "# check every node's migration status\n", + "for NODE in $MIGRATIONS; do\n", + " if [[ $NODE != '(0,0,0)' ]] && [[ $NODE != 'N/E' ]]; then\n", + " $DONE=false\n", + " break;\n", + " fi\n", + "done\n", + "# if migrations not done, pause ASG actions. Otherwise, continue autoscaling termination.\n", + "if [[ $DONE == false ]]; then\n", + " aws autoscaling record-lifecycle-action-heartbeat --region $REGION --lifecycle-action-token $TOKEN --auto-scaling-group-name $ASG --lifecycle-hook-name $HOOK\n", + " else\n", + " aws autoscaling complete-lifecycle-action --region $REGION --lifecycle-action-token $TOKEN --lifecycle-hook-name $HOOK --auto-scaling-group-name $ASG --lifecycle-action-result CONTINUE\n", + "fi\n" + ] ] }, + "mode" : "000744", + "owner" : "root", + "group" : "root" + } + }, + "commands" : { + "01_form_asd_cluster" : { + "command" : "/tmp/aerospike_cluster", + "cwd" : "/tmp" + } + } + } + } + }, + "Properties" : { + "InstanceType" : { "Ref" : "InstanceType"}, + "KeyName" : { "Ref" : "KeyPair" }, + "BlockDeviceMappings" : { "Fn::If" : [ "NotUsingEBS", + { "Ref" : "AWS::NoValue" }, + [ { + "DeviceName" : "/dev/sdg", + "Ebs" : { "VolumeSize" : {"Ref" : "EBS" }, + "VolumeType" : "gp2" } + } ] + ] }, + "IamInstanceProfile" : { "Ref" : "ClusterInstanceProfile" }, + "ImageId" : { "Fn::FindInMap" : [ "RegionMap", {"Ref" : "AWS::Region"} , "name"] }, + "AssociatePublicIpAddress" : "true", + "PlacementTenancy" : { "Ref" : "Tenancy" }, + "SecurityGroups" : [ { "Fn::GetAtt" : [ "InstanceSecurityGroup", "GroupId" ] } ], + "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [ + "#!/bin/bash -xe\n", + "yum update -y aws-cfn-bootstrap\n", + "yum install -y jq\n", + "/opt/aws/bin/cfn-init -v ", + " --stack ", { "Ref" : "AWS::StackName" }, + " --resource LaunchConfig ", + " --region ", { "Ref" : "AWS::Region" }, "\n", + + "/opt/aws/bin/cfn-signal -e $? ", + " --stack ", { "Ref" : "AWS::StackName" }, + " --resource ClusterGroup ", + " --region ", { "Ref" : "AWS::Region" }, "\n" + ] ] } + } + } + }, + "InstanceSecurityGroup" : { + "Type" : "AWS::EC2::SecurityGroup", + "Properties" : { + "GroupDescription" : "Enable ports to access Aerospike", + "VpcId" : { "Ref" : "VPC" }, + "SecurityGroupIngress" : [ { + "IpProtocol" : "tcp", + "FromPort" : "3000", + "ToPort" : "3000", + "CidrIp" : "0.0.0.0/0" + }, + { + "IpProtocol" : "tcp", + "FromPort" : "8081", + "ToPort" : "8081", + "CidrIp" : "0.0.0.0/0" + }, + { + "IpProtocol" : "tcp", + "FromPort" : "22", + "ToPort" : "22", + "CidrIp" : { "Ref" : "PermitSSH" } + + }, + { + "IpProtocol" : "icmp", + "FromPort" : "-1", + "ToPort" : "-1", + "CidrIp" : "0.0.0.0/0" + } ], + "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] + } + }, + "InstanceSecurityGroupIngress" : { + "Type" : "AWS::EC2::SecurityGroupIngress", + "Properties" : { + "GroupId" : { "Fn::GetAtt" : [ "InstanceSecurityGroup", "GroupId"] }, + "IpProtocol" : "tcp", + "FromPort" : "3001", + "ToPort" : "3004", + "SourceSecurityGroupId" : { "Fn::GetAtt" : ["InstanceSecurityGroup", "GroupId"] } + }, + "DependsOn" : "InstanceSecurityGroup" + } + } +} diff --git a/aerospike-topology3.json b/aerospike-topology3.json new file mode 100644 index 0000000..732e9e9 --- /dev/null +++ b/aerospike-topology3.json @@ -0,0 +1,466 @@ +{ + "AWSTemplateFormatVersion" : "2010-09-09", + "Description" : "Template to create an Aerospike cluster", + "Parameters" : { + "KeyPair" : { + "Description" : "Name of the KeyPair that would be used to ssh into the instances", + "Type" : "AWS::EC2::KeyPair::KeyName", + "ConstraintDescription" : "Please specify the name of the keypair that you use to login" + }, + "Tenancy" : { + "Description" : "The tenancy of your instance", + "Type" : "String", + "Default" : "default", + "AllowedValues" : [ "default", "dedicated"] + }, + "NumberOfInstances" : { + "Description" : "Number of instances in the cluster", + "Type" : "Number", + "Default" : "4", + "MinValue" : "1", + "MaxValue" : "15" + }, + "Cloudwatch" : { + "Description" : "Add basic Aerospike metrics to Cloundwatch. Will incur Cloudwatch expenses ~ $24/mo/instance", + "Type" : "String", + "Default" : "no", + "AllowedValues": [ "yes" , "no" ] + }, + "PermitSSH" : { + "Description" : "CIDR block that's permitted to SSH to the Aerospike Cluster", + "Type" : "String", + "AllowedPattern" : "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/([0-9]|[1-2][0-9]|3[0-2]))$", + "ConstraintDescription" : "Must be in CIDR notation. To specify one specifc IPv4 address, append /32. eg: 192.168.1.100/32" + }, + "InstanceType" : { + "Description" : "Type of EC2 instance to launch.", + "Type" : "String", + "Default" : "m3.large", + "AllowedValues" : [ "t2.micro", "t2.small", "t2.medium", "t2.large", + "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", + "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", + "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", + "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", + "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", + "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge" + ] + }, + "EBS" : { + "Description" : "Size of EBS SSD volume in GB. The volume will attach under /dev/sdg. Limit of 16000. Enter 0 to not use EBS.", + "Type" : "Number", + "Default" : "50", + "MinValue" : "0", + "MaxValue" : "16000" + }, + "NamespaceFile" : { + "Description" : "(Optional) Location of your namespace definition. Must be publically downloadable. Will append file directly to end of aerospike.conf", + "Type" : "String" + } + }, + + "Mappings" : { + "RegionMap" : { + "us-east-1" : {"name": "ami-9b3c79f1"}, + "us-west-2" : {"name": "ami-22abba43"}, + "us-west-1" : {"name": "ami-8286e8e2"}, + "eu-west-1" : {"name": "ami-27fc2654"}, + "eu-central-1" : {"name" : "ami-df3220b3"}, + "ap-southeast-1" : {"name": "ami-b58140d6"}, + "ap-southeast-2" : {"name": "ami-e9227b8a"}, + "ap-northeast-1" : {"name": "ami-fd321193"}, + "sa-east-1" : {"name": "ami-1c51eb70"} + } + }, + + "Conditions" : { + "NotUsingEBS" : { "Fn::Equals" : [ { "Ref" : "EBS" }, 0 ] } + }, + + "Resources" : { + "ClusterRole" : { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version" : "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Principal": { + "Service": [ "ec2.amazonaws.com","autoscaling.amazonaws.com" ] + }, + "Action": [ "sts:AssumeRole" ] + } ] + }, + "Path": "/", + "Policies": [ { + "PolicyName": "AerospikeClusterPolicy", + "PolicyDocument": { + "Version" : "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Action": "ec2:DescribeInstances", + "Resource": "*" + } ] + } + },{ + "PolicyName": "AerospikeCloudWatchPolicy", + "PolicyDocument" :{ + "Version" : "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Action": "cloudwatch:PutMetricData", + "Resource": "*" + } ] + } + },{ + "PolicyName": "AerospikeSQSPolicy", + "PolicyDocument" :{ + "Version" : "2012-10-17", + "Statement": [ { + "Effect": "Allow", + "Action": "sqs:*", + "Resource": { "Fn::GetAtt" : ["MigrationSQS", "Arn"]} + } ] + } + },{ + "PolicyName": "AerospikeAutoScalingPolicy", + "PolicyDocument" :{ + "Version" : "2012-10-17", + "Statement" : [ { + "Effect": "Allow", + "Action": "autoscaling:*", + "Resource": "*" + } ] + } + }] + } + }, + + + "ClusterInstanceProfile": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "Path": "/", + "Roles": [ { + "Ref": "ClusterRole" + } ] + } + }, + "MigrationSQS" : { + "Type": "AWS::SQS::Queue", + "Properties": { + "ReceiveMessageWaitTimeSeconds": 10 + } + }, + "MigrationHook": { + "Type": "AWS::AutoScaling::LifecycleHook", + "DependsOn" : "MigrationSQS", + "Properties": { + "AutoScalingGroupName": { "Ref": "ClusterGroup" }, + "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING", + "NotificationTargetARN": { "Fn::GetAtt": [ "MigrationSQS", "Arn" ] }, + "RoleARN": { "Fn::GetAtt": [ "ClusterRole", "Arn" ] } + } + }, + + "ClusterGroup" : { + "Type" : "AWS::AutoScaling::AutoScalingGroup", + "Properties" : { + "LaunchConfigurationName" : { "Ref" : "LaunchConfig" }, + "DesiredCapacity" : { "Ref" : "NumberOfInstances"}, + "MinSize" : "1", + "MaxSize" : "15", + "AvailabilityZones" : [ { "Fn::Select" : [ "0", { "Fn::GetAZs" : { "Ref" : "AWS::Region" } } ] } ], + "VPCZoneIdentifier" : [{ "Ref" : "PublicSubnet" }], + "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"}, "PropagateAtLaunch" : "true" } ] + } + }, + "LaunchConfig" : { + "Type" : "AWS::AutoScaling::LaunchConfiguration", + "Metadata" : { + "AWS::CloudFormation::Init" : { + "config" : { + "files" : { + "/tmp/aerospike_cluster" : { + "content" : { "Fn::Join" : ["", [ + "#!/bin/bash\n", + "echo ClusterInstancesScriptStart > /var/log/awsuserdatascript\n", + " PUBLICIP=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)\n", + " CONF=/etc/aerospike/aerospike.conf\n", + " sed -i \"/port 3000/a \\\t\taccess-address $PUBLICIP virtual\" $CONF\n", + " ###Point to all instances using the mesh-address config option\n", + " PRIVATEIP=$(aws ec2 describe-instances --filter Name=tag-key,Values=StackID Name=tag-value,Values=", { "Ref" : "AWS::StackId" }," --output=text --region=",{ "Ref" : "AWS::Region" }," | grep PRIVATEIPADDRESSES | awk '{print $4}') \n", + " echo $PRIVATEIP >> /var/log/awsuserdatascript\n", + " sed -i '/.*mesh-seed-address-port/d' $CONF\n", + " for i in $PRIVATEIP; do ", + " sed -i \"/interval/i \\\t\tmesh-seed-address-port $i 3002\" $CONF\n", + " CODE=$(curl -Is",{ "Ref" :"NamespaceFile" }," | head -n 1 | cut -d$' ' -f2)\n", + " if [ \"$CODE\" != \"200\" ]; then echo 'Namespace File not found' >> /var/log/awsuserdatascript\n", + " else curl -s ",{ "Ref" :"NamespaceFile" }," >> $CONF; fi\n", + " /etc/init.d/aerospike start\n", + " /etc/init.d/amc start\n", + "echo OtherInstancesScriptFinish >> /var/log/awsuserdatascript\n", + "(crontab -l 2>/dev/null; echo '*/5 * * * * /opt/aerospike/poll_sqs') | crontab -\n", + " if [[ \"",{ "Ref" : "Cloudwatch" },"\" == \"yes\" ]]; then\n", + " (crontab -l 2>/dev/null; echo '*/5 * * * * /opt/aerospike/cloudwatch') | crontab -; fi\n" + ] ] }, + "mode" : "000744", + "owner" : "root", + "group" : "root" + }, + "/opt/aerospike/cloudwatch" : { + "content" : { "Fn::Join" : ["", [ + "#!/bin/bash\n", + "METRICS=$(asinfo -v stats -l)\n", + "NAMESPACE=aerospike\n", + "REGION=",{ "Ref" : "AWS::Region" },"\n", + "INSTANCE=$(curl 169.254.169.254/latest/meta-data/instance-id)\n", + "CLUSTER=",{ "Ref" : "AWS::StackId" },"\n", + "for L in $METRICS; do\n", + " LINE=(${L//=/ })\n", + " case ${LINE[0]} in\n", + " cluster_integrity)\n", + " if [[ ${LINE[1]} != \"true\" ]]; then\n", + " INTEGRITY_ERROR=1\n", + " else\n", + " INTEGRITY_ERROR=0\n", + " fi\n", + " ;;\n", + " total-bytes-memory)\n", + " TBM=${LINE[1]}\n", + " ;;\n", + " used-bytes-memory)\n", + " UBM=${LINE[1]}\n", + " ;;\n", + " total-bytes-disk)\n", + " TBD=${LINE[1]}\n", + " ;;\n", + " used-bytes-disk)\n", + " UBD=${LINE[1]}\n", + " ;;\n", + " objects)\n", + " OBJECTS=${LINE[1]}\n", + " ;;\n", + " *)\n", + " continue\n", + " ;;\n", + " esac\n", + "done\n", + "FM=$(expr $TBM - $UBM)\n", + "FD=$(expr $TBD - $UBD)\n", + "# Submit metrics\n", + "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $INTEGRITY_ERROR --metric-name 'Cluster Integrity'\n", + "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $FM --metric-name 'Free Memory' --unit 'Bytes'\n", + "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $FD --metric-name 'Free Disk' --unit 'Bytes'\n", + "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $OBJECTS --metric-name 'Number of Objects' --unit 'Count'\n" + ] ] }, + "mode" : "000744", + "owner" : "root", + "group" : "root" + }, + "/opt/aerospike/poll_sqs" : { + "content" : { "Fn::Join" : [ "" , [ + "#!/bin/bash\n", + "# This script will prevent autoscaling from terminating\n", + "# this instance until ASD migrations are completed\n", + "set -e\n", + "MYIP=$(curl 169.254.169.254/latest/meta-data/local-ipv4)\n", + "MYNODE=$(curl 169.254.169.254/latest/meta-data/instance-id)\n", + "REGION='",{ "Ref" : "AWS::Region"},"' # CFT\n", + "QUEUE='",{ "Ref" : "MigrationSQS"},"' # CFT\n", + "CLUSTER=$(aws ec2 describe-instances --filter Name=tag-key,Values=StackID Name=tag-value,Values=",{ "Ref":"AWS::StackId"}," --output=text --region=$REGION | grep PRIVATEIPADDRESSES | awk '{print $4}')\n", + "# Find SQS message with termination message\n", + "FOUND=false\n", + "MESSAGE=$(aws sqs receive-message --region $REGION --queue-url $QUEUE --region $REGION --wait-time-seconds 10 --visibility-timeout 2 )\n", "BODY=$(echo $MESSAGE | jq '.Messages[0] .Body')\n", + "RECEIPT=$(echo $MESSAGE | jq --raw-output '.Messages[0] .ReceiptHandle')\n", + "LIFECYCLE=$(eval echo $BODY | jq --raw-output '.LifecycleTransition')\n", "INSTANCE=$(eval echo $BODY | jq --raw-output '.EC2InstanceId')\n", + "if [[ $LIFECYCLE == 'autoscaling:EC2_INSTANCE_TERMINATING' ]] && [[ $INSTANCE == $MYNODE ]]; then\n", + " TOKEN=$(eval echo $BODY | jq --raw-output '.LifecycleActionToken')\n", + " HOOK=$(eval echo $BODY | jq --raw-output '.LifecycleHookName')\n", + " ASG=$(eval echo $BODY | jq --raw-output '.AutoScalingGroupName')\n", + " FOUND=true\n", + " aws sqs delete-message --region $REGION --queue-url $QUEUE --receipt-handle $RECEIPT\n", + "fi\n", + "# If not not found, exit\n", + "if [[ $FOUND == false ]]; then\n", + " exit 0\n", + "fi\n", + "# stop aerospike\n", + "/etc/init.d/aerospike stop\n", + "# Find first node that's not myself\n", + "for I in $CLUSTER; do\n", + " if [[ $I == $MYIP ]]; then\n", + " continue;\n", + " fi\n", + " NODE=$I\n", + " break\n", + "done\n", + "# Grab migration info\n", + "MIGRATIONS=$(asadm -h $NODE -e 'info service'| sed -n '4,/Number/ {/Number/!p}' | awk '{print $8}')\n", + "DONE=true\n", + "# check every node's migration status\n", + "for NODE in $MIGRATIONS; do\n", + " if [[ $NODE != '(0,0,0)' ]] && [[ $NODE != 'N/E' ]]; then\n", + " $DONE=false\n", + " break;\n", + " fi\n", + "done\n", + "# if migrations not done, pause ASG actions. Otherwise, continue autoscaling termination.\n", + "if [[ $DONE == false ]]; then\n", + " aws autoscaling record-lifecycle-action-heartbeat --region $REGION --lifecycle-action-token $TOKEN --auto-scaling-group-name $ASG --lifecycle-hook-name $HOOK\n", + " else\n", + " aws autoscaling complete-lifecycle-action --region $REGION --lifecycle-action-token $TOKEN --lifecycle-hook-name $HOOK --auto-scaling-group-name $ASG --lifecycle-action-result CONTINUE\n", + "fi\n" + ] ] }, + "mode" : "000744", + "owner" : "root", + "group" : "root" + } + }, + "commands" : { + "01_form_asd_cluster" : { + "command" : "/tmp/aerospike_cluster", + "cwd" : "/tmp" + } + } + } + } + }, + "Properties" : { + "InstanceType" : { "Ref" : "InstanceType"}, + "KeyName" : { "Ref" : "KeyPair" }, + "BlockDeviceMappings" : { "Fn::If" : [ "NotUsingEBS", + { "Ref" : "AWS::NoValue" }, + [ { + "DeviceName" : "/dev/sdg", + "Ebs" : { "VolumeSize" : {"Ref" : "EBS" }, + "VolumeType" : "gp2" } + } ] + ] }, + "IamInstanceProfile" : { "Ref" : "ClusterInstanceProfile" }, + "ImageId" : { "Fn::FindInMap" : [ "RegionMap", {"Ref" : "AWS::Region"} , "name"] }, + "AssociatePublicIpAddress" : "true", + "SecurityGroups" : [ { "Fn::GetAtt" : [ "InstanceSecurityGroup", "GroupId" ] } ], + "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [ + "#!/bin/bash -xe\n", + "yum update -y aws-cfn-bootstrap\n", + "yum install -y jq\n", + "/opt/aws/bin/cfn-init -v ", + " --stack ", { "Ref" : "AWS::StackName" }, + " --resource LaunchConfig ", + " --region ", { "Ref" : "AWS::Region" }, "\n", + + "/opt/aws/bin/cfn-signal -e $? ", + " --stack ", { "Ref" : "AWS::StackName" }, + " --resource ClusterGroup ", + " --region ", { "Ref" : "AWS::Region" }, "\n" + ] ] } + } + } + }, + "VPC" : { + "Type" : "AWS::EC2::VPC", + "Properties" : { + "CidrBlock" : "10.0.0.0/16", + "EnableDnsSupport" : "true", + "EnableDnsHostnames" : "true", + "InstanceTenancy": { "Ref" : "Tenancy" }, + "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] + } + }, + + "PublicSubnet" : { + "Type" : "AWS::EC2::Subnet", + "Properties" : { + "VpcId" : { "Ref" : "VPC" }, + "CidrBlock" : "10.0.0.0/24", + "AvailabilityZone" : { "Fn::Select" : [ "0", { "Fn::GetAZs" : { "Ref" : "AWS::Region" } } ] }, + "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] + } + }, + + "InternetGateway" : { + "Type" : "AWS::EC2::InternetGateway", + "Properties" : { + "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] + } + }, + + "GatewayToInternet" : { + "Type" : "AWS::EC2::VPCGatewayAttachment", + "Properties" : { + "VpcId" : { "Ref" : "VPC" }, + "InternetGatewayId" : { "Ref" : "InternetGateway" } + } + }, + + "PublicRouteTable" : { + "Type" : "AWS::EC2::RouteTable", + "Properties" : { + "VpcId" : { "Ref" : "VPC" }, + "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] + } + }, + + "PublicRoute" : { + "Type" : "AWS::EC2::Route", + "DependsOn" : "GatewayToInternet", + "Properties" : { + "RouteTableId" : { "Ref" : "PublicRouteTable" }, + "DestinationCidrBlock" : "0.0.0.0/0", + "GatewayId" : { "Ref" : "InternetGateway" } + } + }, + + "PublicSubnetRouteTableAssociation" : { + "Type" : "AWS::EC2::SubnetRouteTableAssociation", + "Properties" : { + "SubnetId" : { "Ref" : "PublicSubnet" }, + "RouteTableId" : { "Ref" : "PublicRouteTable" } + } + }, + + "InstanceSecurityGroup" : { + "Type" : "AWS::EC2::SecurityGroup", + "Properties" : { + "GroupDescription" : "Enable ports needed by Aerospike", + "VpcId" : { "Ref" : "VPC" }, + "SecurityGroupIngress" : [ { + "IpProtocol" : "tcp", + "FromPort" : "3000", + "ToPort" : "3000", + "CidrIp" : "0.0.0.0/0" + }, + { + "IpProtocol" : "tcp", + "FromPort" : "22", + "ToPort" : "22", + "CidrIp" : { "Ref" : "PermitSSH" } + }, + { + "IpProtocol" : "tcp", + "FromPort" : "8081", + "ToPort" : "8081", + "CidrIp" : "0.0.0.0/0" + }, + { + "IpProtocol" : "icmp", + "FromPort" : "-1", + "ToPort" : "-1", + "CidrIp" : "0.0.0.0/0" + } ], + "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] + } + }, + "InstanceSecurityGroupIngress" : { + "Type" : "AWS::EC2::SecurityGroupIngress", + "Properties" : { + "GroupId" : { "Fn::GetAtt" : [ "InstanceSecurityGroup", "GroupId"] }, + "IpProtocol" : "tcp", + "FromPort" : "3001", + "ToPort" : "3004", + "SourceSecurityGroupId" : { "Fn::GetAtt" : ["InstanceSecurityGroup", "GroupId"] } + }, + "DependsOn" : "InstanceSecurityGroup" + } + } +} diff --git a/custom_namespace.conf b/custom_namespace.conf index 79dbf27..9a92d85 100644 --- a/custom_namespace.conf +++ b/custom_namespace.conf @@ -14,8 +14,8 @@ namespace ExampleNS { # /dev/sdg = EBS. Must have provisioned an EBS volume. # The second entry is used as a shadow device. It is always recommended # to use ephemeral as the primary storage device and EBS as a shadow - # device. See the following article for more detail: + # device. See the following article for more details: # www.aerospike.com/docs/deploy_guides/aws/recommendations - write-block-size 256K + write-block-size 1M } } From d769ad3eee7e0b54cf7ed432109096d36f3000d2 Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Mon, 11 Jan 2016 12:26:15 -0800 Subject: [PATCH 2/7] Updated for 3.7.1 marketplace listing --- aerospike-topology2.json | 19 ++++++++++--------- aerospike-topology3.json | 19 ++++++++++--------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/aerospike-topology2.json b/aerospike-topology2.json index 3821520..6f209fa 100644 --- a/aerospike-topology2.json +++ b/aerospike-topology2.json @@ -68,15 +68,16 @@ "Mappings" : { "RegionMap" : { - "us-east-1" : {"name": "ami-9b3c79f1"}, - "us-west-2" : {"name": "ami-22abba43"}, - "us-west-1" : {"name": "ami-8286e8e2"}, - "eu-west-1" : {"name": "ami-27fc2654"}, - "eu-central-1" : {"name" : "ami-df3220b3"}, - "ap-southeast-1" : {"name": "ami-b58140d6"}, - "ap-southeast-2" : {"name": "ami-e9227b8a"}, - "ap-northeast-1" : {"name": "ami-fd321193"}, - "sa-east-1" : {"name": "ami-1c51eb70"} + "us-east-1" : {"name": "ami-871249ed"}, + "us-west-2" : {"name": "ami-baa8b3db"}, + "us-west-1" : {"name": "ami-3c8aff5c"}, + "eu-west-1" : {"name": "ami-2e6dc55d"}, + "eu-central-1" : {"name" : "ami-b8cbd5d4"}, + "ap-southeast-1" : {"name": "ami-9467aaf7"}, + "ap-southeast-2" : {"name": "ami-894762ea"}, + "ap-northeast-1" : {"name": "ami-358ebc5b"}, + "ap-northeast-2" : {"name": "ami-08b27c66"}, + "sa-east-1" : {"name": "ami-84e160e8"} } }, diff --git a/aerospike-topology3.json b/aerospike-topology3.json index 732e9e9..cf95138 100644 --- a/aerospike-topology3.json +++ b/aerospike-topology3.json @@ -60,15 +60,16 @@ "Mappings" : { "RegionMap" : { - "us-east-1" : {"name": "ami-9b3c79f1"}, - "us-west-2" : {"name": "ami-22abba43"}, - "us-west-1" : {"name": "ami-8286e8e2"}, - "eu-west-1" : {"name": "ami-27fc2654"}, - "eu-central-1" : {"name" : "ami-df3220b3"}, - "ap-southeast-1" : {"name": "ami-b58140d6"}, - "ap-southeast-2" : {"name": "ami-e9227b8a"}, - "ap-northeast-1" : {"name": "ami-fd321193"}, - "sa-east-1" : {"name": "ami-1c51eb70"} + "us-east-1" : {"name": "ami-871249ed"}, + "us-west-2" : {"name": "ami-baa8b3db"}, + "us-west-1" : {"name": "ami-3c8aff5c"}, + "eu-west-1" : {"name": "ami-2e6dc55d"}, + "eu-central-1" : {"name" : "ami-b8cbd5d4"}, + "ap-southeast-1" : {"name": "ami-9467aaf7"}, + "ap-southeast-2" : {"name": "ami-894762ea"}, + "ap-northeast-1" : {"name": "ami-358ebc5b"}, + "ap-northeast-2" : {"name": "ami-08b27c66"}, + "sa-east-1" : {"name": "ami-84e160e8"} } }, From 4a16d36b90db83891de39c6bddc71714aa0cecb0 Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Mon, 11 Jan 2016 12:45:50 -0800 Subject: [PATCH 3/7] Fixed do loop --- aerospike-topology2.json | 2 +- aerospike-topology3.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aerospike-topology2.json b/aerospike-topology2.json index 6f209fa..8909aab 100644 --- a/aerospike-topology2.json +++ b/aerospike-topology2.json @@ -200,7 +200,7 @@ " echo $PRIVATEIP >> /var/log/awsuserdatascript\n", " sed -i '/.*mesh-seed-address-port/d' $CONF\n", " for i in $PRIVATEIP; do ", - " sed -i \"/interval/i \\\t\tmesh-seed-address-port $i 3002\" $CONF\n", + " sed -i \"/interval/i \\\t\tmesh-seed-address-port $i 3002\" $CONF; done\n", " CODE=$(curl -Is",{ "Ref" :"NamespaceFile" }," | head -n 1 | cut -d$' ' -f2)\n", " if [ \"$CODE\" != \"200\" ]; then echo 'Namespace File not found' >> /var/log/awsuserdatascript\n", " else curl -s ",{ "Ref" :"NamespaceFile" }," >> $CONF; fi\n", diff --git a/aerospike-topology3.json b/aerospike-topology3.json index cf95138..eec9ff0 100644 --- a/aerospike-topology3.json +++ b/aerospike-topology3.json @@ -193,7 +193,7 @@ " echo $PRIVATEIP >> /var/log/awsuserdatascript\n", " sed -i '/.*mesh-seed-address-port/d' $CONF\n", " for i in $PRIVATEIP; do ", - " sed -i \"/interval/i \\\t\tmesh-seed-address-port $i 3002\" $CONF\n", + " sed -i \"/interval/i \\\t\tmesh-seed-address-port $i 3002\" $CONF; done\n", " CODE=$(curl -Is",{ "Ref" :"NamespaceFile" }," | head -n 1 | cut -d$' ' -f2)\n", " if [ \"$CODE\" != \"200\" ]; then echo 'Namespace File not found' >> /var/log/awsuserdatascript\n", " else curl -s ",{ "Ref" :"NamespaceFile" }," >> $CONF; fi\n", From 12f4ff63b7a82505eb5225cd74d0e068fb020ec0 Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Tue, 12 Jan 2016 14:43:58 -0800 Subject: [PATCH 4/7] Added client AMI details --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 4b04fb3..db08bd2 100644 --- a/README.md +++ b/README.md @@ -92,3 +92,21 @@ By default ping, Aerospike port 3000 and AMC port 8081 are open globally (0.0.0. ## Pricing The Aerospike AMI is a free subscription. You will be prompted to subscribe to the AMI before this CF template can be used. Pricing is dependant on the instance type used. Please see EC2 pricing [here](https://aws.amazon.com/ec2/pricing/). Cost will increase if launching more than 1 instance. + +# Clients +An AMI pre-loaded with most clients is available for quick development uptake. + +Owner: 262212597706 + +| Region | AMI | +|----------------|--------------| +| us-east-1 | ami-34b6ed5e | +| us-west-1 | ami-c5ed98a5 | +| us-west-2 | ami-01637960 | +| eu-central-1 | ami-06e4fa6a | +| eu-west-1 | ami-cf268ebc | +| ap-northeast-1 | ami-ed152883 | +| ap-northeast-2 | ami-5dce0033 | +| ap-southeast-1 | ami-41844822 | +| ap-southeast-2 | ami-d75673b4 | +| sa-east-1 | ami-14840578 | From 1e0a783b8cfde6760fa11b91db719a39217ab7b9 Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Thu, 28 Jan 2016 09:52:06 -0800 Subject: [PATCH 5/7] Fixed namespace dl typo. AER-4738 --- aerospike-topology2.json | 2 +- aerospike-topology3.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aerospike-topology2.json b/aerospike-topology2.json index 8909aab..f596fff 100644 --- a/aerospike-topology2.json +++ b/aerospike-topology2.json @@ -201,7 +201,7 @@ " sed -i '/.*mesh-seed-address-port/d' $CONF\n", " for i in $PRIVATEIP; do ", " sed -i \"/interval/i \\\t\tmesh-seed-address-port $i 3002\" $CONF; done\n", - " CODE=$(curl -Is",{ "Ref" :"NamespaceFile" }," | head -n 1 | cut -d$' ' -f2)\n", + " CODE=$(curl -Is ",{ "Ref" :"NamespaceFile" }," | head -n 1 | cut -d$' ' -f2)\n", " if [ \"$CODE\" != \"200\" ]; then echo 'Namespace File not found' >> /var/log/awsuserdatascript\n", " else curl -s ",{ "Ref" :"NamespaceFile" }," >> $CONF; fi\n", " /etc/init.d/aerospike start\n", diff --git a/aerospike-topology3.json b/aerospike-topology3.json index eec9ff0..8ea092d 100644 --- a/aerospike-topology3.json +++ b/aerospike-topology3.json @@ -194,7 +194,7 @@ " sed -i '/.*mesh-seed-address-port/d' $CONF\n", " for i in $PRIVATEIP; do ", " sed -i \"/interval/i \\\t\tmesh-seed-address-port $i 3002\" $CONF; done\n", - " CODE=$(curl -Is",{ "Ref" :"NamespaceFile" }," | head -n 1 | cut -d$' ' -f2)\n", + " CODE=$(curl -Is ",{ "Ref" :"NamespaceFile" }," | head -n 1 | cut -d$' ' -f2)\n", " if [ \"$CODE\" != \"200\" ]; then echo 'Namespace File not found' >> /var/log/awsuserdatascript\n", " else curl -s ",{ "Ref" :"NamespaceFile" }," >> $CONF; fi\n", " /etc/init.d/aerospike start\n", From aa572340858f296a5c63a76513400de10463c63a Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Thu, 28 Jan 2016 09:56:12 -0800 Subject: [PATCH 6/7] ASD 3.7.2 update --- aerospike-topology2.json | 20 ++++++++++---------- aerospike-topology3.json | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aerospike-topology2.json b/aerospike-topology2.json index f596fff..a6c70a4 100644 --- a/aerospike-topology2.json +++ b/aerospike-topology2.json @@ -68,16 +68,16 @@ "Mappings" : { "RegionMap" : { - "us-east-1" : {"name": "ami-871249ed"}, - "us-west-2" : {"name": "ami-baa8b3db"}, - "us-west-1" : {"name": "ami-3c8aff5c"}, - "eu-west-1" : {"name": "ami-2e6dc55d"}, - "eu-central-1" : {"name" : "ami-b8cbd5d4"}, - "ap-southeast-1" : {"name": "ami-9467aaf7"}, - "ap-southeast-2" : {"name": "ami-894762ea"}, - "ap-northeast-1" : {"name": "ami-358ebc5b"}, - "ap-northeast-2" : {"name": "ami-08b27c66"}, - "sa-east-1" : {"name": "ami-84e160e8"} + "us-east-1" : {"name": "ami-2d1b3847"}, + "us-west-2" : {"name": "ami-234ca843"}, + "us-west-1" : {"name": "ami-61621501"}, + "eu-west-1" : {"name": "ami-647dc917"}, + "eu-central-1" : {"name" : "ami-5b594137"}, + "ap-southeast-1" : {"name": "ami-34cd0257"}, + "ap-southeast-2" : {"name": "ami-fb5f7b98"}, + "ap-northeast-1" : {"name": "ami-00c5fc6e"}, + "ap-northeast-2" : {"name": "ami-9ee32df0"}, + "sa-east-1" : {"name": "ami-f389099f"} } }, diff --git a/aerospike-topology3.json b/aerospike-topology3.json index 8ea092d..6f7c28f 100644 --- a/aerospike-topology3.json +++ b/aerospike-topology3.json @@ -59,18 +59,18 @@ }, "Mappings" : { - "RegionMap" : { - "us-east-1" : {"name": "ami-871249ed"}, - "us-west-2" : {"name": "ami-baa8b3db"}, - "us-west-1" : {"name": "ami-3c8aff5c"}, - "eu-west-1" : {"name": "ami-2e6dc55d"}, - "eu-central-1" : {"name" : "ami-b8cbd5d4"}, - "ap-southeast-1" : {"name": "ami-9467aaf7"}, - "ap-southeast-2" : {"name": "ami-894762ea"}, - "ap-northeast-1" : {"name": "ami-358ebc5b"}, - "ap-northeast-2" : {"name": "ami-08b27c66"}, - "sa-east-1" : {"name": "ami-84e160e8"} - } + "RegionMap" : { + "us-east-1" : {"name": "ami-2d1b3847"}, + "us-west-2" : {"name": "ami-234ca843"}, + "us-west-1" : {"name": "ami-61621501"}, + "eu-west-1" : {"name": "ami-647dc917"}, + "eu-central-1" : {"name" : "ami-5b594137"}, + "ap-southeast-1" : {"name": "ami-34cd0257"}, + "ap-southeast-2" : {"name": "ami-fb5f7b98"}, + "ap-northeast-1" : {"name": "ami-00c5fc6e"}, + "ap-northeast-2" : {"name": "ami-9ee32df0"}, + "sa-east-1" : {"name": "ami-f389099f"} + } }, "Conditions" : { From 87ac0550d0a3eac11e3349010edd4a465ba565b6 Mon Sep 17 00:00:00 2001 From: Richard Guo Date: Thu, 28 Jan 2016 10:03:14 -0800 Subject: [PATCH 7/7] branch out AWS CFT --- aerospike-cf-hvm-private.json | 459 ---------------- aerospike-cf-hvm.json | 516 ------------------ ...ology2.json => aerospike-existing-vpc.json | 0 ...e-topology3.json => aerospike-new-vpc.json | 0 4 files changed, 975 deletions(-) delete mode 100644 aerospike-cf-hvm-private.json delete mode 100644 aerospike-cf-hvm.json rename aerospike-topology2.json => aerospike-existing-vpc.json (100%) rename aerospike-topology3.json => aerospike-new-vpc.json (100%) diff --git a/aerospike-cf-hvm-private.json b/aerospike-cf-hvm-private.json deleted file mode 100644 index 5ec732a..0000000 --- a/aerospike-cf-hvm-private.json +++ /dev/null @@ -1,459 +0,0 @@ -{ - "AWSTemplateFormatVersion" : "2010-09-09", - "Description" : "Template to create an Aerospike cluster", - "Parameters" : { - "KeyPair" : { - "Description" : "Name of the KeyPair that would be used to ssh into the instances", - "Type" : "AWS::EC2::KeyPair::KeyName", - "ConstraintDescription" : "Please specify the name of the keypair that you use to login" - }, - "AerospikeVersion" : { - "Description" : "Version of Aerospike to deploy", - "Type" : "String", - "Default" : "3.6.3", - "AllowedValues" : [ "3.6.0", "3.6.1", "3.6.2", "3.6.3" ] - }, - "VPC" : { - "Description" : "The VPC to deploy into", - "Type" : "AWS::EC2::VPC::Id" - }, - "VPCSubnet" : { - "Description" : "Choose a subnet from the VPC selected above.", - "Type" : "AWS::EC2::Subnet::Id" - }, - "Cloudwatch" : { - "Description" : "Add basic Aerospike metrics to Cloundwatch. Will incur Cloudwatch expenses ~ $24/mo/instance", - "Type" : "String", - "Default" : "no", - "AllowedValues": [ "yes" , "no" ] - }, - "Tenancy" : { - "Description" : "The tenancy of your instances", - "Type" : "String", - "Default" : "default", - "AllowedValues" : [ "default", "dedicated" ] - }, - "PermitSSH" : { - "Description" : "CIDR block that's permitted to SSH to the Aerospike Cluster", - "Type" : "String", - "AllowedPattern" : "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$", - "ConstraintDescription" : "Must be in CIDR notation. To specify one specifc IPv4 address, append /32. eg: 192.168.1.100/32" - }, - "NumberOfInstances" : { - "Description" : "Number of instances in the cluster", - "Type" : "Number", - "Default" : "4", - "MinValue" : "1", - "MaxValue" : "15" - }, - "InstanceType" : { - "Description" : "Type of EC2 instance to launch.", - "Type" : "String", - "Default" : "m3.large", - "AllowedValues" : [ "t2.micro", "t2.small", "t2.medium", "t2.large", - "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", - "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", - "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", - "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", - "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", - "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge" - ] - }, - "EBS" : { - "Description" : "Size of EBS SSD volume in GB. The volume will attach under /dev/sdg. Limit of 16000. Enter 0 to not use EBS.", - "Type" : "Number", - "Default" : "50", - "MinValue" : "0", - "MaxValue" : "16000" - }, - "NamespaceFile" : { - "Description" : "(Optional) Location of your namespace definition. Must be publically downloadable. Will append file directly to end of aerospike.conf", - "Type" : "String" - } - }, - - "Mappings" : { - "VersionMap" : { - "3.6.0" : { "Virginia" : "ami-71b1371a" , - "Oregon" : "ami-3f4c5f0f" , - "California" : "ami-d139c295" , - "Ireland" : "ami-8dfcd8fa" , - "Franfurt" : "ami-d61d1ccb" , - "Singapore" : "ami-dc1b108e" , - "Sydney" : "ami-8b3579b1" , - "Tokyo" : "ami-56f17e56" , - "SPaulo" : "ami-b9f57fa4" }, - "3.6.1" : { "Virginia" : "ami-43295126" , - "Oregon" : "ami-bd8b938d" , - "California" : "ami-8daa6dc9" , - "Ireland" : "ami-5b38102c" , - "Franfurt" : "ami-74adae69" , - "Singapore" : "ami-980215ca" , - "Sydney" : "ami-0b4d0431" , - "Tokyo" : "ami-547be654" , - "SPaulo" : "ami-9934a084" }, - "3.6.2" : { "Virginia" : "ami-0d4f0568" , - "Oregon" : "ami-9e1cfaad" , - "California" : "ami-2b6aab6f" , - "Ireland" : "ami-b5bb8bc2" , - "Franfurt" : "ami-02f4f91f" , - "Singapore" : "ami-d4435386", - "Sydney" : "ami-e97932d3" , - "Tokyo" : "ami-caea8dca" , - "SPaulo" : "ami-6b198f76" }, - "3.6.3" : { "Virginia" : "ami-e1f9a684" , - "Oregon" : "ami-c60be9f5" , - "California" : "ami-1f1edd5b" , - "Ireland" : "ami-a75967d0" , - "Franfurt" : "ami-7c828e61" , - "Singapore" : "ami-32b7a460" , - "Sydney" : "ami-27f8b21d" , - "Tokyo" : "ami-4e6c004e" , - "SPaulo" : "ami-0b893167" } - }, - "RegionMap" : { - "us-east-1" : {"name": "Virginia"}, - "us-west-2" : {"name": "Oregon"}, - "us-west-1" : {"name": "California"}, - "eu-west-1" : {"name": "Ireland"}, - "eu-central-1" : {"name" : "Franfurt"}, - "ap-southeast-1" : {"name": "Singapore"}, - "ap-southeast-2" : {"name": "Sydney"}, - "ap-northeast-1" : {"name": "Tokyo"}, - "sa-east-1" : {"name": "SPaulo"} - } - }, - - "Conditions" : { - "NotUsingEBS" : { "Fn::Equals" : [ { "Ref" : "EBS" }, 0 ] } - }, - - "Resources" : { - "ClusterRole" : { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Version" : "2012-10-17", - "Statement": [ { - "Effect": "Allow", - "Principal": { - "Service": [ "ec2.amazonaws.com","autoscaling.amazonaws.com" ] - }, - "Action": [ "sts:AssumeRole" ] - } ] - }, - "Path": "/", - "Policies": [ { - "PolicyName": "AerospikeClusterPolicy", - "PolicyDocument": { - "Version" : "2012-10-17", - "Statement": [ { - "Effect": "Allow", - "Action": "ec2:DescribeInstances", - "Resource": "*" - } ] - } - },{ - "PolicyName": "AerospikeCloudWatchPolicy", - "PolicyDocument": { - "Version" : "2012-10-17", - "Statement": [ { - "Effect": "Allow", - "Action": "cloudwatch:PutMetricData", - "Resource": "*" - } ] - } - },{ - "PolicyName": "AerospikeSQSPolicy", - "PolicyDocument" :{ - "Version" : "2012-10-17", - "Statement": [ { - "Effect": "Allow", - "Action": "sqs:*", - "Resource": { "Fn::GetAtt" : ["MigrationSQS", "Arn"]} - } ] - } - },{ - "PolicyName": "AerospikeAutoScalingPolicy", - "PolicyDocument" :{ - "Version" : "2012-10-17", - "Statement" : [ { - "Effect": "Allow", - "Action": "autoscaling:*", - "Resource": "*" - } ] - } - } ] - } - }, - - "ClusterInstanceProfile": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "Path": "/", - "Roles": [ { - "Ref": "ClusterRole" - } ] - } - }, - "MigrationSQS" : { - "Type": "AWS::SQS::Queue", - "Properties": { - "ReceiveMessageWaitTimeSeconds": 10 - } - }, - "MigrationHook": { - "Type": "AWS::AutoScaling::LifecycleHook", - "DependsOn" : "MigrationSQS", - "Properties": { - "AutoScalingGroupName": { "Ref": "ClusterGroup" }, - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING", - "NotificationTargetARN": { "Fn::GetAtt": [ "MigrationSQS", "Arn" ] }, - "RoleARN": { "Fn::GetAtt": [ "ClusterRole", "Arn" ] } - } - }, - - "ClusterGroup" : { - "Type" : "AWS::AutoScaling::AutoScalingGroup", - "Properties" : { - "LaunchConfigurationName" : { "Ref" : "LaunchConfig" }, - "DesiredCapacity" : { "Ref" : "NumberOfInstances"}, - "MinSize" : "1", - "MaxSize" : "15", - "VPCZoneIdentifier" : [{ "Ref" : "VPCSubnet" }], - "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"}, "PropagateAtLaunch" : "true" } ] - } - }, - "LaunchConfig" : { - "Type" : "AWS::AutoScaling::LaunchConfiguration", - "Metadata" : { - "AWS::CloudFormation::Init" : { - "config" : { - "files" : { - "/tmp/aerospike_cluster" : { - "content" : { "Fn::Join" : ["", [ - "#!/bin/bash\n", - "echo ClusterInstancesScriptStart > /var/log/awsuserdatascript\n", - " CONF=/etc/aerospike/aerospike.conf\n", - " ###Point to all instances using the mesh-address config option\n", - " PRIVATEIP=$(aws ec2 describe-instances --filter Name=tag-key,Values=StackID Name=tag-value,Values=", { "Ref" : "AWS::StackId" }," --output=text --region=",{ "Ref" : "AWS::Region" }," | grep PRIVATEIPADDRESSES | awk '{print $4}') \n", - " echo $PRIVATEIP >> /var/log/awsuserdatascript\n", - " sed -i '/.*mesh-seed-address-port/d' $CONF\n", - " for i in $PRIVATEIP; do ", - " sed -i \"/interval/i \\\t\tmesh-seed-address-port $i 3002\" $CONF\n", - " done \n", - " if [ ! -z \"",{ "Ref" :"NamespaceFile" },"\" ]; then\n", - " curl -s ",{ "Ref" :"NamespaceFile" }," >> $CONF; fi\n", - " /etc/init.d/aerospike start\n", - " /etc/init.d/amc start\n", - "echo OtherInstancesScriptFinish >> /var/log/awsuserdatascript\n", - - "(crontab -l 2>/dev/null; echo '*/5 * * * * /opt/aerospike/poll_sqs') | crontab -\n", - " if [[ \"",{ "Ref" : "Cloudwatch" },"\" == \"yes\" ]]; then\n", - " (crontab -l 2>/dev/null; echo '*/5 * * * * /opt/aerospike/cloudwatch') | crontab -; fi\n" - ] ] }, - "mode" : "000744", - "owner" : "root", - "group" : "root" - }, - "/opt/aerospike/cloudwatch" : { - "content" : { "Fn::Join" : ["", [ - "#!/bin/bash\n", - "METRICS=$(asinfo -v stats -l)\n", - "NAMESPACE=aerospike\n", - "REGION=",{ "Ref" : "AWS::Region" },"\n", - "INSTANCE=$(curl 169.254.169.254/latest/meta-data/instance-id)\n", - "CLUSTER=",{ "Ref" : "AWS::StackId" },"\n", - "for L in $METRICS; do\n", - " LINE=(${L//=/ })\n", - " case ${LINE[0]} in\n", - " cluster_integrity)\n", - " if [[ ${LINE[1]} != \"true\" ]]; then\n", - " INTEGRITY_ERROR=1\n", - " else\n", - " INTEGRITY_ERROR=0\n", - " fi\n", - " ;;\n", - " total-bytes-memory)\n", - " TBM=${LINE[1]}\n", - " ;;\n", - " used-bytes-memory)\n", - " UBM=${LINE[1]}\n", - " ;;\n", - " total-bytes-disk)\n", - " TBD=${LINE[1]}\n", - " ;;\n", - " used-bytes-disk)\n", - " UBD=${LINE[1]}\n", - " ;;\n", - " objects)\n", - " OBJECTS=${LINE[1]}\n", - " ;;\n", - " *)\n", - " continue\n", - " ;;\n", - " esac\n", - "done\n", - "FM=$(expr $TBM - $UBM)\n", - "FD=$(expr $TBD - $UBD)\n", - "# Submit metrics\n", - "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $INTEGRITY_ERROR --metric-name 'Cluster Integrity'\n", - "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $FM --metric-name 'Free Memory' --unit 'Bytes'\n", - "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $FD --metric-name 'Free Disk' --unit 'Bytes'\n", - "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $OBJECTS --metric-name 'Number of Objects' --unit 'Count'\n" - ] ] }, - "mode" : "000744", - "owner" : "root", - "group" : "root" - }, - "/opt/aerospike/poll_sqs" : { - "content" : { "Fn::Join" : [ "" , [ - "#!/bin/bash\n", - "# This script will prevent autoscaling from terminating\n", - "# this instance until ASD migrations are completed\n", - "set -e\n", - "MYIP=$(curl 169.254.169.254/latest/meta-data/local-ipv4)\n", - "MYNODE=$(curl 169.254.169.254/latest/meta-data/instance-id)\n", - "REGION='",{ "Ref" : "AWS::Region"},"' # CFT\n", - "QUEUE='",{ "Ref" : "MigrationSQS"},"' # CFT\n", - "CLUSTER=$(aws ec2 describe-instances --filter Name=tag-key,Values=StackID Name=tag-value,Values=",{ "Ref":"AWS::StackId"}," --output=text --region=$REGION | grep PRIVATEIPADDRESSES | awk '{print $4}')\n", - "# Find SQS message with termination message\n", - "FOUND=false\n", - "MESSAGE=$(aws sqs receive-message --region $REGION --queue-url $QUEUE --region $REGION --wait-time-seconds 10 --visibility-timeout 2 )\n", - "BODY=$(echo $MESSAGE | jq '.Messages[0] .Body')\n", - "RECEIPT=$(echo $MESSAGE | jq --raw-output '.Messages[0] .ReceiptHandle')\n", - "LIFECYCLE=$(eval echo $BODY | jq --raw-output '.LifecycleTransition')\n", - "INSTANCE=$(eval echo $BODY | jq --raw-output '.EC2InstanceId')\n", - "if [[ $LIFECYCLE == 'autoscaling:EC2_INSTANCE_TERMINATING' ]] && [[ $INSTANCE == $MYNODE ]]; then\n", - " TOKEN=$(eval echo $BODY | jq --raw-output '.LifecycleActionToken')\n", - " HOOK=$(eval echo $BODY | jq --raw-output '.LifecycleHookName')\n", - " ASG=$(eval echo $BODY | jq --raw-output '.AutoScalingGroupName')\n", - " FOUND=true\n", - " aws sqs delete-message --region $REGION --queue-url $QUEUE --receipt-handle $RECEIPT\n", - "fi\n", - "# If not not found, exit\n", - "if [[ $FOUND == false ]]; then\n", - " exit 0\n", - "fi\n", - "# stop aerospike\n", - "/etc/init.d/aerospike stop\n", - "# Find first node that's not myself\n", - "for I in $CLUSTER; do\n", - " if [[ $I == $MYIP ]]; then\n", - " continue;\n", - " fi\n", - " NODE=$I\n", - " break\n", - "done\n", - "# Grab migration info\n", - "MIGRATIONS=$(asadm -h $NODE -e 'info service'| sed -n '4,/Number/ {/Number/!p}' | awk '{print $8}')\n", - "DONE=true\n", - "# check every node's migration status\n", - "for NODE in $MIGRATIONS; do\n", - " if [[ $NODE != '(0,0,0)' ]] && [[ $NODE != 'N/E' ]]; then\n", - " $DONE=false\n", - " break;\n", - " fi\n", - "done\n", - "# if migrations not done, pause ASG actions. Otherwise, continue autoscaling termination.\n", - "if [[ $DONE == false ]]; then\n", - " aws autoscaling record-lifecycle-action-heartbeat --region $REGION --lifecycle-action-token $TOKEN --auto-scaling-group-name $ASG --lifecycle-hook-name $HOOK\n", - " else\n", - " aws autoscaling complete-lifecycle-action --region $REGION --lifecycle-action-token $TOKEN --lifecycle-hook-name $HOOK --auto-scaling-group-name $ASG --lifecycle-action-result CONTINUE\n", - "fi\n" - ] ] }, - "mode" : "000744", - "owner" : "root", - "group" : "root" - } - - }, - "commands" : { - "01_form_asd_cluster" : { - "command" : "/tmp/aerospike_cluster", - "cwd" : "/tmp" - } - } - } - } - }, - "Properties" : { - "InstanceType" : { "Ref" : "InstanceType"}, - "BlockDeviceMappings" : { "Fn::If" : [ "NotUsingEBS", - { "Ref" : "AWS::NoValue" }, - [ { - "DeviceName" : "/dev/sdg", - "Ebs" : { "VolumeSize" : {"Ref" : "EBS" }, - "VolumeType" : "gp2" } - } ] - ] }, - "KeyName" : { "Ref" : "KeyPair" }, - "IamInstanceProfile" : { "Ref" : "ClusterInstanceProfile" }, - "ImageId" : { "Fn::FindInMap" : [ "VersionMap", {"Ref" : "AerospikeVersion"} , { "Fn::FindInMap" : [ "RegionMap", { "Ref": "AWS::Region"}, "name"] } ] }, - "AssociatePublicIpAddress" : "true", - "PlacementTenancy" : { "Ref" : "Tenancy" }, - "SecurityGroups" : [ { "Fn::GetAtt" : [ "InstanceSecurityGroup", "GroupId" ] } ], - "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [ - "#!/bin/bash -xe\n", - "yum update -y aws-cfn-bootstrap\n", - "yum install -y jq\n", - "/opt/aws/bin/cfn-init -v ", - " --stack ", { "Ref" : "AWS::StackName" }, - " --resource LaunchConfig ", - " --region ", { "Ref" : "AWS::Region" }, "\n", - - "/opt/aws/bin/cfn-signal -e $? ", - " --stack ", { "Ref" : "AWS::StackName" }, - " --resource ClusterGroup ", - " --region ", { "Ref" : "AWS::Region" }, "\n" - ] ] } - } - } - }, - - "InstanceSecurityGroup" : { - "Type" : "AWS::EC2::SecurityGroup", - "Properties" : { - "GroupDescription" : "Enable ports to access Aerospike", - "VpcId" : { "Ref" : "VPC" }, - "SecurityGroupIngress" : [ { - "IpProtocol" : "tcp", - "FromPort" : "3000", - "ToPort" : "3000", - "CidrIp" : "0.0.0.0/0" - }, - { - "IpProtocol" : "tcp", - "FromPort" : "22", - "ToPort" : "22", - "CidrIp" : { "Ref" : "PermitSSH"} - - }, - { - "IpProtocol" : "tcp", - "FromPort" : "8081", - "ToPort" : "8081", - "CidrIp" : "0.0.0.0/0" - }, - { - "IpProtocol" : "icmp", - "FromPort" : "-1", - "ToPort" : "-1", - "CidrIp" : "0.0.0.0/0" - } ], - "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] - } - }, - "InstanceSecurityGroupIngress" : { - "Type" : "AWS::EC2::SecurityGroupIngress", - "Properties" : { - "GroupId" : { "Fn::GetAtt" : [ "InstanceSecurityGroup", "GroupId"] }, - "IpProtocol" : "tcp", - "FromPort" : "3001", - "ToPort" : "3004", - "SourceSecurityGroupId" : { "Fn::GetAtt" : ["InstanceSecurityGroup", "GroupId"] } - }, - "DependsOn" : "InstanceSecurityGroup" - } - } -} diff --git a/aerospike-cf-hvm.json b/aerospike-cf-hvm.json deleted file mode 100644 index 23c3830..0000000 --- a/aerospike-cf-hvm.json +++ /dev/null @@ -1,516 +0,0 @@ -{ - "AWSTemplateFormatVersion" : "2010-09-09", - "Description" : "Template to create an Aerospike cluster", - "Parameters" : { - "KeyPair" : { - "Description" : "Name of the KeyPair that would be used to ssh into the instances", - "Type" : "AWS::EC2::KeyPair::KeyName", - "ConstraintDescription" : "Please specify the name of the keypair that you use to login" - }, - "AerospikeVersion" : { - "Description" : "Version of Aerospike to deploy", - "Type" : "String", - "Default" : "3.6.3", - "AllowedValues" : [ "3.6.0", "3.6.1", "3.6.2", "3.6.3" ] - }, - "Tenancy" : { - "Description" : "The tenancy of your instance", - "Type" : "String", - "Default" : "default", - "AllowedValues" : [ "default", "dedicated"] - }, - "NumberOfInstances" : { - "Description" : "Number of instances in the cluster", - "Type" : "Number", - "Default" : "4", - "MinValue" : "1", - "MaxValue" : "15" - }, - "Cloudwatch" : { - "Description" : "Add basic Aerospike metrics to Cloundwatch. Will incur Cloudwatch expenses ~ $24/mo/instance", - "Type" : "String", - "Default" : "no", - "AllowedValues": [ "yes" , "no" ] - }, - "PermitSSH" : { - "Description" : "CIDR block that's permitted to SSH to the Aerospike Cluster", - "Type" : "String", - "AllowedPattern" : "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\\/([0-9]|[1-2][0-9]|3[0-2]))$", - "ConstraintDescription" : "Must be in CIDR notation. To specify one specifc IPv4 address, append /32. eg: 192.168.1.100/32" - }, - "InstanceType" : { - "Description" : "Type of EC2 instance to launch.", - "Type" : "String", - "Default" : "m3.large", - "AllowedValues" : [ "t2.micro", "t2.small", "t2.medium", "t2.large", - "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge", - "m4.large", "m4.xlarge", "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", - "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge", - "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge", - "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge", - "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge" - ] - }, - "EBS" : { - "Description" : "Size of EBS SSD volume in GB. The volume will attach under /dev/sdg. Limit of 16000. Enter 0 to not use EBS.", - "Type" : "Number", - "Default" : "50", - "MinValue" : "0", - "MaxValue" : "16000" - }, - "NamespaceFile" : { - "Description" : "(Optional) Location of your namespace definition. Must be publically downloadable. Will append file directly to end of aerospike.conf", - "Type" : "String" - } - }, - - "Mappings" : { - "VersionMap" : { - "3.6.0" : { "Virginia" : "ami-71b1371a" , - "Oregon" : "ami-3f4c5f0f" , - "California" : "ami-d139c295" , - "Ireland" : "ami-8dfcd8fa" , - "Franfurt" : "ami-d61d1ccb" , - "Singapore" : "ami-dc1b108e" , - "Sydney" : "ami-8b3579b1" , - "Tokyo" : "ami-56f17e56" , - "SPaulo" : "ami-b9f57fa4" }, - "3.6.1" : { "Virginia" : "ami-43295126" , - "Oregon" : "ami-bd8b938d" , - "California" : "ami-8daa6dc9" , - "Ireland" : "ami-5b38102c" , - "Franfurt" : "ami-74adae69" , - "Singapore" : "ami-980215ca" , - "Sydney" : "ami-0b4d0431" , - "Tokyo" : "ami-547be654" , - "SPaulo" : "ami-9934a084" }, - "3.6.2" : { "Virginia" : "ami-0d4f0568" , - "Oregon" : "ami-9e1cfaad" , - "California" : "ami-2b6aab6f" , - "Ireland" : "ami-b5bb8bc2" , - "Franfurt" : "ami-02f4f91f" , - "Singapore" : "ami-d4435386", - "Sydney" : "ami-e97932d3" , - "Tokyo" : "ami-caea8dca" , - "SPaulo" : "ami-6b198f76" }, - "3.6.3" : { "Virginia" : "ami-e1f9a684" , - "Oregon" : "ami-c60be9f5" , - "California" : "ami-1f1edd5b" , - "Ireland" : "ami-a75967d0" , - "Franfurt" : "ami-7c828e61" , - "Singapore" : "ami-32b7a460" , - "Sydney" : "ami-27f8b21d" , - "Tokyo" : "ami-4e6c004e" , - "SPaulo" : "ami-0b893167" } - }, - "RegionMap" : { - "us-east-1" : {"name": "Virginia"}, - "us-west-2" : {"name": "Oregon"}, - "us-west-1" : {"name": "California"}, - "eu-west-1" : {"name": "Ireland"}, - "eu-central-1" : {"name" : "Franfurt"}, - "ap-southeast-1" : {"name": "Singapore"}, - "ap-southeast-2" : {"name": "Sydney"}, - "ap-northeast-1" : {"name": "Tokyo"}, - "sa-east-1" : {"name": "SPaulo"} - } - }, - - "Conditions" : { - "NotUsingEBS" : { "Fn::Equals" : [ { "Ref" : "EBS" }, 0 ] } - }, - - "Resources" : { - "ClusterRole" : { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Version" : "2012-10-17", - "Statement": [ { - "Effect": "Allow", - "Principal": { - "Service": [ "ec2.amazonaws.com","autoscaling.amazonaws.com" ] - }, - "Action": [ "sts:AssumeRole" ] - } ] - }, - "Path": "/", - "Policies": [ { - "PolicyName": "AerospikeClusterPolicy", - "PolicyDocument": { - "Version" : "2012-10-17", - "Statement": [ { - "Effect": "Allow", - "Action": "ec2:DescribeInstances", - "Resource": "*" - } ] - } - },{ - "PolicyName": "AerospikeCloudWatchPolicy", - "PolicyDocument" :{ - "Version" : "2012-10-17", - "Statement": [ { - "Effect": "Allow", - "Action": "cloudwatch:PutMetricData", - "Resource": "*" - } ] - } - },{ - "PolicyName": "AerospikeSQSPolicy", - "PolicyDocument" :{ - "Version" : "2012-10-17", - "Statement": [ { - "Effect": "Allow", - "Action": "sqs:*", - "Resource": { "Fn::GetAtt" : ["MigrationSQS", "Arn"]} - } ] - } - },{ - "PolicyName": "AerospikeAutoScalingPolicy", - "PolicyDocument" :{ - "Version" : "2012-10-17", - "Statement" : [ { - "Effect": "Allow", - "Action": "autoscaling:*", - "Resource": "*" - } ] - } - }] - } - }, - - "ClusterInstanceProfile": { - "Type": "AWS::IAM::InstanceProfile", - "Properties": { - "Path": "/", - "Roles": [ { - "Ref": "ClusterRole" - } ] - } - }, - "MigrationSQS" : { - "Type": "AWS::SQS::Queue", - "Properties": { - "ReceiveMessageWaitTimeSeconds": 10 - } - }, - "MigrationHook": { - "Type": "AWS::AutoScaling::LifecycleHook", - "DependsOn" : "MigrationSQS", - "Properties": { - "AutoScalingGroupName": { "Ref": "ClusterGroup" }, - "LifecycleTransition": "autoscaling:EC2_INSTANCE_TERMINATING", - "NotificationTargetARN": { "Fn::GetAtt": [ "MigrationSQS", "Arn" ] }, - "RoleARN": { "Fn::GetAtt": [ "ClusterRole", "Arn" ] } - } - }, - - "ClusterGroup" : { - "Type" : "AWS::AutoScaling::AutoScalingGroup", - "DependsOn" : "GatewayToInternet", - "Properties" : { - "LaunchConfigurationName" : { "Ref" : "LaunchConfig" }, - "DesiredCapacity" : { "Ref" : "NumberOfInstances"}, - "MinSize" : "1", - "MaxSize" : "15", - "AvailabilityZones" : [ { "Fn::Select" : [ "0", { "Fn::GetAZs" : { "Ref" : "AWS::Region" } } ] } ], - "VPCZoneIdentifier" : [{ "Ref" : "PublicSubnet" }], - "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"}, "PropagateAtLaunch" : "true" } ] - } - }, - "LaunchConfig" : { - "Type" : "AWS::AutoScaling::LaunchConfiguration", - "DependsOn" : "GatewayToInternet", - "Metadata" : { - "AWS::CloudFormation::Init" : { - "config" : { - "files" : { - "/tmp/aerospike_cluster" : { - "content" : { "Fn::Join" : ["", [ - "#!/bin/bash\n", - "echo ClusterInstancesScriptStart > /var/log/awsuserdatascript\n", - " PUBLICIP=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)\n", - " CONF=/etc/aerospike/aerospike.conf\n", - " sed -i \"/port 3000/a \\\t\taccess-address $PUBLICIP virtual\" $CONF\n", - " ###Point to all instances using the mesh-address config option\n", - " PRIVATEIP=$(aws ec2 describe-instances --filter Name=tag-key,Values=StackID Name=tag-value,Values=", { "Ref" : "AWS::StackId" }," --output=text --region=",{ "Ref" : "AWS::Region" }," | grep PRIVATEIPADDRESSES | awk '{print $4}') \n", - " echo $PRIVATEIP >> /var/log/awsuserdatascript\n", - " sed -i '/.*mesh-seed-address-port/d' $CONF\n", - " for i in $PRIVATEIP; do ", - " sed -i \"/interval/i \\\t\tmesh-seed-address-port $i 3002\" $CONF\n", - " done \n", - " if [ ! -z \"",{ "Ref" :"NamespaceFile" },"\" ]; then\n", - " curl -s ",{ "Ref" :"NamespaceFile" }," >> $CONF; fi\n", - " /etc/init.d/aerospike start\n", - " /etc/init.d/amc start\n", - "echo OtherInstancesScriptFinish >> /var/log/awsuserdatascript\n", - "(crontab -l 2>/dev/null; echo '*/5 * * * * /opt/aerospike/poll_sqs') | crontab -\n", - " if [[ \"",{ "Ref" : "Cloudwatch" },"\" == \"yes\" ]]; then\n", - " (crontab -l 2>/dev/null; echo '*/5 * * * * /opt/aerospike/cloudwatch') | crontab -; fi\n" - ] ] }, - "mode" : "000744", - "owner" : "root", - "group" : "root" - }, - "/opt/aerospike/cloudwatch" : { - "content" : { "Fn::Join" : ["", [ - "#!/bin/bash\n", - "METRICS=$(asinfo -v stats -l)\n", - "NAMESPACE=aerospike\n", - "REGION=",{ "Ref" : "AWS::Region" },"\n", - "INSTANCE=$(curl 169.254.169.254/latest/meta-data/instance-id)\n", - "CLUSTER=",{ "Ref" : "AWS::StackId" },"\n", - "for L in $METRICS; do\n", - " LINE=(${L//=/ })\n", - " case ${LINE[0]} in\n", - " cluster_integrity)\n", - " if [[ ${LINE[1]} != \"true\" ]]; then\n", - " INTEGRITY_ERROR=1\n", - " else\n", - " INTEGRITY_ERROR=0\n", - " fi\n", - " ;;\n", - " total-bytes-memory)\n", - " TBM=${LINE[1]}\n", - " ;;\n", - " used-bytes-memory)\n", - " UBM=${LINE[1]}\n", - " ;;\n", - " total-bytes-disk)\n", - " TBD=${LINE[1]}\n", - " ;;\n", - " used-bytes-disk)\n", - " UBD=${LINE[1]}\n", - " ;;\n", - " objects)\n", - " OBJECTS=${LINE[1]}\n", - " ;;\n", - " *)\n", - " continue\n", - " ;;\n", - " esac\n", - "done\n", - "FM=$(expr $TBM - $UBM)\n", - "FD=$(expr $TBD - $UBD)\n", - "# Submit metrics\n", - "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $INTEGRITY_ERROR --metric-name 'Cluster Integrity'\n", - "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $FM --metric-name 'Free Memory' --unit 'Bytes'\n", - "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $FD --metric-name 'Free Disk' --unit 'Bytes'\n", - "aws cloudwatch --region $REGION put-metric-data --dimensions Cluster=$CLUSTER,Instance=$INSTANCE --namespace $NAMESPACE --value $OBJECTS --metric-name 'Number of Objects' --unit 'Count'\n" - ] ] }, - "mode" : "000744", - "owner" : "root", - "group" : "root" - }, - "/opt/aerospike/poll_sqs" : { - "content" : { "Fn::Join" : [ "" , [ - "#!/bin/bash\n", - "# This script will prevent autoscaling from terminating\n", - "# this instance until ASD migrations are completed\n", - "set -e\n", - "MYIP=$(curl 169.254.169.254/latest/meta-data/local-ipv4)\n", - "MYNODE=$(curl 169.254.169.254/latest/meta-data/instance-id)\n", - "REGION='",{ "Ref" : "AWS::Region"},"' # CFT\n", - "QUEUE='",{ "Ref" : "MigrationSQS"},"' # CFT\n", - "CLUSTER=$(aws ec2 describe-instances --filter Name=tag-key,Values=StackID Name=tag-value,Values=",{ "Ref":"AWS::StackId"}," --output=text --region=$REGION | grep PRIVATEIPADDRESSES | awk '{print $4}')\n", - "# Find SQS message with termination message\n", - "FOUND=false\n", - "MESSAGE=$(aws sqs receive-message --region $REGION --queue-url $QUEUE --region $REGION --wait-time-seconds 10 --visibility-timeout 2 )\n", - "BODY=$(echo $MESSAGE | jq '.Messages[0] .Body')\n", - "RECEIPT=$(echo $MESSAGE | jq --raw-output '.Messages[0] .ReceiptHandle')\n", - "LIFECYCLE=$(eval echo $BODY | jq --raw-output '.LifecycleTransition')\n", - "INSTANCE=$(eval echo $BODY | jq --raw-output '.EC2InstanceId')\n", - "if [[ $LIFECYCLE == 'autoscaling:EC2_INSTANCE_TERMINATING' ]] && [[ $INSTANCE == $MYNODE ]]; then\n", - " TOKEN=$(eval echo $BODY | jq --raw-output '.LifecycleActionToken')\n", - " HOOK=$(eval echo $BODY | jq --raw-output '.LifecycleHookName')\n", - " ASG=$(eval echo $BODY | jq --raw-output '.AutoScalingGroupName')\n", - " FOUND=true\n", - " aws sqs delete-message --region $REGION --queue-url $QUEUE --receipt-handle $RECEIPT\n", - "fi\n", - "# If not not found, exit\n", - "if [[ $FOUND == false ]]; then\n", - " exit 0\n", - "fi\n", - "# stop aerospike\n", - "/etc/init.d/aerospike stop\n", - "# Find first node that's not myself\n", - "for I in $CLUSTER; do\n", - " if [[ $I == $MYIP ]]; then\n", - " continue;\n", - " fi\n", - " NODE=$I\n", - " break\n", - "done\n", - "# Grab migration info\n", - "MIGRATIONS=$(asadm -h $NODE -e 'info service'| sed -n '4,/Number/ {/Number/!p}' | awk '{print $8}')\n", - "DONE=true\n", - "# check every node's migration status\n", - "for NODE in $MIGRATIONS; do\n", - " if [[ $NODE != '(0,0,0)' ]] && [[ $NODE != 'N/E' ]]; then\n", - " $DONE=false\n", - " break;\n", - " fi\n", - "done\n", - "# if migrations not done, pause ASG actions. Otherwise, continue autoscaling termination.\n", - "if [[ $DONE == false ]]; then\n", - " aws autoscaling record-lifecycle-action-heartbeat --region $REGION --lifecycle-action-token $TOKEN --auto-scaling-group-name $ASG --lifecycle-hook-name $HOOK\n", - " else\n", - " aws autoscaling complete-lifecycle-action --region $REGION --lifecycle-action-token $TOKEN --lifecycle-hook-name $HOOK --auto-scaling-group-name $ASG --lifecycle-action-result CONTINUE\n", - "fi\n" - ] ] }, - "mode" : "000744", - "owner" : "root", - "group" : "root" - } - }, - "commands" : { - "01_form_asd_cluster" : { - "command" : "/tmp/aerospike_cluster", - "cwd" : "/tmp" - } - } - } - } - }, - "Properties" : { - "InstanceType" : { "Ref" : "InstanceType"}, - "BlockDeviceMappings" : { "Fn::If" : [ "NotUsingEBS", - { "Ref" : "AWS::NoValue" }, - [ { - "DeviceName" : "/dev/sdg", - "Ebs" : { "VolumeSize" : {"Ref" : "EBS" }, - "VolumeType" : "gp2" } - } ] - ] }, - "KeyName" : { "Ref" : "KeyPair" }, - "IamInstanceProfile" : { "Ref" : "ClusterInstanceProfile" }, - "ImageId" : { "Fn::FindInMap" : [ "VersionMap", {"Ref" : "AerospikeVersion"} , { "Fn::FindInMap" : [ "RegionMap", { "Ref": "AWS::Region"}, "name"] } ] }, - "AssociatePublicIpAddress" : "true", - "SecurityGroups" : [ { "Ref" : "InstanceSecurityGroup" } ], - "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [ - "#!/bin/bash -xe\n", - "yum update -y aws-cfn-bootstrap\n", - "yum install -y jq\n", - "/opt/aws/bin/cfn-init -v ", - " --stack ", { "Ref" : "AWS::StackName" }, - " --resource LaunchConfig ", - " --region ", { "Ref" : "AWS::Region" }, "\n", - - "/opt/aws/bin/cfn-signal -e $? ", - " --stack ", { "Ref" : "AWS::StackName" }, - " --resource ClusterGroup ", - " --region ", { "Ref" : "AWS::Region" }, "\n" - ] ] } - } - } - }, - - "VPC" : { - "Type" : "AWS::EC2::VPC", - "Properties" : { - "CidrBlock" : "10.0.0.0/16", - "EnableDnsSupport" : "true", - "EnableDnsHostnames" : "true", - "InstanceTenancy": { "Ref" : "Tenancy" }, - "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] - } - }, - - "PublicSubnet" : { - "Type" : "AWS::EC2::Subnet", - "Properties" : { - "VpcId" : { "Ref" : "VPC" }, - "CidrBlock" : "10.0.0.0/24", - "AvailabilityZone" : { "Fn::Select" : [ "0", { "Fn::GetAZs" : { "Ref" : "AWS::Region" } } ] }, - "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] - } - }, - - "InternetGateway" : { - "Type" : "AWS::EC2::InternetGateway", - "Properties" : { - "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] - } - }, - - "GatewayToInternet" : { - "Type" : "AWS::EC2::VPCGatewayAttachment", - "Properties" : { - "VpcId" : { "Ref" : "VPC" }, - "InternetGatewayId" : { "Ref" : "InternetGateway" } - } - }, - - "PublicRouteTable" : { - "Type" : "AWS::EC2::RouteTable", - "Properties" : { - "VpcId" : { "Ref" : "VPC" }, - "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] - } - }, - - "PublicRoute" : { - "Type" : "AWS::EC2::Route", - "DependsOn" : "GatewayToInternet", - "Properties" : { - "RouteTableId" : { "Ref" : "PublicRouteTable" }, - "DestinationCidrBlock" : "0.0.0.0/0", - "GatewayId" : { "Ref" : "InternetGateway" } - } - }, - - "PublicSubnetRouteTableAssociation" : { - "Type" : "AWS::EC2::SubnetRouteTableAssociation", - "Properties" : { - "SubnetId" : { "Ref" : "PublicSubnet" }, - "RouteTableId" : { "Ref" : "PublicRouteTable" } - } - }, - - "InstanceSecurityGroup" : { - "Type" : "AWS::EC2::SecurityGroup", - "Properties" : { - "GroupDescription" : "Enable ports needed by Aerospike", - "VpcId" : { "Ref" : "VPC" }, - "SecurityGroupIngress" : [ { - "IpProtocol" : "tcp", - "FromPort" : "3000", - "ToPort" : "3000", - "CidrIp" : "0.0.0.0/0" - }, - { - "IpProtocol" : "tcp", - "FromPort" : "22", - "ToPort" : "22", - "CidrIp" : { "Ref" : "PermitSSH"} - - }, - { - "IpProtocol" : "tcp", - "FromPort" : "8081", - "ToPort" : "8081", - "CidrIp" : "0.0.0.0/0" - - }, - { - "IpProtocol" : "icmp", - "FromPort" : "-1", - "ToPort" : "-1", - "CidrIp" : "0.0.0.0/0" - } ], - "Tags" : [ {"Key" : "StackID", "Value" : { "Ref" : "AWS::StackId"} } ] - } - }, - "InstanceSecurityGroupIngress" : { - "Type" : "AWS::EC2::SecurityGroupIngress", - "Properties" : { - "GroupId" : { "Fn::GetAtt" : [ "InstanceSecurityGroup", "GroupId"] }, - "IpProtocol" : "tcp", - "FromPort" : "3001", - "ToPort" : "3004", - "SourceSecurityGroupId" : { "Fn::GetAtt" : ["InstanceSecurityGroup", "GroupId"] } - }, - "DependsOn" : "InstanceSecurityGroup" - } - } -} diff --git a/aerospike-topology2.json b/aerospike-existing-vpc.json similarity index 100% rename from aerospike-topology2.json rename to aerospike-existing-vpc.json diff --git a/aerospike-topology3.json b/aerospike-new-vpc.json similarity index 100% rename from aerospike-topology3.json rename to aerospike-new-vpc.json