New feature: Support Stack Tags as a separate file (#6)

* bump play + scala.

* stack loader to now also return an option of tags for a given stack.

* stack creator to support passing tags if provided to cloudformation.

* tests passing - process for having tags optionally flow with the stack from a separate file.

* bump AWS SDK to latest, trim down to only AWS libs we use. Bump testing libraries & fought a long fight with Mockito + Play's Spec2 forced integration that brings in a very old mockito.

README.md

@@ -456,7 +456,7 @@ To do this we've authored an overly simple app that makes the API call for your
 
 This software is licensed under the Apache 2 license, quoted below.
 
-Copyright (C) 2014-2016 LifeWay Christian Resources. (https://www.lifeway.com).
+Copyright (C) 2014-2018 LifeWay Christian Resources. (https://www.lifeway.com).
 
 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this project except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0.
 

app/actors/workflow/AmazonCredentials.scala

@@ -52,7 +52,7 @@ class AmazonCredentials extends Actor with ActorLogging {
       val provider: AWSCredentialsProvider = credentialsProvider match {
         case "DefaultAWSCredentialsProviderChain" => new DefaultAWSCredentialsProviderChain()
         case "TypesafeConfigAWSCredentialsProvider" => new TypesafeConfigAWSCredentialsProvider(config)
-        case "InstanceProfileCredentialsProvider" => new InstanceProfileCredentialsProvider()
+        case "InstanceProfileCredentialsProvider" => InstanceProfileCredentialsProvider.getInstance()
         case "ClasspathPropertiesFileCredentialsProvider" => new ClasspathPropertiesFileCredentialsProvider()
         case "EnvironmentVariableCredentialsProvider" => new EnvironmentVariableCredentialsProvider()
         case "SystemPropertiesCredentialsProvider" => new SystemPropertiesCredentialsProvider()

app/actors/workflow/WorkflowManager.scala

@@ -12,6 +12,7 @@ import actors.workflow.steps.RollBackStackSupervisor.{RollBackCompleted, RollBac
 import actors.workflow.steps.TearDownSupervisor.{TearDownCommand, TearDownFinished}
 import actors.workflow.steps.ValidateAndFreezeSupervisor._
 import actors.workflow.steps._
+import actors.workflow.tasks.Tag
 import akka.actor._
 import com.amazonaws.auth.AWSCredentialsProvider
 import com.typesafe.config.ConfigFactory
@@ -69,15 +70,15 @@ class WorkflowManager(logActor: ActorRef, actorFactory: ActorFactory) extends FS
   }
 
   when(AwaitingLoadStackResponse) {
-    case Event(LoadStackResponse(stackJson), DeployDataWithCreds(data, creds)) =>
+    case Event(LoadStackResponse(stackJson, tags), DeployDataWithCreds(data, creds)) =>
       context.unwatch(sender())
       logActor ! LogMessage("Stack JSON data loaded. Querying for existing stack")
 
       val stepData: Map[String, String] = Map("stackFileContents" -> stackJson.toString())
       val validateAndFreezeSupervisor = actorFactory(ValidateAndFreezeSupervisor, context, "validateAndFreezeSupervisor", creds, actorFactory)
       context.watch(validateAndFreezeSupervisor)
       validateAndFreezeSupervisor ! ValidateAndFreezeStackCommand(data.stackPath, data.version)
-      goto(AwaitingStackVerifier) using DeployDataWithCredsWithSteps(data, creds, stepData)
+      goto(AwaitingStackVerifier) using DeployDataWithCredsWithSteps(data, creds, stepData, tags)
   }
 
   when(AwaitingDeleteStackResponse) {
@@ -91,48 +92,48 @@ class WorkflowManager(logActor: ActorRef, actorFactory: ActorFactory) extends FS
   }
 
   when(AwaitingStackVerifier) {
-    case Event(NoExistingStacksExist, DeployDataWithCredsWithSteps(data, creds, stepData)) =>
+    case Event(NoExistingStacksExist, DeployDataWithCredsWithSteps(data, creds, stepData, tags)) =>
       context.unwatch(sender())
       logActor ! LogMessage("No existing stacks found. First-time stack launch")
 
       val stackLauncher = actorFactory(NewStackSupervisor, context, "newStackSupervisor", creds, actorFactory)
       context.watch(stackLauncher)
       stepData.get("stackFileContents") match {
         case Some(stackFile) =>
-          stackLauncher ! NewStackFirstLaunchCommand(data.stackName, data.amiId, data.version, Json.parse(stackFile))
+          stackLauncher ! NewStackFirstLaunchCommand(data.stackName, data.amiId, data.version, Json.parse(stackFile), tags)
         case None => throw new Exception("No stack contents found when attempting to deploy")
       }
       goto(AwaitingStackLaunched)
 
-    case Event(VerifiedAndStackFrozen(oldStackName, oldASGName), DeployDataWithCredsWithSteps(data, creds, stepData)) =>
+    case Event(VerifiedAndStackFrozen(oldStackName, oldASGName), DeployDataWithCredsWithSteps(data, creds, stepData, tags)) =>
       context.unwatch(sender())
 
       val newStepData = stepData + ("oldStackName" -> oldStackName) + ("oldAsgName" -> oldASGName)
       val stackLauncher = actorFactory(NewStackSupervisor, context, "newStackSupervisor", creds, actorFactory)
       context.watch(stackLauncher)
       stepData.get("stackFileContents") match {
         case Some(stackFile) =>
-          stackLauncher ! NewStackUpgradeLaunchCommand(data.stackName, data.amiId, data.version, Json.parse(stackFile), oldStackName, oldASGName)
+          stackLauncher ! NewStackUpgradeLaunchCommand(data.stackName, data.amiId, data.version, Json.parse(stackFile), tags, oldStackName, oldASGName)
         case None => throw new Exception("No stack contents found when attempting to deploy")
       }
       goto(AwaitingStackLaunched) using DeployDataWithCredsWithSteps(data, creds, newStepData)
 
-    case Event(StackVersionAlreadyExists, DeployDataWithCredsWithSteps(data, creds, stepData)) =>
+    case Event(StackVersionAlreadyExists, DeployDataWithCredsWithSteps(data, creds, stepData, _)) =>
       context.unwatch(sender())
       logActor ! LogMessage("Workflow is being stopped - you are trying to redeploy an existing stack version")
       failed()
       stop()
   }
 
   when(AwaitingStackLaunched) {
-    case Event(FirstStackLaunchCompleted(newStackName), DeployDataWithCredsWithSteps(_, _, _)) =>
+    case Event(FirstStackLaunchCompleted(newStackName), DeployDataWithCredsWithSteps(_, _, _, _)) =>
       context.unwatch(sender())
       logActor ! LogMessage(s"The first version of this stack has been successfully deployed. Stack Name: $newStackName")
       logActor ! WorkflowLog.WorkflowCompleted
       context.parent ! WorkflowCompleted
       stop()
 
-    case Event(StackUpgradeLaunchCompleted(newAsgName), DeployDataWithCredsWithSteps(data, creds, stepData)) =>
+    case Event(StackUpgradeLaunchCompleted(newAsgName), DeployDataWithCredsWithSteps(data, creds, stepData, _)) =>
       context.unwatch(sender())
       logActor ! LogMessage("The next version of the stack has been successfully deployed.")
       val tearDownSupervisor = actorFactory(TearDownSupervisor, context, "tearDownSupervisor", creds, actorFactory)
@@ -158,7 +159,7 @@ class WorkflowManager(logActor: ActorRef, actorFactory: ActorFactory) extends FS
   }
 
   when(AwaitingOldStackTearDown) {
-    case Event(TearDownFinished, DeployDataWithCredsWithSteps(_, _, _)) =>
+    case Event(TearDownFinished, DeployDataWithCredsWithSteps(_, _, _, _)) =>
       context.unwatch(sender())
       logActor ! LogMessage("The old stack has been deleted and the new stack's ASG has been unfrozen.")
       logActor ! WorkflowLog.WorkflowCompleted
@@ -267,7 +268,7 @@ object WorkflowManager extends PropFactory {
   case class DeployData(deploy: Deploy) extends WorkflowData
   case class DeployDataWithCreds(deploy: Deploy, creds: AWSCredentialsProvider) extends WorkflowData
   case class DeployDataWithCredsWithSteps(deploy: Deploy, creds: AWSCredentialsProvider,
-                                          stepData: Map[String, String] = Map.empty[String, String]) extends WorkflowData
+                                          stepData: Map[String, String] = Map.empty[String, String], tags: Option[Seq[Tag]] = None) extends WorkflowData
   case class DeleteData(stackName: String) extends WorkflowData
 
   def props(args: Any*): Props = Props(classOf[WorkflowManager], args: _*)

app/actors/workflow/steps/LoadStackSupervisor.scala

@@ -2,13 +2,13 @@ package actors.workflow.steps
 
 import actors.WorkflowLog.{Log, LogMessage}
 import actors.workflow.steps.LoadStackSupervisor.{LoadStackData, LoadStackStates}
-import actors.workflow.tasks.StackLoader
+import actors.workflow.tasks.{StackLoader, Tag}
 import actors.workflow.tasks.StackLoader.{LoadStack, StackLoaded}
 import actors.workflow.{AWSSupervisorStrategy, WorkflowManager}
 import akka.actor._
 import com.amazonaws.auth.AWSCredentialsProvider
 import play.api.libs.json.JsValue
-import utils.{PropFactory, ActorFactory}
+import utils.{ActorFactory, PropFactory}
 
 class LoadStackSupervisor(credentials: AWSCredentialsProvider,
                           actorFactory: ActorFactory) extends FSM[LoadStackStates, LoadStackData] with ActorLogging
@@ -27,10 +27,10 @@ class LoadStackSupervisor(credentials: AWSCredentialsProvider,
   }
 
   when(AwaitingStackData) {
-    case Event(StackLoaded(data), _) =>
+    case Event(StackLoaded(data, tags), _) =>
       context.unwatch(sender())
       context.stop(sender())
-      context.parent ! LoadStackResponse(data)
+      context.parent ! LoadStackResponse(data, tags)
       stop()
   }
 
@@ -61,7 +61,7 @@ object LoadStackSupervisor extends PropFactory {
   //Interaction Messages
   sealed trait LoadStackMessage
   case class LoadStackCommand(bucketName: String, stackPath: String)
-  case class LoadStackResponse(stackData: JsValue)
+  case class LoadStackResponse(stackData: JsValue, tags: Option[Seq[Tag]])
 
   //FSM: States
   sealed trait LoadStackStates

app/actors/workflow/steps/NewStackSupervisor.scala

@@ -28,13 +28,13 @@ class NewStackSupervisor(credentials: AWSCredentialsProvider,
     case Event(msg: NewStackFirstLaunchCommand, Uninitialized) =>
       val stackCreator = actorFactory(StackCreator, context, "stackLauncher", credentials)
       context.watch(stackCreator)
-      stackCreator ! StackCreateCommand(msg.newStackName, msg.imageId, msg.version, msg.stackContent)
+      stackCreator ! StackCreateCommand(msg.newStackName, msg.imageId, msg.version, msg.stackContent, msg.tags)
       goto(AwaitingStackCreatedResponse) using FirstTimeStack(msg.newStackName)
 
     case Event(msg: NewStackUpgradeLaunchCommand, Uninitialized) =>
       val stackCreator = actorFactory(StackCreator, context, "stackLauncher", credentials)
       context.watch(stackCreator)
-      stackCreator ! StackCreateCommand(msg.newStackName, msg.imageId, msg.version, msg.stackContent)
+      stackCreator ! StackCreateCommand(msg.newStackName, msg.imageId, msg.version, msg.stackContent, msg.tags)
       goto(AwaitingStackCreatedResponse) using UpgradeOldStackData(msg.oldStackASG, msg.oldStackName, msg.newStackName)
   }
 
@@ -176,8 +176,8 @@ object NewStackSupervisor extends PropFactory {
   //Interaction Messages
   sealed trait NewStackMessage
   case class NewStackFirstLaunchCommand(newStackName: String, imageId: String, version: Version,
-                                        stackContent: JsValue) extends NewStackMessage
-  case class NewStackUpgradeLaunchCommand(newStackName: String, imageId: String, version: Version, stackContent: JsValue,
+                                        stackContent: JsValue, tags: Option[Seq[Tag]]) extends NewStackMessage
+  case class NewStackUpgradeLaunchCommand(newStackName: String, imageId: String, version: Version, stackContent: JsValue, tags: Option[Seq[Tag]],
                                           oldStackName: String, oldStackASG: String) extends NewStackMessage
   case class FirstStackLaunchCompleted(newStackName: String) extends NewStackMessage
   case class StackUpgradeLaunchCompleted(newAsgName: String) extends NewStackMessage

app/actors/workflow/tasks/StackCreator.scala

@@ -4,7 +4,8 @@ import actors.DeploymentSupervisor.{StackAndAppVersion, AppVersion, Version}
 import actors.workflow.AWSRestartableActor
 import akka.actor.Props
 import com.amazonaws.auth.AWSCredentialsProvider
-import com.amazonaws.services.cloudformation.model.{Capability, CreateStackRequest, Parameter, Tag}
+import com.amazonaws.services.cloudformation.model.{Capability, CreateStackRequest, Parameter}
+import com.amazonaws.services.cloudformation.model.{Tag => AWSTag}
 import play.api.libs.json.JsValue
 import utils.{AmazonCloudFormationService, PropFactory}
 
@@ -14,14 +15,16 @@ class StackCreator(credentials: AWSCredentialsProvider) extends AWSRestartableAc
 
   override def receive: Receive = {
     case launchCommand: StackCreateCommand =>
-      val appVersionTag = new Tag()
+      val appVersionTag = new AWSTag()
                           .withKey("ApplicationVersion")
                           .withValue(launchCommand.version.appVersion)
 
-      val tags = launchCommand.version match {
-        case x:AppVersion => Seq(appVersionTag)
-        case x:StackAndAppVersion => Seq(appVersionTag, new Tag().withKey("StackVersion").withValue(x.stackVersion))
-      }
+      val additionalTags: Seq[AWSTag] = launchCommand.tags.getOrElse(Seq.empty[Tag]).map(t => new AWSTag().withKey(t.key).withValue(t.value))
+
+      val tags = (launchCommand.version match {
+        case _:AppVersion => Seq(appVersionTag)
+        case x:StackAndAppVersion => Seq(appVersionTag, new AWSTag().withKey("StackVersion").withValue(x.stackVersion))
+      }) ++ additionalTags
 
       val params = Seq(
         new Parameter()
@@ -47,7 +50,7 @@ class StackCreator(credentials: AWSCredentialsProvider) extends AWSRestartableAc
 }
 
 object StackCreator extends PropFactory {
-  case class StackCreateCommand(stackName: String, imageId: String, version: Version, stackData: JsValue)
+  case class StackCreateCommand(stackName: String, imageId: String, version: Version, stackData: JsValue, tags: Option[Seq[Tag]])
   case object StackCreateRequestCompleted
 
   override def props(args: Any*): Props = Props(classOf[StackCreator], args: _*)

app/actors/workflow/tasks/StackLoader.scala

@@ -3,13 +3,26 @@ package actors.workflow.tasks
 import actors.workflow.AWSRestartableActor
 import akka.actor.Props
 import com.amazonaws.auth.AWSCredentialsProvider
-import com.amazonaws.services.s3.AmazonS3Client
-import com.amazonaws.services.s3.model.S3Object
+import com.amazonaws.services.s3.AmazonS3
+import com.amazonaws.services.s3.model.{AmazonS3Exception, S3Object}
 import org.apache.commons.io.IOUtils
-import play.api.Logger
-import play.api.libs.json.{JsValue, Json}
+import play.api.libs.json._
+import play.api.libs.functional.syntax._
 import utils.{AmazonS3Service, PropFactory}
 
+case class Tag(key: String, value: String)
+
+object Tag {
+  implicit val reads: Reads[Tag] = (
+    (JsPath \ "Key").read[String] and
+      (JsPath \ "Value").read[String]
+  )(Tag.apply _)
+  implicit val writes: Writes[Tag] = (
+    (JsPath \ "Key").write[String] and
+      (JsPath \ "Value").write[String]
+  )(unlift(Tag.unapply))
+}
+
 class StackLoader(credentials: AWSCredentialsProvider, bucketName: String) extends AWSRestartableActor with AmazonS3Service {
 
   import actors.workflow.tasks.StackLoader._
@@ -18,15 +31,26 @@ class StackLoader(credentials: AWSCredentialsProvider, bucketName: String) exten
     case msg: LoadStack =>
       val client = s3Client(credentials)
       val stackObject: S3Object = client.getObject(bucketName, s"chadash-stacks/${msg.stackPath}.json")
+      val tags = getTags(bucketName, s"chadash-stacks/${msg.stackPath}.tags.json", client)
       val stackFileJson = Json.parse(IOUtils.toByteArray(stackObject.getObjectContent))
 
-      context.parent ! StackLoaded(stackFileJson)
+      context.parent ! StackLoaded(stackFileJson, tags)
+  }
+
+  def getTags(bucketName: String, path: String, client: AmazonS3): Option[Seq[Tag]] = {
+    try {
+      val tagObject: S3Object = client.getObject(bucketName, path)
+      Json.parse(IOUtils.toByteArray(tagObject.getObjectContent)).asOpt[Seq[Tag]]
+    } catch {
+      case _:AmazonS3Exception => None
+      case e:Throwable => throw e
+    }
   }
 }
 
 object StackLoader extends PropFactory {
   case class LoadStack(stackPath: String)
-  case class StackLoaded(stackJson: JsValue)
+  case class StackLoaded(stackJson: JsValue, tags: Option[Seq[Tag]])
 
   override def props(args: Any*): Props = Props(classOf[StackLoader], args: _*)
 }

app/utils/Components.scala

@@ -1,23 +1,23 @@
 package utils
 
 import com.amazonaws.auth.AWSCredentialsProvider
-import com.amazonaws.services.autoscaling.{AmazonAutoScaling, AmazonAutoScalingClient}
-import com.amazonaws.services.cloudformation.{AmazonCloudFormation, AmazonCloudFormationClient}
-import com.amazonaws.services.elasticloadbalancing.{AmazonElasticLoadBalancing, AmazonElasticLoadBalancingClient}
-import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}
+import com.amazonaws.services.autoscaling.{AmazonAutoScaling, AmazonAutoScalingClientBuilder}
+import com.amazonaws.services.cloudformation.{AmazonCloudFormation, AmazonCloudFormationClientBuilder}
+import com.amazonaws.services.elasticloadbalancing.{AmazonElasticLoadBalancing, AmazonElasticLoadBalancingClientBuilder}
+import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder}
 
 trait AmazonAutoScalingService {
-  def autoScalingClient(credentials: AWSCredentialsProvider): AmazonAutoScaling = new AmazonAutoScalingClient(credentials)
+  def autoScalingClient(credentials: AWSCredentialsProvider): AmazonAutoScaling = AmazonAutoScalingClientBuilder.standard().withCredentials(credentials).build()
 }
 
 trait AmazonCloudFormationService {
-  def cloudFormationClient(credentials: AWSCredentialsProvider): AmazonCloudFormation = new AmazonCloudFormationClient(credentials)
+  def cloudFormationClient(credentials: AWSCredentialsProvider): AmazonCloudFormation = AmazonCloudFormationClientBuilder.standard().withCredentials(credentials).build()
 }
 
 trait AmazonElasticLoadBalancingService {
-  def elasticLoadBalancingClient(credentials: AWSCredentialsProvider): AmazonElasticLoadBalancing = new AmazonElasticLoadBalancingClient(credentials)
+  def elasticLoadBalancingClient(credentials: AWSCredentialsProvider): AmazonElasticLoadBalancing = AmazonElasticLoadBalancingClientBuilder.standard().withCredentials(credentials).build()
 }
 
 trait AmazonS3Service {
-  def s3Client(credentials: AWSCredentialsProvider): AmazonS3 = new AmazonS3Client(credentials)
+  def s3Client(credentials: AWSCredentialsProvider): AmazonS3 = AmazonS3ClientBuilder.standard().withCredentials(credentials).build()
 }

build.sbt

@@ -1,27 +1,36 @@
 name := "chadash"
 version := scala.util.Properties.envOrElse("BUILD_VERSION", "DEV")
-scalaVersion := "2.11.6"
+scalaVersion := "2.11.12"
 scalacOptions ++= Seq("-feature", "-target:jvm-1.8")
 
 lazy val root = (project in file(".")).enablePlugins(PlayScala).settings(
   javaOptions in Test += "-Dconfig.file=conf/application.test.conf"
   //FOR DEBUGGING TESTS:
   //,javaOptions in Test += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=9999"
-)
+).settings(libraryDependencies ~= (_.map(excludeSpecs2)))
+val AwsVersion   = "1.11.298"
 
 libraryDependencies ++= Seq(
   ws,
   filters,
-  "com.amazonaws" % "aws-java-sdk" % "1.9.19",
+  "com.amazonaws" % "aws-java-sdk-cloudformation" % AwsVersion,
+  "com.amazonaws" % "aws-java-sdk-s3" % AwsVersion,
+  "com.amazonaws" % "aws-java-sdk-autoscaling" % AwsVersion,
+  "com.amazonaws" % "aws-java-sdk-elasticloadbalancing" % AwsVersion,
   "com.google.code.findbugs" % "jsr305" % "3.0.0",
   "commons-io" % "commons-io" % "2.4",
   "com.typesafe.akka" %% "akka-actor" % "2.3.9",
   "com.typesafe.akka" %% "akka-slf4j" % "2.3.9",
   "com.typesafe.akka" %% "akka-testkit" % "2.3.9" % "test",
-  "org.scalatestplus" %% "play" % "1.1.0" % "test",
+  "org.scalatestplus" %% "play" % "1.2.0" % "test",
+  "org.mockito" % "mockito-core" % "2.16.0" % "test",
   "com.google.inject" % "guice" % "3.0"
 )
 
+def excludeSpecs2(module: ModuleID): ModuleID =
+  module.excludeAll(ExclusionRule(organization =
+    "org.specs2")).exclude("com.novocode", "junit-interface")
+
 //----
 // Create the buildInfo object at compile time
 //----

project/build.properties

@@ -1,4 +1 @@
-#Activator-generated Properties
-#Sat Sep 13 08:19:33 CDT 2014
-template.uuid=9d0f021d-ca8f-4a88-992f-f6468442419e
-sbt.version=0.13.8
+sbt.version=0.13.17

project/plugins.sbt

@@ -1,7 +1,7 @@
 resolvers += "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/"
 
 // The Play plugin
-addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.3.8")
+addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.3.10")
 
 addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.3.2")