build.gradle

buildscript {
  repositories {
    mavenCentral()
  }
  dependencies {
    classpath "com.marklogic:ml-gradle:${mlGradleVersion}"

    classpath "org.json:json:${orgJsonVersion}"
  }
}

plugins {
  id 'net.saliman.properties' version "${netSalimonPropertiesVersion}"
  id 'nu.studer.credentials' version "${nuStuderCredentialsVersion}"
}

apply plugin: "com.marklogic.ml-gradle"

repositories {
  mavenCentral()
}

configurations {
  mlcp
}

dependencies {
  mlcp "com.marklogic:mlcp:${mlcpVersion}"
  mlcp "org.apache.commons:commons-csv:${orgApacheCommonsCsvVersion}"
  mlcp files("lib")
}
tasks.withType(JavaExec) {
    jvmArgs = ['-Xmx1g']
}

def codeVersion = project.getProperty('defaultCodeVersion') // Overridden in determineCodeVersion.

// This task is intended to execute before any that interact with MarkLogic Server.  It switches
// from the default default REST port (8003) to the app services port.  This port is used by the
// majority of the tasks, and has been proven necessary for setBanner (tenant app server may not
// exist).
// An exception is mlLoadModules: we need to switch to the tenant app server to ensure the REST
// options are added to the tenant's modules database.
task startWithAppServicesPort() {
  println ''
  println "Setting mlAppConfig's port to " + project.property("mlAppServicesPort") + " (from " + mlAppConfig.getRestPort() + ")"
  println ''
  mlAppConfig.setRestPort(project.property("mlAppServicesPort").toInteger())
}

// Change ML app config's rest port to that of the app services port.  See startWithAppServicesPort's
// comment for details.
task switchToAppServicesPort() {
  doFirst {
    println ''
    println "Changing mlAppConfig's port from " + mlAppConfig.getRestPort() + " to " + project.property("mlAppServicesPort")
    println ''
    mlAppConfig.setRestPort(project.property("mlAppServicesPort").toInteger())
  }
}

// Change ML app config's rest port to that of the tenant's app server port (one of them).  See
// startWithAppServicesPort's comment for details.
task switchToTenantPort() {
  doFirst {
    println ''
    println "Changing mlAppConfig's port from " + mlAppConfig.getRestPort() + " to " + project.property("mlRestPortGroup1")
    println ''
    mlAppConfig.setRestPort(project.property("mlRestPortGroup1").toInteger())
  }
}
mlLoadModules.dependsOn switchToTenantPort

task determineCodeVersion {
  def applyDefault = false
	def stdOut = new ByteArrayOutputStream()
  try {
    exec {
      commandLine 'sh', './version.sh'
      standardOutput = stdOut
    }
  } catch (Exception e) {
    println ''
    println 'WARNING: Unable to determine the code version via git; using default.'
    println ''
    applyDefault = true
  }

  stdOut = stdOut.toString().trim()
  if (applyDefault || stdOut == 'default') {
    println "Applying default code version of ${codeVersion}"
  } else {
    codeVersion = stdOut
    println "Applying derived code version of ${codeVersion}"
  }
}

// This project's implementation of its custom token feature.
//
// This task needs to run before any ML Gradle task that references ML Gradle configuration files does.  Rather than
// list all the related ML Gradle tasks, we're defining it as the first custom task and allow it to always run.
//
// More info in /docs/lux-backend-build-tool-and-tasks.md
task preprocessRuntimeConfigAndModules() {
  dependsOn 'determineCodeVersion'

  def names = []
  def texts = []
  def text

  // For each name, pull in the content of its file and remove the outer brackets.  Departed from JSON when we picked
  // up some double-quoted values that contained escaped double quotes --the escape character was being dropped.  
  // We now need to double-escape such double quotes *and* read it in as text.  We have also started to use single
  // quotes in the configuration files, but keeping this as a backup.
  //
  // Note: .push() adds the new item to the beginning of the array thus multiple .get(0) calls.
  names.push('contentDatabaseConfGenerated')
  text = file('./config/' + names.get(0) + '.json').text.trim()
  texts.push(text.substring(1, text.length() - 1))

  names.push('codeVersion')
  texts.push(codeVersion)

  println ''
  println 'Resolving custom token(s): ' + names.join(", ")
  println ''

  // Clear target directory first, as there could be obsolete files therein.
  delete './build/main'

  // Copy src/main/ml-config into build directory
  copy {
    from './src/main'
    into './build/main'
    include 'ml-config/**'
    include 'ml-modules/**'
    filter { line -> {
      for (def i = 0; i < names.size(); i++) {
        line = line.replaceAll('@@' + names[i] + '@@', texts[i])
      }
      return line
    }}
  }
}

// Code herein automatically executes.
task preprocessBuildSupportScripts() {
  // Resolve property references while copying into the build dir.
  copy {
    from './scripts/buildSupport'
    into './build/buildSupport'
    include '**'
    filter { line -> {
      // What's an existing/standard/better way to do this?
      for (Map.Entry<String, ?> entry : project.getProperties().entrySet()) {
        line = line.replaceAll('%%' +entry.getKey() + '%%', entry.getValue().toString())
      }
      return line
    }}
  }
}

task copyContentDatabaseConfGenerated(){
  def text = file('./config/contentDatabaseConfGenerated.json').text
  copy{
    from './src/main/templates'
    into './src/main/ml-modules/root/config'
    include 'contentDatabaseConfGenerated.mjs'
    filter{ line -> {
      return line.replaceAll("@@contentDatabaseConfGenerated@@", text)
    }}
  }

}
mlLoadModules.dependsOn copyContentDatabaseConfGenerated

// Developer convenience to copy a database, including from DEV to one's local environment.
// More info in /docs/lux-backend-build-tool-and-tasks.md.
//
// Should this task automatically clear the database first, like importDataFull does?
task copyDatabase(type: com.marklogic.gradle.task.MlcpTask) {
  //check if SSL is enabled and use the properties value
  def useSsl = true
  if (project.hasProperty("mlAppServicesSimpleSsl")) {
    useSsl = project.getProperty("mlAppServicesSimpleSsl").toBoolean();
  }

  classpath = configurations.mlcp
  command = "COPY"

  input_host = project.property("copyDatabaseInputHost")
  input_port = project.property("copyDatabaseInputPort").toInteger()
  input_username = project.property("copyDatabaseInputUsername")
  input_password = credentials.forKey('copyDatabaseInputPassword')
  input_database = project.property("copyDatabaseInputDatabase")

  output_host = project.property("copyDatabaseOutputHost")
  output_port = project.property("copyDatabaseOutputPort").toInteger()
  output_username = project.property("copyDatabaseOutputUsername")
  output_password = credentials.forKey('copyDatabaseOutputPassword')
  output_database = project.property("copyDatabaseOutputDatabase")

  copy_permissions=true
  copy_collections=true
  copy_metadata=true
  copy_properties=true
  copy_quality=true

  fastload=true
  batch_size = project.getProperties().getOrDefault("copyDatabaseBatchSize", "100").toInteger()
  thread_count = project.getProperties().getOrDefault("copyDatabaseThreadCount", "4").toInteger()

  ssl = useSsl
  args = ["-ssl_protocol", "tlsv1.2"]
}

// Import data running in full mode.
//    Database *is* cleared first.
//    MLCP *does* run in fast load mode --therefore it is *not* safe to update documents.
// More info in /docs/lux-backend-build-tool-and-tasks.md
//
// At least on Windows, this task can start to fail once the Gradle cache has enough entries
// to make the generated command too long.  Perhaps we can provide a shorter classpath.
task importDataFull(type: com.marklogic.gradle.task.MlcpTask) {
  //check if SSL is enabled and use the properties value
  def useSsl = true
  if (project.hasProperty("mlAppServicesSimpleSsl")) {
    useSsl = project.getProperty("mlAppServicesSimpleSsl").toBoolean();
  }
  
  classpath = configurations.mlcp
  command = "IMPORT"
  host = project.property("importDataHost")
  port = project.property("importDataPort").toInteger()
  username = project.property("importDataUsername")
  password = credentials.forKey('importDataPassword')
  database = project.property("tenantContentDatabase")
  modules = project.property("tenantModulesDatabase")
  fastload = true // Database cleared for full loads
  input_file_path = project.property("importDataFilePath")
  input_file_type = "delimited_json"
  input_compressed = project.property("importDataFileIsCompressed")
  input_compression_codec = "gzip"
  transform_module "/documentTransforms.sjs"
  transform_function "associateDocToDataSlice"
  uri_id = "id"
  ssl = useSsl
  args = ["-ssl_protocol", "tlsv1.2"]

  dependsOn {
    mlClearDatabase
  }
}

// Import data running in incremental mode:
//    Database is *not* cleared first.
//    MLCP does *not* run in fast load mode --therefore it is safe to update documents.
// More info in /docs/lux-backend-build-tool-and-tasks.md
//
// At least on Windows, this task can start to fail once the Gradle cache has enough entries
// to make the generated command too long.  Perhaps we can provide a shorter classpath.
task importDataIncremental(type: com.marklogic.gradle.task.MlcpTask) {
  //check if SSL is enabled and use the properties value
  def useSsl = true
  if (project.hasProperty("mlAppServicesSimpleSsl")) {
    useSsl = project.getProperty("mlAppServicesSimpleSsl").toBoolean();
  }

  classpath = configurations.mlcp
  command = "IMPORT"

  host = project.property("importDataHost")
  port = project.property("importDataPort").toInteger()
  username = project.property("importDataUsername")
  password = credentials.forKey('importDataPassword')
  database = project.property("tenantContentDatabase")
  modules = project.property("tenantModulesDatabase")
  fastload = false // Database not cleared for incremental loads
  input_file_path = project.property("importDataFilePath")
  input_file_type = "delimited_json"
  input_compressed = project.property("importDataFileIsCompressed")
  input_compression_codec = "gzip"
  transform_module "/documentTransforms.sjs"
  transform_function "associateDocToDataSlice"
  uri_id = "id"
  ssl = useSsl
  args = ["-ssl_protocol", "tlsv1.2"]
}

// Runs the remaining search terms generator
// More info in /docs/lux-backend-build-tool-and-tasks.md
task generateRemainingSearchTerms (type: com.marklogic.gradle.task.MarkLogicTask) {
  doFirst {
    println ""
    println "Generating the remaining search terms..."
    println ""

    def client = mlAppConfig.newDatabaseClient()
    def script = new File('./build/buildSupport/generateRemainingSearchTerms.sjs').getText('UTF-8')
    def result = client.newServerEval().javascript(script).evalAs(String.class)

    println ""
    println result
    println ""
  }
}
mlDeploy.finalizedBy generateRemainingSearchTerms
mlLoadModules.finalizedBy generateRemainingSearchTerms
copyDatabase.finalizedBy generateRemainingSearchTerms

// Runs the related lists configuration generator
// More info in /docs/lux-backend-build-tool-and-tasks.md
task generateRelatedListsConfig (type: com.marklogic.gradle.task.MarkLogicTask) {
  doFirst {
    println ""
    println "Generating the related lists configuration..."
    println ""

    def client = mlAppConfig.newDatabaseClient()
    def script = new File('./build/buildSupport/generateRelatedListsConfig.sjs').getText('UTF-8')
    def result = client.newServerEval().javascript(script).evalAs(String.class)

    println ""
    println result
    println ""
  }
}
generateRemainingSearchTerms.finalizedBy generateRelatedListsConfig

// Runs the advanced search configuration generator
// More info in /docs/lux-backend-build-tool-and-tasks.md
task generateAdvancedSearchConfig (type: com.marklogic.gradle.task.MarkLogicTask) {
  doFirst {
    println ""
    println "Generating the advanced search configuration..."
    println ""

    def client = mlAppConfig.newDatabaseClient()
    def script = new File('./build/buildSupport/generateAdvancedSearchConfig.sjs').getText('UTF-8')
    def result = client.newServerEval().javascript(script).evalAs(String.class)

    println ""
    println result
    println ""
  }
}
generateRemainingSearchTerms.finalizedBy generateAdvancedSearchConfig

task importRestApiOptions(type: com.marklogic.gradle.task.MlcpTask) {
  //check if SSL is enabled and use the properties value
  def useSsl = true
  if (project.hasProperty("mlAppServicesSimpleSsl")) {
    useSsl = project.getProperty("mlAppServicesSimpleSsl").toBoolean();
  }
  
  classpath = configurations.mlcp
  command = "IMPORT"
  host = project.property("mlHost")
  port = project.property("mlRestPortGroup2").toInteger()
  username = project.property("mlUsername")
  password = credentials.forKey('mlPassword')
  database = project.property("tenantModulesDatabase")
  modules = project.property("tenantModulesDatabase")
  fastload = false // need ability to overwrite
  input_file_path = "./src/main/ml-modules/options/lux-options.xml"
  input_file_type = "documents"
  output_uri_replace = ".*/options/lux-options.xml,'/Default/" + project.getProperty("mlAppName") + "-request-group-2/rest-api/options/lux-options.xml'"
  output_permissions "rest-reader-internal,read,rest-admin-internal,update," + project.getProperty("mlAppName") + "-deployer,update"
  ssl = useSsl
  args = ["-ssl_protocol", "tlsv1.2"]
}
generateAdvancedSearchConfig.finalizedBy importRestApiOptions

// Admin required to run this; thus piggybacking mlDeploySecurity.
// More info in /docs/lux-backend-build-tool-and-tasks.md
task setBanner(type: com.marklogic.gradle.task.ServerEvalTask) {
  javascript = new File('./build/buildSupport/setBanner.sjs').getText('UTF-8')
}
mlDeploySecurity.finalizedBy setBanner

task showAppServerCiphers(type: com.marklogic.gradle.task.ServerEvalTask) {
  xquery = new File('./build/buildSupport/showAppServerCiphers.xqy').getText('UTF-8')
}

task showDeprecatedSSLProtocols(type: com.marklogic.gradle.task.ServerEvalTask) {
  xquery = new File('./build/buildSupport/showDeprecatedSSLProtocols.xqy').getText('UTF-8')
}

task disableDeprecatedSSLProtocols(type: com.marklogic.gradle.task.ServerEvalTask) {
  xquery = new File('./build/buildSupport/disableDeprecatedSSLProtocols.xqy').getText('UTF-8')
}
mlDeployServers.finalizedBy disableDeprecatedSSLProtocols

task updateSSLCiphers(type: com.marklogic.gradle.task.ServerEvalTask) {
  xquery = new File('./build/buildSupport/updateSSLCiphers.xqy').getText('UTF-8')
}
mlDeployServers.finalizedBy updateSSLCiphers

// This is a convenience task intended for one with the tenant's deployer role to deploy most of the backend.
// Purposely excluded:
//
//    mlDeploySecurity: insufficient permissions
//    mlLoadData: takes a while and not always needed
//    mlReloadModules: too likely to take the modules database offline; do it manually before this task.
//
task performBaseDeployment() {
  doFirst {
    println ""
    println "Starting Base Deployment"
    println ""
  }
  dependsOn mlDeployGroups, mlDeployDatabases, mlDeployRestApis, mlDeployServers, mlLoadModules
  doLast {
    println ""
    println "Base Deployment Done"
    println ""
  }
}
// It was a little tricky binding mlDeployForestReplicas to performBaseDeployment; adding it above
// could result in an error if the tenant's modules database didn't exist in time.
performBaseDeployment.finalizedBy mlDeployForestReplicas, generateRemainingSearchTerms

// Enables SSL on the builtin servers
// And create the certificate intended for the non-builtin servers
task enableSSL(type: com.marklogic.gradle.task.MarkLogicTask) {
  doFirst {
    def manageConfig = getProject().property("mlManageConfig")
    manageConfig.setScheme("http")
    manageConfig.setConfigureSimpleSsl(false)
    //manageConfig.setAdminScheme("http")
    //manageConfig.setAdminConfigureSimpleSsl(false)

    def adminConfig = getProject().property("mlAdminConfig")
    adminConfig.setScheme("http")
    adminConfig.setConfigureSimpleSsl(false)

    def manageClient = new com.marklogic.mgmt.ManageClient(manageConfig)
    def adminManager = new com.marklogic.mgmt.admin.AdminManager(adminConfig)

    def certManager = new com.marklogic.mgmt.resource.security.CertificateTemplateManager(manageClient)
    certManager.save(builtInAppCert())
    certManager.save(luxAppCert())

    def gtcc = new com.marklogic.appdeployer.command.security.GenerateTemporaryCertificateCommand();
    gtcc.setTemplateIdOrName("built-in-app-cert");
    gtcc.setCommonName("localhost");
    gtcc.execute(new com.marklogic.appdeployer.command.CommandContext(getAppConfig(), manageClient, adminManager));
    gtcc.setTemplateIdOrName("lux-app-cert");
    gtcc.setCommonName("localhost");
    gtcc.execute(new com.marklogic.appdeployer.command.CommandContext(getAppConfig(), manageClient, adminManager));

    adminConfig = getProject().property("mlAdminConfig")
    adminConfig.setScheme("https")
    adminConfig.setConfigureSimpleSsl(true)
    adminManager = new com.marklogic.mgmt.admin.AdminManager(adminConfig)

    manageClient.putJson("/manage/v2/servers/Admin/properties?group-id=Default", '{"ssl-certificate-template": "built-in-app-cert"}')
    adminManager.waitForRestart()
    manageClient.putJson("/manage/v2/servers/App-Services/properties?group-id=Default", '{"ssl-certificate-template": "built-in-app-cert"}')
    adminManager.waitForRestart()
    manageClient.putJson("/manage/v2/servers/Manage/properties?group-id=Default", '{"ssl-certificate-template": "built-in-app-cert"}')
    adminManager.waitForRestart()
  }
}

// Disables SSL on the builtin servers
task disableSSL(type: com.marklogic.gradle.task.MarkLogicTask) {
  doFirst {
    def manageClient = getManageClient()
    manageClient.putJson("/manage/v2/servers/Admin/properties?group-id=Default", '{"ssl-certificate-template": ""}')
    manageClient.putJson("/manage/v2/servers/App-Services/properties?group-id=Default", '{"ssl-certificate-template": ""}')
    manageClient.putJson("/manage/v2/servers/Manage/properties?group-id=Default", '{"ssl-certificate-template": ""}')

    def adminConfig = getProject().property("mlAdminConfig")
    adminConfig.setScheme("http")
    adminConfig.setConfigureSimpleSsl(false)
    def adminManager = new com.marklogic.mgmt.admin.AdminManager(adminConfig)
    adminManager.waitForRestart()

    // def manageConfig = getProject().property("mlManageConfig")
    // manageConfig.setScheme("http")
    // manageConfig.setConfigureSimpleSsl(false)
    // def mgClient = new com.marklogic.mgmt.ManageClient(manageConfig)

    // def certManager = new com.marklogic.mgmt.resource.security.CertificateTemplateManager(mgClient)
    // certManager.delete(builtInAppCert())
    // certManager.delete(luxAppCert())
  }
}

// Returns a certificate request for use on the built-in appservers
def builtInAppCert() {
    return """
    <certificate-template-properties xmlns="http://marklogic.com/manage">
      <template-name>built-in-app-cert</template-name>
      <template-description>Built-In App Servers Certificate Template</template-description>
      <key-type>rsa</key-type>
      <key-options />
      <req>
        <version>0</version>
        <subject>
          <countryName>US</countryName>
          <stateOrProvinceName>CT</stateOrProvinceName>
          <localityName>New Haven</localityName>
          <organizationName>Yale University</organizationName>
          <organizationalUnitName>ITS</organizationalUnitName>
          <emailAddress>xinjian.guo@yale.edu</emailAddress>
        </subject>
      </req>
    </certificate-template-properties>
    """
}

// Returns a certificate request for use on the LUX appservers
def luxAppCert() {
    return """
    <certificate-template-properties xmlns="http://marklogic.com/manage">
      <template-name>lux-app-cert</template-name>
      <template-description>LUX App Servers Certificate Template</template-description>
      <key-type>rsa</key-type>
      <key-options />
      <req>
        <version>0</version>
        <subject>
          <countryName>US</countryName>
          <stateOrProvinceName>CT</stateOrProvinceName>
          <localityName>New Haven</localityName>
          <organizationName>Yale University</organizationName>
          <organizationalUnitName>ITS</organizationalUnitName>
          <emailAddress>xinjian.guo@yale.edu</emailAddress>
        </subject>
      </req>
    </certificate-template-properties>
    """
}

task printCredentials {
  doFirst {
    // Most important ones
    println "mlPassword: " + credentials.forKey('mlPassword')

    // Only required by specific, infrequently-used tasks
    println "copyDatabaseInputPassword: " + credentials.forKey('copyDatabaseInputPassword')
    println "copyDatabaseOutputPassword: " + credentials.forKey('copyDatabaseOutputPassword')
    println "importDataPassword: " + credentials.forKey('importDataPassword')
  }
}

ext {
  // Configure properties based on encrypted credentials
  mlManageConfig.password = credentials.forKey('mlPassword')
  //mlManageConfig.securityPassword = credentials.forKey('mlPassword')  only needed if setting mlSecurityUsername
  mlAdminConfig.password = credentials.forKey('mlPassword')
  mlAppConfig.restAdminPassword =  credentials.forKey('mlPassword')
  mlAppConfig.appServicesPassword = credentials.forKey('mlPassword')

  // Re-initialize the connections to the Admin and Manage servers
  mlManageClient.manageConfig = mlManageConfig
  mlAdminManager.adminConfig = mlAdminConfig
}