build.gradle

import org.apache.tools.ant.filters.ReplaceTokens

apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'download-task'
apply plugin: 'versions'
apply from: 'gradle/functions.gradle'

ext.configDir = new File(rootDir, 'config')
ext.hadoopBinaries = "${System.getProperty('user.home')}/hadoop-binaries"
ext.branchVersion = project.getProperties().get('clusterVersion', '2.4').toLowerCase()

ext.hiveVersionMap = [
        '1.0' : '0.10.0',
        '1.1' : '0.10.0',
        'cdh4': '0.10.0-cdh4.6.0',
        'cdh5': '0.12.0-cdh5.0.0'
].withDefault {
    '0.12.0'
}

ext.hadoopClusters = [
        '0.23': '0.23.10',
        '1.0' : '1.0.4',
        '1.1' : '1.1.2',
        '1.2' : '1.2.1',
        '2.2' : '2.2.0',
        '2.3' : '2.3.0',
        '2.4' : '2.4.0',
        'cdh4': '2.0.0-cdh4.6.0',
        'cdh5': '2.3.0-cdh5.0.0',
].withDefault {
    throw new GradleException("Unknown hadoop version: ${it}")
}

ext.pigVersionMap = [
        'cdh4': '0.11.0-cdh4.6.0',
        'cdh5': '0.12.0-cdh5.0.0'
].withDefault {
    '0.13.0'
}

ext.javaDriverVersion = '2.12.3'

ext.clusterVersion = hadoopClusters[branchVersion]
ext.hiveVersion = hiveVersionMap[branchVersion]
ext.pigVersion = pigVersionMap[branchVersion]
ext.hadoopVersion = '2.4.1'

ext.hadoopHome = "${hadoopBinaries}/hadoop-${clusterVersion}"
ext.hiveHome = "${hadoopBinaries}/hive-${hiveVersion}"
ext.pigHome = "${hadoopBinaries}/pig-${pigVersion}"
ext.dataHome = "${hadoopBinaries}/examples/data"
ext.mongoimport = new File('/mnt/jenkins/mongodb/26/26-release/bin/mongoimport').exists() ?
                  '/mnt/jenkins/mongodb/26/26-release/bin/mongoimport' :
                  '/usr/local/bin/mongoimport';
ext.mongorestore = new File('/mnt/jenkins/mongodb/26/26-release/bin/mongorestore').exists() ?
                  '/mnt/jenkins/mongodb/26/26-release/bin/mongorestore' :
                  '/usr/local/bin/mongorestore';


buildscript {
    repositories {
        mavenLocal()
        mavenCentral()
        maven { url 'https://oss.sonatype.org/content/repositories/snapshots' }
        jcenter()
        maven { url "https://github.com/ben-manes/gradle-versions-plugin/raw/mvnrepo" }
    }
    dependencies {
        classpath 'me.trnl:clirr-gradle-plugin:0.4'
        classpath 'com.antwerkz.github:github-release-gradle-plugin:1.1.0'
        classpath 'de.undercouch:gradle-download-task:1.0'
        classpath 'com.github.ben-manes:gradle-versions-plugin:0.5-beta-2'
    }
}

task wrapper(type: Wrapper) {
    gradleVersion = '1.11'
}

configure(subprojects) {
    apply plugin: 'java'
    apply plugin: 'idea'
    apply plugin: 'eclipse'
    apply plugin: 'checkstyle'
    apply plugin: 'findbugs'

    repositories {
        mavenCentral()
        mavenLocal()
        maven { url "https://repository.cloudera.com/artifactory/cloudera-repos/" }
    }

    group = 'org.mongodb'
    version = '1.4.0-SNAPSHOT'

    sourceCompatibility = JavaVersion.VERSION_1_6
    targetCompatibility = JavaVersion.VERSION_1_6

    configurations {
        tests
        tests.extendsFrom(testCompile)
        
        provided
        provided.extendsFrom(compile)
    }
    
    sourceSets.main.compileClasspath += configurations.provided
    sourceSets.test.compileClasspath += configurations.provided
    sourceSets.test.runtimeClasspath += configurations.provided

    dependencies {
        compile "org.mongodb:mongo-java-driver:${javaDriverVersion}"

        testCompile 'junit:junit:4.11'
        testCompile 'org.hamcrest:hamcrest-all:1.3'
        testCompile 'org.zeroturnaround:zt-exec:1.6'
        testCompile 'com.jayway.awaitility:awaitility:1.6.0'
        testCompile 'commons-daemon:commons-daemon:1.0.15'

        testCompile "org.apache.hadoop:hadoop-hdfs:${hadoopVersion}"
        testCompile "org.apache.hadoop:hadoop-hdfs:${hadoopVersion}:tests"
        testCompile "org.apache.hadoop:hadoop-common:${hadoopVersion}:tests"
        testCompile "org.apache.hadoop:hadoop-yarn-server-tests:${hadoopVersion}:tests"
        testCompile "org.apache.hadoop:hadoop-mapreduce-client-jobclient:${hadoopVersion}:tests"
    }

    /* Compiling */
    tasks.withType(AbstractCompile) {
        options.encoding = 'ISO-8859-1'
        options.fork = true
        options.debug = true
        options.compilerArgs = [/*'-Xlint:deprecation',*/ '-Xlint:-options']
    }

    project.ext.buildingWith = { n ->
        project.hasProperty(n) && project.property(n).toBoolean()
    }

    /* Testing */
    task testJar(type: Jar, dependsOn: testClasses) {
        from sourceSets.test.output
    }

    tasks.withType(Test) {
        maxParallelForks = 1
        systemProperty 'project.version', project.version
        systemProperty 'cluster.version', hadoopClusters[branchVersion]
        systemProperty 'hadoop.version', branchVersion
        systemProperty 'hive.version', hiveVersion
        systemProperty 'pig.version', pigVersion
        systemProperties << System.getProperties()

        beforeTest { descr ->
            logger.info("[Test ${descr.className} > ${descr.name}]")
        }
        
        finalizedBy ':shutdownCluster'
    }

    test.dependsOn ':configureCluster'
    compileJava.dependsOn ':generateScripts'
    clean.dependsOn ':cleanHadoop'

    /* Code quality */
    checkstyle {
        configFile = new File("$configDir/checkstyle.xml")
    }

    checkstyleTest {
        classpath += configurations.compile
        classpath += configurations.testCompile
    }

    checkstyleMain {
        classpath += configurations.compile
    }

    task quickCheck(dependsOn: [checkstyleMain, checkstyleTest] ) << {}
    
    findbugs {
        excludeFilter = new File("$configDir/findbugs-exclude.xml")
        sourceSets = [sourceSets.main]
    }

    tasks.withType(FindBugs) {
        reports {
            xml.enabled = project.buildingWith('xmlReports.enabled')
            html.enabled = !project.buildingWith('xmlReports.enabled')
        }
    }

    javadoc {
        options.version = true
        options.links 'http://docs.oracle.com/javase/6/docs/api/'
        options.links 'http://hadoop.apache.org/docs/r2.4.0/api'
        options.links 'http://api.mongodb.org/java/2.12.3/'
    }

    gradle.taskGraph.whenReady { taskGraph ->
        testJar {
            baseName = "${archivesBaseName}-test"
            mustRunAfter jar
        }
    }

    configurations {
    }

    artifacts {
        tests testJar
    }

    test {
        dependsOn ':configureCluster', ':downloadEnronEmails'
        finalizedBy ':shutdownCluster'
    }
}

configure(subprojects.findAll { it.name.contains('examples/') }) {
    def exampleName = project.name.split('/')[1]
    jar {
        baseName = exampleName
    }
    group += ".mongo-hadoop-examples"

}

project(":core") {
    archivesBaseName = "mongo-hadoop-core"

    dependencies {
        compile "org.apache.hadoop:hadoop-common:${hadoopVersion}"
        compile "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoopVersion}"
        compile "org.apache.hadoop:hadoop-mapreduce-client-common:${hadoopVersion}" 
        compile "org.apache.hadoop:hadoop-mapreduce-client-shuffle:${hadoopVersion}" 
        compile "org.apache.hadoop:hadoop-mapreduce-client-app:${hadoopVersion}"
        compile "org.apache.hadoop:hadoop-mapreduce-client-jobclient:${hadoopVersion}"
    }
}

project(":hive") {
    archivesBaseName = "mongo-hadoop-hive"
    dependencies {
        compile project(":core")

        compile "org.apache.hive:hive-exec:${hiveVersion}"
        compile "org.apache.hive:hive-serde:${hiveVersion}"

        testCompile "org.apache.hive:hive-cli:${hiveVersion}"
        testCompile "org.apache.hive:hive-service:${hiveVersion}"

        testCompile project(path: ':core', configuration: 'tests')
        testCompile("com.nitayjoffe.thirdparty.com.jointhegrid:hive_test:4.0.0")  {
            exclude group: 'org.apache.derby', module: 'derby'
            exclude group: 'org.apache.hadoop', module: 'hadoop-core'
        }
    }
}

project(":pig") {
    archivesBaseName = "mongo-hadoop-pig"
    
    dependencies {
        compile project(":core")
        compile "org.apache.pig:pig:${pigVersion}"

        testCompile project(path: ':core', configuration: 'tests')
        testCompile "org.antlr:antlr:3.5.2"
    }

    processTestResources {
        outputs.upToDateWhen{ false }
        filter ReplaceTokens, tokens: [
                JAVA_DRIVER_JAR : project(':core').configurations.compile.find { it.name.startsWith("mongo-java-driver") }.toString(),
                PROJECT_VERSION : project(':core').version,
                PROJECT_HOME    : project.rootDir.getAbsolutePath(),
        ]
    }

    jar {
        from project(':core').sourceSets.main.output
        from sourceSets.main.output

        configurations.compile.filter {
            it.name.startsWith('mongo-java-driver')
        }.each {
            from zipTree(it)
        }
    }

}

project(":streaming") {
    archivesBaseName = "mongo-hadoop-streaming"

    dependencies {
        compile project(":core")
        compile "org.apache.hadoop:hadoop-streaming:${hadoopVersion}"
    }

    jar {
        from project(':core').sourceSets.main.output
        from sourceSets.main.output

        configurations.compile.filter {
            it.name.startsWith('mongo-java-driver')
        }.each {
            from zipTree(it)
        }
    }
}

project(":flume") {
    dependencies {
        compile project(":core")
        compile("com.cloudera:flume-core:0.9.4-cdh3u3") {
            exclude group: 'org.apache.hadoop', module: 'hadoop-core'
            exclude group: 'com.cloudera.cdh', module: 'hadoop-ant'
        }
    }
}

project(":examples/treasury_yield") {
    uploadArchives.onlyIf { false }
    dependencies {
        compile project(":core")
        testCompile project(path: ':core', configuration: 'tests')
        testCompile project(':streaming')
        testCompile 'org.slf4j:slf4j-jdk14:1.7.7'
    }
}

project(":examples/enron") {
    uploadArchives.onlyIf { false }
    dependencies {
        compile project(":core")
    }
}

project(":examples/sensors") {
    uploadArchives.onlyIf { false }
    dependencies {
        compile project(":core")
    }
}

project(":integration-tests") {
    uploadArchives.onlyIf { false }
    dependencies {
        testCompile project(":core")
        testCompile "${group}:mongo-hadoop-streaming:${version}"
        testCompile 'org.slf4j:slf4j-jdk14:1.7.7'
    }
}

task cleanLogs(type: Delete) {  
    delete fileTree(dir: "logs", exclude: ".keep")  
}

task cleanHadoop(type: Delete, dependsOn: cleanLogs) {
    delete hadoopHome, hiveHome, pigHome
}

task historicalYield(dependsOn: configureCluster) << {
    
    exec() {
        commandLine mongoimport, "-d", "mongo_hadoop", "-c", "yield_historical.in", "--drop",
                    "examples/treasury_yield/src/main/resources/yield_historical_in.json"
    }

    hadoop("examples/treasury_yield/build/libs/treasury_yield-${project(':core').version}.jar",
           "com.mongodb.hadoop.examples.treasury.TreasuryYieldXMLConfig", [
            "mongo.input.uri=mongodb://localhost:27017/mongo_hadoop.yield_historical.in",
            "mongo.output.uri=mongodb://localhost:27017/mongo_hadoop.yield_historical.out"
    ])
}

task sensorData(dependsOn: 'configureCluster') << {
    hadoop("examples/sensors/build/libs/sensors-${project(':core').version}.jar",
           "com.mongodb.hadoop.examples.sensors.Devices", [])

    hadoop("examples/sensors/build/libs/sensors-${project(':core').version}.jar",
           "com.mongodb.hadoop.examples.sensors.Logs", ["io.sort.mb=100"])
}

task downloadEnronEmails() {
    if (!new File("${dataHome}/dump").exists()) {
        println "Downloading Enron email test data to ${dataHome}"
        new File(dataHome).mkdirs()
        def attempts = 0
        while (!new File(dataHome, "enron_mongo.tar.bz2").exists() && attempts < 10) {
            try {
                download {
                    src 'https://s3.amazonaws.com/mongodb-enron-email/enron_mongo.tar.bz2'
                    dest dataHome
                    onlyIfNewer true
                }
            } catch (Exception e) {
                attempts++
                if (attempts == 10) {
                    throw e;
                }
            }
        }
        println "extracting email data"
        copy {
            from(tarTree("${dataHome}/enron_mongo.tar.bz2"))
            into dataHome
        }
    }
}

task enronEmails(dependsOn: [downloadEnronEmails, configureCluster]) << {
    exec() {
        commandLine mongorestore, "-v", "-d", "mongo_hadoop", "--drop", "${dataHome}/dump/enron_mail"
    }

    hadoop("examples/enron/build/libs/enron-${project(':core').version}.jar",
           "com.mongodb.hadoop.examples.enron.EnronMail", ["mongo.input.split_size=64"])
}

def hadoop(jar, className, args) {
    def line = ["${hadoopHome}/bin/hadoop",
                "jar", jar, className,
                //Split settings
                "-Dmongo.input.split_size=8",
                "-Dmongo.job.verbose=true",
    ]
    args.each {
        line << "-D${it}"
    }
    println "Executing hadoop job:\n ${line.join(' \\\n\t')}"
    def hadoopEnv = [:]
    if (clusterVersion.contains("cdh")) {
        hadoopEnv.MAPRED_DIR = 'share/hadoop/mapreduce2'
    }
    hadoopEnv.HADOOP_PREFIX=''
    exec() {
        environment << hadoopEnv
        commandLine line
    }
    shutdownCluster.execute()    
}

apply from: 'gradle/maven-deployment.gradle'