Initial commit of stuff from asmyczek's azkaban repo

build.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<project name="mr-kluj" default="jar-with-deps">
+    <property name="package.name" value="mr-kluj-1.0.0"/>
+    <property name="base.dir" value="."/>
+    <property name="lib.dir" value="${base.dir}/lib"/>
+    <property name="jar-with-deps.dir" value="${base.dir}/dist/jar-with-deps"/>
+
+    <target name="jar-with-deps" description="Make big bad jar">
+        <delete dir="${jar-with-deps.dir}"/>
+        <mkdir dir="${jar-with-deps.dir}"/>
+
+        <exec executable="lein">
+            <arg value="clean"/>
+        </exec>
+        <exec executable="lein">
+            <arg value="javac"/>
+        </exec>
+        <exec executable="lein">
+            <arg value="jar"/>
+        </exec>
+
+        <jar jarfile="${jar-with-deps.dir}/${package.name}.jar" filesetmanifest="merge">
+            <manifest>
+                <attribute name="Main-Class" value="com.linkedin.mr_kluj.GenericClojureJob"/>
+            </manifest>
+            <zipgroupfileset file="mr-kluj-1.0.0-SNAPSHOT.jar"/>
+            <zipgroupfileset dir="${lib.dir}" includes="*.jar"/>
+        </jar>
+    </target>
+</project>

examples/basic_avro_job.clj

@@ -0,0 +1,17 @@
+(use     '(simple-avro schema))
+(require '[com.linkedin.mr-kluj.job :as job]
+         '[com.linkedin.mr-kluj.avro-storage :as avro])
+
+; Output schema 
+(defavro-record NameCount
+	:first avro-string
+	:count avro-int)
+
+(job/run 
+  (job/staged-job ["avro-job" "staging-location"]
+    (avro/avro-storage-input "test.avro")
+    (job/map-mapper (fn [key value context] [[(get value "first") 1]]))
+    (job/create-reducer (fn [key values context] [[nil {"first" key "count" (count values)}]]))
+    (avro/avro-intermediate-data avro-string avro-int)
+    (avro/avro-storage-output "count.avro" nil NameCount)))
+

pom.xml

@@ -0,0 +1,153 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project>
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>mr-kluj</groupId>
+  <artifactId>mr-kluj</artifactId>
+  <version>2.0.1-SNAPSHOT</version>
+  <name>mr-kluj</name>
+  <description>A project to make map reduce jobs in clojure simple(r)</description>
+  <packaging>clojure</packaging>
+  <scm>
+    <connection>scm:git:ssh://[email protected]/metamx/azkaban.git</connection>
+    <developerConnection>scm:git:ssh://[email protected]/metamx/azkaban.git</developerConnection>
+    <url>http://www.github.com/metamx/azkaban</url>
+  </scm>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.clojure</groupId>
+      <artifactId>clojure</artifactId>
+      <version>1.2.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.clojure</groupId>
+      <artifactId>clojure-contrib</artifactId>
+      <version>1.2.0</version>
+    </dependency>
+    <dependency>
+      <groupId>log4j</groupId>
+      <artifactId>log4j</artifactId>
+      <version>1.2.15</version>
+      <exclusions>
+        <exclusion>
+          <artifactId>mail</artifactId>
+          <groupId>javax.mail</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jms</artifactId>
+          <groupId>javax.jms</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jmxtools</artifactId>
+          <groupId>com.sun.jdmk</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jmxri</artifactId>
+          <groupId>com.sun.jmx</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.google.collections</groupId>
+      <artifactId>google-collections</artifactId>
+      <version>1.0-rc2</version>
+    </dependency>
+    <dependency>
+      <groupId>joda-time</groupId>
+      <artifactId>joda-time</artifactId>
+      <version>1.6</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-lang</groupId>
+      <artifactId>commons-lang</artifactId>
+      <version>2.1</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-logging</groupId>
+      <artifactId>commons-logging</artifactId>
+      <version>1.0.4</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-httpclient</groupId>
+      <artifactId>commons-httpclient</artifactId>
+      <version>3.1</version>
+    </dependency>
+    <dependency>
+      <groupId>azkaban</groupId>
+      <artifactId>azkaban-common</artifactId>
+      <version>0.4-mr-kluj</version>
+    </dependency>
+    <dependency>
+      <groupId>simple-avro</groupId>
+      <artifactId>simple-avro</artifactId>
+      <version>0.0.5</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop</artifactId>
+      <version>0.20.3-dev-core</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>voldemort</groupId>
+      <artifactId>voldemort</artifactId>
+      <version>0.70.1</version>
+      <optional>true</optional>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+	<groupId>com.theoryinpractise</groupId>
+	<artifactId>clojure-maven-plugin</artifactId>
+	<version>1.3.7</version>
+	<extensions>true</extensions>
+	<configuration>
+	  <sourceDirectories>
+	    <sourceDirectory>src/main/clj</sourceDirectory>
+	  </sourceDirectories>
+	  <testSourceDirectories>
+	    <testSourceDirectory>test/main/clj</testSourceDirectory>
+	  </testSourceDirectories>
+	  <resources>
+	    <resource><directory>src/main/resources</directory></resource>
+	  </resources>
+	  <testResources>
+	    <testResource><directory>test-resources</directory></testResource>
+	  </testResources>
+	  <namespaces>
+	    <namespace>!.*</namespace>
+	  </namespaces>
+	  <compileDeclaredNamespaceOnly>true</compileDeclaredNamespaceOnly>
+	  <warnOnReflection>true</warnOnReflection>
+	</configuration>
+      </plugin>
+      <plugin>
+	<groupId>org.apache.maven.plugins</groupId>
+	<artifactId>maven-compiler-plugin</artifactId>
+	<configuration>
+	  <source>1.6</source>
+	  <target>1.6</target>
+	</configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>1.4</version>
+        <executions>
+          <execution>
+	    <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+	      <shadedArtifactAttached>true</shadedArtifactAttached>
+              <shadedClassifierName>selfcontained</shadedClassifierName>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

project.clj

@@ -0,0 +1,23 @@
+(defproject mr-kluj "1.0.0-SNAPSHOT"
+  :description "A project to make map reduce jobs in clojure simple(r)"
+  :dependencies [[org.clojure/clojure "1.2.0"]
+                 [org.clojure/clojure-contrib "1.2.0"]
+		 [org.apache.hadoop/hadoop-core "0.20.2"]
+		 [voldemort/voldemort "0.90.li8"]
+     [simple-avro/simple-avro "0.0.5"]
+		 [log4j/log4j "1.2.15" :exclusions [javax.mail/mail
+						    javax.jms/jms
+						    com.sun.jdmk/jmxtools
+						    com.sun.jmx/jmxri]]
+		 [com.google.collections/google-collections "1.0-rc2"]
+		 [joda-time/joda-time "1.6"]
+		 [commons-lang/commons-lang "2.1"]
+		 [commons-logging/commons-logging "1.0.4"]
+		 [commons-httpclient/commons-httpclient "3.1"]
+		 [azkaban/azkaban-common "0.05"]]
+  :aot [com.linkedin.mr-kluj.job
+        com.linkedin.mr-kluj.hadoop-utils
+        com.linkedin.mr-kluj.utils
+        com.linkedin.mr-kluj.avro-storage]
+  :java-source-path "src/main/java"
+  :source-path "src/main/clj")