Skip to content

Commit

Permalink
consolidated patch of Windows compatibility work, prepping for submis…
Browse files Browse the repository at this point in the history 
…sion of pull request
  • Loading branch information
@cnauroth
cnauroth committed Sep 17, 2013
1 parent fbd3aa7 commit c00d1eb
Show file tree
Hide file tree
Showing 10 changed files with 674 additions and 81 deletions.
5 changes: 5 additions & 0 deletions .gitattributes
View file
Original file line number Diff line number Diff line change
@@ -1 +1,6 @@
.archive-version export-subst

# Auto detect text files and perform LF normalization
* text=auto
*.sln text merge=union eol=crlf
*.vcxproj text merge=union eol=crlf
9 changes: 9 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,22 @@ LZO 2.x is required, and most easily installed via the package manager on your s
1. Download the latest LZO release from http://www.oberhumer.com/opensource/lzo/
1. Configure LZO to build a shared library (required) and use a package-specific prefix (optional but recommended): `./configure --enable-shared --prefix /usr/local/lzo-2.06`
1. Build and install LZO: `make && sudo make install`
1. On Windows, you can build lzo2.dll with this command: `B\win64\vc_dll.bat`

Now let's build hadoop-lzo.

C_INCLUDE_PATH=/usr/local/lzo-2.06/include \
LIBRARY_PATH=/usr/local/lzo-2.06/lib \
mvn clean test

Running tests on Windows also requires setting PATH to include the location of lzo2.dll.

set PATH=C:\lzo-2.06;%PATH%

Additionally on Windows, the Hadoop core code requires setting HADOOP_HOME so that the tests can find winutils.exe. If you've built Hadoop trunk in directory C:\hdc, then the following would work.

set HADOOP_HOME=C:\hdc\hadoop-common-project\hadoop-common\target

Once the libs are built and installed, you may want to add them to the class paths and library paths. That is, in hadoop-env.sh, set

export HADOOP_CLASSPATH=/path/to/your/hadoop-lzo-lib.jar
Expand Down
174 changes: 148 additions & 26 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.current.version>2.1.0-beta</hadoop.current.version>
<hadoop.old.version>1.0.4</hadoop.old.version>
</properties>

<profiles>
Expand All @@ -100,13 +102,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.0.4-alpha</version>
<version>${hadoop.current.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.0.4-alpha</version>
<version>${hadoop.current.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
Expand All @@ -120,13 +122,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>1.0.4</version>
<version>${hadoop.old.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-test</artifactId>
<version>1.0.4</version>
<version>${hadoop.old.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
Expand All @@ -147,17 +149,36 @@
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
<executions>
<execution>
<id>set-props</id>
<id>check-platform</id>
<phase>initialize</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<exportAntProperties>true</exportAntProperties>
<target name="set-props" description="sets key properties that are used throughout">
<target name="check-platform">
<condition property="windows">
<os family="windows" />
</condition>
<condition property="non-windows">
<not>
<isset property="windows" />
</not>
</condition>
</target>
</configuration>
</execution>
<execution>
<id>set-props-non-win</id>
<phase>initialize</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<exportAntProperties>true</exportAntProperties>
<target name="set-props-non-win" description="sets key properties that are used throughout" if="non-windows">
<exec executable="sed" inputstring="${os.name}" outputproperty="nonspace.os">
<arg value="s/ /_/g"/>
</exec>
Expand All @@ -170,17 +191,35 @@
</configuration>
</execution>
<execution>
<id>build-info</id>
<id>set-props-win</id>
<phase>initialize</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<exportAntProperties>true</exportAntProperties>
<target name="set-props-win" description="sets key properties that are used throughout" if="windows">
<property name="build.native" value="${project.build.directory}/native/${env.OS}-${env.PLATFORM}"/>
<property name="native.src.dir" value="${basedir}/src/main/native"/>
<property name="test.build.dir" value="${project.build.directory}/test-classes"/>
<property name="test.log.dir" value="${test.build.dir}/logs"/>
</target>
</configuration>
</execution>
<execution>
<id>build-info-non-win</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target name="build-info" description="generates build.properties">
<target name="build-info-non-win" description="generates build.properties" if="non-windows">
<tstamp>
<format property="build_time" pattern="MM/dd/yyyy hh:mm aa" timezone="GMT"/>
</tstamp>
<exec executable="${basedir}/scripts/get_build_revision.sh" outputproperty="build_revision"/>
<exec executable="sh" outputproperty="build_revision">
<arg value="${basedir}/scripts/get_build_revision.sh" />
</exec>
<exec executable="whoami" outputproperty="build_author"/>
<exec executable="uname" outputproperty="build_os">
<arg value="-a"/>
Expand All @@ -197,28 +236,68 @@
</configuration>
</execution>
<execution>
<id>check-native-uptodate</id>
<id>build-info-win</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target name="build-info-win" description="generates build.properties" if="windows">
<tstamp>
<format property="build_time" pattern="MM/dd/yyyy hh:mm aa" timezone="GMT"/>
</tstamp>
<exec executable="sh" outputproperty="build_revision">
<arg value="${basedir}/scripts/get_build_revision.sh" />
</exec>
<propertyfile file="${project.build.outputDirectory}/build.properties"
comment="This file is automatically generated - DO NOT EDIT">
<entry key="build_time" value="${build_time}"/>
<entry key="build_revision" value="${build_revision}"/>
<entry key="build_author" value="${env.USERNAME}"/>
<entry key="build_version" value="${project.version}"/>
<entry key="build_os" value="${env.OS}"/>
</propertyfile>
</target>
</configuration>
</execution>
<execution>
<id>check-native-uptodate-non-win</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<exportAntProperties>true</exportAntProperties>
<target name="check-native-uptodate-non-win" description="checks if native binaries should be rebuilt" if="non-windows">
<uptodate property="native.uptodate" targetfile="${build.native}/lib/libgplcompression.la">
<srcfiles dir="${native.src.dir}" includes="**/*"/>
</uptodate>
</target>
</configuration>
</execution>
<execution>
<id>check-native-uptodate-win</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<exportAntProperties>true</exportAntProperties>
<target name="check-native-uptodate" description="checks if native binaries should be rebuilt">
<uptodate property="native.uptodate" targetfile="${build.native}/lib/libgplcompression.la">
<srcfiles dir="${native.src.dir}" includes="**/*"/>
</uptodate>
</target>
</configuration>
</execution>
<execution>
<id>build-native</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target name="build-native" unless="native.uptodate" description="compiles native code">
<target name="check-native-uptodate-win" description="checks if native binaries should be rebuilt" if="windows">
<uptodate property="native.uptodate" targetfile="${build.native}/lib/gplcompression.dll">
<srcfiles dir="${native.src.dir}" includes="**/*"/>
</uptodate>
</target>
</configuration>
</execution>
<execution>
<id>build-native-non-win</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target name="build-native-non-win" if="non-windows" unless="native.uptodate" description="compiles native code">
<property name="make.cmd" value="make"/>
<property name="build.classes" refid="maven.compile.classpath"/>
<condition property="native.ldflags" value="" else="-Wl,--no-as-needed">
Expand Down Expand Up @@ -258,6 +337,39 @@
</target>
</configuration>
</execution>
<execution>
<id>build-native-win</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target name="build-native-win" unless="native.uptodate" if="windows" description="compiles native code">
<property name="build.classes" refid="maven.compile.classpath"/>

<mkdir dir="${build.native}/lib"/>
<mkdir dir="${build.native}/src/com/hadoop/compression/lzo"/>

<javah classpath="${build.classes}"
destdir="${build.native}/src/com/hadoop/compression/lzo"
force="yes"
verbose="yes">
<class name="com.hadoop.compression.lzo.LzoCompressor" />
<class name="com.hadoop.compression.lzo.LzoDecompressor" />
</javah>

<exec dir="${build.native}" executable="msbuild" failonerror="true">
<arg value="${basedir}/src/main/native/gplcompression.sln" />
<arg value="/nologo" />
<arg value="/p:Configuration=Release" />
<arg value="/p:IntDir=${build.native}/" />
<arg value="/p:OutDir=${build.native}/" />
</exec>

<copy file="${build.native}/gplcompression.dll" todir="${build.native}/lib" />
</target>
</configuration>
</execution>
<execution>
<id>prep-test</id>
<phase>test-compile</phase>
Expand Down Expand Up @@ -356,6 +468,16 @@
</lifecycleMappingMetadata>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
</plugin>
</plugins>
</pluginManagement>
</build>
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/hadoop/compression/lzo/LzoCompressor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ class LzoCompressor implements Compressor {
@SuppressWarnings("unused")
private long lzoCompressor = 0; // The actual lzo compression function.
private int workingMemoryBufLen = 0; // The length of 'working memory' buf.
private long lzoCompressLevelFunc = 0; // The lzo compression level function.
@SuppressWarnings("unused")
private ByteBuffer workingMemoryBuf; // The 'working memory' for lzo.
private int lzoCompressionLevel;
Expand Down
43 changes: 43 additions & 0 deletions src/main/native/gplcompression.sln
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@

Microsoft Visual Studio Solution File, Format Version 11.00
# Visual Studio 2010

# This file is part of Hadoop-Gpl-Compression.
#
# Hadoop-Gpl-Compression is free software: you can redistribute it
# and/or modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation, either version 3 of
# the License, or (at your option) any later version.
#
# Hadoop-Gpl-Compression is distributed in the hope that it will be
# useful, but WITHOUT ANY WARRANTY; without even the implied warranty
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Hadoop-Gpl-Compression. If not, see
# <http://www.gnu.org/licenses/>.

Project("{EAAEC089-7163-4E57-96A1-B61BF66B3F72}") = "gplcompression", "gplcompression.vcxproj", "{A8EC1579-8E95-4C0B-A462-4C3D625CDC39}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{A8EC1579-8E95-4C0B-A462-4C3D625CDC39}.Debug|Win32.ActiveCfg = Release|x64
{A8EC1579-8E95-4C0B-A462-4C3D625CDC39}.Debug|Win32.Build.0 = Release|x64
{A8EC1579-8E95-4C0B-A462-4C3D625CDC39}.Debug|x64.ActiveCfg = Debug|x64
{A8EC1579-8E95-4C0B-A462-4C3D625CDC39}.Debug|x64.Build.0 = Debug|x64
{A8EC1579-8E95-4C0B-A462-4C3D625CDC39}.Release|Win32.ActiveCfg = Release|Win32
{A8EC1579-8E95-4C0B-A462-4C3D625CDC39}.Release|Win32.Build.0 = Release|Win32
{A8EC1579-8E95-4C0B-A462-4C3D625CDC39}.Release|x64.ActiveCfg = Release|x64
{A8EC1579-8E95-4C0B-A462-4C3D625CDC39}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal
Loading

0 comments on commit c00d1eb

Please sign in to comment.