Skip to content

Commit

Permalink
[pytorch] consolidate android gradle build scripts (pytorch#39999)
Browse files Browse the repository at this point in the history 
Summary:
Pull Request resolved: pytorch#39999

Cleaned up the android build scripts. Consolidated common functions into
common.sh. Also made a few minor fixes:

- We should trust build_android.sh doing right about reusing existing
  `build_android_$abi` directory;

- We should clean up `pytorch_android/src/main/jniLibs/` to remove
  broken symbolic links in case custom abi list changes since last build;

Test Plan: Imported from OSS

Differential Revision: D22036926

Pulled By: ljk53

fbshipit-source-id: e93915ee4f195111b6171cdabc667fa0135d5195
  • Loading branch information
@ljk53 @facebook-github-bot
ljk53 authored and facebook-github-bot committed Jun 16, 2020
1 parent 9204d76 commit bcb4479
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 182 deletions.
79 changes: 5 additions & 74 deletions android/build_test_app.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,83 +2,16 @@
set -eux

PYTORCH_DIR="$(cd $(dirname $0)/..; pwd -P)"

PYTORCH_ANDROID_DIR=$PYTORCH_DIR/android
WORK_DIR=$PYTORCH_DIR

echo "PYTORCH_DIR:$PYTORCH_DIR"
echo "WORK_DIR:$WORK_DIR"

echo "ANDROID_HOME:$ANDROID_HOME"
if [ ! -z "$ANDROID_HOME" ]; then
echo "ANDROID_HOME not set; please set it to Android sdk directory"
fi

if [ ! -d $ANDROID_HOME ]; then
echo "ANDROID_HOME not a directory; did you install it under $ANDROID_HOME?"
exit 1
fi

GRADLE_PATH=gradle
GRADLE_NOT_FOUND_MSG="Unable to find gradle, please add it to PATH or set GRADLE_HOME"

if [ ! -x "$(command -v gradle)" ]; then
if [ -z "$GRADLE_HOME" ]; then
echo GRADLE_NOT_FOUND_MSG
exit 1
fi
GRADLE_PATH=$GRADLE_HOME/bin/gradle
if [ ! -f "$GRADLE_PATH" ]; then
echo GRADLE_NOT_FOUND_MSG
exit 1
fi
fi
echo "GRADLE_PATH:$GRADLE_PATH"

ABIS_LIST="armeabi-v7a,arm64-v8a,x86,x86_64"
CUSTOM_ABIS_LIST=false
if [ $# -gt 0 ]; then
ABIS_LIST=$1
CUSTOM_ABIS_LIST=true
fi

echo "ABIS_LIST:$ABIS_LIST"

LIB_DIR=$PYTORCH_ANDROID_DIR/pytorch_android/src/main/jniLibs
INCLUDE_DIR=$PYTORCH_ANDROID_DIR/pytorch_android/src/main/cpp/libtorch_include
mkdir -p $LIB_DIR
rm -f $LIB_DIR/*
mkdir -p $INCLUDE_DIR
source "$PYTORCH_ANDROID_DIR/common.sh"

for abi in $(echo $ABIS_LIST | tr ',' '\n')
do
echo "abi:$abi"

OUT_DIR=$WORK_DIR/build_android_$abi

rm -rf $OUT_DIR
mkdir -p $OUT_DIR

pushd $PYTORCH_DIR
python $PYTORCH_DIR/setup.py clean

ANDROID_ABI=$abi VERBOSE=1 ANDROID_DEBUG_SYMBOLS=1 $PYTORCH_DIR/scripts/build_android.sh -DANDROID_CCACHE=$(which ccache)

cp -R $PYTORCH_DIR/build_android/install/lib $OUT_DIR/
cp -R $PYTORCH_DIR/build_android/install/include $OUT_DIR/

echo "$abi build output lib,include copied to $OUT_DIR"

LIB_LINK_PATH=$LIB_DIR/$abi
INCLUDE_LINK_PATH=$INCLUDE_DIR/$abi

rm -f $LIB_LINK_PATH
rm -f $INCLUDE_LINK_PATH

ln -s $OUT_DIR/lib $LIB_LINK_PATH
ln -s $OUT_DIR/include $INCLUDE_LINK_PATH

done
check_android_sdk
check_gradle
parse_abis_list "$@"
build_android

# To set proxy for gradle add following lines to ./gradle/gradle.properties:
# systemProp.http.proxyHost=...
Expand All @@ -95,5 +28,3 @@ fi
find $PYTORCH_ANDROID_DIR -type f -name *apk

find $PYTORCH_ANDROID_DIR -type f -name *apk | xargs echo "To install apk run: $ANDROID_HOME/platform-tools/adb install -r "

popd
81 changes: 81 additions & 0 deletions android/common.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#!/bin/bash
set -eux

##############################################################################
# Common util functions for Android build scripts.
##############################################################################

if [ -z "$PYTORCH_DIR" ]; then
echo "PYTORCH_DIR not set!"
exit 1
fi

check_android_sdk() {
if [ -z "$ANDROID_HOME" ]; then
echo "ANDROID_HOME not set; please set it to Android sdk directory"
exit 1
fi

if [ ! -d "$ANDROID_HOME" ]; then
echo "ANDROID_HOME not a directory; did you install it under $ANDROID_HOME?"
exit 1
fi
echo "ANDROID_HOME:$ANDROID_HOME"
}

check_gradle() {
GRADLE_PATH=gradle
GRADLE_NOT_FOUND_MSG="Unable to find gradle, please add it to PATH or set GRADLE_HOME"

if [ ! -x "$(command -v gradle)" ]; then
if [ -z "$GRADLE_HOME" ]; then
echo "$GRADLE_NOT_FOUND_MSG"
exit 1
fi
GRADLE_PATH=$GRADLE_HOME/bin/gradle
if [ ! -f "$GRADLE_PATH" ]; then
echo "$GRADLE_NOT_FOUND_MSG"
exit 1
fi
fi
echo "GRADLE_PATH:$GRADLE_PATH"
}

parse_abis_list() {
ABIS_LIST="armeabi-v7a,arm64-v8a,x86,x86_64"
CUSTOM_ABIS_LIST=false
if [ $# -gt 0 ]; then
ABIS_LIST=$1
CUSTOM_ABIS_LIST=true
fi

echo "ABIS_LIST:$ABIS_LIST"
echo "CUSTOM_ABIS_LIST:$CUSTOM_ABIS_LIST"
}

build_android() {
PYTORCH_ANDROID_DIR="$PYTORCH_DIR/android"
BUILD_ROOT="${BUILD_ROOT:-$PYTORCH_DIR}"
echo "BUILD_ROOT:$BUILD_ROOT"

LIB_DIR="$PYTORCH_ANDROID_DIR/pytorch_android/src/main/jniLibs"
INCLUDE_DIR="$PYTORCH_ANDROID_DIR/pytorch_android/src/main/cpp/libtorch_include"

# These directories only contain symbolic links.
rm -rf "$LIB_DIR" && mkdir -p "$LIB_DIR"
rm -rf "$INCLUDE_DIR" && mkdir -p "$INCLUDE_DIR"

for abi in $(echo "$ABIS_LIST" | tr ',' '\n')
do
echo "abi:$abi"
ANDROID_BUILD_ROOT="$BUILD_ROOT/build_android_$abi"
ANDROID_ABI="$abi" \
BUILD_ROOT="$ANDROID_BUILD_ROOT" \
"$PYTORCH_DIR/scripts/build_android.sh" \
-DANDROID_CCACHE="$(which ccache)"

echo "$abi build output lib,include at $ANDROID_BUILD_ROOT/install"
ln -s "$ANDROID_BUILD_ROOT/install/lib" "$LIB_DIR/$abi"
ln -s "$ANDROID_BUILD_ROOT/install/include" "$INCLUDE_DIR/$abi"
done
}
40 changes: 4 additions & 36 deletions android/run_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,42 +4,10 @@ set -eux
PYTORCH_DIR="$(cd $(dirname $0)/..; pwd -P)"
PYTORCH_ANDROID_DIR=$PYTORCH_DIR/android

echo "ANDROID_HOME:$ANDROID_HOME"
if [ ! -z "$ANDROID_HOME" ]; then
echo "ANDROID_HOME not set; please set it to Android sdk directory"
fi

if [ ! -d $ANDROID_HOME ]; then
echo "ANDROID_HOME not a directory; did you install it under $ANDROID_HOME?"
exit 1
fi

echo "ANDROID_NDK:$ANDROID_NDK"
if [ ! -z "$ANDROID_NDK" ]; then
echo "ANDROID_NDK not set; please set it to Android sdk directory"
fi

if [ ! -d $ANDROID_NDK ]; then
echo "ANDROID_NDK not a directory; did you install it under $ANDROID_NDK?"
exit 1
fi

GRADLE_PATH=gradle
GRADLE_NOT_FOUND_MSG="Unable to find gradle, please add it to PATH or set GRADLE_HOME"

if [ ! -x "$(command -v gradle)" ]; then
if [ -z "$GRADLE_HOME" ]; then
echo GRADLE_NOT_FOUND_MSG
exit 1
fi
GRADLE_PATH=$GRADLE_HOME/bin/gradle
if [ ! -f "$GRADLE_PATH" ]; then
echo GRADLE_NOT_FOUND_MSG
exit 1
fi
fi
echo "GRADLE_PATH:$GRADLE_PATH"
source "$PYTORCH_ANDROID_DIR/common.sh"

check_android_sdk
check_gradle

# Run android instrumented tests on x86 emulator

Expand Down Expand Up @@ -70,7 +38,7 @@ cat <<- EOF
$ANDROID_HOME/platform-tools/adb devices
If everything is ok the output will be:
List of devices attached
emulator-5554 device
EOF
Expand Down
88 changes: 16 additions & 72 deletions scripts/build_pytorch_android.sh
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,17 @@
#!/bin/bash
set -eux

##############################################################################
# Master script to build PyTorch Android library with Java bindings.
##############################################################################
# Example usage:
# - Build default AARs:
# scipts/build_pytorch_androis.sh
#
# - Build for specific ABI(s):
# scipts/build_pytorch_androis.sh armeabi-v7a
# scipts/build_pytorch_androis.sh arm64-v8a,x86,x86_64
#
# Script's workflow:
# 1. Builds libtorch for android for specified android abisi (by default for all 4).
# Custom list of android abis can be specified as a bash argument as comma separated list.
Expand All @@ -12,82 +23,16 @@ set -eux
# gradle assembleRelease

PYTORCH_DIR="$(cd $(dirname $0)/..; pwd -P)"

PYTORCH_ANDROID_DIR=$PYTORCH_DIR/android
WORK_DIR=$PYTORCH_DIR

echo "PYTORCH_DIR:$PYTORCH_DIR"
echo "WORK_DIR:$WORK_DIR"

echo "ANDROID_HOME:$ANDROID_HOME"
if [ -z "$ANDROID_HOME" ]; then
echo "ANDROID_HOME not set; please set it to Android sdk directory"
fi

if [ ! -d $ANDROID_HOME ]; then
echo "ANDROID_HOME not a directory; did you install it under $ANDROID_HOME?"
exit 1
fi

GRADLE_PATH=gradle
GRADLE_NOT_FOUND_MSG="Unable to find gradle, please add it to PATH or set GRADLE_HOME"

if [ ! -x "$(command -v gradle)" ]; then
if [ -z "$GRADLE_HOME" ]; then
echo GRADLE_NOT_FOUND_MSG
exit 1
fi
GRADLE_PATH=$GRADLE_HOME/bin/gradle
if [ ! -f "$GRADLE_PATH" ]; then
echo GRADLE_NOT_FOUND_MSG
exit 1
fi
fi
echo "GRADLE_PATH:$GRADLE_PATH"

ABIS_LIST="armeabi-v7a,arm64-v8a,x86,x86_64"
CUSTOM_ABIS_LIST=false
if [ $# -gt 0 ]; then
ABIS_LIST=$1
CUSTOM_ABIS_LIST=true
fi

echo "ABIS_LIST:$ABIS_LIST"

LIB_DIR=$PYTORCH_ANDROID_DIR/pytorch_android/src/main/jniLibs
INCLUDE_DIR=$PYTORCH_ANDROID_DIR/pytorch_android/src/main/cpp/libtorch_include
mkdir -p $LIB_DIR
mkdir -p $INCLUDE_DIR

for abi in $(echo $ABIS_LIST | tr ',' '\n')
do
echo "abi:$abi"

OUT_DIR=$WORK_DIR/build_android_$abi

rm -rf $OUT_DIR
mkdir -p $OUT_DIR

pushd $PYTORCH_DIR
python $PYTORCH_DIR/setup.py clean

ANDROID_ABI=$abi $PYTORCH_DIR/scripts/build_android.sh -DANDROID_CCACHE=$(which ccache)

cp -R $PYTORCH_DIR/build_android/install/lib $OUT_DIR/
cp -R $PYTORCH_DIR/build_android/install/include $OUT_DIR/

echo "$abi build output lib,include copied to $OUT_DIR"

LIB_LINK_PATH=$LIB_DIR/$abi
INCLUDE_LINK_PATH=$INCLUDE_DIR/$abi

rm -f $LIB_LINK_PATH
rm -f $INCLUDE_LINK_PATH

ln -s $OUT_DIR/lib $LIB_LINK_PATH
ln -s $OUT_DIR/include $INCLUDE_LINK_PATH
source "$PYTORCH_ANDROID_DIR/common.sh"

done
check_android_sdk
check_gradle
parse_abis_list "$@"
build_android

# To set proxy for gradle add following lines to ./gradle/gradle.properties:
# systemProp.http.proxyHost=...
Expand All @@ -102,4 +47,3 @@ else
fi

find $PYTORCH_ANDROID_DIR -type f -name *aar | xargs ls -lah
popd

0 comments on commit bcb4479

Please sign in to comment.