From f0b3f61ea7cd88b5e31cf912d7a2df55bd3ccee9 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 9 Oct 2017 07:56:11 +0300 Subject: [PATCH 1/6] Build script for RPM/DEB packages --- build.sh | 241 +++++++++++++++++++++++++++++ resources/freno.conf.skeleton.json | 15 ++ 2 files changed, 256 insertions(+) create mode 100755 build.sh create mode 100644 resources/freno.conf.skeleton.json diff --git a/build.sh b/build.sh new file mode 100755 index 00000000..29d0ced2 --- /dev/null +++ b/build.sh @@ -0,0 +1,241 @@ +#!/bin/bash + +# Simple packaging of freno +# +# Requires fpm: https://github.com/jordansissel/fpm +# + +# This script is based off https://github.com/github/orchestrator/blob/master/build.sh +# hence licensed under the Apache 2.0 license. + +# Copyright 2017 Shlomi Noach, GitHub Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +mydir=$(dirname $0) +GIT_COMMIT=$(git rev-parse HEAD) +RELEASE_VERSION= +RELEASE_SUBVERSION= +TOPDIR=/tmp/freno-release +export RELEASE_VERSION TOPDIR + +usage() { + echo + echo "Usage: $0 [-t target ] [-a arch ] [ -p prefix ] [-h] [-d] [-r]" + echo "Options:" + echo "-h Show this screen" + echo "-t (linux|darwin) Target OS Default:(linux)" + echo "-a (amd64|386) Arch Default:(amd64)" + echo "-d debug output" + echo "-b build only, do not generate packages" + echo "-p build prefix Default:(/usr/bin)" + echo "-r build with race detector" + echo "-v release version (optional; default: content of RELEASE_VERSION file)" + echo "-s release subversion (optional; default: empty)" + echo +} + +function precheck() { + local target="$1" + local build_only="$2" + local ok=0 # return err. so shell exit code + + if [[ "$target" == "linux" ]]; then + if [[ $build_only -eq 0 ]] && [[ ! -x "$( which fpm )" ]]; then + echo "Please install fpm and ensure it is in PATH (typically: 'gem install fpm')" + ok=1 + fi + + if [[ $build_only -eq 0 ]] && [[ ! -x "$( which rpmbuild )" ]]; then + echo "rpmbuild not in PATH, rpm will not be built (OS/X: 'brew install rpm')" + fi + fi + + if [[ -z "$GOPATH" ]]; then + echo "GOPATH not set" + ok=1 + fi + + if [[ ! -x "$( which go )" ]]; then + echo "go binary not found in PATH" + ok=1 + fi + + if [[ $(go version | egrep "go1[.][0123456]") ]]; then + echo "go version is too low. Must use 1.7 or above" + ok=1 + fi + + return $ok +} + +function setuptree() { + local b prefix + prefix="$1" + + mkdir -p $TOPDIR + rm -rf ${TOPDIR:?}/* + b=$( mktemp -d $TOPDIR/frenoXXXXXX ) || return 1 + mkdir -p $b/freno + mkdir -p $b/freno${prefix} + mkdir -p $b/freno/etc/init.d + mkdir -p $b/freno/var/lib + echo $b +} + +function oinstall() { + local builddir prefix + builddir="$1" + prefix="$2" + + cd $mydir + gofmt -s -w go/ + cp ./resources/etc/init.d/freno $builddir/freno/etc/init.d/freno + cp ./resources/freno.conf.skeleton.json $builddir/freno/etc/freno.conf.json + chmod +x $builddir/freno/etc/init.d/freno +} + +function package() { + local target builddir prefix packages + target="$1" + builddir="$2" + prefix="$3" + + cd $TOPDIR + + echo "Release version is ${RELEASE_VERSION} ( ${GIT_COMMIT} )" + + case $target in + 'linux') + echo "Creating Linux Tar package" + tar -C $builddir/freno -czf $TOPDIR/freno-"${RELEASE_VERSION}"-$target-$arch.tar.gz ./ + + echo "Creating Distro full packages" + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n freno -m shlomi-noach --description "Cooperative, highly available throttler service" --url "https://github.com/github/freno" --vendor "GitHub" --license "Apache 2.0" -C $builddir/freno --prefix=/ -t rpm . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n freno -m shlomi-noach --description "Cooperative, highly available throttler service" --url "https://github.com/github/freno" --vendor "GitHub" --license "Apache 2.0" -C $builddir/freno --prefix=/ -t deb --deb-no-default-config-files . + ;; + 'darwin') + echo "Creating Darwin full Package" + tar -C $builddir/freno -czf $TOPDIR/freno-"${RELEASE_VERSION}"-$target-$arch.tar.gz ./ + ;; + esac + + echo "---" + echo "Done. Find releases in $TOPDIR" +} + +function build() { + local target arch builddir gobuild prefix + os="$1" + arch="$2" + builddir="$3" + prefix="$4" + ldflags="-X main.AppVersion=${RELEASE_VERSION} -X main.GitCommit=${GIT_COMMIT}" + echo "Building via $(go version)" + gobuild="go build -i ${opt_race} -ldflags \"$ldflags\" -o $builddir/freno${prefix}/freno/freno go/cmd/freno/main.go" + + case $os in + 'linux') + echo "GOOS=$os GOARCH=$arch $gobuild" | bash + ;; + 'darwin') + echo "GOOS=darwin GOARCH=amd64 $gobuild" | bash + ;; + esac + [[ $(find $builddir/freno${prefix}/freno/ -type f -name freno) ]] && echo "freno binary created" || (echo "Failed to generate freno binary" ; exit 1) + cp $builddir/freno${prefix}/freno/freno $builddir/freno-cli/usr/bin && echo "binary copied to freno-cli" || (echo "Failed to copy freno binary to freno-cli" ; exit 1) + cp $builddir/freno${prefix}/freno/resources/bin/freno-client $builddir/freno-client/usr/bin && echo "freno-client copied to freno-client/" || (echo "Failed to copy freno-client to freno-client/" ; exit 1) +} + +function main() { + local target="$1" + local arch="$2" + local prefix="$3" + local build_only=$4 + local builddir + + if [ -z "${RELEASE_VERSION}" ] ; then + RELEASE_VERSION=$(cat $mydir/RELEASE_VERSION) + fi + RELEASE_VERSION="${RELEASE_VERSION}${RELEASE_SUBVERSION}" + + precheck "$target" "$build_only" + builddir=$( setuptree "$prefix" ) + oinstall "$builddir" "$prefix" + build "$target" "$arch" "$builddir" "$prefix" + [[ $? == 0 ]] || return 1 + if [[ $build_only -eq 0 ]]; then + package "$target" "$builddir" "$prefix" + fi +} + +build_only=0 +opt_race= +while getopts a:t:p:s:v:dbhr flag; do + case $flag in + a) + arch="${OPTARG}" + ;; + t) + target="${OPTARG}" + ;; + h) + usage + exit 0 + ;; + d) + debug=1 + ;; + b) + echo "Build only; no packaging" + build_only=1 + ;; + p) + prefix="${OPTARG}" + ;; + r) + opt_race="-race" + ;; + v) + RELEASE_VERSION="${OPTARG}" + ;; + s) + RELEASE_SUBVERSION="_${OPTARG}" + ;; + ?) + usage + exit 2 + ;; + esac +done + +shift $(( OPTIND - 1 )); + +if [ -z "$target" ]; then + uname=$(uname) + case $uname in + Linux) target=linux;; + Darwin) target=darwin;; + *) echo "Unexpected OS from uname: $uname. Exiting" + exit 1 + esac +fi +arch=${arch:-"amd64"} # default for arch is amd64 but should take from environment +prefix=${prefix:-"/usr/bin"} + +[[ $debug -eq 1 ]] && set -x +main "$target" "$arch" "$prefix" "$build_only" + +echo "freno build done; exit status is $?" diff --git a/resources/freno.conf.skeleton.json b/resources/freno.conf.skeleton.json new file mode 100644 index 00000000..672cf0e4 --- /dev/null +++ b/resources/freno.conf.skeleton.json @@ -0,0 +1,15 @@ +{ + "ListenPort": 9777, + "DefaultRaftPort": "9888", + "RaftDataDir": "/var/lib/freno", + "RaftBind": "", + "RaftNodes": ["", "", ""], + "MemcacheServers": [], + "Stores": { + "MySQL": { + "ThrottleThreshold": 1.0, + "Clusters": { + } + } + } +} From 1942c93e167838be7f5223dbfe40df280691bcfb Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 9 Oct 2017 08:17:15 +0300 Subject: [PATCH 2/6] release version by tag --- build.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/build.sh b/build.sh index 29d0ced2..1363d4af 100755 --- a/build.sh +++ b/build.sh @@ -27,7 +27,6 @@ set -e mydir=$(dirname $0) GIT_COMMIT=$(git rev-parse HEAD) RELEASE_VERSION= -RELEASE_SUBVERSION= TOPDIR=/tmp/freno-release export RELEASE_VERSION TOPDIR @@ -167,9 +166,8 @@ function main() { local builddir if [ -z "${RELEASE_VERSION}" ] ; then - RELEASE_VERSION=$(cat $mydir/RELEASE_VERSION) + RELEASE_VERSION=$(git describe --abbrev=0 --tags) fi - RELEASE_VERSION="${RELEASE_VERSION}${RELEASE_SUBVERSION}" precheck "$target" "$build_only" builddir=$( setuptree "$prefix" ) @@ -212,7 +210,6 @@ while getopts a:t:p:s:v:dbhr flag; do RELEASE_VERSION="${OPTARG}" ;; s) - RELEASE_SUBVERSION="_${OPTARG}" ;; ?) usage From c36b938fe53c21ac2153c4e01bb09b14cba36c50 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 9 Oct 2017 08:20:58 +0300 Subject: [PATCH 3/6] removed unrelated build steps --- build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.sh b/build.sh index 1363d4af..c1ad0185 100755 --- a/build.sh +++ b/build.sh @@ -154,8 +154,6 @@ function build() { ;; esac [[ $(find $builddir/freno${prefix}/freno/ -type f -name freno) ]] && echo "freno binary created" || (echo "Failed to generate freno binary" ; exit 1) - cp $builddir/freno${prefix}/freno/freno $builddir/freno-cli/usr/bin && echo "binary copied to freno-cli" || (echo "Failed to copy freno binary to freno-cli" ; exit 1) - cp $builddir/freno${prefix}/freno/resources/bin/freno-client $builddir/freno-client/usr/bin && echo "freno-client copied to freno-client/" || (echo "Failed to copy freno-client to freno-client/" ; exit 1) } function main() { From 311d11f19c4d254ad865142b869b900ef292f01a Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 9 Oct 2017 08:24:02 +0300 Subject: [PATCH 4/6] fixing freno binary path --- build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index c1ad0185..7aa3b37c 100755 --- a/build.sh +++ b/build.sh @@ -102,8 +102,8 @@ function oinstall() { cd $mydir gofmt -s -w go/ cp ./resources/etc/init.d/freno $builddir/freno/etc/init.d/freno - cp ./resources/freno.conf.skeleton.json $builddir/freno/etc/freno.conf.json chmod +x $builddir/freno/etc/init.d/freno + cp ./resources/freno.conf.skeleton.json $builddir/freno/etc/freno.conf.json } function package() { @@ -143,7 +143,7 @@ function build() { prefix="$4" ldflags="-X main.AppVersion=${RELEASE_VERSION} -X main.GitCommit=${GIT_COMMIT}" echo "Building via $(go version)" - gobuild="go build -i ${opt_race} -ldflags \"$ldflags\" -o $builddir/freno${prefix}/freno/freno go/cmd/freno/main.go" + gobuild="go build -i ${opt_race} -ldflags \"$ldflags\" -o $builddir/freno${prefix}/freno go/cmd/freno/main.go" case $os in 'linux') @@ -153,7 +153,7 @@ function build() { echo "GOOS=darwin GOARCH=amd64 $gobuild" | bash ;; esac - [[ $(find $builddir/freno${prefix}/freno/ -type f -name freno) ]] && echo "freno binary created" || (echo "Failed to generate freno binary" ; exit 1) + [[ $(find $builddir/freno${prefix}/ -type f -name freno) ]] && echo "freno binary created" || (echo "Failed to generate freno binary" ; exit 1) } function main() { From d6b43528a23bc78b9275679144c3a7ed7b549d88 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 9 Oct 2017 08:30:54 +0300 Subject: [PATCH 5/6] sanitizing verion number --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 7aa3b37c..55e6a8d1 100755 --- a/build.sh +++ b/build.sh @@ -164,7 +164,7 @@ function main() { local builddir if [ -z "${RELEASE_VERSION}" ] ; then - RELEASE_VERSION=$(git describe --abbrev=0 --tags) + RELEASE_VERSION=$(git describe --abbrev=0 --tags | tr -d 'v') fi precheck "$target" "$build_only" From 17ea9e04944ca502ec0ae8d6caa849e223af110c Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 9 Oct 2017 08:34:02 +0300 Subject: [PATCH 6/6] fixed datatype of DefaultRaftPort --- doc/high-availability.md | 2 +- resources/freno.conf.sample.json | 2 +- resources/freno.conf.skeleton.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/high-availability.md b/doc/high-availability.md index da487c02..bb635065 100644 --- a/doc/high-availability.md +++ b/doc/high-availability.md @@ -34,7 +34,7 @@ Let's dissect the general section of the [sample config file](../resources/freno ``` { "ListenPort": 9777, - "DefaultRaftPort": "9888", + "DefaultRaftPort": 9888, "RaftDataDir": "/var/lib/freno", "RaftBind": "10.0.0.1", "RaftNodes": ["10.0.0.1", "10.0.0.2", "10.0.0.3"] diff --git a/resources/freno.conf.sample.json b/resources/freno.conf.sample.json index 351f4834..29f6749e 100644 --- a/resources/freno.conf.sample.json +++ b/resources/freno.conf.sample.json @@ -1,6 +1,6 @@ { "ListenPort": 9777, - "DefaultRaftPort": "9888", + "DefaultRaftPort": 9888, "RaftDataDir": "/var/lib/freno", "RaftBind": "10.0.0.1", "RaftNodes": ["10.0.0.1", "10.0.0.2", "10.0.0.3"], diff --git a/resources/freno.conf.skeleton.json b/resources/freno.conf.skeleton.json index 672cf0e4..9d1f1614 100644 --- a/resources/freno.conf.skeleton.json +++ b/resources/freno.conf.skeleton.json @@ -1,6 +1,6 @@ { "ListenPort": 9777, - "DefaultRaftPort": "9888", + "DefaultRaftPort": 9888, "RaftDataDir": "/var/lib/freno", "RaftBind": "", "RaftNodes": ["", "", ""],