Skip to content

Commit

Permalink
Adding compressed harvesting (#370)
Browse files Browse the repository at this point in the history
Signed-off-by: wallentx <[email protected]>
Co-authored-by: Author: Florin Chirica <[email protected]>
Co-authored-by: Harold Brenes <[email protected]>
Co-authored-by: Amine Khaldi <[email protected]>
Co-authored-by: Kyle Altendorf <[email protected]>
Co-authored-by: Chris Marslender <[email protected]>
Co-authored-by: arvidn <[email protected]>
Co-authored-by: Phill Walker <[email protected]>
  • Loading branch information
8 people authored Aug 5, 2023
1 parent 2309cbd commit d963785
Show file tree
Hide file tree
Showing 8 changed files with 820 additions and 92 deletions.
93 changes: 93 additions & 0 deletions .github/actions/fetch_bladebit_harvester.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
#!/usr/bin/env bash
set -eo pipefail
_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
cd "$_dir/../.."

##
# Usage: fetch_bladebit_harvester.sh <linux|macos|windows> <arm64|x86-64>
#
# Use gitbash or similar under Windows.
##
host_os=$1
host_arch=$2

if [[ "${host_os}" != "linux" ]] && [[ "${host_os}" != "macos" ]] && [[ "${host_os}" != "windows" ]]; then
echo >&2 "Unkonwn OS '${host_os}'"
exit 1
fi

if [[ "${host_arch}" != "arm64" ]] && [[ "${host_arch}" != "x86-64" ]]; then
echo >&2 "Unkonwn Architecture '${host_arch}'"
exit 1
fi

## Change this if including a new bladebit release
artifact_ver="v3.0.0"
artifact_base_url="https://github.com/Chia-Network/bladebit/releases/download/v3.0.0"

linux_arm_sha256="5a53d82c2cc22172bfa2267ea9fb53126a527eba7bafd03ddf5503913a61f70c"
linux_x86_sha256="3cdbcf127126d7c61f6da715b25ef73a8420778dd34d56e82ed1865d7a1ebfeb"
macos_arm_sha256="325150951e83be4ee8690be996e6fde0776ff4cca89e39111c97f0aae3f93bf3"
macos_x86_sha256="718ab50a19ea3a8f064f2d09df38720cbb7d89667b599f57f2bca3cdf41c18e9"
windows_sha256="f3e14d02daafaa8e3666ab4666220d3b2859b1c10254bddb38a69da83cc899c5"
## End changes

artifact_ext="tar.gz"
sha_bin="sha256sum"
expected_sha256=

if [[ "$OSTYPE" == "darwin"* ]]; then
sha_bin="shasum -a 256"
fi

case "${host_os}" in
linux)
if [[ "${host_arch}" == "arm64" ]]; then
expected_sha256=$linux_arm_sha256
else
expected_sha256=$linux_x86_sha256
fi
;;
macos)
if [[ "${host_arch}" == "arm64" ]]; then
expected_sha256=$macos_arm_sha256
else
expected_sha256=$macos_x86_sha256
fi
;;
windows)
expected_sha256=$windows_sha256
artifact_ext="zip"
;;
*)
echo >&2 "Unexpected OS '${host_os}'"
exit 1
;;
esac

# Download artifact
artifact_name="green_reaper.${artifact_ext}"
curl -L "${artifact_base_url}/green_reaper-${artifact_ver}-${host_os}-${host_arch}.${artifact_ext}" >"${artifact_name}"

# Validate sha256, if one was given
if [ -n "${expected_sha256}" ]; then
gr_sha256="$(${sha_bin} ${artifact_name} | cut -d' ' -f1)"

if [[ "${gr_sha256}" != "${expected_sha256}" ]]; then
echo >&2 "GreenReaper SHA256 mismatch!"
echo >&2 " Got : '${gr_sha256}'"
echo >&2 " Expected: '${expected_sha256}'"
exit 1
fi
fi

# Unpack artifact
dst_dir="libs/green_reaper"
mkdir -p "${dst_dir}"
if [[ "${artifact_ext}" == "zip" ]]; then
unzip -d "${dst_dir}" "${artifact_name}"
else
pushd "${dst_dir}"
tar -xzvf "../../${artifact_name}"
popd
fi
87 changes: 58 additions & 29 deletions .github/workflows/build-wheels.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ on:
push:
branches:
- main
tags:
- '**'
release:
types: [published]
pull_request:
branches:
- '**'
Expand All @@ -19,6 +19,7 @@ jobs:
build-wheels:
name: Wheel - ${{ matrix.os.name }} ${{ matrix.python.major-dot-minor }} ${{ matrix.arch.name }}
runs-on: ${{ matrix.os.runs-on[matrix.arch.matrix] }}
continue-on-error: true
strategy:
fail-fast: false
matrix:
Expand All @@ -41,29 +42,23 @@ jobs:
runs-on:
intel: [windows-latest]
python:
- major-dot-minor: '3.7'
cibw-build: 'cp37-*'
manylinux:
arm: manylinux2014
intel: manylinux2010
matrix: '3.7'
- major-dot-minor: '3.8'
cibw-build: 'cp38-*'
manylinux:
arm: manylinux2014
intel: manylinux2010
intel: manylinux2014
matrix: '3.8'
- major-dot-minor: '3.9'
cibw-build: 'cp39-*'
manylinux:
arm: manylinux2014
intel: manylinux2010
intel: manylinux2014
matrix: '3.9'
- major-dot-minor: '3.10'
cibw-build: 'cp310-*'
manylinux:
arm: manylinux2014
intel: manylinux2010
intel: manylinux2014
matrix: '3.10'
- major-dot-minor: '3.11'
cibw-build: 'cp311-*'
Expand Down Expand Up @@ -96,9 +91,9 @@ jobs:
arm: [macOS, ARM64]
intel: [macos-latest]
python:
major-dot-minor: '3.7'
cibw-build: 'cp37-*'
matrix: '3.7'
major-dot-minor: '3.8'
cibw-build: 'cp38-*'
matrix: '3.8'
arch:
name: ARM
matrix: arm
Expand All @@ -112,6 +107,12 @@ jobs:
with:
fetch-depth: 0

- name: Set Env
if: env.RUNNER_ARCH != 'ARM64'
uses: Chia-Network/actions/setjobenv@main
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- uses: Chia-Network/actions/setup-python@main
with:
python-version: ${{ matrix.python.major-dot-minor }}
Expand All @@ -120,32 +121,55 @@ jobs:
run: |
pip install pipx
- name: Get Windows Bladebit Harvester Artifact
if: runner.os == 'Windows'
shell: bash
run: |
set -eo pipefail
set -x
.github/actions/fetch_bladebit_harvester.sh windows x86-64
- name: Build and test
env:
CIBW_PRERELEASE_PYTHONS: True
CIBW_BUILD_VERBOSITY_MACOS: 0
CIBW_BUILD_VERBOSITY_LINUX: 0
CIBW_BUILD_VERBOSITY_WINDOWS: 0
CIBW_BUILD_VERBOSITY_MACOS: 1
CIBW_BUILD_VERBOSITY_LINUX: 1
CIBW_BUILD_VERBOSITY_WINDOWS: 1
CIBW_BUILD: ${{ matrix.python.cibw-build }}
CIBW_SKIP: '*-manylinux_i686 *-win32 *-musllinux_*'
CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.python.manylinux['arm'] }}
CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.python.manylinux['intel'] }}
CIBW_ENVIRONMENT_LINUX: "PATH=/project/cmake-3.17.3-Linux-`uname -m`/bin:$PATH"
CIBW_BEFORE_ALL_LINUX: >
curl -L https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-`uname -m`.sh > cmake.sh
&& yes | sh cmake.sh | cat
&& rm -f /usr/bin/cmake
&& which cmake
&& cmake --version
&& uname -a
CIBW_ENVIRONMENT_WINDOWS: "CP_USE_GREEN_REAPER=1"
CIBW_ENVIRONMENT_LINUX: CP_USE_GREEN_REAPER="1"
CIBW_BEFORE_ALL_LINUX: |
set -eo pipefail
set -x
# Get bladebit harvester
set -eo pipefail
ARCH=$(uname -m)
if [[ $ARCH == x86_64 ]]; then
.github/actions/fetch_bladebit_harvester.sh linux x86-64
else
.github/actions/fetch_bladebit_harvester.sh linux arm64
fi
CIBW_BEFORE_BUILD_LINUX: >
python -m pip install --upgrade pip
CIBW_ARCHS_MACOS: ${{ matrix.os.cibw-archs-macos[matrix.arch.matrix] }}
CIBW_BEFORE_ALL_MACOS: >
CIBW_BEFORE_ALL_MACOS: |
brew install gmp boost cmake
# Get bladebit harvester
set -eo pipefail
ARCH=$(uname -m)
if [[ $ARCH == x86_64 ]]; then
.github/actions/fetch_bladebit_harvester.sh macos x86-64
else
.github/actions/fetch_bladebit_harvester.sh macos arm64
fi
CIBW_BEFORE_BUILD_MACOS: >
python -m pip install --upgrade pip
CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=10.14"
CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=10.14 CP_USE_GREEN_REAPER=1"
CIBW_TEST_REQUIRES: pytest
CIBW_TEST_COMMAND: py.test -v {project}/tests
run:
Expand Down Expand Up @@ -274,6 +298,11 @@ jobs:
with:
fetch-depth: 0

- name: Set Env
uses: Chia-Network/actions/setjobenv@main
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- uses: Chia-Network/actions/setup-python@main
with:
python-version: ${{ matrix.python.major-dot-minor }}
Expand All @@ -298,15 +327,15 @@ jobs:
run: pip install twine

- name: Publish distribution to PyPI
if: startsWith(github.event.ref, 'refs/tags') && steps.check_secrets.outputs.HAS_SECRET
if: env.RELEASE == 'true' && steps.check_secrets.outputs.HAS_SECRET
env:
TWINE_USERNAME: __token__
TWINE_NON_INTERACTIVE: 1
TWINE_PASSWORD: ${{ secrets.pypi_password }}
run: twine upload --non-interactive --skip-existing --verbose 'dist/*'

- name: Publish distribution to Test PyPI
if: steps.check_secrets.outputs.HAS_SECRET
if: env.PRE_RELEASE == 'true' && steps.check_secrets.outputs.HAS_SECRET
env:
TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
TWINE_USERNAME: __token__
Expand Down
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,9 @@ build
.mypy_cache
*.whl
venv
build-tsan
build-*
cmake-build*
*.zip
*.tar.gz
libs/
Loading

0 comments on commit d963785

Please sign in to comment.