Skip to content

Enables model ensembles #318

Enables model ensembles

Enables model ensembles #318

Workflow file for this run

name: "Build"
'on':
push:
branches:
- main
- ci-sandbox
tags:
- "v*.*.*"
pull_request:
branches:
- '**'
env:
qt_version: "6.2.1" # only used by build-macos
emsdk_version: 3.1.8 # For use in emscripten build
ccache_basedir: ${{ github.workspace }}
ccache_dir: "${{ github.workspace }}/.ccache"
ccache_compilercheck: content
ccache_compress: 'true'
ccache_compresslevel: 9
ccache_maxsize: 200M
ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache
jobs:
build-wheels:
strategy:
matrix:
os: [ubuntu-20.04, macos-10.15]
fail-fast: false
name: "cibuildwheel / ${{ matrix.os }}"
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- name: Generate ccache_vars for ccache based on machine
shell: bash
id: ccache_vars
run: |-
echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})"
echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')"
- name: Cache-op for build-cache through ccache
uses: actions/cache@v2
with:
path: ${{ env.ccache_dir }}
key: ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }}
restore-keys: |-
ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}
ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}
ccache-cibuildwheel-${{ matrix.os }}
- name: ccache environment setup
run: |-
mkdir -p ${{ env.ccache_dir }}
- name: Inject local version identifier for non tag builds
if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
run: |-
echo "PYTHON_LOCAL_VERSION_IDENTIFIER=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
- name: Apply MacOS patch
if: ${{ startsWith(runner.os, 'mac') }}
run: |
patch -p1 < patches/01-marian-fstream-for-macos.patch
- name: Build wheels
uses: pypa/[email protected]
# to supply options, put them in 'env', like:
env:
CIBW_ENVIRONMENT_LINUX:
BUILD_ARCH=core-avx-i
USE_CCACHE=1
CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}
CCACHE_COMPRESS=${{ env.ccache_compress }}
CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}
CCACHE_MAXSIZE=${{ env.ccache_maxsize }}
PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }}
CCACHE_DIR=/host/${{ env.ccache_dir }}
CCACHE_BASEDIR=/host/${{ env.ccache_basedir }}
CIBW_ENVIRONMENT_MACOS:
BUILD_ARCH=core-avx-i
USE_CCACHE=1
CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}
CCACHE_COMPRESS=${{ env.ccache_compress }}
CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}
CCACHE_MAXSIZE=${{ env.ccache_maxsize }}
PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }}
CCACHE_DIR=${{ env.ccache_dir }}
CCACHE_BASEDIR=${{ env.ccache_basedir }}
MACOSX_DEPLOYMENT_TARGET=10.9
CIBW_BEFORE_BUILD_LINUX: |
yum install -y ccache
# Install Intel MKL.
yum-config-manager -y --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo
yum install -y intel-mkl
chmod -R a+rwx /host/${{ env.ccache_dir }}
ccache -s # Print current cache stats
ccache -z # Zero cache entry
CIBW_BEFORE_BUILD_MACOS: |
brew install openblas protobuf ccache boost pybind11
chmod -R a+rwx ${{ env.ccache_dir }}
ccache -s # Print current cache stats
ccache -z # Zero cache entry
CIBW_BUILD: "cp{36,37,38,39,310}-*manylinux_x86_64 cp{36,37,38,39,310}-macosx_x86_64"
CIBW_BEFORE_TEST: |
ccache -s # Print current ccache stats
CIBW_TEST_COMMAND: |
# The wheels are installed automatically and available.
# Fetch models from translateLocally repository.
python3 -m bergamot download -m en-de-tiny
python3 -m bergamot download -m de-en-tiny
python3 -m bergamot ls
# Fetch models from opus repository.
python3 -m bergamot download -m eng-fin-tiny -r opus
python3 -m bergamot ls -r opus
# Run the sample python script shipped with module
python3 -m bergamot translate --model en-de-tiny <<< "Hello World"
python3 -m bergamot translate --model en-de-tiny de-en-tiny <<< "Hello World"
python3 -m bergamot translate --model eng-fin-tiny --repository opus <<< "Hello World"
- uses: actions/upload-artifact@v2
with:
name: wheels
path: ./wheelhouse/*.whl
upload-wheels:
name: "Upload wheels to PyPI"
runs-on: ubuntu-latest
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
needs: [build-wheels]
steps:
- name: Download artifacts
uses: actions/download-artifact@v2
with:
name: wheels
- name: Publish wheels to PyPI
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
python3 -m pip install twine
twine upload *.whl
build-wasm:
name: "emscripten"
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
- name: Set ccache environment for emcc
run: |
# We are hardcoding this to mtime instead of env pickup. Rest use content.
echo "CCACHE_COMPILER_CHECK=mtime" >> $GITHUB_ENV
echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV
echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV
echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV
echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV
echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV
# https://emscripten.org/docs/compiling/Building-Projects.html#using-a-compiler-wrapper
echo "EM_COMPILER_WRAPPER=ccache" >> $GITHUB_ENV
# This need to be run before setup, so ccache build caching doesn't complain.
- name: Obtain emsdk sources
run: |
git clone --depth 1 https://github.com/emscripten-core/emsdk.git
- name: Cache-op for build-cache through ccache
uses: actions/cache@v2
with:
path: |
${{ env.ccache_dir }}
${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/
key: ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }}
restore-keys: |-
ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}
ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}
ccache-${{ github.job }}-${{ env.emsdk_version }}
- name: Setup Emscripten toolchain
run: |
(cd emsdk && ./emsdk install ${{ env.emsdk_version }} ccache-git-emscripten-64bit)
(cd emsdk && ./emsdk activate ${{ env.emsdk_version }} ccache-git-emscripten-64bit)
# mtime of this file is checked by ccache, we set it to avoid cache misses.
touch -m -d '1 Jan 2021 12:00' emsdk/.emscripten
# These needs to be done in the activated shell.
eval $(./emsdk/emsdk construct_env \
| sed 's/export PATH=\(.*\);/echo \1 >> $GITHUB_PATH;/' \
| sed 's/export \(.*\);/echo \1 >> $GITHUB_ENV;/' );
# This looks more permanent than version pinned, so keeping temporarily to avoid failures.
echo "${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/bin" >> $GITHUB_PATH
- name: Generate ccache_vars for ccache based on machine
shell: bash
id: ccache_vars
run: |-
echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})"
echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')"
- name: Verify Emscripten setup
run: |
emcc --version
emcmake cmake --version
emmake make --version
- name: ccache prolog
run: |-
ccache -s # Print current cache stats
ccache -z # Zero cache entry
- name: "Configure builds"
run: |
mkdir -p build-wasm
cd build-wasm
emcmake cmake -DCOMPILE_WASM=on -DWORMHOLE=off ..
- name: "Compile"
working-directory: build-wasm
run: |
emmake make -j2
- name: ccache epilog
run: |
ccache -s # Print current cache stats
- name: Import GEMM library from a separate wasm module
working-directory: build-wasm
run: bash ../wasm/patch-artifacts-import-gemm-module.sh
# Setup nodejs-18, as nodejs-14 provided by emsdk fails when running
# and newer version of node allows us to use fetch().
- name: Setup nodejs
uses: actions/setup-node@v3
with:
node-version: 18
- name: Test run
working-directory: wasm
run: |
cp ../build-wasm/bergamot-translator-worker.{js,wasm} ./
npm install jsdom
# --unhandled-rejections make the script exit with a non-zero code (at least on node-14).
# So leaving this here.
node --unhandled-rejections=strict node-test.js
# Upload both together.
- name: Upload wasm artifact
uses: actions/upload-artifact@v2
with:
name: wasm-artefacts
if-no-files-found: error
path: |
# Without wormhole
${{github.workspace}}/build-wasm/bergamot-translator-worker.js
${{github.workspace}}/build-wasm/bergamot-translator-worker.wasm
${{github.workspace}}/build-wasm/bergamot-translator-worker.js.bak
upload-wasm:
name: "Upload node package to NPM"
runs-on: ubuntu-latest
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
needs: [build-wasm]
steps:
- name: Download artifacts
uses: actions/download-artifact@v2
with:
name: wasm-artefacts
path: wasm/module/worker
- uses: actions/setup-node@v3
with:
node-version: '18.x'
registry-url: 'https://registry.npmjs.org'
- run: npm ci
- run: npm publish
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
# Try to upload a release using https://github.com/marvinpinto/actions/issues/177#issuecomment-917605585 as a model
release-latest:
name: Release Latest Build
runs-on: ubuntu-latest
needs: [build-wheels, build-wasm]
if: github.ref == 'refs/heads/main'
steps:
- name: Download artifacts
uses: actions/download-artifact@v2
# Leave the below be, it will be useful.
- name: List downloaded assets
run: |
find ./
- name: Update GitHub prerelease
uses: marvinpinto/action-automatic-releases@latest
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
automatic_release_tag: latest
prerelease: true
title: "Latest Build"
files: |
wheels/*.whl
wasm-artefacts/bergamot-translator-worker.js
wasm-artefacts/bergamot-translator-worker.wasm
release-version:
name: Release version
runs-on: ubuntu-latest
needs: [build-wheels, build-wasm]
permissions:
contents: "write"
packages: "write"
pull-requests: "read"
if: startsWith(github.ref, 'refs/tags/v')
steps:
- name: Download artifacts
uses: actions/download-artifact@v2
# Leave the below be, it will be useful.
- name: List downloaded assets
run: |
find ./
- name: Update GitHub release
uses: marvinpinto/action-automatic-releases@latest
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
automatic_release_tag: ${{ github.ref_name }}
prerelease: false
title: "${{ github.ref_name }}"
files: |
wheels/*.whl
wasm-artefacts/bergamot-translator-worker.js
wasm-artefacts/bergamot-translator-worker.wasm
python-checks:
name: "formatting and typechecks"
runs-on: "ubuntu-latest"
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
- name: Install Dependencies
run: |-
python3 -m pip install black isort pytype
- name: "Formatting checks: black, isort"
run: |
python3 -m black --diff --check bindings/python/ setup.py doc/conf.py
python3 -m isort --profile black --diff --check bindings/python setup.py doc/conf.py
- name: "Static typing checks: pytype"
run: |-
python3 -m pytype bindings/python
docs:
runs-on: ubuntu-18.04
needs: [build-wheels]
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: recursive
# Runs javascript to extract push events from both tags and branch (only main, due to workflow trigger)
# converts refs/<>/<name> -> <name>
# eg:
# refs/head/main -> main
# refs/tags/v0.1.0 -> v0.1.0
#
- name: Download artifacts
uses: actions/download-artifact@v2
- name: Extract tag name
id: tag
uses: actions/[email protected]
if: ${{ github.event_name == 'push' }}
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const args = context.payload.ref.split("/");
[refs, category, ...rest] = args;
return rest.join("/");
# Patches the BERGAMOT_VERSION file used by sphinx-docs at run time to
# obtain names like 'main' or 'ci-sandbox' to not confuse with version
# based documentation built separately.
- name: Deploy-time patch version
run: |
echo ${{steps.tag.outputs.result }} > BERGAMOT_VERSION
- name: Set up Doxygen
run: sudo apt-get install -y doxygen
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Set up dependency cache
uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('doc/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
working-directory: ./doc
run: |
python3 -m pip install -r requirements.txt
python3 -m pip install --find-links=${{github.workspace}}/wheels bergamot
- name: Build documentation
working-directory: ./doc
run: sphinx-build -b html ./ build/
- name: Deploy 🚀
uses: JamesIves/[email protected]
if: ${{ github.event_name == 'push' && github.repository == 'browsermt/bergamot-translator' }}
with:
repository-name: 'browsermt/docs'
branch: gh-pages # The branch the action should deploy to.
folder: './doc/build/' # The folder the action should deploy.
target-folder: '${{ steps.tag.outputs.result }}'
ssh-key: ${{ secrets.BERGAMOT_SSH_PRIVATE_KEY }}
# This artifact contains the HTML output of Sphinx only.
# With index.html at the root of the produced zip file.
# For use for maintainers to download the zip and check render of
# documentation while generated at pull-request.
- name: Upload documentation
uses: actions/upload-artifact@v2
if: ${{ github.event_name == 'pull_request'}}
with:
name: api-docs
path: ./doc/build/
if-no-files-found: error