-
Notifications
You must be signed in to change notification settings - Fork 645
172 lines (151 loc) · 5.73 KB
/
build_docs.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
name: Build documentation
on:
pull_request:
push:
branches:
- nightly
- main
- release/*
tags:
- v[0-9]+.[0-9]+.[0-9]
- v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+
workflow_dispatch:
jobs:
build:
strategy:
matrix:
python_version: ["3.10"]
cuda_arch_version: ["11.8"]
fail-fast: false
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
with:
job-name: Build doc
runner: linux.g5.4xlarge.nvidia.gpu
repository: pytorch/audio
gpu-arch-type: cuda
gpu-arch-version: ${{ matrix.cuda_arch_version }}
timeout: 120
upload-artifact: docs
script: |
# Mark Build Directory Safe
git config --global --add safe.directory /__w/audio/audio
# Set up Environment Variables
export PYTHON_VERSION="${{ matrix.python_version }}"
export CU_VERSION="${{ matrix.cuda_arch_version }}"
export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}"
export CODEC_HEADER_VERSION=n11.1.5.2
export COMPUTE_CAPABILITY=86
# Set CHANNEL
if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
export CHANNEL=test
else
export CHANNEL=nightly
fi
# Create Conda Env
conda create --quiet -y --prefix ci_env python="${PYTHON_VERSION}"
conda activate ./ci_env
# Install FFmpeg-dependencies
conda install --quiet -y -c conda-forge yasm x264 gnutls pkg-config lame libopus libvpx openh264 openssl x264
# Build FFmpeg with NVIDIA Video Codec SDK
# TODO cache this
(
git clone --quiet https://git.videolan.org/git/ffmpeg/nv-codec-headers.git
cd nv-codec-headers
git checkout ${CODEC_HEADER_VERSION}
make PREFIX=${CONDA_PREFIX} install
)
(
wget -q https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n5.1.3.tar.gz
tar -xf n5.1.3.tar.gz
cd ./FFmpeg-n5.1.3
export revision=5.1.3
./configure \
--prefix=${CONDA_PREFIX} \
--extra-cflags="-I${CONDA_PREFIX}/include" \
--extra-ldflags="-L${CONDA_PREFIX}/lib" \
--nvccflags="-gencode arch=compute_${COMPUTE_CAPABILITY},code=sm_${COMPUTE_CAPABILITY} -O2" \
--disable-doc \
--enable-rpath \
--disable-static \
--enable-protocol=https \
--enable-gnutls \
--enable-shared \
--enable-gpl \
--enable-nonfree \
--enable-libmp3lame \
--enable-libx264 \
--enable-cuda-nvcc \
--enable-nvenc \
--enable-cuvid \
--enable-nvdec
make clean
make -j > /dev/null 2>&1
make install
# test
# src="https://download.pytorch.org/torchaudio/tutorial-assets/stream-api/NASAs_Most_Scientifically_Complex_Space_Observatory_Requires_Precision-MP4_small.mp4"
# ffmpeg -y -vsync 0 -hwaccel cuvid -hwaccel_output_format cuda -c:v h264_cuvid -resize 360x240 -i "${src}" -c:a copy -c:v h264_nvenc -b:v 5M test.mp4
)
# Install PyTorch
set -ex
set +u # don't know why
conda install \
--yes \
--quiet \
-c "pytorch-${CHANNEL}" \
-c nvidia "pytorch-${CHANNEL}"::pytorch[build="*${CU_VERSION}*"] \
"${CUDATOOLKIT}"
# Install torchaudio
# TODO: Enable NVDec/NVEnc
conda install --quiet -y 'ffmpeg>=4.1' pkg-config
pip --quiet install cmake>=3.18.0 ninja
BUILD_VERSION=$( cut -f 1 -d a version.txt )
BUILD_VERSION=$BUILD_VERSION.dev$(date "+%Y%m%d")
export BUILD_VERSION
FFMPEG_ROOT=${CONDA_PREFIX} USE_FFMPEG=1 USE_CUDA=1 pip install --progress-bar off -v -e . --no-use-pep517
# Install runtime dependencies
pip --quiet install git+https://github.com/kpu/kenlm/ flashlight-text
# Install build tools
conda install --quiet -y -c conda-forge pandoc doxygen pysoundfile
pip install --quiet -r docs/requirements.txt -r docs/requirements-tutorials.txt
# Build docs
export BUILD_GALLERY=true
(cd docs && make html)
cp -rf docs/build/html/* "${RUNNER_DOCS_DIR}"
mv docs/build/html /artifacts/
commit:
if: ${{ (github.repository == 'pytorch/audio') && ((github.event_name == 'push') && (github.ref_name == 'nightly')) }}
permissions:
# Required for `git push`
# Note:
# This is not effective from fork.
# When you debug this, make sure to make a branch on pytorch and
# make PR from there.
contents: write
runs-on: ubuntu-latest
needs: build
steps:
- uses: actions/checkout@v3
with:
ref: gh-pages
fetch-depth: 5
- uses: actions/download-artifact@v3
with:
name: docs
- name: Update nightly doc
run: |
set -x
git config user.name "pytorchbot"
git config user.email "[email protected]"
# When `git clone`, `gh-pages` branch is fetched by default.
# The size of gh-pages grows significantly, so we use ammend and force push
# We add a new commit once a week
if [ "date +%u" = "1" ]; then
git commit --allow-empty -m "placeholder"
fi
# TODO: add tag-based process (need to handle the main directory name)
# Update the main doc
rm -rf main
mv html main
git add --all main || true
git commit --amend -m "auto-generating sphinx docs" || true
git push -f