feat(interactive): Introduce kafka as another wal storage #5428
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GraphScope CI on Local | |
| on: | |
| # Trigger the workflow on push or pull request, | |
| # but only for the main branch | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'CONTRIBUTORS' | |
| - 'LICENSE' | |
| - 'NOTICE.txt' | |
| - '**.md' | |
| - '**.rst' | |
| - 'docs/**' | |
| - 'demo/**' | |
| - 'scripts/**' | |
| - 'tutorials/**' | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - '.github/workflows/local-ci.yml' | |
| - 'Makefile' | |
| - 'analytical_engine/**' | |
| - 'charts/**' | |
| - 'coordinator/**' | |
| - 'interactive_engine/**' | |
| - 'k8s/**' | |
| - 'learning_engine/**' | |
| - 'proto/**' | |
| - 'python/**' | |
| - '!**.md' | |
| - '!**.rst' | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| env: | |
| GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope | |
| jobs: | |
| # JOB to run change detection | |
| changes: | |
| runs-on: ubuntu-20.04 | |
| # Set job outputs to values from filter step | |
| outputs: | |
| gae-python: ${{ steps.filter.outputs.gae-python }} | |
| networkx: ${{ steps.filter.outputs.networkx }} | |
| gie-function-test: ${{ steps.filter.outputs.gie-function-test }} | |
| steps: | |
| # For push it's necessary to checkout the code | |
| - uses: actions/checkout@v4 | |
| - name: Detect the tmate session | |
| run: | | |
| # hack here: get all field of build-wheels which run on self-hosted machine | |
| if sed -n '/self-hosted/,/mini-test/ p' .github/workflows/local-ci.yml | \ | |
| grep -v "grep" - | grep "action-tmate"; then | |
| echo 'WARNING!!!the self-hosted machine cannot run tmate session, please debug it manually' | |
| exit 1 | |
| fi | |
| # For pull requests it's not necessary to checkout the code | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| base: main # Change detection against merge-base with main before push | |
| filters: | | |
| gae-python: | |
| - 'proto/**' | |
| - 'analytical_engine/**' | |
| - 'python/graphscope/analytical/**' | |
| - 'python/graphscope/client/**' | |
| - 'python/graphscope/dataset/**' | |
| - 'python/graphscope/deploy/**' | |
| - 'python/graphscope/framework/**' | |
| - 'python/graphscope/tests/unittest/**' | |
| - 'coordinator/gscoordinator/**' | |
| - '.github/workflows/local-ci.yml' | |
| networkx: | |
| - 'analytical_engine/apps/**' | |
| - 'analytical_engine/frame/**' | |
| - 'analytical_engine/core/**' | |
| - 'python/graphscope/nx/**' | |
| - 'python/requirements.txt' | |
| gie-function-test: | |
| - 'interactive_engine/**' | |
| - 'python/graphscope/interactive/**' | |
| - '.github/workflows/ci.yml' | |
| gie-k8s-failover-test: | |
| - 'interactive_engine/**' | |
| - 'charts/ir-standalone/**' | |
| - name: Cpp Format and Lint Check | |
| run: | | |
| # install clang-format | |
| sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output /usr/bin/clang-format | |
| sudo chmod +x /usr/bin/clang-format | |
| # collect the source files | |
| cd analytical_engine/ | |
| files=$(find ./apps ./benchmarks ./core ./frame ./misc ./test \( -name "*.h" -o -name "*.cc" \)) | |
| # run format | |
| clang-format -i --style=file $(echo $files) | |
| # validate format | |
| function prepend() { while read line; do echo "${1}${line}"; done; } | |
| GIT_DIFF=$(git diff --ignore-submodules) | |
| if [[ -n $GIT_DIFF ]]; then | |
| echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" | |
| echo "| clang-format failures found!" | |
| echo "|" | |
| echo "$GIT_DIFF" | prepend "| " | |
| echo "|" | |
| echo "| Run: " | |
| echo "|" | |
| echo "| make gsa_clformat" | |
| echo "|" | |
| echo "| to fix this error." | |
| echo "|" | |
| echo "| Ensure you are working with clang-format-8, which can be obtained from" | |
| echo "|" | |
| echo "| https://github.com/muttleyxd/clang-tools-static-binaries/releases" | |
| echo "|" | |
| echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" | |
| exit -1 | |
| fi | |
| # validate cpplint | |
| function ec() { [[ "$1" == "-h" ]] && { shift && eval $* > /dev/null 2>&1; ec=$?; echo $ec; } || eval $*; ec=$?; } | |
| # run cpplint | |
| ec ./misc/cpplint.py $(echo $files) | |
| if [[ "$ec" != "0" ]]; then | |
| echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" | |
| echo "| cpplint failures found! Run: " | |
| echo "|" | |
| echo "| make gsa_cpplint" | |
| echo "|" | |
| echo "| to fix this error." | |
| echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" | |
| exit -1 | |
| fi | |
| - name: Python Format and Lint Check | |
| run: | | |
| echo "Checking formatting for $GITHUB_REPOSITORY" | |
| pip3 install -r coordinator/requirements-dev.txt | |
| pushd python | |
| python3 -m isort --check --diff . | |
| python3 -m black --check --diff . | |
| python3 -m flake8 . | |
| popd | |
| pushd coordinator | |
| python3 -m isort --check --diff . | |
| python3 -m black --check --diff . | |
| python3 -m flake8 . | |
| - name: Setup Java11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'zulu' | |
| java-version: '11' | |
| - name: Java Format and Lint Check | |
| run: | | |
| wget https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar | |
| files_to_format=$(git ls-files *.java) | |
| # run formatter in-place | |
| java -jar ${GITHUB_WORKSPACE}/google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $files_to_format | |
| # validate format | |
| function prepend() { while read line; do echo "${1}${line}"; done; } | |
| GIT_DIFF=$(git diff --ignore-submodules) | |
| if [[ -n $GIT_DIFF ]]; then | |
| echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" | |
| echo "| google-java-format failures found!" | |
| echo "|" | |
| echo "$GIT_DIFF" | prepend "| " | |
| echo "|" | |
| echo "| Run: " | |
| echo "|" | |
| echo '| java -jar google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $(git ls-files **/*.java)' | |
| echo "|" | |
| echo "| to fix this error." | |
| echo "|" | |
| echo "| Ensure you are working with google-java-format-1.13.0, which can be obtained from" | |
| echo "|" | |
| echo "| https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar" | |
| echo "|" | |
| echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" | |
| exit -1 | |
| fi | |
| build-wheels: | |
| # Require the user id of the self-hosted is 1001, which may need to be | |
| # configured manually when a new self-hosted runner is added. | |
| runs-on: [self-hosted, ubuntu2004] | |
| needs: [changes] | |
| if: ${{ github.repository == 'alibaba/GraphScope' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| ~/.cache/sccache | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Build Wheels | |
| run: | | |
| #sleep infinity | |
| git submodule update --init | |
| cd ${GITHUB_WORKSPACE}/learning_engine/graph-learn && git submodule update --init third_party/pybind11 | |
| # build graphscope wheels | |
| cd ${GITHUB_WORKSPACE}/k8s/internal | |
| # sudo -E -u runner make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope | |
| make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com | |
| # build client wheels | |
| # sudo -E -u runner make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope | |
| make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com | |
| # package | |
| cd ${GITHUB_WORKSPACE} | |
| sudo chown $(id -u) -R . | |
| tar -zcf client.tar.gz python/dist/wheelhouse/*.whl | |
| tar -zcf graphscope.tar.gz coordinator/dist/ | |
| - name: Upload Artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheel-${{ github.sha }} | |
| path: | | |
| client.tar.gz | |
| graphscope.tar.gz | |
| retention-days: 5 | |
| mini-test: | |
| runs-on: ubuntu-20.04 | |
| if: ${{ github.repository == 'alibaba/GraphScope' }} | |
| needs: [build-wheels] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| - name: Prepare Environment | |
| env: | |
| GS_TEST_DIR: ${{ github.workspace }}/gstest | |
| run: | | |
| # install graphscope-client | |
| cd artifacts | |
| tar -zxf ./wheel-${{ github.sha }}/client.tar.gz | |
| pushd python/dist/wheelhouse | |
| for f in * ; do python3 -m pip install --no-cache-dir $f || true; done | |
| popd | |
| # install graphscope | |
| tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz | |
| pushd coordinator/dist | |
| python3 -m pip install --no-cache-dir ./*.whl | |
| popd | |
| pushd coordinator/dist/wheelhouse | |
| python3 -m pip install --no-cache-dir ./*.whl | |
| popd | |
| # install tensorflow | |
| python3 -m pip install --no-cache-dir pytest pytest-xdist "pandas" --user | |
| # python3 -m pip install --no-cache-dir "tensorflow" --user | |
| # install pytorch | |
| # python3 -m pip install --no-cache-dir "torch<=2.2.1" "networkx<=3.0" --index-url https://download.pytorch.org/whl/cpu | |
| # install java | |
| sudo apt update -y && sudo apt install openjdk-11-jdk -y | |
| # download dataset | |
| git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: false | |
| - name: Run Minimum Test | |
| env: | |
| GS_TEST_DIR: ${{ github.workspace }}/gstest | |
| run: | | |
| python3 -m pytest -d --tx popen//python=python3 \ | |
| -s -v \ | |
| $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest | |
| - name: Upload GIE log | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| name: gie-log | |
| path: ~/.local/log/graphscope | |
| python-unittest: | |
| runs-on: ubuntu-20.04 | |
| if: ${{ github.repository == 'alibaba/GraphScope' }} | |
| needs: [build-wheels, changes] | |
| defaults: | |
| run: | |
| shell: bash --noprofile --norc -eo pipefail {0} | |
| strategy: | |
| matrix: | |
| deployment: ["standalone", "distributed"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: ${{ needs.changes.outputs.gae-python == 'true' }} | |
| - uses: actions/download-artifact@v4 | |
| if: ${{ needs.changes.outputs.gae-python == 'true' }} | |
| with: | |
| path: artifacts | |
| - name: Prepare Environment | |
| if: ${{ needs.changes.outputs.gae-python == 'true' }} | |
| run: | | |
| # install graphscope-client | |
| cd artifacts | |
| tar -zxf ./wheel-${{ github.sha }}/client.tar.gz | |
| pushd python/dist/wheelhouse | |
| for f in * ; do python3 -m pip install --no-cache-dir $f || true; done | |
| popd | |
| # install graphscope | |
| tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz | |
| pushd coordinator/dist | |
| python3 -m pip install --no-cache-dir ./*.whl | |
| popd | |
| pushd coordinator/dist/wheelhouse | |
| python3 -m pip install --no-cache-dir ./*.whl | |
| popd | |
| # install pytest | |
| python3 -m pip install --no-cache-dir pytest pytest-cov pytest-timeout pytest-xdist | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: false | |
| - name: Run Python Test | |
| if: ${{ needs.changes.outputs.gae-python == 'true' }} | |
| env: | |
| GS_TEST_DIR: ${{ github.workspace }}/gstest | |
| DEPLOYMENT: ${{ matrix.deployment }} | |
| run: | | |
| # download dataset | |
| git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} | |
| export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}" | |
| python3 -m pytest -d --tx popen//python=python3 \ | |
| -s -v \ | |
| --cov=graphscope --cov-config=${GITHUB_WORKSPACE}/python/.coveragerc --cov-report=xml --cov-report=term \ | |
| --exitfirst \ | |
| $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/unittest | |
| - name: Upload Coverage | |
| uses: codecov/codecov-action@v4 | |
| continue-on-error: true | |
| with: | |
| files: ./coverage.xml | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| verbose: true | |
| networkx-basic-test: | |
| runs-on: ubuntu-20.04 | |
| needs: [build-wheels, changes] | |
| if: ${{ github.repository == 'alibaba/GraphScope' }} | |
| defaults: | |
| run: | |
| shell: bash | |
| strategy: | |
| matrix: | |
| deployment: ["standalone", "distributed"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: ${{ needs.changes.outputs.networkx == 'true' }} | |
| - uses: dorny/paths-filter@v3 | |
| if: ${{ needs.changes.outputs.networkx == 'true' }} | |
| id: nx-filter | |
| with: | |
| filters: | | |
| basic: | |
| - 'analytical_engine/apps/**' | |
| - 'analytical_engine/frame/**' | |
| - 'analytical_engine/core/**' | |
| - 'python/graphscope/nx/__init__.py' | |
| - 'python/graphscope/nx/classes/**' | |
| - 'python/graphscope/nx/!(tests)' | |
| - 'python/graphscope/nx/tests/!(convert)' | |
| - 'python/requirements.txt' | |
| convert: | |
| - 'python/graphscope/nx/convert.py' | |
| - 'python/graphscope/nx/convert_matrix.py' | |
| - 'python/graphscope/nx/tests/convert/**' | |
| - uses: actions/download-artifact@v4 | |
| if: ${{ needs.changes.outputs.networkx == 'true' }} | |
| with: | |
| path: artifacts | |
| - name: Prepare Environment | |
| if: ${{ needs.changes.outputs.networkx == 'true' }} | |
| env: | |
| GS_TEST_DIR: ${{ github.workspace }}/gstest | |
| run: | | |
| # install graphscope-client | |
| cd artifacts | |
| tar -zxf ./wheel-${{ github.sha }}/client.tar.gz | |
| pushd python/dist/wheelhouse | |
| for f in * ; do python3 -m pip install --no-cache-dir $f || true; done | |
| popd | |
| # install graphscope | |
| tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz | |
| pushd coordinator/dist | |
| python3 -m pip install --no-cache-dir ./*.whl | |
| popd | |
| pushd coordinator/dist/wheelhouse | |
| python3 -m pip install --no-cache-dir ./*.whl | |
| popd | |
| # install pytest | |
| python3 -m pip install --no-cache-dir pytest pytest-xdist | |
| # download dataset | |
| git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: false | |
| - name: Basic Test | |
| if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.basic == 'true' }} | |
| env: | |
| DEPLOYMENT: ${{ matrix.deployment }} | |
| GS_TEST_DIR: ${{ github.workspace }}/gstest | |
| run: | | |
| pip3 show networkx | |
| python3 -m pytest -d --tx popen//python=python3 \ | |
| --exitfirst -s -vvv \ | |
| $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests \ | |
| --ignore=$(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests/convert | |
| - name: Convert Test | |
| if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.convert == 'true' }} | |
| env: | |
| DEPLOYMENT: ${{ matrix.deployment }} | |
| GS_TEST_DIR: ${{ github.workspace }}/gstest | |
| run: | | |
| pip3 show networkx | |
| python3 -m pytest -d --tx popen//python=python3 \ | |
| --exitfirst -s -vvv \ | |
| $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests/convert | |
| networkx-algo-and-generator-test: | |
| runs-on: ubuntu-20.04 | |
| needs: [build-wheels, changes] | |
| if: ${{ github.repository == 'alibaba/GraphScope' }} | |
| strategy: | |
| matrix: | |
| deployment: ["standalone", "distributed"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: ${{ needs.changes.outputs.networkx == 'true' }} | |
| - uses: dorny/paths-filter@v3 | |
| if: ${{ needs.changes.outputs.networkx == 'true' }} | |
| id: nx-filter | |
| with: | |
| filters: | | |
| algorithm: | |
| - 'analytical_engine/apps/**' | |
| - 'analytical_engine/frame/**' | |
| - 'python/graphscope/nx/algorithms/builtin.py' | |
| - 'python/graphscope/nx/algorithms/tests/builtin/**' | |
| generator: | |
| - 'python/graphscope/nx/classes/**' | |
| - 'python/graphscope/nx/utils/**' | |
| - 'python/graphscope/nx/generators/**' | |
| io: | |
| - 'python/graphscope/nx/readwrite/**' | |
| - uses: actions/download-artifact@v4 | |
| if: ${{ needs.changes.outputs.networkx == 'true' }} | |
| with: | |
| path: artifacts | |
| - name: Prepare Environment | |
| if: ${{ needs.changes.outputs.networkx == 'true' }} | |
| env: | |
| GS_TEST_DIR: ${{ github.workspace }}/gstest | |
| run: | | |
| # install graphscope-client | |
| cd artifacts | |
| tar -zxf ./wheel-${{ github.sha }}/client.tar.gz | |
| pushd python/dist/wheelhouse | |
| for f in * ; do python3 -m pip install --no-cache-dir $f || true; done | |
| popd | |
| # install graphscope | |
| tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz | |
| pushd coordinator/dist | |
| python3 -m pip install --no-cache-dir ./*.whl | |
| popd | |
| pushd coordinator/dist/wheelhouse | |
| python3 -m pip install --no-cache-dir ./*.whl | |
| popd | |
| # install pytest | |
| python3 -m pip install --no-cache-dir pytest pytest-xdist | |
| # download dataset | |
| git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} | |
| - name: Algorithm test | |
| if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.algorithm == 'true' }} | |
| env: | |
| DEPLOYMENT: ${{ matrix.deployment }} | |
| GS_TEST_DIR: ${{ github.workspace }}/gstest | |
| run: | | |
| pip3 show networkx | |
| python3 -m pytest -d --tx popen//python=python3 \ | |
| --exitfirst -s -v \ | |
| $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/algorithms/tests/builtin | |
| - name: Generator test | |
| if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.generator == 'true' }} | |
| env: | |
| DEPLOYMENT: ${{ matrix.deployment }} | |
| GS_TEST_DIR: ${{ github.workspace }}/gstest | |
| run: | | |
| pip3 show networkx | |
| python3 -m pytest -d --tx popen//python=python3 \ | |
| --exitfirst -s -v \ | |
| $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/generators/tests | |
| - name: Readwrite test | |
| if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.io == 'true' }} | |
| env: | |
| DEPLOYMENT: ${{ matrix.deployment }} | |
| run: | | |
| python3 -m pytest -d --tx popen//python=python3 \ | |
| --exitfirst -s -v -m "not slow" \ | |
| $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/readwrite/tests |