interactive.yml

name: GraphScope GIE HighQPS DB CI

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  workflow_dispatch:
  push:
    branches:
      - main
    paths:
      - 'flex/**'
      - 'interactive_engine/**'
      - '.github/workflows/interactive.yml'
  pull_request:
    branches:
      - main
    paths:
      - 'flex/**'
      - 'interactive_engine/**'
      - '.github/workflows/interactive.yml'

concurrency:
  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

jobs:
  build-interactive:
    runs-on: ubuntu-20.04
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    container:
      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
    steps:
    - uses: actions/checkout@v4
    - name: Install latest libgrape-lite
      if: false
      run: |
        git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
        cd /tmp/libgrape-lite
        mkdir -p build && cd build
        cmake ..
        make -j$(nproc)
        make install

    - name: Setup tmate session
      if: false
      uses: mxschmitt/action-tmate@v3

    - name: Build Interactive
      env:
        GIE_HOME: ${{ github.workspace }}/interactive_engine/
        HOME: /home/graphscope/
      run: |
        . /home/graphscope/.graphscope_env
        cd ${GITHUB_WORKSPACE}/
        git submodule update --init
        cd ${GITHUB_WORKSPACE}/flex
        mkdir build && cd build
        cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope -DCMAKE_BUILD_TYPE=DEBUG && sudo make -j$(nproc)
        # package the build artifacts
        cd .. && tar -zcf build.tar.gz build
    
    - name: Upload Artifacts
      uses: actions/upload-artifact@v4
      with:
        name: interactive_build-${{ github.sha }}
        path: |
          ${{ github.workspace }}/flex/build.tar.gz


  test-hqps-engine:
    runs-on: ubuntu-20.04
    needs: build-interactive
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    container:
      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
    steps:
    - uses: actions/checkout@v4

    - uses: actions/cache@v4
      with:
        path: ~/.m2/repository
        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
            ${{ runner.os }}-maven-

    - uses: actions/cache@v4
      with:
        path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            ~/.cache/sccache
        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

    - name: Install latest libgrape-lite
      if: false
      run: |
        git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
        cd /tmp/libgrape-lite
        mkdir -p build && cd build
        cmake ..
        make -j$(nproc)
        make install
    
    - name: Download Artifacts
      uses: actions/download-artifact@v4
      with:
        name: interactive_build-${{ github.sha }}

    - name: Setup tmate session
      if: false
      uses: mxschmitt/action-tmate@v3

    - name: Extract build artifacts
      run: |
        cd ${GITHUB_WORKSPACE}
        tar zxf build.tar.gz -C flex && rm build.tar.gz

    - name: Build
      env:
        GIE_HOME: ${{ github.workspace }}/interactive_engine/
        HOME: /home/graphscope/
      run: |
        . /home/graphscope/.graphscope_env
        cd ${GITHUB_WORKSPACE}/

        # cargo
        . /home/graphscope/.cargo/env
        which cargo

        # build compiler
        cd ${GIE_HOME}/
        mvn clean install -Pexperimental -DskipTests -q

    - name: Prepare dataset and workspace
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        # download dataset
        git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR}
        mkdir -p ${INTERACTIVE_WORKSPACE}/data/ldbc
        GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml
        BUILD_LOAD_FILE=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml
        cp ${GRAPH_SCHEMA_YAML} ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml
        cp ${BUILD_LOAD_FILE} ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml
        mkdir -p ${INTERACTIVE_WORKSPACE}/data/movies
        cp ${GS_TEST_DIR}/flex/movies/movies_schema.yaml ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml
        cp ${GS_TEST_DIR}/flex/movies/movies_import.yaml ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml
        mkdir -p ${INTERACTIVE_WORKSPACE}/data/graph_algo
        cp ${GITHUB_WORKSPACE}/flex/tests/interactive/graph_algo_test.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml
        cp ${GITHUB_WORKSPACE}/flex/interactive/examples/graph_algo/import.yaml ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml
        mkdir -p ${INTERACTIVE_WORKSPACE}/data/new_graph_algo
        cp ${GITHUB_WORKSPACE}/flex/interactive/examples/new_graph_algo/graph.yaml ${INTERACTIVE_WORKSPACE}/data/new_graph_algo/graph.yaml
        cp ${GITHUB_WORKSPACE}/flex/interactive/examples/new_graph_algo/import.yaml ${INTERACTIVE_WORKSPACE}/data/new_graph_algo/import.yaml
        mkdir -p ${INTERACTIVE_WORKSPACE}/data/modern_graph
        cp ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml ${INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml

        # load graph
        cd ${GITHUB_WORKSPACE}/flex/build
        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
        GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/ldbc/indices/
        export FLEX_DATA_DIR=../interactive/examples/movies
        GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/movies/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/movies/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/movies/indices/
        export FLEX_DATA_DIR=../interactive/examples/graph_algo
        GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/graph_algo/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/graph_algo/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/graph_algo/indices/
        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/new_graph_algo
        GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/new_graph_algo/graph.yaml -l ${INTERACTIVE_WORKSPACE}/data/new_graph_algo/import.yaml -d ${INTERACTIVE_WORKSPACE}/data/new_graph_algo/indices/
        export FLEX_DATA_DIR=../interactive/examples/modern_graph
        GLOG_v=10 ./bin/bulk_loader -g ${INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml -l ../interactive/examples/modern_graph/bulk_load.yaml -d ${INTERACTIVE_WORKSPACE}/data/modern_graph/indices/

    - name: Test HQPS admin http service
      env: 
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph
        TMP_INTERACTIVE_WORKSPACE: /tmp/temp_workspace
      run: |
        rm -rf ${TMP_INTERACTIVE_WORKSPACE}
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml
        BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/bulk_load.yaml
        mkdir -p ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/
        cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
        GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps
        bash hqps_admin_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ${GS_TEST_DIR}

    - name: Build and test Interactive Java/Python SDK
      env: 
        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph
        TMP_INTERACTIVE_WORKSPACE: /tmp/temp_workspace
      run: |
        cd ${GITHUB_WORKSPACE}/flex/interactive/sdk/

        bash generate_sdk.sh -g java
        bash generate_sdk.sh -g python
        cd java
        mvn clean install -DskipTests -q
        cd ../python
        pip3 install -r requirements.txt
        pip3 install -r test-requirements.txt
        export PATH=${HOME}/.local/bin:$PATH
        python3 setup.py build_proto

        rm -rf ${TMP_INTERACTIVE_WORKSPACE}     
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml
        BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/bulk_load.yaml
        mkdir -p ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/
        cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
        GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps
        sed -i 's/interactive_workspace/temp_workspace/g' ./interactive_config_test.yaml
        bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml java
        bash hqps_sdk_test.sh ${TMP_INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml python
        sed -i 's/temp_workspace/interactive_workspace/g' ./interactive_config_test.yaml
  
    - name: Robustness test 
      env:
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps
        bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ./interactive_config_test_cbo.yaml

    - name: Sample Query test
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        cd ${GITHUB_WORKSPACE}/flex/build
        export FLEX_DATA_DIR=${GS_TEST_DIR}/flex/ldbc-sf01-long-date
        ./bin/bulk_loader -g ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \
        -l ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml -d /tmp/csr-data-dir/
        rm -r /tmp/csr-data-dir/runtime/* 
        rm -r /tmp/csr-data-dir/wal
        ./tests/hqps/query_test ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml \
        /tmp/csr-data-dir/

    - name: Test get graph meta from admin service
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps
        pip3 install argparse
        pip3 install neo4j
        bash hqps_compiler_get_meta_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml

    - name: Run codegen test.
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps
        bash hqps_codegen_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ./interactive_config_test_cbo.yaml

    - name: Test cypher&cpp procedure generation and loading
      env:
        TMP_INTERACTIVE_WORKSPACE: /tmp/temp_workspace
        PLUGIN_DIR: /tmp/temp_workspace/data/modern_graph/plugins
        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph
      run: |
        rm -rf ${TMP_INTERACTIVE_WORKSPACE}
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml
        BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/bulk_load.yaml
        mkdir -p ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/
        cp ${SCHEMA_FILE} ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/graph.yaml
        GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d ${TMP_INTERACTIVE_WORKSPACE}/data/modern_graph/indices/
  
        mkdir -p ${PLUGIN_DIR}
        pip3 install argparse
        
        cd ${GITHUB_WORKSPACE}/flex/bin
        ./load_plan_and_gen.sh -e=hqps -i=../tests/interactive/plus_one.cc -w=/tmp/codegen \
          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml -o=${PLUGIN_DIR} \
          --procedure_name=plus_one \
          --graph_schema_path=../interactive/examples/modern_graph/graph.yaml \
          --procedure_desc="This is test procedure, and the input is a number, and the output is the number plus one."
        
        ./load_plan_and_gen.sh -e=hqps -i=../interactive/sdk/java/src/test/resources/sample_app.cc -w=/tmp/codegen \
          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml -o=${PLUGIN_DIR} \
          --procedure_name=sample_app \
          --graph_schema_path=../interactive/examples/modern_graph/graph.yaml

        ./load_plan_and_gen.sh -e=hqps -i=../interactive/examples/modern_graph/count_vertex_num.cypher -w=/tmp/codegen \
          --ir_conf=${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml  -o=${PLUGIN_DIR} \
          --procedure_name=count_vertex_num \
          --graph_schema_path=../interactive/examples/modern_graph/graph.yaml
        
        # Among the above procedures, the correct input format for each is:
        # count_vertex_num: () -> (num: int64), CypherProcedure.
        # plus_one: (num: int64) -> (num: int64), CppEncoder
        # sample_app: (num: int64) -> (num: int64), kCypherJson

        sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
        sed -i 's/interactive_workspace/temp_workspace/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
        cd ${GITHUB_WORKSPACE}/flex/tests/interactive/
        bash test_plugin_loading.sh ${TMP_INTERACTIVE_WORKSPACE} modern_graph \
           ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml \
           ./modern_graph_schema_v0_0.yaml  ./modern_graph_schema_v0_1.yaml
        sed -i 's/temp_workspace/interactive_workspace/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
        sed -i 's/default_graph: modern_graph/default_graph: movies/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
      
    - name: Let compiler use latest interactive java sdk
      env:
        HOME: /home/graphscope/
      run: |
        . /home/graphscope/.graphscope_env
        . /home/graphscope/.cargo/env
        # replace the <interactive.sdk.version>0.4</interactive.sdk.version> with the latest version in flex/interactive/sdk/java/pom.xml
        sdk_version=$(grep -oPm1 "(?<=<version>)[^<]+" ${GITHUB_WORKSPACE}/flex/interactive/sdk/java/pom.xml)
        sed -i "s/<interactive.sdk.version>.*<\/interactive.sdk.version>/<interactive.sdk.version>${sdk_version}<\/interactive.sdk.version>/" ${GITHUB_WORKSPACE}/interactive_engine/pom.xml
        cd ${GITHUB_WORKSPACE}/interactive_engine/
        mvn clean install -Pexperimental -DskipTests -q

    - name: Test physical plan generation
      run: |
        cd ${GITHUB_WORKSPACE}/interactive_engine
        cat > /tmp/physical_plan_gen_config.yaml <<EOF
        compiler:
          planner:
            is_on: true
            opt: CBO
            rules:
              - FilterIntoJoinRule
              - FilterMatchRule
              - NotMatchToAntiJoinRule
          physical.opt.config: proto
        EOF
        echo "  meta.reader.schema.uri: ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml" >> /tmp/physical_plan_gen_config.yaml
        echo "  meta.reader.statistics.uri: ${GITHUB_WORKSPACE}/interactive_engine/compiler/src/test/resources/statistics/modern_statistics.json" >> /tmp/physical_plan_gen_config.yaml
        mvn clean install -DskipTests -Pgraph-planner-jni
        INTERACTIVE_ENGINE_HOME=${GITHUB_WORKSPACE}/interactive_engine
        ./target/native/test_graph_planner ${INTERACTIVE_ENGINE_HOME}/compiler/target/compiler-0.0.1-SNAPSHOT.jar:${INTERACTIVE_ENGINE_HOME}/compiler/target/libs/ ${INTERACTIVE_ENGINE_HOME}/executor/ir/target/release/libir_core.so \
           ${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml ${INTERACTIVE_ENGINE_HOME}/compiler/src/test/resources/statistics/modern_statistics.json \
          "MATCH(n) return count(n);" /tmp/physical_plan_gen_config.yaml

    - name: Run End-to-End cypher adhoc ldbc query test
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        HOME : /home/graphscope/
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
        export ENGINE_TYPE=hiactor
        # change the default_graph config in ./interactive_config_test.yaml to ldbc
        sed -i 's/default_graph: movies/default_graph: ldbc/g' ./interactive_config_test.yaml
        bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
              ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml

    - name: Run End-to-End cypher adhoc movie query test
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        HOME : /home/graphscope/
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
        export ENGINE_TYPE=hiactor
        # change the default_graph config in ./interactive_config_test.yaml to movies
        sed -i 's/default_graph: ldbc/default_graph: movies/g' ./interactive_config_test.yaml
        bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} movies \
          ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml

    - name: Run End-to-End cypher adhoc graph_algo query test
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        HOME : /home/graphscope/
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
        export ENGINE_TYPE=hiactor
        # change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/interactive_config.yaml to graph_algo
        sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./interactive_config_test.yaml
        bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \
          ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml

    - name: Run Gremlin test on modern graph
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        HOME : /home/graphscope/
        INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
      run: |
        cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
        export ENGINE_TYPE=hiactor
        sed -i 's/default_graph: graph_algo/default_graph: modern_graph/g' ./interactive_config_test.yaml
        bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} modern_graph \
          ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml gremlin

  test-build-flex:
    runs-on: ubuntu-22.04

    strategy:
      matrix:
        os: [ubuntu-22.04]
        platform: [x86_64]

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Install dependencies
      run: |
        cd ${GITHUB_WORKSPACE}
        python3 -m pip install --upgrade pip && python3 -m pip install click
        python3 ./gsctl.py install-deps dev-interactive

    - name: Setup tmate session
      if: false
      uses: mxschmitt/action-tmate@v3

    - name: Build
      run: |
        source ${HOME}/.graphscope_env
        cd ${GITHUB_WORKSPACE}/flex
        git submodule update --init
        mkdir build && cd build # only test default build
        cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DBUILD_DOC=OFF && sudo make -j 4

  # test the different combination of cmake options: -DBUILD_TEST=ON/OFF, -DBUILD_ODPS_FRAGMENT_LOADER=ON/OFF
  test-cmake-options:
    runs-on: ubuntu-20.04
    container:
      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
    strategy:
      matrix:
        BUILD_TEST: [ON, OFF]
        BUILD_ODPS_FRAGMENT_LOADER: [ON, OFF]
    steps:
    - uses: actions/checkout@v4

    - name: Build
      run: |
        cd ${GITHUB_WORKSPACE}/flex
        git submodule update --init
        mkdir build && cd build
        cmake .. -DBUILD_TEST=${{ matrix.BUILD_TEST }} \
                 -DBUILD_ODPS_FRAGMENT_LOADER=${{ matrix.BUILD_ODPS_FRAGMENT_LOADER }}
        sudo make -j4

  test-AOCC-compilation:
    runs-on: ubuntu-20.04
    container:
      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
    steps:
    - uses: actions/checkout@v4

    - name: Download aocc compiler
      run: |
        cd ${GITHUB_WORKSPACE}/flex
        wget https://download.amd.com/developer/eula/aocc/aocc-4-1/aocc-compiler-4.1.0_1_amd64.deb
        sudo apt-get update && sudo apt-get install -y libncurses-dev libncurses5-dev
        sudo dpkg -i aocc-compiler-4.1.0_1_amd64.deb
    
    - name: Test AOCC compiler
      env:
        CC: clang
        CXX: clang++
        OMPI_CC: clang
        OMPI_CXX: clang++
      run: |
        . /opt/AMD/aocc-compiler-4.1.0/setenv_AOCC.sh
        cd ${GITHUB_WORKSPACE}/flex
        git submodule update --init
        mkdir build && cd build
        cmake .. -DBUILD_DOC=OFF -DCMAKE_BUILD_TYPE=DEBUG
        sudo make -j4

  test-flex:
    runs-on: ubuntu-20.04
    needs: build-interactive
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    container:
      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
    steps:
    - uses: actions/checkout@v4

    - name: Install latest libgrape-lite
      if: false
      run: |
        git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
        cd /tmp/libgrape-lite
        mkdir -p build && cd build
        cmake ..
        make -j$(nproc)
        make install

    - name: Download Artifacts
      uses: actions/download-artifact@v4
      with:
        name: interactive_build-${{ github.sha }}

    - name: Extract build artifacts
      run: |
        cd ${GITHUB_WORKSPACE}
        tar zxf build.tar.gz -C flex && rm build.tar.gz

    - name: Test GRIN on mutable csr 
      run: |
        git submodule update --init
        cd flex/engines/graph_db/grin
        mkdir build && cd build
        cmake .. && sudo make -j$(nproc)
        export FLEX_DATA_DIR=../../../../interactive/examples/modern_graph/
        ${GITHUB_WORKSPACE}/flex/build/bin/bulk_loader -g ../../../../interactive/examples/modern_graph/graph.yaml -l ../../../../interactive/examples/modern_graph/bulk_load.yaml -d ./data/
        rm -r ./data/wal
        rm -r ./data/runtime/*
        ./run_grin_test 'flex://schema_file=../../../../interactive/examples/modern_graph/graph.yaml&data_dir=./data/'

    - name: Prepare test dataset
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest/
      run: |
        git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR}

    - name: Test String primary key on modern graph
      env:
        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph/
      run: |
        rm -rf /tmp/csr-data-dir/
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=../tests/rt_mutable_graph/modern_graph_string_pk.yaml 
        BULK_LOAD_FILE=../interactive/examples/modern_graph/bulk_load.yaml
        GLOG_v=10 ./bin/bulk_loader  -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/

    - name: Test String edge property on modern graph
      env:
        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph/
      run: |
        rm -rf /tmp/csr-data-dir/
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=../tests/rt_mutable_graph/modern_graph_string_edge.yaml 
        BULK_LOAD_FILE=../interactive/examples/modern_graph/bulk_load.yaml
        GLOG_v=10 ./bin/bulk_loader  -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/
        GLOG_v=10 ./tests/rt_mutable_graph/string_edge_property_test ${SCHEMA_FILE} /tmp/csr-data-dir/
    

    - name: Test update transaction
      env:
        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph/
      run: |
        rm -rf /tmp/csr-data-dir/
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=../tests/rt_mutable_graph/modern_graph_string_edge.yaml 
        BULK_LOAD_FILE=../interactive/examples/modern_graph/bulk_load.yaml
        GLOG_v=10 ./bin/bulk_loader  -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/
        GLOG_v=10 ./tests/rt_mutable_graph/test_update_transaction ${SCHEMA_FILE} /tmp/csr-data-dir/

    - name: Test multiple properties edge
      env:
        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph/
      run: |
        rm -rf /tmp/csr-data-dir/
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=../tests/rt_mutable_graph/modern_graph_multiple_props_edge.yaml
        BULK_LOAD_FILE=../tests/rt_mutable_graph/modern_graph_multiple_props_edge_import.yaml
        GLOG_v=10 ./bin/bulk_loader  -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/
        GLOG_v=10 ./tests/rt_mutable_graph/multiple_props_edge_test ${SCHEMA_FILE} /tmp/csr-data-dir/

    - name: Test schema parsing and loading on modern graph
      env:
        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph/
      run: |
        rm -rf /tmp/csr-data-dir/
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=../tests/rt_mutable_graph/modern_graph_unified_schema.yaml
        BULK_LOAD_FILE=../interactive/examples/modern_graph/bulk_load.yaml
        GLOG_v=10 ./bin/bulk_loader  -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/

    - name: Test build empty graph
      run: |
        rm -rf /tmp/csr-data-dir/
        cd ${GITHUB_WORKSPACE}/flex/build/
        GLOG_v=10 ./tests/rt_mutable_graph/test_empty_graph /tmp/csr-data-dir/
    - name: Test ACID
      run: |
        rm -rf /tmp/csr-data-dir/
        cd ${GITHUB_WORKSPACE}/flex/build/
        GLOG_v=10 ./tests/rt_mutable_graph/test_acid 8 /tmp/csr-data-dir/

    - name: Test Graph Loading on modern graph
      env:
        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/modern_graph/
      run: |
        rm -rf /tmp/csr-data-dir/
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=../interactive/examples/modern_graph/graph.yaml
        BULK_LOAD_FILE=../interactive/examples/modern_graph/bulk_load.yaml
        GLOG_v=10 ./bin/bulk_loader  -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/

    - name: Test Graph Loading on type_test graph
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest/
        FLEX_DATA_DIR: ${{ github.workspace }}/gstest/flex/type_test/
      run: |
        # remove modern graph indices
        rm -rf /tmp/csr-data-dir/

        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=${GS_TEST_DIR}/flex/type_test/graph.yaml
        BULK_LOAD_FILE=${GS_TEST_DIR}/flex/type_test/import.yaml
        GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/ -p 2

    - name: Test Graph Loading on LDBC SNB sf0.1
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest/
        FLEX_DATA_DIR: ${{ github.workspace }}/gstest/flex/ldbc-sf01-long-date/
      run: |
        # remove previous graph indices
        rm -rf /tmp/csr-data-dir/

        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=${FLEX_DATA_DIR}/audit_graph_schema.yaml
        BULK_LOAD_FILE=${FLEX_DATA_DIR}/audit_bulk_load.yaml
        GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/ -p 2

    - name: Test graph load on movie graph
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest/
        FLEX_DATA_DIR: ${{ github.workspace }}/flex/interactive/examples/movies/
      run: |
        rm -rf /tmp/csr-data-dir/
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/tests/rt_mutable_graph/movie_schema_test.yaml
        BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/tests/rt_mutable_graph/movie_import_test.yaml
        GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/
    
    - name: Test graph loading with different delimiter
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest/
        FLEX_DATA_DIR: ${{ github.workspace }}/gstest/flex/modern_graph_tab_delimiter/
      run: |
        rm -rf /tmp/csr-data-dir/
        cd ${GITHUB_WORKSPACE}/flex/build/
        SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml
        BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/bulk_load.yaml
        sed -i 's/|/\\t/g' ${BULK_LOAD_FILE}
        GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/