From 3a84d24cf0b86876bdef62014c0036f148ec10e9 Mon Sep 17 00:00:00 2001 From: fmhoeger Date: Sat, 10 Feb 2024 20:35:01 -0600 Subject: [PATCH 1/8] WIP --- .github/workflows/main.yml | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c5db392c0..8da174daf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,15 +12,15 @@ on: jobs: build-and-test: - name: Test PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, CLN=${{ matrix.cln-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }} + name: Test PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }} runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.12"] bitcoind-version: ["26.0"] - cln-version: ["master", "v23.11"] experimental: [1] deprecated: [0] @@ -29,16 +29,7 @@ jobs: with: submodules: recursive - - name: Checkout c-lightning@${{ matrix.cln-version }} - uses: actions/checkout@v4 - with: - repository: 'ElementsProject/lightning' - path: 'lightning' - ref: ${{ matrix.cln-version }} - submodules: 'recursive' - fetch-depth: 0 # Required for pyln versions to be recognized - - - name: Download runtime dependencies + - name: Download & install Bitcoin binaries run: | export BITCOIND_VERSION=${{ matrix.bitcoind-version }} wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz @@ -46,8 +37,20 @@ jobs: sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin rm -rf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz bitcoin-${BITCOIND_VERSION} - - name: Compile & install c-lightning@master + - name: Checkout c-lightning@v23.11 + uses: actions/checkout@v4 + with: + repository: 'ElementsProject/lightning' + path: 'lightning' + ref: v23.11 + submodules: 'recursive' + fetch-depth: 0 # Required for pyln versions to be recognized + + - name: Download & install c-lightning binaries run: | + curl -s -L $(curl -s https://api.github.com/repos/ElementsProject/lightning/releases/latest | jq '.assets[] | select(.name | contains("22.04")) | .browser_download_url' | tr -d '\"') | tar xJ -C . + sudo mv usr/bin/* /usr/local/bin + rm -r usr export EXPERIMENTAL_FEATURES=${{ matrix.experimental }} export COMPAT=${{ matrix.deprecated }} export VALGRIND=0 @@ -79,9 +82,9 @@ jobs: pip install --user -U -r requirements.txt pip install --user contrib/pyln-client contrib/pyln-testing flaky - ./configure --disable-valgrind - make -j 16 - sudo make install + # ./configure --disable-valgrind + # make -j 16 + # sudo make install - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 From f30392eff86ac85c6fd54cc0ed72b8531b977468 Mon Sep 17 00:00:00 2001 From: fmhoeger Date: Sat, 10 Feb 2024 20:35:39 -0600 Subject: [PATCH 2/8] Set on-push branch to 407-use-cln-latest-release-binaries --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8da174daf..d92a15f62 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,7 +7,7 @@ concurrency: on: push: - branches: [ master ] + branches: [ 407-use-cln-latest-release-binaries ] pull_request: jobs: From 54ffc642c667dc2c30c46bcfdef6c4c1170961fb Mon Sep 17 00:00:00 2001 From: fmhoeger Date: Sun, 11 Feb 2024 22:49:46 -0600 Subject: [PATCH 3/8] WIP --- .github/workflows/main.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d92a15f62..b1c7940c5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,9 +48,10 @@ jobs: - name: Download & install c-lightning binaries run: | - curl -s -L $(curl -s https://api.github.com/repos/ElementsProject/lightning/releases/latest | jq '.assets[] | select(.name | contains("22.04")) | .browser_download_url' | tr -d '\"') | tar xJ -C . - sudo mv usr/bin/* /usr/local/bin - rm -r usr + url=$(curl -s https://api.github.com/repos/ElementsProject/lightning/releases/latest | jq '.assets[] | select(.name | contains("22.04")) | .browser_download_url' | tr -d '\"') + wget $url + sudo tar -xvf ${url##*/} -C /usr/local --strip-components=2 + export EXPERIMENTAL_FEATURES=${{ matrix.experimental }} export COMPAT=${{ matrix.deprecated }} export VALGRIND=0 From c2f75fe94dc84a64ae118fc11f7be583c3497902 Mon Sep 17 00:00:00 2001 From: fmhoeger Date: Sun, 11 Feb 2024 22:56:29 -0600 Subject: [PATCH 4/8] Cleanup apt-get install --- .github/workflows/main.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b1c7940c5..af8171ca2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,17 +56,8 @@ jobs: export COMPAT=${{ matrix.deprecated }} export VALGRIND=0 sudo apt-get install -y \ - build-essential \ - gettext \ - libpq-dev \ - libsodium-dev \ - libsqlite3-dev \ - net-tools \ - postgresql \ - protobuf-compiler \ python3 \ python3-pip \ - zlib1g-dev cd lightning pip3 install --user -U \ @@ -83,10 +74,6 @@ jobs: pip install --user -U -r requirements.txt pip install --user contrib/pyln-client contrib/pyln-testing flaky - # ./configure --disable-valgrind - # make -j 16 - # sudo make install - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From ffb7a1795e816d95972d0e44e38850cf33997e76 Mon Sep 17 00:00:00 2001 From: fmhoeger Date: Sun, 11 Feb 2024 23:18:41 -0600 Subject: [PATCH 5/8] Add nightly GitHub CI workflow for master branch --- .github/workflows/nightly.yml | 107 ++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..e7748d34b --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,107 @@ +name: Nightly integration tests (master) + +on: + schedule: + - cron: "21 2 * * *" + +jobs: + nightly-build-and-test: + name: Test PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }} + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + bitcoind-version: ["26.0"] + experimental: [1] + deprecated: [0] + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Download Bitcoin & install binaries + run: | + export BITCOIND_VERSION=${{ matrix.bitcoind-version }} + wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz + tar -xzf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz + sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin + rm -rf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz bitcoin-${BITCOIND_VERSION} + + - name: Checkout Core Lightning + uses: actions/checkout@v4 + with: + repository: 'ElementsProject/lightning' + path: 'lightning' + ref: master + submodules: 'recursive' + fetch-depth: 0 # Required for pyln versions to be recognized + + - name: Compile & install Core Lightning + run: | + export EXPERIMENTAL_FEATURES=${{ matrix.experimental }} + export COMPAT=${{ matrix.deprecated }} + export VALGRIND=0 + sudo apt-get install -y \ + build-essential \ + gettext \ + libpq-dev \ + libsodium-dev \ + libsqlite3-dev \ + net-tools \ + postgresql \ + protobuf-compiler \ + python3 \ + python3-pip \ + zlib1g-dev + + cd lightning + pip3 install --user -U \ + pip \ + poetry \ + wheel \ + blinker \ + pytest-custom-exit-code==0.3.0 \ + pytest-json-report + + poetry install + poetry update + poetry export --without-hashes -f requirements.txt --output requirements.txt + pip install --user -U -r requirements.txt + pip install --user contrib/pyln-client contrib/pyln-testing flaky + + ./configure --disable-valgrind + make -j 16 + sudo make install + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Test with pytest + run: | + export EXPERIMENTAL_FEATURES=${{ matrix.experimental }} + export COMPAT=${{ matrix.deprecated }} + export SLOW_MACHINE=1 + export TEST_DEBUG=1 + export TRAVIS=1 + export CLN_PATH=${{ github.workspace }}/lightning + pip3 install --upgrade pip + pip3 install --user -U virtualenv pip > /dev/null + python3 .ci/test.py + + gather: + # A dummy task that depends on the full matrix of tests, and + # signals successful completion. Used for the PR status to pass + # before merging. + name: CI completion + runs-on: ubuntu-22.04 + needs: + - nightly-build-and-test + steps: + - name: Complete + run: | + echo Nightly CI completed successfully \ No newline at end of file From c2369c4bdb069e9e3606f01be79d1f3acf7d0634 Mon Sep 17 00:00:00 2001 From: fmhoeger Date: Mon, 12 Feb 2024 01:36:44 -0600 Subject: [PATCH 6/8] Update main workflow --- .github/workflows/main.yml | 56 ++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index af8171ca2..99fde7fbf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: Integration Tests +name: Integration Tests (latest) # Cancel duplicate jobs concurrency: @@ -7,7 +7,7 @@ concurrency: on: push: - branches: [ 407-use-cln-latest-release-binaries ] + branches: [ master ] pull_request: jobs: @@ -29,7 +29,7 @@ jobs: with: submodules: recursive - - name: Download & install Bitcoin binaries + - name: Download Bitcoin ${{ matrix.bitcoind-version }} & install binaries run: | export BITCOIND_VERSION=${{ matrix.bitcoind-version }} wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz @@ -37,24 +37,27 @@ jobs: sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin rm -rf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz bitcoin-${BITCOIND_VERSION} - - name: Checkout c-lightning@v23.11 + - name: Download Core Lightning latest & install binaries + run: | + url=$(curl -s https://api.github.com/repos/ElementsProject/lightning/releases/latest | jq '.assets[] | select(.name | contains("22.04")) | .browser_download_url' | tr -d '\"') + wget $url + sudo tar -xvf ${url##*/} -C /usr/local --strip-components=2 + + - name: Checkout Core Lightning uses: actions/checkout@v4 with: repository: 'ElementsProject/lightning' path: 'lightning' - ref: v23.11 + fetch-depth: 0 # fetch all history for all branches and tags submodules: 'recursive' - fetch-depth: 0 # Required for pyln versions to be recognized - - name: Download & install c-lightning binaries - run: | - url=$(curl -s https://api.github.com/repos/ElementsProject/lightning/releases/latest | jq '.assets[] | select(.name | contains("22.04")) | .browser_download_url' | tr -d '\"') - wget $url - sudo tar -xvf ${url##*/} -C /usr/local --strip-components=2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - export EXPERIMENTAL_FEATURES=${{ matrix.experimental }} - export COMPAT=${{ matrix.deprecated }} - export VALGRIND=0 + - name: Install Core Lightning Python package dependencies + run: | sudo apt-get install -y \ python3 \ python3-pip \ @@ -74,22 +77,29 @@ jobs: pip install --user -U -r requirements.txt pip install --user contrib/pyln-client contrib/pyln-testing flaky - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Test with pytest + - name: Run pytest tests run: | - export EXPERIMENTAL_FEATURES=${{ matrix.experimental }} + export CLN_PATH=${{ github.workspace }}/lightning export COMPAT=${{ matrix.deprecated }} + export EXPERIMENTAL_FEATURES=${{ matrix.experimental }} export SLOW_MACHINE=1 export TEST_DEBUG=1 export TRAVIS=1 - export CLN_PATH=${{ github.workspace }}/lightning + export VALGRIND=0 + pip3 install --upgrade pip pip3 install --user -U virtualenv pip > /dev/null - python3 .ci/test.py + + # Checkout the branch in order to be able to collect the changed plugins below with 'git diff' + git fetch && git checkout ${{ github.base_ref }} + + # Collect the plugins that have changed + echo "Using git diff ${{github.base_ref}} ${{github.sha}}" + plugin_dirs=$(git diff --name-only ${{ github.base_ref }} ${{ github.event.after }} | cut -d "/" -f1 | uniq | grep -v '^\.' | while read -r line; do [ -d "$line" ] && [[ "$line" != "archived" ]] && echo $line; done) + + # Run the tests + echo Testing plugins: $(echo "$plugin_dirs") + python3 .ci/test.py $(echo "$plugin_dirs") gather: # A dummy task that depends on the full matrix of tests, and From 0ec7921da49319c56177aab24afaac0088078601 Mon Sep 17 00:00:00 2001 From: fmhoeger Date: Tue, 13 Feb 2024 13:36:48 -0600 Subject: [PATCH 7/8] Update main workflow --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 99fde7fbf..0ec21fb77 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,6 +96,7 @@ jobs: # Collect the plugins that have changed echo "Using git diff ${{github.base_ref}} ${{github.sha}}" plugin_dirs=$(git diff --name-only ${{ github.base_ref }} ${{ github.event.after }} | cut -d "/" -f1 | uniq | grep -v '^\.' | while read -r line; do [ -d "$line" ] && [[ "$line" != "archived" ]] && echo $line; done) + git checkout v23.11 # Run the tests echo Testing plugins: $(echo "$plugin_dirs") From d787d887f8f86be36c702ac82ccc7bbfd6022ff5 Mon Sep 17 00:00:00 2001 From: fmhoeger Date: Tue, 13 Feb 2024 13:51:23 -0600 Subject: [PATCH 8/8] Update nightly workflow --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e7748d34b..188eb84b0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -104,4 +104,4 @@ jobs: steps: - name: Complete run: | - echo Nightly CI completed successfully \ No newline at end of file + echo Nightly CI completed successfully