407 Use cln latest release binaries and add nightly master

.github/workflows/main.yml

@@ -1,4 +1,4 @@
-name: Integration Tests
+name: Integration Tests (latest)
 
 # Cancel duplicate jobs
 concurrency:
@@ -12,15 +12,15 @@ on:
 
 jobs:
   build-and-test:
-    name: Test PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, CLN=${{ matrix.cln-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }}
+    name: Test PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }}
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.12"]
         bitcoind-version: ["26.0"]
-        cln-version: ["master", "v23.11"]
         experimental: [1]
         deprecated: [0]
 
@@ -29,40 +29,38 @@ jobs:
       with:
         submodules: recursive
 
-    - name: Checkout c-lightning@${{ matrix.cln-version }}
-      uses: actions/checkout@v4
-      with:
-        repository: 'ElementsProject/lightning'
-        path: 'lightning'
-        ref: ${{ matrix.cln-version }}
-        submodules: 'recursive'
-        fetch-depth: 0  # Required for pyln versions to be recognized
-
-    - name: Download runtime dependencies
+    - name: Download Bitcoin ${{ matrix.bitcoind-version }} & install binaries
       run: |
         export BITCOIND_VERSION=${{ matrix.bitcoind-version }}
         wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz
         tar -xzf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz
         sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin
         rm -rf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz bitcoin-${BITCOIND_VERSION}
 
-    - name: Compile & install c-lightning@master
+    - name: Download Core Lightning latest & install binaries
+      run: |
+        url=$(curl -s https://api.github.com/repos/ElementsProject/lightning/releases/latest | jq '.assets[] | select(.name | contains("22.04")) | .browser_download_url' | tr -d '\"')
+        wget $url
+        sudo tar -xvf ${url##*/} -C /usr/local --strip-components=2
+
+    - name: Checkout Core Lightning
+      uses: actions/checkout@v4
+      with:
+        repository: 'ElementsProject/lightning'
+        path: 'lightning'
+        fetch-depth: 0  # fetch all history for all branches and tags
+        submodules: 'recursive'
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Core Lightning Python package dependencies
       run: |
-        export EXPERIMENTAL_FEATURES=${{ matrix.experimental }}
-        export COMPAT=${{ matrix.deprecated }}
-        export VALGRIND=0
         sudo apt-get install -y \
-          build-essential \
-          gettext \
-          libpq-dev \
-          libsodium-dev \
-          libsqlite3-dev \
-          net-tools \
-          postgresql \
-          protobuf-compiler \
           python3 \
           python3-pip \
-          zlib1g-dev
 
         cd lightning
         pip3 install --user -U \
@@ -79,26 +77,30 @@ jobs:
         pip install --user -U -r requirements.txt
         pip install --user contrib/pyln-client contrib/pyln-testing flaky
 
-        ./configure --disable-valgrind
-        make -j 16
-        sudo make install
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Test with pytest
+    - name: Run pytest tests
       run: |
-        export EXPERIMENTAL_FEATURES=${{ matrix.experimental }}
+        export CLN_PATH=${{ github.workspace }}/lightning
         export COMPAT=${{ matrix.deprecated }}
+        export EXPERIMENTAL_FEATURES=${{ matrix.experimental }}
         export SLOW_MACHINE=1
         export TEST_DEBUG=1
         export TRAVIS=1
-        export CLN_PATH=${{ github.workspace }}/lightning
+        export VALGRIND=0
+
         pip3 install --upgrade pip
         pip3 install --user -U virtualenv pip > /dev/null
-        python3 .ci/test.py
+
+        # Checkout the branch in order to be able to collect the changed plugins below with 'git diff'
+        git fetch && git checkout ${{ github.base_ref }}
+
+        # Collect the plugins that have changed
+        echo "Using git diff ${{github.base_ref}} ${{github.sha}}"
+        plugin_dirs=$(git diff --name-only ${{ github.base_ref }} ${{ github.event.after }} | cut -d "/" -f1 | uniq | grep -v '^\.' | while read -r line; do [ -d "$line" ] && [[ "$line" != "archived" ]] && echo $line; done)
+        git checkout v23.11
+
+        # Run the tests
+        echo Testing plugins: $(echo "$plugin_dirs")
+        python3 .ci/test.py $(echo "$plugin_dirs")
 
   gather:
     # A dummy task that depends on the full matrix of tests, and

.github/workflows/nightly.yml

@@ -0,0 +1,107 @@
+name: Nightly integration tests (master)
+
+on:
+  schedule:
+    - cron: "21 2 * * *"
+
+jobs:
+  nightly-build-and-test:
+    name: Test PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        bitcoind-version: ["26.0"]
+        experimental: [1]
+        deprecated: [0]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Download Bitcoin & install binaries
+      run: |
+        export BITCOIND_VERSION=${{ matrix.bitcoind-version }}
+        wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIND_VERSION}/bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz
+        tar -xzf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz
+        sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin
+        rm -rf bitcoin-${BITCOIND_VERSION}-x86_64-linux-gnu.tar.gz bitcoin-${BITCOIND_VERSION}
+
+    - name: Checkout Core Lightning
+      uses: actions/checkout@v4
+      with:
+        repository: 'ElementsProject/lightning'
+        path: 'lightning'
+        ref: master
+        submodules: 'recursive'
+        fetch-depth: 0  # Required for pyln versions to be recognized
+
+    - name: Compile & install Core Lightning
+      run: |
+        export EXPERIMENTAL_FEATURES=${{ matrix.experimental }}
+        export COMPAT=${{ matrix.deprecated }}
+        export VALGRIND=0
+        sudo apt-get install -y \
+          build-essential \
+          gettext \
+          libpq-dev \
+          libsodium-dev \
+          libsqlite3-dev \
+          net-tools \
+          postgresql \
+          protobuf-compiler \
+          python3 \
+          python3-pip \
+          zlib1g-dev
+
+        cd lightning
+        pip3 install --user -U \
+          pip \
+          poetry \
+          wheel \
+          blinker \
+          pytest-custom-exit-code==0.3.0 \
+          pytest-json-report
+
+        poetry install
+        poetry update
+        poetry export --without-hashes -f requirements.txt --output requirements.txt
+        pip install --user -U -r requirements.txt
+        pip install --user contrib/pyln-client contrib/pyln-testing flaky
+
+        ./configure --disable-valgrind
+        make -j 16
+        sudo make install
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Test with pytest
+      run: |
+        export EXPERIMENTAL_FEATURES=${{ matrix.experimental }}
+        export COMPAT=${{ matrix.deprecated }}
+        export SLOW_MACHINE=1
+        export TEST_DEBUG=1
+        export TRAVIS=1
+        export CLN_PATH=${{ github.workspace }}/lightning
+        pip3 install --upgrade pip
+        pip3 install --user -U virtualenv pip > /dev/null
+        python3 .ci/test.py
+
+  gather:
+    # A dummy task that depends on the full matrix of tests, and
+    # signals successful completion. Used for the PR status to pass
+    # before merging.
+    name: CI completion
+    runs-on: ubuntu-22.04
+    needs:
+      - nightly-build-and-test
+    steps:
+      - name: Complete
+        run: |
+          echo Nightly CI completed successfully