Merge develop into 2.8

.github/workflows/release.yml

@@ -9,6 +9,68 @@ on:
       - 'v*.*.*'
 
 jobs:
+  test-latest-submodules:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository with submodules
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          # This fetches only a single branch by default, so additional fetch is needed
+          fetch-depth: 0 # Optionally, set to 0 to fetch all history for all branches and tags
+
+      - name: Initialize and update submodule
+        run: |
+          git submodule update --init --recursive
+
+      - name: Check if submodules are up to date
+        shell: bash
+        run: |
+          NOTEBOOKS_PATH=docs/notebooks
+          FAQ_PATH=docs/faq
+
+          # Checking out Notebooks submodule with the same branch as the main project/develop
+          echo "Checking $NOTEBOOKS_PATH for updates..."
+          cd $NOTEBOOKS_PATH
+          NOTEBOOKS_CURRENT_COMMIT=$(git rev-parse HEAD)
+          echo $(git fetch --all --verbose)
+          echo $(git remote get-url origin)
+          if git show-ref --verify refs/remotes/origin/develop; then
+           echo "Branch develop exists."
+          else
+            echo "::error::Branch develop does not exist on remote."
+            exit 1
+          fi
+          NOTEBOOKS_LATEST_COMMIT=$(git rev-parse refs/remotes/origin/develop)
+          echo "NOTEBOOKS_LATEST_COMMIT: $NOTEBOOKS_LATEST_COMMIT"
+          echo "NOTEBOOKS_CURRENT_COMMIT: $NOTEBOOKS_CURRENT_COMMIT"
+
+
+          cd ../..
+          if [ "$NOTEBOOKS_LATEST_COMMIT" != "$NOTEBOOKS_CURRENT_COMMIT" ]; then
+            echo "::error ::Submodule $NOTEBOOKS_PATH is not up to date with the develop branch. Please update it."
+            exit 1
+          else
+            echo "Submodule $NOTEBOOKS_PATH is up to date with the develop branch."
+          fi
+
+          # Checking FAQs only on the develop branch.
+          echo "Checking $FAQ_PATH for updates..."
+          cd $FAQ_PATH
+          FAQ_CURRENT_COMMIT=$(git rev-parse HEAD)
+          echo $(git fetch --all --verbose)
+          echo $(git remote get-url origin)
+          FAQ_LATEST_COMMIT=$(git rev-parse refs/remotes/origin/develop)
+          echo "FAQ_LATEST_COMMIT: $FAQ_LATEST_COMMIT"
+          echo "FAQ_CURRENT_COMMIT: $FAQ_CURRENT_COMMIT"
+          cd ../..
+          if [ "$FAQ_LATEST_COMMIT" != "$FAQ_CURRENT_COMMIT" ]; then
+            echo "::error ::Submodule $FAQ_PATH is not up to date. Please update it."
+            exit 1
+          else
+            echo "Submodule $FAQ_PATH is up to date."
+          fi
+
   github-release:
     runs-on: ubuntu-latest
     steps:

.github/workflows/run_tests.yml

@@ -27,86 +27,6 @@ jobs:
           pip install pre-commit
           pre-commit run # this should be really more agressive
 
-  test-latest-submodules:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository with submodules
-        uses: actions/checkout@v4
-        with:
-          submodules: 'recursive'
-          # This fetches only a single branch by default, so additional fetch is needed
-          fetch-depth: 0 # Optionally, set to 0 to fetch all history for all branches and tags
-
-      - name: Determine current branch or PR ref
-        id: get_branch
-        run: |
-          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
-            # Extract the base branch of the PR
-            BRANCH_NAME="${{ github.event.pull_request.base.ref }}"
-            echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
-          else
-            # Assume it's a push event, extract the branch name from $GITHUB_REF
-            BRANCH_NAME=$(echo $GITHUB_REF | sed 's|refs/heads/||')
-            echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
-          fi
-          # Now echoing the BRANCH_NAME to verify
-          echo "BRANCH_NAME: $BRANCH_NAME"
-        shell: bash
-        env:
-          GITHUB_REF: ${{ github.ref }}
-
-      - name: Initialize and update submodule
-        run: |
-          git submodule update --init --recursive
-
-      - name: Check if submodules are up to date
-        shell: bash
-        run: |
-          NOTEBOOKS_PATH=docs/notebooks
-          FAQ_PATH=docs/faq
-
-          # Checking out Notebooks submodule with the same branch as the main project
-          echo "Checking $NOTEBOOKS_PATH for updates..."
-          cd $NOTEBOOKS_PATH
-          NOTEBOOKS_CURRENT_COMMIT=$(git rev-parse HEAD)
-          echo $(git fetch --all --verbose)
-          echo $(git remote get-url origin)
-          if git show-ref --verify refs/remotes/origin/$BRANCH_NAME; then
-           echo "Branch $BRANCH_NAME exists."
-          else
-            echo "::error::Branch $BRANCH_NAME does not exist on remote."
-            exit 1
-          fi
-          NOTEBOOKS_LATEST_COMMIT=$(git rev-parse refs/remotes/origin/${{ env.BRANCH_NAME }})
-          echo "NOTEBOOKS_LATEST_COMMIT: $NOTEBOOKS_LATEST_COMMIT"
-          echo "NOTEBOOKS_CURRENT_COMMIT: $NOTEBOOKS_CURRENT_COMMIT"
-
-
-          cd ../..
-          if [ "$NOTEBOOKS_LATEST_COMMIT" != "$NOTEBOOKS_CURRENT_COMMIT" ]; then
-            echo "::error ::Submodule $NOTEBOOKS_PATH is not up to date with the ${{ env.BRANCH_NAME }} branch. Please update it."
-            exit 1
-          else
-            echo "Submodule $NOTEBOOKS_PATH is up to date with the ${{ env.BRANCH_NAME }} branch."
-          fi
-
-          # Checking FAQs only on the develop branch.
-          echo "Checking $FAQ_PATH for updates..."
-          cd $FAQ_PATH
-          FAQ_CURRENT_COMMIT=$(git rev-parse HEAD)
-          echo $(git fetch --all --verbose)
-          echo $(git remote get-url origin)
-          FAQ_LATEST_COMMIT=$(git rev-parse refs/remotes/origin/develop)
-          echo "FAQ_LATEST_COMMIT: $FAQ_LATEST_COMMIT"
-          echo "FAQ_CURRENT_COMMIT: $FAQ_CURRENT_COMMIT"
-          cd ../..
-          if [ "$FAQ_LATEST_COMMIT" != "$FAQ_CURRENT_COMMIT" ]; then
-            echo "::error ::Submodule $FAQ_PATH is not up to date. Please update it."
-            exit 1
-          else
-            echo "Submodule $FAQ_PATH is up to date."
-          fi
-
   build:
     name: Python ${{ matrix.python-version }} - ${{ matrix.platform }}
     runs-on: ${{ matrix.platform }}

.github/workflows/sync-to-readthedocs-repo.yaml

@@ -42,6 +42,16 @@ jobs:
           token: ${{ secrets.GH_PAT }}
           ref: ${{ needs.extract_branch_or_tag.outputs.ref_name }}
 
+      - name: Push corresponding reference to mirror repo if a branch
+        if: contains(github.ref, 'refs/heads/')
+        run: |
+          git fetch --unshallow origin ${{ needs.extract_branch_or_tag.outputs.ref_name }}
+          git pull origin ${{ needs.extract_branch_or_tag.outputs.ref_name }}
+          git remote add mirror https://github.com/flexcompute-readthedocs/tidy3d-docs.git
+          git push mirror ${{ needs.extract_branch_or_tag.outputs.ref_name }} --force # overwrites always
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       # Conditional Checkout for Tag
       - name: Checkout Tag if tag-triggered-sync
         if: contains(github.ref, 'refs/tags/')
@@ -53,11 +63,11 @@ jobs:
           ref: ${{ needs.extract_branch_or_tag.outputs.ref_name }}
           fetch-tags: true
 
-      - name: Push corresponding reference to mirror repo
+
+      - name: Push corresponding reference to mirror repo if a tag
+        if: contains(github.ref, 'refs/tags/')
         run: |
-          git fetch --unshallow origin ${{ needs.extract_branch_or_tag.outputs.ref_name }}
-          git pull origin ${{ needs.extract_branch_or_tag.outputs.ref_name }}
           git remote add mirror https://github.com/flexcompute-readthedocs/tidy3d-docs.git
           git push mirror ${{ needs.extract_branch_or_tag.outputs.ref_name }} --force # overwrites always
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test_daily_latest_submodule.yaml

@@ -0,0 +1,107 @@
+name: "tidy3d-submodule-daily-tests"
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 9 * * *' # Runs at 9am UK-time every day
+
+
+jobs:
+  test-latest-submodules:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository with submodules
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          # This fetches only a single branch by default, so additional fetch is needed
+          fetch-depth: 0 # Optionally, set to 0 to fetch all history for all branches and tags
+
+      - name: Initialize and update submodule
+        run: |
+          git submodule update --init --recursive
+
+      - name: Determine the latest pre/2.* branch
+        id: get_latest_pre_branch
+        run: |
+          # Fetch all branches
+          git fetch --all --quiet
+
+          # List all remote branches for debugging purposes
+          echo "Available branches:"
+          git branch -r
+
+          # List branches matching the pre/2.* pattern
+          BRANCHES=$(git branch -r | grep 'origin/pre/2\.' | sed 's|origin/||' | sort -V)
+
+          # Debugging: Print out the matched branches
+          echo "Matched branches with pre/2.* pattern:"
+          echo "$BRANCHES"
+
+          # Identify the latest branch
+          LATEST_BRANCH=$(echo "$BRANCHES" | tail -n 1)
+
+          # Set the latest branch as an environment variable
+          echo "LATEST_BRANCH=$LATEST_BRANCH" >> $GITHUB_ENV
+          echo "Latest pre/2.* branch is: $LATEST_BRANCH"
+
+      - name: Check submodules for multiple branches
+        shell: bash
+        run: |
+          BRANCHES=("develop" $LATEST_BRANCH) # Add your branches here
+
+          for BRANCH in "${BRANCHES[@]}"; do
+            echo "Analyzing branch: $BRANCH"
+
+            # Fetch all branches and tags
+            git fetch --all --verbose
+
+            # Checkout the branch
+            git checkout $BRANCH
+
+            NOTEBOOKS_PATH=docs/notebooks
+            FAQ_PATH=docs/faq
+
+            # Checking Notebooks submodule
+            echo "Checking $NOTEBOOKS_PATH for updates..."
+            cd $NOTEBOOKS_PATH
+            NOTEBOOKS_CURRENT_COMMIT=$(git rev-parse HEAD)
+            echo $(git fetch --all --verbose)
+            echo $(git remote get-url origin)
+            if git show-ref --verify refs/remotes/origin/$BRANCH; then
+              echo "Branch $BRANCH exists."
+            else
+              echo "::error::Branch $BRANCH does not exist on remote."
+              exit 1
+            fi
+            NOTEBOOKS_LATEST_COMMIT=$(git rev-parse refs/remotes/origin/${BRANCH})
+            echo "NOTEBOOKS_LATEST_COMMIT: $NOTEBOOKS_LATEST_COMMIT"
+            echo "NOTEBOOKS_CURRENT_COMMIT: $NOTEBOOKS_CURRENT_COMMIT"
+
+            cd ../..
+            if [ "$NOTEBOOKS_LATEST_COMMIT" != "$NOTEBOOKS_CURRENT_COMMIT" ]; then
+              echo "::error::Submodule $NOTEBOOKS_PATH is not up to date with the $BRANCH branch. Please update it."
+              exit 1
+            else
+              echo "Submodule $NOTEBOOKS_PATH is up to date with the $BRANCH branch."
+            fi
+
+            # Checking FAQs only on the develop branch
+            if [[ "$BRANCH" == "develop" ]]; then
+              echo "Checking $FAQ_PATH for updates..."
+              cd $FAQ_PATH
+              FAQ_CURRENT_COMMIT=$(git rev-parse HEAD)
+              echo $(git fetch --all --verbose)
+              echo $(git remote get-url origin)
+              FAQ_LATEST_COMMIT=$(git rev-parse refs/remotes/origin/develop)
+              echo "FAQ_LATEST_COMMIT: $FAQ_LATEST_COMMIT"
+              echo "FAQ_CURRENT_COMMIT: $FAQ_CURRENT_COMMIT"
+              cd ../..
+              if [ "$FAQ_LATEST_COMMIT" != "$FAQ_CURRENT_COMMIT" ]; then
+                echo "::error::Submodule $FAQ_PATH is not up to date. Please update it."
+                exit 1
+              else
+                echo "Submodule $FAQ_PATH is up to date."
+              fi
+            fi
+          done

.github/workflows/test_pr_latest_submodule.yaml

@@ -0,0 +1,72 @@
+name: "tidy3d-submodule-PR-tests"
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ latest ]
+  pull_request:
+    branches:
+      - latest
+
+jobs:
+  test-latest-submodules:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository with submodules
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          # This fetches only a single branch by default, so additional fetch is needed
+          fetch-depth: 0 # Optionally, set to 0 to fetch all history for all branches and tags
+
+      - name: Initialize and update submodule
+        run: |
+          git submodule update --init --recursive
+
+      - name: Check if submodules are up to date
+        shell: bash
+        run: |
+          NOTEBOOKS_PATH=docs/notebooks
+          FAQ_PATH=docs/faq
+
+          # Checking out Notebooks submodule with the same branch as the main project
+          echo "Checking $NOTEBOOKS_PATH for updates..."
+          cd $NOTEBOOKS_PATH
+          NOTEBOOKS_CURRENT_COMMIT=$(git rev-parse HEAD)
+          echo $(git fetch --all --verbose)
+          echo $(git remote get-url origin)
+          if git show-ref --verify refs/remotes/origin/develop; then
+           echo "Branch develop exists."
+          else
+            echo "::error::Branch develop does not exist on remote."
+            exit 1
+          fi
+          NOTEBOOKS_LATEST_COMMIT=$(git rev-parse refs/remotes/origin/develop)
+          echo "NOTEBOOKS_LATEST_COMMIT: $NOTEBOOKS_LATEST_COMMIT"
+          echo "NOTEBOOKS_CURRENT_COMMIT: $NOTEBOOKS_CURRENT_COMMIT"
+
+
+          cd ../..
+          if [ "$NOTEBOOKS_LATEST_COMMIT" != "$NOTEBOOKS_CURRENT_COMMIT" ]; then
+            echo "::error ::Submodule $NOTEBOOKS_PATH is not up to date with the develop branch. Please update it."
+            exit 1
+          else
+            echo "Submodule $NOTEBOOKS_PATH is up to date with the develop branch."
+          fi
+
+          # Checking FAQs only on the develop branch.
+          echo "Checking $FAQ_PATH for updates..."
+          cd $FAQ_PATH
+          FAQ_CURRENT_COMMIT=$(git rev-parse HEAD)
+          echo $(git fetch --all --verbose)
+          echo $(git remote get-url origin)
+          FAQ_LATEST_COMMIT=$(git rev-parse refs/remotes/origin/develop)
+          echo "FAQ_LATEST_COMMIT: $FAQ_LATEST_COMMIT"
+          echo "FAQ_CURRENT_COMMIT: $FAQ_CURRENT_COMMIT"
+          cd ../..
+          if [ "$FAQ_LATEST_COMMIT" != "$FAQ_CURRENT_COMMIT" ]; then
+            echo "::error ::Submodule $FAQ_PATH is not up to date. Please update it."
+            exit 1
+          else
+            echo "Submodule $FAQ_PATH is up to date."
+          fi

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.4.8"
+    rev: "v0.5.5"
     hooks:
      - id: ruff
        args: [ --fix ]