diff --git a/.github/actions/numba_cache/action.yml b/.github/actions/numba_cache/action.yml
new file mode 100644
index 00000000..808bce84
--- /dev/null
+++ b/.github/actions/numba_cache/action.yml
@@ -0,0 +1,63 @@
+name: Numba cache setup
+description: "Tries to restore the numba cache and sets relevant environment variables for numba to use"
+
+inputs:
+  cache_name:
+    description: "The name of the cache"
+    required: true
+  runner_os:
+    description: "The runner os"
+    required: true
+  python_version:
+    description: "The python version"
+    required: true
+  restore_cache:
+    description: "Whether to restore the cache"
+    required: false
+    default: "true"
+
+runs:
+  using: "composite"
+  steps:
+    # Set the location for numba cache to exist
+    - name: Set numba cache env
+      run: echo "NUMBA_CACHE_DIR=${{ github.workspace }}/.numba_cache" >> $GITHUB_ENV
+      shell: bash
+
+    # Sets the cpu name for numba to use
+    - name: Set numba cpu env
+      run: echo "NUMBA_CPU_NAME=generic" >> $GITHUB_ENV
+      shell: bash
+
+    # Sets the cpu features for numba to use
+    - name: Set numba cpu features env
+      run: echo "NUMBA_CPU_FEATURES=+64bit +adx +aes +avx +avx2 -avx512bf16 -avx512bitalg -avx512bw -avx512cd -avx512dq -avx512er -avx512f -avx512ifma -avx512pf -avx512vbmi -avx512vbmi2 -avx512vl -avx512vnni -avx512vpopcntdq +bmi +bmi2 -cldemote -clflushopt -clwb -clzero +cmov +cx16 +cx8 -enqcmd +f16c +fma -fma4 +fsgsbase +fxsr -gfni +invpcid -lwp +lzcnt +mmx +movbe -movdir64b -movdiri -mwaitx +pclmul -pconfig -pku +popcnt -prefetchwt1 +prfchw -ptwrite -rdpid +rdrnd +rdseed +rtm +sahf -sgx -sha -shstk +sse +sse2 +sse3 +sse4.1 +sse4.2 -sse4a +ssse3 -tbm -vaes -vpclmulqdq -waitpkg -wbnoinvd -xop +xsave -xsavec +xsaveopt -xsaves" >> $GITHUB_ENV
+      shell: bash
+
+    # Set the CICD_RUNNING env so that numba knows it is running in a CI environment
+    - name: Set CICD_RUNNING env
+      run: echo "CICD_RUNNING=1" >> $GITHUB_ENV
+      shell: bash
+
+    # Get current date for cache restore
+    - name: Get Current Date
+      run: echo "CURRENT_DATE=$(date +'%d/%m/%Y')" >> $GITHUB_ENV
+      shell: bash
+
+      # GNU tar on windows runs much faster than the default BSD tar
+    - name: Use GNU tar instead BSD tar if Windows
+      if: ${{ inputs.runner_os == 'Windows' }}
+      shell: cmd
+      run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%"
+
+    # Restore cache if restore_cache is true
+    - name: Restore cache
+      uses: actions/cache/restore@v4
+      if: ${{ inputs.restore_cache == 'true' }}
+      with:
+        path: ${{ github.workspace }}/.numba_cache
+        # Try restore using today's date
+        key: numba-${{ inputs.cache_name }}-${{ inputs.runner_os }}-${{ inputs.python_version }}-${{ env.CURRENT_DATE }}
+        # If cant restore with today's date try another cache (without date)
+        restore-keys: |
+          numba-${{ inputs.cache_name }}-${{ inputs.runner_os }}-${{ inputs.python_version }}-
diff --git a/.github/workflows/issue_comment_edited.yml b/.github/workflows/issue_comment_edited.yml
new file mode 100644
index 00000000..5b7ba972
--- /dev/null
+++ b/.github/workflows/issue_comment_edited.yml
@@ -0,0 +1,33 @@
+name: Issue Comment Posted
+
+on:
+  issue_comment:
+    types: [edited]
+
+jobs:
+  check-pr-labels:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: build_tools
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install PyGithub
+        run: pip install -Uq PyGithub
+
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.PR_APP_ID }}
+          private-key: ${{ secrets.PR_APP_KEY }}
+
+      - name: Assign issue
+        run: python build_tools/comment_pr_labeler.py
+        env:
+          CONTEXT_GITHUB: ${{ toJson(github) }}
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
diff --git a/.github/workflows/issue_comment_posted.yml b/.github/workflows/issue_comment_posted.yml
new file mode 100644
index 00000000..8cd7ce6c
--- /dev/null
+++ b/.github/workflows/issue_comment_posted.yml
@@ -0,0 +1,33 @@
+name: Issue Comment Posted
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  self-assign:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: build_tools
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install PyGithub
+        run: pip install -Uq PyGithub
+
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.PR_APP_ID }}
+          private-key: ${{ secrets.PR_APP_KEY }}
+
+      - name: Assign issue
+        run: python build_tools/issue_assign.py
+        env:
+          CONTEXT_GITHUB: ${{ toJson(github) }}
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
diff --git a/.github/workflows/periodic_github_maintenace.yml b/.github/workflows/periodic_github_maintenace.yml
new file mode 100644
index 00000000..01569ca5
--- /dev/null
+++ b/.github/workflows/periodic_github_maintenace.yml
@@ -0,0 +1,34 @@
+name: GitHub Maintenance
+
+on:
+  schedule:
+    # Run on 1st and 15th of every month at 01:00 AM UTC
+    - cron:  "0 1 1,15 * * *"
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  contents: write
+
+jobs:
+  stale_branches:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.PR_APP_ID }}
+          private-key: ${{ secrets.PR_APP_KEY }}
+
+      - name: Stale Branches
+        uses: crs-k/stale-branches@v5.0.0
+        with:
+          repo-token: ${{ steps.app-token.outputs.token }}
+          days-before-stale: 365
+          days-before-delete: 99999
+          comment-updates: true
+          tag-committer: true
+          stale-branch-label: "stale branch"
+          compare-branches: "info"
+          pr-check: true
diff --git a/.github/workflows/periodic_tests.yml b/.github/workflows/periodic_tests.yml
index 54278375..d4ded820 100644
--- a/.github/workflows/periodic_tests.yml
+++ b/.github/workflows/periodic_tests.yml
@@ -49,6 +49,14 @@ jobs:
         with:
           python-version: "3.10"
 
+      - name: Use numba cache to set env variables but not restore cache
+        uses: ./.github/actions/numba_cache
+        with:
+          cache_name: "run-notebook-examples"
+          runner_os: ${{ runner.os }}
+          python_version: "3.10"
+          restore_cache: "false"
+
       - name: Install
         uses: nick-fields/retry@v3
         with:
@@ -60,13 +68,19 @@ jobs:
         run: build_tools/run_examples.sh
         shell: bash
 
+      - name: Save new cache
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ github.workspace }}/.numba_cache
+          key: numba-run-notebook-examples-${{ runner.os }}-3.10-${{ env.CURRENT_DATE }}
+
   pytest:
     runs-on: ${{ matrix.os }}
 
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-20.04, macOS-13, windows-2022 ]
+        os: [ ubuntu-20.04, macos-14, windows-2022 ]
         python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
 
     steps:
@@ -76,6 +90,14 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Use numba cache to set env variables but not restore cache
+        uses: ./.github/actions/numba_cache
+        with:
+          cache_name: "pytest"
+          runner_os: ${{ runner.os }}
+          python_version: ${{ matrix.python-version }}
+          restore_cache: "false"
+
       - name: Install
         uses: nick-fields/retry@v3
         with:
@@ -89,6 +111,12 @@ jobs:
       - name: Tests
         run: python -m pytest
 
+      - name: Save new cache
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ github.workspace }}/.numba_cache
+          key: numba-pytest-${{ runner.os }}-${{ matrix.python-version}}-${{ env.CURRENT_DATE }}
+
   codecov:
     runs-on: ubuntu-20.04
 
@@ -113,7 +141,7 @@ jobs:
         run: python -m pip list
 
       - name: Run tests
-        run: python -m pytest --cov=aeon --cov-report=xml
+        run: python -m pytest --cov=aeon --cov-report=xml --timeout 1800
 
       - uses: codecov/codecov-action@v4
         env:
diff --git a/.github/workflows/pr_examples.yml b/.github/workflows/pr_examples.yml
index 821c4eaa..a5a1e8d1 100644
--- a/.github/workflows/pr_examples.yml
+++ b/.github/workflows/pr_examples.yml
@@ -28,6 +28,13 @@ jobs:
         with:
           python-version: "3.10"
 
+      - name: Restore numba cache
+        uses: ./.github/actions/numba_cache
+        with:
+            cache_name: "run-notebook-examples"
+            runner_os: ${{ runner.os }}
+            python_version: "3.10"
+
       - name: Install
         uses: nick-fields/retry@v3
         with:
@@ -36,5 +43,5 @@ jobs:
           command: python -m pip install .[all_extras,dev,binder,unstable_extras]
 
       - name: Run example notebooks
-        run: build_tools/run_examples.sh
+        run: build_tools/run_examples.sh ${{ github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'full examples run') }}
         shell: bash
diff --git a/.github/workflows/pr_opened.yml b/.github/workflows/pr_opened.yml
index 0a28bd98..efe1cbaa 100644
--- a/.github/workflows/pr_opened.yml
+++ b/.github/workflows/pr_opened.yml
@@ -34,11 +34,17 @@ jobs:
 
       - name: Label pull request
         id: label-pr
-        run: python build_tools/pr_labeler.py ${{ steps.app-token.outputs.token }}
+        run: python build_tools/pr_labeler.py
         env:
           CONTEXT_GITHUB: ${{ toJson(github) }}
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
 
       - name: Write pull request comment
-        run: python build_tools/pr_open_commenter.py ${{ steps.app-token.outputs.token }} ${{ steps.label-pr.outputs.title-labels }} ${{ steps.label-pr.outputs.title-labels-new }} ${{ steps.label-pr.outputs.content-labels }} ${{ steps.label-pr.outputs.content-labels-status }}
+        run: python build_tools/pr_open_commenter.py
         env:
           CONTEXT_GITHUB: ${{ toJson(github) }}
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
+          TITLE_LABELS: ${{ steps.label-pr.outputs.title-labels }}
+          TITLE_LABELS_NEW: ${{ steps.label-pr.outputs.title-labels-new }}
+          CONTENT_LABELS: ${{ steps.label-pr.outputs.content-labels }}
+          CONTENT_LABELS_STATUS: ${{ steps.label-pr.outputs.content-labels-status }}
diff --git a/.github/workflows/pr_pre_commit.yml b/.github/workflows/pr_pre_commit.yml
index c77981e4..8030aa9e 100644
--- a/.github/workflows/pr_pre_commit.yml
+++ b/.github/workflows/pr_pre_commit.yml
@@ -17,13 +17,22 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
+      - uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.PR_APP_ID }}
+          private-key: ${{ secrets.PR_APP_KEY }}
+
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ steps.app-token.outputs.token }}
 
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 
-      - uses: tj-actions/changed-files@v44.5.2
+      - uses: tj-actions/changed-files@v44
         id: changed-files
 
       - name: List changed files
@@ -40,5 +49,9 @@ jobs:
         with:
           extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }}
 
-      - if: ${{ failure() && github.event_name == 'pull_request' && github.event.pull_request.draft == false }}
-        uses: pre-commit-ci/lite-action@v1.0.2
+      - if: ${{ failure() && github.event_name == 'pull_request' && github.event.pull_request.draft == false && !contains(github.event.pull_request.labels.*.name, 'stop pre-commit fixes')}}
+        name: Push pre-commit fixes
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: Automatic `pre-commit` fixes
+          commit_user_name: tsml-actions-bot[bot]
diff --git a/.github/workflows/pr_pytest.yml b/.github/workflows/pr_pytest.yml
index 89db0540..0061cde8 100644
--- a/.github/workflows/pr_pytest.yml
+++ b/.github/workflows/pr_pytest.yml
@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-20.04, macOS-13, windows-2022 ]
+        os: [ ubuntu-20.04, macos-14, windows-2022 ]
         python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
 
     steps:
@@ -33,6 +33,13 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Restore numba cache
+        uses: ./.github/actions/numba_cache
+        with:
+          cache_name: "pytest"
+          runner_os: ${{ runner.os }}
+          python_version: ${{ matrix.python-version }}
+
       - name: Install
         uses: nick-fields/retry@v3
         with:
@@ -67,7 +74,7 @@ jobs:
           command: python -m pip install .[all_extras,dev,unstable_extras]
 
       - name: Tests
-        run: python -m pytest --cov=tsml_eval --cov-report=xml
+        run: python -m pytest --cov=tsml_eval --cov-report=xml --timeout 1800
 
       - uses: codecov/codecov-action@v4
         env:
diff --git a/.github/workflows/precommit_autoupdate.yml b/.github/workflows/precommit_autoupdate.yml
index 04615ce4..2f25e607 100644
--- a/.github/workflows/precommit_autoupdate.yml
+++ b/.github/workflows/precommit_autoupdate.yml
@@ -17,7 +17,7 @@ jobs:
         with:
           python-version: "3.10"
 
-      - uses: browniebroke/pre-commit-autoupdate-action@v1.0.0
+      - uses: browniebroke/pre-commit-autoupdate-action@v1
 
       - if: always()
         uses: actions/create-github-app-token@v1
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e60f56cd..db4d8a03 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -51,7 +51,7 @@ jobs:
 
     strategy:
       matrix:
-        os: [ ubuntu-20.04, macOS-13, windows-2022 ]
+        os: [ ubuntu-20.04, macos-14, windows-2022 ]
         python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
 
     steps:
diff --git a/build_tools/comment_pr_labeler.py b/build_tools/comment_pr_labeler.py
new file mode 100644
index 00000000..e87425f3
--- /dev/null
+++ b/build_tools/comment_pr_labeler.py
@@ -0,0 +1,44 @@
+"""Labels PRs based on bot comment checkboxes."""
+
+import json
+import os
+import sys
+
+from github import Github
+
+context_dict = json.loads(os.getenv("CONTEXT_GITHUB"))
+
+repo = context_dict["repository"]
+g = Github(os.getenv("GITHUB_TOKEN"))
+repo = g.get_repo(repo)
+issue_number = context_dict["event"]["issue"]["number"]
+issue = repo.get_issue(number=issue_number)
+comment_body = context_dict["event"]["comment"]["body"]
+comment_user = context_dict["event"]["comment"]["user"]["login"]
+labels = [label.name for label in issue.get_labels()]
+
+if comment_user != "tsml-actions-bot[bot]":
+    sys.exit(0)
+
+
+def check_label_option(label, option):
+    """Add or remove a label based on a checkbox in a comment."""
+    if f"- [x] {option}" in comment_body:
+        if label not in labels:
+            issue.add_to_labels(label)
+    elif f"- [ ] {option}" in comment_body:
+        if label in labels:
+            issue.remove_from_labels(label)
+
+
+label_options = [
+    ("full pytest actions", "Run `pre-commit` checks for all files"),
+    ("full examples run", "Run all notebook example tests"),
+    (
+        "stop pre-commit fixes",
+        "Stop automatic `pre-commit` fixes (always disabled for drafts)",
+    ),
+]
+
+for option in label_options:
+    check_label_option(option[0], option[1])
diff --git a/build_tools/issue_assign.py b/build_tools/issue_assign.py
new file mode 100644
index 00000000..790d7e05
--- /dev/null
+++ b/build_tools/issue_assign.py
@@ -0,0 +1,31 @@
+"""Script for the GitHub issue self-assign bot.
+
+It checks if a comment on an issue or PR includes the trigger
+phrase (as defined) and a mentioned user.
+If it does, it assigns the issue/PR to the mentioned user.
+"""
+
+import json
+import os
+import re
+
+from github import Github
+
+context_dict = json.loads(os.getenv("CONTEXT_GITHUB"))
+
+repo = context_dict["repository"]
+g = Github(os.getenv("GITHUB_TOKEN"))
+repo = g.get_repo(repo)
+issue_number = context_dict["event"]["issue"]["number"]
+issue = repo.get_issue(number=issue_number)
+comment_body = context_dict["event"]["comment"]["body"]
+
+# Assign tagged used to the issue if the comment includes the trigger phrase
+body = comment_body.lower()
+if "@tsml-actions-bot" in body and "assign" in body:
+    mentioned_users = re.findall(r"@[a-zA-Z0-9_-]+", comment_body)
+    mentioned_users = [user[1:] for user in mentioned_users]
+    mentioned_users.remove("tsml-actions-bot")
+
+    for user in mentioned_users:
+        issue.add_to_assignees(user)
diff --git a/build_tools/pr_labeler.py b/build_tools/pr_labeler.py
index 383e514a..ae4c0798 100644
--- a/build_tools/pr_labeler.py
+++ b/build_tools/pr_labeler.py
@@ -15,7 +15,7 @@
 context_dict = json.loads(os.getenv("CONTEXT_GITHUB"))
 
 repo = context_dict["repository"]
-g = Github(sys.argv[1])
+g = Github(os.getenv("GITHUB_TOKEN"))
 repo = g.get_repo(repo)
 pr_number = context_dict["event"]["number"]
 pr = repo.get_pull(number=pr_number)
diff --git a/build_tools/pr_open_commenter.py b/build_tools/pr_open_commenter.py
index bc6be5a1..564e346e 100644
--- a/build_tools/pr_open_commenter.py
+++ b/build_tools/pr_open_commenter.py
@@ -12,7 +12,7 @@
 context_dict = json.loads(os.getenv("CONTEXT_GITHUB"))
 
 repo = context_dict["repository"]
-g = Github(sys.argv[1])
+g = Github(os.getenv("GITHUB_TOKEN"))
 repo = g.get_repo(repo)
 pr_number = context_dict["event"]["number"]
 pr = repo.get_pull(number=pr_number)
@@ -20,11 +20,10 @@
 if "[bot]" in pr.user.login:
     sys.exit(0)
 
-print(sys.argv[2:])  # noqa
-title_labels = sys.argv[2][1:-1].split(",")
-title_labels_new = sys.argv[3][1:-1].split(",")
-content_labels = sys.argv[4][1:-1].split(",")
-content_labels_status = sys.argv[5]
+title_labels = os.getenv("TITLE_LABELS")[1:-1].replace("'", "").split(",")
+title_labels_new = os.getenv("TITLE_LABELS_NEW")[1:-1].replace("'", "").split(",")
+content_labels = os.getenv("CONTENT_LABELS")[1:-1].replace("'", "").split(",")
+content_labels_status = os.getenv("CONTENT_LABELS_STATUS")
 
 replacement_labels = [
     ("tsmlresearchresources", "tsml research resources"),
@@ -105,5 +104,13 @@
 {content_labels_str}
 
 The [Checks](https://github.com/time-series-machine-learning/tsml-eval/pull/{pr_number}/checks) tab will show the status of our automated tests. You can click on individual test runs in the tab or "Details" in the panel below to see more information if there is a failure.
+
+### PR CI actions
+
+These checkboxes will add labels to enable/disable CI functionality for this PR. This may not take effect immediately, and a new commit may be required to run the new configuration.
+
+- [ ] Run `pre-commit` checks for all files
+- [ ] Run all notebook example tests
+- [ ] Stop automatic `pre-commit` fixes (always disabled for drafts)
     """  # noqa
 )
diff --git a/build_tools/run_examples.sh b/build_tools/run_examples.sh
index c4a3ef41..12532d87 100755
--- a/build_tools/run_examples.sh
+++ b/build_tools/run_examples.sh
@@ -9,9 +9,9 @@ excluded=(
   "tsml_eval/publications/y2023/distance_based_clustering/package_distance_timing.ipynb"
   "examples/regression_experiments.ipynb"
 )
-#if [ "$1" = true ]; then
-#  excluded+=()
-#fi
+if [ "$1" = true ]; then
+  excluded+=()
+fi
 
 shopt -s lastpipe
 notebooks=()
diff --git a/tsml_eval/testing/_cicd_numba_caching.py b/tsml_eval/testing/_cicd_numba_caching.py
new file mode 100644
index 00000000..64af39d7
--- /dev/null
+++ b/tsml_eval/testing/_cicd_numba_caching.py
@@ -0,0 +1,113 @@
+"""CICD numba caching functions."""
+
+import pickle
+import subprocess
+
+import numba.core.caching
+
+
+def get_invalid_numba_files():
+    """Get the files that have been changed since the last commit.
+
+    This function is used to get the files that have been changed. This is needed
+    because custom logic to save the numba cache has been implemented for numba.
+    This function returns the file names that have been changed and if they appear
+    in here any numba functions cache are invalidated.
+
+    Returns
+    -------
+    list
+        List of file names that have been changed.
+    """
+    subprocess.run(["git", "fetch", "origin", "main"], check=True)
+
+    result = subprocess.run(
+        ["git", "diff", "--name-only", "origin/main"],
+        check=True,
+        capture_output=True,
+        text=True,  # Makes the output text instead of bytes
+    )
+
+    files = result.stdout.split("\n")
+
+    files = [file for file in files if file]
+
+    clean_files = []
+
+    for temp in files:
+        if temp.endswith(".py"):
+            clean_files.append((temp.split("/")[-1]).strip(".py"))
+
+    return clean_files
+
+
+# Retry the git fetch and git diff commands in case of failure
+retry = 0
+while retry < 3:
+    try:
+        CHANGED_FILES = get_invalid_numba_files()
+        break
+    except subprocess.CalledProcessError:
+        retry += 1
+
+# If the retry count is reached, raise an error
+if retry == 3:
+    raise Exception("Failed to get the changed files from git.")
+
+
+def custom_load_index(self):
+    """Overwrite load index method for numba.
+
+    This is used to overwrite the numba internal logic to allow for caching during
+    the cicd run. Numba traditionally checks the timestamp of the file and if it
+    has changed it invalidates the cache. This is not ideal for the cicd run as
+    the cache restore is always before the files (since it is cloned in) and
+    thus the cache is always invalidated. This custom method ignores the timestamp
+    element and instead just checks the file name. This isn't as fine grained as numba
+    but it is better to invalidate more and make sure the right function has been
+    compiled than try to be too clever and miss some.
+
+    Returns
+    -------
+    dict
+        Dictionary of the cached functions.
+    """
+    try:
+        with open(self._index_path, "rb") as f:
+            version = pickle.load(f)
+            data = f.read()
+    except FileNotFoundError:
+        return {}
+    if version != self._version:
+        return {}
+    stamp, overloads = pickle.loads(data)
+    cache_filename = self._index_path.split("/")[-1].split("-")[0].split(".")[0]
+    if stamp[1] != self._source_stamp[1] or cache_filename in CHANGED_FILES:
+        return {}
+    else:
+        return overloads
+
+
+original_load_index = numba.core.caching.IndexDataCacheFile._load_index
+numba.core.caching.IndexDataCacheFile._load_index = custom_load_index
+
+
+# Force all numba functions to be cached
+original_jit = numba.core.decorators._jit
+
+
+def custom_njit(*args, **kwargs):
+    """Force jit to cache.
+
+    This is used for libraries like stumpy that doesn't cache by default. This
+    function will force all functions running to be cache'd
+    """
+    target = kwargs["targetoptions"]
+    # This target can't be cached
+    if "no_cpython_wrapper" not in target:
+        kwargs["cache"] = True
+    return original_jit(*args, **kwargs)
+
+
+# Overwrite the jit function with the custom version
+numba.core.decorators._jit = custom_njit
diff --git a/tsml_eval/testing/testing_config.py b/tsml_eval/testing/testing_config.py
new file mode 100644
index 00000000..f23f0de4
--- /dev/null
+++ b/tsml_eval/testing/testing_config.py
@@ -0,0 +1,6 @@
+"""Test configuration."""
+
+import os
+
+if os.environ.get("CICD_RUNNING") == "1":
+    import tsml_eval.testing._cicd_numba_caching  # noqa: F401