From dc4bbd749a2b281e81faf7a99a2903bef278beca Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 12:35:44 -0400 Subject: [PATCH 01/12] fix: Fix string formatting in version function --- src/nbiatoolkit/nbia_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nbiatoolkit/nbia_cli.py b/src/nbiatoolkit/nbia_cli.py index 946977b..23f4d52 100644 --- a/src/nbiatoolkit/nbia_cli.py +++ b/src/nbiatoolkit/nbia_cli.py @@ -18,7 +18,7 @@ def version(): - f = """ + f = r""" _ ______ _______ ______ ____ _ __ / | / / __ )/ _/ |/_ __/___ ____ / / /__(_) /_ / |/ / __ |/ // /| | / / / __ \/ __ \/ / //_/ / __/ From 2f858265dd467267a0079919315280e62c34b173 Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 13:47:05 -0400 Subject: [PATCH 02/12] build: Add timeout for Unit-Tests job and checkout code with tag in Continuous-Deployment job --- .github/workflows/main.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1c3b36f..0697e9e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,6 +11,7 @@ on: jobs: Unit-Tests: runs-on: ${{ matrix.os }} + timeout-minutes: 10 # Consider increasing timeout strategy: matrix: os: [ubuntu-latest, macos-latest, macos-14] @@ -101,7 +102,7 @@ jobs: needs: [Unit-Tests] # if pulling to main, deploy to PyPI - if: github.ref == 'refs/heads/main' + # if: github.ref == 'refs/heads/main' # Set up operating system runs-on: ubuntu-latest @@ -140,8 +141,10 @@ jobs: if: needs.Continuous-Deployment.outputs.released == 'true' runs-on: ubuntu-latest steps: - - name: Checkout code + - name: Checkout the code with tag ${{ needs.Continuous-Deployment.outputs.tag }} uses: actions/checkout@v3 + with: + ref: ${{ needs.Continuous-Deployment.outputs.tag }} - name: Set up Python 3.12 uses: actions/setup-python@v4 @@ -170,24 +173,22 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + with: + ref: ${{ needs.Continuous-Deployment.outputs.tag }} - name: Set up QEMU - if: steps.release.outputs.released == 'true' uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - if: steps.release.outputs.released == 'true' uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - if: steps.release.outputs.released == 'true' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to the GitHub Container Registry - if: steps.release.outputs.released == 'true' uses: docker/login-action@v3 with: registry: ghcr.io From 6ff1e962c981f3a481728b2b0ba03c4d3d9edcc7 Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 13:47:35 -0400 Subject: [PATCH 03/12] build: Add development branch to CI/CD workflow --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0697e9e..8a32420 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ name: CI-CD on: push: # push to any branch * - branches: [ main ] + branches: [ main , development ] pull_request: branches: [ main , development] From 272a9f52be5f6e1a5f2474c5cc433000a17fa4b6 Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 13:49:48 -0400 Subject: [PATCH 04/12] fix: testing gha --- .github/workflows/main.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8a32420..9726be9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,14 +11,12 @@ on: jobs: Unit-Tests: runs-on: ${{ matrix.os }} - timeout-minutes: 10 # Consider increasing timeout + timeout-minutes: 15 # Consider increasing timeout + strategy: matrix: os: [ubuntu-latest, macos-latest, macos-14] python-version: ["3.12", "3.11", "3.10"] - # include: - # - os: ubuntu-latest - # python-version: "3.9" steps: - uses: actions/checkout@v3 @@ -124,12 +122,6 @@ jobs: with: fetch-depth: 0 - # This action uses Python Semantic Release v8 - # What this action does: - # - Determines the next version number based on the commit history - # - Creates a new tag with the new version number - # - Pushes the new tag to GitHub - # - Creates a GitHub release with the new version number - name: Python Semantic Release id: release uses: python-semantic-release/python-semantic-release@master From 71e68e02ce047331fe1adc7f5a658f9899c8d356 Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 13:57:21 -0400 Subject: [PATCH 05/12] feat: release on development --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3304de1..87302d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ changelog_file = "docs/project_info/CHANGELOG.md" exclude_commit_types = ["docs", "style", "refactor", "test", "chore"] [tool.semantic_release.branches.main] -match = "(main|master)" +match = "(main|master|development)" [tool.semantic_release.commit_parser_options] From b813e2a3e82d281f3cdfcb88415100ca451b43a2 Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 13:57:40 -0400 Subject: [PATCH 06/12] feat: release on development --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9726be9..84591a5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -97,7 +97,7 @@ jobs: issues: write pull-requests: write - needs: [Unit-Tests] + # needs: [Unit-Tests] # if pulling to main, deploy to PyPI # if: github.ref == 'refs/heads/main' From e2e9b74c9b790be49387df0fd596da24b4afde2b Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 1 Apr 2024 18:00:01 +0000 Subject: [PATCH 07/12] chore(sem-ver): 1.2.0 --- docs/project_info/CHANGELOG.md | 27 +++++++++++++++++++++++++++ pyproject.toml | 2 +- src/nbiatoolkit/nbia.py | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/project_info/CHANGELOG.md b/docs/project_info/CHANGELOG.md index bc07df8..e9a4c83 100644 --- a/docs/project_info/CHANGELOG.md +++ b/docs/project_info/CHANGELOG.md @@ -2,6 +2,29 @@ +## v1.2.0 (2024-04-01) + +### Build + +* build: Add development branch to CI/CD workflow ([`6ff1e96`](https://github.com/jjjermiah/nbia-toolkit/commit/6ff1e962c981f3a481728b2b0ba03c4d3d9edcc7)) + +* build: Add timeout for Unit-Tests job and checkout code with tag in Continuous-Deployment job ([`2f85826`](https://github.com/jjjermiah/nbia-toolkit/commit/2f858265dd467267a0079919315280e62c34b173)) + +### Feature + +* feat: release on development ([`b813e2a`](https://github.com/jjjermiah/nbia-toolkit/commit/b813e2a3e82d281f3cdfcb88415100ca451b43a2)) + +* feat: release on development ([`71e68e0`](https://github.com/jjjermiah/nbia-toolkit/commit/71e68e02ce047331fe1adc7f5a658f9899c8d356)) + +### Fix + +* fix: testing gha ([`272a9f5`](https://github.com/jjjermiah/nbia-toolkit/commit/272a9f52be5f6e1a5f2474c5cc433000a17fa4b6)) + +### Unknown + +* Merge remote-tracking branch 'origin' into development ([`2ed6d37`](https://github.com/jjjermiah/nbia-toolkit/commit/2ed6d37f9d776992bb9bbe23239ce639083aae53)) + + ## v1.1.0 (2024-04-01) ### Build @@ -14,6 +37,8 @@ ### Chore +* chore(sem-ver): 1.1.0 ([`c41b230`](https://github.com/jjjermiah/nbia-toolkit/commit/c41b2304acd2fc0360ac26ef4736bf10e774871b)) + * chore: Update README: 1.0.1 ([`1b7508f`](https://github.com/jjjermiah/nbia-toolkit/commit/1b7508f515ce2820c5b232810fb26660448e66a4)) ### Documentation @@ -34,6 +59,8 @@ ### Fix +* fix: Fix string formatting in version function ([`dc4bbd7`](https://github.com/jjjermiah/nbia-toolkit/commit/dc4bbd749a2b281e81faf7a99a2903bef278beca)) + * fix: python 3.9 only on ubuntu ([`1adb7b8`](https://github.com/jjjermiah/nbia-toolkit/commit/1adb7b8c66a6d73fb27374b557eb3b76d9d0c01c)) * fix: Update GitHub Actions workflow to include Ubuntu 3.9 only ([`8d376d3`](https://github.com/jjjermiah/nbia-toolkit/commit/8d376d35f8ef52386330dd8907f35a6ad4a15f4f)) diff --git a/pyproject.toml b/pyproject.toml index 2798f1a..58407f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "nbiatoolkit" -version = "1.1.0" +version = "1.2.0" description = "A python package to query the National Biomedical Imaging Archive (NBIA) database." authors = ["Jermiah Joseph"] license = "MIT" diff --git a/src/nbiatoolkit/nbia.py b/src/nbiatoolkit/nbia.py index 2591681..2e8795a 100644 --- a/src/nbiatoolkit/nbia.py +++ b/src/nbiatoolkit/nbia.py @@ -38,7 +38,7 @@ from datetime import datetime # set __version__ variable -__version__ = "1.1.0" +__version__ = "1.2.0" def downloadSingleSeries( From 6cb12bc08c7434b086f3931de3fa9099b9792c76 Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 14:02:07 -0400 Subject: [PATCH 08/12] fix: Update branch filter in CI/CD workflow --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 84591a5..bc16e15 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,9 +4,9 @@ name: CI-CD on: push: # push to any branch * - branches: [ main , development ] + branches: [ main ] pull_request: - branches: [ main , development] + branches: [ main , development ] jobs: Unit-Tests: @@ -97,10 +97,10 @@ jobs: issues: write pull-requests: write - # needs: [Unit-Tests] + needs: [Unit-Tests] # if pulling to main, deploy to PyPI - # if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' # Set up operating system runs-on: ubuntu-latest From 695f660de7b52e51adefea139a0fc4eafe0a3726 Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 14:07:52 -0400 Subject: [PATCH 09/12] fix: docker setup to test images --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bc16e15..7dc9ac4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -239,7 +239,8 @@ jobs: docker_tag: ["latest", "${{ needs.Continuous-Deployment.outputs.tag }}"] steps: - - uses: actions/checkout@v3 + - name: Setup Docker to pull images + uses: docker/setup-buildx-action@v3 - name: Install using Docker run: | From dfc41fa6d9e8a194596885dddb90d7375938c7b2 Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 15:56:28 -0400 Subject: [PATCH 10/12] feat: Add DICOM tags module and update DICOM sorting functionality --- src/nbiatoolkit/__init__.py | 1 + src/nbiatoolkit/dicomsort/__init__.py | 3 +- src/nbiatoolkit/dicomsort/dicomsort.py | 7 +++- src/nbiatoolkit/dicomtags/__init__.py | 2 ++ src/nbiatoolkit/dicomtags/tags.py | 30 +++++++++++++++-- src/nbiatoolkit/nbia.py | 46 +++++++++++++++++++++++++- 6 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/nbiatoolkit/__init__.py b/src/nbiatoolkit/__init__.py index 80805ca..a86b56a 100644 --- a/src/nbiatoolkit/__init__.py +++ b/src/nbiatoolkit/__init__.py @@ -13,6 +13,7 @@ from .auth import OAuth2 from .logger.logger import setup_logger from .utils.nbia_endpoints import NBIA_ENDPOINTS +from .dicomtags import * # define the __all__ variable __all__ = [ diff --git a/src/nbiatoolkit/dicomsort/__init__.py b/src/nbiatoolkit/dicomsort/__init__.py index 9701563..aa1c3e3 100644 --- a/src/nbiatoolkit/dicomsort/__init__.py +++ b/src/nbiatoolkit/dicomsort/__init__.py @@ -8,7 +8,7 @@ # ] -from .dicomsort import DICOMSorter +from .dicomsort import DICOMSorter, generateFilePathFromDICOMAttributes from .helper_functions import parseDICOMKeysFromFormat, sanitizeFileName, _truncateUID __all__ = [ @@ -16,4 +16,5 @@ "sanitizeFileName", "_truncateUID", "DICOMSorter", + "generateFilePathFromDICOMAttributes", ] diff --git a/src/nbiatoolkit/dicomsort/dicomsort.py b/src/nbiatoolkit/dicomsort/dicomsort.py index 3bbc294..d9834df 100644 --- a/src/nbiatoolkit/dicomsort/dicomsort.py +++ b/src/nbiatoolkit/dicomsort/dicomsort.py @@ -14,6 +14,7 @@ import shutil from .helper_functions import parseDICOMKeysFromFormat, sanitizeFileName, _truncateUID from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Union def get_dicom_files(sourceDir) -> list[str]: @@ -39,7 +40,7 @@ def read_in_dicom_file(filePath: str) -> pydicom.FileDataset: def generateFilePathFromDICOMAttributes( - dataset: pydicom.FileDataset, + dataset: pydicom.Dataset, targetPattern: str, truncateUID: bool, sanitizeFilename: bool, @@ -57,6 +58,10 @@ def generateFilePathFromDICOMAttributes( # Retrieve the attribute value if it exists or default to a placeholder string value = str(getattr(dataset, key, "Unknown" + key)) + # if value is exactly "UnknownInstanceNumber", replace it with "1" + if value == "UnknownInstanceNumber": + value = "1" + value = ( _truncateUID(uid=value, lastDigits=5) if key.endswith("UID") and truncateUID diff --git a/src/nbiatoolkit/dicomtags/__init__.py b/src/nbiatoolkit/dicomtags/__init__.py index 84a7a4e..561ea0e 100644 --- a/src/nbiatoolkit/dicomtags/__init__.py +++ b/src/nbiatoolkit/dicomtags/__init__.py @@ -10,6 +10,7 @@ subsetSeriesTags, getReferencedFrameOfReferenceSequence, getReferencedSeriesUIDS, + extract_ROI_info, ) __all__ = [ @@ -21,4 +22,5 @@ "subsetSeriesTags", "getReferencedFrameOfReferenceSequence", "getReferencedSeriesUIDS", + "extract_ROI_info", ] diff --git a/src/nbiatoolkit/dicomtags/tags.py b/src/nbiatoolkit/dicomtags/tags.py index 35ba825..33b8678 100644 --- a/src/nbiatoolkit/dicomtags/tags.py +++ b/src/nbiatoolkit/dicomtags/tags.py @@ -1,6 +1,6 @@ from math import log -from pydicom.datadict import dictionary_VR -from pydicom.datadict import tag_for_keyword +import pydicom +from pydicom.datadict import dictionary_VR, tag_for_keyword import pandas as pd from typing import List @@ -342,6 +342,32 @@ def extract_ROI_info(StructureSetROISequence) -> dict[str, dict[str, str]]: return ROISet +def generateFileDatasetFromTags(tags_df: pd.DataFrame) -> pydicom.Dataset: + """ + Generate a pydicom Dataset object from a DataFrame of DICOM tags. + + Args: + tags_df (pd.DataFrame): DataFrame containing DICOM tags. + + Returns: + pydicom.Dataset: A pydicom Dataset object containing the DICOM tags. + """ + + # Create a new FileDataset + ds = pydicom.Dataset() + + for _, row in tags_df.iterrows(): + tag = convert_element_to_int(row["element"]) + value = row["data"] + if tag == -1: + continue + VR = element_VR_lookup(row["element"])[1] + + ds.add_new(tag=tag, VR=VR, value=value) + + return ds + + # def getRTSTRUCT_ROI_info(seriesUID: str) -> dict[str, dict[str, str]]: # """ # Given a SeriesInstanceUID of an RTSTRUCT, retrieves the ROI information. diff --git a/src/nbiatoolkit/nbia.py b/src/nbiatoolkit/nbia.py index 2e8795a..c67e72b 100644 --- a/src/nbiatoolkit/nbia.py +++ b/src/nbiatoolkit/nbia.py @@ -4,7 +4,9 @@ import re import zipfile from tempfile import TemporaryDirectory -from .dicomsort import DICOMSorter + +from pydicom import Dataset, FileDataset +from .dicomsort import DICOMSorter, generateFilePathFromDICOMAttributes import multiprocessing from .auth import OAuth2 @@ -26,6 +28,7 @@ getReferencedSeriesUIDS, extract_ROI_info, getSequenceElement, + generateFileDatasetFromTags, ) import pandas as pd @@ -637,6 +640,47 @@ def getRefSeriesUIDs( return getReferencedSeriesUIDS(series_tags_df=tags_df) + def generateFilePathFromDICOMTags( + self, + SeriesInstanceUID: str, + filePattern: str = "%PatientName/%Modality-%SeriesNumber-%SeriesInstanceUID/%InstanceNumber.dcm", + ) -> str: + """ + Generates a file path from DICOM tags. + + Args: + SeriesInstanceUID (str): The Series Instance UID of the DICOM series. + filePattern (str, optional): The file pattern to use for generating the file path. Defaults to "%PatientName/%Modality-%SeriesNumber-%SeriesInstanceUID/%InstanceNumber.dcm". + + Returns: + str: The generated file path. + + Note: + This only considers the first instance of the series. + Meant to be used to determine the dirname of the series files. + """ + self.logger.debug("Getting DICOM tags for series %s", SeriesInstanceUID) + tags_df = self.getDICOMTags( + SeriesInstanceUID=SeriesInstanceUID, + return_type=ReturnType.DATAFRAME, + ) + + if type(tags_df) != pd.DataFrame: + raise ValueError("DICOM Tags not df or not found in the response.") + + self.logger.debug("Generating file path from DICOM tags") + ds: Dataset = generateFileDatasetFromTags(tags_df=tags_df) + filePath: str = generateFilePathFromDICOMAttributes( + dataset=ds, + targetPattern=filePattern, + truncateUID=True, + sanitizeFilename=True, + ) + self.logger.debug( + "Generated file path: %s for series %s", filePath, SeriesInstanceUID + ) + return filePath + def downloadSeries( self, SeriesInstanceUID: Union[str, list], From 97f0c3185ed7af2cfb9169482e421fe8035523ce Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 15:58:04 -0400 Subject: [PATCH 11/12] build: Update Docker image tag in CI workflow --- .github/workflows/main.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7dc9ac4..9a84b04 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -236,7 +236,6 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - docker_tag: ["latest", "${{ needs.Continuous-Deployment.outputs.tag }}"] steps: - name: Setup Docker to pull images @@ -244,5 +243,5 @@ jobs: - name: Install using Docker run: | - docker pull ${{ secrets.DOCKERHUB_USERNAME }}/nbiatoolkit:${{ matrix.docker_tag }}; - docker run --rm ${{ secrets.DOCKERHUB_USERNAME }}/nbiatoolkit:${{ matrix.docker_tag }} NBIAToolkit + docker pull ${{ secrets.DOCKERHUB_USERNAME }}/nbiatoolkit:${{ needs.Continuous-Deployment.outputs.tag }}; + docker run --rm ${{ secrets.DOCKERHUB_USERNAME }}/nbiatoolkit:${{ needs.Continuous-Deployment.outputs.tag }} NBIAToolkit From dd537732acf5065853be49019a23f9407cfb7849 Mon Sep 17 00:00:00 2001 From: Jermiah Joseph Date: Mon, 1 Apr 2024 15:59:49 -0400 Subject: [PATCH 12/12] fix: Update PyPi installation command --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9a84b04..6b913da 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -227,7 +227,7 @@ jobs: - name: Install using PyPi run: | - pip install nbiatoolkit; + pip install nbiatoolkit==${{ needs.Continuous-Deployment.outputs.version }} NBIAToolkit Test-Docker-Image: