From 85f753b428604b4811fe2f7b9f1d32ad61c721f5 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Mon, 26 Jul 2021 16:32:27 -0400 Subject: [PATCH 01/13] cnn masking for hifib0 --- DiffusionPreprocessing/scripts/run_topup.sh | 62 ++++++--------------- 1 file changed, 16 insertions(+), 46 deletions(-) diff --git a/DiffusionPreprocessing/scripts/run_topup.sh b/DiffusionPreprocessing/scripts/run_topup.sh index 27945383d..b733c396f 100755 --- a/DiffusionPreprocessing/scripts/run_topup.sh +++ b/DiffusionPreprocessing/scripts/run_topup.sh @@ -14,16 +14,16 @@ ${FSLDIR}/bin/topup --imain=${workingdir}/Pos_Neg_b0 --datain=${workingdir}/acqp dimt=$(${FSLDIR}/bin/fslval ${workingdir}/Pos_b0 dim4) dimt=$((${dimt} + 1)) -# echo "Applying topup to get a hifi b0" -# ${FSLDIR}/bin/fslroi ${workingdir}/Pos_b0 ${workingdir}/Pos_b01 0 1 -# ${FSLDIR}/bin/fslroi ${workingdir}/Neg_b0 ${workingdir}/Neg_b01 0 1 -# ${FSLDIR}/bin/applytopup --imain=${workingdir}/Pos_b01,${workingdir}/Neg_b01 --topup=${workingdir}/topup_Pos_Neg_b0 --datain=${workingdir}/acqparams.txt --inindex=1,${dimt} --out=${workingdir}/hifib0 -# -# if [ ! -f ${workingdir}/hifib0.nii.gz ]; then -# echo "run_topup.sh -- ERROR -- ${FSLDIR}/bin/applytopup failed to generate ${workingdir}/hifib0.nii.gz" -# # Need to add mechanism whereby scripts that invoke this script (run_topup.sh) -# # check for a return code to determine success or failure -# fi +echo "Applying topup to get a hifi b0" +${FSLDIR}/bin/fslroi ${workingdir}/Pos_b0 ${workingdir}/Pos_b01 0 1 +${FSLDIR}/bin/fslroi ${workingdir}/Neg_b0 ${workingdir}/Neg_b01 0 1 +${FSLDIR}/bin/applytopup --imain=${workingdir}/Pos_b01,${workingdir}/Neg_b01 --topup=${workingdir}/topup_Pos_Neg_b0 --datain=${workingdir}/acqparams.txt --inindex=1,${dimt} --out=${workingdir}/hifib0 + +if [ ! -f ${workingdir}/hifib0.nii.gz ]; then + echo "run_topup.sh -- ERROR -- ${FSLDIR}/bin/applytopup failed to generate ${workingdir}/hifib0.nii.gz" + # Need to add mechanism whereby scripts that invoke this script (run_topup.sh) + # check for a return code to determine success or failure +fi ${FSLDIR}/bin/imrm ${workingdir}/Pos_b0* @@ -32,46 +32,16 @@ ${FSLDIR}/bin/imrm ${workingdir}/Neg_b0* # echo "Running BET on the hifi b0" # ${FSLDIR}/bin/bet ${workingdir}/hifib0 ${workingdir}/nodif_brain -m -f 0.2 -# echo "Applying PNL invented CNN masking tool to obtain b0 brain mask" -# cnn_mask_exe=`which dwi_masking.py` -# if [ ! -z $cnn_mask_exe ]; then -# echo "CNN-Diffusion-MRIBrain-Segmentation/pipeline/dwi_masking.py is not available in PATH" -# exit 1 -# fi -# -# # TODO -# $cnn_mask_exe -i ${workingdir}/hifib0 -f $(dirname $cnn_mask_exe)/../model_folder -o ${workingdir}/nodif_brain - - -# define the masks in PA,AP order (pos,neg) -# obtain 107 masks -IFS=' ' read -ra masks_107 <<< $MASKS_107 -${FSLDIR}/bin/applytopup --imain=${masks_107[0]},${masks_107[1]} --topup=${workingdir}/topup_Pos_Neg_b0 --datain=${workingdir}/acqparams.txt --inindex=1,${dimt} --out=${workingdir}/mask_107 --verbose --method=jac --interp=trilinear - -# obtain 99 masks -IFS=' ' read -ra masks_99 <<< $MASKS_99 -${FSLDIR}/bin/applytopup --imain=${masks_99[0]},${masks_99[1]} --topup=${workingdir}/topup_Pos_Neg_b0 --datain=${workingdir}/acqparams.txt --inindex=1,${dimt} --out=${workingdir}/mask_99 --verbose --method=jac --interp=trilinear - -# take their union -pushd . -cd ${workingdir} - -fslmaths mask_107 -abs mask_107 -fslmaths mask_99 -abs mask_99 -fslmaths mask_107 -add mask_99 nodif_brain_mask -odt char -fslmaths nodif_brain_mask -bin nodif_brain_mask - -# filter the resultant mask -if [ -z `which maskfilter.py` ]; then - echo "pnlNipype/scripts/maskfilter.py is not available in PATH" +echo "Applying PNL invented CNN masking tool to obtain b0 brain mask" +cnn_mask_exe=`which dwi_masking.py` +if [ ! -z $cnn_mask_exe ]; then + echo "CNN-Diffusion-MRIBrain-Segmentation/pipeline/dwi_masking.py is not available in PATH" exit 1 fi -# maskfilter.py nodif_brain.nii.gz 2 nodif_brain.nii.gz -${FSLDIR}/bin/imrm ${workingdir}/mask_107 -${FSLDIR}/bin/imrm ${workingdir}/mask_99 +# TODO +$cnn_mask_exe -i ${workingdir}/hifib0 -f $(dirname $cnn_mask_exe)/../model_folder -o ${workingdir}/nodif_brain -popd if [ ! -f ${workingdir}/nodif_brain.nii.gz ]; then echo "run_topup.sh -- ERROR -- ${FSLDIR}/bin/bet failed to generate ${workingdir}/nodif_brain.nii.gz" From af9dc9027b7ae394d9aeb83831cd7e99daef816d Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Tue, 27 Jul 2021 12:11:21 -0400 Subject: [PATCH 02/13] correct if condition for cnn_mask_exe --- DiffusionPreprocessing/scripts/run_topup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DiffusionPreprocessing/scripts/run_topup.sh b/DiffusionPreprocessing/scripts/run_topup.sh index b733c396f..1ff73be94 100755 --- a/DiffusionPreprocessing/scripts/run_topup.sh +++ b/DiffusionPreprocessing/scripts/run_topup.sh @@ -34,12 +34,11 @@ ${FSLDIR}/bin/imrm ${workingdir}/Neg_b0* echo "Applying PNL invented CNN masking tool to obtain b0 brain mask" cnn_mask_exe=`which dwi_masking.py` -if [ ! -z $cnn_mask_exe ]; then +if [ -z $cnn_mask_exe ]; then echo "CNN-Diffusion-MRIBrain-Segmentation/pipeline/dwi_masking.py is not available in PATH" exit 1 fi -# TODO $cnn_mask_exe -i ${workingdir}/hifib0 -f $(dirname $cnn_mask_exe)/../model_folder -o ${workingdir}/nodif_brain From 5d1aa5256b1a415138ce40b615a252a1b7a9e888 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Tue, 27 Jul 2021 14:04:31 -0400 Subject: [PATCH 03/13] activate removal of intensity normalized data --- DiffusionPreprocessing/scripts/run_eddy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DiffusionPreprocessing/scripts/run_eddy.sh b/DiffusionPreprocessing/scripts/run_eddy.sh index 98a685fe3..b9ff1bba5 100755 --- a/DiffusionPreprocessing/scripts/run_eddy.sh +++ b/DiffusionPreprocessing/scripts/run_eddy.sh @@ -627,8 +627,8 @@ main() { rm -r ${workingdir}/../rawdata # remove intensity normalized data - # rm ${workingdir}/Pos* - # rm ${workingdir}/Neg* + rm ${workingdir}/Pos* + rm ${workingdir}/Neg* # Redoing eddy without --repol flag to obtain not outlier replaced <=500 bshell if [ ! -z "${extra_eddy_args}" ]; then From 997e0f0a68bc68797f6818de276a3cc7199630f7 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Tue, 27 Jul 2021 14:16:36 -0400 Subject: [PATCH 04/13] adjust w/space --- DiffusionPreprocessing/scripts/run_eddy.sh | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/DiffusionPreprocessing/scripts/run_eddy.sh b/DiffusionPreprocessing/scripts/run_eddy.sh index b9ff1bba5..c490d6aa4 100755 --- a/DiffusionPreprocessing/scripts/run_eddy.sh +++ b/DiffusionPreprocessing/scripts/run_eddy.sh @@ -632,19 +632,19 @@ main() { # Redoing eddy without --repol flag to obtain not outlier replaced <=500 bshell if [ ! -z "${extra_eddy_args}" ]; then - for extra_eddy_arg in ${extra_eddy_args}; do - - if [ $extra_eddy_arg = '--repol' ]; then - log_Msg "Redoing eddy without --repol flag to obtain not outlier replaced b<=500 shells" - eddy_command="${HCPPIPEDIR}/DiffusionPreprocessing/scripts/eddy_pnl_repol.py ${eddy_command}" - log_Msg "${eddy_command}" - ${eddy_command} - eddyReturnValue=$? - log_Msg "Completed with return value: ${eddyReturnValue}" - fi + for extra_eddy_arg in ${extra_eddy_args}; do + + if [ $extra_eddy_arg = '--repol' ]; then + log_Msg "Redoing eddy without --repol flag to obtain not outlier replaced b<=500 shells" + eddy_command="${HCPPIPEDIR}/DiffusionPreprocessing/scripts/eddy_pnl_repol.py ${eddy_command}" + log_Msg "${eddy_command}" + ${eddy_command} + eddyReturnValue=$? + log_Msg "Completed with return value: ${eddyReturnValue}" + fi - done - fi + done + fi exit ${eddyReturnValue} From 2a848679fabcca7349fbf07114661e73dca22e7b Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Tue, 27 Jul 2021 22:15:37 -0400 Subject: [PATCH 05/13] update messages to show b=550 --- DiffusionPreprocessing/scripts/eddy_pnl_repol.py | 2 +- DiffusionPreprocessing/scripts/run_eddy.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DiffusionPreprocessing/scripts/eddy_pnl_repol.py b/DiffusionPreprocessing/scripts/eddy_pnl_repol.py index da53a5f1b..4e5aebd9d 100755 --- a/DiffusionPreprocessing/scripts/eddy_pnl_repol.py +++ b/DiffusionPreprocessing/scripts/eddy_pnl_repol.py @@ -43,7 +43,7 @@ def main(): if '--repol' in eddy_openmp_params and len(ind): print('\nDoing eddy_openmp/cuda again without --repol option ' - 'to obtain eddy correction w/o outlier replacement for b<=500 shells\n') + f'to obtain eddy correction w/o outlier replacement for b<={REPOL_BSHELL_GREATER} shells\n') eddy_openmp_params.remove('--repol') print(eddy_openmp_params) diff --git a/DiffusionPreprocessing/scripts/run_eddy.sh b/DiffusionPreprocessing/scripts/run_eddy.sh index c490d6aa4..cf70d618a 100755 --- a/DiffusionPreprocessing/scripts/run_eddy.sh +++ b/DiffusionPreprocessing/scripts/run_eddy.sh @@ -635,7 +635,7 @@ main() { for extra_eddy_arg in ${extra_eddy_args}; do if [ $extra_eddy_arg = '--repol' ]; then - log_Msg "Redoing eddy without --repol flag to obtain not outlier replaced b<=500 shells" + log_Msg "Redoing eddy without --repol flag to obtain not outlier replaced b<=550 shells" eddy_command="${HCPPIPEDIR}/DiffusionPreprocessing/scripts/eddy_pnl_repol.py ${eddy_command}" log_Msg "${eddy_command}" ${eddy_command} From bab336c9817a143165409dbea928c25adc9c09e0 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Wed, 28 Jul 2021 15:57:29 -0400 Subject: [PATCH 06/13] direction about final results issue #3 --- DiffusionPreprocessing/DiffPreprocPipeline.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DiffusionPreprocessing/DiffPreprocPipeline.sh b/DiffusionPreprocessing/DiffPreprocPipeline.sh index 1cae0d4c0..fed795a3d 100755 --- a/DiffusionPreprocessing/DiffPreprocPipeline.sh +++ b/DiffusionPreprocessing/DiffPreprocPipeline.sh @@ -544,6 +544,14 @@ main() { log_Msg "eddy_cmd: ${eddy_cmd}" ${eddy_cmd} + echo """ + Observe the following output files in ${StudyFolder}/${Subject}: + dwi: Diffusion/eddy/eddy_unwarped_images.nii.gz + bvals: Diffusion/eddy/Pos_Neg.bvals + bvecs: Diffusion/eddy/eddy_unwarped_images.eddy_rotated_bvecs + mask: Diffusion/eddy/nodif_brain_mask.nii.gz + """ + exit log_Msg "Invoking Post-Eddy Steps" From c782fe0e1e5520c467ae10df1864a4f8917a7846 Mon Sep 17 00:00:00 2001 From: Tashrif Billah Date: Wed, 28 Jul 2021 16:44:44 -0400 Subject: [PATCH 07/13] remove colon from path of git diff-index --- show_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/show_version b/show_version index be03068a6..f82961ed1 100755 --- a/show_version +++ b/show_version @@ -104,7 +104,7 @@ if [[ -d "$HCPPIPEDIR"/.git ]] && which git &> /dev/null then modified=no #ignore modification of things in Examples? - if ! (cd "$HCPPIPEDIR"; git diff-index --quiet HEAD -- ':!/Examples/') + if ! (cd "$HCPPIPEDIR"; git diff-index HEAD -- '!Examples/') then modified=YES verstring="$verstring"-MOD From 2d1789d1a854dcbf702b5772f35250597906a321 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Wed, 28 Jul 2021 17:00:39 -0400 Subject: [PATCH 08/13] rm normalized data after 2nd eddy --- DiffusionPreprocessing/scripts/run_eddy.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/DiffusionPreprocessing/scripts/run_eddy.sh b/DiffusionPreprocessing/scripts/run_eddy.sh index cf70d618a..122727c2f 100755 --- a/DiffusionPreprocessing/scripts/run_eddy.sh +++ b/DiffusionPreprocessing/scripts/run_eddy.sh @@ -626,10 +626,6 @@ main() { # remove rawdata and release space rm -r ${workingdir}/../rawdata - # remove intensity normalized data - rm ${workingdir}/Pos* - rm ${workingdir}/Neg* - # Redoing eddy without --repol flag to obtain not outlier replaced <=500 bshell if [ ! -z "${extra_eddy_args}" ]; then for extra_eddy_arg in ${extra_eddy_args}; do @@ -646,6 +642,9 @@ main() { done fi + # remove intensity normalized data + rm ${workingdir}/Pos_Neg.nii.gz + exit ${eddyReturnValue} } From 616a528e250ce9bbe216b6e6c4b0ee4812c061c2 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Wed, 28 Jul 2021 17:02:21 -0400 Subject: [PATCH 09/13] elaborate CNN masking program --- DiffusionPreprocessing/scripts/run_topup.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/DiffusionPreprocessing/scripts/run_topup.sh b/DiffusionPreprocessing/scripts/run_topup.sh index 1ff73be94..15490bfb4 100755 --- a/DiffusionPreprocessing/scripts/run_topup.sh +++ b/DiffusionPreprocessing/scripts/run_topup.sh @@ -39,7 +39,18 @@ if [ -z $cnn_mask_exe ]; then exit 1 fi -$cnn_mask_exe -i ${workingdir}/hifib0 -f $(dirname $cnn_mask_exe)/../model_folder -o ${workingdir}/nodif_brain +pushd . +cd ${workingdir} +listpre=b0_list +realpath hifib0.nii.gz > ${listpre}.txt +$cnn_mask_exe -i ${listpre}.txt -f $(dirname $cnn_mask_exe)/../model_folder +# rename +mv hifib0_bse-multi_BrainMask.nii.gz nodif_brain_mask.nii.gz +# cleanup +rm ${listpre}* +rm *_cases_* +rm -r slicesdir_multi/ +popd if [ ! -f ${workingdir}/nodif_brain.nii.gz ]; then From 9fa236c25dd795cbb8ba34c876d58aef7e262030 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Wed, 28 Jul 2021 23:20:16 -0400 Subject: [PATCH 10/13] revert git diff-index command https://github.com/Washington-University/HCPpipelines/pull/215#issuecomment-888677341 --- show_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/show_version b/show_version index f82961ed1..be03068a6 100755 --- a/show_version +++ b/show_version @@ -104,7 +104,7 @@ if [[ -d "$HCPPIPEDIR"/.git ]] && which git &> /dev/null then modified=no #ignore modification of things in Examples? - if ! (cd "$HCPPIPEDIR"; git diff-index HEAD -- '!Examples/') + if ! (cd "$HCPPIPEDIR"; git diff-index --quiet HEAD -- ':!/Examples/') then modified=YES verstring="$verstring"-MOD From 0099054f489d7349a21d91c1ff1a27b9d95acf22 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Fri, 18 Nov 2022 10:59:32 -0500 Subject: [PATCH 11/13] status msg: change bet to dwi_masking.py --- DiffusionPreprocessing/scripts/run_topup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DiffusionPreprocessing/scripts/run_topup.sh b/DiffusionPreprocessing/scripts/run_topup.sh index 15490bfb4..e568bbc92 100755 --- a/DiffusionPreprocessing/scripts/run_topup.sh +++ b/DiffusionPreprocessing/scripts/run_topup.sh @@ -54,7 +54,7 @@ popd if [ ! -f ${workingdir}/nodif_brain.nii.gz ]; then - echo "run_topup.sh -- ERROR -- ${FSLDIR}/bin/bet failed to generate ${workingdir}/nodif_brain.nii.gz" + echo "run_topup.sh -- ERROR -- CNN-Diffusion-MRIBrain-Segmentation/pipeline/dwi_masking.py failed to generate ${workingdir}/nodif_brain.nii.gz" # Need to add mechanism whereby scripts that invoke this script (run_topup.sh) # check for a return code to determine success or failure fi From 93a59641e0ef4046c2c7477db9c51bdf8189adb6 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" Date: Tue, 22 Nov 2022 15:04:26 -0500 Subject: [PATCH 12/13] condition on nodif_brain_mask.nii.gz nodif_brain.nii.gz is not generated at all --- DiffusionPreprocessing/scripts/run_topup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DiffusionPreprocessing/scripts/run_topup.sh b/DiffusionPreprocessing/scripts/run_topup.sh index e568bbc92..4e2fc461d 100755 --- a/DiffusionPreprocessing/scripts/run_topup.sh +++ b/DiffusionPreprocessing/scripts/run_topup.sh @@ -53,8 +53,8 @@ rm -r slicesdir_multi/ popd -if [ ! -f ${workingdir}/nodif_brain.nii.gz ]; then - echo "run_topup.sh -- ERROR -- CNN-Diffusion-MRIBrain-Segmentation/pipeline/dwi_masking.py failed to generate ${workingdir}/nodif_brain.nii.gz" +if [ ! -f ${workingdir}/nodif_brain_mask.nii.gz ]; then + echo "run_topup.sh -- ERROR -- CNN-Diffusion-MRIBrain-Segmentation/pipeline/dwi_masking.py failed to generate ${workingdir}/nodif_brain_mask.nii.gz" # Need to add mechanism whereby scripts that invoke this script (run_topup.sh) # check for a return code to determine success or failure fi From 043e68c8db9f067e45daf3f3c98474ab947cdee6 Mon Sep 17 00:00:00 2001 From: "Billah, Tashrif" <35086881+tashrifbillah@users.noreply.github.com> Date: Tue, 17 Sep 2024 09:40:32 -0400 Subject: [PATCH 13/13] let CNN dwi_masking.py run without -f model_folder --- DiffusionPreprocessing/scripts/run_topup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DiffusionPreprocessing/scripts/run_topup.sh b/DiffusionPreprocessing/scripts/run_topup.sh index 4e2fc461d..0ecdfa2a5 100755 --- a/DiffusionPreprocessing/scripts/run_topup.sh +++ b/DiffusionPreprocessing/scripts/run_topup.sh @@ -43,7 +43,7 @@ pushd . cd ${workingdir} listpre=b0_list realpath hifib0.nii.gz > ${listpre}.txt -$cnn_mask_exe -i ${listpre}.txt -f $(dirname $cnn_mask_exe)/../model_folder +$cnn_mask_exe -i ${listpre}.txt # rename mv hifib0_bse-multi_BrainMask.nii.gz nodif_brain_mask.nii.gz # cleanup