From 21094bb46a3aef8cb3d8096408df00aa710af8af Mon Sep 17 00:00:00 2001 From: blakeNaccarato Date: Wed, 2 Oct 2024 10:40:11 -0700 Subject: [PATCH] Enable hydra --- .dvc/config | 2 + conf/config.yaml | 2 + conf/stage/defaults.yaml | 11 ++++ conf/stage/trackpy.yaml | 11 ++++ docs/notebooks/find_objects.ipynb | 7 +-- dvc.yaml | 101 +++++++----------------------- params.yaml | 23 +++---- 7 files changed, 63 insertions(+), 94 deletions(-) create mode 100644 conf/config.yaml create mode 100644 conf/stage/defaults.yaml create mode 100644 conf/stage/trackpy.yaml diff --git a/.dvc/config b/.dvc/config index d37c9141..766068ed 100644 --- a/.dvc/config +++ b/.dvc/config @@ -1,4 +1,6 @@ [core] remote = origin +[hydra] + enabled = True ['remote "origin"'] url = gs://boilercv diff --git a/conf/config.yaml b/conf/config.yaml new file mode 100644 index 00000000..04aea136 --- /dev/null +++ b/conf/config.yaml @@ -0,0 +1,2 @@ +defaults: + - stage: "defaults.yaml" diff --git a/conf/stage/defaults.yaml b/conf/stage/defaults.yaml new file mode 100644 index 00000000..b3c54750 --- /dev/null +++ b/conf/stage/defaults.yaml @@ -0,0 +1,11 @@ +scale: 1.3 +marker_scale: 20.0 +precision: 3 +display_rows: 12 +sample: "2024-07-18T17-44-35" +only_sample: "--no-only-sample" +load_src_from_outs: "--no-load-src-from-outs" +frame_count: 0 +frame_step: 1 +compare_with_trackpy: "--no-compare-with-trackpy" +guess_diameter: 21 diff --git a/conf/stage/trackpy.yaml b/conf/stage/trackpy.yaml new file mode 100644 index 00000000..2f9b8a0d --- /dev/null +++ b/conf/stage/trackpy.yaml @@ -0,0 +1,11 @@ +scale: 1.3 +marker_scale: 20.0 +precision: 3 +display_rows: 12 +sample: "2024-07-18T17-44-35" +only_sample: "--no-only-sample" +load_src_from_outs: "--no-load-src-from-outs" +frame_count: 0 +frame_step: 1 +compare_with_trackpy: "--compare-with-trackpy" +guess_diameter: 21 diff --git a/docs/notebooks/find_objects.ipynb b/docs/notebooks/find_objects.ipynb index 255005f6..6358430e 100644 --- a/docs/notebooks/find_objects.ipynb +++ b/docs/notebooks/find_objects.ipynb @@ -57,8 +57,6 @@ "\"\"\"Notebook stage parameters.\"\"\"\n", "MODE = get_mode()\n", "\"\"\"Notebook execution mode.\"\"\"\n", - "PREVIEW_FRAME_COUNT = 50\n", - "\"\"\"Number of preview frames.\"\"\"\n", "Params.hide()" ] }, @@ -76,7 +74,7 @@ "if isinstance(PARAMS, str):\n", " params = Params.model_validate_json(PARAMS)\n", "elif MODE == \"docs\":\n", - " PREVIEW_FRAME_COUNT = 10\n", + " preview_frame_count = 10\n", " params = Params(\n", " context=init(mode=MODE),\n", " compare_with_trackpy=True,\n", @@ -94,6 +92,7 @@ "frames_slice = slices.get(FRAME, slice(None))\n", "contours = get_contours_df2(one(params.contours)).loc[IndexSlice[frames_slice, :], :]\n", "frames = contours.reset_index()[FRAME].unique()\n", + "preview_frame_count = round(0.619233215798799 * len(frames) ** 0.447632153789354)\n", "\n", "# # ? Produce reduced-size docs data\n", "# from pathlib import Path\n", @@ -153,7 +152,7 @@ "data.plots.composite, ax = subplots()\n", "filled_path = one(params.filled)\n", "with load_video(\n", - " filled_path, slices={FRAME: frames[:: (len(frames) // PREVIEW_FRAME_COUNT)]}\n", + " filled_path, slices={FRAME: frames[:: int(len(frames) // preview_frame_count)]}\n", ") as video:\n", " filled_preview = scale_bool(video)\n", " composite_video = scale_bool(video).max(FRAME).values\n", diff --git a/dvc.yaml b/dvc.yaml index 75a8e85e..c64274f5 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -1,131 +1,92 @@ stages: skip_cloud: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage skip-cloud --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows}" - params: - - scale - - marker_scale - - precision - - display_rows + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage skip-cloud --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows}" + params: ["stage"] outs: - data/cines: cache: false persist: true push: false convert: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage convert --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows}" + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage convert --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows}" deps: - packages/pipeline/boilercv_pipeline/stages/convert - data/cines - params: - - scale - - marker_scale - - precision - - display_rows + params: ["stage"] outs: - data/large_sources: cache: false persist: true push: false binarize: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage binarize --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows}" + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage binarize --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows}" deps: - packages/pipeline/boilercv_pipeline/stages/binarize - data/large_sources - params: - - scale - - marker_scale - - precision - - display_rows + params: ["stage"] outs: - data/sources: persist: true - data/rois: persist: true preview_gray: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage preview-gray --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows}" + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage preview-gray --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows}" deps: - packages/pipeline/boilercv_pipeline/stages/preview_gray - data/large_sources - params: - - scale - - marker_scale - - precision - - display_rows + params: ["stage"] outs: - data/previews/gray_preview.nc: persist: true preview_binarized: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage preview-binarized --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows}" + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage preview-binarized --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows}" deps: - packages/pipeline/boilercv_pipeline/stages/preview_binarized - data/sources - data/rois - params: - - scale - - marker_scale - - precision - - display_rows + params: ["stage"] outs: - data/previews/binarized_preview.nc: persist: true find_contours: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage find-contours --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows}" + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage find-contours --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows}" deps: - packages/pipeline/boilercv_pipeline/stages/find_contours - data/sources - data/rois - params: - - scale - - marker_scale - - precision - - display_rows + params: ["stage"] outs: - data/contours: persist: true fill: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage fill --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows}" + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage fill --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows}" deps: - packages/pipeline/boilercv_pipeline/stages/fill - data/sources - data/rois - data/contours - params: - - scale - - marker_scale - - precision - - display_rows + params: ["stage"] outs: - data/filled: persist: true preview_filled: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage preview-filled --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows}" + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage preview-filled --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows}" deps: - packages/pipeline/boilercv_pipeline/stages/preview_filled - data/filled - data/rois - params: - - scale - - marker_scale - - precision - - display_rows + params: ["stage"] outs: - data/previews/filled_preview.nc: persist: true get_thermal_data: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage get-thermal-data --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows} --sample ${sample} ${only_sample} ${load_src_from_outs}" + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage get-thermal-data --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows} --sample ${stage.sample} ${stage.only_sample} ${stage.load_src_from_outs}" deps: - packages/pipeline/boilercv_pipeline/stages/get_thermal_data - docs/notebooks/get_thermal_data.ipynb - data/thermal - data/models - params: - - scale - - marker_scale - - precision - - display_rows - - sample - - only_sample - - load_src_from_outs + params: ["stage"] outs: - data/e230920/thermal.h5: persist: true @@ -134,23 +95,13 @@ stages: - data/e230920/thermal_plots/subcool_superheat.png - data/e230920/thermal_plots/superheat.png find_objects: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage find-objects --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows} --sample ${sample} ${only_sample} --frame-count ${frame_count} --frame-step ${frame_step} ${compare_with_trackpy} --guess-diameter ${guess_diameter}" + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage find-objects --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows} --sample ${stage.sample} ${stage.only_sample} --frame-count ${stage.frame_count} --frame-step ${stage.frame_step} ${stage.compare_with_trackpy} --guess-diameter ${stage.guess_diameter}" deps: - packages/pipeline/boilercv_pipeline/stages/find_objects - docs/notebooks/find_objects.ipynb - data/filled - data/contours - params: - - scale - - marker_scale - - precision - - display_rows - - sample - - only_sample - - frame_count - - frame_step - - compare_with_trackpy - - guess_diameter + params: ["stage"] outs: - data/e230920/objects: persist: true @@ -172,22 +123,14 @@ stages: - data/e230920/objects_plots/composite_2024-07-18T18-40-58.png - data/e230920/objects_plots/composite_2024-07-18T18-49-55.png find_tracks: - cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage find-tracks --scale ${scale} --marker-scale ${marker_scale} --precision ${precision} --display-rows ${display_rows} --sample ${sample} ${only_sample} --frame-count ${frame_count} --frame-step ${frame_step}" + cmd: pwsh -Command "./Invoke-Uv.ps1 boilercv-pipeline stage find-tracks --scale ${stage.scale} --marker-scale ${stage.marker_scale} --precision ${stage.precision} --display-rows ${stage.display_rows} --sample ${stage.sample} ${stage.only_sample} --frame-count ${stage.frame_count} --frame-step ${stage.frame_step}" deps: - packages/pipeline/boilercv_pipeline/stages/find_tracks - docs/notebooks/find_tracks.ipynb - data/filled - data/e230920/objects - data/e230920/thermal.h5 - params: - - scale - - marker_scale - - precision - - display_rows - - sample - - only_sample - - frame_count - - frame_step + params: ["stage"] outs: - data/e230920/tracks: persist: true diff --git a/params.yaml b/params.yaml index 24eaa265..48932a56 100644 --- a/params.yaml +++ b/params.yaml @@ -1,11 +1,12 @@ -scale: 1.3 -marker_scale: 20.0 -precision: 3 -display_rows: 12 -sample: 2024-07-18T17-44-35 -only_sample: --no-only-sample -load_src_from_outs: --no-load-src-from-outs -frame_count: 0 -frame_step: 1 -compare_with_trackpy: --no-compare-with-trackpy -guess_diameter: 21 +stage: + scale: 1.3 + marker_scale: 20.0 + precision: 3 + display_rows: 12 + sample: "2024-07-18T17-44-35" + only_sample: "--no-only-sample" + load_src_from_outs: "--no-load-src-from-outs" + frame_count: 0 + frame_step: 1 + compare_with_trackpy: "--no-compare-with-trackpy" + guess_diameter: 21