Merge pull request #563 from PaulHax/camera-bounds

Load image bounded by camera frustum

src/Context/ImageActorContext.js

@@ -2,9 +2,6 @@ class ImageActorContext {
   // MultiscaleSpatialImage to be visualized
   image = null
 
-  // The target image scale
-  targetScale = null
-
   // The successfully loaded scale
   loadedScale = null
 
@@ -45,10 +42,16 @@ class ImageActorContext {
   // Map of image intensity component to array of [minValue, maxValue] for
   // mapping colors
   colorRanges = new Map()
+  // Keep growing component ranges as new parts of the image are loaded
+  // Map of image intensity component to Boolean
+  colorRangesAutoAdjust = null
 
   // Map of image intensity component to array of [minBound, maxBound] for
   // limiting the color range in the UI
   colorRangeBounds = new Map()
+  // Keep growing component bounds as new parts of the image are loaded
+  // Map of image intensity component to Boolean
+  colorRangeBoundsAutoAdjust = null
 
   // Map of the image intensity component to an object representing the
   // piecewise function. This object has two properties: range and nodes.

src/Context/MainMachineContext.js

@@ -52,15 +52,16 @@ class MainMachineContext {
 
   // Cropping planes widget enabled
   croppingPlanesEnabled = false
+  areCroppingPlanesTouched = false
 
-  // Cropping planes. These typically define square or a cube of a region of
+  // Cropping planes. These typically define a box containing a region of
   // interest in space. The visualization is cropped outside of these planes.
   // Each is characterized with: { origin, normal }.
   //
   // origin: x,y,z point at a point in the plane
   // normal: 3-component vector defining the normal to the plane
   //
-  // An array of six planes that are ordered, when the planes are axis aligned:
+  // An example: An array of six planes. When the planes are axis aligned:
   // -x, +x, -y, +y, -z, +z
   croppingPlanes = null
 

src/IO/ConglomerateMultiscaleSpatialImage.js

@@ -21,11 +21,18 @@ export class ConglomerateMultiscaleSpatialImage extends MultiscaleSpatialImage {
     this.images = images
   }
 
-  async buildImage(scale, bounds) {
-    // Run sequentially rather than Promise.all to avoid hang
+  async getImage(scale, worldBounds = []) {
+    this.worldBoundsForBuildImage = worldBounds
+    return super.getImage(scale, worldBounds)
+  }
+
+  async buildImage(scale /*bounds*/) {
+    // Run sequentially rather than Promise.all to avoid hang during chunk fetching
     const builtImages = []
     for (const image of this.images) {
-      builtImages.push(await image.buildImage(scale, bounds))
+      builtImages.push(
+        await image.getImage(scale, this.worldBoundsForBuildImage)
+      )
     }
     return builtImages
   }

src/IO/MultiscaleSpatialImage.js

@@ -135,16 +135,17 @@ export const worldBoundsToIndexBounds = ({
   fullIndexBounds,
   worldToIndex,
 }) => {
+  const fullIndexBoundsWithZCT = ensuredDims([0, 1], CXYZT, fullIndexBounds)
   if (!bounds || bounds.length === 0) {
     // no bounds, return full image
-    return fullIndexBounds
+    return fullIndexBoundsWithZCT
   }
 
   const imageBounds = transformBounds(worldToIndex, bounds)
   // clamp to existing integer indexes
   const imageBoundsByDim = chunkArray(2, imageBounds)
   const spaceBounds = ['x', 'y', 'z'].map((dim, idx) => {
-    const [min, max] = fullIndexBounds.get(dim)
+    const [min, max] = fullIndexBoundsWithZCT.get(dim)
     const [bmin, bmax] = imageBoundsByDim[idx]
     return [
       dim,
@@ -154,7 +155,7 @@ export const worldBoundsToIndexBounds = ({
       ],
     ]
   })
-  const ctBounds = ['c', 't'].map(dim => [dim, fullIndexBounds.get(dim)])
+  const ctBounds = ['c', 't'].map(dim => [dim, fullIndexBoundsWithZCT.get(dim)])
   return new Map([...spaceBounds, ...ctBounds])
 }
 
@@ -374,15 +375,10 @@ class MultiscaleSpatialImage {
     const indexToWorld = await this.scaleIndexToWorld(scale)
 
     const { dims } = this.scaleInfo[scale]
-    const fullIndexBounds = ensuredDims(
-      [0, 1],
-      CXYZT,
-      this.getIndexBounds(scale)
-    )
     const indexBounds = orderBy(dims)(
       worldBoundsToIndexBounds({
         bounds: worldBounds,
-        fullIndexBounds,
+        fullIndexBounds: this.getIndexBounds(scale),
         worldToIndex: mat4.invert([], indexToWorld),
       })
     )

src/Rendering/Images/createImageRenderingActor.js

@@ -3,6 +3,24 @@ import { assign, createMachine, forwardTo } from 'xstate'
 const getLoadedImage = actorContext =>
   actorContext.image ?? actorContext.labelImage
 
+const assignColorRange = assign({
+  images: (
+    { images },
+    { data: { name, component, range, keepAutoAdjusting = false } }
+  ) => {
+    const { colorRanges, colorRangesAutoAdjust } = images.actorContext.get(name)
+
+    colorRanges.set(component, range)
+
+    colorRangesAutoAdjust.set(
+      component,
+      colorRangesAutoAdjust.get(component) && keepAutoAdjusting
+    )
+
+    return images
+  },
+})
+
 const assignUpdateRenderedName = assign({
   images: (context, event) => {
     const images = context.images
@@ -45,32 +63,39 @@ const assignLoadedScale = assign({
   },
 })
 
+const assignClearHistograms = assign({
+  images: ({ images }, { data }) => {
+    const name = data.name ?? data
+    const actorContext = images.actorContext.get(name)
+    actorContext.histograms = new Map()
+    return images
+  },
+})
+
 const LOW_FPS = 10.0
-const JUST_ACCEPTABLE_FPS = 30.0
+const HIGH_FPS = 30.0
 
-// Return true if finest scale or right scale (to stop loading of finer scale)
+// Return true if finest scale or already backed off coarser or FPS is in Goldilocks zone
 function finestScaleOrScaleJustRight(context) {
   const { loadedScale } = context.images.actorContext.get(
     context.images.updateRenderedName
   )
   return (
     loadedScale === 0 ||
     context.hasScaledCoarser ||
-    (LOW_FPS < context.main.fps && context.main.fps < JUST_ACCEPTABLE_FPS)
+    (LOW_FPS < context.main.fps && context.main.fps < HIGH_FPS)
   )
 }
 
-function scaleTooHigh(context) {
+function isFpsLow(context) {
   return context.main.fps <= LOW_FPS
 }
 
-function scaleTooHighAndMostCoarse(context) {
+function isLoadedScaleMostCoarse(context) {
   const actorContext = context.images.actorContext.get(
     context.images.updateRenderedName
   )
-  const { coarsestScale } = getLoadedImage(actorContext)
-  const { loadedScale } = actorContext
-  return scaleTooHigh(context) && loadedScale === coarsestScale
+  return getLoadedImage(actorContext).coarsestScale === actorContext.loadedScale
 }
 
 const assignIsFramerateScalePickingOn = assign({
@@ -81,6 +106,21 @@ const assignIsFramerateScalePickingOn = assign({
   },
 })
 
+const KNOWN_ERRORS = [
+  'Voxel count over max at scale',
+  "DataCloneError: Failed to execute 'postMessage' on 'Worker': Data cannot be cloned, out of memory.",
+]
+
+const checkIsKnownErrorOrThrow = (c, { data: error }) => {
+  if (
+    KNOWN_ERRORS.some(knownMessage => error.message.startsWith(knownMessage))
+  ) {
+    console.warn(`Could not update image : ${error.message}`)
+  } else {
+    throw error
+  }
+}
+
 const sendRenderedImageAssigned = (
   context,
   { data: { name, loadedScale } }
@@ -124,7 +164,10 @@ const eventResponses = {
     actions: 'mapToPiecewiseFunctionNodes',
   },
   IMAGE_COLOR_RANGE_CHANGED: {
-    actions: 'applyColorRange',
+    actions: [assignColorRange, 'applyColorRange'],
+  },
+  IMAGE_COLOR_RANGE_BOUNDS_CHANGED: {
+    actions: ['applyColorRangeBounds'],
   },
   IMAGE_COLOR_MAP_SELECTED: {
     actions: 'applyColorMap',
@@ -175,7 +218,10 @@ const eventResponses = {
   // updateRenderedImage->applyRenderedImage->updateCroppingParametersFromImage->CROPPING_PLANES_CHANGED
   // because image size may change across scales.
   CROPPING_PLANES_CHANGED_BY_USER: {
-    target: 'imageBoundsDebouncing',
+    target: 'debouncedImageUpdate',
+  },
+  CAMERA_MODIFIED: {
+    target: 'debouncedImageUpdate',
   },
 }
 
@@ -188,7 +234,7 @@ const createUpdatingImageMachine = options => {
         checkingUpdateNeeded: {
           always: [
             { cond: 'isImageUpdateNeeded', target: 'loadingImage' },
-            { target: '#updatingImageMachine.afterUpdatingImage' },
+            { target: '#updatingImageMachine.loadedImage' },
           ],
           exit: assign({ isUpdateForced: false }),
         },
@@ -198,27 +244,23 @@ const createUpdatingImageMachine = options => {
             id: 'updateRenderedImage',
             src: 'updateRenderedImage',
             onDone: {
-              target: '#updatingImageMachine.afterUpdatingImage',
+              target: '#updatingImageMachine.loadedImage',
               actions: [
                 'assignRenderedImage',
                 assignLoadedScale,
+                assignClearHistograms,
                 'applyRenderedImage',
                 sendRenderedImageAssigned,
               ],
             },
             onError: {
-              actions: [
-                (c, event) => {
-                  console.warn(`Could not update image : ${event.data.stack}`)
-                },
-                assignCoarserScale,
-              ],
+              actions: [checkIsKnownErrorOrThrow, assignCoarserScale],
               target: 'checkingUpdateNeeded',
             },
           },
         },
 
-        afterUpdatingImage: {
+        loadedImage: {
           always: [
             {
               cond: 'isFramerateScalePickingOn',
@@ -235,17 +277,18 @@ const createUpdatingImageMachine = options => {
           on: {
             FPS_UPDATED: [
               {
-                cond: scaleTooHighAndMostCoarse, // FPS too slow but nothing to do about it
-                target: 'finished',
+                cond: c =>
+                  [isFpsLow, isLoadedScaleMostCoarse].every(cond => cond(c)),
+                target: 'finished', // FPS too slow but nothing to do about it
               },
               {
-                cond: scaleTooHigh, // FPS too slow
+                cond: isFpsLow,
                 actions: assignCoarserScale, // back off
                 target: 'checkingUpdateNeeded',
               },
               {
-                cond: finestScaleOrScaleJustRight, // found good scale
-                target: 'finished',
+                cond: finestScaleOrScaleJustRight,
+                target: 'finished', // found good scale
               },
               {
                 actions: assignFinerScale, // try harder
@@ -282,7 +325,7 @@ const createImageRenderingActor = (options, context /*, event*/) => {
           },
         },
 
-        imageBoundsDebouncing: {
+        debouncedImageUpdate: {
           on: {
             ...eventResponses,
           },

src/Rendering/Images/createImagesRenderingMachine.js

@@ -24,6 +24,15 @@ function spawnImageRenderingActor(options) {
   })
 }
 
+const sendEventToAllActors = () =>
+  actions.pure(({ images: { imageRenderingActors } }, event) =>
+    Array.from(imageRenderingActors.values()).map(actor =>
+      send(event, {
+        to: actor,
+      })
+    )
+  )
+
 function createImagesRenderingMachine(options, context) {
   const { imageRenderingActor } = options
 
@@ -46,6 +55,14 @@ function createImagesRenderingMachine(options, context) {
           },
         },
         active: {
+          invoke: {
+            id: 'cameraModifiedWatcher',
+            src: context => send =>
+              context.itkVtkView
+                .getRenderer()
+                .getActiveCamera()
+                .onModified(() => send('CAMERA_MODIFIED')).unsubscribe, // return cleanup func
+          },
           on: {
             IMAGE_ASSIGNED: {
               actions: spawnImageRenderingActor(imageRenderingActor),
@@ -96,6 +113,11 @@ function createImagesRenderingMachine(options, context) {
                 to: (c, e) => `imageRenderingActor-${e.data.name}`,
               }),
             },
+            IMAGE_COLOR_RANGE_BOUNDS_CHANGED: {
+              actions: send((_, e) => e, {
+                to: (c, e) => `imageRenderingActor-${e.data.name}`,
+              }),
+            },
             IMAGE_COLOR_RANGE_CHANGED: {
               actions: send((_, e) => e, {
                 to: (c, e) => `imageRenderingActor-${e.data.name}`,
@@ -162,14 +184,10 @@ function createImagesRenderingMachine(options, context) {
               }),
             },
             CROPPING_PLANES_CHANGED_BY_USER: {
-              // send to all image actors
-              actions: actions.pure(({ images: { imageRenderingActors } }) =>
-                Array.from(imageRenderingActors.values()).map(actor =>
-                  send('CROPPING_PLANES_CHANGED_BY_USER', {
-                    to: actor,
-                  })
-                )
-              ),
+              actions: sendEventToAllActors(),
+            },
+            CAMERA_MODIFIED: {
+              actions: sendEventToAllActors(),
             },
           },
         },