feat: add setModelTransformation capability to 360 image collection (#…

…4350)

* feat: add setting model transformation to a 360 image collection

* fix: make LOD structure factor in model transform

* fix: make intersection logic respect collection transform

* fix: setting transform now also affects visualization box

* fix: adaptive sizing not reacting to transform

* fix: support scaling 360 collection

* fix: not returning cached matrix

* chore: remove test code

* chore: update API

* fix: redraw when transform is updated

* fix: update 360 visual test

* fix: use collection transform in proximity culling

viewer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cognite/reveal",
-  "version": "4.10.8",
+  "version": "4.11.0",
   "description": "WebGL based 3D viewer for CAD and point clouds processed in Cognite Data Fusion.",
   "homepage": "https://github.com/cognitedata/reveal/tree/master/viewer",
   "repository": {

viewer/packages/360-images/src/Image360Facade.ts

@@ -115,19 +115,29 @@ export class Image360Facade<T> {
   }
 
   public intersect(coords: THREE.Vector2, camera: THREE.Camera): Image360Entity | undefined {
-    this._rayCaster.setFromCamera(coords, camera);
     const cameraDirection = camera.getWorldDirection(new THREE.Vector3());
     const cameraPosition = camera.position.clone();
-
-    const intersections = this._image360Collections
-      .flatMap(getImage360Entities)
-      .filter(hasVisibleIcon)
-      .map(entity => getIntersection(entity, this._rayCaster.ray))
-      .filter(hasIntersection)
-      .map(intersectionToCameraSpace)
-      .filter(isInFrontOfCamera)
-      .sort(byDistanceToCamera)
-      .map(selectEntity);
+    const collectionMatrix = new THREE.Matrix4();
+
+    const intersections = this._image360Collections.flatMap(collection =>
+      getImage360Entities(collection)
+        .filter(hasVisibleIcon)
+        .map(
+          getIntersector(
+            getTransformedRay(
+              this._rayCaster,
+              coords,
+              camera,
+              getWorldToModelCollectionMatrix(collection, collectionMatrix)
+            )
+          )
+        )
+        .filter(hasIntersection)
+        .map(intersectionToCameraSpace)
+        .filter(isInFrontOfCamera)
+        .sort(byDistanceToCamera)
+        .map(selectEntity)
+    );
 
     return first(intersections);
 
@@ -139,8 +149,28 @@ export class Image360Facade<T> {
       return entity.icon.getVisible() && !entity.image360Visualization.visible;
     }
 
-    function getIntersection(entity: Image360Entity, ray: THREE.Ray): [Image360Entity, THREE.Vector3 | null] {
-      return [entity, entity.icon.intersect(ray)];
+    function getIntersector(ray: THREE.Ray): (entity: Image360Entity) => [Image360Entity, THREE.Vector3 | null] {
+      return (entity: Image360Entity) => [entity, entity.icon.intersect(ray)];
+    }
+
+    function getTransformedRay(
+      rayCaster: THREE.Raycaster,
+      coords: THREE.Vector2,
+      camera: THREE.Camera,
+      transform: THREE.Matrix4
+    ): THREE.Ray {
+      rayCaster.setFromCamera(coords, camera);
+      rayCaster.ray.applyMatrix4(transform);
+      return rayCaster.ray;
+    }
+
+    function getWorldToModelCollectionMatrix(
+      collection: DefaultImage360Collection,
+      collectionMatrix: THREE.Matrix4
+    ): THREE.Matrix4 {
+      collection.getModelTransformation(collectionMatrix);
+      collectionMatrix.invert();
+      return collectionMatrix;
     }
 
     function hasIntersection(

viewer/packages/360-images/src/collection/DefaultImage360Collection.ts

@@ -30,6 +30,7 @@ import { Image360AnnotationFilter } from '../annotation/Image360AnnotationFilter
 import { Image360 } from '../entity/Image360';
 import { Image360Revision } from '../entity/Image360Revision';
 import { ImageAssetLinkAnnotationInfo } from '@reveal/data-providers';
+import { Matrix4 } from 'three';
 
 type Image360Events = 'image360Entered' | 'image360Exited';
 
@@ -64,6 +65,7 @@ export class DefaultImage360Collection implements Image360Collection {
   private _isCollectionVisible: boolean;
   private readonly _collectionId: string;
   private readonly _collectionLabel: string | undefined;
+  private readonly _setNeedsRedraw: () => void;
 
   get id(): string {
     return this._collectionId;
@@ -101,7 +103,8 @@ export class DefaultImage360Collection implements Image360Collection {
     entities: Image360Entity[],
     icons: IconCollection,
     annotationFilter: Image360AnnotationFilter,
-    image360DataProvider: Image360DataProvider
+    image360DataProvider: Image360DataProvider,
+    setNeedsRedraw: () => void
   ) {
     this._collectionId = collectionId;
     this._collectionLabel = collectionLabel;
@@ -110,6 +113,17 @@ export class DefaultImage360Collection implements Image360Collection {
     this._isCollectionVisible = true;
     this._annotationFilter = annotationFilter;
     this._image360DataProvider = image360DataProvider;
+    this._setNeedsRedraw = setNeedsRedraw;
+  }
+
+  public getModelTransformation(out?: Matrix4): Matrix4 {
+    return this._icons.getTransform(out);
+  }
+
+  public setModelTransformation(matrix: THREE.Matrix4): void {
+    this._icons.setTransform(matrix);
+    this.image360Entities.forEach(entity => entity.setWorldTransform(matrix));
+    this._setNeedsRedraw();
   }
   /**
    * Subscribes to events on 360 Image datasets. There are several event types:

viewer/packages/360-images/src/collection/Image360Collection.ts

@@ -10,6 +10,7 @@ import { Image360Revision } from '../entity/Image360Revision';
 import { IdEither } from '@cognite/sdk';
 import { Image360Annotation } from '../annotation/Image360Annotation';
 import { ImageAssetLinkAnnotationInfo } from '@reveal/data-providers';
+import { Matrix4 } from 'three';
 
 /**
  * Filter for finding annotations related to an asset
@@ -84,6 +85,17 @@ export interface Image360Collection {
    */
   targetRevisionDate: Date | undefined;
 
+  /**
+   * Sets the transformation matrix to be applied to the collection.
+   * @param matrix The transformation matrix to be applied to the collection.
+   */
+  setModelTransformation(matrix: Matrix4): void;
+
+  /**
+   * Gets the transformation matrix of the collection
+   */
+  getModelTransformation(out?: Matrix4): Matrix4;
+
   /**
    * Specify parameters used to determine the number of icons that are visible when entering 360 Images.
    * @param radius Only icons within the given radius will be made visible.

viewer/packages/360-images/src/collection/Image360CollectionFactory.ts

@@ -22,11 +22,13 @@ export class Image360CollectionFactory<T> {
   private readonly _onBeforeSceneRendered: EventTrigger<BeforeSceneRenderedDelegate>;
   private readonly _iconsOptions: IconsOptions | undefined;
   private readonly _device: DeviceDescriptor;
+  private readonly _setNeedsRedraw: () => void;
 
   constructor(
     image360DataProvider: Image360Provider<T>,
     sceneHandler: SceneHandler,
     onBeforeSceneRendered: EventTrigger<BeforeSceneRenderedDelegate>,
+    setNeedsRedraw: () => void,
     device: DeviceDescriptor,
     iconsOptions?: IconsOptions
   ) {
@@ -35,6 +37,7 @@ export class Image360CollectionFactory<T> {
     this._onBeforeSceneRendered = onBeforeSceneRendered;
     this._iconsOptions = iconsOptions;
     this._device = device;
+    this._setNeedsRedraw = setNeedsRedraw;
   }
 
   public async create(
@@ -86,7 +89,8 @@ export class Image360CollectionFactory<T> {
       entities,
       collectionIcons,
       annotationFilterer,
-      this._image360DataProvider
+      this._image360DataProvider,
+      this._setNeedsRedraw
     );
 
     function isDefined(

viewer/packages/360-images/src/entity/Image360Entity.ts

@@ -12,14 +12,15 @@ import { Image360VisualizationBox } from './Image360VisualizationBox';
 import { ImageAnnotationObject } from '../annotation/ImageAnnotationObject';
 import { Overlay3DIcon } from '@reveal/3d-overlays';
 import { Image360AnnotationFilter } from '../annotation/Image360AnnotationFilter';
-import { Color } from 'three';
+import { Color, Matrix4 } from 'three';
 
 import cloneDeep from 'lodash/cloneDeep';
 
 export class Image360Entity implements Image360 {
   private readonly _revisions: Image360RevisionEntity[];
   private readonly _imageMetadata: Image360EventDescriptor;
-  private readonly _transform: THREE.Matrix4;
+  private readonly _modelTransform: THREE.Matrix4;
+  private readonly _worldTransform: THREE.Matrix4;
   private readonly _image360Icon: Overlay3DIcon;
   private readonly _image360VisualizationBox: Image360VisualizationBox;
   private _activeRevision: Image360RevisionEntity;
@@ -31,7 +32,7 @@ export class Image360Entity implements Image360 {
    * @returns model-to-world transform of the 360 Image
    */
   get transform(): THREE.Matrix4 {
-    return this._transform.clone();
+    return this._worldTransform.clone();
   }
 
   /**
@@ -76,11 +77,12 @@ export class Image360Entity implements Image360 {
     icon: Overlay3DIcon,
     device: DeviceDescriptor
   ) {
-    this._transform = transform;
+    this._modelTransform = transform;
+    this._worldTransform = transform.clone();
     this._image360Icon = icon;
     this._imageMetadata = image360Metadata;
 
-    this._image360VisualizationBox = new Image360VisualizationBox(this._transform, sceneHandler, device);
+    this._image360VisualizationBox = new Image360VisualizationBox(this._modelTransform, sceneHandler, device);
     this._image360VisualizationBox.visible = false;
 
     this._revisions = image360Metadata.imageRevisions.map(
@@ -90,6 +92,11 @@ export class Image360Entity implements Image360 {
     this._activeRevision = this.getMostRecentRevision();
   }
 
+  public setWorldTransform(matrix: Matrix4): void {
+    this._worldTransform.copy(matrix).multiply(this._modelTransform);
+    this._image360VisualizationBox.setWorldTransform(matrix);
+  }
+
   /**
    * List all historical images for this entity.
    * @returns A list of available revisions.

viewer/packages/360-images/src/entity/Image360VisualizationBox.ts

@@ -27,6 +27,7 @@ export class Image360VisualizationBox implements Image360Visualization {
   private readonly _textureLoader: THREE.TextureLoader;
   private readonly _faceMaterialOrder: Image360Face['face'][] = ['left', 'right', 'top', 'bottom', 'front', 'back'];
   private readonly _annotationsGroup: THREE.Group = new THREE.Group();
+  private readonly _localTransform: THREE.Matrix4;
 
   get opacity(): number {
     return this._visualizationState.opacity;
@@ -84,7 +85,8 @@ export class Image360VisualizationBox implements Image360Visualization {
   }
 
   constructor(worldTransform: THREE.Matrix4, sceneHandler: SceneHandler, device: DeviceDescriptor) {
-    this._worldTransform = worldTransform;
+    this._localTransform = worldTransform.clone();
+    this._worldTransform = worldTransform.clone();
     this._sceneHandler = sceneHandler;
     this._device = device;
     this._textureLoader = new THREE.TextureLoader();
@@ -96,6 +98,15 @@ export class Image360VisualizationBox implements Image360Visualization {
     };
   }
 
+  public setWorldTransform(matrix: THREE.Matrix4): void {
+    this._worldTransform.copy(matrix).multiply(this._localTransform);
+
+    if (this._visualizationMesh) {
+      this._visualizationMesh.position.setFromMatrixPosition(this._worldTransform);
+      this._visualizationMesh.rotation.setFromRotationMatrix(this._worldTransform);
+    }
+  }
+
   public loadImages(textures: Image360Texture[]): void {
     if (this._visualizationMesh) {
       this._faceMaterialOrder.forEach((face, index) => {
@@ -118,7 +129,8 @@ export class Image360VisualizationBox implements Image360Visualization {
     const boxGeometry = new THREE.BoxGeometry(1, 1, 1);
     const visualizationMesh = new THREE.Mesh(boxGeometry, this._faceMaterials);
     visualizationMesh.renderOrder = this._visualizationState.renderOrder;
-    visualizationMesh.applyMatrix4(this._worldTransform);
+    visualizationMesh.position.setFromMatrixPosition(this._worldTransform);
+    visualizationMesh.rotation.setFromRotationMatrix(this._worldTransform);
     visualizationMesh.scale.copy(this._visualizationState.scale);
     visualizationMesh.visible = this._visualizationState.visible;
     visualizationMesh.add(this._annotationsGroup);

viewer/packages/360-images/src/icons/IconCollection.ts

@@ -108,13 +108,25 @@ export class IconCollection {
     sceneHandler.addObject3D(iconsSprites);
   }
 
+  public setTransform(transform: Matrix4): void {
+    this._pointsObject.setTransform(transform);
+    this._icons.forEach(icon => icon.setWorldTransform(transform));
+  }
+
+  public getTransform(out?: Matrix4): Matrix4 {
+    return this._pointsObject.getTransform(out);
+  }
+
   private setIconClustersByLOD(octree: IconOctree, iconSprites: OverlayPointsObject): BeforeSceneRenderedDelegate {
     const projection = new Matrix4();
     const frustum = new Frustum();
     const screenSpaceAreaThreshold = 0.04;
     const minimumLevel = 3;
     return ({ camera }) => {
-      projection.copy(camera.projectionMatrix).multiply(camera.matrixWorldInverse);
+      projection
+        .copy(camera.projectionMatrix)
+        .multiply(camera.matrixWorldInverse)
+        .multiply(this._pointsObject.getTransform());
       const nodesLOD = octree.getLODByScreenArea(screenSpaceAreaThreshold, projection, minimumLevel);
 
       frustum.setFromProjectionMatrix(projection);
@@ -142,12 +154,18 @@ export class IconCollection {
   }
 
   private computeProximityPoints(octree: IconOctree, iconSprites: OverlayPointsObject): BeforeSceneRenderedDelegate {
+    const cameraModelSpacePosition = new Vector3();
+    const worldTransform = new Matrix4();
     return ({ camera }) => {
+      this._pointsObject.getTransform(worldTransform);
+      worldTransform.invert();
+      cameraModelSpacePosition.copy(camera.position).applyMatrix4(worldTransform);
+
       const points =
         this._proximityRadius === Infinity
           ? this._icons
           : octree
-              .findPoints(camera.position, this._proximityRadius)
+              .findPoints(cameraModelSpacePosition, this._proximityRadius)
               .map(pointContainer => {
                 return pointContainer.data;
               })
@@ -156,7 +174,8 @@ export class IconCollection {
       const closestPoints = points
         .sort((a, b) => {
           return (
-            a.getPosition().distanceToSquared(camera.position) - b.getPosition().distanceToSquared(camera.position)
+            a.getPosition().distanceToSquared(cameraModelSpacePosition) -
+            b.getPosition().distanceToSquared(cameraModelSpacePosition)
           );
         })
         .slice(0, this._proximityPointLimit + 1); //Add 1 to account for self.
@@ -204,7 +223,7 @@ export class IconCollection {
 
     icons.forEach(icon =>
       icon.on('selected', () => {
-        this._hoverSprite.position.copy(icon.getPosition());
+        this._hoverSprite.position.copy(icon.getPosition().clone().applyMatrix4(this.getTransform()));
         this._hoverSprite.scale.set(icon.adaptiveScale * 2, icon.adaptiveScale * 2, 1);
       })
     );

viewer/packages/360-images/unit-tests/Image360CollectionFactory.test.ts

@@ -60,6 +60,7 @@ describe(Image360CollectionFactory.name, () => {
       mock360ImageProvider.object(),
       mockSceneHandler.object(),
       new EventTrigger<BeforeSceneRenderedDelegate>(),
+      () => {},
       desktopDevice,
       { platformMaxPointsSize: 256 }
     );