Do not overwrite reserved visibility flags in ogre2 (Fortress) (#783)

Calling Sensor::SetVisibilityMask in ogre2 with bits 31 or 30 set (which are reserved by OgreNext) would cause wrong rendering. PR #760 already tries to fix this issue on a localized problem: heightmap in lidar. This is the general solution. This PR simply uses PassSceneDef::setVisibilityMask instead of directly setting PassSceneDef::mVisibilityMask to mask out the reserved flags. Ogre2Camera::setVisibilityMask has been modified to warn when the reserved flags are set. Other cameras (e.g. Lidar's Ogre2GpuRays) won't warn though, since they don't derive from Ogre2Camera. I cannot add such warning without breaking the ABI. Signed-off-by: Matias N. Goldberg <[email protected]> Co-authored-by: Ian Chen <[email protected]>
gazebosim · Dec 12, 2022 · 090d079 · 090d079
1 parent b3df123
commit 090d079
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 16 deletions.
diff --git a/ogre2/src/Ogre2Camera.cc b/ogre2/src/Ogre2Camera.cc
@@ -376,6 +376,12 @@ Ogre::Camera *Ogre2Camera::OgreCamera() const
 //////////////////////////////////////////////////
 void Ogre2Camera::SetVisibilityMask(uint32_t _mask)
 {
+  if (_mask & ~Ogre::VisibilityFlags::RESERVED_VISIBILITY_FLAGS)
+  {
+    ignerr << "Ogre2Camera::SetVisibilityMask: Mask bits " << std::hex
+           << ~Ogre::VisibilityFlags::RESERVED_VISIBILITY_FLAGS << std::dec
+           << " are set but will be ignored by ogre2 backend." << std::endl;
+  }
   BaseSensor::SetVisibilityMask(_mask);
   if (this->renderTexture)
     this->renderTexture->SetVisibilityMask(_mask);

diff --git a/ogre2/src/Ogre2DepthCamera.cc b/ogre2/src/Ogre2DepthCamera.cc
@@ -718,7 +718,7 @@ void Ogre2DepthCamera::CreateDepthTexture()
             static_cast<Ogre::CompositorPassSceneDef *>(
             colorTargetDef->addPass(Ogre::PASS_SCENE));
         passScene->mShadowNode = this->dataPtr->kShadowNodeName;
-        passScene->mVisibilityMask = IGN_VISIBILITY_ALL;
+        passScene->setVisibilityMask(IGN_VISIBILITY_ALL);
         passScene->mIncludeOverlays = false;
         passScene->mFirstRQ = 0u;
         passScene->mLastRQ = 2u;
@@ -755,7 +755,7 @@ void Ogre2DepthCamera::CreateDepthTexture()
         Ogre::CompositorPassSceneDef *passScene =
             static_cast<Ogre::CompositorPassSceneDef *>(
             colorTargetDef->addPass(Ogre::PASS_SCENE));
-        passScene->mVisibilityMask = IGN_VISIBILITY_ALL;
+        passScene->setVisibilityMask(IGN_VISIBILITY_ALL);
         // todo(anyone) PbsMaterialsShadowNode is hardcoded.
         // Although this may be just fine
         passScene->mShadowNode = this->dataPtr->kShadowNodeName;
@@ -777,8 +777,8 @@ void Ogre2DepthCamera::CreateDepthTexture()
         this->FarClipPlane(),
         this->FarClipPlane()));
       // depth texute does not contain particles
-      passScene->mVisibilityMask = IGN_VISIBILITY_ALL
-          & ~Ogre2ParticleEmitter::kParticleVisibilityFlags;
+      passScene->setVisibilityMask(
+        IGN_VISIBILITY_ALL & ~Ogre2ParticleEmitter::kParticleVisibilityFlags);
     }
 
     Ogre::CompositorTargetDef *particleTargetDef =
@@ -791,8 +791,8 @@ void Ogre2DepthCamera::CreateDepthTexture()
           particleTargetDef->addPass(Ogre::PASS_SCENE));
       passScene->setAllLoadActions(Ogre::LoadAction::Clear);
       passScene->setAllClearColours(Ogre::ColourValue::Black);
-      passScene->mVisibilityMask =
-          Ogre2ParticleEmitter::kParticleVisibilityFlags;
+      passScene->setVisibilityMask(
+        Ogre2ParticleEmitter::kParticleVisibilityFlags);
     }
 
     // rt0 target - converts depth to xyz

diff --git a/ogre2/src/Ogre2GpuRays.cc b/ogre2/src/Ogre2GpuRays.cc
@@ -961,11 +961,9 @@ void Ogre2GpuRays::Setup1stPass()
           colorTargetDef->addPass(Ogre::PASS_SCENE));
       passScene->setAllLoadActions(Ogre::LoadAction::Clear);
       passScene->setAllClearColours(Ogre::ColourValue(0, 0, 0));
-      // set visibility mask and '&' it with IGN_VISIBILITY_ALL (0x0FFFFFFF)
-      // to make sure the fist 4 bits are 0 otherwise lidar will not be able
-      // to detect heightmaps
-      passScene->mVisibilityMask = (this->VisibilityMask() & IGN_VISIBILITY_ALL)
-          & ~Ogre2ParticleEmitter::kParticleVisibilityFlags;
+      passScene->setVisibilityMask(
+        this->VisibilityMask() &
+        ~Ogre2ParticleEmitter::kParticleVisibilityFlags);
     }
 
     Ogre::CompositorTargetDef *particleTargetDef =

diff --git a/ogre2/src/Ogre2RenderTarget.cc b/ogre2/src/Ogre2RenderTarget.cc
@@ -59,8 +59,8 @@ class Ogre2RenderTargetCompositorListener :
       Ogre::Viewport *vp = scenePass->getCamera()->getLastViewport();
       if (vp == nullptr) return;
       // make sure we do not alter the reserved visibility flags
-      uint32_t f = this->ogreRenderTarget->VisibilityMask() |
-          ~Ogre::VisibilityFlags::RESERVED_VISIBILITY_FLAGS;
+      uint32_t f = this->ogreRenderTarget->VisibilityMask() &
+                   Ogre::VisibilityFlags::RESERVED_VISIBILITY_FLAGS;
       // apply the new visibility mask
       uint32_t flags = f & vp->getVisibilityMask();
       vp->_setVisibilityMask(flags, vp->getLightVisibilityMask());

diff --git a/ogre2/src/Ogre2SelectionBuffer.cc b/ogre2/src/Ogre2SelectionBuffer.cc
@@ -347,7 +347,7 @@ void Ogre2SelectionBuffer::CreateRTTBuffer()
         colorTargetDef->addPass(Ogre::PASS_SCENE));
     passScene->setAllLoadActions(Ogre::LoadAction::Clear);
     passScene->setAllClearColours(Ogre::ColourValue::Black);
-    passScene->mVisibilityMask = IGN_VISIBILITY_SELECTABLE;
+    passScene->setVisibilityMask(IGN_VISIBILITY_SELECTABLE);
   }
 
   Ogre::CompositorTargetDef *targetDef = nodeDef->addTargetPass("rt");

diff --git a/ogre2/src/Ogre2ThermalCamera.cc b/ogre2/src/Ogre2ThermalCamera.cc
@@ -791,8 +791,8 @@ void Ogre2ThermalCamera::CreateThermalTexture()
       passScene->setAllLoadActions(Ogre::LoadAction::Clear);
       passScene->setAllClearColours(Ogre::ColourValue(0, 0, 0));
       // thermal camera should not see particles
-      passScene->mVisibilityMask = IGN_VISIBILITY_ALL &
-          ~Ogre2ParticleEmitter::kParticleVisibilityFlags;
+      passScene->setVisibilityMask(
+        IGN_VISIBILITY_ALL & ~Ogre2ParticleEmitter::kParticleVisibilityFlags);
     }
 
     // rt_input target - converts to thermal