Fix shared depth buffers leaking (#103)

Use a reference count system to track which shared depth buffers are in
use, and release those that reach 0 ref count

Additionally, added TextureGpu::getSourceType for useful stats tracking

Patch originally by SolarPortal.
See https://forums.ogre3d.org/viewtopic.php?p=548396#p548396

Modifications by Matias N. Goldberg:
- Original patch exposed several functions as public which shouldn't be
- Renamed "DepthBuffer" to "SharedDepthBuffer", since RenderSystem
doesn't manage all depth buffers. It only manages automatically
generated ones
- Moved mSourceType so that it takes advantage of preexisting padding
- _dereferenceSharedDepthBuffer original version accepted any input
(even non-shared depth buffers) thus adding zombie entries to
mSharedDepthBufferRefs. It later relied on destroySharedDepthBuffer's
mSourceType to avoid destroying these. Fixed by only accepting shared
depth buffer as input
- Due to these zombie entries, negative ref count was possible. Got rid
of this and ref count is now unsigned. Underflow is considered an Ogre
bug and will assert
- Assert if user tries to call _setDepthBufferDefaults on a shared
depth buffer
- Original version did not release Stencil-only buffers (at the time
only iOS supports this)
- Added mSharedDepthBufferZeroRefCandidates so that the cost of
_cleanupDepthBuffers (which gets called every frame) is almost always
O(1), rather than being O(N) where N is the number of live shared depth
buffers.
- Ported changes to Metal

OgreMain/include/OgreRenderSystem.h

@@ -55,6 +55,8 @@ namespace Ogre
 
     typedef vector<TextureGpu*>::type TextureGpuVec;
     typedef map< uint16, TextureGpuVec >::type DepthBufferMap2;
+    typedef map<TextureGpu *, uint16>::type DepthBufferRefMap;
+    typedef set<TextureGpu *>::type TextureGpuSet;
 
     /// Enum describing the ways to generate texture coordinates
     enum TexCoordCalcMethod
@@ -755,9 +757,17 @@ namespace Ogre
 
     protected:
         virtual TextureGpu* createDepthBufferFor( TextureGpu *colourTexture, bool preferDepthTexture,
-                                                  PixelFormatGpu depthBufferFormat );
+                                                  PixelFormatGpu depthBufferFormat, uint16 poolId );
 
+        /// Detroys a depth buffer associated in the pool. If no texture is found the it skips.
+        void destroySharedDepthBuffer( TextureGpu *depthTexture );
+        void referenceSharedDepthBuffer( TextureGpu *depthBuffer );
     public:
+        void _cleanupDepthBuffers( void );
+        /// Releases the reference count on a shared depth buffer.
+        /// Does nothing if input is not a shared depth buffer.
+        void _dereferenceSharedDepthBuffer( TextureGpu *depthBuffer );
+
         virtual TextureGpu* getDepthBufferFor( TextureGpu *colourTexture, uint16 poolId,
                                                bool preferDepthTexture,
                                                PixelFormatGpu depthBufferFormat );
@@ -1394,6 +1404,8 @@ namespace Ogre
         void destroyAllRenderPassDescriptors(void);
 
         DepthBufferMap2 mDepthBufferPool2;
+        DepthBufferRefMap mSharedDepthBufferRefs;
+        TextureGpuSet mSharedDepthBufferZeroRefCandidates;
 
         typedef set<RenderPassDescriptor*>::type RenderPassDescriptorSet;
         RenderPassDescriptorSet mRenderPassDescs;

OgreMain/include/OgreTextureGpu.h

@@ -157,6 +157,21 @@ namespace Ogre
             PoolOwner           = 1u << 13u
         };
     }
+
+    namespace TextureSourceType
+    {
+        enum TextureSourceType
+        {
+            Standard,           /// Regular texture
+            Shadow,             /// Created by compositor, for shadow mapping
+            Compositor,         /// Created by compositor
+            PoolOwner,          /// TextureFlags::PoolOwner is set
+            SharedDepthBuffer,  /// Created automatically, may be shared and reused by multiple colour
+                                /// targets
+            NumTextureSourceTypes
+        };
+    }
+
     /**
     @remarks
         Internal layout of data in memory:
@@ -208,6 +223,14 @@ namespace Ogre
         /// our actual data is, inside a texture array which we do not own.
         uint16      mInternalSliceStart;
 
+        /// This setting is for where the texture is created, e.g. its a compositor texture, a shadow
+        /// texture or standard texture loaded for a mesh etc...
+        ///
+        /// This value is merely for statistical tracking purposes
+        ///
+        /// @see    TextureSourceType::TextureSourceType
+        uint8 mSourceType;
+
         /// This setting can only be altered if mResidencyStatus == OnStorage).
         TextureTypes::TextureTypes  mTextureType;
         PixelFormatGpu              mPixelFormat;
@@ -305,6 +328,10 @@ namespace Ogre
         TextureTypes::TextureTypes getTextureType(void) const;
         TextureTypes::TextureTypes getInternalTextureType(void) const;
 
+        void _setSourceType( uint8 type );
+        /// @copydoc TextureGpu::mSourceType
+        uint8 getSourceType( void ) const;
+
         /** Sets the pixel format.
         @remarks
             If prefersLoadingFromFileAsSRGB() returns true, the format may not be fully honoured

OgreMain/src/Compositor/OgreCompositorShadowNode.cpp

@@ -91,6 +91,19 @@ namespace Ogre
         mShadowMapCameras.reserve( definition->mShadowMapTexDefinitions.size() );
         mLocalTextures.reserve( mLocalTextures.size() + definition->mShadowMapTexDefinitions.size() );
 
+        // Set all textures to shadow tex source
+        {
+            CompositorChannelVec::iterator tempIt = mLocalTextures.begin();
+            CompositorChannelVec::iterator tempEn = mLocalTextures.end();
+            while( tempIt != tempEn )
+            {
+                TextureGpu *refTex = *tempIt;
+                if( refTex )
+                    refTex->_setSourceType( TextureSourceType::Shadow );
+                ++tempIt;
+            }
+        }
+
         SceneManager *sceneManager = workspace->getSceneManager();
         SceneNode *pseudoRootNode = 0;
 

OgreMain/src/Compositor/OgreTextureDefinition.cpp

@@ -340,6 +340,7 @@ namespace Ogre
                 (textureDef.depthOrSlices == 1u || textureDef.textureType > TextureTypes::Type2D ) &&
                 (textureDef.depthOrSlices == 6 || textureDef.textureType != TextureTypes::TypeCube) );
 
+        tex->_setSourceType( TextureSourceType::Compositor );
         tex->setResolution( width, height, textureDef.depthOrSlices );
         if( textureDef.format != PFG_UNKNOWN )
             tex->setPixelFormat( textureDef.format );

OgreMain/src/Compositor/Pass/OgreCompositorPass.cpp

@@ -831,8 +831,24 @@ namespace Ogre
             mNumPassesLeft = mDefinition->mNumInitialPasses;
 
             //Reset texture pointers and setup RenderPassDescriptor again
-            mRenderPassDesc->mDepth.texture = 0;
-            mRenderPassDesc->mStencil.texture = 0;
+			RenderSystem *renderSystem = mParentNode->getRenderSystem();
+            if( mRenderPassDesc->mDepth.texture )
+            {
+                renderSystem->_dereferenceSharedDepthBuffer( mRenderPassDesc->mDepth.texture );
+                mRenderPassDesc->mDepth.texture = 0;
+
+                if( mRenderPassDesc->mStencil.texture &&
+                    mRenderPassDesc->mStencil.texture == mRenderPassDesc->mDepth.texture )
+                {
+                    renderSystem->_dereferenceSharedDepthBuffer( mRenderPassDesc->mStencil.texture );
+                    mRenderPassDesc->mStencil.texture = 0;
+                }
+            }
+            if( mRenderPassDesc->mStencil.texture )
+            {
+                renderSystem->_dereferenceSharedDepthBuffer( mRenderPassDesc->mStencil.texture );
+                mRenderPassDesc->mStencil.texture = 0;
+            }
 
             for( int i=0; i<OGRE_MAX_MULTIPLE_RENDER_TARGETS; ++i )
             {

OgreMain/src/OgreRenderSystem.cpp

@@ -449,6 +449,24 @@ namespace Ogre {
         assert( itor != mRenderPassDescs.end() && "Already destroyed?" );
         if( itor != mRenderPassDescs.end() )
             mRenderPassDescs.erase( itor );
+
+        if( renderPassDesc->mDepth.texture )
+        {
+            _dereferenceSharedDepthBuffer( renderPassDesc->mDepth.texture );
+
+            if( renderPassDesc->mStencil.texture &&
+                renderPassDesc->mStencil.texture == renderPassDesc->mDepth.texture )
+            {
+                _dereferenceSharedDepthBuffer( renderPassDesc->mStencil.texture );
+                renderPassDesc->mStencil.texture = 0;
+            }
+        }
+        if( renderPassDesc->mStencil.texture )
+        {
+            _dereferenceSharedDepthBuffer( renderPassDesc->mStencil.texture );
+            renderPassDesc->mStencil.texture = 0;
+        }
+
         delete renderPassDesc;
     }
     //---------------------------------------------------------------------
@@ -502,8 +520,71 @@ namespace Ogre {
         mUavRenderingDirty = true;
     }
     //---------------------------------------------------------------------
+    void RenderSystem::destroySharedDepthBuffer( TextureGpu *depthBuffer )
+    {
+        TextureGpuVec &bufferVec = mDepthBufferPool2[depthBuffer->getDepthBufferPoolId()];
+        TextureGpuVec::iterator itor = std::find( bufferVec.begin(), bufferVec.end(), depthBuffer );
+        if( itor != bufferVec.end() )
+        {
+            efficientVectorRemove( bufferVec, itor );
+            mTextureGpuManager->destroyTexture( depthBuffer );
+        }
+    }
+    //---------------------------------------------------------------------
+    void RenderSystem::_cleanupDepthBuffers( void )
+    {
+        TextureGpuSet::const_iterator itor = mSharedDepthBufferZeroRefCandidates.begin();
+        TextureGpuSet::const_iterator endt = mSharedDepthBufferZeroRefCandidates.end();
+
+        while( itor != endt )
+        {
+            // When a shared depth buffer ends up in mSharedDepthBufferZeroRefCandidates,
+            // it's because its ref. count reached 0. However it may have been reacquired.
+            // We need to check its reference count is still 0 before deleting it.
+            DepthBufferRefMap::iterator itMap = mSharedDepthBufferRefs.find( *itor );
+            if( itMap != mSharedDepthBufferRefs.end() && itMap->second == 0u )
+            {
+                destroySharedDepthBuffer( *itor );
+                mSharedDepthBufferRefs.erase( itMap );
+            }
+            ++itor;
+        }
+
+        mSharedDepthBufferZeroRefCandidates.clear();
+    }
+    //---------------------------------------------------------------------
+    void RenderSystem::referenceSharedDepthBuffer( TextureGpu *depthBuffer )
+    {
+        OGRE_ASSERT_MEDIUM( depthBuffer->getSourceType() == TextureSourceType::SharedDepthBuffer );
+        OGRE_ASSERT_MEDIUM( mSharedDepthBufferRefs.find( depthBuffer ) != mSharedDepthBufferRefs.end() );
+        ++mSharedDepthBufferRefs[depthBuffer];
+    }
+    //---------------------------------------------------------------------
+    void RenderSystem::_dereferenceSharedDepthBuffer( TextureGpu *depthBuffer )
+    {
+        if( !depthBuffer )
+            return;
+
+        DepthBufferRefMap::iterator itor = mSharedDepthBufferRefs.find( depthBuffer );
+
+        if( itor != mSharedDepthBufferRefs.end() )
+        {
+            OGRE_ASSERT_MEDIUM( depthBuffer->getSourceType() == TextureSourceType::SharedDepthBuffer );
+            OGRE_ASSERT_LOW( itor->second > 0u && "Releasing a shared depth buffer too much" );
+            --itor->second;
+
+            if( itor->second == 0u )
+                mSharedDepthBufferZeroRefCandidates.insert( depthBuffer );
+        }
+        else
+        {
+            // This is not a shared depth buffer (e.g. one created by the user)
+            OGRE_ASSERT_MEDIUM( depthBuffer->getSourceType() != TextureSourceType::SharedDepthBuffer );
+        }
+    }
+    //---------------------------------------------------------------------
     TextureGpu* RenderSystem::createDepthBufferFor( TextureGpu *colourTexture, bool preferDepthTexture,
-                                                    PixelFormatGpu depthBufferFormat )
+                                                    PixelFormatGpu depthBufferFormat, uint16 poolId )
     {
         uint32 textureFlags = TextureFlags::RenderToTexture;
 
@@ -517,13 +598,16 @@ namespace Ogre {
         TextureGpu *retVal = mTextureGpuManager->createTexture( depthBufferName.c_str(),
                                                                 GpuPageOutStrategy::Discard,
                                                                 textureFlags, TextureTypes::Type2D );
-
         retVal->setResolution( colourTexture->getWidth(), colourTexture->getHeight() );
         retVal->setPixelFormat( depthBufferFormat );
+        retVal->_setDepthBufferDefaults( poolId, preferDepthTexture, depthBufferFormat );
+        retVal->_setSourceType( TextureSourceType::SharedDepthBuffer );
         retVal->setSampleDescription( colourTexture->getRequestedSampleDescription() );
 
         retVal->_transitionTo( GpuResidency::Resident, (uint8*)0 );
 
+        // Start reference count on the depth buffer here
+        mSharedDepthBufferRefs[retVal] = 1u;
         return retVal;
     }
     //---------------------------------------------------------------------
@@ -544,7 +628,7 @@ namespace Ogre {
         if( poolId == DepthBuffer::POOL_NON_SHAREABLE )
         {
             TextureGpu *retVal = createDepthBufferFor( colourTexture, preferDepthTexture,
-                                                       depthBufferFormat );
+                                                       depthBufferFormat, poolId );
             return retVal;
         }
 
@@ -562,6 +646,7 @@ namespace Ogre {
                 (*itor)->supportsAsDepthBufferFor( colourTexture ) )
             {
                 retVal = *itor;
+                referenceSharedDepthBuffer( retVal );
             }
             else
             {
@@ -573,7 +658,8 @@ namespace Ogre {
         //Not found yet? Create a new one!
         if( !retVal )
         {
-            retVal = createDepthBufferFor( colourTexture, preferDepthTexture, depthBufferFormat );
+            retVal =
+                createDepthBufferFor( colourTexture, preferDepthTexture, depthBufferFormat, poolId );
             mDepthBufferPool2[poolId].push_back( retVal );
 
             if( !retVal )
@@ -640,6 +726,11 @@ namespace Ogre {
         mHwOcclusionQueries.clear();
 
         destroyAllRenderPassDescriptors();
+        _cleanupDepthBuffers();
+        OGRE_ASSERT_LOW( mSharedDepthBufferRefs.empty() &&
+                         "destroyAllRenderPassDescriptors followed by _cleanupDepthBuffers should've "
+                         "emptied mSharedDepthBufferRefs. Please report this bug to "
+                         "https://github.com/OGRECave/ogre-next/issues/" );
 
         OGRE_DELETE mTextureGpuManager;
         mTextureGpuManager = 0;

OgreMain/src/OgreRoot.cpp

@@ -1545,8 +1545,14 @@ namespace Ogre {
         // This belongs here, as all render targets must be updated before events are
         // triggered, otherwise targets could be mismatched.  This could produce artifacts,
         // for instance, with shadows.
-        for (SceneManagerEnumerator::SceneManagerIterator it = getSceneManagerIterator(); it.hasMoreElements(); it.moveNext())
+        for( SceneManagerEnumerator::SceneManagerIterator it = getSceneManagerIterator();
+             it.hasMoreElements(); it.moveNext() )
+        {
             it.peekNextValue()->_handleLodEvents();
+        }
+
+        // Release all the depth buffers which are no longer in use
+        mActiveRenderer->_cleanupDepthBuffers();
 
         return ret;
     }

OgreMain/src/OgreTextureGpu.cpp

@@ -58,6 +58,7 @@ namespace Ogre
         mDepthOrSlices( 0 ),
         mNumMipmaps( 1 ),
         mInternalSliceStart( 0 ),
+        mSourceType( TextureSourceType::Standard ),
         mTextureType( initialType ),
         mPixelFormat( PFG_UNKNOWN ),
         mTextureFlags( textureFlags ),
@@ -261,6 +262,10 @@ namespace Ogre
         return mInternalSliceStart;
     }
     //-----------------------------------------------------------------------------------
+    void TextureGpu::_setSourceType( uint8 type ) { mSourceType = type; }
+    //-----------------------------------------------------------------------------------
+    uint8 TextureGpu::getSourceType( void ) const { return mSourceType; }
+    //-----------------------------------------------------------------------------------
     void TextureGpu::setSampleDescription( SampleDescription desc )
     {
         assert( mResidencyStatus == GpuResidency::OnStorage );

OgreMain/src/OgreTextureGpuManager.cpp

@@ -279,6 +279,7 @@ namespace Ogre
         newPool.usedMemory = 0;
         newPool.usedSlots.reserve( numSlices );
 
+        newPool.masterTexture->_setSourceType( TextureSourceType::PoolOwner );
         newPool.masterTexture->setResolution( width, height, numSlices );
         newPool.masterTexture->setPixelFormat( pixelFormat );
         newPool.masterTexture->setNumMipmaps( numMipmaps );
@@ -347,6 +348,7 @@ namespace Ogre
 
         TextureGpu *retVal = createTextureImpl( pageOutStrategy, idName, textureFlags, initialType );
         retVal->setTexturePoolId( poolId );
+        retVal->_setSourceType( TextureSourceType::Standard );
 
         mEntriesMutex.lock();
         mEntries[idName] = ResourceEntry( name, aliasName, resourceGroup, retVal, filters );
@@ -1884,6 +1886,7 @@ namespace Ogre
                                                        TextureFlags::PoolOwner,
                                                        TextureTypes::Type2DArray );
             const uint16 numSlices = mTextureGpuManagerListener->getNumSlicesFor( texture, this );
+            newPool.masterTexture->_setSourceType( TextureSourceType::PoolOwner );
 
             newPool.manuallyReserved = false;
             newPool.usedMemory = 0;

RenderSystems/Direct3D11/include/OgreD3D11RenderSystem.h

@@ -215,7 +215,7 @@ namespace Ogre
         virtual void endRenderPassDescriptor(void);
 
         TextureGpu* createDepthBufferFor( TextureGpu *colourTexture, bool preferDepthTexture,
-                                          PixelFormatGpu depthBufferFormat );
+                                          PixelFormatGpu depthBufferFormat, uint16 poolId );
 
         const String& getName(void) const;
 

RenderSystems/Direct3D11/src/OgreD3D11RenderSystem.cpp

@@ -1581,7 +1581,7 @@ namespace Ogre
     }
     //-----------------------------------------------------------------------------------
     TextureGpu* D3D11RenderSystem::createDepthBufferFor( TextureGpu *colourTexture, bool preferDepthTexture,
-                                                         PixelFormatGpu depthBufferFormat )
+                                                         PixelFormatGpu depthBufferFormat, uint16 poolId )
     {
         if( depthBufferFormat == PFG_UNKNOWN )
         {
@@ -1592,7 +1592,7 @@ namespace Ogre
         }
 
         return RenderSystem::createDepthBufferFor( colourTexture, preferDepthTexture,
-                                                   depthBufferFormat );
+                                                   depthBufferFormat, poolId );
     }
     //---------------------------------------------------------------------
     void D3D11RenderSystem::_notifyWindowDestroyed( Window *window )

RenderSystems/Direct3D11/src/OgreD3D11TextureGpu.cpp

@@ -780,6 +780,8 @@ namespace Ogre
             uint16 depthBufferPoolId, bool preferDepthTexture, PixelFormatGpu desiredDepthBufferFormat )
     {
         assert( isRenderToTexture() );
+        OGRE_ASSERT_MEDIUM( mSourceType != TextureSourceType::SharedDepthBuffer &&
+                            "Cannot call _setDepthBufferDefaults on a shared depth buffer!" );
         mDepthBufferPoolId          = depthBufferPoolId;
         mPreferDepthTexture         = preferDepthTexture;
         mDesiredDepthBufferFormat   = desiredDepthBufferFormat;

RenderSystems/GL3Plus/include/OgreGL3PlusRenderSystem.h

@@ -261,7 +261,7 @@ namespace Ogre {
         virtual void endRenderPassDescriptor(void);
 
         TextureGpu* createDepthBufferFor( TextureGpu *colourTexture, bool preferDepthTexture,
-                                          PixelFormatGpu depthBufferFormat );
+                                          PixelFormatGpu depthBufferFormat, uint16 poolId );
 
         /** See
             RenderSystem