Skip to content

Commit

Permalink
avoid ANRs and frame freezes due long pipelines compilations - set de…
Browse files Browse the repository at this point in the history 
…adline and then defer compilation requests to next frame
  • Loading branch information
@eugenegff
Eugene Golushkov authored and eugenegff committed Jan 29, 2025
1 parent 2e3a146 commit e5216ef
Show file tree
Hide file tree
Showing 25 changed files with 192 additions and 66 deletions.
3 changes: 2 additions & 1 deletion Components/Hlms/Pbs/include/OgreHlmsPbs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,8 @@ namespace Ogre
const HlmsCache *createShaderCacheEntry( uint32 renderableHash, const HlmsCache &passCache,
uint32 finalHash,
const QueuedRenderable &queuedRenderable,
HlmsCache *reservedStubEntry, size_t tid ) override;
HlmsCache *reservedStubEntry, uint64 deadline,
size_t tid ) override;

HlmsDatablock *createDatablockImpl( IdString datablockName, const HlmsMacroblock *macroblock,
const HlmsBlendblock *blendblock,
Expand Down
5 changes: 3 additions & 2 deletions Components/Hlms/Pbs/src/OgreHlmsPbs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -590,12 +590,13 @@ namespace Ogre
const HlmsCache *HlmsPbs::createShaderCacheEntry( uint32 renderableHash, const HlmsCache &passCache,
uint32 finalHash,
const QueuedRenderable &queuedRenderable,
HlmsCache *reservedStubEntry, const size_t tid )
HlmsCache *reservedStubEntry, uint64 deadline,
const size_t tid )
{
OgreProfileExhaustive( "HlmsPbs::createShaderCacheEntry" );

const HlmsCache *retVal = Hlms::createShaderCacheEntry(
renderableHash, passCache, finalHash, queuedRenderable, reservedStubEntry, tid );
renderableHash, passCache, finalHash, queuedRenderable, reservedStubEntry, deadline, tid );

if( mShaderProfile != "glsl" )
{
Expand Down
3 changes: 2 additions & 1 deletion Components/Hlms/Unlit/include/OgreHlmsUnlit.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ namespace Ogre
const HlmsCache *createShaderCacheEntry( uint32 renderableHash, const HlmsCache &passCache,
uint32 finalHash,
const QueuedRenderable &queuedRenderable,
HlmsCache *reservedStubEntry, size_t tid ) override;
HlmsCache *reservedStubEntry, uint64 deadline,
size_t tid ) override;

HlmsDatablock *createDatablockImpl( IdString datablockName, const HlmsMacroblock *macroblock,
const HlmsBlendblock *blendblock,
Expand Down
7 changes: 4 additions & 3 deletions Components/Hlms/Unlit/src/OgreHlmsUnlit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,12 +208,13 @@ namespace Ogre
const HlmsCache *HlmsUnlit::createShaderCacheEntry( uint32 renderableHash,
const HlmsCache &passCache, uint32 finalHash,
const QueuedRenderable &queuedRenderable,
HlmsCache *reservedStubEntry, const size_t tid )
HlmsCache *reservedStubEntry, uint64 deadline,
const size_t tid )
{
OgreProfileExhaustive( "HlmsUnlit::createShaderCacheEntry" );

const HlmsCache *retVal = Hlms::createShaderCacheEntry(
renderableHash, passCache, finalHash, queuedRenderable, reservedStubEntry, tid );
renderableHash, passCache, finalHash, queuedRenderable, reservedStubEntry, deadline, tid );

if( mShaderProfile != "glsl" )
{
Expand Down Expand Up @@ -730,7 +731,7 @@ namespace Ogre
const uint32 hash = uint32( it - mPassCache.begin() ) << HlmsBits::PassShift;

// Fill the buffers
HlmsCache retVal( hash, mType, HlmsPso() );
HlmsCache retVal( hash, mType, HLMS_CACHE_FLAGS_NONE, HlmsPso() );
retVal.setProperties = mT[kNoTid].setProperties;
retVal.pso.pass = passCache.passPso;

Expand Down
8 changes: 5 additions & 3 deletions OgreMain/include/OgreHlms.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -513,8 +513,8 @@ namespace Ogre
virtual const HlmsCache *createShaderCacheEntry( uint32 renderableHash,
const HlmsCache &passCache, uint32 finalHash,
const QueuedRenderable &queuedRenderable,
HlmsCache *reservedStubEntry,
size_t threadIdx );
HlmsCache *reservedStubEntry, uint64 deadline,
size_t threadIdx );

enum PropertiesMergeStatus
{
Expand Down Expand Up @@ -936,14 +936,16 @@ namespace Ogre
See lastReturnedValue from getMaterial()
@param reservedStubEntry
The stub cache entry (return value of getMaterial()) to fill.
@param deadline
Deadline in ms, after which long shader compilation could be skipped to avoid frame stutter.
@param queuedRenderable
See getMaterial()
@param renderableHash
@param finalHash
@param tid
Thread idx of caller
*/
void compileStubEntry( const HlmsCache &passCache, HlmsCache *reservedStubEntry,
void compileStubEntry( const HlmsCache &passCache, HlmsCache *reservedStubEntry, uint64 deadline,
QueuedRenderable queuedRenderable, uint32 renderableHash,
uint32 finalHash, size_t tid );

Expand Down
15 changes: 12 additions & 3 deletions OgreMain/include/OgreHlmsCommon.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ namespace Ogre
/** Up to 8 different HLMS generator types are allowed. The maximum values must be in sync
with ShaderBits in RenderQueue.cpp (the higher 3 bits)
*/
enum HlmsTypes
enum HlmsTypes : uchar
{
HLMS_LOW_LEVEL, ///< Proxy that redirects to a regular Material
HLMS_PBS, ///< Physically Based Shader Generator
Expand All @@ -201,18 +201,27 @@ namespace Ogre
HLMS_COMPUTE,
};

enum HlmsCacheFlags : uchar
{
HLMS_CACHE_FLAGS_NONE = 0,
HLMS_CACHE_FLAGS_COMPILATION_REQUIRED = 1,
HLMS_CACHE_FLAGS_COMPILATION_REQUESTED = 2,
};

struct HlmsCache
{
uint32 hash;
HlmsTypes type;
HlmsCacheFlags flags;
HlmsPropertyVec setProperties;

HlmsPso pso;

HlmsCache() : hash( 0 ), type( HLMS_MAX ) {}
HlmsCache( uint32 _hash, HlmsTypes _type, const HlmsPso &_pso ) :
HlmsCache() : hash( 0 ), type( HLMS_MAX ), flags( HLMS_CACHE_FLAGS_NONE ) {}
HlmsCache( uint32 _hash, HlmsTypes _type, HlmsCacheFlags _flags, const HlmsPso &_pso ) :
hash( _hash ),
type( _type ),
flags( _flags ),
pso( _pso )
{
}
Expand Down
3 changes: 2 additions & 1 deletion OgreMain/include/OgreHlmsLowLevel.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@ namespace Ogre
const HlmsCache *createShaderCacheEntry( uint32 renderableHash, const HlmsCache &passCache,
uint32 finalHash,
const QueuedRenderable &queuedRenderable,
HlmsCache *reservedStubEntry, size_t tid ) override;
HlmsCache *reservedStubEntry, uint64 deadline,
size_t tid ) override;

HlmsDatablock *createDatablockImpl( IdString datablockName, const HlmsMacroblock *macroblock,
const HlmsBlendblock *blendblock,
Expand Down
9 changes: 8 additions & 1 deletion OgreMain/include/OgreRenderQueue.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ namespace Ogre
LightweightMutex mMutex;
Semaphore mSemaphore;
std::atomic<bool> mKeepCompiling;
uint64 mCompilationDeadline; // 16ms in the future, or (uint64)-1
std::atomic<uint64> mCompilationIncompleteCounter; // at least one was skipped

bool mExceptionFound; // GUARDED_BY( mMutex )
std::exception_ptr mThreadedException; // GUARDED_BY( mMutex )
Expand All @@ -99,11 +101,16 @@ namespace Ogre
inline void pushRequest( const Request &&request )
{
ScopedLock lock( mMutex );
request.reservedStubEntry->flags = HLMS_CACHE_FLAGS_COMPILATION_REQUESTED;
mRequests.emplace_back( request );
mSemaphore.increment();
}

inline void pushWarmUpRequest( const Request &&request ) { mRequests.emplace_back( request ); }
inline void pushWarmUpRequest( const Request &&request )
{
request.reservedStubEntry->flags = HLMS_CACHE_FLAGS_COMPILATION_REQUESTED;
mRequests.emplace_back( request );
}

/** Starts worker threads (job queue) so they start accepting work every time pushRequest()
gets called and will keep compiling those shaders until stopAndWait() is called.
Expand Down
15 changes: 14 additions & 1 deletion OgreMain/include/OgreRenderSystem.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -905,7 +905,17 @@ namespace Ogre

virtual void executeResourceTransition( const ResourceTransitionArray &rstCollection ) {}

virtual void _hlmsPipelineStateObjectCreated( HlmsPso *newPso ) {}
/// PSO creation on Vulkan could be skipped after exhausting per-frame time budget
void setPsoRequestsTimeout( int ms ) { mPsoRequestsTimeout = ms; }
int getPsoRequestsTimeout() const { return mPsoRequestsTimeout; }
uint64 getIncompletePsoRequestsCounter() const { return mIncompletePsoRequestsCounter; }
void _notifyIncompletePsoRequests( uint64 count ) { mIncompletePsoRequestsCounter += count; }

/// return false for recoverable errors, for example for exhausted per-frame time budget
virtual bool _hlmsPipelineStateObjectCreated( HlmsPso *newPso, uint64 deadline = (uint64)-1 )
{
return true;
}
virtual void _hlmsPipelineStateObjectDestroyed( HlmsPso *pso ) {}
virtual void _hlmsMacroblockCreated( HlmsMacroblock *newBlock ) {}
virtual void _hlmsMacroblockDestroyed( HlmsMacroblock *block ) {}
Expand Down Expand Up @@ -1708,6 +1718,9 @@ namespace Ogre

bool mReverseDepth;
bool mInvertedClipSpaceY;

int mPsoRequestsTimeout; // ms, per frame, or 0 to disable
uint64 mIncompletePsoRequestsCounter;
};
/** @} */
/** @} */
Expand Down
39 changes: 27 additions & 12 deletions OgreMain/src/OgreHlms.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2036,7 +2036,7 @@ namespace Ogre
//-----------------------------------------------------------------------------------
HlmsCache *Hlms::addStubShaderCache( uint32 hash )
{
HlmsCache cache( hash, mType, HlmsPso() );
HlmsCache cache( hash, mType, HLMS_CACHE_FLAGS_COMPILATION_REQUIRED, HlmsPso() );
HlmsCacheVec::iterator it =
std::lower_bound( mShaderCache.begin(), mShaderCache.end(), &cache, OrderCacheByHash );

Expand All @@ -2054,7 +2054,7 @@ namespace Ogre
{
ScopedLock lock( mMutex );

HlmsCache cache( hash, mType, pso );
HlmsCache cache( hash, mType, HLMS_CACHE_FLAGS_NONE, pso );
HlmsCacheVec::iterator it =
std::lower_bound( mShaderCache.begin(), mShaderCache.end(), &cache, OrderCacheByHash );

Expand All @@ -2070,7 +2070,7 @@ namespace Ogre
//-----------------------------------------------------------------------------------
const HlmsCache *Hlms::getShaderCache( uint32 hash ) const
{
HlmsCache cache( hash, mType, HlmsPso() );
HlmsCache cache( hash, mType, HLMS_CACHE_FLAGS_NONE, HlmsPso() );
HlmsCacheVec::const_iterator it =
std::lower_bound( mShaderCache.begin(), mShaderCache.end(), &cache, OrderCacheByHash );

Expand Down Expand Up @@ -2510,7 +2510,8 @@ namespace Ogre
const HlmsCache *Hlms::createShaderCacheEntry( uint32 renderableHash, const HlmsCache &passCache,
uint32 finalHash,
const QueuedRenderable &queuedRenderable,
HlmsCache *reservedStubEntry, const size_t tid )
HlmsCache *reservedStubEntry, uint64 deadline,
const size_t tid )
{
OgreProfileExhaustive( "Hlms::createShaderCacheEntry" );

Expand Down Expand Up @@ -2659,7 +2660,7 @@ namespace Ogre
LogManager::getSingleton().logMessage(
"Compiling new PSO for datablock: " + datablock->getName().getFriendlyText(), LML_TRIVIAL );
#endif
mRenderSystem->_hlmsPipelineStateObjectCreated( &pso );
bool rsPsoCreated = mRenderSystem->_hlmsPipelineStateObjectCreated( &pso, deadline );

if( reservedStubEntry )
{
Expand All @@ -2668,7 +2669,12 @@ namespace Ogre
!reservedStubEntry->pso.tesselationHullShader &&
!reservedStubEntry->pso.tesselationDomainShader &&
!reservedStubEntry->pso.pixelShader && "Race condition?" );
reservedStubEntry->pso = pso;
if( rsPsoCreated )
reservedStubEntry->pso = pso;
reservedStubEntry->flags =
!rsPsoCreated && reservedStubEntry->flags == HLMS_CACHE_FLAGS_COMPILATION_REQUESTED
? HLMS_CACHE_FLAGS_COMPILATION_REQUIRED // recoverable error (timeout?), reschedule
: HLMS_CACHE_FLAGS_NONE;
}

const HlmsCache *retVal =
Expand Down Expand Up @@ -3558,7 +3564,7 @@ namespace Ogre

const uint32 hash = static_cast<uint32>( it - mPassCache.begin() ) << HlmsBits::PassShift;

HlmsCache retVal( hash, mType, HlmsPso() );
HlmsCache retVal( hash, mType, HLMS_CACHE_FLAGS_NONE, HlmsPso() );
retVal.setProperties = mT[kNoTid].setProperties;
retVal.pso.pass = passCache.passPso;

Expand Down Expand Up @@ -3712,8 +3718,8 @@ namespace Ogre
// Low level is a special case because it doesn't (yet?) support parallel compilation
if( !parallelQueue || mType == HLMS_LOW_LEVEL )
{
lastReturnedValue = createShaderCacheEntry( hash[0], passCache, finalHash,
queuedRenderable, nullptr, kNoTid );
lastReturnedValue = createShaderCacheEntry(
hash[0], passCache, finalHash, queuedRenderable, nullptr, (uint64)-1, kNoTid );
}
else
{
Expand All @@ -3725,17 +3731,26 @@ namespace Ogre
{ &passCache, stubEntry, queuedRenderable, hash[0], finalHash } );
}
}
else if( lastReturnedValue->flags == HLMS_CACHE_FLAGS_COMPILATION_REQUIRED )
{
// Stub entry was created for previous frame, but compilation was skipped
// due to exhausted time budget, and attempt should be repeated
HlmsCache *stubEntry = const_cast<HlmsCache *>( lastReturnedValue );

parallelQueue->pushRequest(
{ &passCache, stubEntry, queuedRenderable, hash[0], finalHash } );
}
}

return lastReturnedValue;
}
//-----------------------------------------------------------------------------------
void Hlms::compileStubEntry( const HlmsCache &passCache, HlmsCache *reservedStubEntry,
QueuedRenderable queuedRenderable, uint32 renderableHash,
uint32 finalHash, size_t tid )
uint64 deadline, QueuedRenderable queuedRenderable,
uint32 renderableHash, uint32 finalHash, size_t tid )
{
createShaderCacheEntry( renderableHash, passCache, finalHash, queuedRenderable,
reservedStubEntry, tid );
reservedStubEntry, deadline, tid );
}
//-----------------------------------------------------------------------------------
uint32 Hlms::getMaterialSerial01( uint32 lastReturnedValue, const HlmsCache &passCache,
Expand Down
4 changes: 2 additions & 2 deletions OgreMain/src/OgreHlmsLowLevel.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ namespace Ogre
const HlmsCache *HlmsLowLevel::createShaderCacheEntry( uint32 renderableHash,
const HlmsCache &passCache, uint32 finalHash,
const QueuedRenderable &queuedRenderable,
HlmsCache *reservedStubEntry,
HlmsCache *reservedStubEntry, uint64 deadline,
const size_t tid )
{
Renderable *renderable = queuedRenderable.renderable;
Expand Down Expand Up @@ -130,7 +130,7 @@ namespace Ogre

applyStrongMacroblockRules( pso );

mRenderSystem->_hlmsPipelineStateObjectCreated( &pso );
mRenderSystem->_hlmsPipelineStateObjectCreated( &pso, (uint64)-1 );

if( reservedStubEntry )
{
Expand Down
2 changes: 1 addition & 1 deletion OgreMain/src/OgrePsoCacheHelper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ namespace Ogre
mPsoCache.insert( itor, 1u, entry );
itor = mPsoCache.begin() + idx;

mRenderSystem->_hlmsPipelineStateObjectCreated( &itor->pso );
mRenderSystem->_hlmsPipelineStateObjectCreated( &itor->pso, (uint64)-1 );
}

mLastFinalHash = finalHash;
Expand Down
Loading

0 comments on commit e5216ef

Please sign in to comment.