Skip to content

Commit

Permalink
[Invoker] Check the file system disk space usage to avoid that invoke…
Browse files Browse the repository at this point in the history
…rs run out of space (cont.)
  • Loading branch information
steffenrost committed Sep 11, 2023
1 parent 5a18e1c commit f8b120d
Showing 1 changed file with 6 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,8 @@ class InvokerReactive(

private val rootfs = "/"
private val logsfs = "/logs"
private val fspcentmax = 85
private val rootfspcentmax = sys.env.getOrElse("ROOT_FS_MAX", "98").toInt
private val logfspcentmax = sys.env.getOrElse("LOG_FS_MAX", "98").toInt
private var rootfspcent = 0
private var logsfspcent = 0

Expand All @@ -249,8 +250,8 @@ class InvokerReactive(
logging.warn(
this,
s"invoker fs space: " +
s"'$rootfsraw ($rootfspcentraw($rootfspcent($fspcentmax)))', " +
s"'$logsfsraw ($logsfspcentraw($logsfspcent($fspcentmax)))', " +
s"'$rootfsraw ($rootfspcentraw($rootfspcent($rootfspcentmax)))', " +
s"'$logsfsraw ($logsfspcentraw($logsfspcent($logfspcentmax)))', " +
s"invoker container pool: " +
s"freePoolSize: ${poolState.free} containers, " +
s"busyPoolSize: ${poolState.busy} containers, " +
Expand Down Expand Up @@ -503,6 +504,7 @@ class InvokerReactive(
}

private val healthProducer = msgProvider.getProducer(config)
rootfspcentmax
Scheduler.scheduleWaitAtMost(1.seconds)(() => {
// ping only if monitor is ready when enabled
if (!imageMonitorEnabled || imageMonitor.isReady) {
Expand All @@ -512,7 +514,7 @@ class InvokerReactive(
PingMessage(
instance = instance,
isBlacklisted = namespaceBlacklist.isBlacklisted(instance.displayedName.getOrElse("")),
hasDiskPressure = rootfspcent >= fspcentmax || logsfspcent >= fspcentmax,
hasDiskPressure = rootfspcent >= rootfspcentmax || logsfspcent >= logfspcentmax,
rootfspcent = rootfspcent,
logsfspcent = logsfspcent,
running = poolState.busy + poolState.waiting))
Expand Down

0 comments on commit f8b120d

Please sign in to comment.