Merge pull request #2139 from tgstation/HealthCheckFix [TGSDeploy]

v6.14.1: Fix Docker health checks

.github/workflows/ci-pipeline.yml

@@ -2322,9 +2322,9 @@ jobs:
 
   update-nix:
     name: Update Nix SHA
-    needs: deploy-tgs
+    needs: changelog-regen
     runs-on: ubuntu-latest
-    if: (!(cancelled() || failure())) && needs.deploy-tgs.result == 'success'
+    if: (!(cancelled() || failure())) && needs.changelog-regen.result == 'success'
     steps:
       - name: Install Native Packages # Name checked in rerunFlakyTests.js
         run: |

build/Dockerfile

@@ -79,6 +79,6 @@ COPY --from=build /repo/build/tgs.docker.sh tgs.sh
 
 VOLUME ["/config_data", "/tgs_logs", "/app/lib"]
 
-HEALTHCHECK CMD --curl --fail http://localhost:5000/health || exit
+HEALTHCHECK --start-interval=60s CMD curl --fail http://localhost:5000/health || exit
 
 ENTRYPOINT ["./tgs.sh"]

build/Version.props

@@ -3,7 +3,7 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="WebpanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>6.14.0</TgsCoreVersion>
+    <TgsCoreVersion>6.14.1</TgsCoreVersion>
     <TgsConfigVersion>5.5.0</TgsConfigVersion>
     <TgsRestVersion>10.12.1</TgsRestVersion>
     <TgsGraphQLVersion>0.5.0</TgsGraphQLVersion>

src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs

@@ -30,6 +30,11 @@ sealed class GitHubClientFactory : IGitHubClientFactory, IDisposable
 		/// <remarks>Set to app installation token lifetime, which is the lowest. See https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/generating-an-installation-access-token-for-a-github-app.</remarks>
 		const uint ClientCacheHours = 1;
 
+		/// <summary>
+		/// Minutes before tokens expire before not using them.
+		/// </summary>
+		const uint AppTokenExpiryGraceMinutes = 15;
+
 		/// <summary>
 		/// The <see cref="clientCache"/> <see cref="KeyValuePair{TKey, TValue}.Key"/> used in place of <see langword="null"/> when accessing a configuration-based client with no token set in <see cref="GeneralConfiguration.GitHubAccessToken"/>.
 		/// </summary>
@@ -56,9 +61,9 @@ sealed class GitHubClientFactory : IGitHubClientFactory, IDisposable
 		readonly GeneralConfiguration generalConfiguration;
 
 		/// <summary>
-		/// Cache of created <see cref="GitHubClient"/>s and last used times, keyed by access token.
+		/// Cache of created <see cref="GitHubClient"/>s and last used/expiry times, keyed by access token.
 		/// </summary>
-		readonly Dictionary<string, (GitHubClient Client, DateTimeOffset LastUsed)> clientCache;
+		readonly Dictionary<string, (GitHubClient Client, DateTimeOffset LastUsed, DateTimeOffset? Expiry)> clientCache;
 
 		/// <summary>
 		/// The <see cref="SemaphoreSlim"/> used to guard access to <see cref="clientCache"/>.
@@ -83,7 +88,7 @@ public GitHubClientFactory(
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
 
-			clientCache = new Dictionary<string, (GitHubClient, DateTimeOffset)>();
+			clientCache = new Dictionary<string, (GitHubClient, DateTimeOffset, DateTimeOffset?)>();
 			clientCacheSemaphore = new SemaphoreSlim(1, 1);
 		}
 
@@ -137,13 +142,20 @@ public async ValueTask<IGitHubClient> CreateClient(string accessToken, Cancellat
 				else
 					cacheKey = accessString;
 
-				cacheHit = clientCache.TryGetValue(cacheKey, out var tuple);
-
 				var now = DateTimeOffset.UtcNow;
-				if (!cacheHit)
+				cacheHit = clientCache.TryGetValue(cacheKey, out var tuple);
+				var tokenValid = cacheHit && (!tuple.Expiry.HasValue || tuple.Expiry.Value <= now);
+				if (!tokenValid)
 				{
+					if (cacheHit)
+					{
+						logger.LogDebug("Previously cached GitHub token has expired!");
+						clientCache.Remove(cacheKey);
+					}
+
 					logger.LogTrace("Creating new GitHubClient...");
 
+					DateTimeOffset? expiry = null;
 					if (accessString != null)
 					{
 						if (accessString.StartsWith(RepositorySettings.TgsAppPrivateKeyPrefix))
@@ -177,6 +189,7 @@ public async ValueTask<IGitHubClient> CreateClient(string accessToken, Cancellat
 								var installToken = await client.GitHubApps.CreateInstallationToken(installation.Id);
 
 								client.Credentials = new Credentials(installToken.Token);
+								expiry = installToken.ExpiresAt.AddMinutes(-AppTokenExpiryGraceMinutes);
 							}
 							catch (Exception ex)
 							{
@@ -193,7 +206,7 @@ public async ValueTask<IGitHubClient> CreateClient(string accessToken, Cancellat
 					else
 						client = CreateUnauthenticatedClient();
 
-					clientCache.Add(cacheKey, (Client: client, LastUsed: now));
+					clientCache.Add(cacheKey, (Client: client, LastUsed: now, Expiry: expiry));
 					lastUsed = null;
 				}
 				else
@@ -213,7 +226,7 @@ public async ValueTask<IGitHubClient> CreateClient(string accessToken, Cancellat
 						continue; // save the hash lookup
 
 					tuple = clientCache[key];
-					if (tuple.LastUsed <= purgeAfter)
+					if (tuple.LastUsed <= purgeAfter || (tuple.Expiry.HasValue && tuple.Expiry.Value <= now))
 					{
 						clientCache.Remove(key);
 						++purgeCount;