Merge pull request #2137 from tgstation/dev [TGSDeploy]

v6.14.0

README.md

@@ -256,7 +256,9 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will
 
 - `General:ConfigVersion`: Suppresses warnings about out of date config versions. You should change this after updating TGS to one with a new config version. The current version can be found on the releases page for your server version.
 
-- `General:MinimumPasswordLength`: Minimum password length requirement for database users
+- `General:MinimumPasswordLength`: Minimum password length requirement for database users.
+
+- `General:PrometheusPort`: Port Prometheus metrics are published on under /metrics. This can be set to the same value as the `ApiPort`, just note that accessing it does not require authentication.
 
 - `General:ValidInstancePaths`: Array meant to limit the directories in which instances may be created.
 

build/Dockerfile

@@ -58,11 +58,12 @@ RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \
 
 FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim
 
-#needed for byond
+#needed for byond, curl for healthchecks
 RUN apt-get update \
 	&& apt-get install -y \
 		gcc-multilib \
 		gdb \
+		curl \
 	&& rm -rf /var/lib/apt/lists/*
 
 EXPOSE 5000
@@ -78,4 +79,6 @@ COPY --from=build /repo/build/tgs.docker.sh tgs.sh
 
 VOLUME ["/config_data", "/tgs_logs", "/app/lib"]
 
+HEALTHCHECK CMD --curl --fail http://localhost:5000/health || exit
+
 ENTRYPOINT ["./tgs.sh"]

build/TestCommon.props

@@ -13,14 +13,14 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <!-- Usage: Hard to say what exactly this is for, but not including it removes the test icon and breaks vstest.console.exe for some reason -->
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
     <!-- Usage: Dependency mocking for tests -->
     <!-- Pinned: Be VERY careful about updating https://github.com/moq/moq/issues/1372 -->
     <PackageReference Include="Moq" Version="4.20.72" />
     <!-- Usage: MSTest execution -->
-    <PackageReference Include="MSTest.TestAdapter" Version="3.7.3" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.8.0" />
     <!-- Usage: MSTest asserts etc... -->
-    <PackageReference Include="MSTest.TestFramework" Version="3.7.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.8.0" />
   </ItemGroup>
 
 </Project>

build/Version.props

@@ -3,8 +3,8 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="WebpanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>6.13.0</TgsCoreVersion>
-    <TgsConfigVersion>5.4.0</TgsConfigVersion>
+    <TgsCoreVersion>6.14.0</TgsCoreVersion>
+    <TgsConfigVersion>5.5.0</TgsConfigVersion>
     <TgsRestVersion>10.12.1</TgsRestVersion>
     <TgsGraphQLVersion>0.5.0</TgsGraphQLVersion>
     <TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>

src/Tgstation.Server.Api/Tgstation.Server.Api.csproj

@@ -28,10 +28,9 @@
     <!-- Usage: HTTP constants reference -->
     <PackageReference Include="Microsoft.AspNetCore.Http.Extensions" Version="2.3.0" />
     <!-- Usage: Decoding the 'nbf' property of JWTs -->
-    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.4.0" />
+    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.5.0" />
     <!-- Usage: Data model annotating -->
     <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
-    <PackageReference Include="System.Text.Encodings.Web" Version="9.0.1" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj

@@ -9,7 +9,7 @@
 
   <ItemGroup>
     <!-- GraphQL connector and code generator -->
-    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.1" />
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.2" />
     <PackageReference Include="StrawberryShake.Server" Version="15.0.3" />
   </ItemGroup>
 

src/Tgstation.Server.Client/Tgstation.Server.Client.csproj

@@ -11,9 +11,9 @@
 
   <ItemGroup>
     <!-- Usage: Connecting to SignalR hubs in API -->
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="9.0.1" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="9.0.2" />
     <!-- Usage: Using target JSON serializer for API -->
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.0" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.2" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj

@@ -13,9 +13,9 @@
 
   <ItemGroup>
     <!-- Usage: Identifying if we're running under SystemD -->
-    <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="9.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="9.0.2" />
     <!-- Usage: Console logging plugin -->
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.2" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj

@@ -21,21 +21,21 @@
     <!-- Usage: Command line argument support -->
     <PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="4.1.1" />
     <!-- Usage: Identifies when we are running in the context of the Windows SCM -->
-    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="9.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="9.0.2" />
     <!-- Usage: Windows event log logging plugin -->
-    <PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="9.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="9.0.2" />
     <!-- Usage: Console logging plugin -->
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.2" />
     <!-- Usage: Updated transitive dependency of Core.System.ServiceProcess -->
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <!-- Usage: Updated transitive dependency of Core.System.ServiceProcess -->
-    <PackageReference Include="System.Drawing.Common" Version="9.0.1" />
+    <PackageReference Include="System.Drawing.Common" Version="9.0.2" />
     <!-- Usage: Updated transitive dependency, unable to tell what of -->
     <PackageReference Include="System.Private.Uri" Version="4.3.2" />
     <!-- Usage: OS identification -->
     <PackageReference Include="System.Runtime.InteropServices" Version="4.3.0" />
     <!-- Usage: Windows Service Manager intergration -->
-    <PackageReference Include="System.ServiceProcess.ServiceController" Version="9.0.1" />
+    <PackageReference Include="System.ServiceProcess.ServiceController" Version="9.0.2" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj

@@ -10,7 +10,7 @@
 
   <ItemGroup>
     <!-- Usage: Logging abstractions -->
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.2" />
     <!-- Usage: POSIX support for signals -->
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
   </ItemGroup>

src/Tgstation.Server.Host/.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "dotnet-ef": {
-      "version": "8.0.11",
+      "version": "8.0.13",
       "commands": [
         "dotnet-ef"
       ]

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -8,6 +8,8 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Common.Extensions;
@@ -105,6 +107,21 @@ sealed class DreamMaker : IDreamMaker
 		/// </summary>
 		readonly Api.Models.Instance metadata;
 
+		/// <summary>
+		/// The number of attempted deployments.
+		/// </summary>
+		readonly Counter attemptedDeployments;
+
+		/// <summary>
+		/// The number of successful deployments.
+		/// </summary>
+		readonly Counter successfulDeployments;
+
+		/// <summary>
+		/// The number of failed deployments.
+		/// </summary>
+		readonly Counter failedDeployments;
+
 		/// <summary>
 		/// <see langword="lock"/> <see cref="object"/> for <see cref="deploying"/>.
 		/// </summary>
@@ -149,6 +166,7 @@ static string FormatExceptionForUsers(Exception exception)
 		/// <param name="repositoryManager">The value of <see cref="repositoryManager"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
 		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> to use.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		/// <param name="sessionConfiguration">The value of <see cref="sessionConfiguration"/>.</param>
 		/// <param name="metadata">The value of <see cref="metadata"/>.</param>
@@ -164,6 +182,7 @@ public DreamMaker(
 			IRepositoryManager repositoryManager,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
 			IAsyncDelayer asyncDelayer,
+			IMetricFactory metricFactory,
 			ILogger<DreamMaker> logger,
 			SessionConfiguration sessionConfiguration,
 			Api.Models.Instance metadata)
@@ -177,12 +196,17 @@ public DreamMaker(
 			this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor));
 			this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer));
 			this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager));
-			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
+			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
+			ArgumentNullException.ThrowIfNull(metricFactory);
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration));
 			this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
 
+			successfulDeployments = metricFactory.CreateCounter("tgs_successful_deployments", "The number of deployments that have completed successfully");
+			failedDeployments = metricFactory.CreateCounter("tgs_failed_deployments", "The number of deployments that have failed");
+			attemptedDeployments = metricFactory.CreateCounter("tgs_total_deployments", "The number of deployments that have been attempted");
+
 			deploymentLock = new object();
 		}
 
@@ -205,9 +229,12 @@ public async ValueTask DeploymentProcess(
 				deploying = true;
 			}
 
+			attemptedDeployments.Inc();
+
 			currentChatCallback = null;
 			currentDreamMakerOutput = null;
 			Models.CompileJob? compileJob = null;
+			bool success = false;
 			try
 			{
 				string? repoOwner = null;
@@ -351,6 +378,7 @@ await databaseContextFactory.UseContext(
 							{
 								var chatNotificationAction = currentChatCallback!(null, compileJob.Output!);
 								await compileJobConsumer.LoadCompileJob(compileJob, chatNotificationAction, cancellationToken);
+								success = true;
 							}
 							catch
 							{
@@ -406,6 +434,10 @@ await databaseContextFactory.UseContext(
 			finally
 			{
 				deploying = false;
+				if (success)
+					successfulDeployments.Inc();
+				else
+					failedDeployments.Inc();
 			}
 		}
 #pragma warning restore CA1506

src/Tgstation.Server.Host/Components/InstanceFactory.cs

@@ -1,10 +1,14 @@
 using System;
+using System.Collections.Generic;
+using System.Globalization;
 using System.Threading;
 using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
+using Prometheus;
+
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Chat.Commands;
 using Tgstation.Server.Host.Components.Deployment;
@@ -140,6 +144,11 @@ sealed class InstanceFactory : IInstanceFactory
 		/// </summary>
 		readonly IDotnetDumpService dotnetDumpService;
 
+		/// <summary>
+		/// The <see cref="IMetricFactory"/> for the <see cref="InstanceFactory"/>.
+		/// </summary>
+		readonly IMetricFactory metricFactory;
+
 		/// <summary>
 		/// The <see cref="GeneralConfiguration"/> for the <see cref="InstanceFactory"/>.
 		/// </summary>
@@ -183,6 +192,7 @@ sealed class InstanceFactory : IInstanceFactory
 		/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
 		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
 		/// <param name="dotnetDumpService">The value of <see cref="dotnetDumpService"/>.</param>
+		/// <param name="metricFactory">The value of <see cref="metricFactory"/>.</param>
 		/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
 		/// <param name="sessionConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="sessionConfiguration"/>.</param>
 		public InstanceFactory(
@@ -208,6 +218,7 @@ public InstanceFactory(
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
 			IAsyncDelayer asyncDelayer,
 			IDotnetDumpService dotnetDumpService,
+			IMetricFactory metricFactory,
 			IOptions<GeneralConfiguration> generalConfigurationOptions,
 			IOptions<SessionConfiguration> sessionConfigurationOptions)
 		{
@@ -233,6 +244,7 @@ public InstanceFactory(
 			this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
 			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			this.dotnetDumpService = dotnetDumpService ?? throw new ArgumentNullException(nameof(dotnetDumpService));
+			this.metricFactory = metricFactory ?? throw new ArgumentNullException(nameof(metricFactory));
 			generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
 			sessionConfiguration = sessionConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(sessionConfigurationOptions));
 		}
@@ -264,6 +276,13 @@ public async ValueTask<IInstance> CreateInstance(IBridgeRegistrar bridgeRegistra
 			var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics");
 			var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration");
 
+			var metricFactory = this.metricFactory.WithLabels(
+				new Dictionary<string, string>
+				{
+					{ "instance_name", metadata.Name! },
+					{ "instance_id", metadata.Id!.Value.ToString(CultureInfo.InvariantCulture) },
+				});
+
 			var configuration = new StaticFiles.Configuration(
 				configurationIoManager,
 				synchronousIOManager,
@@ -323,6 +342,7 @@ public async ValueTask<IInstance> CreateInstance(IBridgeRegistrar bridgeRegistra
 							eventConsumer,
 							asyncDelayer,
 							dotnetDumpService,
+							metricFactory,
 							loggerFactory,
 							loggerFactory.CreateLogger<SessionControllerFactory>(),
 							sessionConfiguration,
@@ -337,6 +357,7 @@ public async ValueTask<IInstance> CreateInstance(IBridgeRegistrar bridgeRegistra
 							diagnosticsIOManager,
 							configuration, // watchdog doesn't need itself as an event consumer
 							remoteDeploymentManagerFactory,
+							metricFactory,
 							metadata,
 							metadata.DreamDaemonSettings!);
 						try
@@ -357,6 +378,7 @@ public async ValueTask<IInstance> CreateInstance(IBridgeRegistrar bridgeRegistra
 								repoManager,
 								remoteDeploymentManagerFactory,
 								asyncDelayer,
+								metricFactory,
 								loggerFactory.CreateLogger<DreamMaker>(),
 								sessionConfiguration,
 								metadata);