Skip to content

Commit

Permalink
Provide a way to override operand image, env, and resources in CRD
Browse files Browse the repository at this point in the history 
Signed-off-by: Michael Edgar <[email protected]>
  • Loading branch information
@MikeEdgar
MikeEdgar committed Jan 9, 2025
1 parent 49e2ad1 commit ed01509
Show file tree
Hide file tree
Showing 6 changed files with 259 additions and 26 deletions.
24 changes: 24 additions & 0 deletions operator/src/main/java/com/github/streamshub/console/api/v1alpha1/spec/ConsoleSpec.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import java.util.List;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import com.github.streamshub.console.api.v1alpha1.spec.containers.Containers;
import com.github.streamshub.console.api.v1alpha1.spec.metrics.MetricsSource;
import com.github.streamshub.console.api.v1alpha1.spec.security.GlobalSecurity;

Expand All @@ -27,6 +29,16 @@ public class ConsoleSpec {
@Required
String hostname;

@JsonPropertyDescription("""
Templates for Console instance containers. The templates allow \
users to specify how the Kubernetes resources are generated.
""")
Containers containers;

@JsonPropertyDescription("""
DEPRECATED: Image overrides to be used for the API and UI servers. \
Use `containers` property instead.
""")
Images images;

GlobalSecurity security;
Expand All @@ -37,6 +49,10 @@ public class ConsoleSpec {

List<KafkaCluster> kafkaClusters = new ArrayList<>();

@JsonPropertyDescription("""
DEPRECATED: Environment variables which should be applied to the API container. \
Use `containers` property instead.
""")
List<EnvVar> env;

public String getHostname() {
Expand All @@ -47,6 +63,14 @@ public void setHostname(String hostname) {
this.hostname = hostname;
}

public Containers getContainers() {
return containers;
}

public void setContainers(Containers containers) {
this.containers = containers;
}

public Images getImages() {
return images;
}
Expand Down
49 changes: 49 additions & 0 deletions ...in/java/com/github/streamshub/console/api/v1alpha1/spec/containers/ContainerTemplate.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package com.github.streamshub.console.api.v1alpha1.spec.containers;

import java.util.List;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;

import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.ResourceRequirements;
import io.sundr.builder.annotations.Buildable;

@Buildable
@JsonInclude(JsonInclude.Include.NON_NULL)
public class ContainerTemplate {

@JsonPropertyDescription("Container image to be used for the container")
private String image;

@JsonPropertyDescription("CPU and memory resources to reserve.")
private ResourceRequirements resources;

@JsonPropertyDescription("Environment variables which should be applied to the container.")
private List<EnvVar> env;

public String getImage() {
return image;
}

public void setImage(String image) {
this.image = image;
}

public ResourceRequirements getResources() {
return resources;
}

public void setResources(ResourceRequirements resources) {
this.resources = resources;
}

public List<EnvVar> getEnv() {
return env;
}

public void setEnv(List<EnvVar> env) {
this.env = env;
}

}
35 changes: 35 additions & 0 deletions .../src/main/java/com/github/streamshub/console/api/v1alpha1/spec/containers/Containers.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package com.github.streamshub.console.api.v1alpha1.spec.containers;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;

import io.sundr.builder.annotations.Buildable;

@Buildable
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Containers {

@JsonPropertyDescription("Template for the Console API server container. " +
"The template allows users to specify how the Kubernetes resources are generated.")
ContainerTemplate api;

@JsonPropertyDescription("Template for the Console UI server container. " +
"The template allows users to specify how the Kubernetes resources are generated.")
ContainerTemplate ui;

public ContainerTemplate getApi() {
return api;
}

public void setApi(ContainerTemplate api) {
this.api = api;
}

public ContainerTemplate getUi() {
return ui;
}

public void setUi(ContainerTemplate ui) {
this.ui = ui;
}
}
30 changes: 24 additions & 6 deletions operator/src/main/java/com/github/streamshub/console/dependents/ConsoleDeployment.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package com.github.streamshub.console.dependents;

import java.util.ArrayList;
import java.util.Collections;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand All @@ -13,6 +13,8 @@

import com.github.streamshub.console.api.v1alpha1.Console;
import com.github.streamshub.console.api.v1alpha1.spec.Images;
import com.github.streamshub.console.api.v1alpha1.spec.containers.ContainerTemplate;
import com.github.streamshub.console.api.v1alpha1.spec.containers.Containers;
import com.github.streamshub.console.dependents.discriminators.ConsoleLabelDiscriminator;

import io.fabric8.kubernetes.api.model.EnvVar;
Expand Down Expand Up @@ -61,17 +63,29 @@ protected Deployment desired(Console primary, Context<Console> context) {
String name = instanceName(primary);
String configSecretName = secret.instanceName(primary);

var imagesSpec = Optional.ofNullable(primary.getSpec().getImages());
String imageAPI = imagesSpec.map(Images::getApi).orElse(defaultAPIImage);
String imageUI = imagesSpec.map(Images::getUi).orElse(defaultUIImage);
var containers = Optional.ofNullable(primary.getSpec().getContainers());
var templateAPI = containers.map(Containers::getApi);
var templateUI = containers.map(Containers::getUi);
// deprecated
var images = Optional.ofNullable(primary.getSpec().getImages());

var envVars = new ArrayList<>(coalesce(primary.getSpec().getEnv(), Collections::emptyList));
String imageAPI = templateAPI.map(ContainerTemplate::getImage)
.or(() -> images.map(Images::getApi))
.orElse(defaultAPIImage);
String imageUI = templateUI.map(ContainerTemplate::getImage)
.or(() -> images.map(Images::getUi))
.orElse(defaultUIImage);

var trustResources = getTrustResources("TrustStoreResources", context);
List<EnvVar> envVars = new ArrayList<>();
envVars.addAll(coalesce(primary.getSpec().getEnv(), Collections::emptyList));
envVars.addAll(templateAPI.map(ContainerTemplate::getEnv).orElseGet(Collections::emptyList));
envVars.addAll(getResourcesByType(trustResources, EnvVar.class));

var trustResourcesUI = getTrustResources("TrustStoreResourcesUI", context);
var envVarsUI = getResourcesByType(trustResourcesUI, EnvVar.class);
List<EnvVar> envVarsUI = new ArrayList<>();
envVarsUI.addAll(templateUI.map(ContainerTemplate::getEnv).orElseGet(Collections::emptyList));
envVarsUI.addAll(getResourcesByType(trustResourcesUI, EnvVar.class));

return desired.edit()
.editMetadata()
Expand All @@ -98,15 +112,19 @@ protected Deployment desired(Console primary, Context<Console> context) {
.endSecret()
.endVolume()
.addAllToVolumes(getResourcesByType(trustResources, Volume.class))
// Set API container image options
.editMatchingContainer(c -> "console-api".equals(c.getName()))
.withImage(imageAPI)
.withImagePullPolicy(pullPolicy(imageAPI))
.withResources(templateAPI.map(ContainerTemplate::getResources).orElse(null))
.addAllToVolumeMounts(getResourcesByType(trustResources, VolumeMount.class))
.addAllToEnv(envVars)
.endContainer()
// Set UI container image options
.editMatchingContainer(c -> "console-ui".equals(c.getName()))
.withImage(imageUI)
.withImagePullPolicy(pullPolicy(imageUI))
.withResources(templateUI.map(ContainerTemplate::getResources).orElse(null))
.editMatchingEnv(env -> "NEXTAUTH_URL".equals(env.getName()))
.withValue(getAttribute(context, ConsoleIngress.NAME + ".url", String.class))
.endEnv()
Expand Down
134 changes: 114 additions & 20 deletions operator/src/test/java/com/github/streamshub/console/ConsoleReconcilerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
Expand All @@ -27,8 +28,11 @@
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarBuilder;
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.api.model.Quantity;
import io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.api.model.SecretBuilder;
import io.fabric8.kubernetes.api.model.Volume;
Expand Down Expand Up @@ -79,6 +83,92 @@ void testBasicConsoleReconciliation() {
awaitReady(consoleCR);
}

@Test
void testConsoleReconciliationWithContainerOverrides() {
Console consoleCR = new ConsoleBuilder()
.withMetadata(new ObjectMetaBuilder()
.withName("console-1")
.withNamespace("ns2")
.build())
.withNewSpec()
.withHostname("console.example.com")
.addNewMetricsSource()
.withName("metrics")
.withType(Type.STANDALONE)
.withUrl("http://prometheus.example.com")
.endMetricsSource()
.withNewImages()
.withApi("deprecated-api-image")
.withUi("deprecated-ui-image")
.endImages()
.addToEnv(new EnvVarBuilder()
.withName("DEPRECATED_API_VAR")
.withValue("value0")
.build())
.withNewContainers()
.withNewApi()
.withImage("custom-api-image")
.withResources(new ResourceRequirementsBuilder()
.withRequests(Map.of("cpu", Quantity.parse("250m")))
.withLimits(Map.of("cpu", Quantity.parse("500m")))
.build())
.addToEnv(new EnvVarBuilder()
.withName("CUSTOM_API_VAR")
.withValue("value1")
.build())
.endApi()
.withNewUi()
.withImage("custom-ui-image")
.withResources(new ResourceRequirementsBuilder()
.withRequests(Map.of("cpu", Quantity.parse("100m")))
.withLimits(Map.of("cpu", Quantity.parse("200m")))
.build())
.addToEnv(new EnvVarBuilder()
.withName("CUSTOM_UI_VAR")
.withValue("value2")
.build())
.endUi()
.endContainers()
.addNewKafkaCluster()
.withName(kafkaCR.getMetadata().getName())
.withNamespace(kafkaCR.getMetadata().getNamespace())
.withListener(kafkaCR.getSpec().getKafka().getListeners().get(0).getName())
.withMetricsSource("metrics")
.endKafkaCluster()
.endSpec()
.build();

client.resource(consoleCR).create();

awaitDependentsNotReady(consoleCR, "ConsoleIngress");
setConsoleIngressReady(consoleCR);
awaitDependentsNotReady(consoleCR, "ConsoleDeployment");
var consoleDeployment = setDeploymentReady(consoleCR, ConsoleDeployment.NAME);
var consoleContainers = consoleDeployment.getSpec().getTemplate().getSpec().getContainers();
var apiContainer = consoleContainers.get(0);

assertEquals("custom-api-image", apiContainer.getImage());
assertEquals(new ResourceRequirementsBuilder()
.withRequests(Map.of("cpu", Quantity.parse("250m")))
.withLimits(Map.of("cpu", Quantity.parse("500m")))
.build(), apiContainer.getResources());
assertEquals(4, apiContainer.getEnv().size()); // 2 overrides + 2 from YAML template
assertEquals("value0", apiContainer.getEnv().stream()
.filter(e -> e.getName().equals("DEPRECATED_API_VAR")).map(EnvVar::getValue).findFirst().orElseThrow());
assertEquals("value1", apiContainer.getEnv().stream()
.filter(e -> e.getName().equals("CUSTOM_API_VAR")).map(EnvVar::getValue).findFirst().orElseThrow());

var uiContainer = consoleContainers.get(1);
assertEquals("custom-ui-image", uiContainer.getImage());
assertEquals(new ResourceRequirementsBuilder()
.withRequests(Map.of("cpu", Quantity.parse("100m")))
.withLimits(Map.of("cpu", Quantity.parse("200m")))
.build(), uiContainer.getResources());
assertEquals(7, uiContainer.getEnv().size()); // 1 override + 6 from YAML template
assertEquals("value2", uiContainer.getEnv().stream()
.filter(e -> e.getName().equals("CUSTOM_UI_VAR")).map(EnvVar::getValue).findFirst().orElseThrow());
}

@Test
void testConsoleReconciliationWithInvalidListenerName() {
Console consoleCR = new ConsoleBuilder()
Expand Down Expand Up @@ -757,31 +847,22 @@ void testConsoleReconciliationWithTrustStores() {
var volumes = podSpec.getVolumes().stream().collect(Collectors.toMap(Volume::getName, Function.identity()));
assertEquals(4, volumes.size()); // cache, config + 2 volumes for truststores

var metricsVolName = "metrics-source-truststore-example-prometheus";
var registryVolName = "schema-registry-truststore-example-registry";

var metricsVolume = volumes.get(metricsVolName);
assertEquals("metrics-source-truststore.example-prometheus.content", metricsVolume.getSecret().getItems().get(0).getKey());
assertEquals("metrics-source-truststore.example-prometheus.jks", metricsVolume.getSecret().getItems().get(0).getPath());

var registryVolume = volumes.get(registryVolName);
assertEquals("schema-registry-truststore.example-registry.content", registryVolume.getSecret().getItems().get(0).getKey());
assertEquals("schema-registry-truststore.example-registry.pem", registryVolume.getSecret().getItems().get(0).getPath());

var mounts = containerSpecAPI.getVolumeMounts().stream().collect(Collectors.toMap(VolumeMount::getName, Function.identity()));
assertEquals(4, mounts.size(), mounts::toString);

var metricsMount = mounts.get(metricsVolName);
var metricsMountPath = "/etc/ssl/metrics-source-truststore.example-prometheus.jks";
assertEquals(metricsMountPath, metricsMount.getMountPath());
assertEquals("metrics-source-truststore.example-prometheus.jks", metricsMount.getSubPath());
var envVars = containerSpecAPI.getEnv().stream().collect(Collectors.toMap(EnvVar::getName, Function.identity()));

var registryMount = mounts.get(registryVolName);
var registryMountPath = "/etc/ssl/schema-registry-truststore.example-registry.pem";
assertEquals(registryMountPath, registryMount.getMountPath());
assertEquals("schema-registry-truststore.example-registry.pem", registryMount.getSubPath());
var metricsVolName = "metrics-source-truststore-example-prometheus";
var metricsMountPath = "/etc/ssl/metrics-source-truststore.example-prometheus.jks";

var envVars = containerSpecAPI.getEnv().stream().collect(Collectors.toMap(EnvVar::getName, Function.identity()));
assertKeyToPath(
"metrics-source-truststore.example-prometheus.content",
"metrics-source-truststore.example-prometheus.jks",
volumes.get(metricsVolName).getSecret().getItems().get(0));
assertMounthPaths(
metricsMountPath,
"metrics-source-truststore.example-prometheus.jks",
mounts.get(metricsVolName));

var metricsTrustPath = envVars.get("QUARKUS_TLS__METRICS_SOURCE_EXAMPLE_PROMETHEUS__TRUST_STORE_JKS_PATH");
assertEquals(metricsMountPath, metricsTrustPath.getValue());
Expand All @@ -792,6 +873,19 @@ void testConsoleReconciliationWithTrustStores() {
assertEquals("console-1-console-secret", metricsPasswordSource.getValueFrom().getSecretKeyRef().getName());
assertEquals("metrics-source-truststore.example-prometheus.password", metricsPasswordSource.getValueFrom().getSecretKeyRef().getKey());

var registryVolName = "schema-registry-truststore-example-registry";
var registryMountPath = "/etc/ssl/schema-registry-truststore.example-registry.pem";

assertKeyToPath(
"schema-registry-truststore.example-registry.content",
"schema-registry-truststore.example-registry.pem",
volumes.get(registryVolName).getSecret().getItems().get(0));

assertMounthPaths(
registryMountPath,
"schema-registry-truststore.example-registry.pem",
mounts.get(registryVolName));

var registryTrustPath = envVars.get("QUARKUS_TLS__SCHEMA_REGISTRY_EXAMPLE_REGISTRY__TRUST_STORE_PEM_CERTS");
assertEquals(registryMountPath, registryTrustPath.getValue());
}
Expand Down
Loading

0 comments on commit ed01509

Please sign in to comment.