Skip to content

Commit

Permalink
Oppleg for versjonsstyring av malar (#1060)
Browse files Browse the repository at this point in the history
* Konfig for Unleash for brevbaker

* Unleash-konfig

* Skriv om så oppsett og overstyring av feature toggle kun kan gjerast ved initialisering, og dermed unngå risikoen for at nokon på eit tidspunkt misbruker overrides-funksjonaliteten

* Gjer om builder frå objekt til klasse, så testane kan køyre utan å gå i frø

* POC-commit for å vise korleis det kan funke

* Som det er no blir malane lese inn kun ved oppstart, og toggle-styring av dei blir kun oppdatert ved oppstart av systemet. Det er ikkje bra nok, så refaktoriserer for å leggje til rette for oppdatering undervegs

* POC-commit for å vise korleis det kan funke

* Revert "POC-commit for å vise korleis det kan funke"

This reverts commit 0303a8a.

* Revert "POC-commit for å vise korleis det kan funke"

This reverts commit 0aa973b.

* Kommentar for å tydeleggjera tenkt bruk

* Konfig for Unleash for brevbaker også i docker-compose

* Satt opp fetch-secrets også for azuread, det treng vi jo også for å køyre brevbaker no

* Presiserer appname i bygg også

* Skrur på fullstendig logging av stacktrace ved feil undervegs i gradle

* Effektiviserer bygga våre ved å skru på gradle-caching

* Nytt forsøk på fullverdig logging av feil

* Litt ekstra indireksjon for å unngå å initialisere Unleash unødig i testsamanheng

* Oppsett for testane

* Vi kan jo bruke Unleash-sdken sin innebygde fake heller enn å lage vår eigen

* Unngå unødig builder-metode-bruk

* Ekstra mykje event-logging for integrasjonstestane

* Prøver å pinpointe feilen

* Kanskje det var manglande miljøvariablar som var problemet

* Roar ned logginga no når ting funkar

* Rettar feil mappenamn

* Oppdaterer unleash-token-referanse

* Prøver litt meir flytande kotlinsk måte å gjera det på

* Tar bort debuglogging som eigentleg ikkje er så nyttig

* Gjer template-map privat, så vi ikkje risikerer at nokon bruker den ved eit uhell seinare

* Loggar når oppsettet er ferdig på ordentleg måte

* Får ut rett kode også for metadata

* Rettar opp i fetch-secrets
  • Loading branch information
madsop-nav authored Nov 20, 2024
1 parent 771d0e7 commit 12afdc4
Show file tree
Hide file tree
Showing 21 changed files with 346 additions and 52 deletions.
8 changes: 6 additions & 2 deletions .github/workflows/pdfbygger-brevbaker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -145,10 +145,14 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
brevbaker:
env:
NAIS_APP_NAME: brevbaker
PDF_BUILDER_URL: "http://pdf-bygger:8080"
AZURE_OPENID_CONFIG_JWKS_URI: "http://vtp-pensjon:8060/rest/isso/oauth2/connect/jwk_uri"
AZURE_OPENID_CONFIG_ISSUER: "https://login.microsoftonline.com/966ac572-f5b7-4bbe-aa88-c76419c0f851/v2.0"
AZURE_APP_CLIENT_ID: "brevbaker-123"
UNLEASH_SERVER_API_ENV: "test"
UNLEASH_SERVER_API_URL: "http://localhost"
UNLEASH_SERVER_API_TOKEN: "Token123"
JAVA_TOOL_OPTIONS: "-Dio.ktor.development=true"
ports:
- 8080:8080
Expand Down Expand Up @@ -228,7 +232,7 @@ jobs:
env:
"ENVIRONMENT": "dev-gcp:brevbaker"
"CLUSTER": "dev-gcp"
"RESOURCE": "pensjon-brevbaker/.nais/nais.yaml"
"RESOURCE": "pensjon-brevbaker/.nais/unleash-api-token-dev-q2.yaml,pensjon-brevbaker/.nais/nais.yaml"
"VAR": image=${{ needs.build-brevbaker.outputs.image }}
"VARS": "pensjon-brevbaker/.nais/dev.yaml"

Expand Down Expand Up @@ -266,7 +270,7 @@ jobs:
env:
"ENVIRONMENT": "prod-gcp:brevbaker"
"CLUSTER": "prod-gcp"
"RESOURCE": "pensjon-brevbaker/.nais/nais.yaml"
"RESOURCE": "pensjon-brevbaker/.nais/unleash-api-token-prod.yaml,pensjon-brevbaker/.nais/nais.yaml"
"VAR": image=${{ needs.build-brevbaker.outputs.image }}
"VARS": "pensjon-brevbaker/.nais/prod.yaml"

Expand Down
3 changes: 3 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ services:
- "5016:5006"
pensjon-brevbaker:
environment:
- NAIS_APP_NAME=brevbaker
- PDF_BUILDER_URL=http://pdf-bygger:8080
- AZURE_OPENID_CONFIG_JWKS_URI=https://login.microsoftonline.com/966ac572-f5b7-4bbe-aa88-c76419c0f851/discovery/v2.0/keys
- AZURE_OPENID_CONFIG_ISSUER=https://login.microsoftonline.com/966ac572-f5b7-4bbe-aa88-c76419c0f851/v2.0
Expand All @@ -23,6 +24,8 @@ services:
ports:
- "8080:8080"
- "5015:5005" # debug port
env_file:
- ./pensjon-brevbaker/secrets/unleash.env
skribenten-backend:
profiles:
- skribenten
Expand Down
3 changes: 2 additions & 1 deletion fetch-secrets.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@

(cd skribenten-backend && ./fetch-secrets.sh)
(cd tjenestebuss-integrasjon && ./fetch-secrets.sh)
(cd skribenten-web/bff && python3 setup_local_azure_secrets.py)
(cd skribenten-web/bff && python3 setup_local_azure_secrets.py)
(cd pensjon-brevbaker && ./fetch-secrets.sh)
4 changes: 3 additions & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,6 @@ micrometerVersion=1.12.5
apiModelVersion=101
jacksonJsr310Version=2.17.0
mockkVersion=1.13.10
exposedVersion=0.55.0
exposedVersion=0.55.0
org.gradle.logging.stacktrace=all
org.gradle.caching=true
3 changes: 2 additions & 1 deletion pensjon-brevbaker/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@
*.iml
*.ipr
*.iws
/docs
/docs
/secrets
3 changes: 3 additions & 0 deletions pensjon-brevbaker/.nais/dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ ingresses:
groups:
- "8bb9b8d1-f46a-4ade-8ee8-5895eccdf8cf" #0000-GA-PENSJON_SAKSBEHANDLER

externalHosts:
- teampensjon-unleash-api.nav.cloud.nais.io

preAuthorized:
- name: pensjon-pen-q0
namespace: pensjon-q0
Expand Down
4 changes: 3 additions & 1 deletion pensjon-brevbaker/.nais/nais.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -75,4 +75,6 @@ spec:
- name: PDF_BUILDER_URL
value: "http://pensjon-pdf-bygger:8080"
- name: PDF_BYGGER_MAX_RETRIES
value: "250"
value: "250"
envFrom:
- secret: "{{ app-name }}-unleash-api-token"
3 changes: 3 additions & 0 deletions pensjon-brevbaker/.nais/prod.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ ingresses:
groups:
- "0af3955f-df85-4eb0-b5b2-45bf2c8aeb9e" #0000-GA-PENSJON_SAKSBEHANDLER

externalHosts:
- teampensjon-unleash-api.nav.cloud.nais.io

preAuthorized:
- name: pensjon-pen
namespace: pensjondeployer
Expand Down
15 changes: 15 additions & 0 deletions pensjon-brevbaker/.nais/unleash-api-token-dev-q2.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: unleash.nais.io/v1
kind: ApiToken
metadata:
name: {{ app-name }}
namespace: pensjonsbrev
spec:
unleashInstance:
apiVersion: unleash.nais.io/v1
kind: RemoteUnleash
name: teampensjon
secretName: {{ app-name }}-unleash-api-token

# Specify which environment the API token should be created for.
# Can be one of: development, or production.
environment: development
15 changes: 15 additions & 0 deletions pensjon-brevbaker/.nais/unleash-api-token-prod.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
apiVersion: unleash.nais.io/v1
kind: ApiToken
metadata:
name: {{ app-name }}
namespace: pensjonsbrev
spec:
unleashInstance:
apiVersion: unleash.nais.io/v1
kind: RemoteUnleash
name: teampensjon
secretName: {{ app-name }}-unleash-api-token

# Specify which environment the API token should be created for.
# Can be one of: development, or production.
environment: production
15 changes: 15 additions & 0 deletions pensjon-brevbaker/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import org.gradle.api.tasks.testing.logging.TestLogEvent
import org.jetbrains.kotlin.gradle.dsl.JvmTarget

val javaTarget: String by System.getProperties()
Expand Down Expand Up @@ -55,19 +56,31 @@ tasks {
useJUnitPlatform {
excludeTags = setOf("integration-test", "manual-test")
}
testLogging {
events(TestLogEvent.PASSED, TestLogEvent.SKIPPED, TestLogEvent.FAILED, TestLogEvent.STANDARD_ERROR)
exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL
}
}

task<Test>("integrationTest") {
group = LifecycleBasePlugin.VERIFICATION_GROUP
useJUnitPlatform {
includeTags = setOf("integration-test")
}
testLogging {
events(TestLogEvent.PASSED, TestLogEvent.SKIPPED, TestLogEvent.FAILED, TestLogEvent.STANDARD_ERROR)
exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL
}
}
task<Test>("manualTest") {
group = LifecycleBasePlugin.VERIFICATION_GROUP
useJUnitPlatform {
includeTags = setOf("manual-test")
}
testLogging {
events(TestLogEvent.PASSED, TestLogEvent.SKIPPED, TestLogEvent.FAILED, TestLogEvent.STANDARD_ERROR)
exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL
}
}
}

Expand Down Expand Up @@ -109,6 +122,8 @@ dependencies {
because("we require deserialization/serialization of java.time.LocalDate")
}

implementation("io.getunleash:unleash-client-java:9.2.4")

// Metrics
implementation("io.ktor:ktor-server-metrics:$ktorVersion")
implementation("io.ktor:ktor-server-metrics-micrometer:$ktorVersion")
Expand Down
70 changes: 70 additions & 0 deletions pensjon-brevbaker/fetch-secrets.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/bin/bash

KUBE_CLUSTER="dev-gcp"

function checkKubectl() {
echo "Verify kubectl, may take some time..."
output="$(kubectl --context $KUBE_CLUSTER version 2>&1)"
status=$?

if [ $status -gt 0 ] ; then
if echo "$output" | grep -q "command not found" ; then
echo "ERROR: You need to install kubectl: $output"
echo "Howto: https://doc.nais.io/basics/access/"
return $status

elif echo "$output" | grep -q "Unable to connect to the server" ; then
error_msg="$(echo "$output" | grep "Unable to connect to the server" | cut -d':' -f2)"
echo "ERROR: Cannot connect to kubernetes cluster $KUBE_CLUSTER: $error_msg"
echo "Have you remembered to connect naisdevice? (see https://doc.nais.io/basics/access/)"
return 1

elif echo "$output" | grep -q "error: You must be logged in" ; then
echo "ERROR: Not logged in to the cluster. Use 'gcloud auth login' (see https://doc.nais.io/basics/access/)."
return 1

else
echo "WARN: Got unknown error from kubectl, but will attempt to fetch secrets anyway."
return 0

fi
elif echo "$output" | grep -q "Client Version" && echo "$output" | grep -q "Server Version" ; then
echo "kubectl: OK "
return 0
else
echo "WARN: Got unexpected output from 'kubectl version', but will attempt to fetch secrets anyway."
return 0
fi
}

checkKubectl || exit 1
jq --version || (
echo "ERROR: You need to install the jq CLI tool on your machine: https://stedolan.github.io/jq/" && exit 1
) || exit 1
which base64 || (
echo "ERROR: You need to install the base64 tool on your machine. (brew install base64 on macOS)" && exit 1
) || exit 1

function getSecret() {
local secret_name="$1"
local output_name="$2"

echo ""
kubectl --context $KUBE_CLUSTER -n pensjonsbrev get secret "${secret_name}" -o json | jq '.data | map_values(@base64d)' > secrets/"${output_name}".json

echo "Creating ${output_name}.env file from ${output_name}.json..."
jq -r 'to_entries|map("\(.key)=\(.value|tostring)")|.[]' secrets/"${output_name}".json > secrets/"${output_name}".env
echo "${output_name}.env file created in the \"secrets\" folder."

}

mkdir -p secrets

# AzureAD
secret_name="$(kubectl --context $KUBE_CLUSTER -n pensjonsbrev get azureapp pensjon-brevbaker -o=jsonpath='{.spec.secretName}')"
getSecret "$secret_name" azuread

# Unleash ApiToken
getSecret pensjon-brevbaker-unleash-api-token unleash

echo "All secrets are fetched and stored in the \"secrets\" folder."
Original file line number Diff line number Diff line change
@@ -1,18 +1,28 @@
package no.nav.pensjon.brev

import com.fasterxml.jackson.core.JacksonException
import io.ktor.http.*
import io.ktor.serialization.jackson.*
import io.ktor.server.application.*
import io.ktor.server.auth.*
import io.ktor.server.plugins.*
import io.ktor.server.plugins.callid.*
import io.ktor.server.plugins.calllogging.*
import io.ktor.server.plugins.contentnegotiation.*
import io.ktor.server.plugins.statuspages.*
import io.ktor.server.request.*
import io.ktor.server.response.*
import io.ktor.util.date.*
import io.ktor.http.ContentType
import io.ktor.http.HttpStatusCode
import io.ktor.serialization.jackson.jackson
import io.ktor.server.application.Application
import io.ktor.server.application.ApplicationStopPreparing
import io.ktor.server.application.ServerReady
import io.ktor.server.application.install
import io.ktor.server.application.log
import io.ktor.server.auth.Authentication
import io.ktor.server.config.ApplicationConfig
import io.ktor.server.plugins.BadRequestException
import io.ktor.server.plugins.ParameterConversionException
import io.ktor.server.plugins.callid.CallId
import io.ktor.server.plugins.callid.callIdMdc
import io.ktor.server.plugins.callid.generate
import io.ktor.server.plugins.calllogging.CallLogging
import io.ktor.server.plugins.calllogging.processingTimeMillis
import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
import io.ktor.server.plugins.statuspages.StatusPages
import io.ktor.server.request.path
import io.ktor.server.response.respond
import io.ktor.util.date.getTimeMillis
import no.nav.pensjon.brev.Metrics.configureMetrics
import no.nav.pensjon.brev.api.ParseLetterDataException
import no.nav.pensjon.brev.converters.LetterResponseFileConverter
Expand Down Expand Up @@ -51,25 +61,34 @@ fun Application.brevbakerModule() {
// Work-around to print proper error message when call.receive<T> fails.
exception<BadRequestException> { call, cause ->
if (cause.cause is JacksonException) {
call.respond(HttpStatusCode.BadRequest, cause.cause?.message ?: "Failed to deserialize json body: unknown reason")
call.respond(
HttpStatusCode.BadRequest,
cause.cause?.message ?: "Failed to deserialize json body: unknown reason"
)
} else {
call.respond(HttpStatusCode.BadRequest, cause.message ?: "Unknown failure")
}
}
exception<LatexTimeoutException>{ call, cause ->
exception<LatexTimeoutException> { call, cause ->
call.application.log.info("Latex compilation timed out", cause)
call.respond(HttpStatusCode.ServiceUnavailable, cause.message ?: "Timed out while compiling latex")
}
exception<LatexCompileException>{ call, cause ->
exception<LatexCompileException> { call, cause ->
call.application.log.info("Latex compilation failed with internal server error", cause)
call.respond(HttpStatusCode.InternalServerError, cause.message ?: "Latex compilation failed")
}
exception<LatexInvalidException>{ call, cause ->
exception<LatexInvalidException> { call, cause ->
call.application.log.info("Latex compilation failed due to invalid latex", cause)
call.respond(HttpStatusCode.InternalServerError, cause.message ?: "Latex compilation failed due to invalid latex")
call.respond(
HttpStatusCode.InternalServerError,
cause.message ?: "Latex compilation failed due to invalid latex"
)
}
exception<ParameterConversionException> { call, cause ->
call.respond(HttpStatusCode.BadRequest, cause.message?: "Failed to convert path parameter to required type: unknown cause")
call.respond(
HttpStatusCode.BadRequest,
cause.message ?: "Failed to convert path parameter to required type: unknown cause"
)
}
exception<ParseLetterDataException> { call, cause ->
call.respond(HttpStatusCode.BadRequest, cause.message ?: "Failed to deserialize letterData: Unknown cause")
Expand Down Expand Up @@ -103,6 +122,18 @@ fun Application.brevbakerModule() {
maxRetries = brevbakerConfig.propertyOrNull("pdfByggerMaxRetries")?.getString()?.toInt() ?: 30,
)

FeatureToggleHandler.configure {
with(brevbakerConfig.config("unleash")) {
appName = stringProperty("appName")
environment = stringProperty("environment")
this@configure.host = stringProperty("host")
apiToken = stringProperty("apiToken")
}
}

configureMetrics()
brevbakerRouting(jwtConfigs.map { it.name }.toTypedArray(), latexCompilerService)
}
monitor.subscribe(ServerReady) { it.log.info("Ferdig med å sette opp applikasjonen") }
}

private fun ApplicationConfig.stringProperty(path: String): String = this.property(path).getString()
Loading

0 comments on commit 12afdc4

Please sign in to comment.