diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a367766..da0fb47 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -28,6 +28,11 @@ jobs:
- name: Install clj runtime
run: .github/workflows/install-binaries.sh
+ - name: Start Redis
+ uses: supercharge/redis-github-action@1.4.0
+ with:
+ redis-version: 6.2
+
- name: Run tests
env:
GATEWAY_URL: https://gateway.test.surfeduhub.nl/
diff --git a/.nvd-suppressions.xml b/.nvd-suppressions.xml
new file mode 100644
index 0000000..c31f715
--- /dev/null
+++ b/.nvd-suppressions.xml
@@ -0,0 +1,13 @@
+
+
+
+
+ 5.0
+ CVE-2023-46120
+
+
+
+
diff --git a/deps.edn b/deps.edn
index 68c66fe..134a778 100644
--- a/deps.edn
+++ b/deps.edn
@@ -4,6 +4,8 @@
ch.qos.logback/logback-classic {:mvn/version "1.5.8"}
cheshire/cheshire {:mvn/version "5.13.0"}
com.fasterxml.jackson.core/jackson-databind {:mvn/version "2.17.2"}
+ com.nilenso/goose {:mvn/version "0.5.3"}
+ com.taoensso/carmine {:mvn/version "3.4.1"}
compojure/compojure {:mvn/version "1.7.1"}
nl.jomco/clj-http-status-codes {:mvn/version "0.1"}
nl.jomco/envopts {:mvn/version "0.0.4"}
diff --git a/readme.md b/readme.md
index 0c61bae..8618923 100644
--- a/readme.md
+++ b/readme.md
@@ -25,8 +25,10 @@ SURF_CONEXT_CLIENT_ID SurfCONEXT client id for validation service
SURF_CONEXT_CLIENT_SECRET SurfCONEXT client secret for validation service
SURF_CONEXT_INTROSPECTION_ENDPOINT SurfCONEXT introspection endpoint
ALLOWED_CLIENT_IDS Comma separated list of allowed SurfCONEXT client ids.
+MAX_TOTAL_REQUESTS Maximum number of requests that validator is allowed to make before raising an error
OOAPI_VERSION Ooapi version to pass through to gateway
SERVER_PORT Starts the app server on this port
+JOB_STATUS_EXPIRY_SECONDS Number of seconds before job status in Redis expires
```
## Build
diff --git a/src/nl/surf/eduhub/validator/service/api.clj b/src/nl/surf/eduhub/validator/service/api.clj
new file mode 100644
index 0000000..0b81744
--- /dev/null
+++ b/src/nl/surf/eduhub/validator/service/api.clj
@@ -0,0 +1,70 @@
+;; This file is part of eduhub-validator-service
+;;
+;; Copyright (C) 2024 SURFnet B.V.
+;;
+;; This program is free software: you can redistribute it and/or
+;; modify it under the terms of the GNU Affero General Public License
+;; as published by the Free Software Foundation, either version 3 of
+;; the License, or (at your option) any later version.
+;;
+;; This program is distributed in the hope that it will be useful, but
+;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+;; Affero General Public License for more details.
+;;
+;; You should have received a copy of the GNU Affero General Public
+;; License along with this program. If not, see
+;; .
+
+(ns nl.surf.eduhub.validator.service.api
+ (:require [clojure.string :as str]
+ [compojure.core :refer [GET POST defroutes]]
+ [compojure.route :as route]
+ [nl.jomco.http-status-codes :as http-status]
+ [nl.surf.eduhub.validator.service.authentication :as auth]
+ [nl.surf.eduhub.validator.service.checker :as checker]
+ [nl.surf.eduhub.validator.service.jobs.client :as jobs-client]
+ [nl.surf.eduhub.validator.service.jobs.status :as status]
+ [ring.middleware.defaults :refer [api-defaults wrap-defaults]]
+ [ring.middleware.json :refer [wrap-json-response]]))
+
+(defroutes app-routes
+ (GET "/status/:uuid" [uuid]
+ {:load-status true, :uuid uuid})
+ (POST "/endpoints/:endpoint-id/config" [endpoint-id]
+ {:checker true :endpoint-id endpoint-id})
+ (POST "/endpoints/:endpoint-id/paths" [endpoint-id profile]
+ {:validator true :endpoint-id endpoint-id :profile profile})
+ (route/not-found "Not Found"))
+
+;; Many response handlers have the same structure - with this function they can be written inline.
+;; `activate-handler?` is a function that takes a request and returns a boolean which determines if
+;; the current handler should be activated (or skipped).
+;; `response-handler` takes an intermediate response and processes it into the next step.
+(defn wrap-response-handler [app activate-handler? response-handler]
+ (fn [req]
+ (let [resp (app req)]
+ (if (activate-handler? resp)
+ (response-handler resp)
+ resp))))
+
+;; Turn the contents of a job status (stored in redis) into an http response.
+(defn- job-status-handler [{:keys [redis-conn] :as _config}]
+ (fn handle-job-status [resp]
+ (let [job-status (status/load-status redis-conn (:uuid resp))]
+ (if (empty? job-status)
+ {:status http-status/not-found}
+ {:status http-status/ok :body job-status}))))
+
+;; Compose the app from the routes and the wrappers. Authentication can be disabled for testing purposes.
+(defn compose-app [{:keys [introspection-endpoint-url introspection-basic-auth allowed-client-ids] :as config} auth-enabled]
+ (let [allowed-client-id-set (set (str/split allowed-client-ids #","))
+ auth-opts {:auth-enabled (boolean auth-enabled)}]
+ (-> app-routes
+ (wrap-response-handler :checker #(checker/check-endpoint (:endpoint-id %) config))
+ (wrap-response-handler :validator #(jobs-client/enqueue-validation (:endpoint-id %) (:profile %) config))
+ (wrap-response-handler :load-status (job-status-handler config))
+ (auth/wrap-allowed-clients-checker allowed-client-id-set auth-opts)
+ (auth/wrap-authentication introspection-endpoint-url introspection-basic-auth auth-opts)
+ wrap-json-response
+ (wrap-defaults api-defaults))))
diff --git a/src/nl/surf/eduhub/validator/service/authentication.clj b/src/nl/surf/eduhub/validator/service/authentication.clj
index 7b8cb42..7a1971b 100644
--- a/src/nl/surf/eduhub/validator/service/authentication.clj
+++ b/src/nl/surf/eduhub/validator/service/authentication.clj
@@ -110,17 +110,20 @@
If no bearer token is provided, the request is executed without a client-id."
; auth looks like {:user client-id :pass client-secret}
- [f introspection-endpoint auth]
+ [f introspection-endpoint auth {:keys [auth-enabled]}]
(let [authenticator (memo/ttl (make-token-authenticator introspection-endpoint auth) :ttl/threshold 60000)] ; 1 minute
(fn authentication [request]
- (if-let [token (bearer-token request)]
- (handle-request-with-token request f (authenticator token))
+ (if auth-enabled
+ (if-let [token (bearer-token request)]
+ (handle-request-with-token request f (authenticator token))
+ (f request))
(f request)))))
-(defn wrap-allowed-clients-checker [f allowed-client-id-set]
+(defn wrap-allowed-clients-checker [f allowed-client-id-set {:keys [auth-enabled]}]
{:pre [(set? allowed-client-id-set)]}
(fn allowed-clients-checker [{:keys [client-id] :as request}]
- (if (and client-id (allowed-client-id-set client-id))
+ (if (or (not auth-enabled)
+ (and client-id (allowed-client-id-set client-id)))
(f request)
{:body (if client-id "Unknown client id" "No client-id found")
:status http-status/forbidden})))
diff --git a/src/nl/surf/eduhub/validator/service/checker.clj b/src/nl/surf/eduhub/validator/service/checker.clj
new file mode 100644
index 0000000..1183aed
--- /dev/null
+++ b/src/nl/surf/eduhub/validator/service/checker.clj
@@ -0,0 +1,47 @@
+(ns nl.surf.eduhub.validator.service.checker
+ (:require [babashka.json :as json]
+ [clojure.tools.logging :as log]
+ [nl.jomco.http-status-codes :as http-status]
+ [nl.surf.eduhub.validator.service.validate :as validate]))
+
+(defn- handle-check-endpoint-response [status body endpoint-id]
+ (condp = status
+ ;; If the validator doesn't have the right credentials for the gateway, manifested by a 401 response,
+ ;; we'll return a 502 error and log it.
+ http-status/unauthorized
+ {:status http-status/bad-gateway :body {:valid false
+ :message "Incorrect credentials for gateway"}}
+
+ ;; If the gateway returns OK we assume we've gotten a json envelope response and check the response status
+ ;; of the endpoint.
+ http-status/ok
+ (let [envelope (json/read-str body)
+ envelope-status (get-in envelope [:gateway :endpoints (keyword endpoint-id) :responseCode])]
+ {:status http-status/ok
+ :body (if (= "200" (str envelope-status))
+ {:valid true}
+ ;; If the endpoint denied the gateway's request or otherwise returned a response outside the 200 range
+ ;; we return 200 ok and return the status of the endpoint and the message of the gateway in our response
+ {:valid false
+ :message (str "Endpoint validation failed with status: " envelope-status)})})
+
+ ;; If the gateway returns something else than a 200 or a 401, treat it similar to an error
+ (let [error-msg (str "Unexpected response status received from gateway: " status)]
+ (log/error error-msg)
+ {:status http-status/internal-server-error
+ :body {:valid false
+ :message error-msg}})))
+
+;; The endpoint checker from phase 1. This connects to an endpoint via the gateway and checks if it receives
+;; a valid response.
+(defn check-endpoint [endpoint-id config]
+ (try
+ (let [{:keys [status body]} (validate/check-endpoint endpoint-id config)]
+ ;; If the client credentials for the validator are incorrect, the wrap-allowed-clients-checker
+ ;; middleware has already returned 401 forbidden and execution doesn't get here.
+ (handle-check-endpoint-response status body endpoint-id))
+ (catch Throwable e
+ (log/error e "Internal error in validator-service")
+ {:status http-status/internal-server-error
+ :body {:valid false
+ :message "Internal error in validator-service"}})))
diff --git a/src/nl/surf/eduhub/validator/service/config.clj b/src/nl/surf/eduhub/validator/service/config.clj
index 596cb77..e5fc3ea 100644
--- a/src/nl/surf/eduhub/validator/service/config.clj
+++ b/src/nl/surf/eduhub/validator/service/config.clj
@@ -30,13 +30,22 @@
:in [:gateway-basic-auth :pass]]
:allowed-client-ids ["Comma separated list of allowed SurfCONEXT client ids." :str
:in [:allowed-client-ids]]
+ :max-total-requests ["Maximum number of requests that validator is allowed to make before raising an error" :int
+ :default 10000
+ :in [:max-total-requests]]
:surf-conext-client-id ["SurfCONEXT client id for validation service" :str
:in [:introspection-basic-auth :user]]
:surf-conext-client-secret ["SurfCONEXT client secret for validation service" :str
:in [:introspection-basic-auth :pass]]
:surf-conext-introspection-endpoint ["SurfCONEXT introspection endpoint" :str
:in [:introspection-endpoint-url]]
+ :redis-uri ["URI to redis" :str
+ :default "redis://localhost"
+ :in [:redis-conn :spec :uri]]
:server-port ["Starts the app server on this port" :int]
+ :job-status-expiry-seconds ["Number of seconds before job status in Redis expires" :int
+ :default (* 3600 24 14)
+ :in [:expiry-seconds]]
:ooapi-version ["Ooapi version to pass through to gateway" :str
:in [:ooapi-version]]})
@@ -68,3 +77,13 @@
(defn load-config-from-env [env-map]
(-> (reduce file-secret-loader-reducer env-map env-keys-with-alternate-file-secret)
(envopts/opts opt-specs)))
+
+(defn validate-and-load-config [env]
+ (let [[config errs] (load-config-from-env env)]
+ (when errs
+ (.println *err* "Error in environment configuration")
+ (.println *err* (envopts/errs-description errs))
+ (.println *err* "Available environment vars:")
+ (.println *err* (envopts/specs-description opt-specs))
+ (System/exit 1))
+ config))
diff --git a/src/nl/surf/eduhub/validator/service/jobs/client.clj b/src/nl/surf/eduhub/validator/service/jobs/client.clj
new file mode 100644
index 0000000..6935956
--- /dev/null
+++ b/src/nl/surf/eduhub/validator/service/jobs/client.clj
@@ -0,0 +1,29 @@
+(ns nl.surf.eduhub.validator.service.jobs.client
+ (:require [clojure.tools.logging :as log]
+ [goose.brokers.redis.broker :as broker]
+ [goose.client :as c]
+ [goose.retry :as retry]
+ [nl.surf.eduhub.validator.service.jobs.status :as status]
+ [nl.surf.eduhub.validator.service.jobs.worker :as worker])
+ (:import [java.util UUID]))
+
+(defn job-error-handler [_cfg _job ex]
+ (log/error ex "Error in job"))
+
+(def client-opts (assoc c/default-opts
+ :broker (broker/new-producer broker/default-opts)
+ :retry-opts (assoc retry/default-opts :error-handler-fn-sym `job-error-handler)))
+
+;; Enqueue the validate-endpoint call in the worker queue.
+(defn enqueue-validation
+ [endpoint-id profile {:keys [redis-conn gateway-basic-auth gateway-url ooapi-version max-total-requests] :as _config}]
+ (let [uuid (str (UUID/randomUUID))
+ prof (or profile "rio")
+ opts {:basic-auth gateway-basic-auth
+ :base-url gateway-url
+ :max-total-requests max-total-requests
+ :ooapi-version ooapi-version
+ :profile prof}]
+ (status/set-status-fields redis-conn uuid "pending" {:endpoint-id endpoint-id, :profile prof} nil)
+ (c/perform-async client-opts `worker/validate-endpoint endpoint-id uuid opts)
+ {:status 200 :body {:job-status "pending" :uuid uuid}}))
diff --git a/src/nl/surf/eduhub/validator/service/jobs/status.clj b/src/nl/surf/eduhub/validator/service/jobs/status.clj
new file mode 100644
index 0000000..ad3021b
--- /dev/null
+++ b/src/nl/surf/eduhub/validator/service/jobs/status.clj
@@ -0,0 +1,28 @@
+(ns nl.surf.eduhub.validator.service.jobs.status
+ (:require [taoensso.carmine :as car])
+ (:import [java.time Instant]))
+
+(defn status-key [uuid] (str "validation:" uuid))
+
+(def job-status "job-status")
+
+;; Updates the status of the job status entry of job `uuid` to `new-status`.
+;; Also sets a timestamp named after the new status (e.g. finished-at)
+;; Also sets any values in `key-value-map` in the redis hash.
+;; If `expires-in-seconds` is set, (re)sets the expiry.
+(defn set-status-fields [redis-conn uuid new-status key-value-map expires-in-seconds]
+ (let [v (assoc key-value-map
+ job-status new-status
+ (str new-status "-at") (-> (System/currentTimeMillis)
+ Instant/ofEpochMilli
+ str))
+ key (status-key uuid)]
+ (car/wcar redis-conn (car/hmset* key v))
+ (when expires-in-seconds
+ (car/wcar redis-conn (car/expire key expires-in-seconds)))))
+
+;; Loads the job status as a clojure map for the job with given uuid.
+(defn load-status [redis-conn uuid]
+ (let [result (car/wcar redis-conn (car/hgetall (status-key uuid)))]
+ (when-not (empty? result)
+ (into {} (apply hash-map result)))))
diff --git a/src/nl/surf/eduhub/validator/service/jobs/worker.clj b/src/nl/surf/eduhub/validator/service/jobs/worker.clj
new file mode 100644
index 0000000..4c307fb
--- /dev/null
+++ b/src/nl/surf/eduhub/validator/service/jobs/worker.clj
@@ -0,0 +1,20 @@
+(ns nl.surf.eduhub.validator.service.jobs.worker
+ (:require [clojure.tools.logging :as log]
+ [nl.surf.eduhub.validator.service.jobs.status :as status]
+ [nl.surf.eduhub.validator.service.validate :as validate]))
+
+;; A worker thread running in the background
+;; Called by the workers. Runs the validate-endpoint function
+;; and updates the values in the job status.
+;; opts should contain: basic-auth ooapi-version base-url profile
+(defn validate-endpoint [endpoint-id uuid {:keys [config] :as opts}]
+ (let [{:keys [redis-conn expiry-seconds]} config]
+ (assert redis-conn)
+ (try
+ (let [html (validate/validate-endpoint endpoint-id opts)]
+ ;; assuming everything went ok, save html in status, update status and set expiry to value configured in ENV
+ (status/set-status-fields redis-conn uuid "finished" {"html-report" html} expiry-seconds))
+ (catch Throwable ex
+ ;; otherwise set status to error, include error message and also set expiry
+ (log/error ex "Validate endpoint threw an exception")
+ (status/set-status-fields redis-conn uuid "failed" {"error" (str ex)} expiry-seconds)))))
diff --git a/src/nl/surf/eduhub/validator/service/main.clj b/src/nl/surf/eduhub/validator/service/main.clj
index 4fd9a0f..6c29bfe 100644
--- a/src/nl/surf/eduhub/validator/service/main.clj
+++ b/src/nl/surf/eduhub/validator/service/main.clj
@@ -18,70 +18,14 @@
(ns nl.surf.eduhub.validator.service.main
(:gen-class)
- (:require [babashka.json :as json]
- [clojure.string :as str]
- [clojure.tools.logging :as log]
- [compojure.core :refer [GET defroutes]]
- [compojure.route :as route]
- [environ.core :refer [env]]
- [nl.jomco.envopts :as envopts]
- [nl.jomco.http-status-codes :as http-status]
- [nl.surf.eduhub.validator.service.authentication :as auth]
+ (:require [environ.core :refer [env]]
+ [goose.brokers.redis.broker :as broker]
+ [goose.worker :as w]
+ [nl.surf.eduhub.validator.service.api :as api]
[nl.surf.eduhub.validator.service.config :as config]
- [nl.surf.eduhub.validator.service.validate :as validate]
- [ring.adapter.jetty :refer [run-jetty]]
- [ring.middleware.defaults :refer [api-defaults wrap-defaults]]
- [ring.middleware.json :refer [wrap-json-response]]))
-
-(defn check-endpoint [endpoint-id config]
- (try
- (let [{:keys [status body]} (validate/check-endpoint endpoint-id config)]
- ;; If the client credentials for the validator are incorrect, the wrap-allowed-clients-checker
- ;; middleware has already returned 401 forbidden and execution doesn't get here.
- (condp = status
- ;; If the validator doesn't have the right credentials for the gateway, manifested by a 401 response,
- ;; we'll return a 502 error and log it.
- http-status/unauthorized
- {:status http-status/bad-gateway :body {:valid false
- :message "Incorrect credentials for gateway"}}
-
- ;; If the gateway returns OK we assume we've gotten a json envelope response and check the response status
- ;; of the endpoint.
- http-status/ok
- (let [envelope (json/read-str body)
- envelope-status (get-in envelope [:gateway :endpoints (keyword endpoint-id) :responseCode])]
- {:status http-status/ok
- :body (if (= "200" (str envelope-status))
- {:valid true}
- ;; If the endpoint denied the gateway's request or otherwise returned a response outside the 200 range
- ;; we return 200 ok and return the status of the endpoint and the message of the gateway in our response
- {:valid false
- :message (str "Endpoint validation failed with status: " envelope-status)})})
-
- ;; If the gateway returns something else than a 200 or a 401, treat it similar to an error
- (do
- (log/error "Unexpected response status received from gateway: " status)
- {:status http-status/internal-server-error
- :body {:valid false
- :message (str "Unexpected response status received from gateway: " status)}})))
- (catch Throwable e
- (log/error e "Exception in validate-endpoint")
- {:status http-status/internal-server-error
- :body {:valid false
- :message "Internal error in validator-service"}})))
-
-(defroutes app-routes
- (GET "/endpoints/:endpoint-id/config" [endpoint-id]
- (fn [_] {:validator true :endpoint-id endpoint-id}))
- (route/not-found "Not Found"))
-
-(defn wrap-validator [app config]
- (fn [req]
- (let [{:keys [validator endpoint-id] :as resp} (app req)]
- (if validator
- (check-endpoint endpoint-id config)
- resp))))
+ [ring.adapter.jetty :refer [run-jetty]]))
+;; Starts a Jetty server on given port.
(defn start-server [routes {:keys [server-port] :as _config}]
(let [server (run-jetty routes {:port server-port :join? false})
handler ^Runnable (fn [] (.stop server))]
@@ -91,20 +35,14 @@
server))
(defn -main [& _]
- (let [[config errs] (config/load-config-from-env env)]
- (when errs
- (.println *err* "Error in environment configuration")
- (.println *err* (envopts/errs-description errs))
- (.println *err* "Available environment vars:")
- (.println *err* (envopts/specs-description config/opt-specs))
- (System/exit 1))
- (let [introspection-endpoint (:introspection-endpoint-url config)
- introspection-auth (:introspection-basic-auth config)
- allowed-client-id-set (set (str/split (:allowed-client-ids config) #","))]
- (start-server (-> app-routes
- (wrap-validator config)
- (auth/wrap-allowed-clients-checker allowed-client-id-set)
- (auth/wrap-authentication introspection-endpoint introspection-auth)
- wrap-json-response
- (wrap-defaults api-defaults))
- config))))
+ (let [config (config/validate-and-load-config env)]
+ ;; set config as global var (read-only) so that the workers can access it
+ (w/start (assoc w/default-opts
+ :broker (broker/new-consumer broker/default-opts)
+ :middlewares (fn [app]
+ (fn [opts job]
+ ;; adds config to map at the last of the args in job
+ (let [job-args (assoc-in (vec (:args job)) [2 :config] config)
+ new-job (assoc job :args job-args)]
+ (app opts new-job))))))
+ (start-server (api/compose-app config :auth-enabled) config)))
diff --git a/src/nl/surf/eduhub/validator/service/validate.clj b/src/nl/surf/eduhub/validator/service/validate.clj
index 0740d39..d744b3e 100644
--- a/src/nl/surf/eduhub/validator/service/validate.clj
+++ b/src/nl/surf/eduhub/validator/service/validate.clj
@@ -22,9 +22,11 @@
[nl.jomco.apie.main :as apie])
(:import [java.io File]))
+;; Validates whether the endpoint is working and reachable at all.
(defn check-endpoint
"Performs a synchronous validation via the eduhub-validator"
[endpoint-id {:keys [gateway-url gateway-basic-auth ooapi-version] :as _config}]
+ {:pre [gateway-url]}
(let [url (str gateway-url (if (.endsWith gateway-url "/") "" "/") "courses")
opts {:headers {"x-route" (str "endpoint=" endpoint-id)
"accept" (str "application/json; version=" ooapi-version)
@@ -33,23 +35,19 @@
:throw false}]
(http/get url opts)))
-
-(defn- temp-file [fname ext]
- (let [tmpfile (File/createTempFile fname ext)]
- (.deleteOnExit tmpfile)
- tmpfile))
-
+;; Uses the ooapi validator to validate an endpoint.
+;; Returns the generated HTML report.
(defn validate-endpoint
"Returns the HTML validation report as a String."
- [endpoint-id {:keys [basic-auth ooapi-version base-url profile] :as opts}]
+ [endpoint-id {:keys [basic-auth ooapi-version max-total-requests base-url profile] :as opts}]
{:pre [endpoint-id basic-auth ooapi-version base-url profile]}
- (let [report-file (temp-file "report" ".html")
+ (let [report-file (File/createTempFile "report" ".html")
report-path (.getAbsolutePath report-file)
- observations-file (temp-file "observations" ".edn")
+ observations-file (File/createTempFile "observations" ".edn")
observations-path (.getAbsolutePath observations-file)
defaults {:bearer-token nil,
:no-report? false,
- :max-total-requests 5,
+ :max-total-requests max-total-requests,
:report-path report-path,
:headers {:x-route (str "endpoint=" endpoint-id),
:accept (str "application/json; version=" ooapi-version),
diff --git a/test/nl/surf/eduhub/validator/service/authentication_test.clj b/test/nl/surf/eduhub/validator/service/authentication_test.clj
index 5f6e3e6..467410b 100644
--- a/test/nl/surf/eduhub/validator/service/authentication_test.clj
+++ b/test/nl/surf/eduhub/validator/service/authentication_test.clj
@@ -63,8 +63,8 @@
(let [body {:client (:client-id req)}]
{:status http-status/ok
:body body}))
- (authentication/wrap-allowed-clients-checker allowed-client-id-set)
- (authentication/wrap-authentication introspection-endpoint basic-auth)))
+ (authentication/wrap-allowed-clients-checker allowed-client-id-set {:auth-enabled true})
+ (authentication/wrap-authentication introspection-endpoint basic-auth {:auth-enabled true})))
(deftest token-validator
;; This binds the *dynamic* http client in clj-http.client
diff --git a/test/nl/surf/eduhub/validator/service/checker_test.clj b/test/nl/surf/eduhub/validator/service/checker_test.clj
new file mode 100644
index 0000000..e2fdb6b
--- /dev/null
+++ b/test/nl/surf/eduhub/validator/service/checker_test.clj
@@ -0,0 +1,60 @@
+;; This file is part of eduhub-validator-service
+;;
+;; Copyright (C) 2024 SURFnet B.V.
+;;
+;; This program is free software: you can redistribute it and/or
+;; modify it under the terms of the GNU Affero General Public License
+;; as published by the Free Software Foundation, either version 3 of
+;; the License, or (at your option) any later version.
+;;
+;; This program is distributed in the hope that it will be useful, but
+;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+;; Affero General Public License for more details.
+;;
+;; You should have received a copy of the GNU Affero General Public
+;; License along with this program. If not, see
+;; .
+
+(ns nl.surf.eduhub.validator.service.checker-test
+ (:require [babashka.http-client :as http]
+ [babashka.json :as json]
+ [clojure.test :refer [deftest is]]
+ [environ.core :refer [env]]
+ [nl.surf.eduhub.validator.service.api :as api]
+ [nl.surf.eduhub.validator.service.config :as config]
+ [nl.surf.eduhub.validator.service.config-test :as config-test]))
+
+(def test-config
+ (first (config/load-config-from-env (merge config-test/default-env env))))
+
+(def app (api/compose-app test-config false))
+
+(defn- response-match [actual req]
+ (is (= actual
+ (-> (app req)
+ (select-keys [:status :body]) ; don't test headers
+ (update :body json/read-str))))) ; json easier to test after parsing
+
+(deftest test-validate-correct
+ (with-redefs [http/request (fn [_] {:status 200
+ :body (json/write-str (assoc-in {} [:gateway :endpoints :google.com :responseCode] 200))})]
+ (response-match {:status 200 :body {:valid true}}
+ {:uri "/endpoints/google.com/config" :request-method :post})))
+
+(deftest test-validate-failed-endpoint
+ (with-redefs [http/request (fn [_] {:status 200
+ :body (json/write-str (assoc-in {} [:gateway :endpoints :google.com :responseCode] 500))})]
+ (response-match {:status 200
+ :body {:valid false :message "Endpoint validation failed with status: 500"}}
+ {:uri "/endpoints/google.com/config" :request-method :post})))
+
+(deftest test-unexpected-gateway-status
+ (with-redefs [http/request (fn [_] {:status 500 :body {:message "mocked response"}})]
+ (response-match {:status 500 :body {:message "Unexpected response status received from gateway: 500", :valid false}}
+ {:uri "/endpoints/google.com/config" :request-method :post})))
+
+(deftest test-validate-fails
+ (with-redefs [http/request (fn [_] {:status 401 :body "mocked response"})]
+ (response-match {:status 502 :body {:message "Incorrect credentials for gateway", :valid false}}
+ {:uri "/endpoints/google.com/config" :request-method :post})))
diff --git a/test/nl/surf/eduhub/validator/service/config_test.clj b/test/nl/surf/eduhub/validator/service/config_test.clj
index 847d00f..8d8cd24 100644
--- a/test/nl/surf/eduhub/validator/service/config_test.clj
+++ b/test/nl/surf/eduhub/validator/service/config_test.clj
@@ -26,6 +26,7 @@
:gateway-basic-auth-user "default",
:gateway-basic-auth-pass "default",
:gateway-url "https://gateway.test.surfeduhub.nl/",
+ :max-total-requests "5",
:ooapi-version "default",
:surf-conext-client-id "default",
:surf-conext-client-secret "default",
@@ -38,7 +39,10 @@
:gateway-basic-auth {:pass "default", :user "john200"},
:introspection-basic-auth {:pass "default", :user "default"},
:introspection-endpoint-url "default"
- :server-port 3002})
+ :max-total-requests 5,
+ :server-port 3002
+ :redis-conn {:spec {:uri "redis://localhost"}}
+ :expiry-seconds 1209600})
(defn- test-env [env]
(-> default-env
diff --git a/test/nl/surf/eduhub/validator/service/jobs/client_test.clj b/test/nl/surf/eduhub/validator/service/jobs/client_test.clj
new file mode 100644
index 0000000..2a30e37
--- /dev/null
+++ b/test/nl/surf/eduhub/validator/service/jobs/client_test.clj
@@ -0,0 +1,79 @@
+(ns nl.surf.eduhub.validator.service.jobs.client-test
+ (:require [babashka.http-client :as http]
+ [babashka.json :as json]
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is testing]]
+ [environ.core :refer [env]]
+ [goose.client :as c]
+ [nl.jomco.http-status-codes :as http-status]
+ [nl.surf.eduhub.validator.service.api :as api]
+ [nl.surf.eduhub.validator.service.config :as config]
+ [nl.surf.eduhub.validator.service.config-test :as config-test]
+ [nl.surf.eduhub.validator.service.test-helper :as test-helper]))
+
+(def test-config
+ (first (config/load-config-from-env (merge config-test/default-env env))))
+
+(def app (api/compose-app test-config false))
+
+(defn- make-status-call [uuid]
+ (let [{:keys [status body]}
+ (-> (app {:uri (str "/status/" uuid) :request-method :get})
+ (update :body json/read-str)
+ (select-keys [:status :body]))]
+ (is (= http-status/ok status))
+ body))
+
+(defn- pop-queue! [atm]
+ (let [old-val @atm]
+ (when-not (empty? old-val)
+ (let [item (peek old-val)
+ new-val (pop old-val)]
+ (if (compare-and-set! atm old-val new-val)
+ item
+ (pop-queue! atm))))))
+
+(deftest test-queue
+ (testing "initial call to api"
+ ;; mock c/perform-async
+ (let [jobs-atom (atom [])
+ dirname "test/fixtures/validate_correct"
+ vcr (test-helper/make-playbacker dirname)]
+ (with-redefs [c/perform-async (fn [_job-opts & args]
+ (swap! jobs-atom conj args))]
+ ;; make endpoint call
+ (let [resp (app {:uri "/endpoints/google.com/paths" :request-method :post})]
+ (is (= {:headers {"Content-Type" "application/json; charset=utf-8"}, :status 200}
+ (select-keys resp [:headers :status])))
+ ;; assert status OK
+ (is (= http-status/ok (:status resp)))
+ ;; assert job queued
+ (is (= 1 (count @jobs-atom)))
+ ;; assert json response with uuid
+ (let [{:keys [job-status uuid]} (-> resp :body (json/read-str))]
+ ;; assert job status pending
+ (is (= job-status "pending"))
+ ;; make http request to status
+ (is (= {:job-status "pending" :profile "rio" :endpoint-id "google.com"}
+ (-> (make-status-call uuid)
+ (test-helper/validate-timestamp :pending-at))))
+
+ ;; run the first job in the queue
+ (testing "run worker"
+ ;; mock http/request
+ (with-redefs [http/request (fn wrap-vcr [req] (vcr req))]
+ ;; run worker
+ (let [[fname & args] (pop-queue! jobs-atom)]
+ (apply (resolve fname) (assoc-in (vec args) [2 :config] test-config)))
+
+ (let [body (-> (make-status-call uuid)
+ (test-helper/validate-timestamp :pending-at)
+ (test-helper/validate-timestamp :finished-at))]
+
+ ;; assert status response with status finished and html report
+ (is (= {:job-status "finished" :profile "rio" :endpoint-id "google.com"}
+ (dissoc body :html-report)))
+ (let [html-report (:html-report body)]
+ (is (string? html-report))
+ (when html-report
+ (is (str/includes? html-report "5 observations have no issues")))))))))))))
diff --git a/test/nl/surf/eduhub/validator/service/main_test.clj b/test/nl/surf/eduhub/validator/service/main_test.clj
deleted file mode 100644
index 937ec46..0000000
--- a/test/nl/surf/eduhub/validator/service/main_test.clj
+++ /dev/null
@@ -1,52 +0,0 @@
-;; This file is part of eduhub-validator-service
-;;
-;; Copyright (C) 2024 SURFnet B.V.
-;;
-;; This program is free software: you can redistribute it and/or
-;; modify it under the terms of the GNU Affero General Public License
-;; as published by the Free Software Foundation, either version 3 of
-;; the License, or (at your option) any later version.
-;;
-;; This program is distributed in the hope that it will be useful, but
-;; WITHOUT ANY WARRANTY; without even the implied warranty of
-;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-;; Affero General Public License for more details.
-;;
-;; You should have received a copy of the GNU Affero General Public
-;; License along with this program. If not, see
-;; .
-
-(ns nl.surf.eduhub.validator.service.main-test
- (:require [babashka.http-client :as http]
- [clojure.data.json :as json]
- [clojure.test :refer [deftest is]]
- [nl.surf.eduhub.validator.service.main :as main]))
-
-(def app (main/wrap-validator main/app-routes {:gateway-url "http://gateway.dev.surf.nl"}))
-
-(deftest test-validate-correct
- (with-redefs [http/request (fn [_] {:status 200
- :body (json/write-str (assoc-in {} [:gateway :endpoints :google.com :responseCode] 200))})]
- (is (= {:status 200
- :body {:valid true} }
- (app {:uri "/endpoints/google.com/config" :request-method :get})))))
-
-(deftest test-validate-failed-endpoint
- (with-redefs [http/request (fn [_] {:status 200
- :body (json/write-str (assoc-in {} [:gateway :endpoints :google.com :responseCode] 500))})]
- (is (= {:status 200
- :body {:valid false
- :message "Endpoint validation failed with status: 500"} }
- (app {:uri "/endpoints/google.com/config" :request-method :get})))))
-
-(deftest test-unexpected-gateway-status
- (with-redefs [http/request (fn [_] {:status 500 :body "mocked response"})]
- (is (= {:status 500
- :body {:message "Unexpected response status received from gateway: 500", :valid false}}
- (app {:uri "/endpoints/google.com/config" :request-method :get})))))
-
-(deftest test-validate-fails
- (with-redefs [http/request (fn [_] {:status 401 :body "mocked response"})]
- (is (= {:status 502
- :body {:message "Incorrect credentials for gateway", :valid false}}
- (app {:uri "/endpoints/google.com/config" :request-method :get})))))
diff --git a/test/nl/surf/eduhub/validator/service/test_helper.clj b/test/nl/surf/eduhub/validator/service/test_helper.clj
index 2b68c02..4500f79 100644
--- a/test/nl/surf/eduhub/validator/service/test_helper.clj
+++ b/test/nl/surf/eduhub/validator/service/test_helper.clj
@@ -19,9 +19,17 @@
(ns nl.surf.eduhub.validator.service.test-helper
(:require [clojure.edn :as edn]
[clojure.java.io :as io]
+ [clojure.test :refer [is]]
[clojure.pprint :refer [pprint]])
(:import [java.io PushbackReader]))
+(defn validate-timestamp [m k]
+ (let [ts (k m)]
+ (is (string? ts) (str k " value must be set in " (prn-str m)))
+ (when (string? ts)
+ (is (re-matches #"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z" ts))))
+ (dissoc m k))
+
(defn make-playbacker [dir]
(let [count-atom (atom 0)]
(fn playbacker [_req]
diff --git a/test/nl/surf/eduhub/validator/service/validate_test.clj b/test/nl/surf/eduhub/validator/service/validate_test.clj
index 9868ecf..8587aca 100644
--- a/test/nl/surf/eduhub/validator/service/validate_test.clj
+++ b/test/nl/surf/eduhub/validator/service/validate_test.clj
@@ -37,13 +37,15 @@
(deftest test-validate-correct
(let [dirname "test/fixtures/validate_correct"
http-handler http/request
+ {:keys [gateway-basic-auth gateway-url max-total-requests]} test-config
vcr (if (.exists (io/file dirname))
(test-helper/make-playbacker dirname)
(test-helper/make-recorder dirname http-handler))]
(with-redefs [http/request (fn wrap-vcr [req] (vcr req))]
- (let [opts {:basic-auth (:gateway-basic-auth test-config)
- :base-url (:gateway-url test-config)
- :ooapi-version 5
+ (let [opts {:basic-auth gateway-basic-auth
+ :base-url gateway-url
+ :max-total-requests max-total-requests
+ :ooapi-version 5
:profile "rio"}]
(is (str/includes? (validate/validate-endpoint "demo04.test.surfeduhub.nl" opts)
"5 observations have no issues"))))))