diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a367766..da0fb47 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,11 @@ jobs: - name: Install clj runtime run: .github/workflows/install-binaries.sh + - name: Start Redis + uses: supercharge/redis-github-action@1.4.0 + with: + redis-version: 6.2 + - name: Run tests env: GATEWAY_URL: https://gateway.test.surfeduhub.nl/ diff --git a/.nvd-suppressions.xml b/.nvd-suppressions.xml new file mode 100644 index 0000000..c31f715 --- /dev/null +++ b/.nvd-suppressions.xml @@ -0,0 +1,13 @@ + + + + + 5.0 + CVE-2023-46120 + + + + diff --git a/deps.edn b/deps.edn index 68c66fe..134a778 100644 --- a/deps.edn +++ b/deps.edn @@ -4,6 +4,8 @@ ch.qos.logback/logback-classic {:mvn/version "1.5.8"} cheshire/cheshire {:mvn/version "5.13.0"} com.fasterxml.jackson.core/jackson-databind {:mvn/version "2.17.2"} + com.nilenso/goose {:mvn/version "0.5.3"} + com.taoensso/carmine {:mvn/version "3.4.1"} compojure/compojure {:mvn/version "1.7.1"} nl.jomco/clj-http-status-codes {:mvn/version "0.1"} nl.jomco/envopts {:mvn/version "0.0.4"} diff --git a/readme.md b/readme.md index 0c61bae..8618923 100644 --- a/readme.md +++ b/readme.md @@ -25,8 +25,10 @@ SURF_CONEXT_CLIENT_ID SurfCONEXT client id for validation service SURF_CONEXT_CLIENT_SECRET SurfCONEXT client secret for validation service SURF_CONEXT_INTROSPECTION_ENDPOINT SurfCONEXT introspection endpoint ALLOWED_CLIENT_IDS Comma separated list of allowed SurfCONEXT client ids. +MAX_TOTAL_REQUESTS Maximum number of requests that validator is allowed to make before raising an error OOAPI_VERSION Ooapi version to pass through to gateway SERVER_PORT Starts the app server on this port +JOB_STATUS_EXPIRY_SECONDS Number of seconds before job status in Redis expires ``` ## Build diff --git a/src/nl/surf/eduhub/validator/service/api.clj b/src/nl/surf/eduhub/validator/service/api.clj new file mode 100644 index 0000000..0b81744 --- /dev/null +++ b/src/nl/surf/eduhub/validator/service/api.clj @@ -0,0 +1,70 @@ +;; This file is part of eduhub-validator-service +;; +;; Copyright (C) 2024 SURFnet B.V. +;; +;; This program is free software: you can redistribute it and/or +;; modify it under the terms of the GNU Affero General Public License +;; as published by the Free Software Foundation, either version 3 of +;; the License, or (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, but +;; WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;; Affero General Public License for more details. +;; +;; You should have received a copy of the GNU Affero General Public +;; License along with this program. If not, see +;; . + +(ns nl.surf.eduhub.validator.service.api + (:require [clojure.string :as str] + [compojure.core :refer [GET POST defroutes]] + [compojure.route :as route] + [nl.jomco.http-status-codes :as http-status] + [nl.surf.eduhub.validator.service.authentication :as auth] + [nl.surf.eduhub.validator.service.checker :as checker] + [nl.surf.eduhub.validator.service.jobs.client :as jobs-client] + [nl.surf.eduhub.validator.service.jobs.status :as status] + [ring.middleware.defaults :refer [api-defaults wrap-defaults]] + [ring.middleware.json :refer [wrap-json-response]])) + +(defroutes app-routes + (GET "/status/:uuid" [uuid] + {:load-status true, :uuid uuid}) + (POST "/endpoints/:endpoint-id/config" [endpoint-id] + {:checker true :endpoint-id endpoint-id}) + (POST "/endpoints/:endpoint-id/paths" [endpoint-id profile] + {:validator true :endpoint-id endpoint-id :profile profile}) + (route/not-found "Not Found")) + +;; Many response handlers have the same structure - with this function they can be written inline. +;; `activate-handler?` is a function that takes a request and returns a boolean which determines if +;; the current handler should be activated (or skipped). +;; `response-handler` takes an intermediate response and processes it into the next step. +(defn wrap-response-handler [app activate-handler? response-handler] + (fn [req] + (let [resp (app req)] + (if (activate-handler? resp) + (response-handler resp) + resp)))) + +;; Turn the contents of a job status (stored in redis) into an http response. +(defn- job-status-handler [{:keys [redis-conn] :as _config}] + (fn handle-job-status [resp] + (let [job-status (status/load-status redis-conn (:uuid resp))] + (if (empty? job-status) + {:status http-status/not-found} + {:status http-status/ok :body job-status})))) + +;; Compose the app from the routes and the wrappers. Authentication can be disabled for testing purposes. +(defn compose-app [{:keys [introspection-endpoint-url introspection-basic-auth allowed-client-ids] :as config} auth-enabled] + (let [allowed-client-id-set (set (str/split allowed-client-ids #",")) + auth-opts {:auth-enabled (boolean auth-enabled)}] + (-> app-routes + (wrap-response-handler :checker #(checker/check-endpoint (:endpoint-id %) config)) + (wrap-response-handler :validator #(jobs-client/enqueue-validation (:endpoint-id %) (:profile %) config)) + (wrap-response-handler :load-status (job-status-handler config)) + (auth/wrap-allowed-clients-checker allowed-client-id-set auth-opts) + (auth/wrap-authentication introspection-endpoint-url introspection-basic-auth auth-opts) + wrap-json-response + (wrap-defaults api-defaults)))) diff --git a/src/nl/surf/eduhub/validator/service/authentication.clj b/src/nl/surf/eduhub/validator/service/authentication.clj index 7b8cb42..7a1971b 100644 --- a/src/nl/surf/eduhub/validator/service/authentication.clj +++ b/src/nl/surf/eduhub/validator/service/authentication.clj @@ -110,17 +110,20 @@ If no bearer token is provided, the request is executed without a client-id." ; auth looks like {:user client-id :pass client-secret} - [f introspection-endpoint auth] + [f introspection-endpoint auth {:keys [auth-enabled]}] (let [authenticator (memo/ttl (make-token-authenticator introspection-endpoint auth) :ttl/threshold 60000)] ; 1 minute (fn authentication [request] - (if-let [token (bearer-token request)] - (handle-request-with-token request f (authenticator token)) + (if auth-enabled + (if-let [token (bearer-token request)] + (handle-request-with-token request f (authenticator token)) + (f request)) (f request))))) -(defn wrap-allowed-clients-checker [f allowed-client-id-set] +(defn wrap-allowed-clients-checker [f allowed-client-id-set {:keys [auth-enabled]}] {:pre [(set? allowed-client-id-set)]} (fn allowed-clients-checker [{:keys [client-id] :as request}] - (if (and client-id (allowed-client-id-set client-id)) + (if (or (not auth-enabled) + (and client-id (allowed-client-id-set client-id))) (f request) {:body (if client-id "Unknown client id" "No client-id found") :status http-status/forbidden}))) diff --git a/src/nl/surf/eduhub/validator/service/checker.clj b/src/nl/surf/eduhub/validator/service/checker.clj new file mode 100644 index 0000000..1183aed --- /dev/null +++ b/src/nl/surf/eduhub/validator/service/checker.clj @@ -0,0 +1,47 @@ +(ns nl.surf.eduhub.validator.service.checker + (:require [babashka.json :as json] + [clojure.tools.logging :as log] + [nl.jomco.http-status-codes :as http-status] + [nl.surf.eduhub.validator.service.validate :as validate])) + +(defn- handle-check-endpoint-response [status body endpoint-id] + (condp = status + ;; If the validator doesn't have the right credentials for the gateway, manifested by a 401 response, + ;; we'll return a 502 error and log it. + http-status/unauthorized + {:status http-status/bad-gateway :body {:valid false + :message "Incorrect credentials for gateway"}} + + ;; If the gateway returns OK we assume we've gotten a json envelope response and check the response status + ;; of the endpoint. + http-status/ok + (let [envelope (json/read-str body) + envelope-status (get-in envelope [:gateway :endpoints (keyword endpoint-id) :responseCode])] + {:status http-status/ok + :body (if (= "200" (str envelope-status)) + {:valid true} + ;; If the endpoint denied the gateway's request or otherwise returned a response outside the 200 range + ;; we return 200 ok and return the status of the endpoint and the message of the gateway in our response + {:valid false + :message (str "Endpoint validation failed with status: " envelope-status)})}) + + ;; If the gateway returns something else than a 200 or a 401, treat it similar to an error + (let [error-msg (str "Unexpected response status received from gateway: " status)] + (log/error error-msg) + {:status http-status/internal-server-error + :body {:valid false + :message error-msg}}))) + +;; The endpoint checker from phase 1. This connects to an endpoint via the gateway and checks if it receives +;; a valid response. +(defn check-endpoint [endpoint-id config] + (try + (let [{:keys [status body]} (validate/check-endpoint endpoint-id config)] + ;; If the client credentials for the validator are incorrect, the wrap-allowed-clients-checker + ;; middleware has already returned 401 forbidden and execution doesn't get here. + (handle-check-endpoint-response status body endpoint-id)) + (catch Throwable e + (log/error e "Internal error in validator-service") + {:status http-status/internal-server-error + :body {:valid false + :message "Internal error in validator-service"}}))) diff --git a/src/nl/surf/eduhub/validator/service/config.clj b/src/nl/surf/eduhub/validator/service/config.clj index 596cb77..e5fc3ea 100644 --- a/src/nl/surf/eduhub/validator/service/config.clj +++ b/src/nl/surf/eduhub/validator/service/config.clj @@ -30,13 +30,22 @@ :in [:gateway-basic-auth :pass]] :allowed-client-ids ["Comma separated list of allowed SurfCONEXT client ids." :str :in [:allowed-client-ids]] + :max-total-requests ["Maximum number of requests that validator is allowed to make before raising an error" :int + :default 10000 + :in [:max-total-requests]] :surf-conext-client-id ["SurfCONEXT client id for validation service" :str :in [:introspection-basic-auth :user]] :surf-conext-client-secret ["SurfCONEXT client secret for validation service" :str :in [:introspection-basic-auth :pass]] :surf-conext-introspection-endpoint ["SurfCONEXT introspection endpoint" :str :in [:introspection-endpoint-url]] + :redis-uri ["URI to redis" :str + :default "redis://localhost" + :in [:redis-conn :spec :uri]] :server-port ["Starts the app server on this port" :int] + :job-status-expiry-seconds ["Number of seconds before job status in Redis expires" :int + :default (* 3600 24 14) + :in [:expiry-seconds]] :ooapi-version ["Ooapi version to pass through to gateway" :str :in [:ooapi-version]]}) @@ -68,3 +77,13 @@ (defn load-config-from-env [env-map] (-> (reduce file-secret-loader-reducer env-map env-keys-with-alternate-file-secret) (envopts/opts opt-specs))) + +(defn validate-and-load-config [env] + (let [[config errs] (load-config-from-env env)] + (when errs + (.println *err* "Error in environment configuration") + (.println *err* (envopts/errs-description errs)) + (.println *err* "Available environment vars:") + (.println *err* (envopts/specs-description opt-specs)) + (System/exit 1)) + config)) diff --git a/src/nl/surf/eduhub/validator/service/jobs/client.clj b/src/nl/surf/eduhub/validator/service/jobs/client.clj new file mode 100644 index 0000000..6935956 --- /dev/null +++ b/src/nl/surf/eduhub/validator/service/jobs/client.clj @@ -0,0 +1,29 @@ +(ns nl.surf.eduhub.validator.service.jobs.client + (:require [clojure.tools.logging :as log] + [goose.brokers.redis.broker :as broker] + [goose.client :as c] + [goose.retry :as retry] + [nl.surf.eduhub.validator.service.jobs.status :as status] + [nl.surf.eduhub.validator.service.jobs.worker :as worker]) + (:import [java.util UUID])) + +(defn job-error-handler [_cfg _job ex] + (log/error ex "Error in job")) + +(def client-opts (assoc c/default-opts + :broker (broker/new-producer broker/default-opts) + :retry-opts (assoc retry/default-opts :error-handler-fn-sym `job-error-handler))) + +;; Enqueue the validate-endpoint call in the worker queue. +(defn enqueue-validation + [endpoint-id profile {:keys [redis-conn gateway-basic-auth gateway-url ooapi-version max-total-requests] :as _config}] + (let [uuid (str (UUID/randomUUID)) + prof (or profile "rio") + opts {:basic-auth gateway-basic-auth + :base-url gateway-url + :max-total-requests max-total-requests + :ooapi-version ooapi-version + :profile prof}] + (status/set-status-fields redis-conn uuid "pending" {:endpoint-id endpoint-id, :profile prof} nil) + (c/perform-async client-opts `worker/validate-endpoint endpoint-id uuid opts) + {:status 200 :body {:job-status "pending" :uuid uuid}})) diff --git a/src/nl/surf/eduhub/validator/service/jobs/status.clj b/src/nl/surf/eduhub/validator/service/jobs/status.clj new file mode 100644 index 0000000..ad3021b --- /dev/null +++ b/src/nl/surf/eduhub/validator/service/jobs/status.clj @@ -0,0 +1,28 @@ +(ns nl.surf.eduhub.validator.service.jobs.status + (:require [taoensso.carmine :as car]) + (:import [java.time Instant])) + +(defn status-key [uuid] (str "validation:" uuid)) + +(def job-status "job-status") + +;; Updates the status of the job status entry of job `uuid` to `new-status`. +;; Also sets a timestamp named after the new status (e.g. finished-at) +;; Also sets any values in `key-value-map` in the redis hash. +;; If `expires-in-seconds` is set, (re)sets the expiry. +(defn set-status-fields [redis-conn uuid new-status key-value-map expires-in-seconds] + (let [v (assoc key-value-map + job-status new-status + (str new-status "-at") (-> (System/currentTimeMillis) + Instant/ofEpochMilli + str)) + key (status-key uuid)] + (car/wcar redis-conn (car/hmset* key v)) + (when expires-in-seconds + (car/wcar redis-conn (car/expire key expires-in-seconds))))) + +;; Loads the job status as a clojure map for the job with given uuid. +(defn load-status [redis-conn uuid] + (let [result (car/wcar redis-conn (car/hgetall (status-key uuid)))] + (when-not (empty? result) + (into {} (apply hash-map result))))) diff --git a/src/nl/surf/eduhub/validator/service/jobs/worker.clj b/src/nl/surf/eduhub/validator/service/jobs/worker.clj new file mode 100644 index 0000000..4c307fb --- /dev/null +++ b/src/nl/surf/eduhub/validator/service/jobs/worker.clj @@ -0,0 +1,20 @@ +(ns nl.surf.eduhub.validator.service.jobs.worker + (:require [clojure.tools.logging :as log] + [nl.surf.eduhub.validator.service.jobs.status :as status] + [nl.surf.eduhub.validator.service.validate :as validate])) + +;; A worker thread running in the background +;; Called by the workers. Runs the validate-endpoint function +;; and updates the values in the job status. +;; opts should contain: basic-auth ooapi-version base-url profile +(defn validate-endpoint [endpoint-id uuid {:keys [config] :as opts}] + (let [{:keys [redis-conn expiry-seconds]} config] + (assert redis-conn) + (try + (let [html (validate/validate-endpoint endpoint-id opts)] + ;; assuming everything went ok, save html in status, update status and set expiry to value configured in ENV + (status/set-status-fields redis-conn uuid "finished" {"html-report" html} expiry-seconds)) + (catch Throwable ex + ;; otherwise set status to error, include error message and also set expiry + (log/error ex "Validate endpoint threw an exception") + (status/set-status-fields redis-conn uuid "failed" {"error" (str ex)} expiry-seconds))))) diff --git a/src/nl/surf/eduhub/validator/service/main.clj b/src/nl/surf/eduhub/validator/service/main.clj index 4fd9a0f..6c29bfe 100644 --- a/src/nl/surf/eduhub/validator/service/main.clj +++ b/src/nl/surf/eduhub/validator/service/main.clj @@ -18,70 +18,14 @@ (ns nl.surf.eduhub.validator.service.main (:gen-class) - (:require [babashka.json :as json] - [clojure.string :as str] - [clojure.tools.logging :as log] - [compojure.core :refer [GET defroutes]] - [compojure.route :as route] - [environ.core :refer [env]] - [nl.jomco.envopts :as envopts] - [nl.jomco.http-status-codes :as http-status] - [nl.surf.eduhub.validator.service.authentication :as auth] + (:require [environ.core :refer [env]] + [goose.brokers.redis.broker :as broker] + [goose.worker :as w] + [nl.surf.eduhub.validator.service.api :as api] [nl.surf.eduhub.validator.service.config :as config] - [nl.surf.eduhub.validator.service.validate :as validate] - [ring.adapter.jetty :refer [run-jetty]] - [ring.middleware.defaults :refer [api-defaults wrap-defaults]] - [ring.middleware.json :refer [wrap-json-response]])) - -(defn check-endpoint [endpoint-id config] - (try - (let [{:keys [status body]} (validate/check-endpoint endpoint-id config)] - ;; If the client credentials for the validator are incorrect, the wrap-allowed-clients-checker - ;; middleware has already returned 401 forbidden and execution doesn't get here. - (condp = status - ;; If the validator doesn't have the right credentials for the gateway, manifested by a 401 response, - ;; we'll return a 502 error and log it. - http-status/unauthorized - {:status http-status/bad-gateway :body {:valid false - :message "Incorrect credentials for gateway"}} - - ;; If the gateway returns OK we assume we've gotten a json envelope response and check the response status - ;; of the endpoint. - http-status/ok - (let [envelope (json/read-str body) - envelope-status (get-in envelope [:gateway :endpoints (keyword endpoint-id) :responseCode])] - {:status http-status/ok - :body (if (= "200" (str envelope-status)) - {:valid true} - ;; If the endpoint denied the gateway's request or otherwise returned a response outside the 200 range - ;; we return 200 ok and return the status of the endpoint and the message of the gateway in our response - {:valid false - :message (str "Endpoint validation failed with status: " envelope-status)})}) - - ;; If the gateway returns something else than a 200 or a 401, treat it similar to an error - (do - (log/error "Unexpected response status received from gateway: " status) - {:status http-status/internal-server-error - :body {:valid false - :message (str "Unexpected response status received from gateway: " status)}}))) - (catch Throwable e - (log/error e "Exception in validate-endpoint") - {:status http-status/internal-server-error - :body {:valid false - :message "Internal error in validator-service"}}))) - -(defroutes app-routes - (GET "/endpoints/:endpoint-id/config" [endpoint-id] - (fn [_] {:validator true :endpoint-id endpoint-id})) - (route/not-found "Not Found")) - -(defn wrap-validator [app config] - (fn [req] - (let [{:keys [validator endpoint-id] :as resp} (app req)] - (if validator - (check-endpoint endpoint-id config) - resp)))) + [ring.adapter.jetty :refer [run-jetty]])) +;; Starts a Jetty server on given port. (defn start-server [routes {:keys [server-port] :as _config}] (let [server (run-jetty routes {:port server-port :join? false}) handler ^Runnable (fn [] (.stop server))] @@ -91,20 +35,14 @@ server)) (defn -main [& _] - (let [[config errs] (config/load-config-from-env env)] - (when errs - (.println *err* "Error in environment configuration") - (.println *err* (envopts/errs-description errs)) - (.println *err* "Available environment vars:") - (.println *err* (envopts/specs-description config/opt-specs)) - (System/exit 1)) - (let [introspection-endpoint (:introspection-endpoint-url config) - introspection-auth (:introspection-basic-auth config) - allowed-client-id-set (set (str/split (:allowed-client-ids config) #","))] - (start-server (-> app-routes - (wrap-validator config) - (auth/wrap-allowed-clients-checker allowed-client-id-set) - (auth/wrap-authentication introspection-endpoint introspection-auth) - wrap-json-response - (wrap-defaults api-defaults)) - config)))) + (let [config (config/validate-and-load-config env)] + ;; set config as global var (read-only) so that the workers can access it + (w/start (assoc w/default-opts + :broker (broker/new-consumer broker/default-opts) + :middlewares (fn [app] + (fn [opts job] + ;; adds config to map at the last of the args in job + (let [job-args (assoc-in (vec (:args job)) [2 :config] config) + new-job (assoc job :args job-args)] + (app opts new-job)))))) + (start-server (api/compose-app config :auth-enabled) config))) diff --git a/src/nl/surf/eduhub/validator/service/validate.clj b/src/nl/surf/eduhub/validator/service/validate.clj index 0740d39..d744b3e 100644 --- a/src/nl/surf/eduhub/validator/service/validate.clj +++ b/src/nl/surf/eduhub/validator/service/validate.clj @@ -22,9 +22,11 @@ [nl.jomco.apie.main :as apie]) (:import [java.io File])) +;; Validates whether the endpoint is working and reachable at all. (defn check-endpoint "Performs a synchronous validation via the eduhub-validator" [endpoint-id {:keys [gateway-url gateway-basic-auth ooapi-version] :as _config}] + {:pre [gateway-url]} (let [url (str gateway-url (if (.endsWith gateway-url "/") "" "/") "courses") opts {:headers {"x-route" (str "endpoint=" endpoint-id) "accept" (str "application/json; version=" ooapi-version) @@ -33,23 +35,19 @@ :throw false}] (http/get url opts))) - -(defn- temp-file [fname ext] - (let [tmpfile (File/createTempFile fname ext)] - (.deleteOnExit tmpfile) - tmpfile)) - +;; Uses the ooapi validator to validate an endpoint. +;; Returns the generated HTML report. (defn validate-endpoint "Returns the HTML validation report as a String." - [endpoint-id {:keys [basic-auth ooapi-version base-url profile] :as opts}] + [endpoint-id {:keys [basic-auth ooapi-version max-total-requests base-url profile] :as opts}] {:pre [endpoint-id basic-auth ooapi-version base-url profile]} - (let [report-file (temp-file "report" ".html") + (let [report-file (File/createTempFile "report" ".html") report-path (.getAbsolutePath report-file) - observations-file (temp-file "observations" ".edn") + observations-file (File/createTempFile "observations" ".edn") observations-path (.getAbsolutePath observations-file) defaults {:bearer-token nil, :no-report? false, - :max-total-requests 5, + :max-total-requests max-total-requests, :report-path report-path, :headers {:x-route (str "endpoint=" endpoint-id), :accept (str "application/json; version=" ooapi-version), diff --git a/test/nl/surf/eduhub/validator/service/authentication_test.clj b/test/nl/surf/eduhub/validator/service/authentication_test.clj index 5f6e3e6..467410b 100644 --- a/test/nl/surf/eduhub/validator/service/authentication_test.clj +++ b/test/nl/surf/eduhub/validator/service/authentication_test.clj @@ -63,8 +63,8 @@ (let [body {:client (:client-id req)}] {:status http-status/ok :body body})) - (authentication/wrap-allowed-clients-checker allowed-client-id-set) - (authentication/wrap-authentication introspection-endpoint basic-auth))) + (authentication/wrap-allowed-clients-checker allowed-client-id-set {:auth-enabled true}) + (authentication/wrap-authentication introspection-endpoint basic-auth {:auth-enabled true}))) (deftest token-validator ;; This binds the *dynamic* http client in clj-http.client diff --git a/test/nl/surf/eduhub/validator/service/checker_test.clj b/test/nl/surf/eduhub/validator/service/checker_test.clj new file mode 100644 index 0000000..e2fdb6b --- /dev/null +++ b/test/nl/surf/eduhub/validator/service/checker_test.clj @@ -0,0 +1,60 @@ +;; This file is part of eduhub-validator-service +;; +;; Copyright (C) 2024 SURFnet B.V. +;; +;; This program is free software: you can redistribute it and/or +;; modify it under the terms of the GNU Affero General Public License +;; as published by the Free Software Foundation, either version 3 of +;; the License, or (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, but +;; WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;; Affero General Public License for more details. +;; +;; You should have received a copy of the GNU Affero General Public +;; License along with this program. If not, see +;; . + +(ns nl.surf.eduhub.validator.service.checker-test + (:require [babashka.http-client :as http] + [babashka.json :as json] + [clojure.test :refer [deftest is]] + [environ.core :refer [env]] + [nl.surf.eduhub.validator.service.api :as api] + [nl.surf.eduhub.validator.service.config :as config] + [nl.surf.eduhub.validator.service.config-test :as config-test])) + +(def test-config + (first (config/load-config-from-env (merge config-test/default-env env)))) + +(def app (api/compose-app test-config false)) + +(defn- response-match [actual req] + (is (= actual + (-> (app req) + (select-keys [:status :body]) ; don't test headers + (update :body json/read-str))))) ; json easier to test after parsing + +(deftest test-validate-correct + (with-redefs [http/request (fn [_] {:status 200 + :body (json/write-str (assoc-in {} [:gateway :endpoints :google.com :responseCode] 200))})] + (response-match {:status 200 :body {:valid true}} + {:uri "/endpoints/google.com/config" :request-method :post}))) + +(deftest test-validate-failed-endpoint + (with-redefs [http/request (fn [_] {:status 200 + :body (json/write-str (assoc-in {} [:gateway :endpoints :google.com :responseCode] 500))})] + (response-match {:status 200 + :body {:valid false :message "Endpoint validation failed with status: 500"}} + {:uri "/endpoints/google.com/config" :request-method :post}))) + +(deftest test-unexpected-gateway-status + (with-redefs [http/request (fn [_] {:status 500 :body {:message "mocked response"}})] + (response-match {:status 500 :body {:message "Unexpected response status received from gateway: 500", :valid false}} + {:uri "/endpoints/google.com/config" :request-method :post}))) + +(deftest test-validate-fails + (with-redefs [http/request (fn [_] {:status 401 :body "mocked response"})] + (response-match {:status 502 :body {:message "Incorrect credentials for gateway", :valid false}} + {:uri "/endpoints/google.com/config" :request-method :post}))) diff --git a/test/nl/surf/eduhub/validator/service/config_test.clj b/test/nl/surf/eduhub/validator/service/config_test.clj index 847d00f..8d8cd24 100644 --- a/test/nl/surf/eduhub/validator/service/config_test.clj +++ b/test/nl/surf/eduhub/validator/service/config_test.clj @@ -26,6 +26,7 @@ :gateway-basic-auth-user "default", :gateway-basic-auth-pass "default", :gateway-url "https://gateway.test.surfeduhub.nl/", + :max-total-requests "5", :ooapi-version "default", :surf-conext-client-id "default", :surf-conext-client-secret "default", @@ -38,7 +39,10 @@ :gateway-basic-auth {:pass "default", :user "john200"}, :introspection-basic-auth {:pass "default", :user "default"}, :introspection-endpoint-url "default" - :server-port 3002}) + :max-total-requests 5, + :server-port 3002 + :redis-conn {:spec {:uri "redis://localhost"}} + :expiry-seconds 1209600}) (defn- test-env [env] (-> default-env diff --git a/test/nl/surf/eduhub/validator/service/jobs/client_test.clj b/test/nl/surf/eduhub/validator/service/jobs/client_test.clj new file mode 100644 index 0000000..2a30e37 --- /dev/null +++ b/test/nl/surf/eduhub/validator/service/jobs/client_test.clj @@ -0,0 +1,79 @@ +(ns nl.surf.eduhub.validator.service.jobs.client-test + (:require [babashka.http-client :as http] + [babashka.json :as json] + [clojure.string :as str] + [clojure.test :refer [deftest is testing]] + [environ.core :refer [env]] + [goose.client :as c] + [nl.jomco.http-status-codes :as http-status] + [nl.surf.eduhub.validator.service.api :as api] + [nl.surf.eduhub.validator.service.config :as config] + [nl.surf.eduhub.validator.service.config-test :as config-test] + [nl.surf.eduhub.validator.service.test-helper :as test-helper])) + +(def test-config + (first (config/load-config-from-env (merge config-test/default-env env)))) + +(def app (api/compose-app test-config false)) + +(defn- make-status-call [uuid] + (let [{:keys [status body]} + (-> (app {:uri (str "/status/" uuid) :request-method :get}) + (update :body json/read-str) + (select-keys [:status :body]))] + (is (= http-status/ok status)) + body)) + +(defn- pop-queue! [atm] + (let [old-val @atm] + (when-not (empty? old-val) + (let [item (peek old-val) + new-val (pop old-val)] + (if (compare-and-set! atm old-val new-val) + item + (pop-queue! atm)))))) + +(deftest test-queue + (testing "initial call to api" + ;; mock c/perform-async + (let [jobs-atom (atom []) + dirname "test/fixtures/validate_correct" + vcr (test-helper/make-playbacker dirname)] + (with-redefs [c/perform-async (fn [_job-opts & args] + (swap! jobs-atom conj args))] + ;; make endpoint call + (let [resp (app {:uri "/endpoints/google.com/paths" :request-method :post})] + (is (= {:headers {"Content-Type" "application/json; charset=utf-8"}, :status 200} + (select-keys resp [:headers :status]))) + ;; assert status OK + (is (= http-status/ok (:status resp))) + ;; assert job queued + (is (= 1 (count @jobs-atom))) + ;; assert json response with uuid + (let [{:keys [job-status uuid]} (-> resp :body (json/read-str))] + ;; assert job status pending + (is (= job-status "pending")) + ;; make http request to status + (is (= {:job-status "pending" :profile "rio" :endpoint-id "google.com"} + (-> (make-status-call uuid) + (test-helper/validate-timestamp :pending-at)))) + + ;; run the first job in the queue + (testing "run worker" + ;; mock http/request + (with-redefs [http/request (fn wrap-vcr [req] (vcr req))] + ;; run worker + (let [[fname & args] (pop-queue! jobs-atom)] + (apply (resolve fname) (assoc-in (vec args) [2 :config] test-config))) + + (let [body (-> (make-status-call uuid) + (test-helper/validate-timestamp :pending-at) + (test-helper/validate-timestamp :finished-at))] + + ;; assert status response with status finished and html report + (is (= {:job-status "finished" :profile "rio" :endpoint-id "google.com"} + (dissoc body :html-report))) + (let [html-report (:html-report body)] + (is (string? html-report)) + (when html-report + (is (str/includes? html-report "5 observations have no issues"))))))))))))) diff --git a/test/nl/surf/eduhub/validator/service/main_test.clj b/test/nl/surf/eduhub/validator/service/main_test.clj deleted file mode 100644 index 937ec46..0000000 --- a/test/nl/surf/eduhub/validator/service/main_test.clj +++ /dev/null @@ -1,52 +0,0 @@ -;; This file is part of eduhub-validator-service -;; -;; Copyright (C) 2024 SURFnet B.V. -;; -;; This program is free software: you can redistribute it and/or -;; modify it under the terms of the GNU Affero General Public License -;; as published by the Free Software Foundation, either version 3 of -;; the License, or (at your option) any later version. -;; -;; This program is distributed in the hope that it will be useful, but -;; WITHOUT ANY WARRANTY; without even the implied warranty of -;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -;; Affero General Public License for more details. -;; -;; You should have received a copy of the GNU Affero General Public -;; License along with this program. If not, see -;; . - -(ns nl.surf.eduhub.validator.service.main-test - (:require [babashka.http-client :as http] - [clojure.data.json :as json] - [clojure.test :refer [deftest is]] - [nl.surf.eduhub.validator.service.main :as main])) - -(def app (main/wrap-validator main/app-routes {:gateway-url "http://gateway.dev.surf.nl"})) - -(deftest test-validate-correct - (with-redefs [http/request (fn [_] {:status 200 - :body (json/write-str (assoc-in {} [:gateway :endpoints :google.com :responseCode] 200))})] - (is (= {:status 200 - :body {:valid true} } - (app {:uri "/endpoints/google.com/config" :request-method :get}))))) - -(deftest test-validate-failed-endpoint - (with-redefs [http/request (fn [_] {:status 200 - :body (json/write-str (assoc-in {} [:gateway :endpoints :google.com :responseCode] 500))})] - (is (= {:status 200 - :body {:valid false - :message "Endpoint validation failed with status: 500"} } - (app {:uri "/endpoints/google.com/config" :request-method :get}))))) - -(deftest test-unexpected-gateway-status - (with-redefs [http/request (fn [_] {:status 500 :body "mocked response"})] - (is (= {:status 500 - :body {:message "Unexpected response status received from gateway: 500", :valid false}} - (app {:uri "/endpoints/google.com/config" :request-method :get}))))) - -(deftest test-validate-fails - (with-redefs [http/request (fn [_] {:status 401 :body "mocked response"})] - (is (= {:status 502 - :body {:message "Incorrect credentials for gateway", :valid false}} - (app {:uri "/endpoints/google.com/config" :request-method :get}))))) diff --git a/test/nl/surf/eduhub/validator/service/test_helper.clj b/test/nl/surf/eduhub/validator/service/test_helper.clj index 2b68c02..4500f79 100644 --- a/test/nl/surf/eduhub/validator/service/test_helper.clj +++ b/test/nl/surf/eduhub/validator/service/test_helper.clj @@ -19,9 +19,17 @@ (ns nl.surf.eduhub.validator.service.test-helper (:require [clojure.edn :as edn] [clojure.java.io :as io] + [clojure.test :refer [is]] [clojure.pprint :refer [pprint]]) (:import [java.io PushbackReader])) +(defn validate-timestamp [m k] + (let [ts (k m)] + (is (string? ts) (str k " value must be set in " (prn-str m))) + (when (string? ts) + (is (re-matches #"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z" ts)))) + (dissoc m k)) + (defn make-playbacker [dir] (let [count-atom (atom 0)] (fn playbacker [_req] diff --git a/test/nl/surf/eduhub/validator/service/validate_test.clj b/test/nl/surf/eduhub/validator/service/validate_test.clj index 9868ecf..8587aca 100644 --- a/test/nl/surf/eduhub/validator/service/validate_test.clj +++ b/test/nl/surf/eduhub/validator/service/validate_test.clj @@ -37,13 +37,15 @@ (deftest test-validate-correct (let [dirname "test/fixtures/validate_correct" http-handler http/request + {:keys [gateway-basic-auth gateway-url max-total-requests]} test-config vcr (if (.exists (io/file dirname)) (test-helper/make-playbacker dirname) (test-helper/make-recorder dirname http-handler))] (with-redefs [http/request (fn wrap-vcr [req] (vcr req))] - (let [opts {:basic-auth (:gateway-basic-auth test-config) - :base-url (:gateway-url test-config) - :ooapi-version 5 + (let [opts {:basic-auth gateway-basic-auth + :base-url gateway-url + :max-total-requests max-total-requests + :ooapi-version 5 :profile "rio"}] (is (str/includes? (validate/validate-endpoint "demo04.test.surfeduhub.nl" opts) "5 observations have no issues"))))))