Web UI

deps.edn

@@ -7,11 +7,11 @@
            com.nilenso/goose                           {:mvn/version "0.5.3"}
            com.taoensso/carmine                        {:mvn/version "3.4.1"}
            compojure/compojure                         {:mvn/version "1.7.1"}
+           hiccup/hiccup                               {:mvn/version "2.0.0-RC1"}
            nl.jomco/clj-http-status-codes              {:mvn/version "0.1"}
            nl.jomco/envopts                            {:mvn/version "0.0.4"}
            nl.surf/apie                                {:git/url "https://github.com/SURFnet/apie.git"
-                                                        :git/tag "v0.2.1"
-                                                        :git/sha "d1326bc79884351315c8220e1f71bfc94dee4433"}
+                                                        :git/sha "4f912f114d29e3edf61b4b331416fa2b90cbeaa6"}
            nl.surfnet/eduhub-validator                 {:git/url "https://github.com/SURFnet/eduhub-validator"
                                                         :git/sha "fe0b456e42abe218fe375b1a5f183bee46a6e222"
                                                         :git/branch "main"}

readme.md

@@ -29,6 +29,7 @@ MAX_TOTAL_REQUESTS                  Maximum number of requests that validator is
 OOAPI_VERSION                       Ooapi version to pass through to gateway
 SERVER_PORT                         Starts the app server on this port
 JOB_STATUS_EXPIRY_SECONDS           Number of seconds before job status in Redis expires
+SPIDER_TIMEOUT_MILLIS               Maximum number of milliseconds before spider timeout.
 ```
 
 ## Build

resources/public/javascript/status.js

@@ -0,0 +1,17 @@
+// Define the endpoint and polling interval (in milliseconds)
+const endpoint = `${rootUrl}/status/${validationUuid}`; // Replace with your actual endpoint
+const pollInterval = 2000; // Poll every 5 seconds
+
+// Reload page and stop polling when status no longer pending
+function pollJobStatus() {
+    fetch(endpoint)
+        .then(response => response.json())
+        .then(data => {
+            if (data['job-status'] !== 'pending') {
+                // Stop polling
+                clearInterval(polling);
+                document.location.reload();
+            }
+        })
+        .catch(error => console.error('Error fetching job status:', error));
+}

resources/public/stylesheets/all.css

@@ -0,0 +1,81 @@
+body {
+  font-family: Arial, sans-serif;
+  background-color: #f4f4f4;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  height: 100vh;
+  margin: 0;
+}
+.profile-container {
+  background-color: #fff;
+  border-radius: 8px;
+  padding: 20px;
+  box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
+  max-width: 400px;
+  width: 100%;
+  text-align: center;
+}
+h1 {
+  font-size: 24px;
+  margin-bottom: 10px;
+  color: #333;
+}
+p {
+  font-size: 16px;
+  color: #666;
+  margin: 8px 0;
+}
+/* status styling */
+.status {
+  font-size: 16px;
+  margin: 10px 0;
+}
+.finished { color: green; }
+.failed { color: red; }
+.pending { color: grey; }
+
+/* Style for external links */
+a[target="_blank"]::after {
+  content: " ↗";  /* Unicode symbol for an upward-right arrow */
+  font-size: 0.8em;
+  color: #555;    /* A subtle gray color for the icon */
+  margin-left: 5px;
+}
+
+a[target="_blank"]:hover::after {
+  color: #000;    /* Darken the arrow when hovering over the link */
+}
+.delete-button:hover {
+  background-color: #c0392b;
+}
+.external-link {
+  text-decoration: none;
+  font-size: 16px;
+  color: #3498db;
+}
+a.external-link::after {
+  content: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewbox='0 0 12.2 12.2' width='14' height='14'%3E%3Cpath d='M5.7 0v1.5h4L4.5 6.7l1 1.1 5.3-5.2v3.9h1.4V0z'/%3E%3Cpath fill='none' d='M3.4 6.7l3-2.9H1.5v7h7V5.9l-3 2.9z'/%3E%3Cpath d='M8.5 5.9v4.9h-7v-7h4.9l1.5-1.6H0v10h10V4.4z'/%3E%3C/svg%3E");
+  margin-left: 0.25em;
+}
+.external-link:hover {
+  color: #2980b9;
+}
+.report-button {
+  background-color: #007BFF;
+}
+.button {
+  color: white;
+  border: none;
+  padding: 10px 0px;
+  font-size: 16px;
+  width: 190px;
+  cursor: pointer;
+  border-radius: 5px;
+  margin: 15px 0;
+  display: block;
+  text-decoration: none;
+}
+.delete-button {
+  background-color: #e74c3c;
+}

src/nl/surf/eduhub/validator/service/api.clj

@@ -18,53 +18,99 @@
 
 (ns nl.surf.eduhub.validator.service.api
   (:require [clojure.string :as str]
-            [compojure.core :refer [GET POST defroutes]]
+            [compojure.core :refer [GET POST]]
             [compojure.route :as route]
             [nl.jomco.http-status-codes :as http-status]
             [nl.surf.eduhub.validator.service.authentication :as auth]
             [nl.surf.eduhub.validator.service.checker :as checker]
             [nl.surf.eduhub.validator.service.jobs.client :as jobs-client]
             [nl.surf.eduhub.validator.service.jobs.status :as status]
+            [nl.surf.eduhub.validator.service.views.status :as views.status]
             [ring.middleware.defaults :refer [api-defaults wrap-defaults]]
-            [ring.middleware.json :refer [wrap-json-response]]))
-
-(defroutes app-routes
-           (GET "/status/:uuid" [uuid]
-             {:load-status true, :uuid uuid})
-           (POST "/endpoints/:endpoint-id/config" [endpoint-id]
-             {:checker true :endpoint-id endpoint-id})
-           (POST "/endpoints/:endpoint-id/paths" [endpoint-id profile]
-             {:validator true :endpoint-id endpoint-id :profile profile})
-           (route/not-found "Not Found"))
+            [ring.middleware.json :refer [wrap-json-response]]
+            [ring.middleware.resource :refer [wrap-resource]]))
 
 ;; Many response handlers have the same structure - with this function they can be written inline.
 ;; `activate-handler?` is a function that takes a request and returns a boolean which determines if
 ;; the current handler should be activated (or skipped).
 ;; `response-handler` takes an intermediate response and processes it into the next step.
-(defn wrap-response-handler [app activate-handler? response-handler]
+(defn wrap-response-handler [app action response-handler config]
   (fn [req]
     (let [resp (app req)]
-      (if (activate-handler? resp)
-        (response-handler resp)
+      (if (= action (:action resp))
+        (response-handler (dissoc resp :action) config)
         resp))))
 
 ;; Turn the contents of a job status (stored in redis) into an http response.
-(defn- job-status-handler [{:keys [redis-conn] :as _config}]
-  (fn handle-job-status [resp]
-    (let [job-status (status/load-status redis-conn (:uuid resp))]
-      (if (empty? job-status)
-        {:status http-status/not-found}
-        {:status http-status/ok :body job-status}))))
+(defn- job-status-handler [uuid {:keys [redis-conn] :as _config}]
+  (let [job-status (status/load-status redis-conn uuid)]
+    (if (empty? job-status)
+      {:status http-status/not-found}
+      {:status http-status/ok :body (dissoc job-status :html-report)})))
 
-;; Compose the app from the routes and the wrappers. Authentication can be disabled for testing purposes.
-(defn compose-app [{:keys [introspection-endpoint-url introspection-basic-auth allowed-client-ids] :as config} auth-enabled]
-  (let [allowed-client-id-set  (set (str/split allowed-client-ids #","))
-        auth-opts              {:auth-enabled (boolean auth-enabled)}]
-    (-> app-routes
-        (wrap-response-handler :checker    #(checker/check-endpoint (:endpoint-id %) config))
-        (wrap-response-handler :validator  #(jobs-client/enqueue-validation (:endpoint-id %) (:profile %) config))
-        (wrap-response-handler :load-status (job-status-handler config))
-        (auth/wrap-allowed-clients-checker allowed-client-id-set auth-opts)
-        (auth/wrap-authentication introspection-endpoint-url introspection-basic-auth auth-opts)
-        wrap-json-response
+(defn- view-report-handler [uuid {:keys [redis-conn] :as _config} {:keys [download]}]
+  (let [validation (status/load-status redis-conn uuid)]
+    (if (= "finished" (:job-status validation))
+      {:status http-status/ok :body (:html-report validation) :download download}
+      {:status http-status/see-other :headers {"Location" (str "/view/status/" uuid)}})))
+
+(defn- delete-report-handler [uuid {:keys [redis-conn] :as _config}]
+  (status/delete-status redis-conn uuid)
+  {:status http-status/see-other :headers {"Location" (str "/view/status/" uuid)}})
+
+(defn- view-status-handler [uuid {:keys [redis-conn] :as config}]
+  (let [validation (status/load-status redis-conn uuid)]
+    (if validation
+      {:status http-status/ok :body (views.status/render (assoc validation :uuid uuid) config)}
+      {:status http-status/not-found :body (views.status/render-not-found)})))
+
+(defn wrap-html-response [app]
+  (fn html-response [req]
+    (let [resp (app req)]
+      (cond
+        (and (string? (:body resp))
+             (:download resp))
+        (update-in resp [:headers] merge
+                   {"Content-Type" "text/html; charset=UTF-8"
+                    "Content-Disposition" "attachment; filename=\"validation-report.html\""})
+
+        (string? (:body resp))
+        (assoc-in resp [:headers "Content-Type"] "text/html; charset=UTF-8")
+
+        :else resp))))
+
+(defn public-routes [config]
+  (-> (compojure.core/routes
+        (GET "/status/:uuid" [uuid]
+          (job-status-handler uuid config))
+        (GET "/view/report/:uuid" [uuid]
+          (view-report-handler uuid config {:download false}))
+        (GET "/download/report/:uuid" [uuid]
+          (view-report-handler uuid config {:download true}))
+        (GET "/view/status/:uuid" [uuid]
+          (view-status-handler uuid config))
+        (POST "/delete/report/:uuid" [uuid]
+          (delete-report-handler uuid config)))
+      (wrap-resource "public")
+      (wrap-html-response)
+      (wrap-json-response)
+      (wrap-defaults api-defaults)))
+
+(defn private-routes [{:keys [introspection-endpoint-url introspection-basic-auth allowed-client-ids] :as config} auth-disabled]
+  (let [allowed-client-id-set (set (str/split allowed-client-ids #","))
+        auth-opts             {:auth-disabled (boolean auth-disabled)}]
+    (-> (compojure.core/routes
+          (POST "/endpoints/:endpoint-id/config" [endpoint-id]
+            (checker/check-endpoint endpoint-id config))
+          (POST "/endpoints/:endpoint-id/paths" [endpoint-id profile]
+            (jobs-client/enqueue-validation endpoint-id profile config)))
+        (auth/wrap-authentication introspection-endpoint-url introspection-basic-auth allowed-client-id-set auth-opts)
+        (wrap-json-response)
         (wrap-defaults api-defaults))))
+
+;; Compose the app from the routes and the wrappers. Authentication can be disabled for testing purposes.
+(defn compose-app [config auth-disabled]
+  (compojure.core/routes
+    (public-routes config)
+    (private-routes config auth-disabled)
+    (route/not-found "Not Found")))

src/nl/surf/eduhub/validator/service/authentication.clj

@@ -40,8 +40,7 @@
             [cheshire.core :as json]
             [clojure.core.memoize :as memo]
             [clojure.tools.logging :as log]
-            [nl.jomco.http-status-codes :as http-status]
-            [ring.util.response :as response]))
+            [nl.jomco.http-status-codes :as http-status]))
 
 (defn bearer-token
   [{{:strs [authorization]} :headers}]
@@ -89,15 +88,6 @@
                         token
                         auth)))
 
-(defn- handle-request-with-token [request request-handler client-id]
-  (if (nil? client-id)
-    (response/status http-status/forbidden)
-    ;; set client-id on request and response (for tracing)
-    (-> request
-        (assoc :client-id client-id)
-        request-handler
-        (assoc :client-id client-id))))
-
 (defn wrap-authentication
   "Authenticate calls to ring handler `f` using `token-authenticator`.
 
@@ -110,20 +100,12 @@
 
   If no bearer token is provided, the request is executed without a client-id."
   ; auth looks like {:user client-id :pass client-secret}
-  [f introspection-endpoint auth {:keys [auth-enabled]}]
+  [app introspection-endpoint auth allowed-client-id-set {:keys [auth-disabled]}]
   (let [authenticator (memo/ttl (make-token-authenticator introspection-endpoint auth) :ttl/threshold 60000)] ; 1 minute
     (fn authentication [request]
-      (if auth-enabled
-        (if-let [token (bearer-token request)]
-          (handle-request-with-token request f (authenticator token))
-          (f request))
-        (f request)))))
-
-(defn wrap-allowed-clients-checker [f allowed-client-id-set {:keys [auth-enabled]}]
-  {:pre [(set? allowed-client-id-set)]}
-  (fn allowed-clients-checker [{:keys [client-id] :as request}]
-    (if (or (not auth-enabled)
-            (and client-id (allowed-client-id-set client-id)))
-      (f request)
-      {:body (if client-id "Unknown client id" "No client-id found")
-       :status http-status/forbidden})))
+      (let [client-id (some-> request bearer-token authenticator)]
+        (if (or auth-disabled
+                (allowed-client-id-set client-id))
+          (app request)
+          {:body   (if client-id "Unknown client id" "No client-id found")
+           :status http-status/forbidden})))))

src/nl/surf/eduhub/validator/service/checker.clj

@@ -29,7 +29,7 @@
 (defn check-endpoint [endpoint-id config]
   (try
     (let [{:keys [status body]} (validate/check-endpoint endpoint-id config)]
-      ;; If the client credentials for the validator are incorrect, the wrap-allowed-clients-checker
+      ;; If the client credentials for the validator are incorrect, the wrap-authentication
       ;; middleware has already returned 401 forbidden and execution doesn't get here.
       (handle-check-endpoint-response status body endpoint-id))
     (catch Throwable e

src/nl/surf/eduhub/validator/service/config.clj

@@ -46,12 +46,17 @@
    :job-status-expiry-seconds          ["Number of seconds before job status in Redis expires" :int
                                         :default (* 3600 24 14)
                                         :in [:expiry-seconds]]
+   :validator-service-root-url         ["Root url for the web view; does not include path" :str
+                                        :in [:root-url]]
    :ooapi-version                      ["Ooapi version to pass through to gateway" :str
-                                        :in [:ooapi-version]]})
+                                        :in [:ooapi-version]]
+   :spider-timeout-millis              ["Maximum number of milliseconds before spider timeout." :int
+                                        :default 3600000
+                                        :in [:spider-timeout-millis]]})
 
 (defn- file-secret-loader-reducer [env-map value-key]
   (let [file-key (keyword (str (name value-key) "-file"))
-        path (file-key env-map)]
+        path     (file-key env-map)]
     (cond
       (nil? path)
       env-map

src/nl/surf/eduhub/validator/service/jobs/client.clj

@@ -26,4 +26,4 @@
               :profile            prof}]
     (status/set-status-fields redis-conn uuid "pending" {:endpoint-id endpoint-id, :profile prof} nil)
     (c/perform-async client-opts `worker/validate-endpoint endpoint-id uuid opts)
-    {:status 200 :body {:job-status "pending" :uuid uuid}}))
+    {:status 200 :body {:job-status "pending" :uuid uuid, :web-url (str "http://localhost:3002/view/status/" uuid)}}))

src/nl/surf/eduhub/validator/service/jobs/status.clj

@@ -25,4 +25,7 @@
 (defn load-status [redis-conn uuid]
   (let [result (car/wcar redis-conn (car/hgetall (status-key uuid)))]
     (when-not (empty? result)
-      (into {} (apply hash-map result)))))
+      (update-keys (apply hash-map result) keyword))))
+
+(defn delete-status [redis-conn uuid]
+  (car/wcar redis-conn (car/del (status-key uuid))))

src/nl/surf/eduhub/validator/service/main.clj

@@ -27,7 +27,9 @@
 
 ;; Starts a Jetty server on given port.
 (defn start-server [routes {:keys [server-port] :as _config}]
-  (let [server (run-jetty routes {:port server-port :join? false})
+  (let [server  (-> routes
+
+                    (run-jetty {:port server-port :join? false}))
         handler ^Runnable (fn [] (.stop server))]
     ;; Add a shutdown hook to stop Jetty on JVM exit (Ctrl+C)
     (.addShutdownHook (Runtime/getRuntime)
@@ -45,4 +47,4 @@
                                (let [job-args (assoc-in (vec (:args job)) [2 :config] config)
                                      new-job (assoc job :args job-args)]
                                  (app opts new-job))))))
-    (start-server (api/compose-app config :auth-enabled) config)))
+    (start-server (api/compose-app config (not :auth-disabled)) config)))