From dcd4b49eb41aa89291de9aa540d87666380cf423 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jan 2024 18:04:53 +0000 Subject: [PATCH 01/19] Bump follow-redirects from 1.14.8 to 1.15.4 in /frontend Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.14.8 to 1.15.4. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.14.8...v1.15.4) --- updated-dependencies: - dependency-name: follow-redirects dependency-type: indirect ... Signed-off-by: dependabot[bot] --- frontend/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/yarn.lock b/frontend/yarn.lock index a55f0e9c..e5cf1e4e 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -7065,12 +7065,12 @@ __metadata: linkType: hard "follow-redirects@npm:^1.0.0": - version: 1.14.8 - resolution: "follow-redirects@npm:1.14.8" + version: 1.15.4 + resolution: "follow-redirects@npm:1.15.4" peerDependenciesMeta: debug: optional: true - checksum: 40c67899c2e3149a27e8b6498a338ff27f39fe138fde8d7f0756cb44b073ba0bfec3d52af28f20c5bdd67263d564d0d8d7b5efefd431de95c18c42f7b4aef457 + checksum: e178d1deff8b23d5d24ec3f7a94cde6e47d74d0dc649c35fc9857041267c12ec5d44650a0c5597ef83056ada9ea6ca0c30e7c4f97dbf07d035086be9e6a5b7b6 languageName: node linkType: hard From 3e7fbdcd3403d6a91829157e7029fe48f57bf3cd Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Tue, 30 Jan 2024 10:17:03 +0100 Subject: [PATCH 02/19] Added more documentation and logging to submission handler --- .../rest/handler/submission/SubmissionHandler.kt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/src/main/kotlin/dev/dres/api/rest/handler/submission/SubmissionHandler.kt b/backend/src/main/kotlin/dev/dres/api/rest/handler/submission/SubmissionHandler.kt index e1a3bb42..c5c17a52 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/handler/submission/SubmissionHandler.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/handler/submission/SubmissionHandler.kt @@ -48,7 +48,17 @@ class SubmissionHandler(private val store: TransientEntityStore) : PostRestHandl path = "/api/v2/submit/{evaluationId}", methods = [HttpMethod.POST], operationId = OpenApiOperation.AUTO_GENERATE, - requestBody = OpenApiRequestBody([OpenApiContent(ApiClientSubmission::class)], required = true), + requestBody = OpenApiRequestBody([OpenApiContent(ApiClientSubmission::class)], required = true, + description = + "Some notes regarding the submission format. " + + "At least one answerSet is required, taskId, taskName are inferred if not provided," + + " at least one answer is required, mediaItemCollectionName is inferred if not provided," + + " start and end should be provided in milliseconds." + + "For most evaluation setups, an answer is built in one of the three following ways:" + + " A) only text is required: just provide the text property with a meaningful entry" + + " B) only a mediaItemName is required: just provide the mediaItemName, optionally with the collection name." + + " C) a specific portion of a mediaItem is required: provide mediaItemName, start and end, optionally with collection name" + ), pathParams = [ OpenApiParam( "evaluationId", @@ -99,6 +109,7 @@ class SubmissionHandler(private val store: TransientEntityStore) : PostRestHandl val apiSubmission = try { runManager.postSubmission(rac, apiClientSubmission) } catch (e: SubmissionRejectedException) { + logger.info("Submission was rejected by submission filter.") throw ErrorStatusException(412, e.message ?: "Submission rejected by submission filter.", ctx) } catch (e: IllegalRunStateException) { logger.info("Submission was received while run manager was not accepting submissions.") From d31250637a985b78de5cf8e268ee9221f5f6d62c Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Tue, 6 Feb 2024 17:18:45 +0100 Subject: [PATCH 03/19] Removed LegacySubmissionHandler --- .../main/kotlin/dev/dres/api/rest/RestApi.kt | 3 - .../submission/LegacySubmissionHandler.kt | 298 ------------------ 2 files changed, 301 deletions(-) delete mode 100644 backend/src/main/kotlin/dev/dres/api/rest/handler/submission/LegacySubmissionHandler.kt diff --git a/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt b/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt index b14d85bc..847b2b6a 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt @@ -21,7 +21,6 @@ import dev.dres.api.rest.handler.log.ResultLogHandler import dev.dres.api.rest.handler.preview.* import dev.dres.api.rest.handler.template.* import dev.dres.api.rest.handler.scores.ListEvaluationScoreHandler -import dev.dres.api.rest.handler.submission.LegacySubmissionHandler import dev.dres.api.rest.handler.submission.SubmissionHandler import dev.dres.api.rest.handler.system.CurrentTimeHandler import dev.dres.api.rest.handler.system.InfoHandler @@ -51,7 +50,6 @@ import org.eclipse.jetty.server.* import org.eclipse.jetty.util.thread.QueuedThreadPool import org.slf4j.LoggerFactory import org.slf4j.MarkerFactory -import kotlin.math.min /** * This is a singleton instance of the RESTful API @@ -154,7 +152,6 @@ object RestApi { GetTeamLogoHandler(), // Submission - LegacySubmissionHandler(store, cache), SubmissionHandler(store), // Log diff --git a/backend/src/main/kotlin/dev/dres/api/rest/handler/submission/LegacySubmissionHandler.kt b/backend/src/main/kotlin/dev/dres/api/rest/handler/submission/LegacySubmissionHandler.kt deleted file mode 100644 index c20958c9..00000000 --- a/backend/src/main/kotlin/dev/dres/api/rest/handler/submission/LegacySubmissionHandler.kt +++ /dev/null @@ -1,298 +0,0 @@ -package dev.dres.api.rest.handler.submission - -import dev.dres.api.rest.AccessManager -import dev.dres.api.rest.types.users.ApiRole -import dev.dres.api.rest.handler.AccessManagedRestHandler -import dev.dres.api.rest.handler.GetRestHandler -import dev.dres.api.rest.types.evaluation.submission.* -import dev.dres.api.rest.types.status.ErrorStatus -import dev.dres.api.rest.types.status.ErrorStatusException -import dev.dres.api.rest.types.status.SuccessfulSubmissionsStatus -import dev.dres.api.rest.types.template.tasks.options.ApiSubmissionOption -import dev.dres.api.rest.types.template.tasks.options.ApiTaskOption -import dev.dres.data.model.admin.UserId -import dev.dres.data.model.template.task.options.DbTaskOption -import dev.dres.data.model.media.* -import dev.dres.data.model.media.time.TemporalPoint -import dev.dres.data.model.run.RunActionContext -import dev.dres.data.model.run.RunActionContext.Companion.runActionContext -import dev.dres.data.model.submissions.* -import dev.dres.mgmt.cache.CacheManager -import dev.dres.run.InteractiveRunManager -import dev.dres.run.audit.AuditLogSource -import dev.dres.run.audit.AuditLogger -import dev.dres.run.exceptions.IllegalRunStateException -import dev.dres.run.exceptions.IllegalTeamIdException -import dev.dres.run.filter.SubmissionRejectedException -import dev.dres.utilities.extensions.sessionToken -import io.javalin.http.Context -import io.javalin.openapi.* -import jetbrains.exodus.database.TransientEntityStore -import kotlinx.dnq.query.* -import org.slf4j.LoggerFactory - -/** - * An [GetRestHandler] used to process [DbSubmission]s. - * - * This endpoint strictly considers [DbSubmission]s to contain single [DbAnswerSet]s. - * - * @author Luca Rossetto - * @author Loris Sauter - * @version 2.0.0 - */ -class LegacySubmissionHandler(private val store: TransientEntityStore, private val cache: CacheManager) : - GetRestHandler, AccessManagedRestHandler { - - /** [LegacySubmissionHandler] requires [ApiRole.PARTICIPANT]. */ - override val permittedRoles = setOf(ApiRole.PARTICIPANT) - - /** All [LegacySubmissionHandler]s are part of the v1 API. */ - override val apiVersion = "v1" - - override val route = "submit" - - private val logger = LoggerFactory.getLogger(this.javaClass) - - companion object { - const val PARAMETER_NAME_COLLECTION = "collection" - const val PARAMETER_NAME_ITEM = "item" - const val PARAMETER_NAME_SHOT = "shot" - const val PARAMETER_NAME_FRAME = "frame" - const val PARAMETER_NAME_TIMECODE = "timecode" - const val PARAMETER_NAME_TEXT = "text" - } - - @OpenApi( - summary = "Endpoint to accept submissions", - path = "/api/v1/submit", - operationId = OpenApiOperation.AUTO_GENERATE, - queryParams = [ - OpenApiParam( - PARAMETER_NAME_COLLECTION, - String::class, - "Collection identifier. Optional, in which case the default collection for the run will be considered.", - allowEmptyValue = true - ), - OpenApiParam(PARAMETER_NAME_ITEM, String::class, "Identifier for the actual media object or media file."), - OpenApiParam( - PARAMETER_NAME_TEXT, - String::class, - "Text to be submitted. ONLY for tasks with target type TEXT. If this parameter is provided, it superseeds all athers.", - allowEmptyValue = true, - required = false - ), - OpenApiParam( - PARAMETER_NAME_FRAME, - Int::class, - "Frame number for media with temporal progression (e.g., video).", - allowEmptyValue = true, - required = false - ), - OpenApiParam( - PARAMETER_NAME_SHOT, - Int::class, - "Shot number for media with temporal progression (e.g., video).", - allowEmptyValue = true, - required = false - ), - OpenApiParam( - PARAMETER_NAME_TIMECODE, - String::class, - "Timecode for media with temporal progression (e.g,. video).", - allowEmptyValue = true, - required = false - ), - OpenApiParam("session", String::class, "Session Token") - ], - tags = ["Submission"], - responses = [ - OpenApiResponse("200", [OpenApiContent(SuccessfulSubmissionsStatus::class)]), - OpenApiResponse("202", [OpenApiContent(SuccessfulSubmissionsStatus::class)]), - OpenApiResponse("400", [OpenApiContent(ErrorStatus::class)]), - OpenApiResponse("401", [OpenApiContent(ErrorStatus::class)]), - OpenApiResponse("404", [OpenApiContent(ErrorStatus::class)]), - OpenApiResponse("412", [OpenApiContent(ErrorStatus::class)]) - ], - methods = [HttpMethod.GET], - deprecated = true, - description = "This has been the submission endpoint for version 1. Please refrain from using it and migrate to the v2 endpoint." - ) - override fun doGet(ctx: Context): SuccessfulSubmissionsStatus { - /* Obtain run action context and parse submission. */ - val rac = ctx.runActionContext() - val (run, submission) = this.store.transactional(true) { - val run = getEligibleRunManager(rac, ctx) - run to toSubmission(rac, run, ctx) - } - - /* Post submission. */ - val apiSubmission = try { - run.postSubmission(rac, submission) - } catch (e: SubmissionRejectedException) { - throw ErrorStatusException(412, e.message ?: "Submission rejected by submission filter.", ctx) - } catch (e: IllegalRunStateException) { - logger.info("Submission was received while run manager was not accepting submissions.") - throw ErrorStatusException(400, "Run manager is in wrong state and cannot accept any more submission.", ctx) - } catch (e: IllegalTeamIdException) { - logger.info("Submission with unknown team id '${submission.teamId}' was received.") - throw ErrorStatusException(400, "Run manager does not know the given teamId ${submission.teamId}.", ctx) - } finally { - AuditLogger.submission( - submission, - rac.evaluationId!!, - AuditLogSource.REST, - ctx.sessionToken(), - ctx.ip() - ) - } - - /* Lookup verdict for submission and return it. */ - return when (apiSubmission.answers.first().status) { - ApiVerdictStatus.CORRECT -> SuccessfulSubmissionsStatus(ApiVerdictStatus.CORRECT, "Submission correct!") - ApiVerdictStatus.WRONG -> SuccessfulSubmissionsStatus( - ApiVerdictStatus.WRONG, - "Submission incorrect! Try again" - ) - - ApiVerdictStatus.INDETERMINATE -> { - ctx.status(202) /* HTTP Accepted. */ - SuccessfulSubmissionsStatus(ApiVerdictStatus.INDETERMINATE, "Submission received. Waiting for verdict!") - } - - ApiVerdictStatus.UNDECIDABLE -> SuccessfulSubmissionsStatus( - ApiVerdictStatus.UNDECIDABLE, - "Submission undecidable. Try again!" - ) - - else -> throw ErrorStatusException(500, "Unsupported submission status. This is very unusual!", ctx) - } - - } - - /** - * Returns the [InteractiveRunManager] that is eligible for the given [UserId] and [Context] - * - * @param rac The [RunActionContext] used for the lookup. - * @param ctx The current [Context]. - */ - private fun getEligibleRunManager(rac: RunActionContext, ctx: Context): InteractiveRunManager { - val managers = - AccessManager.getRunManagerForUser(rac.userId).filterIsInstance(InteractiveRunManager::class.java).filter { - it.currentTask(rac)?.isRunning == true - } - if (managers.isEmpty()) throw ErrorStatusException( - 404, - "There is currently no eligible evaluation with an active task.", - ctx - ) - if (managers.size > 1) throw ErrorStatusException( - 409, - "More than one possible evaluation found: ${managers.joinToString { it.template.name }}", - ctx - ) - return managers.first() - } - - /** - * Converts the user request tu a [ApiClientSubmission]. - * - * @param rac The [RunActionContext] used for the conversion. - * @param runManager The [InteractiveRunManager] - * @param ctx The HTTP [Context] - */ - private fun toSubmission( - rac: RunActionContext, - runManager: InteractiveRunManager, - ctx: Context - ): ApiClientSubmission { - val map = ctx.queryParamMap() - - /* If text is supplied, it supersedes other parameters */ - val textParam = map[PARAMETER_NAME_TEXT]?.first() - val itemParam = map[PARAMETER_NAME_ITEM]?.first() - - val answer = if (textParam != null) { - ApiClientAnswer(text = textParam) - } else if (itemParam != null) { - val collection = map[PARAMETER_NAME_COLLECTION]?.first().let { name -> DbMediaCollection.filter { it.name eq name }.firstOrNull()?.id } ?: - runManager.currentTaskTemplate(rac).collectionId - val item = DbMediaItem.filter { (it.name eq itemParam) and (it.collection.id eq collection) }.singleOrNull() - ?: throw ErrorStatusException(404, "Item '$itemParam' not found'", ctx) - val range: Pair? = when { - map.containsKey(PARAMETER_NAME_SHOT) && item.type == DbMediaType.VIDEO -> { - val shot = map[PARAMETER_NAME_SHOT]?.first()!! - val time = item.segments.filter { it.name eq shot } - .firstOrNull()?.range?.toMilliseconds() - ?: throw ErrorStatusException( - 400, - "Shot '${item.name}.${map[PARAMETER_NAME_SHOT]?.first()!!}' not found.", - ctx - ) - time.first to time.second - } - - map.containsKey(PARAMETER_NAME_FRAME) && item.type == DbMediaType.VIDEO -> { - val fps = item.fps - ?: throw IllegalStateException("Missing media item fps information prevented mapping from frame number to milliseconds.") - val time = TemporalPoint.Frame( - map[PARAMETER_NAME_FRAME]?.first()?.toIntOrNull() - ?: throw ErrorStatusException( - 400, - "Parameter '$PARAMETER_NAME_FRAME' must be a number.", - ctx - ), - fps - ) - val ms = time.toMilliseconds() - ms to ms - - } - - map.containsKey(PARAMETER_NAME_TIMECODE) -> { - val fps = item.fps - ?: throw IllegalStateException("Missing media item fps information prevented mapping from frame number to milliseconds.") - val time = - TemporalPoint.Millisecond( - TemporalPoint.Timecode.timeCodeToMilliseconds(map[PARAMETER_NAME_TIMECODE]?.first()!!, fps) - ?: throw ErrorStatusException( - 400, - "'${map[PARAMETER_NAME_TIMECODE]?.first()!!}' is not a valid time code", - ctx - ) - ) - val ms = time.toMilliseconds() - ms to ms - - } - - else -> null - } - - /* Assign information to submission. */ - if (range != null) { - ApiClientAnswer(mediaItemName = itemParam, start = range.first, end = range.second) - } else { - ApiClientAnswer(mediaItemName = itemParam) - } - } else { - throw ErrorStatusException(404, "Required submission parameters are missing (content not set)!", ctx) - } - - /* Generate and return ApiClientSubmission. */ - return ApiClientSubmission(listOf(ApiClientAnswerSet(answers = listOf(answer)))) - } - - /** - * Triggers generation of a preview image for the provided [DbSubmission]. - * - * @param answerSet The [DbAnswerSet] to generate preview for. - */ - private fun generatePreview(answerSet: AnswerSet) { - if (answerSet.answers().firstOrNull()?.type() != AnswerType.TEMPORAL) return - if (answerSet.answers().firstOrNull()?.item == null) return - val item = - DbMediaItem.query((DbMediaItem::id eq answerSet.answers().firstOrNull()?.item!!.mediaItemId)).firstOrNull() - ?: return - this.cache.asyncPreviewImage(item, answerSet.answers().firstOrNull()?.start ?: 0) - } -} From 847daf89873762f5494165d40e5c21e053e07fed Mon Sep 17 00:00:00 2001 From: Luca Rossetto Date: Tue, 6 Feb 2024 17:26:03 +0100 Subject: [PATCH 04/19] Fixed redundant string escapement in ScoreDownloadHandler --- .../handler/download/ScoreDownloadHandler.kt | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/backend/src/main/kotlin/dev/dres/api/rest/handler/download/ScoreDownloadHandler.kt b/backend/src/main/kotlin/dev/dres/api/rest/handler/download/ScoreDownloadHandler.kt index 07505ff3..489b4381 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/handler/download/ScoreDownloadHandler.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/handler/download/ScoreDownloadHandler.kt @@ -18,10 +18,14 @@ import kotlinx.dnq.query.firstOrNull * @author Ralph Gasser * @version 1.0.0 */ -class ScoreDownloadHandler : AbstractDownloadHandler(), GetRestHandler { +class ScoreDownloadHandler : AbstractDownloadHandler(), GetRestHandler { override val route = "download/evaluation/{evaluationId}/scores" + override fun doGet(ctx: Context) { + //nop + } + @OpenApi( summary = "Provides a CSV download with the scores for a given evaluation.", path = "/api/v2/download/evaluation/{evaluationId}/scores", @@ -38,7 +42,7 @@ class ScoreDownloadHandler : AbstractDownloadHandler(), GetRestHandler { ], methods = [HttpMethod.GET] ) - override fun doGet(ctx: Context): String { + override fun get(ctx: Context) { val manager = ctx.eligibleManagerForId() val rac = ctx.runActionContext() @@ -46,14 +50,16 @@ class ScoreDownloadHandler : AbstractDownloadHandler(), GetRestHandler { ctx.contentType("text/csv") ctx.header("Content-Disposition", "attachment; filename=\"scores-${manager.id}.csv\"") - /* Prepare and return response. */ - return "startTime,task,group,team,score\n" + manager.tasks(rac).filter { - it.started != null - }.sortedBy { - it.started - }.flatMap { task -> - task.scorer.scores().map { "${task.started},\"${task.template.name}\",\"${task.template.taskGroup}\",\"${manager.template.teams.firstOrNull { t -> t.id == it.first }?.name ?: "???"}\",${it.third}" } - }.joinToString(separator = "\n") + /* Prepare and send response. */ + ctx.result( + "startTime,task,group,team,score\n" + manager.tasks(rac).filter { + it.started != null + }.sortedBy { + it.started + }.flatMap { task -> + task.scorer.scores() + .map { "${task.started},\"${task.template.name}\",\"${task.template.taskGroup}\",\"${manager.template.teams.firstOrNull { t -> t.id == it.first }?.name ?: "???"}\",${it.third}" } + }.joinToString(separator = "\n") + ) } - } From 0c4988f717354a114954a1d1d9ab8ebb135a8e77 Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Tue, 5 Mar 2024 17:35:49 +0100 Subject: [PATCH 05/19] Fixed #459 by adding support for evaluation-based viewer-roled users, similarly to judge-roled users --- backend/build.gradle | 2 + .../types/template/ApiEvaluationTemplate.kt | 1 + .../model/template/DbEvaluationTemplate.kt | 5 + .../kotlin/dev/dres/mgmt/TemplateManager.kt | 13 +- .../utilities/extensions/ContextExtensions.kt | 14 +- build.gradle | 2 +- doc/oas-client.json | 154 +----------------- doc/oas.json | 154 +----------------- .../template-builder-components.module.ts | 6 +- .../viewers-list/viewers-list.component.html | 41 +++++ .../viewers-list/viewers-list.component.scss | 0 .../viewers-list/viewers-list.component.ts | 94 +++++++++++ .../template-builder.component.html | 5 +- .../template-builder.component.scss | 15 ++ 14 files changed, 207 insertions(+), 299 deletions(-) create mode 100644 frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.html create mode 100644 frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.scss create mode 100644 frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.ts diff --git a/backend/build.gradle b/backend/build.gradle index 03b4e080..425dd016 100644 --- a/backend/build.gradle +++ b/backend/build.gradle @@ -96,6 +96,8 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: version_log4j implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: version_log4j implementation group: 'org.apache.logging.log4j', name: 'log4j-jul', version: version_log4j + implementation 'io.github.microutils:kotlin-logging-jvm:2.0.11' + ////// FastUtil implementation group: 'it.unimi.dsi', name: 'fastutil', version: version_fastutil diff --git a/backend/src/main/kotlin/dev/dres/api/rest/types/template/ApiEvaluationTemplate.kt b/backend/src/main/kotlin/dev/dres/api/rest/types/template/ApiEvaluationTemplate.kt index 14b8b77b..cf0ec5d4 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/types/template/ApiEvaluationTemplate.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/types/template/ApiEvaluationTemplate.kt @@ -32,6 +32,7 @@ data class ApiEvaluationTemplate( val teams: List, val teamGroups: List, val judges: List, + val viewers: List ) { @get:JsonIgnore diff --git a/backend/src/main/kotlin/dev/dres/data/model/template/DbEvaluationTemplate.kt b/backend/src/main/kotlin/dev/dres/data/model/template/DbEvaluationTemplate.kt index 4d1950b0..5701c991 100644 --- a/backend/src/main/kotlin/dev/dres/data/model/template/DbEvaluationTemplate.kt +++ b/backend/src/main/kotlin/dev/dres/data/model/template/DbEvaluationTemplate.kt @@ -64,6 +64,9 @@ class DbEvaluationTemplate(entity: Entity) : PersistentEntity(entity), Evaluatio /** The [DbUser]s that act as judge for this [DbEvaluationTemplate] */ val judges by xdLink0_N(DbUser, onDelete = OnDeletePolicy.CLEAR, onTargetDelete = OnDeletePolicy.CLEAR) + /** The [DbUser]s that can view / spectate this [DbEvaluationTemplate] */ + val viewers by xdLink0_N(DbUser, onDelete = OnDeletePolicy.CLEAR, onTargetDelete = OnDeletePolicy.CLEAR) + /** The [DbTaskTemplate]s contained in this [DbEvaluationTemplate]*/ val tasks by xdChildren0_N(DbTaskTemplate::evaluation) @@ -91,6 +94,7 @@ class DbEvaluationTemplate(entity: Entity) : PersistentEntity(entity), Evaluatio teamGroups = this.teamGroups.asSequence().map { it.toApi() }.toList(), teams = this.teams.asSequence().map { it.toApi() }.toList(), judges = this.judges.asSequence().map { it.id }.toList(), + viewers = this.viewers.asSequence().map{ it.id }.toList(), tasks = this.tasks.sortedBy(DbTaskTemplate::idx).asSequence().map { it.toApi() }.toList(), ) @@ -107,6 +111,7 @@ class DbEvaluationTemplate(entity: Entity) : PersistentEntity(entity), Evaluatio created = this@DbEvaluationTemplate.created modified = this@DbEvaluationTemplate.modified judges.addAll(this@DbEvaluationTemplate.judges) + viewers.addAll(this@DbEvaluationTemplate.viewers) } /* Copy task types. */ diff --git a/backend/src/main/kotlin/dev/dres/mgmt/TemplateManager.kt b/backend/src/main/kotlin/dev/dres/mgmt/TemplateManager.kt index 75b71a19..eb81106c 100644 --- a/backend/src/main/kotlin/dev/dres/mgmt/TemplateManager.kt +++ b/backend/src/main/kotlin/dev/dres/mgmt/TemplateManager.kt @@ -322,11 +322,22 @@ object TemplateManager { dbEvaluationTemplate.judges.removeAll(DbUser.query(not(DbUser::id.containsIn(*judgeIds)))) for (userId in judgeIds) { val user = DbUser.filter { it.id eq userId }.firstOrNull() - ?: throw IllegalArgumentException("Unknown user $userId for evaluation ${apiEvaluationTemplate.id}.") + ?: throw IllegalArgumentException("Unknown judge user $userId for evaluation ${apiEvaluationTemplate.id}.") if (!dbEvaluationTemplate.judges.contains(user)) { dbEvaluationTemplate.judges.add(user) } } + + /* Update viewer information */ + val viewerIds = apiEvaluationTemplate.viewers.toTypedArray() + dbEvaluationTemplate.viewers.removeAll(DbUser.query(not(DbUser::id.containsIn(*viewerIds)))) + for (userId in viewerIds){ + val user = DbUser.filter{it.id eq userId}.firstOrNull() + ?: throw IllegalArgumentException("Unknown viewer user $userId for evaluation ${apiEvaluationTemplate.id}.") + if( !(dbEvaluationTemplate.viewers.contains(user))){ + dbEvaluationTemplate.viewers.add(user) + } + } } /** diff --git a/backend/src/main/kotlin/dev/dres/utilities/extensions/ContextExtensions.kt b/backend/src/main/kotlin/dev/dres/utilities/extensions/ContextExtensions.kt index f3a4da76..67650ea3 100644 --- a/backend/src/main/kotlin/dev/dres/utilities/extensions/ContextExtensions.kt +++ b/backend/src/main/kotlin/dev/dres/utilities/extensions/ContextExtensions.kt @@ -129,6 +129,8 @@ inline fun Context.eligibleManagerForId(): T { val manager = RunExecutor.managerForId(evaluationId) as? T ?: throw ErrorStatusException(404, "Evaluation $evaluationId not found.", this) return if (this.isJudge() && manager.template.judges.contains(userId)) { manager + } else if (this.isViewer() && manager.template.viewers.contains(userId)) { + manager } else if (this.isParticipant() && manager.template.hasParticipant(userId)) { manager } else if (this.isAdmin()) { @@ -168,7 +170,17 @@ fun Context.isJudge(): Boolean { return roles.contains(ApiRole.JUDGE) && !roles.contains(ApiRole.ADMIN) } +/** + * Checks if user associated with current [Context] has [ApiRole.VIEWER]. + * + * @return True if current user has [ApiRole.VIEWER] + */ +fun Context.isViewer(): Boolean { + val roles = AccessManager.rolesOfSession(this.sessionToken()) + return roles.contains(ApiRole.VIEWER) && !roles.contains(ApiRole.ADMIN) +} + /** * Returns the remote IP address of a context, taking forwarding headers into account */ -fun Context.realIP(): String = this.header("X-Forwarded-For") ?: this.ip() \ No newline at end of file +fun Context.realIP(): String = this.header("X-Forwarded-For") ?: this.ip() diff --git a/build.gradle b/build.gradle index 1a43f94c..fb097951 100644 --- a/build.gradle +++ b/build.gradle @@ -32,7 +32,7 @@ openApiGenerate { outputDir = file("${project.projectDir}/frontend/openapi").toString() configOptions = [ npmName: '@dres-openapi/api', - ngVersion: '13.2.3', + ngVersion: '15.2.9', snapshot: 'true', /// I suggest to remove this, as soon as we automate this, enumPropertyNaming: 'original' ] diff --git a/doc/oas-client.json b/doc/oas-client.json index 4083d741..609fc5ad 100644 --- a/doc/oas-client.json +++ b/doc/oas-client.json @@ -6,151 +6,6 @@ "version" : "2.0.0-RC4" }, "paths" : { - "/api/v1/submit" : { - "get" : { - "tags" : [ "Submission" ], - "summary" : "Endpoint to accept submissions", - "description" : "This has been the submission endpoint for version 1. Please refrain from using it and migrate to the v2 endpoint.", - "operationId" : "getApiV1Submit", - "parameters" : [ { - "name" : "collection", - "in" : "query", - "description" : "Collection identifier. Optional, in which case the default collection for the run will be considered.", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : true, - "schema" : { - "type" : "string" - } - }, { - "name" : "item", - "in" : "query", - "description" : "Identifier for the actual media object or media file.", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : false, - "schema" : { - "type" : "string" - } - }, { - "name" : "text", - "in" : "query", - "description" : "Text to be submitted. ONLY for tasks with target type TEXT. If this parameter is provided, it superseeds all athers.", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : true, - "schema" : { - "type" : "string" - } - }, { - "name" : "frame", - "in" : "query", - "description" : "Frame number for media with temporal progression (e.g., video).", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : true, - "schema" : { - "type" : "integer", - "format" : "int32" - } - }, { - "name" : "shot", - "in" : "query", - "description" : "Shot number for media with temporal progression (e.g., video).", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : true, - "schema" : { - "type" : "integer", - "format" : "int32" - } - }, { - "name" : "timecode", - "in" : "query", - "description" : "Timecode for media with temporal progression (e.g,. video).", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : true, - "schema" : { - "type" : "string" - } - }, { - "name" : "session", - "in" : "query", - "description" : "Session Token", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : false, - "schema" : { - "type" : "string" - } - } ], - "responses" : { - "200" : { - "description" : "OK", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/SuccessfulSubmissionsStatus" - } - } - } - }, - "202" : { - "description" : "Accepted", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/SuccessfulSubmissionsStatus" - } - } - } - }, - "400" : { - "description" : "Bad Request", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/ErrorStatus" - } - } - } - }, - "401" : { - "description" : "Unauthorized", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/ErrorStatus" - } - } - } - }, - "404" : { - "description" : "Not Found", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/ErrorStatus" - } - } - } - }, - "412" : { - "description" : "Precondition Failed", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/ErrorStatus" - } - } - } - } - }, - "deprecated" : true, - "security" : [ ] - } - }, "/api/v2/client/evaluation/currentTask/{evaluationId}" : { "get" : { "tags" : [ "Evaluation Client" ], @@ -739,6 +594,7 @@ } } ], "requestBody" : { + "description" : "Some notes regarding the submission format. At least one answerSet is required, taskId, taskName are inferred if not provided, at least one answer is required, mediaItemCollectionName is inferred if not provided, start and end should be provided in milliseconds.For most evaluation setups, an answer is built in one of the three following ways: A) only text is required: just provide the text property with a meaningful entry B) only a mediaItemName is required: just provide the mediaItemName, optionally with the collection name. C) a specific portion of a mediaItem is required: provide mediaItemName, start and end, optionally with collection name", "content" : { "application/json" : { "schema" : { @@ -2107,9 +1963,15 @@ "items" : { "type" : "string" } + }, + "viewers" : { + "type" : "array", + "items" : { + "type" : "string" + } } }, - "required" : [ "id", "name", "taskTypes", "taskGroups", "tasks", "teams", "teamGroups", "judges" ] + "required" : [ "id", "name", "taskTypes", "taskGroups", "tasks", "teams", "teamGroups", "judges", "viewers" ] }, "ApiEvaluationTemplateOverview" : { "type" : "object", diff --git a/doc/oas.json b/doc/oas.json index a43f6c1f..740ce610 100644 --- a/doc/oas.json +++ b/doc/oas.json @@ -13,151 +13,6 @@ "version" : "2.0.0-RC4" }, "paths" : { - "/api/v1/submit" : { - "get" : { - "tags" : [ "Submission" ], - "summary" : "Endpoint to accept submissions", - "description" : "This has been the submission endpoint for version 1. Please refrain from using it and migrate to the v2 endpoint.", - "operationId" : "getApiV1Submit", - "parameters" : [ { - "name" : "collection", - "in" : "query", - "description" : "Collection identifier. Optional, in which case the default collection for the run will be considered.", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : true, - "schema" : { - "type" : "string" - } - }, { - "name" : "item", - "in" : "query", - "description" : "Identifier for the actual media object or media file.", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : false, - "schema" : { - "type" : "string" - } - }, { - "name" : "text", - "in" : "query", - "description" : "Text to be submitted. ONLY for tasks with target type TEXT. If this parameter is provided, it superseeds all athers.", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : true, - "schema" : { - "type" : "string" - } - }, { - "name" : "frame", - "in" : "query", - "description" : "Frame number for media with temporal progression (e.g., video).", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : true, - "schema" : { - "type" : "integer", - "format" : "int32" - } - }, { - "name" : "shot", - "in" : "query", - "description" : "Shot number for media with temporal progression (e.g., video).", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : true, - "schema" : { - "type" : "integer", - "format" : "int32" - } - }, { - "name" : "timecode", - "in" : "query", - "description" : "Timecode for media with temporal progression (e.g,. video).", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : true, - "schema" : { - "type" : "string" - } - }, { - "name" : "session", - "in" : "query", - "description" : "Session Token", - "required" : false, - "deprecated" : false, - "allowEmptyValue" : false, - "schema" : { - "type" : "string" - } - } ], - "responses" : { - "200" : { - "description" : "OK", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/SuccessfulSubmissionsStatus" - } - } - } - }, - "202" : { - "description" : "Accepted", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/SuccessfulSubmissionsStatus" - } - } - } - }, - "400" : { - "description" : "Bad Request", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/ErrorStatus" - } - } - } - }, - "401" : { - "description" : "Unauthorized", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/ErrorStatus" - } - } - } - }, - "404" : { - "description" : "Not Found", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/ErrorStatus" - } - } - } - }, - "412" : { - "description" : "Precondition Failed", - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/ErrorStatus" - } - } - } - } - }, - "deprecated" : true, - "security" : [ ] - } - }, "/api/v2/client/evaluation/currentTask/{evaluationId}" : { "get" : { "tags" : [ "Evaluation Client" ], @@ -4348,6 +4203,7 @@ } } ], "requestBody" : { + "description" : "Some notes regarding the submission format. At least one answerSet is required, taskId, taskName are inferred if not provided, at least one answer is required, mediaItemCollectionName is inferred if not provided, start and end should be provided in milliseconds.For most evaluation setups, an answer is built in one of the three following ways: A) only text is required: just provide the text property with a meaningful entry B) only a mediaItemName is required: just provide the mediaItemName, optionally with the collection name. C) a specific portion of a mediaItem is required: provide mediaItemName, start and end, optionally with collection name", "content" : { "application/json" : { "schema" : { @@ -6521,9 +6377,15 @@ "items" : { "type" : "string" } + }, + "viewers" : { + "type" : "array", + "items" : { + "type" : "string" + } } }, - "required" : [ "id", "name", "taskTypes", "taskGroups", "tasks", "teams", "teamGroups", "judges" ] + "required" : [ "id", "name", "taskTypes", "taskGroups", "tasks", "teams", "teamGroups", "judges", "viewers" ] }, "ApiEvaluationTemplateOverview" : { "type" : "object", diff --git a/frontend/src/app/template/template-builder/components/template-builder-components.module.ts b/frontend/src/app/template/template-builder/components/template-builder-components.module.ts index 0c0459cd..60999034 100644 --- a/frontend/src/app/template/template-builder/components/template-builder-components.module.ts +++ b/frontend/src/app/template/template-builder/components/template-builder-components.module.ts @@ -48,6 +48,7 @@ import { MatDialogModule } from "@angular/material/dialog"; import { ColorPickerModule } from "ngx-color-picker"; import {CdkDrag, CdkDropList} from '@angular/cdk/drag-drop'; import { MatCardModule } from "@angular/material/card"; +import { ViewersListComponent } from './viewers-list/viewers-list.component'; @NgModule({ @@ -67,7 +68,8 @@ import { MatCardModule } from "@angular/material/card"; QueryDescriptionMediaItemVideoFormFieldComponent, QueryDescriptionExternalVideoFormFieldComponent, QueryDescriptionExternalImageFormFieldComponent, - TeamBuilderDialogComponent + TeamBuilderDialogComponent, + ViewersListComponent ], imports: [ CommonModule, @@ -100,7 +102,7 @@ import { MatCardModule } from "@angular/material/card"; TaskTypesListComponent, TaskGroupsListComponent, TaskTemplatesListComponent, - TaskTemplateEditorComponent] + TaskTemplateEditorComponent, ViewersListComponent] }) export class TemplateBuilderComponentsModule { } diff --git a/frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.html b/frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.html new file mode 100644 index 00000000..db5b5122 --- /dev/null +++ b/frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.html @@ -0,0 +1,41 @@ +
+

Viewers

+
+ + + +
+ + Viewer Selection + + + {{user.username}} + + +
+ +
+ + + + + + + + + + + + + + + +
Name{{(userForId(user) | async)?.username}}Action
diff --git a/frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.scss b/frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.scss new file mode 100644 index 00000000..e69de29b diff --git a/frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.ts b/frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.ts new file mode 100644 index 00000000..866a31d0 --- /dev/null +++ b/frontend/src/app/template/template-builder/components/viewers-list/viewers-list.component.ts @@ -0,0 +1,94 @@ +import { Component, OnDestroy, OnInit, ViewChild } from "@angular/core"; +import { AbstractTemplateBuilderComponent } from "../abstract-template-builder.component"; +import { ApiRole, ApiUser, TemplateService, UserService } from "../../../../../../openapi"; +import { MatTable } from "@angular/material/table"; +import { Observable } from "rxjs"; +import { TemplateBuilderService } from "../../template-builder.service"; +import { ActivatedRoute } from "@angular/router"; +import { MatSnackBar } from "@angular/material/snack-bar"; +import { map, shareReplay, tap } from "rxjs/operators"; +import { MatAutocompleteSelectedEvent } from "@angular/material/autocomplete"; + +@Component({ + selector: "app-viewers-list", + templateUrl: "./viewers-list.component.html", + styleUrls: ["./viewers-list.component.scss"] +}) +export class ViewersListComponent extends AbstractTemplateBuilderComponent implements OnInit, OnDestroy { + + /** The table to use for the "list" */ + @ViewChild("table") + table: MatTable; + + /** The users that are available, i.e. other except those in the list */ + availableUsers: Observable; + + /** The columns in the table */ + displayedColumns: string[] = ["name", "action"]; + + /** The initially empty list of users in the list */ + users: Observable> = new Observable>((x) => x.next([])); + + constructor( + private userService: UserService, + builderService: TemplateBuilderService, + route: ActivatedRoute, + templateService: TemplateService, + snackBar: MatSnackBar + ) { + super(builderService, route, templateService, snackBar); + this.refreshAvailableUsers(); + } + + addUser(event: MatAutocompleteSelectedEvent){ + if(this.builderService.getTemplate().viewers.includes(event.option.value.id)){ + // We ignore a possible add when the user is already in the list + return; + } + this.builderService.getTemplate().viewers.push(event.option.value.id); + this.builderService.update(); + this.table.renderRows(); + } + + remove(userId: string){ + this.builderService.getTemplate().viewers.splice(this.builderService.getTemplate().viewers.indexOf(userId),1); + this.builderService.update(); + this.table.renderRows(); + } + + userForId(id: string){ + return this.availableUsers.pipe(map((users) => users.find((u) => u.id === id))) + } + + displayUser(user: ApiUser){ + return user.username + } + + ngOnInit() { + this.onInit() + } + + ngOnDestroy() { + this.onDestroy() + } + + onChange() { + this.users = this.builderService.templateAsObservable().pipe( + map((t) => { + if(t){ + return t.viewers; + }else{ + return []; + } + }), + tap(_ => this.table?.renderRows()) + ) + } + + refreshAvailableUsers(){ + this.availableUsers = this.userService.getApiV2UserList().pipe( + map((users) => users.filter((user) => user.role === ApiRole.VIEWER)), + shareReplay(1) + ) + } +} diff --git a/frontend/src/app/template/template-builder/template-builder.component.html b/frontend/src/app/template/template-builder/template-builder.component.html index 030f85af..4b8ce192 100644 --- a/frontend/src/app/template/template-builder/template-builder.component.html +++ b/frontend/src/app/template/template-builder/template-builder.component.html @@ -50,10 +50,11 @@

Edit evaluation template {{(builderService.templateAsObservable() | async)?. -
+
- + +
diff --git a/frontend/src/app/template/template-builder/template-builder.component.scss b/frontend/src/app/template/template-builder/template-builder.component.scss index b30329d1..4b61fd76 100644 --- a/frontend/src/app/template/template-builder/template-builder.component.scss +++ b/frontend/src/app/template/template-builder/template-builder.component.scss @@ -10,6 +10,10 @@ grid-area: center; } +.content-center-right{ + grid-area: center-right; +} + .tb-container-2col { display: grid; grid-template-columns: 1fr 1fr; @@ -80,3 +84,14 @@ min-width: 0; } } + +.tb-container-2col-2col-1col-1col { + display: grid; + grid-template-columns: 1fr 1fr 1fr 1fr 1fr 1fr; + grid-template-areas: "left left center center center-right right"; + column-gap: 1em; + + > * { + min-width: 0; + } +} From 995843b4e8b10c1e25258010c9fc5995b2ab0538 Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Wed, 6 Mar 2024 08:51:35 +0100 Subject: [PATCH 06/19] Removed weirdly temporarily required dependency --- backend/build.gradle | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/build.gradle b/backend/build.gradle index 425dd016..4a818013 100644 --- a/backend/build.gradle +++ b/backend/build.gradle @@ -90,14 +90,11 @@ dependencies { ////// Jaffree ffmpeg wrapper implementation group: 'com.github.kokorin.jaffree', name: 'jaffree', version: version_jaffree - ////// Log4J implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: version_log4j implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: version_log4j implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: version_log4j implementation group: 'org.apache.logging.log4j', name: 'log4j-jul', version: version_log4j - implementation 'io.github.microutils:kotlin-logging-jvm:2.0.11' - ////// FastUtil implementation group: 'it.unimi.dsi', name: 'fastutil', version: version_fastutil From a7085e84d99c77275bab1b8e9dad944a6f9c667d Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Wed, 6 Mar 2024 12:32:19 +0100 Subject: [PATCH 07/19] Initial migration to javalin6 --- backend/build.gradle | 10 +- backend/src/main/kotlin/dev/dres/DRES.kt | 2 +- .../kotlin/dev/dres/api/rest/AccessManager.kt | 12 +- .../dev/dres/api/rest/ClientOpenApiPlugin.kt | 88 +++++----- .../dev/dres/api/rest/ClientSwaggerPlugin.kt | 40 +++-- .../main/kotlin/dev/dres/api/rest/RestApi.kt | 150 +++++++++--------- build.gradle | 4 +- gradle.properties | 4 +- 8 files changed, 169 insertions(+), 141 deletions(-) diff --git a/backend/build.gradle b/backend/build.gradle index 4a818013..f72c9ad1 100644 --- a/backend/build.gradle +++ b/backend/build.gradle @@ -47,8 +47,6 @@ compileTestKotlin { dependencies { - def javalinOpenapi = '5.6.1' - ///// Frontend files (produced by sub-project). implementation frontendClasspath(project(path: ":frontend", configuration: 'frontendFiles')) @@ -64,11 +62,11 @@ dependencies { ////// Javalin implementation group: 'io.javalin', name: 'javalin', version: version_javalin - kapt("io.javalin.community.openapi:openapi-annotation-processor:$javalinOpenapi") + kapt("io.javalin.community.openapi:openapi-annotation-processor:$version_javalinopenapi") - implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: javalinOpenapi - implementation group: 'io.javalin.community.openapi', name:'javalin-swagger-plugin', version: javalinOpenapi - implementation group: 'io.javalin.community.ssl', name: 'ssl-plugin', version: version_javalin + implementation group: 'io.javalin.community.openapi', name: 'javalin-openapi-plugin', version: version_javalinopenapi + implementation group: 'io.javalin.community.openapi', name:'javalin-swagger-plugin', version: version_javalinopenapi + implementation group: 'io.javalin.community.ssl', name: 'ssl-plugin', version: version_javalinssl ////// Bcrypt implementation group: 'org.mindrot', name: 'jbcrypt', version: version_bcrypt diff --git a/backend/src/main/kotlin/dev/dres/DRES.kt b/backend/src/main/kotlin/dev/dres/DRES.kt index ed9150a0..7356213b 100644 --- a/backend/src/main/kotlin/dev/dres/DRES.kt +++ b/backend/src/main/kotlin/dev/dres/DRES.kt @@ -42,7 +42,7 @@ import kotlin.system.exitProcess */ object DRES { /** Version of DRES. */ - const val VERSION = "2.0.0-RC4" + const val VERSION = "2.0.0-RC5" /** Application root; should be relative to JAR file or classes path. */ val APPLICATION_ROOT: Path = diff --git a/backend/src/main/kotlin/dev/dres/api/rest/AccessManager.kt b/backend/src/main/kotlin/dev/dres/api/rest/AccessManager.kt index 7e926924..784fac71 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/AccessManager.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/AccessManager.kt @@ -39,6 +39,16 @@ object AccessManager { } } + fun hasAccess(ctx: Context): Boolean { + val permittedRoles = ctx.routeRoles() + return when { + permittedRoles.isEmpty() -> true + permittedRoles.contains(ApiRole.ANYONE) -> true + rolesOfSession(ctx.sessionToken()).any { it in permittedRoles } -> true + else -> false + } + } + /** An internal [ConcurrentHashMap] that maps [SessionToken]s to [ApiRole]s. */ private val sessionRoleMap = HashMap>() @@ -155,4 +165,4 @@ object AccessManager { fun getRunManagerForUser(userId: UserId): Set = this.locks.read { return this.usersToRunMap[userId] ?: emptySet() } -} \ No newline at end of file +} diff --git a/backend/src/main/kotlin/dev/dres/api/rest/ClientOpenApiPlugin.kt b/backend/src/main/kotlin/dev/dres/api/rest/ClientOpenApiPlugin.kt index b256ba99..093662d6 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/ClientOpenApiPlugin.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/ClientOpenApiPlugin.kt @@ -3,55 +3,57 @@ package dev.dres.api.rest import com.fasterxml.jackson.databind.node.ObjectNode import dev.dres.DRES import io.javalin.openapi.CookieAuth -import io.javalin.openapi.plugin.OpenApiConfiguration import io.javalin.openapi.plugin.OpenApiPlugin -import io.javalin.openapi.plugin.OpenApiPluginConfiguration import io.javalin.openapi.plugin.SecurityComponentConfiguration -class ClientOpenApiPlugin : OpenApiPlugin(OpenApiPluginConfiguration() - .withDocumentationPath("/client-oas") - .withDefinitionConfiguration { _, u -> - u.withOpenApiInfo { t -> - t.title = "DRES Client API" - t.version = DRES.VERSION - t.description = "Client API for DRES (Distributed Retrieval Evaluation Server), Version ${DRES.VERSION}" - } - u.withSecurity( - SecurityComponentConfiguration() - .withSecurityScheme("CookieAuth", CookieAuth(AccessManager.SESSION_COOKIE_NAME)) - ) - u.withDefinitionProcessor { doc -> - - val blacklist = setOf( - "/external/", - "/collection", - "/run", - "/audit", - "/mediaItem", - "/score", - "/user/list", - "/user/session/", - "/evaluation/admin", - "/evaluation/template", - "/evaluation/{evaluationId}/judge", - "/evaluation/{evaluationId}/vote", - "/evaluation/{evaluationId}/submission", - "/evaluation/{evaluationId}/task", - "/evaluation/{evaluationId}/{taskId}", - "/download", - "/mediaitem", - "/template", - "/preview", - "/status/info" +class ClientOpenApiPlugin : OpenApiPlugin({ + it + .withDocumentationPath("/clientapi.json") + .withDefinitionConfiguration { _, u -> + u.withOpenApiInfo { t -> + t.title = "DRES Client API" + t.version = DRES.VERSION + t.description = + "Client API for DRES (Distributed Retrieval Evaluation Server), Version ${DRES.VERSION}" + } + u.withSecurity( + SecurityComponentConfiguration() + .withSecurityScheme("CookieAuth", CookieAuth(AccessManager.SESSION_COOKIE_NAME)) ) + u.withDefinitionProcessor { doc -> + + val blacklist = setOf( + "/external/", + "/collection", + "/run", + "/audit", + "/mediaItem", + "/score", + "/user/list", + "/user/session/", + "/evaluation/admin", + "/evaluation/template", + "/evaluation/{evaluationId}/judge", + "/evaluation/{evaluationId}/vote", + "/evaluation/{evaluationId}/submission", + "/evaluation/{evaluationId}/task", + "/evaluation/{evaluationId}/{taskId}", + "/download", + "/mediaitem", + "/template", + "/preview", + "/status/info" + ) - val relevantRoutes = - doc["paths"].fields().asSequence().filter { blacklist.none { b -> it.key.contains(b) } }.map { it.key } - .toList() + val relevantRoutes = + doc["paths"].fields().asSequence() + .filter { blacklist.none { b -> it.key.contains(b) } }.map { it.key } + .toList() - (doc["paths"] as ObjectNode).retain(relevantRoutes) + (doc["paths"] as ObjectNode).retain(relevantRoutes) - doc.toPrettyString() + doc.toPrettyString() + } } - }) +}) diff --git a/backend/src/main/kotlin/dev/dres/api/rest/ClientSwaggerPlugin.kt b/backend/src/main/kotlin/dev/dres/api/rest/ClientSwaggerPlugin.kt index fe7bb5a8..a6d7c24f 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/ClientSwaggerPlugin.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/ClientSwaggerPlugin.kt @@ -1,27 +1,37 @@ package dev.dres.api.rest import io.javalin.Javalin +import io.javalin.config.JavalinConfig import io.javalin.openapi.plugin.swagger.SwaggerConfiguration import io.javalin.openapi.plugin.swagger.SwaggerHandler import io.javalin.openapi.plugin.swagger.SwaggerPlugin import io.javalin.plugin.Plugin -class ClientSwaggerPlugin : Plugin { - override fun apply(app: Javalin) { - val swaggerHandler = SwaggerHandler( - title = "DRES Client API", - documentationPath = "/client-oas", - swaggerVersion = SwaggerConfiguration().version, - validatorUrl = "https://validator.swagger.io/validator", - routingPath = app.cfg.routing.contextPath, - basePath = null, - tagsSorter = "'alpha'", - operationsSorter = "'alpha'", - customJavaScriptFiles = emptyList(), - customStylesheetFiles = emptyList() - ) +class ClientSwaggerPlugin : SwaggerPlugin() { + + override fun name(): String { + return this.javaClass.simpleName + } + + override fun onStart(config: JavalinConfig) { +// val swaggerHandler = SwaggerHandler( +// title = "DRES Client API", +// documentationPath = "/client-oas", +// swaggerVersion = SwaggerConfiguration().version, +// validatorUrl = "https://validator.swagger.io/validator", +//// routingPath = app.cfg.routing.contextPath, +// basePath = null, +// tagsSorter = "'alpha'", +// operationsSorter = "'alpha'", +// customJavaScriptFiles = emptyList(), +// customStylesheetFiles = emptyList() +// ) +// +// config.router.apiBuilder { +// get("/swagger-client", swaggerHandler) +// } + - app.get("/swagger-client", swaggerHandler) } } diff --git a/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt b/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt index 847b2b6a..42ce991e 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt @@ -36,14 +36,14 @@ import dev.dres.utilities.NamedThreadFactory import io.javalin.Javalin import io.javalin.apibuilder.ApiBuilder.* import io.javalin.http.staticfiles.Location -import io.javalin.community.ssl.SSLPlugin +import io.javalin.community.ssl.SslPlugin import io.javalin.http.Cookie +import io.javalin.http.HttpStatus import io.javalin.http.SameSite import io.javalin.openapi.CookieAuth import io.javalin.openapi.OpenApiContact import io.javalin.openapi.OpenApiLicense import io.javalin.openapi.plugin.* -import io.javalin.openapi.plugin.swagger.SwaggerConfiguration import io.javalin.openapi.plugin.swagger.SwaggerPlugin import jetbrains.exodus.database.TransientEntityStore import org.eclipse.jetty.server.* @@ -222,73 +222,67 @@ object RestApi { ScoreDownloadHandler() ) - javalin = Javalin.create { + javalin = Javalin.create { it -> it.jsonMapper(KotlinxJsonMapper) - it.plugins.enableCors { cors -> - cors.add { corsPluginConfig -> + it.bundledPlugins.enableCors { cors -> + cors.addRule { corsPluginConfig -> corsPluginConfig.reflectClientOrigin = true // anyHost() has similar implications and might be used in production? I'm not sure how to cope with production and dev here simultaneously corsPluginConfig.allowCredentials = true } } - it.plugins.register( - OpenApiPlugin( - OpenApiPluginConfiguration() - .withDocumentationPath("/swagger-docs") - .withDefinitionConfiguration { _, u -> - u.withOpenApiInfo { t -> - t.title = "DRES API" - t.version = DRES.VERSION - t.description = + it.registerPlugin( + OpenApiPlugin{ oapConfig -> + oapConfig + .withDocumentationPath("/openapi.json") + .withDefinitionConfiguration { version, openApiDef -> + openApiDef + .withInfo { info -> + info.title = "DRES API" + info.version = DRES.VERSION + info.description = "API for DRES (Distributed Retrieval Evaluation Server), Version ${DRES.VERSION}" val contact = OpenApiContact() contact.url = "https://dres.dev" contact.name = "The DRES Dev Team" - t.contact = contact + info.contact = contact val license = OpenApiLicense() license.name = "MIT" - // license.identifier = "MIT" - t.license = license - } - u.withSecurity( + info.license = license + } + + .withSecurity( SecurityComponentConfiguration() .withSecurityScheme("CookieAuth", CookieAuth(AccessManager.SESSION_COOKIE_NAME)) ) } - - - ) - ) - - it.plugins.register(ClientOpenApiPlugin()) - it.plugins.register( - SwaggerPlugin( - SwaggerConfiguration().apply { - //this.version = "4.10.3" - this.documentationPath = "/swagger-docs" - this.uiPath = "/swagger-ui" - } - ) + } ) - it.plugins.register( - ClientSwaggerPlugin() - ) + it.registerPlugin(ClientOpenApiPlugin()) + it.registerPlugin(SwaggerPlugin{ swaggerConfig -> + swaggerConfig.documentationPath = "/openapi.json" + swaggerConfig.uiPath = "/swagger-ui" + }) + it.registerPlugin(SwaggerPlugin{ swaggerConfig -> + swaggerConfig.documentationPath = "/clientapi.json" + swaggerConfig.uiPath = "/swagger-client" + swaggerConfig.title = "Client Swagger UI" + }) it.http.defaultContentType = "application/json" it.http.prefer405over404 = true it.http.maxRequestSize = 20 * 1024 * 1024 //20mb - it.jetty.server { setupHttpServer() } - it.accessManager(AccessManager::manage) + it.jetty.threadPool = pool it.staticFiles.add("html", Location.CLASSPATH) it.spaRoot.addFile("/vote", "vote/index.html") it.spaRoot.addFile("/", "html/index.html") if (config.enableSsl) { - val ssl = SSLPlugin { conf -> + val ssl = SslPlugin { conf -> conf.keystoreFromPath(config.keystorePath, config.keystorePassword) conf.http2 = true conf.secure = true @@ -296,10 +290,54 @@ object RestApi { conf.securePort = config.httpsPort conf.sniHostCheck = false } - it.plugins.register(ssl) + it.registerPlugin(ssl) } + it.router.apiBuilder{ + path("api") { + apiRestHandlers.groupBy { it.apiVersion }.forEach { apiGroup -> + path(apiGroup.key) { + apiGroup.value.forEach { handler -> + path(handler.route) { + val permittedRoles = if (handler is AccessManagedRestHandler) { + handler.permittedRoles.toTypedArray() + } else { + arrayOf(ApiRole.ANYONE) + } + + if (handler is GetRestHandler<*>) { + get(handler::get, *permittedRoles) + } + + if (handler is PostRestHandler<*>) { + post(handler::post, *permittedRoles) + } + + if (handler is PatchRestHandler<*>) { + patch(handler::patch, *permittedRoles) + } + + if (handler is DeleteRestHandler<*>) { + delete(handler::delete, *permittedRoles) + } + + } + } + } + } + //ws("ws/run", runExecutor) + } + } + + }.beforeMatched{ctx -> + /* BeforeMatched handlers are only matched if the request will be matched */ + /* See https://javalin.io/migration-guide-javalin-5-to-6 */ + if(! AccessManager.hasAccess(ctx)){ + ctx.status(HttpStatus.UNAUTHORIZED) + ctx.skipRemainingHandlers() + } }.before { ctx -> + /* Before are matched before every request, including static files */ //check for session cookie val cookieId = ctx.cookie(AccessManager.SESSION_COOKIE_NAME) @@ -332,40 +370,8 @@ object RestApi { ctx.header("Cache-Control", "no-store") } - }.routes { - path("api") { - apiRestHandlers.groupBy { it.apiVersion }.forEach { apiGroup -> - path(apiGroup.key) { - apiGroup.value.forEach { handler -> - path(handler.route) { - val permittedRoles = if (handler is AccessManagedRestHandler) { - handler.permittedRoles.toTypedArray() - } else { - arrayOf(ApiRole.ANYONE) - } - - if (handler is GetRestHandler<*>) { - get(handler::get, *permittedRoles) - } - - if (handler is PostRestHandler<*>) { - post(handler::post, *permittedRoles) - } - - if (handler is PatchRestHandler<*>) { - patch(handler::patch, *permittedRoles) - } - if (handler is DeleteRestHandler<*>) { - delete(handler::delete, *permittedRoles) - } - } - } - } - } - //ws("ws/run", runExecutor) - } }.error(401) { it.json(ErrorStatus("Unauthorized request!")) }.exception(Exception::class.java) { e, ctx -> diff --git a/build.gradle b/build.gradle index fb097951..d6ce0806 100644 --- a/build.gradle +++ b/build.gradle @@ -13,8 +13,8 @@ plugins { } /// Variables used for Open API generation. -def fullOAS = 'http://localhost:8080/swagger-docs' -def clientOAS = 'http://localhost:8080/client-oas' +def fullOAS = 'http://localhost:8080/openapi.json' +def clientOAS = 'http://localhost:8080/clientapi.json' def oasFile = "${project.projectDir}/doc/oas.json" def clientOasFile = "${project.projectDir}/doc/oas-client.json" diff --git a/gradle.properties b/gradle.properties index 86cd4682..4052de18 100644 --- a/gradle.properties +++ b/gradle.properties @@ -12,7 +12,9 @@ version_clikt=4.1.0 version_fastutil=8.5.12 version_fuel=2.3.1 version_jaffree=2022.06.03 -version_javalin=5.6.3 +version_javalin=6.1.3 +version_javalinopenapi=6.1.3 +version_javalinssl=6.1.3 version_jline3=3.22.0 version_junit=5.9.1 version_kotlin=1.8.22 From bf4bd4f660c1dfaff5e92fa5a461f433462b8adc Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Wed, 6 Mar 2024 12:37:23 +0100 Subject: [PATCH 08/19] Regenerated openapi specs with javalin6 and preventing viewers to see non-accessible evaluations --- .../evaluation/viewer/AbstractEvaluationViewerHandler.kt | 2 ++ doc/oas-client.json | 4 ++-- doc/oas.json | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/viewer/AbstractEvaluationViewerHandler.kt b/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/viewer/AbstractEvaluationViewerHandler.kt index dc41195f..e75afe7a 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/viewer/AbstractEvaluationViewerHandler.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/viewer/AbstractEvaluationViewerHandler.kt @@ -10,6 +10,7 @@ import dev.dres.run.RunExecutor import dev.dres.run.RunManager import dev.dres.utilities.extensions.isJudge import dev.dres.utilities.extensions.isParticipant +import dev.dres.utilities.extensions.isViewer import dev.dres.utilities.extensions.userId import io.javalin.http.Context import io.javalin.security.RouteRole @@ -58,6 +59,7 @@ abstract class AbstractEvaluationViewerHandler: RestHandler, AccessManagedRestHa m.template.teams.flatMap { it.users }.any { it.id == ctx.userId() } } ctx.isJudge() -> managers.filter { m -> m.template.judges.any { u -> u == ctx.userId() } } + ctx.isViewer() -> managers.filter { m -> m.template.viewers.any{u -> u == ctx.userId()}} else -> managers } } diff --git a/doc/oas-client.json b/doc/oas-client.json index 609fc5ad..37ca2dc6 100644 --- a/doc/oas-client.json +++ b/doc/oas-client.json @@ -2,8 +2,8 @@ "openapi" : "3.0.3", "info" : { "title" : "DRES Client API", - "description" : "Client API for DRES (Distributed Retrieval Evaluation Server), Version 2.0.0-RC4", - "version" : "2.0.0-RC4" + "version" : "2.0.0-RC5", + "description" : "Client API for DRES (Distributed Retrieval Evaluation Server), Version 2.0.0-RC5" }, "paths" : { "/api/v2/client/evaluation/currentTask/{evaluationId}" : { diff --git a/doc/oas.json b/doc/oas.json index 740ce610..d62691e4 100644 --- a/doc/oas.json +++ b/doc/oas.json @@ -2,15 +2,15 @@ "openapi" : "3.0.3", "info" : { "title" : "DRES API", - "description" : "API for DRES (Distributed Retrieval Evaluation Server), Version 2.0.0-RC4", + "version" : "2.0.0-RC5", + "description" : "API for DRES (Distributed Retrieval Evaluation Server), Version 2.0.0-RC5", "contact" : { "name" : "The DRES Dev Team", "url" : "https://dres.dev" }, "license" : { "name" : "MIT" - }, - "version" : "2.0.0-RC4" + } }, "paths" : { "/api/v2/client/evaluation/currentTask/{evaluationId}" : { From b2a3dfd682b4bf961f45d4c14cc699bc45199271 Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Thu, 7 Mar 2024 15:21:17 +0100 Subject: [PATCH 09/19] Slightly reworked media collection CLI --- .../dres/api/cli/MediaCollectionCommand.kt | 157 ++++++++++++------ .../dev/dres/api/rest/ClientOpenApiPlugin.kt | 2 +- .../main/kotlin/dev/dres/api/rest/RestApi.kt | 55 +++--- .../dev/dres/data/model/config/Config.kt | 2 + .../dev/dres/mgmt/MediaCollectionManager.kt | 6 +- 5 files changed, 149 insertions(+), 73 deletions(-) diff --git a/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt b/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt index 70025e19..8d7e81c6 100644 --- a/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt +++ b/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt @@ -4,6 +4,9 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.NoOpCliktCommand import com.github.ajalt.clikt.core.subcommands +import com.github.ajalt.clikt.parameters.groups.mutuallyExclusiveOptions +import com.github.ajalt.clikt.parameters.groups.required +import com.github.ajalt.clikt.parameters.groups.single import com.github.ajalt.clikt.parameters.options.* import com.github.ajalt.clikt.parameters.types.enum import com.github.ajalt.clikt.parameters.types.float @@ -14,9 +17,7 @@ import com.github.kokorin.jaffree.StreamType import com.github.kokorin.jaffree.ffprobe.FFprobe import com.github.kokorin.jaffree.ffprobe.FFprobeResult import com.jakewharton.picnic.table -import dev.dres.api.rest.types.collection.ApiMediaItem -import dev.dres.api.rest.types.collection.ApiMediaSegment -import dev.dres.api.rest.types.collection.ApiMediaType +import dev.dres.api.rest.types.collection.* import dev.dres.data.model.config.Config import dev.dres.data.model.media.* import dev.dres.mgmt.MediaCollectionManager @@ -69,6 +70,7 @@ class MediaCollectionCommand(private val config: Config) : return mapOf( "ls" to listOf("list"), "remove" to listOf("delete"), + "rm" to listOf("delete"), "drop" to listOf("delete") ) } @@ -84,7 +86,24 @@ class MediaCollectionCommand(private val config: Config) : } /** - * + * Wrapper to both handle [CollectionId] and collection name as a [String]. + */ + sealed class CollectionAddress { + data class IdAddress(val id: CollectionId): CollectionAddress(){ + override fun toString(): String { + return "CollectionAddress(id=$id)" + } + } + data class NameAddress(val name: String): CollectionAddress(){ + override fun toString(): String { + return "CollectionAddress(name=$name)" + } + } + } + + /** + * Base command for media collection related commands. + * Provides infrastructure to get the collection specified by the user */ abstract inner class AbstractCollectionCommand( name: String, @@ -92,8 +111,44 @@ class MediaCollectionCommand(private val config: Config) : ) : CliktCommand(name = name, help = help, printHelpOnEmptyArgs = true) { - /** The [CollectionId] of the [DbMediaCollection] affected by this [AbstractCollectionCommand]. */ - protected val id: CollectionId by option("-i", "--id", help = "ID of a media collection.").required() + /** The [CollectionAddress] of the [DbMediaCollection] affected by this [AbstractCollectionCommand]. */ + protected val collectionAddress: CollectionAddress by mutuallyExclusiveOptions( + option( + "-i", + "--id", + help = "ID of a media collection." + ).convert { CollectionAddress.IdAddress(it) }, + option( + "-c", + "--collection", + help = "The collection name" + ).convert { CollectionAddress.NameAddress(it) } + ).single().required() + + /** + * Resolves the [ApiMediaCollection] which is addressed by the given [CollectionAddress] + */ + protected fun resolve(addr: CollectionAddress): ApiMediaCollection { + val col = when (addr) { + is CollectionAddress.IdAddress -> { + MediaCollectionManager.getCollection(addr.id) + } + + is CollectionAddress.NameAddress -> { + MediaCollectionManager.getCollectionByName(addr.name) + } + } + if (col != null) { + return col + } else { + throw IllegalArgumentException("Could not find media collection for address $addr") + } + } + + protected fun resolvePopulated(addr: CollectionAddress): ApiPopulatedMediaCollection { + return MediaCollectionManager.getPopulatedCollection(resolve(addr).id!!) + ?: throw IllegalArgumentException("Could not find media collection for address $addr") + } } @@ -140,13 +195,13 @@ class MediaCollectionCommand(private val config: Config) : inner class Delete : AbstractCollectionCommand("delete", help = "Deletes a media collection.") { override fun run() { - - if (MediaCollectionManager.deleteCollection(this.id) == null) { - println("Failed to delete collection; specified collection not found.") - } else { + try{ + MediaCollectionManager.deleteCollection(resolve(collectionAddress).id!!) + ?: throw IllegalArgumentException("Could not find media collection for address $collectionAddress") println("Collection deleted successfully.") + }catch(ex: IllegalArgumentException){ + println(ex.message) } - } } @@ -174,11 +229,10 @@ class MediaCollectionCommand(private val config: Config) : ) override fun run() { - - val collection = MediaCollectionManager.getCollection(id) - - if (collection == null) { - println("Failed to update collection; specified collection not found.") + val collection = try { + resolve(collectionAddress) + }catch(ex: IllegalArgumentException){ + println(ex.message) return } @@ -250,9 +304,10 @@ class MediaCollectionCommand(private val config: Config) : override fun run() { /* Find media collection. */ - val collection = MediaCollectionManager.getPopulatedCollection(id) - if (collection == null) { - println("Collection not found.") + val collection = try{ + resolvePopulated(collectionAddress) + }catch(ex: IllegalArgumentException){ + println(ex.message) return } @@ -305,9 +360,10 @@ class MediaCollectionCommand(private val config: Config) : ) { override fun run() { /* Find media collection. */ - val collection = MediaCollectionManager.getPopulatedCollection(id) - if (collection == null) { - println("Collection not found.") + val collection = try{ + resolvePopulated(collectionAddress) + }catch(ex: IllegalArgumentException){ + println(ex.message) return } @@ -373,9 +429,10 @@ class MediaCollectionCommand(private val config: Config) : /* Find media collection and note base path */ - val collection = MediaCollectionManager.getCollection(id) - if (collection == null) { - println("Collection not found.") + val collection = try{ + resolve(collectionAddress) + }catch(ex: IllegalArgumentException){ + println(ex.message) return } @@ -413,7 +470,7 @@ class MediaCollectionCommand(private val config: Config) : type = ApiMediaType.IMAGE, name = it.fileName.nameWithoutExtension, location = relativePath.toString(), - collectionId = id, + collectionId = collection.id!!, mediaItemId = "" //is generated anew anyway ) } @@ -445,7 +502,7 @@ class MediaCollectionCommand(private val config: Config) : location = relativePath.toString(), durationMs = duration, fps = fps, - collectionId = id, + collectionId = collection.id!!, mediaItemId = "" //is generated anew anyway ) } @@ -466,7 +523,7 @@ class MediaCollectionCommand(private val config: Config) : } - MediaCollectionManager.addMediaItems(id, items) + MediaCollectionManager.addMediaItems(collection.id!!, items) } @@ -510,9 +567,10 @@ class MediaCollectionCommand(private val config: Config) : } /* Find media collection. */ - val collection = MediaCollectionManager.getPopulatedCollection(id) - if (collection == null) { - println("Collection not found.") + val collection = try{ + resolvePopulated(collectionAddress) + }catch(ex: IllegalArgumentException){ + println(ex.message) return } @@ -564,9 +622,10 @@ class MediaCollectionCommand(private val config: Config) : override fun run() { /* Find media collection. */ - val collection = MediaCollectionManager.getCollection(id) - if (collection == null) { - println("Collection not found.") + val collection = try{ + resolve(collectionAddress) + }catch(ex: IllegalArgumentException){ + println(ex.message) return } @@ -583,7 +642,7 @@ class MediaCollectionCommand(private val config: Config) : location = this@AddItem.path, durationMs = this@AddItem.duration, fps = this@AddItem.fps, - collectionId = id, + collectionId = collection.id!!, mediaItemId = "" ) @@ -611,9 +670,10 @@ class MediaCollectionCommand(private val config: Config) : override fun run() { /* Find media collection. */ - val collection = MediaCollectionManager.getPopulatedCollection(id) - if (collection == null) { - println("Collection not found.") + val collection = try{ + resolvePopulated(collectionAddress) + }catch(ex: IllegalArgumentException){ + println(ex.message) return } @@ -659,11 +719,13 @@ class MediaCollectionCommand(private val config: Config) : } /* Find media collection. */ - val collection = MediaCollectionManager.getCollection(id) - if (collection == null) { - println("Collection not found.") + val collection = try{ + resolve(collectionAddress) + }catch(ex: IllegalArgumentException){ + println(ex.message) return } + /* Load file. */ Files.newInputStream(this.input, StandardOpenOption.READ).use { ips -> val chunks = csvReader().readAllWithHeader(ips).chunked(transactionChunkSize) @@ -678,12 +740,12 @@ class MediaCollectionCommand(private val config: Config) : location = row.getValue("location"), durationMs = row["duration"]?.toLongOrNull(), fps = row["fps"]?.toFloatOrNull(), - collectionId = id, + collectionId = collection.id!!, mediaItemId = "" ) } - MediaCollectionManager.addMediaItems(id, items) + MediaCollectionManager.addMediaItems(collection.id!!, items) } } @@ -722,9 +784,10 @@ class MediaCollectionCommand(private val config: Config) : /* Find media collection. */ - val collection = MediaCollectionManager.getCollection(id) - if (collection == null) { - println("Collection not found.") + val collection = try{ + resolve(collectionAddress) + }catch(ex: IllegalArgumentException){ + println(ex.message) return } /* Load file. */ @@ -743,7 +806,7 @@ class MediaCollectionCommand(private val config: Config) : .sortedBy { it.mediaItemName } .asSequence().chunked(transactionChunkSize) .forEach { chunk -> - inserted += MediaCollectionManager.addSegments(id, chunk) + inserted += MediaCollectionManager.addSegments(collection.id!!, chunk) } } diff --git a/backend/src/main/kotlin/dev/dres/api/rest/ClientOpenApiPlugin.kt b/backend/src/main/kotlin/dev/dres/api/rest/ClientOpenApiPlugin.kt index 093662d6..0db480ce 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/ClientOpenApiPlugin.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/ClientOpenApiPlugin.kt @@ -10,7 +10,7 @@ class ClientOpenApiPlugin : OpenApiPlugin({ it .withDocumentationPath("/clientapi.json") .withDefinitionConfiguration { _, u -> - u.withOpenApiInfo { t -> + u.withInfo { t -> t.title = "DRES Client API" t.version = DRES.VERSION t.description = diff --git a/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt b/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt index 42ce991e..7856d1d5 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt @@ -1,6 +1,7 @@ package dev.dres.api.rest import GetTaskHintHandler +import com.fasterxml.jackson.databind.node.ObjectNode import dev.dres.DRES import dev.dres.api.rest.handler.* import dev.dres.api.rest.handler.collection.* @@ -235,39 +236,44 @@ object RestApi { } it.registerPlugin( - OpenApiPlugin{ oapConfig -> + OpenApiPlugin { oapConfig -> oapConfig .withDocumentationPath("/openapi.json") .withDefinitionConfiguration { version, openApiDef -> openApiDef .withInfo { info -> - info.title = "DRES API" - info.version = DRES.VERSION - info.description = - "API for DRES (Distributed Retrieval Evaluation Server), Version ${DRES.VERSION}" - val contact = OpenApiContact() - contact.url = "https://dres.dev" - contact.name = "The DRES Dev Team" - info.contact = contact - val license = OpenApiLicense() - license.name = "MIT" - info.license = license - } - - .withSecurity( - SecurityComponentConfiguration() - .withSecurityScheme("CookieAuth", CookieAuth(AccessManager.SESSION_COOKIE_NAME)) - ) + info.title = "DRES API" + info.version = DRES.VERSION + info.description = + "API for DRES (Distributed Retrieval Evaluation Server), Version ${DRES.VERSION}" + val contact = OpenApiContact() + contact.url = "https://dres.dev" + contact.name = "The DRES Dev Team" + info.contact = contact + val license = OpenApiLicense() + license.name = "MIT" + info.license = license + } + + .withSecurity( + SecurityComponentConfiguration() + .withSecurityScheme( + "CookieAuth", + CookieAuth(AccessManager.SESSION_COOKIE_NAME) + ) + ) } } ) + it.registerPlugin(ClientOpenApiPlugin()) - it.registerPlugin(SwaggerPlugin{ swaggerConfig -> + + it.registerPlugin(SwaggerPlugin { swaggerConfig -> swaggerConfig.documentationPath = "/openapi.json" swaggerConfig.uiPath = "/swagger-ui" }) - it.registerPlugin(SwaggerPlugin{ swaggerConfig -> + it.registerPlugin(SwaggerPlugin { swaggerConfig -> swaggerConfig.documentationPath = "/clientapi.json" swaggerConfig.uiPath = "/swagger-client" swaggerConfig.title = "Client Swagger UI" @@ -293,7 +299,9 @@ object RestApi { it.registerPlugin(ssl) } - it.router.apiBuilder{ + + + it.router.apiBuilder { path("api") { apiRestHandlers.groupBy { it.apiVersion }.forEach { apiGroup -> path(apiGroup.key) { @@ -329,10 +337,10 @@ object RestApi { } } - }.beforeMatched{ctx -> + }.beforeMatched { ctx -> /* BeforeMatched handlers are only matched if the request will be matched */ /* See https://javalin.io/migration-guide-javalin-5-to-6 */ - if(! AccessManager.hasAccess(ctx)){ + if (!AccessManager.hasAccess(ctx)) { ctx.status(HttpStatus.UNAUTHORIZED) ctx.skipRemainingHandlers() } @@ -371,7 +379,6 @@ object RestApi { } - }.error(401) { it.json(ErrorStatus("Unauthorized request!")) }.exception(Exception::class.java) { e, ctx -> diff --git a/backend/src/main/kotlin/dev/dres/data/model/config/Config.kt b/backend/src/main/kotlin/dev/dres/data/model/config/Config.kt index 9a54ad30..e950a352 100644 --- a/backend/src/main/kotlin/dev/dres/data/model/config/Config.kt +++ b/backend/src/main/kotlin/dev/dres/data/model/config/Config.kt @@ -34,6 +34,8 @@ data class Config( val auditLocation: Path = DRES.APPLICATION_ROOT.parent.resolve("audit"), /** Whether API endpoints for external media should be available */ val externalMediaEndpointsEnabled: Boolean = false, + /** Whether to enable http3 protocol, requires enableSsl to be true, to have an effect. THIS IS CURRENTLY NOT YET SUPPORTED */ + val enableHttp3: Boolean = false, ) { companion object{ diff --git a/backend/src/main/kotlin/dev/dres/mgmt/MediaCollectionManager.kt b/backend/src/main/kotlin/dev/dres/mgmt/MediaCollectionManager.kt index f6ff40a3..d273dee9 100644 --- a/backend/src/main/kotlin/dev/dres/mgmt/MediaCollectionManager.kt +++ b/backend/src/main/kotlin/dev/dres/mgmt/MediaCollectionManager.kt @@ -27,6 +27,10 @@ object MediaCollectionManager { DbMediaCollection.query(DbMediaCollection::id eq collectionId).firstOrNull()?.toApi() } + fun getCollectionByName(name: String): ApiMediaCollection? = this.store.transactional(true){ + DbMediaCollection.query(DbMediaCollection::name eq name).firstOrNull()?.toApi() + } + fun getPopulatedCollection(collectionId: CollectionId): ApiPopulatedMediaCollection? = this.store.transactional(true) { val collection = DbMediaCollection.query(DbMediaCollection::id eq collectionId).firstOrNull() @@ -200,4 +204,4 @@ object MediaCollectionManager { counter } -} \ No newline at end of file +} From 892eb95f47b2ca1ebacdc576034273c9313d2fe7 Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Thu, 7 Mar 2024 16:10:55 +0100 Subject: [PATCH 10/19] Addresing compiler warnings --- backend/build.gradle | 2 ++ .../dev/dres/api/cli/EvaluationCommand.kt | 1 - .../main/kotlin/dev/dres/api/rest/RestApi.kt | 5 +---- .../evaluation/scores/TeamGroupScoreHandler.kt | 2 +- .../evaluation/viewer/GetTaskHintHandler.kt | 2 +- .../template/task/options/DbConfiguredOption.kt | 17 ++++++----------- .../run/InteractiveSynchronousRunManager.kt | 4 ++-- .../src/main/kotlin/dev/dres/run/RunExecutor.kt | 4 ++-- 8 files changed, 15 insertions(+), 22 deletions(-) diff --git a/backend/build.gradle b/backend/build.gradle index f72c9ad1..9a398a7e 100644 --- a/backend/build.gradle +++ b/backend/build.gradle @@ -116,8 +116,10 @@ dependencies { kapt { correctErrorTypes true useBuildCache true + targetCompatibility = 11 } + test { useJUnitPlatform() } diff --git a/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt b/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt index 9774e535..eb852a6f 100644 --- a/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt +++ b/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt @@ -254,7 +254,6 @@ class EvaluationCommand(internal val store: TransientEntityStore) : NoOpCliktCom return@transactional } - val mapper = jacksonObjectMapper() Files.newBufferedWriter(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE).use { /*val writer = if (this.pretty) { mapper.writerWithDefaultPrettyPrinter() diff --git a/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt b/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt index 7856d1d5..2223860d 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt @@ -75,9 +75,6 @@ object RestApi { * @param cache The [CacheManager] instance used to access the media cache. */ fun init(config: Config, store: TransientEntityStore, cache: CacheManager) { - - val runExecutor = RunExecutor - /** * The list of API operations, each as a handler. * Did you follow our convention? @@ -239,7 +236,7 @@ object RestApi { OpenApiPlugin { oapConfig -> oapConfig .withDocumentationPath("/openapi.json") - .withDefinitionConfiguration { version, openApiDef -> + .withDefinitionConfiguration { _, openApiDef -> openApiDef .withInfo { info -> info.title = "DRES API" diff --git a/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/scores/TeamGroupScoreHandler.kt b/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/scores/TeamGroupScoreHandler.kt index b6cf2684..fa1c4509 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/scores/TeamGroupScoreHandler.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/scores/TeamGroupScoreHandler.kt @@ -44,7 +44,7 @@ class TeamGroupScoreHandler : AbstractScoreHandler(), GetRestHandler + val sequence = this.hints.groupBy { it.type }.flatMap { (_, hints) -> var index = 0 hints.sortedBy { it.start ?: 0 }.flatMap { val ret = mutableListOf(it.toContentElement()) diff --git a/backend/src/main/kotlin/dev/dres/data/model/template/task/options/DbConfiguredOption.kt b/backend/src/main/kotlin/dev/dres/data/model/template/task/options/DbConfiguredOption.kt index 6b4d44b3..558f4c5c 100644 --- a/backend/src/main/kotlin/dev/dres/data/model/template/task/options/DbConfiguredOption.kt +++ b/backend/src/main/kotlin/dev/dres/data/model/template/task/options/DbConfiguredOption.kt @@ -27,40 +27,35 @@ class DbConfiguredOption(entity: Entity) : XdEntity(entity) { /** * Tries to parse a named parameter as [Boolean]. Returns null, if the parameter is not set or cannot be converted. * - * @param name Name of the parameter to return. * @return [Boolean] or null. */ - fun getAsBool(name: String): Boolean? = this.value.toBooleanStrictOrNull() + fun getAsBool(): Boolean? = this.value.toBooleanStrictOrNull() /** * Tries to parse a named parameter as [Int]. Returns null, if the parameter is not set or cannot be converted. * - * @param name Name of the parameter to return. * @return [Int] or null. */ - fun getAsInt(name: String): Int? = this.value.toIntOrNull() + fun getAsInt(): Int? = this.value.toIntOrNull() /** * Tries to parse a named parameter as [Long]. Returns null, if the parameter is not set or cannot be converted. * - * @param name Name of the parameter to return. * @return [Long] or null. */ - fun getAsLong(name: String): Long? = this.value.toLongOrNull() + fun getAsLong(): Long? = this.value.toLongOrNull() /** * Tries to parse a named parameter as [Float]. Returns null, if the parameter is not set or cannot be converted. * - * @param name Name of the parameter to return. * @return [Float] or null. */ - fun getAsFloat(name: String): Float? = this.value.toFloatOrNull() + fun getAsFloat(): Float? = this.value.toFloatOrNull() /** * Tries to parse a named parameter as [Double]. Returns null, if the parameter is not set or cannot be converted. * - * @param name Name of the parameter to return. * @return [Double] or null. */ - fun getAsDouble(name: String): Double? = this.value.toDoubleOrNull() -} \ No newline at end of file + fun getAsDouble(): Double? = this.value.toDoubleOrNull() +} diff --git a/backend/src/main/kotlin/dev/dres/run/InteractiveSynchronousRunManager.kt b/backend/src/main/kotlin/dev/dres/run/InteractiveSynchronousRunManager.kt index 53949ce7..67f11f1e 100644 --- a/backend/src/main/kotlin/dev/dres/run/InteractiveSynchronousRunManager.kt +++ b/backend/src/main/kotlin/dev/dres/run/InteractiveSynchronousRunManager.kt @@ -471,9 +471,9 @@ class InteractiveSynchronousRunManager( val currentTemplateId = this.evaluation.getCurrentTaskTemplate().id if (taskTemplateId == currentTemplateId) { - val status = this.evaluation.currentTaskRun?.status +// val status = this.evaluation.currentTaskRun?.status // if (status == ApiTaskStatus.PREPARING) { - /* Since the viewer does sent the ready message too early, we cannot care whether the task is (already) preparing or not) */ + /* Since the viewer does send the ready message too early, we cannot care whether the task is (already) preparing or not */ this.readyLatch.register(viewerInfo) //avoid redying previously untracked viewers this.readyLatch.setReady(viewerInfo) // } diff --git a/backend/src/main/kotlin/dev/dres/run/RunExecutor.kt b/backend/src/main/kotlin/dev/dres/run/RunExecutor.kt index ab3f018f..7c740aca 100644 --- a/backend/src/main/kotlin/dev/dres/run/RunExecutor.kt +++ b/backend/src/main/kotlin/dev/dres/run/RunExecutor.kt @@ -86,7 +86,7 @@ object RunExecutor { /** A thread that cleans after [RunManager] have finished. */ private val cleanerThread = Thread { while (!this@RunExecutor.executor.isShutdown) { - var stamp = this@RunExecutor.runManagerLock.readLock() +// var stamp = this@RunExecutor.runManagerLock.readLock() this@RunExecutor.runManagerLock.read { //try { this@RunExecutor.results.entries.removeIf { entry -> @@ -233,4 +233,4 @@ object RunExecutor { this.executor.shutdownNow() } -} \ No newline at end of file +} From 8c7e39cfe0a72bbac85eaec90d93c443efd018c9 Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Thu, 7 Mar 2024 16:47:17 +0100 Subject: [PATCH 11/19] Added check if ffmpeg is there and executable --- backend/src/main/kotlin/dev/dres/DRES.kt | 7 +++++++ .../dev/dres/api/cli/EvaluationCommand.kt | 8 +++++--- .../kotlin/dev/dres/mgmt/cache/CacheManager.kt | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/backend/src/main/kotlin/dev/dres/DRES.kt b/backend/src/main/kotlin/dev/dres/DRES.kt index 7356213b..7479b825 100644 --- a/backend/src/main/kotlin/dev/dres/DRES.kt +++ b/backend/src/main/kotlin/dev/dres/DRES.kt @@ -1,5 +1,6 @@ package dev.dres +import com.github.kokorin.jaffree.ffmpeg.FFmpeg import dev.dres.api.cli.Cli import dev.dres.api.cli.OpenApiCommand import dev.dres.api.rest.RestApi @@ -29,6 +30,10 @@ import kotlinx.dnq.util.initMetaData import java.io.File import java.nio.file.Path import java.nio.file.Paths +import kotlin.io.path.exists +import kotlin.io.path.isDirectory +import kotlin.io.path.isExecutable +import kotlin.io.path.listDirectoryEntries import kotlin.system.exitProcess @@ -100,6 +105,8 @@ object DRES { println("Starting DRES (application: $APPLICATION_ROOT, data: $DATA_ROOT)") println("Initializing...") + + /* Initialize Xodus based data store. */ val store = this.prepareDatabase() diff --git a/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt b/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt index eb852a6f..49e8eff4 100644 --- a/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt +++ b/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt @@ -31,7 +31,8 @@ import java.nio.file.StandardOpenOption * @author Ralph Gasser * @version 2.0.0 */ -class EvaluationCommand(internal val store: TransientEntityStore) : NoOpCliktCommand(name = "evaluation") { +class EvaluationCommand(internal val store: TransientEntityStore) : + NoOpCliktCommand(name = "evaluation") { init { subcommands( @@ -49,9 +50,10 @@ class EvaluationCommand(internal val store: TransientEntityStore) : NoOpCliktCom override fun aliases(): Map> { return mapOf( "ls" to listOf("ongoing"), - "la" to listOf("list"), + "ll" to listOf("list"), "remove" to listOf("delete"), - "drop" to listOf("delete") + "drop" to listOf("delete"), + "rm" to listOf("delete") ) } diff --git a/backend/src/main/kotlin/dev/dres/mgmt/cache/CacheManager.kt b/backend/src/main/kotlin/dev/dres/mgmt/cache/CacheManager.kt index e6db3caf..07542f97 100644 --- a/backend/src/main/kotlin/dev/dres/mgmt/cache/CacheManager.kt +++ b/backend/src/main/kotlin/dev/dres/mgmt/cache/CacheManager.kt @@ -26,6 +26,11 @@ import java.nio.file.Path import java.nio.file.StandardOpenOption import java.util.concurrent.* import javax.imageio.ImageIO +import kotlin.io.path.exists +import kotlin.io.path.isDirectory +import kotlin.io.path.isExecutable +import kotlin.io.path.listDirectoryEntries +import kotlin.system.exitProcess /** * A [CacheManager] used to manager and generate, access and manage cached image and video previews. @@ -53,6 +58,19 @@ class CacheManager(private val config: Config, private val store: TransientEntit init { println("Found FFmpeg at ${this.ffmpegBin}...") + /* Validating that FFmpeg path exists and is a directory */ + if(!this.ffmpegBin.exists() || !this.ffmpegBin.isDirectory()){ + System.err.println("ERROR: FFmpeg path ${this.ffmpegBin} does not exist! Shutting down!") + exitProcess(-100) + } + + /* Validating that FFmpeg and FFprobe exist and are executable */ + this.ffmpegBin.listDirectoryEntries("ff*").forEach { + if(!it.exists() || !it.isExecutable()){ + System.err.println("ERROR: $it in ${this.ffmpegBin} does not exist or is not executable! Shutting down!") + exitProcess(-101) + } + } if (!Files.exists(cacheLocation)) { println("Created cache location at ${this.cacheLocation}...") Files.createDirectories(cacheLocation) From 80f131504e9aedc4b793f8eb910b7e2a28e57683 Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Fri, 8 Mar 2024 10:30:37 +0100 Subject: [PATCH 12/19] Added slightly more verbose error handling during segment import --- .../dres/api/cli/MediaCollectionCommand.kt | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt b/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt index 8d7e81c6..5f338b54 100644 --- a/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt +++ b/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt @@ -796,17 +796,21 @@ class MediaCollectionCommand(private val config: Config) : val rows: kotlin.collections.List> = csvReader().readAllWithHeader(ips) println("Done! Reading ${rows.size} rows") - rows.mapNotNull { - val segmentName = it["name"] ?: return@mapNotNull null - val videoName = it["video"] ?: return@mapNotNull null - val start = it["start"]?.toIntOrNull() ?: return@mapNotNull null - val end = it["end"]?.toIntOrNull() ?: return@mapNotNull null - ApiMediaSegment(videoName, segmentName, start, end) + rows.mapIndexedNotNull { index, row -> + val segmentName = row["name"] ?: return@mapIndexedNotNull null + val videoName = row["video"] ?: return@mapIndexedNotNull null + val start = row["start"]?.toIntOrNull() ?: return@mapIndexedNotNull null + val end = row["end"]?.toIntOrNull() ?: return@mapIndexedNotNull null + index to ApiMediaSegment(videoName, segmentName, start, end) } - .sortedBy { it.mediaItemName } + .sortedBy { it.second.mediaItemName } .asSequence().chunked(transactionChunkSize) .forEach { chunk -> - inserted += MediaCollectionManager.addSegments(collection.id!!, chunk) + try { + inserted += MediaCollectionManager.addSegments(collection.id!!, chunk.map { it.second }) + }catch(ex: RuntimeException){ + System.err.println("An error (${ex.javaClass.simpleName}) occurred during ingesting from rows ${chunk.first().first} to ${chunk.last().first}: ${ex.message} ") + } } } From 04100f8fda2ae6d9ebd734acdeab4c3f69d4945c Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Fri, 8 Mar 2024 10:34:02 +0100 Subject: [PATCH 13/19] Added error logging to previous change --- .../main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt b/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt index 5f338b54..41cc23a3 100644 --- a/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt +++ b/backend/src/main/kotlin/dev/dres/api/cli/MediaCollectionCommand.kt @@ -809,7 +809,12 @@ class MediaCollectionCommand(private val config: Config) : try { inserted += MediaCollectionManager.addSegments(collection.id!!, chunk.map { it.second }) }catch(ex: RuntimeException){ - System.err.println("An error (${ex.javaClass.simpleName}) occurred during ingesting from rows ${chunk.first().first} to ${chunk.last().first}: ${ex.message} ") + val errorMsg = "An error (${ex.javaClass.simpleName}) occurred during ingesting from rows ${chunk.first().first} to ${chunk.last().first}" + System.err.println("$errorMsg: ${ex.message}") + this@MediaCollectionCommand.logger.error( + this@MediaCollectionCommand.logMarker, + errorMsg, ex + ) } } } From 381f25158ab33d57281564ccb1d94d6d5319700d Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Fri, 8 Mar 2024 11:12:20 +0100 Subject: [PATCH 14/19] Addressed comment regarding check for ffmpeg / ffprobe --- .../kotlin/dev/dres/mgmt/cache/CacheManager.kt | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/src/main/kotlin/dev/dres/mgmt/cache/CacheManager.kt b/backend/src/main/kotlin/dev/dres/mgmt/cache/CacheManager.kt index 07542f97..cce77993 100644 --- a/backend/src/main/kotlin/dev/dres/mgmt/cache/CacheManager.kt +++ b/backend/src/main/kotlin/dev/dres/mgmt/cache/CacheManager.kt @@ -26,10 +26,7 @@ import java.nio.file.Path import java.nio.file.StandardOpenOption import java.util.concurrent.* import javax.imageio.ImageIO -import kotlin.io.path.exists -import kotlin.io.path.isDirectory -import kotlin.io.path.isExecutable -import kotlin.io.path.listDirectoryEntries +import kotlin.io.path.* import kotlin.system.exitProcess /** @@ -66,9 +63,14 @@ class CacheManager(private val config: Config, private val store: TransientEntit /* Validating that FFmpeg and FFprobe exist and are executable */ this.ffmpegBin.listDirectoryEntries("ff*").forEach { - if(!it.exists() || !it.isExecutable()){ - System.err.println("ERROR: $it in ${this.ffmpegBin} does not exist or is not executable! Shutting down!") - exitProcess(-101) + /* Slightly convoluted in order to not hassle with OS dependent things like .exe extension on Windows */ + val isFFmpeg = it.nameWithoutExtension.startsWith("ffmpeg") && it.nameWithoutExtension.endsWith("ffmpeg") + val isFFprobe = it.nameWithoutExtension.startsWith("ffprobe") && it.nameWithoutExtension.endsWith("ffprobe") + if(isFFprobe || isFFmpeg){ + if(!it.exists() || !it.isExecutable()){ + System.err.println("ERROR: $it in ${this.ffmpegBin} does not exist or is not executable! Shutting down!") + exitProcess(-101) + } } } if (!Files.exists(cacheLocation)) { From fd3c3873419f8c0cfc8bb99073cc7bff0972a95a Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Fri, 8 Mar 2024 14:22:26 +0100 Subject: [PATCH 15/19] Added kotlin (beta-support) for code analyse workflow --- .github/workflows/codeql-analysis.yml | 2 +- .../dev/dres/api/rest/ClientSwaggerPlugin.kt | 37 ------------------- 2 files changed, 1 insertion(+), 38 deletions(-) delete mode 100644 backend/src/main/kotlin/dev/dres/api/rest/ClientSwaggerPlugin.kt diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 48f30957..e787ae5f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'javascript' ] #TODO add kotlin as soon as it is supported + language: [ 'javascript', 'kotlin' ] #TODO kotlin support is currently in beta # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://git.io/codeql-language-support diff --git a/backend/src/main/kotlin/dev/dres/api/rest/ClientSwaggerPlugin.kt b/backend/src/main/kotlin/dev/dres/api/rest/ClientSwaggerPlugin.kt deleted file mode 100644 index a6d7c24f..00000000 --- a/backend/src/main/kotlin/dev/dres/api/rest/ClientSwaggerPlugin.kt +++ /dev/null @@ -1,37 +0,0 @@ -package dev.dres.api.rest - -import io.javalin.Javalin -import io.javalin.config.JavalinConfig -import io.javalin.openapi.plugin.swagger.SwaggerConfiguration -import io.javalin.openapi.plugin.swagger.SwaggerHandler -import io.javalin.openapi.plugin.swagger.SwaggerPlugin -import io.javalin.plugin.Plugin - -class ClientSwaggerPlugin : SwaggerPlugin() { - - override fun name(): String { - return this.javaClass.simpleName - } - - override fun onStart(config: JavalinConfig) { -// val swaggerHandler = SwaggerHandler( -// title = "DRES Client API", -// documentationPath = "/client-oas", -// swaggerVersion = SwaggerConfiguration().version, -// validatorUrl = "https://validator.swagger.io/validator", -//// routingPath = app.cfg.routing.contextPath, -// basePath = null, -// tagsSorter = "'alpha'", -// operationsSorter = "'alpha'", -// customJavaScriptFiles = emptyList(), -// customStylesheetFiles = emptyList() -// ) -// -// config.router.apiBuilder { -// get("/swagger-client", swaggerHandler) -// } - - - } - -} From 58bdd6f187c5ea36d2664e897f8f4cff829fd0b2 Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Fri, 8 Mar 2024 14:22:36 +0100 Subject: [PATCH 16/19] Addressed comments in PR #460 --- backend/src/main/kotlin/dev/dres/DRES.kt | 13 ++++--------- .../src/main/kotlin/dev/dres/api/rest/RestApi.kt | 12 +++++------- .../evaluation/scores/TeamGroupScoreHandler.kt | 11 ++++------- .../handler/evaluation/viewer/GetTaskHintHandler.kt | 2 ++ 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/backend/src/main/kotlin/dev/dres/DRES.kt b/backend/src/main/kotlin/dev/dres/DRES.kt index 7479b825..8db32b7b 100644 --- a/backend/src/main/kotlin/dev/dres/DRES.kt +++ b/backend/src/main/kotlin/dev/dres/DRES.kt @@ -1,21 +1,20 @@ package dev.dres -import com.github.kokorin.jaffree.ffmpeg.FFmpeg import dev.dres.api.cli.Cli import dev.dres.api.cli.OpenApiCommand import dev.dres.api.rest.RestApi -import dev.dres.data.model.config.Config import dev.dres.data.model.admin.DbRole import dev.dres.data.model.admin.DbUser +import dev.dres.data.model.config.Config import dev.dres.data.model.media.* +import dev.dres.data.model.run.* +import dev.dres.data.model.submissions.* import dev.dres.data.model.template.DbEvaluationTemplate import dev.dres.data.model.template.task.* import dev.dres.data.model.template.task.options.* import dev.dres.data.model.template.team.DbTeam import dev.dres.data.model.template.team.DbTeamAggregator import dev.dres.data.model.template.team.DbTeamGroup -import dev.dres.data.model.run.* -import dev.dres.data.model.submissions.* import dev.dres.mgmt.MediaCollectionManager import dev.dres.mgmt.TemplateManager import dev.dres.mgmt.admin.UserManager @@ -30,10 +29,6 @@ import kotlinx.dnq.util.initMetaData import java.io.File import java.nio.file.Path import java.nio.file.Paths -import kotlin.io.path.exists -import kotlin.io.path.isDirectory -import kotlin.io.path.isExecutable -import kotlin.io.path.listDirectoryEntries import kotlin.system.exitProcess @@ -47,7 +42,7 @@ import kotlin.system.exitProcess */ object DRES { /** Version of DRES. */ - const val VERSION = "2.0.0-RC5" + const val VERSION = "2.0.0" /** Application root; should be relative to JAR file or classes path. */ val APPLICATION_ROOT: Path = diff --git a/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt b/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt index 2223860d..43bbb0b3 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/RestApi.kt @@ -1,7 +1,5 @@ package dev.dres.api.rest -import GetTaskHintHandler -import com.fasterxml.jackson.databind.node.ObjectNode import dev.dres.DRES import dev.dres.api.rest.handler.* import dev.dres.api.rest.handler.collection.* @@ -20,34 +18,34 @@ import dev.dres.api.rest.handler.judgement.* import dev.dres.api.rest.handler.log.QueryLogHandler import dev.dres.api.rest.handler.log.ResultLogHandler import dev.dres.api.rest.handler.preview.* -import dev.dres.api.rest.handler.template.* import dev.dres.api.rest.handler.scores.ListEvaluationScoreHandler import dev.dres.api.rest.handler.submission.SubmissionHandler import dev.dres.api.rest.handler.system.CurrentTimeHandler import dev.dres.api.rest.handler.system.InfoHandler import dev.dres.api.rest.handler.system.LoginHandler import dev.dres.api.rest.handler.system.LogoutHandler +import dev.dres.api.rest.handler.template.* import dev.dres.api.rest.handler.users.* import dev.dres.api.rest.types.status.ErrorStatus import dev.dres.api.rest.types.users.ApiRole import dev.dres.data.model.config.Config import dev.dres.mgmt.cache.CacheManager -import dev.dres.run.RunExecutor import dev.dres.utilities.NamedThreadFactory import io.javalin.Javalin import io.javalin.apibuilder.ApiBuilder.* -import io.javalin.http.staticfiles.Location import io.javalin.community.ssl.SslPlugin import io.javalin.http.Cookie import io.javalin.http.HttpStatus import io.javalin.http.SameSite +import io.javalin.http.staticfiles.Location import io.javalin.openapi.CookieAuth import io.javalin.openapi.OpenApiContact import io.javalin.openapi.OpenApiLicense -import io.javalin.openapi.plugin.* +import io.javalin.openapi.plugin.OpenApiPlugin +import io.javalin.openapi.plugin.SecurityComponentConfiguration import io.javalin.openapi.plugin.swagger.SwaggerPlugin import jetbrains.exodus.database.TransientEntityStore -import org.eclipse.jetty.server.* +import org.eclipse.jetty.server.Server import org.eclipse.jetty.util.thread.QueuedThreadPool import org.slf4j.LoggerFactory import org.slf4j.MarkerFactory diff --git a/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/scores/TeamGroupScoreHandler.kt b/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/scores/TeamGroupScoreHandler.kt index fa1c4509..c2445c5c 100644 --- a/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/scores/TeamGroupScoreHandler.kt +++ b/backend/src/main/kotlin/dev/dres/api/rest/handler/evaluation/scores/TeamGroupScoreHandler.kt @@ -42,13 +42,10 @@ class TeamGroupScoreHandler : AbstractScoreHandler(), GetRestHandler Date: Fri, 8 Mar 2024 14:28:27 +0100 Subject: [PATCH 17/19] Re-added evaluation export on CLI --- .../main/kotlin/dev/dres/api/cli/EvaluationCommand.kt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt b/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt index 49e8eff4..584f1c15 100644 --- a/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt +++ b/backend/src/main/kotlin/dev/dres/api/cli/EvaluationCommand.kt @@ -1,6 +1,6 @@ package dev.dres.api.cli -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.databind.ObjectMapper import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.NoOpCliktCommand import com.github.ajalt.clikt.core.subcommands @@ -249,21 +249,22 @@ class EvaluationCommand(internal val store: TransientEntityStore) : /** Flag indicating whether export should be pretty printed.*/ private val pretty: Boolean by option("-p", "--pretty", help = "Flag indicating whether exported JSON should be pretty printed.").flag("-u", "--ugly", default = true) + + override fun run() = this@EvaluationCommand.store.transactional(true) { val evaluation = DbEvaluation.query(DbEvaluation::id eq this.id).firstOrNull() if (evaluation == null) { println("Evaluation ${this.id} does not seem to exist.") return@transactional } - + val mapper = ObjectMapper() Files.newBufferedWriter(path, StandardOpenOption.WRITE, StandardOpenOption.CREATE).use { - /*val writer = if (this.pretty) { + val writer = if (this.pretty) { mapper.writerWithDefaultPrettyPrinter() } else { mapper.writer() } - writer.writeValue(it, run)*/ - // TODO: Export must be re-conceived based on API classes. + writer.writeValue(it, evaluation.toApi()) } println("Successfully exported evaluation ${this.id} to $path.") } From 8f6345e22c97d9179efcedc289848b7a14633dee Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Fri, 8 Mar 2024 14:32:51 +0100 Subject: [PATCH 18/19] Updated github codeql workflow to v2, as v1 is deprecated --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e787ae5f..a34ec611 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -42,7 +42,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -53,7 +53,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -67,4 +67,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 From 823d9b14647e1dc97a0f01776490f7866b29265c Mon Sep 17 00:00:00 2001 From: Loris Sauter Date: Fri, 8 Mar 2024 14:50:26 +0100 Subject: [PATCH 19/19] Removed kotlin support, as it seems to be broken --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index a34ec611..a7e3cbc0 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'javascript', 'kotlin' ] #TODO kotlin support is currently in beta + language: [ 'javascript' ] #TODO kotlin support is currently in beta # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://git.io/codeql-language-support