From f6772fe0a10b07968e40c29a2b5026314e1cc545 Mon Sep 17 00:00:00 2001 From: Redwanul Haque Sourave Date: Sat, 11 Mar 2023 03:00:50 -0500 Subject: [PATCH] Refactor JsonCreate and XmlCreate to extract common logic --- .../kotlin/org/sirix/rest/SirixVerticle.kt | 17 +- .../sirix/rest/crud/AbstractCreateHandler.kt | 35 +++ .../org/sirix/rest/crud/CreateHandler.kt | 55 ++++ .../org/sirix/rest/crud/json/JsonCreate2.kt | 258 ++++++++++++++++++ .../org/sirix/rest/crud/xml/XMLCreate2.kt | 220 +++++++++++++++ 5 files changed, 577 insertions(+), 8 deletions(-) create mode 100644 bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/AbstractCreateHandler.kt create mode 100644 bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/CreateHandler.kt create mode 100644 bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/json/JsonCreate2.kt create mode 100644 bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/xml/XMLCreate2.kt diff --git a/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/SirixVerticle.kt b/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/SirixVerticle.kt index 12944a91e..1ae8691db 100644 --- a/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/SirixVerticle.kt +++ b/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/SirixVerticle.kt @@ -213,18 +213,19 @@ class SirixVerticle : CoroutineVerticle() { GetHandler(location, keycloak, authz).handle(it) } - put("/:database").consumes("application/xml").coroutineHandler { + put("/:database").coroutineHandler { Auth(keycloak, authz, AuthRole.CREATE).handle(it) it.next() }.handler(BodyHandler.create()).coroutineHandler { - XmlCreate(location, false).handle(it) - } - put("/:database").consumes("application/json").coroutineHandler { - Auth(keycloak, authz, AuthRole.CREATE).handle(it) - it.next() - }.coroutineHandler { - JsonCreate(location, true).handle(it) + + CreateHandler(location, false).handle(it) } +// put("/:database").consumes("application/json").coroutineHandler { +// Auth(keycloak, authz, AuthRole.CREATE).handle(it) +// it.next() +// }.coroutineHandler { +// JsonCreate(location, true).handle(it) +// } delete("/:database").coroutineHandler { Auth(keycloak, authz, AuthRole.DELETE).handle(it) diff --git a/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/AbstractCreateHandler.kt b/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/AbstractCreateHandler.kt new file mode 100644 index 000000000..7aa225f4f --- /dev/null +++ b/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/AbstractCreateHandler.kt @@ -0,0 +1,35 @@ +package org.sirix.rest.crud + +import io.vertx.core.Context +import io.vertx.core.Promise +import io.vertx.ext.web.Route +import io.vertx.ext.web.RoutingContext +import io.vertx.kotlin.coroutines.await +import org.sirix.access.DatabaseConfiguration +import org.sirix.api.Database +import org.sirix.api.ResourceSession +import org.sirix.api.json.JsonResourceSession +import java.nio.file.Files +import java.nio.file.Path + +abstract class AbstractCreateHandler(location: Path, createMultipleResources: Boolean) { + +// abstract suspend fun handle(ctx: RoutingContext): Route + abstract suspend fun shredder(databaseName: String, resource: String, ctx: RoutingContext) + abstract suspend fun createMultipleResources(databaseName: String, ctx: RoutingContext) + abstract suspend fun createDatabaseIfNotExists(dbFile: Path, context: Context) : DatabaseConfiguration? + abstract suspend fun insertResource(dbFile: Path, resPathName: String, ctx: RoutingContext) + + protected suspend fun prepareDatabasePath(dbFile: Path, context: Context): DatabaseConfiguration? { + return context.executeBlocking{ promise: Promise -> + val dbExists = Files.exists(dbFile) + + if(!dbExists) { + Files.createDirectories(dbFile.parent) + } + + val dbConfig = DatabaseConfiguration(dbFile) + promise.complete(dbConfig) + }.await() + } +} \ No newline at end of file diff --git a/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/CreateHandler.kt b/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/CreateHandler.kt new file mode 100644 index 000000000..63e3a9c4a --- /dev/null +++ b/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/CreateHandler.kt @@ -0,0 +1,55 @@ +package org.sirix.rest.crud + +import io.vertx.core.http.HttpHeaders +import io.vertx.ext.web.Route +import io.vertx.ext.web.RoutingContext +import org.sirix.access.DatabasesInternals +import org.sirix.rest.crud.json.JsonCreate2 +import org.sirix.rest.crud.xml.XMLCreate2 +import org.sirix.utils.LogWrapper +import org.slf4j.LoggerFactory +import java.nio.file.Path + +private val logger = LogWrapper(LoggerFactory.getLogger(DatabasesInternals::class.java)) + +class CreateHandler(private val location: Path, + private val createMultipleResources: Boolean) { + suspend fun handle(ctx: RoutingContext): Route { + var abstractCreateHandler: AbstractCreateHandler? = null + when(ctx.request().getHeader(HttpHeaders.CONTENT_TYPE)) { + "application/json" -> { + abstractCreateHandler = JsonCreate2(location, createMultipleResources) + } + + "application/xml" -> { + abstractCreateHandler = XMLCreate2(location, createMultipleResources) + } + } + + val databaseName = ctx.pathParam("database") + val resource = ctx.pathParam("resource") + + val dbFile = location.resolve(databaseName) + val vertxContext = ctx.vertx().orCreateContext + + abstractCreateHandler?.createDatabaseIfNotExists(dbFile, vertxContext) + + if (resource == null) { + ctx.response().setStatusCode(201).end() + return ctx.currentRoute() + } + + if (databaseName == null) { + throw IllegalArgumentException("Database name and resource data to store not given.") + } + + if (createMultipleResources) { + abstractCreateHandler?.createMultipleResources(databaseName, ctx) + ctx.response().setStatusCode(201).end() + return ctx.currentRoute() + } + + abstractCreateHandler?.shredder(databaseName, resource, ctx) + return ctx.currentRoute() + } +} \ No newline at end of file diff --git a/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/json/JsonCreate2.kt b/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/json/JsonCreate2.kt new file mode 100644 index 000000000..caa0e40e4 --- /dev/null +++ b/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/json/JsonCreate2.kt @@ -0,0 +1,258 @@ +package org.sirix.rest.crud.json + +import io.vertx.core.Context +import io.vertx.core.Promise +import io.vertx.core.file.OpenOptions +import io.vertx.core.file.impl.FileResolver +import io.vertx.core.parsetools.JsonParser +import io.vertx.ext.web.Route +import io.vertx.ext.web.RoutingContext +import io.vertx.ext.web.handler.BodyHandler +import io.vertx.kotlin.coroutines.await +import io.vertx.kotlin.coroutines.dispatcher +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.sirix.access.DatabaseConfiguration +import org.sirix.access.Databases +import org.sirix.access.DatabasesInternals +import org.sirix.access.ResourceConfiguration +import org.sirix.access.trx.node.HashType +import org.sirix.api.Database +import org.sirix.api.json.JsonResourceSession +import org.sirix.rest.KotlinJsonStreamingShredder +import org.sirix.rest.crud.AbstractCreateHandler +import org.sirix.rest.crud.Revisions +import org.sirix.rest.crud.SirixDBUser +import org.sirix.service.json.serialize.JsonSerializer +import org.sirix.service.json.shredder.JsonShredder +import org.sirix.utils.LogWrapper +import org.slf4j.LoggerFactory +import java.io.StringWriter +import java.nio.file.Files +import java.nio.file.Path +import java.util.* + + +private const val MAX_NODES_TO_SERIALIZE = 5000 + +private val logger = LogWrapper(LoggerFactory.getLogger(DatabasesInternals::class.java)) + +class JsonCreate2(private val location: Path, private val createMultipleResources: Boolean): AbstractCreateHandler(location, createMultipleResources) { +// override suspend fun handle(ctx: RoutingContext): Route { +// val databaseName = ctx.pathParam("database") +// val resource = ctx.pathParam("resource") +// +// val dbFile = location.resolve(databaseName) +// val vertxContext = ctx.vertx().orCreateContext +// createDatabaseIfNotExists(dbFile, vertxContext) +// +// if (resource == null) { +// ctx.response().setStatusCode(201).end() +// return ctx.currentRoute() +// } +// +// if (databaseName == null) { +// throw IllegalArgumentException("Database name and resource data to store not given.") +// } +// +// if (createMultipleResources) { +// createMultipleResources(databaseName, ctx) +// ctx.response().setStatusCode(201).end() +// return ctx.currentRoute() +// } +// +// shredder(databaseName, resource, ctx) +// +// return ctx.currentRoute() +// } + + override suspend fun shredder(databaseName: String, resPathName: String, ctx: RoutingContext) { + val dbFile = location.resolve(databaseName) + insertResource(dbFile, resPathName, ctx) + } + + override suspend fun createMultipleResources(databaseName: String, ctx: RoutingContext) { + val dbFile = location.resolve(databaseName) + val sirixDBUser = SirixDBUser.create(ctx) + + ctx.vertx().executeBlocking { + val database = Databases.openJsonDatabase(dbFile, sirixDBUser) + + database.use { + BodyHandler.create().handle(ctx) + val fileResolver = FileResolver() + val hashType = ctx.queryParam("hashType").getOrNull(0) ?: "NONE" + ctx.fileUploads().forEach { fileUpload -> + val fileName = fileUpload.fileName() + val resConfig = ResourceConfiguration.Builder(fileName).useDeweyIDs(true).hashKind( + HashType.valueOf(hashType.uppercase()) + ).build() + + val resourceHasBeenCreated = createResourceIfNotExisting( + database, + resConfig + ) + + if (resourceHasBeenCreated) { + val manager = database.beginResourceSession(fileName) + + manager.use { + insertJsonSubtreeAsFirstChild( + manager, + fileResolver.resolveFile(fileUpload.uploadedFileName()).toPath() + ) + } + } + } + } + }.await() + + ctx.response().setStatusCode(201).end() + } + + override suspend fun createDatabaseIfNotExists(dbFile: Path, context: Context): + DatabaseConfiguration? { + val dbConfig = prepareDatabasePath(dbFile, context) + return context.executeBlocking{ promise: Promise -> + if(!Databases.existsDatabase(dbFile)) { + Databases.createJsonDatabase(dbConfig) + } + promise.complete(dbConfig) + }.await() + } + + override suspend fun insertResource( + dbFile: Path, resPathName: String, + ctx: RoutingContext + ) { + ctx.request().pause() +// val fileResolver = FileResolver() +// +// val filePath = withContext(Dispatchers.IO) { +// fileResolver.resolveFile(Files.createTempFile(UUID.randomUUID().toString(), null).toString()) +// } +// +// val file = ctx.vertx().fileSystem().open( +// filePath.toString(), +// OpenOptions() +// ).await() +// +// ctx.request().resume() +// ctx.request().pipeTo(file).await() + + withContext(Dispatchers.IO) { + var body: String? = null + val sirixDBUser = SirixDBUser.create(ctx) + val database = Databases.openJsonDatabase(dbFile, sirixDBUser) + + database.use { + val commitTimestampAsString = ctx.queryParam("commitTimestamp").getOrNull(0) + val hashType = ctx.queryParam("hashType").getOrNull(0) ?: "NONE" + val resConfig = + ResourceConfiguration.Builder(resPathName).useDeweyIDs(true) + .hashKind(HashType.valueOf(hashType.uppercase())) + .customCommitTimestamps(commitTimestampAsString != null) + .build() + + val resourceHasBeenCreated = createResourceIfNotExisting( + database, + resConfig + ) + + val manager = database.beginResourceSession(resPathName) + + manager.use { + val maxNodeKey = if (resourceHasBeenCreated) { + insertJsonSubtreeAsFirstChild(manager, ctx) + } else { + val rtx = manager.beginNodeReadOnlyTrx() + + rtx.use { + return@use rtx.maxNodeKey + } + } +// ctx.vertx().fileSystem().delete(pathToFile.toAbsolutePath().toString()).await() + + if (maxNodeKey < MAX_NODES_TO_SERIALIZE) { + body = serializeJson(manager, ctx) + } else { + ctx.response().setStatusCode(200) + } + } + } + + if (body != null) { + ctx.response().end(body) + } else { + ctx.response().end() + } + } + } + + private fun serializeJson( + manager: JsonResourceSession, + routingCtx: RoutingContext + ): String { + val out = StringWriter() + val serializerBuilder = JsonSerializer.newBuilder(manager, out) + val serializer = serializerBuilder.build() + + return JsonSerializeHelper().serialize( + serializer, + out, + routingCtx, + manager, + intArrayOf(1), + null + ) + } + + private suspend fun insertJsonSubtreeAsFirstChild( + manager: JsonResourceSession, + ctx: RoutingContext + ): Long { + val commitMessage = ctx.queryParam("commitMessage").getOrNull(0) + val commitTimestampAsString = ctx.queryParam("commitTimestamp").getOrNull(0) + val commitTimestamp = if (commitTimestampAsString == null) { + null + } else { + Revisions.parseRevisionTimestamp(commitTimestampAsString).toInstant() + } + + val wtx = manager.beginNodeTrx() + return wtx.use { + val jsonParser = JsonParser.newParser(ctx.request()) + val future = KotlinJsonStreamingShredder(wtx, jsonParser).call() + ctx.request().resume() + future.await() + wtx.commit(commitMessage, commitTimestamp) + return@use wtx.maxNodeKey + } + } + + + private fun createResourceIfNotExisting( + database: Database, + resConfig: ResourceConfiguration?, + ): Boolean { + logger.debug("Try to create resource: $resConfig") + return database.createResource(resConfig) + } + + private fun insertJsonSubtreeAsFirstChild( + manager: JsonResourceSession, + resFileToStore: Path + ): Long { + val wtx = manager.beginNodeTrx() + return wtx.use { + val eventReader = JsonShredder.createFileReader(resFileToStore) + eventReader.use { + wtx.insertSubtreeAsFirstChild(eventReader) + } + wtx.maxNodeKey + } + } + + +} \ No newline at end of file diff --git a/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/xml/XMLCreate2.kt b/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/xml/XMLCreate2.kt new file mode 100644 index 000000000..9b21357bd --- /dev/null +++ b/bundles/sirix-rest-api/src/main/kotlin/org/sirix/rest/crud/xml/XMLCreate2.kt @@ -0,0 +1,220 @@ +package org.sirix.rest.crud.xml + +import io.vertx.core.Context +import io.vertx.core.Promise +import io.vertx.core.file.OpenOptions +import io.vertx.core.file.impl.FileResolver +import io.vertx.ext.web.Route +import io.vertx.ext.web.RoutingContext +import io.vertx.ext.web.handler.BodyHandler +import io.vertx.kotlin.coroutines.await +import io.vertx.kotlin.coroutines.dispatcher +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.sirix.access.DatabaseConfiguration +import org.sirix.access.Databases +import org.sirix.access.ResourceConfiguration +import org.sirix.access.trx.node.HashType +import org.sirix.api.Database +import org.sirix.api.xml.XmlNodeTrx +import org.sirix.api.xml.XmlResourceSession +import org.sirix.rest.crud.AbstractCreateHandler +import org.sirix.rest.crud.Revisions +import org.sirix.rest.crud.SirixDBUser +import org.sirix.service.xml.serialize.XmlSerializer +import org.sirix.service.xml.shredder.XmlShredder +import java.io.ByteArrayOutputStream +import java.io.FileInputStream +import java.lang.IllegalStateException +import java.nio.file.Files +import java.nio.file.Path +import java.util.* + +class XMLCreate2(private val location: Path, private val createMultipleResources: Boolean): AbstractCreateHandler(location, createMultipleResources) { +// public override suspend fun handle(ctx: RoutingContext): Route { +// val databaseName = ctx.pathParam("database") +// val resource = ctx.pathParam("resource") +// +// val dbFile = location.resolve(databaseName) +// val vertxContext = ctx.vertx().orCreateContext +// createDatabaseIfNotExists(dbFile, vertxContext) +// +// if (resource == null) { +// ctx.response().setStatusCode(201).end() +// return ctx.currentRoute() +// } +// +// if (databaseName == null) { +// throw IllegalArgumentException("Database name and resource data to store not given.") +// } +// +// if (createMultipleResources) { +// createMultipleResources(databaseName, ctx) +// ctx.response().setStatusCode(201).end() +// return ctx.currentRoute() +// } +// +// shredder(databaseName, resource, ctx) +// return ctx.currentRoute() +// } + + override suspend fun shredder(databaseName: String, resPathName: String, ctx: RoutingContext) { + val dbFile = location.resolve(databaseName) + insertResource(dbFile, resPathName, ctx) + } + + override suspend fun createMultipleResources(databaseName: String, ctx: RoutingContext) { + val dbFile = location.resolve(databaseName) + val context = ctx.vertx().orCreateContext + val dispatcher = ctx.vertx().dispatcher() + createDatabaseIfNotExists(dbFile, context) + + val sirixDBUser = SirixDBUser.create(ctx) + + withContext(Dispatchers.IO) { + val database = Databases.openXmlDatabase(dbFile, sirixDBUser) + + database.use { + BodyHandler.create().handle(ctx) + val fileResolver = FileResolver() + ctx.fileUploads().forEach { fileUpload -> + val fileName = fileUpload.fileName() + val hashType = ctx.queryParam("hashType").getOrNull(0) ?: "NONE" + val resConfig = + ResourceConfiguration.Builder(fileName).useDeweyIDs(true) + .hashKind(HashType.valueOf(hashType.uppercase())).build() + + createOrRemoveAndCreateResource(database, resConfig, fileName, dispatcher) + + val manager = database.beginResourceSession(fileName) + + manager.use { + insertXmlSubtreeAsFirstChild( + manager, + fileResolver.resolveFile(fileUpload.uploadedFileName()).toPath(), + ctx + ) + } + } + } + + ctx.response().setStatusCode(200).end() + } + } + + override suspend fun createDatabaseIfNotExists(dbFile: Path, context: Context): + DatabaseConfiguration? { + val dbConfig = prepareDatabasePath(dbFile, context) + return context.executeBlocking{ promise: Promise -> + if(!Databases.existsDatabase(dbFile)) { + Databases.createXmlDatabase(dbConfig) + } + promise.complete(dbConfig) + }.await() + } + + override suspend fun insertResource( + dbFile: Path, resPathName: String, + ctx: RoutingContext + ) { + val dispatcher = ctx.vertx().dispatcher() + ctx.request().pause() + val fileResolver = FileResolver() + + val filePath = withContext(Dispatchers.IO) { + fileResolver.resolveFile(Files.createTempFile(UUID.randomUUID().toString(), null).toString()) + } + + val file = ctx.vertx().fileSystem().open( + filePath.toString(), + OpenOptions() + ).await() + ctx.request().resume() + ctx.request().pipeTo(file).await() + + withContext(Dispatchers.IO) { + var body: String? = null + val sirixDBUser = SirixDBUser.create(ctx) + val database = Databases.openXmlDatabase(dbFile, sirixDBUser) + + database.use { + val hashType = ctx.queryParam("hashType").getOrNull(0) ?: "NONE" + val commitTimestampAsString = ctx.queryParam("commitTimestamp").getOrNull(0) + val resConfig = + ResourceConfiguration.Builder(resPathName).hashKind(HashType.valueOf(hashType.uppercase())) + .customCommitTimestamps(commitTimestampAsString != null).build() + createOrRemoveAndCreateResource(database, resConfig, resPathName, dispatcher) + val manager = database.beginResourceSession(resPathName) + + manager.use { + val pathToFile = filePath.toPath() + val maxNodeKey = insertXmlSubtreeAsFirstChild(manager, pathToFile.toAbsolutePath(), ctx) + + ctx.vertx().fileSystem().delete(filePath.toString()).await() + + if (maxNodeKey < 5000) { + body = serializeXml(manager, ctx) + } else { + ctx.response().setStatusCode(200) + } + } + } + + if (body != null) { + ctx.response().end(body) + } else { + ctx.response().end() + } + } + } + + private suspend fun createOrRemoveAndCreateResource( + database: Database, + resConfig: ResourceConfiguration?, + resPathName: String, dispatcher: CoroutineDispatcher + ) { + withContext(dispatcher) { + if (!database.createResource(resConfig)) { + database.removeResource(resPathName) + database.createResource(resConfig) + } + } + } + private fun insertXmlSubtreeAsFirstChild( + manager: XmlResourceSession, + resFileToStore: Path, + ctx: RoutingContext + ): Long { + val commitMessage = ctx.queryParam("commitMessage").getOrNull(0) + val commitTimestampAsString = ctx.queryParam("commitTimestamp").getOrNull(0) + val commitTimestamp = if (commitTimestampAsString == null) { + null + } else { + Revisions.parseRevisionTimestamp(commitTimestampAsString).toInstant() + } + + val wtx = manager.beginNodeTrx() + return wtx.use { + val inputStream = FileInputStream(resFileToStore.toFile()) + return@use inputStream.use { + val eventStream = XmlShredder.createFileReader(inputStream) + wtx.insertSubtreeAsFirstChild(eventStream, XmlNodeTrx.Commit.No) + eventStream.close() + wtx.commit(commitMessage, commitTimestamp) + wtx.maxNodeKey + } + } + } + private fun serializeXml( + manager: XmlResourceSession, + routingCtx: RoutingContext + ): String { + val out = ByteArrayOutputStream() + val serializerBuilder = XmlSerializer.XmlSerializerBuilder(manager, out) + val serializer = serializerBuilder.emitIDs().emitRESTful().emitRESTSequence().prettyPrint().build() + + return XmlSerializeHelper().serializeXml(serializer, out, routingCtx, manager, null) + } + +} \ No newline at end of file