diff --git a/.gitignore b/.gitignore index 47dfa3073..6aa5b837c 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,5 @@ resources/ #IIIF iiif-media-*/ + +cineast-api/src/main/python/__pycache__/ diff --git a/cineast-api/build.gradle b/cineast-api/build.gradle index 82f47d4e5..bf300b77f 100644 --- a/cineast-api/build.gradle +++ b/cineast-api/build.gradle @@ -111,6 +111,8 @@ dependencies { implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin', version: version_jackson implementation group: 'de.svenkubiak', name: 'jBCrypt', version: version_jbcrypt implementation group: 'org.vitrivr', name: 'cineast-proto', version: version_cineast_proto + implementation group: 'tech.molecules', name: 'external-umap-java', version: '1.0' + } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java index 0d54af496..857b0e562 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/APIEndpoint.java @@ -50,6 +50,7 @@ import org.vitrivr.cineast.api.rest.handlers.actions.tag.FindTagsAllGetHandler; import org.vitrivr.cineast.api.rest.handlers.actions.tag.FindTagsByIdsPostHandler; import org.vitrivr.cineast.api.rest.handlers.actions.tag.FindTagsGetHandler; +import org.vitrivr.cineast.api.rest.handlers.actions.vector.LoadVectorsForIdsPostHandler; import org.vitrivr.cineast.api.rest.handlers.interfaces.DeleteRestHandler; import org.vitrivr.cineast.api.rest.handlers.interfaces.DocumentedRestHandler; import org.vitrivr.cineast.api.rest.handlers.interfaces.GetRestHandler; @@ -427,7 +428,9 @@ private void registerRestOperations() { new SelectFromTablePostHandler(), new CountRowsGetHandler(), /* Status */ - new StatusInvocationHandler() + new StatusInvocationHandler(), + /* Vector */ + new LoadVectorsForIdsPostHandler(Config.sharedConfig().getDatabase().getSelectorSupplier()) )); } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java index 2ae194b4c..1158710a1 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/Main.java @@ -1,7 +1,14 @@ package org.vitrivr.cineast.api; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.concurrent.LinkedBlockingDeque; -import java.util.logging.Logger; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderJob; import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderWorker; @@ -13,83 +20,117 @@ import org.vitrivr.cineast.standalone.monitoring.PrometheusServer; import org.vitrivr.cineast.standalone.util.CLI; + public class Main { + private static final Logger LOGGER = LogManager.getLogger(); - /** - * Entrypoint for Cineast API application. - * - * @param args Program arguments. - */ - public static void main(String[] args) { - /* (Force) load application config. */ - if (args.length == 0) { - System.out.println("No config path given, loading default config '" + DEFAULT_CONFIG_PATH + "'"); - if (Config.loadConfig(DEFAULT_CONFIG_PATH) == null) { - System.err.println("Failed to load Cineast configuration from '" + DEFAULT_CONFIG_PATH + "'. Cineast API will shutdown..."); - System.exit(1); - } - } + /** + * Entrypoint for Cineast API application. + * + * @param args Program arguments. + */ + public static void main(String[] args) { + /* (Force) load application config. */ + if (args.length == 0) { + System.out.println("No config path given, loading default config '" + DEFAULT_CONFIG_PATH + "'"); + if (Config.loadConfig(DEFAULT_CONFIG_PATH) == null) { + System.err.println("Failed to load Cineast configuration from '" + DEFAULT_CONFIG_PATH + "'. Cineast API will shutdown..."); + System.exit(1); + } + } - /* (Force) load application config. */ - if (args.length != 0) { - if (Config.loadConfig(args[0]) == null) { - System.err.println("Failed to load Cineast configuration from '" + args[0] + "'. Cineast API will shutdown..."); - System.exit(1); - } - } + /* (Force) load application config. */ + if (args.length != 0) { + if (Config.loadConfig(args[0]) == null) { + System.err.println("Failed to load Cineast configuration from '" + args[0] + "'. Cineast API will shutdown..."); + System.exit(1); + } + } - /* Start API endpoint. */ - try { - APIEndpoint.getInstance().start(); - } catch (Throwable e) { - e.printStackTrace(); - System.err.println("Failed to initialize API endpoint due to an exception: " + e.getMessage()); - } + /* Start API endpoint. */ + try { + APIEndpoint.getInstance().start(); + } catch (Throwable e) { + e.printStackTrace(); + System.err.println("Failed to initialize API endpoint due to an exception: " + e.getMessage()); + } - /* Start gRPC endpoint. */ - try { - GRPCEndpoint.start(); - } catch (Throwable e) { - e.printStackTrace(); - System.err.println("Failed to initialize gRPC endpoint due to an exception: " + e.getMessage()); - } + /* Start gRPC endpoint. */ + try { + GRPCEndpoint.start(); + } catch (Throwable e) { + e.printStackTrace(); + System.err.println("Failed to initialize gRPC endpoint due to an exception: " + e.getMessage()); + } - /* Initialize Monitoring */ - try { - PrometheusServer.initialize(); - } catch (Throwable e) { - e.printStackTrace(); - System.err.println("Failed to initialize Monitoring due to an exception: " + e.getMessage()); - } - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - System.out.println("Shutting down endpoints..."); - APIEndpoint.stop(); - GRPCEndpoint.stop(); - PrometheusServer.stopServer(); - if (RenderWorker.getRenderJobQueue() != null) { - RenderWorker.getRenderJobQueue().add(new RenderJob(JobControlCommand.SHUTDOWN_WORKER)); - } - System.out.println("Goodbye!"); - })); - - if (Config.sharedConfig().getExtractor().getEnableRenderWorker()) { - /* Initialize Renderer */ - var renderThread = new Thread(new RenderWorker(new LinkedBlockingDeque<>()), "RenderWorker"); - renderThread.start(); - } + /* Initialize Monitoring */ + try { + PrometheusServer.initialize(); + } catch (Throwable e) { + e.printStackTrace(); + System.err.println("Failed to initialize Monitoring due to an exception: " + e.getMessage()); + } + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + System.out.println("Shutting down endpoints..."); + APIEndpoint.stop(); + GRPCEndpoint.stop(); + PrometheusServer.stopServer(); + if (RenderWorker.getRenderJobQueue() != null) { + RenderWorker.getRenderJobQueue().add(new RenderJob(JobControlCommand.SHUTDOWN_WORKER)); + } + System.out.println("Goodbye!"); + })); + + if (Config.sharedConfig().getExtractor().getEnableRenderWorker()) { + /* Initialize Renderer */ + var renderThread = new Thread(new RenderWorker(new LinkedBlockingDeque<>()), "RenderWorker"); + renderThread.start(); + } + + if (Config.sharedConfig().getApi().getEnableExternalClip()) { + /* Startup Clip Python Endpoint */ + // TODO: Make this configurable + Path condaEnvironmentPath = Path.of( "C:/Users/walten0000/.conda/envs/openclip/python.exe"); + Path scriptPath = Path.of( "./cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py"); + + var processParameters = new ArrayList(); + processParameters.add(condaEnvironmentPath.toString()); + processParameters.add(scriptPath.toString()); + + var processBuilder = new ProcessBuilder(processParameters); + //processBuilder.command("python", "./cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py"); + processBuilder.redirectErrorStream(true); + + var processBuilderThread = new Thread(()->{ + Process process = null; + try { + process = processBuilder.start(); + process.info(); + } catch (IOException e) { + throw new RuntimeException(e); + } + var reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + LOGGER.info("Starting OpenClip Python Endpoint"); + reader.lines().forEach(l -> LOGGER.info("External Clip: " + l)); + + } + ); + processBuilderThread.start(); + + } - try { - /* Start Cineast CLI in interactive mode (blocking). */ - if (Config.sharedConfig().getApi().getEnableCli()) { - CLI.start(CineastCli.class); - } else { - while (true) { - Thread.sleep(100); + try { + /* Start Cineast CLI in interactive mode (blocking). */ + if (Config.sharedConfig().getApi().getEnableCli()) { + CLI.start(CineastCli.class); + } else { + while (true) { + Thread.sleep(100); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); } - } - } catch (InterruptedException e) { - e.printStackTrace(); + System.exit(0); } - System.exit(0); - } } diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/VectorLookup.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/VectorLookup.java new file mode 100644 index 000000000..a5597dcac --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/query/VectorLookup.java @@ -0,0 +1,8 @@ +package org.vitrivr.cineast.api.messages.query; + +import java.util.Map; +import org.vitrivr.cineast.api.messages.lookup.IdList; + +public record VectorLookup(IdList ids, String feature, String projection, Map properties) { + +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorList.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorList.java new file mode 100644 index 000000000..9901c030c --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorList.java @@ -0,0 +1,7 @@ +package org.vitrivr.cineast.api.messages.result; + +import java.util.List; + +public record IdVectorList(List points) { + +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorPair.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorPair.java new file mode 100644 index 000000000..ffe46a260 --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/messages/result/IdVectorPair.java @@ -0,0 +1,5 @@ +package org.vitrivr.cineast.api.messages.result; + +public record IdVectorPair(String id, float[] vector){ + +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/vector/LoadVectorsForIdsPostHandler.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/vector/LoadVectorsForIdsPostHandler.java new file mode 100644 index 000000000..f28691a0c --- /dev/null +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/rest/handlers/actions/vector/LoadVectorsForIdsPostHandler.java @@ -0,0 +1,133 @@ +package org.vitrivr.cineast.api.rest.handlers.actions.vector; + +import io.javalin.http.Context; +import io.javalin.plugin.openapi.dsl.OpenApiBuilder; +import io.javalin.plugin.openapi.dsl.OpenApiDocumentation; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.vitrivr.cineast.api.messages.query.VectorLookup; +import org.vitrivr.cineast.api.messages.result.IdVectorList; +import org.vitrivr.cineast.api.messages.result.IdVectorPair; +import org.vitrivr.cineast.api.rest.handlers.interfaces.ParsingPostRestHandler; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.db.DBSelector; +import org.vitrivr.cineast.core.db.DBSelectorSupplier; +import tagbio.umap.Umap; + +public class LoadVectorsForIdsPostHandler implements ParsingPostRestHandler { + + private final DBSelectorSupplier selectorSupply; + + public LoadVectorsForIdsPostHandler(DBSelectorSupplier selectorSupply) { + this.selectorSupply = selectorSupply; + } + + @Override + public OpenApiDocumentation docs() { + return OpenApiBuilder.document() + .operation(op -> { + op.summary("Loads the vectors of a particular feature, applies optional projection"); + op.description("Loads the vectors of a particular feature, applies optional projection"); + op.operationId("loadVectors"); + op.addTagsItem("Vectors"); + }) + .body(inClass()) + .json("200", outClass()); + } + + @Override + public IdVectorList performPost(VectorLookup input, Context ctx) { + + DBSelector selector = this.selectorSupply.get(); + selector.open("feature_" + input.feature()); + List> rows = selector.getRows("feature", input.ids().ids(), "vector_lookup"); + + List ids = new ArrayList<>(input.ids().ids().size()); + List vectors = new ArrayList<>(input.ids().ids().size()); + + for (Map row : rows) { + ids.add(row.get("id").getString()); + vectors.add(row.get("feature").getFloatArray()); + } + + selector.close(); + + switch (input.projection().toLowerCase()) { + case "umap" -> { + + Umap umap = new Umap(); + + umap.setMetric( + input.properties().getOrDefault("metric", "cosine") + ); + + umap.setNumberComponents( + Integer.parseInt( + input.properties().getOrDefault("components", "3") + ) + ); + + umap.setNumberNearestNeighbours( + Integer.parseInt( + input.properties().getOrDefault("nearestNeighbours", "15") + ) + ); + + umap.setThreads( + Integer.parseInt( + input.properties().getOrDefault("threads", Runtime.getRuntime().availableProcessors() + "") + ) + ); + + float[][] data = new float[vectors.size()][]; + + for (int i = 0; i < vectors.size(); ++i) { + data[i] = vectors.get(i); + } + + float[][] transformed = umap.fitTransform(data); + + List pairs = new ArrayList<>(ids.size()); + + for (int i = 0; i < ids.size(); ++i) { + pairs.add(new IdVectorPair(ids.get(i), transformed[i])); + } + + return new IdVectorList(pairs); + + } + case "tsne" -> throw new IllegalStateException("tsne projection not implemented"); + + + case "raw" -> { + + List pairs = new ArrayList<>(ids.size()); + + for (int i = 0; i < ids.size(); ++i) { + pairs.add(new IdVectorPair(ids.get(i), vectors.get(i))); + } + + return new IdVectorList(pairs); + + } + } + + throw new IllegalArgumentException("Projection " + input.projection() + " not known"); + } + + @Override + public Class inClass() { + return VectorLookup.class; + } + + @Override + public Class outClass() { + return IdVectorList.class; + } + + @Override + public String route() { + return "/find/vectors"; + } +} diff --git a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java index b4dde917d..be9e4d8ce 100644 --- a/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java +++ b/cineast-api/src/main/java/org/vitrivr/cineast/api/util/QueryUtil.java @@ -279,7 +279,7 @@ public static List retrieveFeaturesForIDByCategory(String id, String cat final DBSelector selector = Config.sharedConfig().getDatabase().getSelectorSupplier().get(); List _return = new ArrayList<>(); retrievalRuntimeConfig.getRetrieversByCategory(category).forEach((ObjectDoubleProcedure) (retriever, weight) -> { - retriever.getTableNames().forEach(tableName -> { + retriever.getEntityNames().forEach(tableName -> { selector.open(tableName); List> rows = selector.getRows(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(id)); rows.stream().map(row -> row.get(FEATURE_COLUMN_QUALIFIER).toObject()).forEach(_return::add); @@ -346,7 +346,7 @@ private static Map>> getFeaturesForCat Map>> _return = new HashMap<>(); retrievalRuntimeConfig.getRetrieversByCategory(category).forEach((ObjectDoubleProcedure) (retriever, weight) -> { - retriever.getTableNames().forEach(tableName -> _return.put(tableName, getFeaturesFromEntity(tableName, ids))); + retriever.getEntityNames().forEach(tableName -> _return.put(tableName, getFeaturesFromEntity(tableName, ids))); }); return _return; diff --git a/cineast-api/src/main/python/open_clip_lion_text_feature_proxy.py b/cineast-api/src/main/python/open_clip_lion_text_feature_proxy.py new file mode 100644 index 000000000..03a863d72 --- /dev/null +++ b/cineast-api/src/main/python/open_clip_lion_text_feature_proxy.py @@ -0,0 +1,123 @@ +import base64 +import os.path + +import torch +import open_clip +from flask import Flask, request +import json +from PIL import Image +import base64 +from io import BytesIO +from datetime import datetime +import requests +import argparse + + +# parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +# parser.add_argument('--device', type=str, help='Device to use for feature extraction.', default='cpu') +# parser.add_argument('--port', type=int, help='Port to listen on.', default=8888) +# +# args = parser.parse_args() +model, preprocess_train , preprocess_val = open_clip.create_model_and_transforms('xlm-roberta-base-ViT-B-32', pretrained='laion5b_s13b_b90k') +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = model.to(device) +tokenizer = open_clip.get_tokenizer('xlm-roberta-base-ViT-B-32') + +app = Flask(__name__) +app.secret_key = 'BAD_SECRET_KEY' + + +@app.route('/heartbeat', methods=['GET']) +def hearthbeat(): + return "Beat dev" + + +@app.route('/', methods=['POST', 'GET']) +def handle_request(): + if request.method == 'POST': + query = request.form['query'] + if query is None: + return "[]" + + return json.dumps(feature(query).tolist()) + + return "requests only supported vis POST" + + +@app.route('/image', methods=['POST']) +def handle_image_request(): + image_data = request.form['image'] + debug_storeimage = False + + dateTimeObj = datetime.now() + file_name_for_base64_data = dateTimeObj.strftime("%d-%b-%Y--(%H-%M-%S)") + + print("try to embed image :" + file_name_for_base64_data) + + # File naming process for directory form data. + # We are taken the last 8 characters from the url string. + # file_name_for_regular_data = url[-10:-4] + featureVector = None + try: + # Base64 DATA + if "data:image/jpeg;base64," in image_data: + base_string = image_data.replace("data:image/jpeg;base64,", "") + decoded_img = base64.b64decode(base_string) + img = Image.open(BytesIO(decoded_img)) + featureVector = imagefeature(img) + if debug_storeimage == True: + file_name = file_name_for_base64_data + ".jpg" + img.save(f"./debugImages/{file_name}", "jpg") + + + # Base64 DATA + elif "data:image/png;base64," in image_data: + base_string = image_data.replace("data:image/png;base64,", "") + decoded_img = base64.b64decode(base_string) + img = Image.open(BytesIO(decoded_img)) + featureVector = imagefeature(img) + if debug_storeimage == True: + file_name = file_name_for_base64_data + ".png" + img.save(f"./debugImages/{file_name}", "png") + + + # Regular URL Form DATA + else: + response = requests.get(image_data) + img = Image.open(BytesIO(response.content)).convert("RGB") + featureVector = imagefeature(img) + featureVector = imagefeature(img) + if debug_storeimage == True: + file_name = "file_name_for_regular_data" + ".jpg" + img.save(f"./debugImages/{file_name}", "jpeg") + + status = "Image has been succesfully sent to the server." + except Exception as ex: + status = "Error! = " + str(ex) + print("Error! = " + str(ex)) + return status + + + return json.dumps(featureVector.tolist()) + + +def imagefeature(imagequery): + device = "cuda" if torch.cuda.is_available() else "cpu" + image = preprocess_val(imagequery).unsqueeze(0).to(device) + with torch.no_grad(): + image_features = model.encode_image(image) + image_features /= image_features.norm(dim=-1, keepdim=True) + return image_features.cpu().numpy().flatten() + + +def feature(query): + device = "cuda" if torch.cuda.is_available() else "cpu" + text = tokenizer(query).to(device) + with torch.no_grad(): + text_features = model.encode_text(text) + text_features /= text_features.norm(dim=-1, keepdim=True) + return text_features.cpu().numpy().flatten() + + +if __name__ == '__main__': + app.run(port=8888) diff --git a/cineast-api/src/main/python/openclip.yaml b/cineast-api/src/main/python/openclip.yaml new file mode 100644 index 000000000..ca9833b7a --- /dev/null +++ b/cineast-api/src/main/python/openclip.yaml @@ -0,0 +1,128 @@ +name: openclip +channels: + - defaults +dependencies: + - abseil-cpp=20211102.0=hd77b12b_0 + - aiohttp=3.8.5=py311h2bbff1b_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - arrow-cpp=11.0.0=ha81ea56_2 + - async-timeout=4.0.2=py311haa95532_0 + - attrs=23.1.0=py311haa95532_0 + - aws-c-common=0.6.8=h2bbff1b_1 + - aws-c-event-stream=0.1.6=hd77b12b_6 + - aws-checksums=0.1.11=h2bbff1b_2 + - aws-sdk-cpp=1.8.185=hd77b12b_1 + - blas=1.0=mkl + - boost-cpp=1.82.0=h59b6b97_2 + - bottleneck=1.3.5=py311h5bb9823_0 + - brotli-python=1.0.9=py311hd77b12b_7 + - bzip2=1.0.8=he774522_0 + - c-ares=1.19.1=h2bbff1b_0 + - ca-certificates=2023.08.22=haa95532_0 + - certifi=2023.7.22=py311haa95532_0 + - cffi=1.15.1=py311h2bbff1b_3 + - chardet=4.0.0=py311haa95532_1003 + - click=8.1.7=py311haa95532_0 + - colorama=0.4.6=py311haa95532_0 + - cryptography=41.0.3=py311h89fc84f_0 + - datasets=2.12.0=py311haa95532_0 + - dill=0.3.6=py311haa95532_0 + - filelock=3.9.0=py311haa95532_0 + - flask=2.2.2=py311haa95532_0 + - frozenlist=1.3.3=py311h2bbff1b_0 + - gflags=2.2.2=ha925a31_0 + - glog=0.5.0=hd77b12b_0 + - grpc-cpp=1.48.2=hfe90ff0_1 + - huggingface_hub=0.17.3=py311haa95532_0 + - idna=3.4=py311haa95532_0 + - importlib-metadata=6.0.0=py311haa95532_0 + - intel-openmp=2023.1.0=h59b6b97_46319 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jinja2=3.1.2=py311haa95532_0 + - libboost=1.82.0=h3399ecb_2 + - libbrotlicommon=1.0.9=h2bbff1b_7 + - libbrotlidec=1.0.9=h2bbff1b_7 + - libbrotlienc=1.0.9=h2bbff1b_7 + - libcurl=8.1.1=h86230a5_0 + - libevent=2.1.12=h56d1f94_1 + - libffi=3.4.4=hd77b12b_0 + - libprotobuf=3.20.3=h23ce68f_0 + - libssh2=1.10.0=he2ea4bf_2 + - libthrift=0.15.0=h4364b78_2 + - libuv=1.44.2=h2bbff1b_0 + - lz4-c=1.9.4=h2bbff1b_0 + - markupsafe=2.1.1=py311h2bbff1b_0 + - mkl=2023.1.0=h6b88ed4_46357 + - mkl-service=2.4.0=py311h2bbff1b_1 + - mkl_fft=1.3.8=py311h2bbff1b_0 + - mkl_random=1.2.4=py311h59b6b97_0 + - mpmath=1.3.0=py311haa95532_0 + - multidict=6.0.2=py311h2bbff1b_0 + - multiprocess=0.70.14=py311haa95532_0 + - networkx=3.1=py311haa95532_0 + - ninja=1.10.2=haa95532_5 + - ninja-base=1.10.2=h6d14046_5 + - numexpr=2.8.7=py311h1fcbade_0 + - numpy=1.26.0=py311hdab7c0b_0 + - numpy-base=1.26.0=py311hd01c5d8_0 + - openssl=3.0.11=h2bbff1b_2 + - orc=1.7.4=h623e30f_1 + - pandas=2.1.1=py311hf62ec03_0 + - pip=23.3=py311haa95532_0 + - pyarrow=11.0.0=py311h8a3a540_1 + - pycparser=2.21=pyhd3eb1b0_0 + - pyopenssl=23.2.0=py311haa95532_0 + - pysocks=1.7.1=py311haa95532_0 + - python=3.11.5=he1021f5_0 + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python-xxhash=2.0.2=py311h2bbff1b_1 + - pytz=2023.3.post1=py311haa95532_0 + - pyyaml=6.0.1=py311h2bbff1b_0 + - re2=2022.04.01=hd77b12b_0 + - regex=2023.10.3=py311h2bbff1b_0 + - requests=2.31.0=py311haa95532_0 + - responses=0.13.3=pyhd3eb1b0_0 + - safetensors=0.4.0=py311hcbdf901_0 + - setuptools=68.0.0=py311haa95532_0 + - six=1.16.0=pyhd3eb1b0_1 + - snappy=1.1.9=h6c2663c_0 + - sqlite=3.41.2=h2bbff1b_0 + - sympy=1.11.1=py311haa95532_0 + - tbb=2021.8.0=h59b6b97_0 + - tk=8.6.12=h2bbff1b_0 + - tokenizers=0.13.3=py311h49fca51_0 + - transformers=4.32.1=py311haa95532_0 + - typing-extensions=4.7.1=py311haa95532_0 + - typing_extensions=4.7.1=py311haa95532_0 + - tzdata=2023c=h04d1e81_0 + - utf8proc=2.6.1=h2bbff1b_0 + - vc=14.2=h21ff451_1 + - vs2015_runtime=14.27.29016=h5e58377_2 + - waitress=2.0.0=pyhd3eb1b0_0 + - werkzeug=2.2.3=py311haa95532_0 + - wheel=0.41.2=py311haa95532_0 + - win_inet_pton=1.1.0=py311haa95532_0 + - xxhash=0.8.0=h2bbff1b_3 + - xz=5.4.2=h8cc25b3_0 + - yaml=0.2.5=he774522_0 + - yarl=1.8.1=py311h2bbff1b_0 + - zipp=3.11.0=py311haa95532_0 + - zlib=1.2.13=h8cc25b3_0 + - zstd=1.5.5=hd43e919_0 + - pip: + - charset-normalizer==3.3.1 + - fsspec==2023.10.0 + - ftfy==6.1.1 + - huggingface-hub==0.18.0 + - open-clip-torch==2.23.0 + - packaging==23.2 + - pillow==10.1.0 + - protobuf==4.24.4 + - sentencepiece==0.1.99 + - timm==0.9.8 + - torch==2.1.0 + - torchvision==0.16.0 + - tqdm==4.66.1 + - urllib3==2.0.7 + - wcwidth==0.2.8 diff --git a/cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py b/cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py new file mode 100644 index 000000000..a9e0075c6 --- /dev/null +++ b/cineast-api/src/main/python/serve_open_clip_lion_text_feature_proxy.py @@ -0,0 +1,19 @@ +from waitress import serve +import open_clip_lion_text_feature_proxy as app +import argparse + + +def main(args): + print(f"Serving on {args.ip}, listen on {args.port}. Clip on {args.device}") + serve(app.app, host=args.ip, port=args.port) + + +if __name__ == '__main__': + print("start server") + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--device', type=str, help='Device to use for feature extraction.', default='cpu') + parser.add_argument('--port', type=int, help='Port to listen on.', default=8888) + parser.add_argument('--ip', type=str, help='Ip to serve on.', default='127.0.0.1') + args = parser.parse_args() + + main(args) diff --git a/cineast-core/build.gradle b/cineast-core/build.gradle index fd3a3dfd6..a607353f3 100644 --- a/cineast-core/build.gradle +++ b/cineast-core/build.gradle @@ -81,7 +81,7 @@ signing { sign publishing.publications.mavenJava } -project.ext.lwjglVersion = "3.3.1" +project.ext.lwjglVersion = "3.3.3" switch (OperatingSystem.current()) { case OperatingSystem.LINUX: diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/CorrespondenceFunctionEnum.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/CorrespondenceFunctionEnum.java new file mode 100644 index 000000000..1257c3110 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/CorrespondenceFunctionEnum.java @@ -0,0 +1,5 @@ +package org.vitrivr.cineast.core.data; + +public enum CorrespondenceFunctionEnum { + IDENTITY, LINEAR, HYPERBOLIC +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java index 1601f5779..ec0e142a4 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/MediaType.java @@ -12,7 +12,7 @@ public enum MediaType { VIDEO(0, "v", "video"), IMAGE(1, "i", "image"), AUDIO(2, "a", "audio"), - MODEL3D(3, "m", "3dmodel"), + MODEL3D(3, "m", "m3d"), TEXTUREMODEL3D(5, "mt", "3dtexturemodel"), IMAGE_SEQUENCE(4, "is", "imagesequence"), UNKNOWN(99, "u", "unknown"); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/IModel.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/IModel.java index 954f05b98..80a44478a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/IModel.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/IModel.java @@ -36,4 +36,6 @@ public interface IModel { */ List getAllNormals(); + boolean usesNonDefaultTexture(); + } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Material.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Material.java index 874c68904..5a7221818 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Material.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Material.java @@ -10,7 +10,8 @@ import org.vitrivr.cineast.core.data.m3d.texturemodel.util.MinimalBoundingBox; /** - * The Material contains all meshes and the texture that are drawn with on the meshes Further it contains the diffuse color of the material + * The Material contains all meshes and the texture that are drawn with on the meshes Further it contains the diffuse + * color of the material */ public class Material { @@ -25,6 +26,7 @@ public class Material { * Texture that drawn on all meshes */ private Texture texture; + private Texture normalMapTexture; /** * DEFAULT_COLOR is black and 100% opaque @@ -34,7 +36,11 @@ public class Material { /** * diffuseColor is the color that is drawn on the meshes when no texture is present */ - private Vector4f diffuseColor; + private final Vector4f diffuseColor; + + private final Vector4f ambientColor; + private float reflectance; + private final Vector4f specularColor; /** * Empty material that can be used as a placeholder. @@ -47,7 +53,10 @@ public class Material { public Material() { this.meshes = new ArrayList<>(); this.texture = new Texture(); + this.normalMapTexture = null; this.diffuseColor = DEFAULT_COLOR; + this.ambientColor = new Vector4f(0.0f, 0.0f, 0.0f, 1.0f);; + this.specularColor = DEFAULT_COLOR; } /** @@ -116,6 +125,26 @@ public void setTexture(Texture texture) { this.texture = texture; } + + public boolean hasNonDefaultTexture(){ + return !this.texture.isDefault(); + } + + /** + * @param texture sets the texture to this material + */ + public void setNormalTexture(Texture texture) { + this.normalMapTexture = texture; + } + + public Texture getNormalMapTexture() { + return this.normalMapTexture; + } + + public boolean hasNormalMapTexture() { + return this.normalMapTexture != null; + } + /** * @return the diffuse color of this material */ @@ -127,19 +156,42 @@ public Vector4f getDiffuseColor() { * @param diffuseColor sets the diffuse color of this material */ public void setDiffuseColor(Vector4f diffuseColor) { - this.diffuseColor = diffuseColor; + this.diffuseColor.set(diffuseColor); + } + + + public Vector4f getAmbientColor() { + return this.ambientColor; + } + + public void setAmbientColor(Vector4f ambientColor) { + this.ambientColor.set(ambientColor); + } + + public float getReflectance() { + return this.reflectance; + } + + public void setReflectance(float reflectance) { + this.reflectance = reflectance; + } + + public Vector4f getSpecularColor() { + return this.specularColor; + } + + public void setSpecularColor(Vector4f specularColor) { + this.specularColor.set(specularColor); } /** - * closes all resources the material uses - * Calls close on all containing classes + * closes all resources the material uses Calls close on all containing classes */ public void close() { this.meshes.forEach(Mesh::close); this.meshes.clear(); this.texture.close(); this.texture = null; - this.diffuseColor = null; LOGGER.trace("Closed Material"); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Mesh.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Mesh.java index e5d67e655..9beeee9ab 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Mesh.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Mesh.java @@ -60,11 +60,21 @@ public class Mesh { /** * List of all vertices normals in the mesh - * TODO: not used yet, will be used for vertex shading */ - @SuppressWarnings("all") private final float[] normals; + + /** + * List of all vertices normals in the mesh + */ + private final float[] tangents; + + /** + * List of all vertices normals in the mesh + */ + private final float[] bitangents; + + /** * MinimalBoundingBox that encloses the mesh */ @@ -84,12 +94,15 @@ public class Mesh { * @param textureCoordinates List of all texture coordinates in the mesh * @param idx List of all vertices ids. */ - public Mesh(float[] positions, float[] normals, float[] textureCoordinates, int[] idx) { + public Mesh(float[] positions, float[] normals, float[] tangents, float[] bitangents, float[] textureCoordinates, int[] idx) { //Stores all the data this.positions = positions; this.idx = idx; this.numVertices = idx.length; this.normals = normals; + this.tangents = tangents; + this.bitangents = bitangents; + // List to store results of face normals calculation this.facenormals = new ArrayList<>(this.numVertices / 3); //this.areas = new ArrayList<>(positions.length / 3); @@ -97,8 +110,8 @@ public Mesh(float[] positions, float[] normals, float[] textureCoordinates, int[ // Calculate face normals // ic increments by 3 because a face is defined by 3 vertices - for (var ic = 0; ic < this.idx.length; ic += 3) { - if (normals == null) { + for (var ic = 0; ic < this.idx.length-2; ic += 3) { + if (normals == null || normals.length == 0) { // Add zero vector if there are no vertex normals this.facenormals.add(new Vector3f(0f, 0f, 0f)); } else { @@ -111,7 +124,7 @@ public Mesh(float[] positions, float[] normals, float[] textureCoordinates, int[ var vn2 = new Vector3f(normals[idx[ic + 1] * 3], normals[idx[ic + 1] * 3 + 1], normals[idx[ic + 1] * 3 + 2]); var vn3 = new Vector3f(normals[idx[ic + 2] * 3], normals[idx[ic + 2] * 3 + 1], normals[idx[ic + 2] * 3 + 2]); // Instance the face normal - var fn = new Vector3f(0, 0, 0); + var fn = new Vector3f(0.0F, 0.0F, 0.0F); // Calculate the direction of the face normal by averaging the three vertex normals fn.add(vn1).add(vn2).add(vn3).div(3).normalize(); // Instance the face area @@ -163,10 +176,20 @@ public int[] getIdx() { /** * @return list containing all face normals */ - public List getNormals() { + public List getFaceNormals() { return this.facenormals; } + public float[] getVerticesNormals() { + return this.normals; + } + public float[] getTangents() { + return this.tangents; + } + public float[] getBitangents() { + return this.bitangents; + } + /** * @return the MinimalBoundingBox which contains the scaling factor to norm and the translation to origin (0,0,0) */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Model.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Model.java index 797e680fe..0fa970d28 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Model.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Model.java @@ -6,7 +6,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.joml.Vector3f; -import org.joml.Vector4f; import org.vitrivr.cineast.core.data.m3d.texturemodel.util.MinimalBoundingBox; /** @@ -109,7 +108,7 @@ public List getMaterials() { @Override public List getAllNormals() { var normals = new ArrayList(); - this.materials.forEach(m -> m.getMeshes().forEach(mesh -> normals.addAll(mesh.getNormals()))); + this.materials.forEach(m -> m.getMeshes().forEach(mesh -> normals.addAll(mesh.getFaceNormals()))); return normals; } @@ -123,4 +122,12 @@ public void close(){ this.entities.clear(); LOGGER.trace("Closed model {}", this.id); } + + public boolean usesNonDefaultTexture() { + var nonDefaultTexture = false; + for (var material : this.materials) { + nonDefaultTexture |= material.hasNonDefaultTexture(); + } + return nonDefaultTexture; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/ModelLoader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/ModelLoader.java index b550226ff..2923a4d1a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/ModelLoader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/ModelLoader.java @@ -1,331 +1,398 @@ package org.vitrivr.cineast.core.data.m3d.texturemodel; -import static org.lwjgl.assimp.Assimp.AI_MATKEY_COLOR_DIFFUSE; -import static org.lwjgl.assimp.Assimp.aiGetMaterialColor; -import static org.lwjgl.assimp.Assimp.aiGetMaterialTexture; -import static org.lwjgl.assimp.Assimp.aiImportFile; -import static org.lwjgl.assimp.Assimp.aiProcess_CalcTangentSpace; -import static org.lwjgl.assimp.Assimp.aiProcess_FixInfacingNormals; -import static org.lwjgl.assimp.Assimp.aiProcess_GlobalScale; -import static org.lwjgl.assimp.Assimp.aiProcess_JoinIdenticalVertices; -import static org.lwjgl.assimp.Assimp.aiProcess_LimitBoneWeights; -import static org.lwjgl.assimp.Assimp.aiProcess_PreTransformVertices; -import static org.lwjgl.assimp.Assimp.aiProcess_Triangulate; -import static org.lwjgl.assimp.Assimp.aiReleaseImport; -import static org.lwjgl.assimp.Assimp.aiReturn_SUCCESS; -import static org.lwjgl.assimp.Assimp.aiTextureType_DIFFUSE; -import static org.lwjgl.assimp.Assimp.aiTextureType_NONE; - import java.io.File; import java.nio.IntBuffer; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.*; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.joml.Vector4f; -import org.lwjgl.assimp.AIColor4D; -import org.lwjgl.assimp.AIFace; -import org.lwjgl.assimp.AIMaterial; -import org.lwjgl.assimp.AIMesh; -import org.lwjgl.assimp.AIString; -import org.lwjgl.assimp.AIVector3D; +import org.lwjgl.assimp.*; import org.lwjgl.system.MemoryStack; +import org.vitrivr.cineast.core.data.m3d.texturemodel.util.TextureLoadException; +import org.vitrivr.cineast.core.data.m3d.texturemodel.util.TimeLimitedFunc; + +import static org.lwjgl.assimp.Assimp.*; public final class ModelLoader { - private static final Logger LOGGER = LogManager.getLogger(); - - /** - * Loads a model from a file. Generates all the standard flags for Assimp. For more details see Assimp. - *
    - *
  • aiProcess_GenSmoothNormals: - * This is ignored if normals are already there at the time this flag - * is evaluated. Model importers try to load them from the source file, so - * they're usually already there. - * This flag may not be specified together with - * #aiProcess_GenNormals. There's a configuration option, - * #AI_CONFIG_PP_GSN_MAX_SMOOTHING_ANGLE which allows you to specify - * an angle maximum for the normal smoothing algorithm. Normals exceeding - * this limit are not smoothed, resulting in a 'hard' seam between two faces. - * Using a decent angle here (e.g. 80 degrees) results in very good visual - * appearance. - *
    • - *
  • aiProcess_JoinIdenticalVertices:
    • - *
  • aiProcess_Triangulate By default the imported mesh data might contain faces with more than 3 - * indices. For rendering you'll usually want all faces to be triangles. - * This post processing step splits up faces with more than 3 indices into - * triangles. Line and point primitives are *not* modified! If you want - * 'triangles only' with no other kinds of primitives, try the following - * solution: - *
      - *
    • Specify both #aiProcess_Triangulate and #aiProcess_SortByPType
      • - * Ignore all point and line meshes when you process assimp's output - *
    - *
    • - *
  • aiProcess_FixInf acingNormals: - * This step tries to determine which meshes have normal vectors that are facing inwards and inverts them. - * The algorithm is simple but effective: the bounding box of all vertices + their normals is compared against - * the volume of the bounding box of all vertices without their normals. This works well for most objects, problems might occur with - * planar surfaces. However, the step tries to filter such cases. - * The step inverts all in-facing normals. Generally it is recommended to enable this step, although the result is not always correct. - *
    • - *
  • aiProcess_CalcTangentSpace: - * Calculates the tangents and bi tangents for the imported meshes - * Does nothing if a mesh does not have normals. - * You might want this post processing step to be executed if you plan to use tangent space calculations such as normal mapping applied to the meshes. - * There's an importer property, AI_CONFIG_PP_CT_MAX_SMOOTHING_ANGLE, which allows you to specify a maximum smoothing angle for the algorithm. - * However, usually you'll want to leave it at the default value. - *
    • - *
  • aiProcess_LimitBoneWeights: - * Limits the number of bones simultaneously affecting a single vertex to a maximum value. - * If any vertex is affected by more than the maximum number of bones, - * the least important vertex weights are removed and the remaining vertex weights are normalized so that the weights still sum up to 1. - * The default bone weight limit is 4 (defined as AI_LBW_MAX_WEIGHTS in config.h), - * but you can use the AI_CONFIG_PP_LBW_MAX_WEIGHTS importer property to supply your own limit to the post processing step. - * If you intend to perform the skinning in hardware, this post processing step might be of interest to you. - *
    • - *
  • aiProcess_PreTransformVertices: - * Removes the node graph and pre-transforms all vertices with the local transformation matrices of their nodes. - * If the resulting scene can be reduced to a single mesh, with a single material, no lights, and no cameras, - * then the output scene will contain only a root node (with no children) that references the single mesh. - * Otherwise, the output scene will be reduced to a root node with a single level of child nodes, each one referencing one mesh, - * and each mesh referencing one material - * In either case, for rendering, you can simply render all meshes in order - you don't need to pay attention to local transformations and the node hierarchy. - * Animations are removed during this step - * This step is intended for applications without a scenegraph. - * The step CAN cause some problems: if e.g. a mesh of the asset contains normals and another, using the same material index, - * does not, they will be brought together, but the first mesh's part of the normal list is zeroed. However, these artifacts are rare. - *
    • - *
- * - * @param modelId The ID of the model. - * @param modelPath Path to the model file. - * @return Model object. - */ - public static Model loadModel(String modelId, String modelPath) { - var model = loadModel(modelId, modelPath, - aiProcess_JoinIdenticalVertices | - aiProcess_GlobalScale | - aiProcess_FixInfacingNormals | - aiProcess_Triangulate | - aiProcess_CalcTangentSpace | - aiProcess_LimitBoneWeights | - aiProcess_PreTransformVertices); - LOGGER.trace("Try return Model 2"); - return model; - } - - /** - * Loads a model from a file. 1. Loads the model file to an aiScene. 2. Process all Materials. 3. Process all Meshes. 3.1 Process all Vertices. 3.2 Process all Normals. 3.3 Process all Textures. 3.4 Process all Indices. - * - * @param modelId Arbitrary unique ID of the model. - * @param modelPath Path to the model file. - * @param flags Flags for the model loading process. - * @return Model object. - */ - @SuppressWarnings("NullAway") - public static Model loadModel(String modelId, String modelPath, int flags) { - LOGGER.trace("Try loading file {} from {}", modelId, modelPath); - - var file = new File(modelPath); - if (!file.exists()) { - throw new RuntimeException("Model path does not exist [" + modelPath + "]"); - } - var modelDir = file.getParent(); + private static final Logger LOGGER = LogManager.getLogger(); - LOGGER.trace("Loading aiScene"); + /** + * Loads a model from a file. Generates all the standard flags for Assimp. For more details see Assimp. + *
    + *
  • aiProcess_GenSmoothNormals: + * This is ignored if normals are already there at the time this flag + * is evaluated. Model importers try to load them from the source file, so + * they're usually already there. + * This flag may not be specified together with + * #aiProcess_GenNormals. There's a configuration option, + * #AI_CONFIG_PP_GSN_MAX_SMOOTHING_ANGLE which allows you to specify + * an angle maximum for the normal smoothing algorithm. Normals exceeding + * this limit are not smoothed, resulting in a 'hard' seam between two faces. + * Using a decent angle here (e.g. 80 degrees) results in very good visual + * appearance. + *
    • + *
  • aiProcess_JoinIdenticalVertices:
    • + *
  • aiProcess_Triangulate By default the imported mesh data might contain faces with more than 3 + * indices. For rendering you'll usually want all faces to be triangles. + * This post processing step splits up faces with more than 3 indices into + * triangles. Line and point primitives are *not* modified! If you want + * 'triangles only' with no other kinds of primitives, try the following + * solution: + *
      + *
    • Specify both #aiProcess_Triangulate and #aiProcess_SortByPType
      • + * Ignore all point and line meshes when you process assimp's output + *
    + *
    • + *
  • aiProcess_FixInf acingNormals: + * This step tries to determine which meshes have normal vectors that are facing inwards and inverts them. + * The algorithm is simple but effective: the bounding box of all vertices + their normals is compared against + * the volume of the bounding box of all vertices without their normals. This works well for most objects, problems might occur with + * planar surfaces. However, the step tries to filter such cases. + * The step inverts all in-facing normals. Generally it is recommended to enable this step, although the result is not always correct. + *
    • + *
  • aiProcess_CalcTangentSpace: + * Calculates the tangents and bi tangents for the imported meshes + * Does nothing if a mesh does not have normals. + * You might want this post processing step to be executed if you plan to use tangent space calculations such as normal mapping applied to the meshes. + * There's an importer property, AI_CONFIG_PP_CT_MAX_SMOOTHING_ANGLE, which allows you to specify a maximum smoothing angle for the algorithm. + * However, usually you'll want to leave it at the default value. + *
    • + *
  • aiProcess_LimitBoneWeights: + * Limits the number of bones simultaneously affecting a single vertex to a maximum value. + * If any vertex is affected by more than the maximum number of bones, + * the least important vertex weights are removed and the remaining vertex weights are normalized so that the weights still sum up to 1. + * The default bone weight limit is 4 (defined as AI_LBW_MAX_WEIGHTS in config.h), + * but you can use the AI_CONFIG_PP_LBW_MAX_WEIGHTS importer property to supply your own limit to the post processing step. + * If you intend to perform the skinning in hardware, this post processing step might be of interest to you. + *
    • + *
  • aiProcess_PreTransformVertices: + * Removes the node graph and pre-transforms all vertices with the local transformation matrices of their nodes. + * If the resulting scene can be reduced to a single mesh, with a single material, no lights, and no cameras, + * then the output scene will contain only a root node (with no children) that references the single mesh. + * Otherwise, the output scene will be reduced to a root node with a single level of child nodes, each one referencing one mesh, + * and each mesh referencing one material + * In either case, for rendering, you can simply render all meshes in order - you don't need to pay attention to local transformations and the node hierarchy. + * Animations are removed during this step + * This step is intended for applications without a scenegraph. + * The step CAN cause some problems: if e.g. a mesh of the asset contains normals and another, using the same material index, + * does not, they will be brought together, but the first mesh's part of the normal list is zeroed. However, these artifacts are rare. + *
    • + *
+ * + * @param modelId The ID of the model. + * @param modelPath Path to the model file. + * @return Model object. + */ + public static Model loadModel(String modelId, String modelPath) throws TextureLoadException, TimeoutException { + var model = loadModel(modelId, modelPath, + aiProcess_JoinIdenticalVertices | + aiProcess_GlobalScale | + aiProcess_FixInfacingNormals | + aiProcess_Triangulate | + aiProcess_CalcTangentSpace | + aiProcess_LimitBoneWeights | + aiProcess_PreTransformVertices | + aiProcess_GenSmoothNormals | + aiProcess_FindInvalidData | + aiProcess_FindDegenerates | + aiProcess_ValidateDataStructure - // DO NOT USE AUTOCLOSEABLE TRY CATCH FOR AI-SCENE!!! THIS WILL CAUSE A FATAL ERROR ON NTH (199) ITERATION! - // RAPHAEL WALTENSPUEL 2023-01-20 - var aiScene = aiImportFile(modelPath, flags); - if (aiScene == null) { - throw new RuntimeException("Error loading model [modelPath: " + modelPath + "]"); + ); + LOGGER.trace("Try return Model 2"); + return model; } - var numMaterials = aiScene.mNumMaterials(); - List materialList = new ArrayList<>(); - for (var ic = 0; ic < numMaterials; ic++) { - //TODO: Warning - var aiMaterial = AIMaterial.create(aiScene.mMaterials().get(ic)); - LOGGER.trace("Try processing material {}", ic); - materialList.add(ModelLoader.processMaterial(aiMaterial, modelDir)); - } + /** + * Loads a model from a file. 1. Loads the model file to an aiScene. 2. Process all Materials. 3. Process all Meshes. + * 3.1 Process all Vertices. 3.2 Process all Normals. 3.3 Process all Textures. 3.4 Process all Indices. + * + * @param modelId Arbitrary unique ID of the model. + * @param modelPath Path to the model file. + * @param flags Flags for the model loading process. + * @return Model object. + */ + @SuppressWarnings("NullAway") + public static Model loadModel(String modelId, String modelPath, int flags) throws TextureLoadException, TimeoutException { + LOGGER.trace("Try loading file {} from {}", modelId, modelPath); - var numMeshes = aiScene.mNumMeshes(); - var aiMeshes = aiScene.mMeshes(); - var defaultMaterial = new Material(); - for (var ic = 0; ic < numMeshes; ic++) { - LOGGER.trace("Try create AI Mesh {}", ic); - //TODO: Warning - var aiMesh = AIMesh.create(aiMeshes.get(ic)); - var mesh = ModelLoader.processMesh(aiMesh); - LOGGER.trace("Try get Material idx"); - var materialIdx = aiMesh.mMaterialIndex(); - Material material; - if (materialIdx >= 0 && materialIdx < materialList.size()) { - material = materialList.get(materialIdx); - } else { - material = defaultMaterial; - } - LOGGER.trace("Try add Material to Mesh"); - material.addMesh(mesh); - } + var file = new File(modelPath); + if (!file.exists()) { + throw new RuntimeException("Model path does not exist [" + modelPath + "]"); + } + var modelDir = file.getParent(); - if (!defaultMaterial.getMeshes().isEmpty()) { - LOGGER.trace("Try add default Material"); - materialList.add(defaultMaterial); - } + LOGGER.trace("Loading aiScene"); + + + // DO NOT USE AUTOCLOSEABLE TRY CATCH FOR AI-SCENE!!! THIS WILL CAUSE A FATAL ERROR ON NTH (199) ITERATION! + // RAPHAEL WALTENSPUEL 2023-01-20 + var tlf = new TimeLimitedFunc<>(120, () -> aiImportFile(modelPath, flags)); + var aiScene = tlf.runWithTimeout(); + if (aiScene == null) { + throw new TextureLoadException("Error loading model [modelPath: " + modelPath + "]"); + } + + + + + + var numMaterials = aiScene.mNumMaterials(); + List materialList = new ArrayList<>(); + for (var ic = 0; ic < numMaterials; ic++) { + //TODO: Warning + var aiMaterial = AIMaterial.create(aiScene.mMaterials().get(ic)); + LOGGER.trace("Try processing material {}", ic); + materialList.add(ModelLoader.processMaterial(aiMaterial, modelDir)); + } + + var numMeshes = aiScene.mNumMeshes(); + var aiMeshes = aiScene.mMeshes(); + var defaultMaterial = new Material(); + for (var ic = 0; ic < numMeshes; ic++) { + LOGGER.trace("Try create AI Mesh {}", ic); + //TODO: Warning + var aiMesh = AIMesh.create(aiMeshes.get(ic)); + var mesh = ModelLoader.processMesh(aiMesh); + LOGGER.trace("Try get Material idx"); + var materialIdx = aiMesh.mMaterialIndex(); + Material material; + if (materialIdx >= 0 && materialIdx < materialList.size()) { + material = materialList.get(materialIdx); + } else { + material = defaultMaterial; + } + LOGGER.trace("Try add Material to Mesh"); + material.addMesh(mesh); + } + + if (!defaultMaterial.getMeshes().isEmpty()) { + LOGGER.trace("Try add default Material"); + materialList.add(defaultMaterial); + } + + LOGGER.trace("Try instantiate Model"); + aiReleaseImport(aiScene); - LOGGER.trace("Try instantiate Model"); - aiReleaseImport(aiScene); - - var model = new Model(modelId, materialList); - LOGGER.trace("Try return Model"); - return model; - } - - /** - * Convert indices from aiMesh to int array. - * - * @param aiMesh aiMesh to process. - * @return flattened int array of indices. - */ - private static int[] processIndices(AIMesh aiMesh) { - LOGGER.trace("Start processing indices"); - List indices = new ArrayList<>(); - var numFaces = aiMesh.mNumFaces(); - var aiFaces = aiMesh.mFaces(); - for (var ic = 0; ic < numFaces; ic++) { - AIFace aiFace = aiFaces.get(ic); - IntBuffer buffer = aiFace.mIndices(); - while (buffer.remaining() > 0) { - indices.add(buffer.get()); - } + var model = new Model(modelId, materialList); + LOGGER.trace("Try return Model"); + return model; } - LOGGER.trace("End processing indices"); - return indices.stream().mapToInt(Integer::intValue).toArray(); - } - - /** - * Convert an AIMaterial to a Material. Loads the diffuse color and texture. - * - * @param aiMaterial aiMaterial to process. - * @param modelDir Path to the model file. - * @return flattened float array of vertices. - */ - private static Material processMaterial(AIMaterial aiMaterial, String modelDir) { - LOGGER.trace("Start processing material"); - var material = new Material(); - try (MemoryStack stack = MemoryStack.stackPush()) { - AIColor4D color = AIColor4D.create(); - - //** Diffuse color if no texture is present - int result = aiGetMaterialColor( - aiMaterial, AI_MATKEY_COLOR_DIFFUSE, aiTextureType_NONE, 0, color); - if (result == aiReturn_SUCCESS) { - material.setDiffuseColor(new Vector4f(color.r(), color.g(), color.b(), color.a())); - } - - //** Try load texture - AIString aiTexturePath = AIString.calloc(stack); - aiGetMaterialTexture(aiMaterial, aiTextureType_DIFFUSE, 0, aiTexturePath, (IntBuffer) null, - null, null, null, null, null); - var texturePath = aiTexturePath.dataString(); - //TODO: Warning - if (texturePath != null && texturePath.length() > 0) { - material.setTexture(new Texture(modelDir + File.separator + new File(texturePath).toPath())); - material.setDiffuseColor(Material.DEFAULT_COLOR); - } - - return material; + + /** + * Convert indices from aiMesh to int array. + * + * @param aiMesh aiMesh to process. + * @return flattened int array of indices. + */ + private static int[] processIndices(AIMesh aiMesh) { + LOGGER.trace("Start processing indices"); + List indices = new ArrayList<>(); + var numFaces = aiMesh.mNumFaces(); + var aiFaces = aiMesh.mFaces(); + for (var ic = 0; ic < numFaces; ic++) { + AIFace aiFace = aiFaces.get(ic); + IntBuffer buffer = aiFace.mIndices(); + while (buffer.remaining() > 0) { + indices.add(buffer.get()); + } + } + LOGGER.trace("End processing indices"); + return indices.stream().mapToInt(Integer::intValue).toArray(); } - } - - /** - * Convert aiMesh to a Mesh. Loads the vertices, normals, texture coordinates and indices. - * Instantiates a new Mesh object. - * @param aiMesh aiMesh to process. - * @return flattened float array of normals. - */ - private static Mesh processMesh(AIMesh aiMesh) { - LOGGER.trace("Start processing mesh"); - var vertices = processVertices(aiMesh); - var normals = processNormals(aiMesh); - var textCoords = processTextCoords(aiMesh); - var indices = processIndices(aiMesh); - - // Texture coordinates may not have been populated. We need at least the empty slots - if (textCoords.length == 0) { - var numElements = (vertices.length / 3) * 2; - textCoords = new float[numElements]; + + /** + * Convert an AIMaterial to a Material. Loads the diffuse color and texture. + * + * @param aiMaterial aiMaterial to process. + * @param modelDir Path to the model file. + * @return flattened float array of vertices. + */ + private static Material processMaterial(AIMaterial aiMaterial, String modelDir) { + LOGGER.trace("Start processing material"); + var material = new Material(); + try (var stack = MemoryStack.stackPush()) { + var color = AIColor4D.create(); + + var result = aiGetMaterialColor(aiMaterial, AI_MATKEY_COLOR_AMBIENT, aiTextureType_NONE, 0, color); + if (result == aiReturn_SUCCESS) { + material.setAmbientColor(new Vector4f(color.r(), color.g(), color.b(), color.a())); + } + result = aiGetMaterialColor(aiMaterial, AI_MATKEY_COLOR_DIFFUSE, aiTextureType_NONE, 0, color); + if (result == aiReturn_SUCCESS) { + material.setDiffuseColor(new Vector4f(color.r(), color.g(), color.b(), color.a())); + } + result = aiGetMaterialColor(aiMaterial, AI_MATKEY_COLOR_DIFFUSE, aiTextureType_NONE, 0, color); + if (result == aiReturn_SUCCESS) { + material.setSpecularColor(new Vector4f(color.r(), color.g(), color.b(), color.a())); + } + + var reflectance = 0.0f; + var shininess = new float[]{0.0f}; + var pMax = new int[]{1}; + result = aiGetMaterialFloatArray(aiMaterial, AI_MATKEY_SHININESS_STRENGTH, aiTextureType_NONE, 0, shininess, + pMax); + if (result == aiReturn_SUCCESS) { + reflectance = shininess[0]; + } + material.setReflectance(reflectance); + + //** Try load texture + var aiTexturePath = AIString.calloc(stack); + aiGetMaterialTexture(aiMaterial, aiTextureType_DIFFUSE, 0, aiTexturePath, (IntBuffer) null, + null, null, null, null, null); + var texturePath = aiTexturePath.dataString(); + //TODO: Warning + if (texturePath != null && texturePath.length() > 0) { + material.setTexture(new Texture(modelDir + File.separator + new File(texturePath).toPath())); + material.setDiffuseColor(Material.DEFAULT_COLOR); + } + + // Try Load NormalMap + var aiNormalMapPath = AIString.calloc(stack); + aiGetMaterialTexture(aiMaterial, aiTextureType_NORMALS, 0, aiNormalMapPath, (IntBuffer) null, + null, null, null, null, null); + var normalMapPath = aiNormalMapPath.dataString(); + if (normalMapPath != null && normalMapPath.length() > 0) { + material.setNormalTexture(new Texture(modelDir + File.separator + new File(normalMapPath).toPath())); + } + return material; + } } - LOGGER.trace("End processing mesh"); - return new Mesh(vertices, normals, textCoords, indices); - } - - /** - * Convert normals from aiMesh to float array. - * - * @param aiMesh aiMesh to process. - * @return flattened float array of normals. - */ - private static float[] processNormals(AIMesh aiMesh) { - LOGGER.trace("Start processing Normals"); - var buffer = aiMesh.mNormals(); - if (buffer == null) { - return null; + + /** + * Convert aiMesh to a Mesh. Loads the vertices, normals, texture coordinates and indices. Instantiates a new Mesh + * object. + * + * @param aiMesh aiMesh to process. + * @return flattened float array of normals. + */ + private static Mesh processMesh(AIMesh aiMesh) { + LOGGER.trace("Start processing mesh"); + var vertices = processVertices(aiMesh); + var normals = processNormals(aiMesh); + var textCoords = processTextCoords(aiMesh); + var indices = processIndices(aiMesh); + var tangents = processTangents(aiMesh, normals); + var bitangents = processBitangents(aiMesh, normals); + + + // Texture coordinates may not have been populated. We need at least the empty slots + if (textCoords.length == 0) { + var numElements = (vertices.length / 3) * 2; + textCoords = new float[numElements]; + } + LOGGER.trace("End processing mesh"); + return new Mesh(vertices, normals, tangents, bitangents, textCoords, indices); } - var data = new float[buffer.remaining() * 3]; - var pos = 0; - while (buffer.remaining() > 0) { - var normal = buffer.get(); - data[pos++] = normal.x(); - data[pos++] = normal.y(); - data[pos++] = normal.z(); + + /** + * Convert normals from aiMesh to float array. + * + * @param aiMesh aiMesh to process. + * @return flattened float array of normals. + */ + private static float[] processNormals(AIMesh aiMesh) { + LOGGER.trace("Start processing Normals"); + var buffer = aiMesh.mNormals(); + if (buffer == null) { + return new float[aiMesh.mNumVertices() * 3]; + } + var data = new float[buffer.remaining() * 3]; + var pos = 0; + while (buffer.remaining() > 0) { + var normal = buffer.get(); + data[pos++] = normal.x(); + data[pos++] = normal.y(); + data[pos++] = normal.z(); + } + return data; } - return data; - } - - /** - * Convert texture coordinates from aiMesh to float array. - * @param aiMesh aiMesh to process. - * @return flattened float array of texture coordinates. - */ - private static float[] processTextCoords(AIMesh aiMesh) { - LOGGER.trace("Start processing Coordinates"); - var buffer = aiMesh.mTextureCoords(0); - if (buffer == null) { - return new float[]{}; + + /** + * Convert texture coordinates from aiMesh to float array. + * + * @param aiMesh aiMesh to process. + * @return flattened float array of texture coordinates. + */ + private static float[] processTextCoords(AIMesh aiMesh) { + LOGGER.trace("Start processing Coordinates"); + var buffer = aiMesh.mTextureCoords(0); + if (buffer == null) { + return new float[]{}; + } + float[] data = new float[buffer.remaining() * 2]; + int pos = 0; + while (buffer.remaining() > 0) { + AIVector3D textCoord = buffer.get(); + data[pos++] = textCoord.x(); + data[pos++] = 1 - textCoord.y(); + } + return data; } - float[] data = new float[buffer.remaining() * 2]; - int pos = 0; - while (buffer.remaining() > 0) { - AIVector3D textCoord = buffer.get(); - data[pos++] = textCoord.x(); - data[pos++] = 1 - textCoord.y(); + + /** + * Convert vertices from aiMesh to float array. + * + * @param aiMesh aiMesh to process. + * @return flattened float array of vertices. + */ + private static float[] processVertices(AIMesh aiMesh) { + LOGGER.trace("Start processing Vertices"); + AIVector3D.Buffer buffer = aiMesh.mVertices(); + float[] data = new float[buffer.remaining() * 3]; + int pos = 0; + while (buffer.remaining() > 0) { + AIVector3D textCoord = buffer.get(); + data[pos++] = textCoord.x(); + data[pos++] = textCoord.y(); + data[pos++] = textCoord.z(); + } + + return data; } - return data; - } - - /** - * Convert vertices from aiMesh to float array. - * - * @param aiMesh aiMesh to process. - * @return flattened float array of vertices. - */ - private static float[] processVertices(AIMesh aiMesh) { - LOGGER.trace("Start processing Vertices"); - AIVector3D.Buffer buffer = aiMesh.mVertices(); - float[] data = new float[buffer.remaining() * 3]; - int pos = 0; - while (buffer.remaining() > 0) { - AIVector3D textCoord = buffer.get(); - data[pos++] = textCoord.x(); - data[pos++] = textCoord.y(); - data[pos++] = textCoord.z(); + + private static float[] processBitangents(AIMesh aiMesh, float[] normals) { + var buffer = aiMesh.mBitangents(); + if (buffer == null) { + return new float[normals.length]; + } + var data = new float[buffer.remaining() * 3]; + var pos = 0; + while (buffer.remaining() > 0) { + var bitangent = buffer.get(); + data[pos++] = bitangent.x(); + data[pos++] = bitangent.y(); + data[pos++] = bitangent.z(); + } + + if (data.length == 0) { + data = new float[normals.length]; + } + return data; } - return data; - } + private static float[] processTangents(AIMesh aiMesh, float[] normals) { + var buffer = aiMesh.mTangents(); + if (buffer == null) { + return new float[normals.length]; + } + var data = new float[buffer.remaining() * 3]; + var pos = 0; + while (buffer.remaining() > 0) { + var tangent = buffer.get(); + data[pos++] = tangent.x(); + data[pos++] = tangent.y(); + data[pos++] = tangent.z(); + } + if (data.length == 0) { + data = new float[normals.length]; + } + return data; + } } \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Texture.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Texture.java index ce4e9c392..55d242088 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Texture.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/Texture.java @@ -2,6 +2,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.lwjgl.vulkan.video.StdVideoEncodeH264SliceHeader; /** * This class represents a texture. @@ -47,6 +48,10 @@ public String getTexturePath() { return this.texturePath; } + public boolean isDefault(){ + return this.texturePath.equals(DEFAULT_TEXTURE); + } + /** * Releases all resources associated with this Texture. */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/util/TextureLoadException.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/util/TextureLoadException.java new file mode 100644 index 000000000..78315b264 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/util/TextureLoadException.java @@ -0,0 +1,13 @@ +package org.vitrivr.cineast.core.data.m3d.texturemodel.util; + +public class TextureLoadException extends Exception +{ + // Parameterless Constructor + public TextureLoadException() {} + + // Constructor that accepts a message + public TextureLoadException(String message) + { + super(message); + } +} \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/util/TimeLimitedFunc.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/util/TimeLimitedFunc.java new file mode 100644 index 000000000..b9ffa0d56 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/m3d/texturemodel/util/TimeLimitedFunc.java @@ -0,0 +1,67 @@ +package org.vitrivr.cineast.core.data.m3d.texturemodel.util; + +import java.util.concurrent.*; +import java.util.function.Supplier; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import static org.lwjgl.assimp.Assimp.aiImportFile; + +public class TimeLimitedFunc { + + private static final Logger LOGGER = LogManager.getLogger(); + + /** Time limit in seconds */ + private final long timeLimit; + + /** Supplier of T, which is the function to be executed and timed */ + private final Supplier sup; + + /** Executor service to run the task */ + final ExecutorService executor = Executors.newSingleThreadExecutor(); + + /** + * Return value + */ + T value = null; + + + /** + * Constructor for TimeLimitedFunc + * @param timeLimit time limit in seconds + * @param sup Supplier of T + */ + public TimeLimitedFunc(long timeLimit, Supplier sup) { + this.timeLimit = timeLimit; + this.sup = sup; + } + + /** + * Run the function with a time limit + * @return T + * @throws TimeoutException if the function takes longer than the time limit + */ + public T runWithTimeout() throws TimeoutException { + LOGGER.trace("Start running function with time limit of " + this.timeLimit + " seconds."); + Runnable task = () -> { + this.value = sup.get(); + }; + Future future = executor.submit(task); + try { + future.get(this.timeLimit, TimeUnit.SECONDS); + } catch (TimeoutException e) { + LOGGER.error("Error Timeout ", e); + if (!executor.isTerminated()) { + executor.shutdownNow(); + } + throw e; + } catch (Exception e) { + LOGGER.error("Error Timeout ", e); + e.printStackTrace(); + } + LOGGER.trace("Function completed within time limit."); + return this.value; + } +} + diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java index fa8083b60..1d647f7f2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/data/segments/SegmentContainer.java @@ -26,7 +26,7 @@ /** * A {@link SegmentContainer} mainly serves two purposes: *

- * During the offline phase, it is passed to an {@link Extractor}, and during the online phase, it is passed to a {@link Retriever}. + * During the offline phase, it is passed to an {@link Extractor} (and represents the media from which features are supposed to be extracted), and during the online phase, it is passed to a {@link Retriever} (and represents the query of a user). */ public interface SegmentContainer extends IdProvider, diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java index 35d7f4cdc..c115d40d0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/DBSelector.java @@ -86,18 +86,9 @@ default List getNearestNeighboursGeneric(int k, P List> getNearestNeighbourRows(int k, float[] vector, String column, ReadableQueryConfig queryConfig); /** - * SELECT 'vectorname' from entity where 'column' = 'value' + * SELECT 'featureColName' from entity where 'column' = 'value' */ - List getFeatureVectors(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig); - - /** - * Conversion to PrimitiveTypeProviders is expensive so underlying classes should feel free to override if they wish to optimize for performance - *

- * takes the float[] method by default - */ - default List getFeatureVectorsGeneric(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig qc) { - return getFeatureVectors(column, value, vectorName, qc).stream().map(FloatArrayTypeProvider::new).collect(Collectors.toList()); - } + List getFeatures(String column, PrimitiveTypeProvider value, String featureColName, ReadableQueryConfig queryConfig); /** * {@link #getRows(String, Iterable, String)} @@ -309,7 +300,7 @@ default List> getAll(String order, int skip, } /** - * Get all rows from the tables (SELECT * FROM table) + * Get all rows from the entities (SELECT * FROM table) */ List> getAll(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java index abd5859f6..cf2a4c321 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/ImporterSelector.java @@ -5,7 +5,6 @@ import java.io.File; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -32,6 +31,7 @@ public abstract class ImporterSelector> implements DBSelec private static final Logger LOGGER = LogManager.getLogger(); private final File baseDirectory; private File file; + protected ImporterSelector(File baseDirectory) { this.baseDirectory = baseDirectory; } @@ -159,8 +159,8 @@ public List> getNearestNeighbourRows(int k, f } @Override - public List getFeatureVectors(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { - ArrayList _return = new ArrayList<>(1); + public List getFeatures(String column, PrimitiveTypeProvider value, String featureColName, ReadableQueryConfig queryConfig) { + ArrayList _return = new ArrayList<>(1); if (value == null || value.getString().isEmpty()) { return _return; @@ -172,11 +172,11 @@ public List getFeatureVectors(String column, PrimitiveTypeProvider valu if (!map.containsKey(column)) { continue; } - if (!map.containsKey(vectorName)) { + if (!map.containsKey(featureColName)) { continue; } if (value.equals(map.get(column).getString())) { - _return.add(PrimitiveTypeProvider.getSafeFloatArray(map.get(vectorName))); + _return.add(map.get(featureColName)); } } @@ -185,19 +185,33 @@ public List getFeatureVectors(String column, PrimitiveTypeProvider valu @Override public List> getRows(String column, Iterable values, String dbQueryId) { - if (values == null) { - return new ArrayList<>(0); + ArrayList> _return = new ArrayList<>(1); + + if (values == null || !values.iterator().hasNext()) { + return _return; } - ArrayList tmp = new ArrayList<>(); + ArrayList valueList = new ArrayList<>(); for (PrimitiveTypeProvider value : values) { - tmp.add(value); + valueList.add(value); } - PrimitiveTypeProvider[] valueArr = new PrimitiveTypeProvider[tmp.size()]; - tmp.toArray(valueArr); + Importer importer = newImporter(this.file); + Map map; + while ((map = importer.readNextAsMap()) != null) { + if (!map.containsKey(column)) { + continue; + } + for (var primitiveTypeProvider : valueList) { + if (primitiveTypeProvider.equals(map.get(column))) { + _return.add(map); + break; + } + } - return this.getRows(column, Arrays.asList(valueArr)); + } + + return _return; } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java index ca1cd35e7..4d5945bbc 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/NoDBSelector.java @@ -32,7 +32,7 @@ public List> getNearestNeighbourRows(int k, f } @Override - public List getFeatureVectors(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { + public List getFeatures(String column, PrimitiveTypeProvider value, String featureColName, ReadableQueryConfig queryConfig) { return new ArrayList<>(0); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentOperator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentOperator.java index 3649eba41..c0a39e305 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentOperator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/PersistentOperator.java @@ -22,7 +22,7 @@ public interface PersistentOperator { * * @return Tables which this {@link PersistentOperator} uses to store/access its data. */ - default List getTableNames() { + default List getEntityNames() { return new ArrayList<>(0); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java index b8c8e0f0e..fdb304ee3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailEntityCreator.java @@ -182,11 +182,11 @@ public boolean createSegmentEntity() { } @Override - public boolean createMetadataEntity(String tableName) { + public boolean createMetadataEntity(String entityName) { final long txId = this.cottontail.client.begin(); try { /* Create entity. */ - final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + tableName; + final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; final CreateEntity entity = new CreateEntity(fqn) .column(MediaObjectMetadataDescriptor.FIELDNAMES[0], Type.STRING, -1, false) .column(MediaObjectMetadataDescriptor.FIELDNAMES[1], Type.STRING, -1, false) @@ -196,8 +196,8 @@ public boolean createMetadataEntity(String tableName) { this.cottontail.client.create(entity); /* Create Index. */ - this.createIndex(tableName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.BTREE, txId); - this.createIndex(tableName, MediaObjectMetadataDescriptor.FIELDNAMES[2], IndexType.BTREE, txId); + this.createIndex(entityName, MediaObjectMetadataDescriptor.FIELDNAMES[0], IndexType.BTREE, txId); + this.createIndex(entityName, MediaObjectMetadataDescriptor.FIELDNAMES[2], IndexType.BTREE, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { @@ -207,11 +207,11 @@ public boolean createMetadataEntity(String tableName) { } @Override - public boolean createSegmentMetadataEntity(String tableName) { + public boolean createSegmentMetadataEntity(String entityName) { final long txId = this.cottontail.client.begin(); try { /* Create entity. */ - final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + tableName; + final String fqn = CottontailWrapper.CINEAST_SCHEMA + "." + entityName; final CreateEntity entity = new CreateEntity(fqn) .column(MediaSegmentMetadataDescriptor.FIELDNAMES[0], Type.STRING, -1, false) .column(MediaSegmentMetadataDescriptor.FIELDNAMES[1], Type.STRING, -1, false) @@ -221,8 +221,8 @@ public boolean createSegmentMetadataEntity(String tableName) { this.cottontail.client.create(entity); /* Create Index. */ - this.createIndex(tableName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.BTREE, txId); - this.createIndex(tableName, MediaSegmentMetadataDescriptor.FIELDNAMES[2], IndexType.BTREE, txId); + this.createIndex(entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], IndexType.BTREE, txId); + this.createIndex(entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[2], IndexType.BTREE, txId); this.cottontail.client.commit(txId); return true; } catch (StatusRuntimeException e) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java index 7a95ac6f3..e6fe48a4b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/cottontaildb/CottontailSelector.java @@ -6,6 +6,7 @@ import static org.vitrivr.cineast.core.util.CineastConstants.KEY_COL_NAME; import static org.vitrivr.cineast.core.util.DBQueryIdGenerator.generateQueryId; +import com.google.protobuf.TextFormat; import io.grpc.Status; import io.grpc.StatusRuntimeException; import java.util.ArrayList; @@ -228,6 +229,7 @@ public List getNearestNeighboursGeneric(int k, fl return handleNearestNeighbourResponse(this.cottontail.client.query(query), distanceElementClass); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getNearestNeighboursGeneric(): {}", e.getMessage()); + LOGGER.trace("Query: {}", query.getBuilder().toString()); return new ArrayList<>(0); } } @@ -249,28 +251,10 @@ public List> getNearestNeighbourRows(int k, f } @Override - public List getFeatureVectors(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { - final Query query = new Query(this.fqn).select(vectorName, null).where(new Expression(column, "==", value.toObject())).queryId(generateQueryId("get-fv", queryConfig)); + public List getFeatures(String column, PrimitiveTypeProvider value, String featureColName, ReadableQueryConfig queryConfig) { + final Query query = new Query(this.fqn).select(featureColName, null).where(new Expression(column, "==", value.toObject())).queryId(generateQueryId("get-fv-gen", queryConfig)); try { - final TupleIterator results = this.cottontail.client.query(query); - final List _return = new LinkedList<>(); - while (results.hasNext()) { - final Tuple t = results.next(); - _return.add(t.asFloatVector(vectorName)); - } - return _return; - } catch (StatusRuntimeException e) { - LOGGER.warn("Error occurred during query execution in getFeatureVectors(): {}", e.getMessage()); - return new ArrayList<>(0); - } - } - - - @Override - public List getFeatureVectorsGeneric(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig qc) { - final Query query = new Query(this.fqn).select(vectorName, null).where(new Expression(column, "==", value.toObject())).queryId(generateQueryId("get-fv-gen", qc)); - try { - return toSingleCol(this.cottontail.client.query(query), vectorName); + return toSingleCol(this.cottontail.client.query(query), featureColName); } catch (StatusRuntimeException e) { LOGGER.warn("Error occurred during query execution in getFeatureVectorsGeneric(): {}", e.getMessage()); return new ArrayList<>(0); @@ -324,6 +308,7 @@ private List> getRowsHelper(String fieldName, return new ArrayList<>(0); } } + @Override public List> getFulltextRows(int rows, String column, ReadableQueryConfig queryConfig, String... terms) { @@ -332,7 +317,7 @@ public List> getFulltextRows(int rows, String /* TODO Cottontail calls this a distance in its documentation, but it's actually a score. See the tests - that's why we order DESC and not ASC */ final Query query = new Query(this.fqn) - .select("id", null) + .select("*", null) .fulltext(column, predicate, DB_DISTANCE_VALUE_QUALIFIER) .queryId(generateQueryId("ft-rows", queryConfig)) .order(DB_DISTANCE_VALUE_QUALIFIER, Direction.DESC) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java index 089cc9af1..d8683e83f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/AbstractMetadataReader.java @@ -26,14 +26,14 @@ public abstract class AbstractMetadataReader extends AbstractEntityReader { private static final Logger LOGGER = LogManager.getLogger(); - private final String tableName; + private final String entityName; private final String idColName; - public AbstractMetadataReader(DBSelector selector, String tableName, String idColName) { + public AbstractMetadataReader(DBSelector selector, String entityName, String idColName) { super(selector); - this.tableName = tableName; + this.entityName = entityName; this.idColName = idColName; - this.selector.open(tableName); + this.selector.open(entityName); } public static List sanitizeIds(List ids) { @@ -80,7 +80,7 @@ public List findBySpec(List ids, List sp StopWatch watch = StopWatch.createStarted(); ids = sanitizeIds(ids); spec = sanitizeSpec(spec); - String dbQueryId = DBQueryIdGenerator.generateQueryId("find-md-spec-" + tableName, queryId); + String dbQueryId = DBQueryIdGenerator.generateQueryId("find-md-spec-" + entityName, queryId); List> results = selector.getMetadataByIdAndSpec(ids, spec, idColName, dbQueryId, null); LOGGER.debug("Performed metadata lookup for {} ids in {} ms. {} results.", ids.size(), watch.getTime(TimeUnit.MILLISECONDS), results.size()); return mapToResultList(results); @@ -101,7 +101,7 @@ public List findBySpec(List spec, String queryId } StopWatch watch = StopWatch.createStarted(); spec = sanitizeSpec(spec); - String dbQueryId = DBQueryIdGenerator.generateQueryId("find-my-spec-" + tableName, queryId); + String dbQueryId = DBQueryIdGenerator.generateQueryId("find-my-spec-" + entityName, queryId); List> results = selector.getMetadataBySpec(spec, dbQueryId, null); LOGGER.debug("Performed metadata lookup in {} ms. {} results.", watch.getTime(TimeUnit.MILLISECONDS), results.size()); return mapToResultList(results); @@ -144,12 +144,12 @@ public List sanitizeSpec(List el.type() != MetadataType.OBJECT)) { + if (Objects.equals(this.entityName, MediaObjectMetadataDescriptor.ENTITY) && spec.stream().anyMatch(el -> el.type() != MetadataType.OBJECT)) { LOGGER.trace("provided spec-list includes non-object tuples, but this is an object reader. These will be ignored."); spec = spec.stream().filter(el -> el.type() == MetadataType.OBJECT).collect(Collectors.toList()); } // filter non-segment specs if this is a segment reader - if (Objects.equals(this.tableName, MediaSegmentMetadataDescriptor.ENTITY) && spec.stream().anyMatch(el -> el.type() != MetadataType.SEGMENT)) { + if (Objects.equals(this.entityName, MediaSegmentMetadataDescriptor.ENTITY) && spec.stream().anyMatch(el -> el.type() != MetadataType.SEGMENT)) { LOGGER.trace("provided spec-list includes non-segment tuples, but this is a segment reader. These will be ignored."); spec = spec.stream().filter(el -> el.type() == MetadataType.SEGMENT).collect(Collectors.toList()); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectMetadataReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectMetadataReader.java index 20ef50de3..8289b1bb2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectMetadataReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaObjectMetadataReader.java @@ -11,8 +11,8 @@ public MediaObjectMetadataReader(DBSelector selector) { this(selector, MediaObjectMetadataDescriptor.ENTITY); } - public MediaObjectMetadataReader(DBSelector selector, String tableName) { - super(selector, tableName, MediaObjectMetadataDescriptor.FIELDNAMES[0]); + public MediaObjectMetadataReader(DBSelector selector, String entityName) { + super(selector, entityName, MediaObjectMetadataDescriptor.FIELDNAMES[0]); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentMetadataReader.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentMetadataReader.java index a59f901ad..f9bba9726 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentMetadataReader.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/reader/MediaSegmentMetadataReader.java @@ -11,8 +11,8 @@ public MediaSegmentMetadataReader(DBSelector selector) { this(selector, MediaSegmentMetadataDescriptor.ENTITY); } - public MediaSegmentMetadataReader(DBSelector selector, String tableName) { - super(selector, tableName, MediaSegmentMetadataDescriptor.FIELDNAMES[0]); + public MediaSegmentMetadataReader(DBSelector selector, String entityName) { + super(selector, entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[0]); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java index 806c79b37..d6113e94b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/AbstractBatchedEntityWriter.java @@ -5,6 +5,8 @@ import java.util.List; import java.util.Queue; import java.util.concurrent.ArrayBlockingQueue; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.db.PersistencyWriter; import org.vitrivr.cineast.core.db.PersistentTuple; @@ -24,6 +26,8 @@ public abstract class AbstractBatchedEntityWriter implements Closeable { */ protected PersistencyWriter writer; + private static final Logger LOGGER = LogManager.getLogger(); + protected AbstractBatchedEntityWriter(PersistencyWriter writer) { this.batch = writer.supportedBatchSize() > 1; if (this.batch) { @@ -79,15 +83,18 @@ public final void flush() { */ @Override public final void close() { - if (this.writer != null) { - if (this.batch && this.buffer.size() > 0) { - this.flush(); - } + if (this.writer == null) { + LOGGER.trace("Underlying writer is null, not doing anything upon close"); + return; + } + if (this.batch && this.buffer.size() > 0) { + LOGGER.debug("Flushing writer upon close"); + this.flush(); + } - if (this.writer != null) { - this.writer.close(); - this.writer = null; - } + if (this.writer != null) { + this.writer.close(); + this.writer = null; } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java index 404658ae4..14c2a97fa 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaObjectMetadataWriter.java @@ -8,17 +8,17 @@ public class MediaObjectMetadataWriter extends AbstractBatchedEntityWriter { - private final String tableName; + private final String entityName; public MediaObjectMetadataWriter(PersistencyWriter writer) { this(writer, MediaObjectMetadataDescriptor.ENTITY); } - public MediaObjectMetadataWriter(PersistencyWriter writer, String tableName) { + public MediaObjectMetadataWriter(PersistencyWriter writer, String entityName) { super(writer); - this.tableName = tableName; + this.entityName = entityName; this.writer.setFieldNames(MediaObjectMetadataDescriptor.FIELDNAMES); - this.writer.open(tableName); + this.writer.open(entityName); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentMetadataWriter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentMetadataWriter.java index 1a0551f2e..2f6cef96d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentMetadataWriter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/dao/writer/MediaSegmentMetadataWriter.java @@ -7,18 +7,18 @@ public class MediaSegmentMetadataWriter extends AbstractBatchedEntityWriter { - private final String tableName; + private final String entityName; - public MediaSegmentMetadataWriter(PersistencyWriter writer, String testSegMetaTableName) { + public MediaSegmentMetadataWriter(PersistencyWriter writer, String testSegMetaEntityName) { super(writer); - this.tableName = testSegMetaTableName; + this.entityName = testSegMetaEntityName; this.init(); } @Override protected void init() { this.writer.setFieldNames(MediaSegmentMetadataDescriptor.FIELDNAMES); - this.writer.open(tableName); + this.writer.open(entityName); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryEntityCreator.java index aa6b9b73c..fca8f62ca 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/memory/InMemoryEntityCreator.java @@ -30,8 +30,8 @@ public boolean createMultiMediaObjectsEntity() { } @Override - public boolean createMetadataEntity(String tableName) { - return this.store.createEntity(tableName, + public boolean createMetadataEntity(String entityName) { + return this.store.createEntity(entityName, MediaObjectMetadataDescriptor.FIELDNAMES[0], MediaObjectMetadataDescriptor.FIELDNAMES[1], MediaObjectMetadataDescriptor.FIELDNAMES[2], @@ -40,8 +40,8 @@ public boolean createMetadataEntity(String tableName) { } @Override - public boolean createSegmentMetadataEntity(String tableName) { - return this.store.createEntity(tableName, + public boolean createSegmentMetadataEntity(String entityName) { + return this.store.createEntity(entityName, MediaSegmentMetadataDescriptor.FIELDNAMES[0], MediaSegmentMetadataDescriptor.FIELDNAMES[1], MediaSegmentMetadataDescriptor.FIELDNAMES[2], diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyEntityCreator.java index f674c70be..e96e1e2c9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenyEntityCreator.java @@ -103,10 +103,10 @@ public boolean createSegmentEntity() { } @Override - public boolean createMetadataEntity(String tableName) { - final String entityName = PolyphenyWrapper.CINEAST_SCHEMA + "." + tableName; + public boolean createMetadataEntity(String entityName) { + var fullEntityName = PolyphenyWrapper.CINEAST_SCHEMA + "." + entityName; try (final Statement stmt = this.wrapper.connection.createStatement()) { - stmt.execute("CREATE TABLE " + entityName + " (" + + stmt.execute("CREATE TABLE " + fullEntityName + " (" + MediaObjectMetadataDescriptor.FIELDNAMES[0] + " VARCHAR(255) NOT NULL," + /* object_id */ MediaObjectMetadataDescriptor.FIELDNAMES[1] + " VARCHAR(255) NOT NULL," + /* domain */ MediaObjectMetadataDescriptor.FIELDNAMES[2] + " VARCHAR(255) NOT NULL," + /* key */ @@ -117,16 +117,16 @@ public boolean createMetadataEntity(String tableName) { /* TODO: Create index on object_id and domain. */ return true; } catch (SQLException e) { - LOGGER.error("Error occurred while creating entity {}: {}", entityName, e); + LOGGER.error("Error occurred while creating entity {}: {}", fullEntityName, e); return false; } } @Override - public boolean createSegmentMetadataEntity(String tableName) { - final String entityName = PolyphenyWrapper.CINEAST_SCHEMA + "." + tableName; + public boolean createSegmentMetadataEntity(String entityName) { + final String fullEntityName = PolyphenyWrapper.CINEAST_SCHEMA + "." + entityName; try (final Statement stmt = this.wrapper.connection.createStatement()) { - stmt.execute("CREATE TABLE " + entityName + " (" + + stmt.execute("CREATE TABLE " + fullEntityName + " (" + MediaSegmentMetadataDescriptor.FIELDNAMES[0] + " VARCHAR(255) NOT NULL," + /* segment_id */ MediaSegmentMetadataDescriptor.FIELDNAMES[1] + " VARCHAR(255) NOT NULL," + /* domain */ MediaSegmentMetadataDescriptor.FIELDNAMES[2] + " VARCHAR(255) NOT NULL ," + /* key */ @@ -137,7 +137,7 @@ public boolean createSegmentMetadataEntity(String tableName) { /* TODO: Create index on object_id and domain. */ return true; } catch (SQLException e) { - LOGGER.error("Error occurred while creating entity {}: {}", entityName, e); + LOGGER.error("Error occurred while creating entity {}: {}", fullEntityName, e); return false; } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java index dac5a2e13..a3f6f5b0c 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/polypheny/PolyphenySelector.java @@ -2,6 +2,7 @@ import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import boofcv.core.image.GConvertImage; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -25,6 +26,8 @@ import org.vitrivr.cineast.core.data.providers.primitive.BooleanProviderImpl; import org.vitrivr.cineast.core.data.providers.primitive.ByteProviderImpl; import org.vitrivr.cineast.core.data.providers.primitive.DoubleProviderImpl; +import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayProviderImpl; +import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider; import org.vitrivr.cineast.core.data.providers.primitive.FloatProviderImpl; import org.vitrivr.cineast.core.data.providers.primitive.IntProviderImpl; import org.vitrivr.cineast.core.data.providers.primitive.LongProviderImpl; @@ -226,15 +229,17 @@ public List> getNearestNeighbourRows(int k, f } @Override - public List getFeatureVectors(String column, PrimitiveTypeProvider value, String vectorName, ReadableQueryConfig queryConfig) { + public List getFeatures(String column, PrimitiveTypeProvider value, String featureColName, ReadableQueryConfig queryConfig) { try (final PreparedStatement statement = this.prepareStatement(column, RelationalOperator.EQ, List.of(value))) { /* Execute query and return results. */ - final List _return = new LinkedList<>(); + final List _return = new LinkedList<>(); try (final ResultSet rs = statement.executeQuery()) { while (rs.next()) { - final Object converted = rs.getArray(vectorName).getArray(); + final Object converted = rs.getArray(featureColName).getArray(); if (converted instanceof float[]) { - _return.add((float[]) converted); + _return.add(new FloatArrayTypeProvider((float[]) converted)); + } else { + LOGGER.warn("unsupported type {}", converted.getClass().getSimpleName()); } } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityCreator.java index 09ef21f28..21a56d601 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/EntityCreator.java @@ -38,7 +38,7 @@ default boolean createMetadataEntity() { *
* Do not use unless you know what you are doing, usage of {@link #createMetadataEntity()} is heavily recommended */ - boolean createMetadataEntity(String tableName); + boolean createMetadataEntity(String entityName); /** @@ -53,7 +53,7 @@ default boolean createSegmentMetadataEntity() { *
* Do not use unless you know what you are doing, usage of {@link #createSegmentMetadataEntity()} is heavily recommended */ - boolean createSegmentMetadataEntity(String tableName); + boolean createSegmentMetadataEntity(String entityName); /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java index 18e6646c6..699c04057 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/db/setup/NoEntityCreator.java @@ -14,7 +14,7 @@ public boolean createMetadataEntity() { } @Override - public boolean createMetadataEntity(String tableName) { + public boolean createMetadataEntity(String entityName) { return false; } @@ -24,7 +24,7 @@ public boolean createSegmentMetadataEntity() { } @Override - public boolean createSegmentMetadataEntity(String tableName) { + public boolean createSegmentMetadataEntity(String entityName) { return false; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/TextureModelDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/TextureModelDecoder.java index 6b220cc01..29dd84542 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/TextureModelDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/m3d/TextureModelDecoder.java @@ -5,127 +5,136 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.config.CacheConfig; import org.vitrivr.cineast.core.config.DecoderConfig; +import org.vitrivr.cineast.core.data.m3d.texturemodel.util.TextureLoadException; import org.vitrivr.cineast.core.extraction.decode.general.Decoder; import org.vitrivr.cineast.core.data.m3d.texturemodel.Model; import org.vitrivr.cineast.core.data.m3d.texturemodel.ModelLoader; public class TextureModelDecoder implements Decoder { - /** - * Default logging facility. - */ - private static final Logger LOGGER = LogManager.getLogger(); - - /** - * HashSet containing all the mime-types supported by this ImageDecoder instance. - */ - private static final Set supportedFiles; - - static { - HashSet tmp = new HashSet<>(); - tmp.add("application/3dt-gltf"); - tmp.add("application/3dt-obj"); - supportedFiles = Collections.unmodifiableSet(tmp); - } - - /** - * Path to the input file. - */ - private Path inputFile; - - /** - * Flag indicating whether or not the Decoder is done decoding and the content has been obtained. - */ - private AtomicBoolean complete = new AtomicBoolean(false); - - /** - * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. - * - * @param path Path to the file that should be decoded. - * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. - * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} - * @return True if initialization was successful, false otherwise. - */ - @Override - public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { - this.inputFile = path; - this.complete.set(false); - return true; - } - - /** - * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. - * - * @return Content of type T. - */ - @Override - public Model getNext() { - Model model = null; - - try { - model = ModelLoader.loadModel( this.inputFile.toString() ,this.inputFile.toString()); - } catch (NumberFormatException e) { - LOGGER.error("Could not decode OBJ file {} because one of the tokens could not be converted to a valid number.", this.inputFile.toString()); - model = null; - } catch (ArrayIndexOutOfBoundsException e) { - LOGGER.error("Could not decode OBJ file {} because one of the faces points to invalid vertex indices.", this.inputFile.toString()); - model = null; - } finally { - this.complete.set(true); + /** + * Default logging facility. + */ + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * HashSet containing all the mime-types supported by this ImageDecoder instance. + */ + private static final Set supportedFiles; + + static { + HashSet tmp = new HashSet<>(); + tmp.add("application/3dt-gltf"); + tmp.add("application/3dt-obj"); + supportedFiles = Collections.unmodifiableSet(tmp); + } + + /** + * Path to the input file. + */ + private Path inputFile; + + /** + * Flag indicating whether or not the Decoder is done decoding and the content has been obtained. + */ + private AtomicBoolean complete = new AtomicBoolean(false); + + /** + * Initializes the decoder with a file. This is a necessary step before content can be retrieved from the decoder by means of the getNext() method. + * + * @param path Path to the file that should be decoded. + * @param decoderConfig {@link DecoderConfig} used by this {@link Decoder}. + * @param cacheConfig The {@link CacheConfig} used by this {@link Decoder} + * @return True if initialization was successful, false otherwise. + */ + @Override + public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheConfig) { + this.inputFile = path; + this.complete.set(false); + return true; + } + + /** + * Fetches the next piece of content of type T and returns it. This method can be safely invoked until complete() returns false. From which on this method will return null. + * + * @return Content of type T. + */ + @Override + public Model getNext() { + Model model = null; + + try { + model = ModelLoader.loadModel(this.inputFile.toString(), this.inputFile.toString()); + } catch (NumberFormatException e) { + LOGGER.error("Could not decode file {} because one of the tokens could not be converted to a valid number.", this.inputFile.toString()); + model = null; + } catch (ArrayIndexOutOfBoundsException e) { + LOGGER.error("Could not decode file {} because one of the faces points to invalid vertex indices.", this.inputFile.toString()); + model = null; + } catch (TextureLoadException e) { + LOGGER.error("Could not decode file {} because one of the faces points to invalid vertex indices.", this.inputFile.toString()); + model = null; + } catch (Exception e) { + LOGGER.error("Could not decode file {} because an unexpected error occurred.", this.inputFile.toString(), e); + model = null; + + } finally { + this.complete.set(true); + } + + return model; + } + + /** + * Returns the total number of content pieces T this decoder can return for a given file. + */ + @Override + public int count() { + return 1; } - return model; - } - - /** - * Returns the total number of content pieces T this decoder can return for a given file. - */ - @Override - public int count() { - return 1; - } - - /** - * Closes the Decoder. This method should cleanup and relinquish all resources. - *

- * Note: It is unsafe to re-use a Decoder after it has been closed. - */ - @Override - public void close() { - } - - /** - * Indicates whether or not a particular instance of the Decoder interface can be re-used or not. This property can be leveraged to reduce the memory-footpring of the application. - * - * @return True if re-use is possible, false otherwise. - */ - @Override - public boolean canBeReused() { - return true; - } - - /** - * Indicates whether or not the current decoder instance is complete i.e. if there is content left that can be obtained. - * - * @return true if there is still content, false otherwise. - */ - @Override - public boolean complete() { - return this.complete.get(); - } - - /** - * Returns a set of the mime/types of supported files. - * - * @return Set of the mime-type of file formats that are supported by the current Decoder instance. - */ - @Override - public Set supportedFiles() { - return supportedFiles; - } + /** + * Closes the Decoder. This method should cleanup and relinquish all resources. + *

+ * Note: It is unsafe to re-use a Decoder after it has been closed. + */ + @Override + public void close() { + } + + /** + * Indicates whether or not a particular instance of the Decoder interface can be re-used or not. This property can be leveraged to reduce the memory-footpring of the application. + * + * @return True if re-use is possible, false otherwise. + */ + @Override + public boolean canBeReused() { + return true; + } + + /** + * Indicates whether or not the current decoder instance is complete i.e. if there is content left that can be obtained. + * + * @return true if there is still content, false otherwise. + */ + @Override + public boolean complete() { + return this.complete.get(); + } + + /** + * Returns a set of the mime/types of supported files. + * + * @return Set of the mime-type of file formats that are supported by the current Decoder instance. + */ + @Override + public Set supportedFiles() { + return supportedFiles; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java index 32c9bdd03..1325f5752 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java @@ -395,6 +395,7 @@ public boolean init(Path path, DecoderConfig decoderConfig, CacheConfig cacheCon this.factory = cacheConfig.sharedCachedDataFactory(); /* Initialize the AVFormatContext. */ + this.pFormatCtx = null; this.pFormatCtx = avformat.avformat_alloc_context(); /* Open video file. */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/ParentFolderNameObjectIdGenerator.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/ParentFolderNameObjectIdGenerator.java index 8c0ec34bd..a96ba51d7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/ParentFolderNameObjectIdGenerator.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/idgenerator/ParentFolderNameObjectIdGenerator.java @@ -21,16 +21,23 @@ public class ParentFolderNameObjectIdGenerator implements ObjectIdGenerator { * Prefix property name. Can be set to false to prevent type prefix in object ID. */ private static final String PROPERTY_NAME_PREFIX = "prefix"; + private static final String PROPERTY_NAME_DEPTH = "depth"; private final boolean prefix; + private final int depth; public ParentFolderNameObjectIdGenerator() { prefix = true; + depth = 0; } public ParentFolderNameObjectIdGenerator(Map properties) { String prefixProp = properties.get(PROPERTY_NAME_PREFIX); - prefix = prefixProp == null || prefixProp.equalsIgnoreCase("true"); + this.prefix = prefixProp == null || prefixProp.equalsIgnoreCase("true"); + + String depthProp = properties.get(PROPERTY_NAME_DEPTH); + this.depth = depthProp == null ? 0 : Integer.parseInt(depthProp); + assert depth >= 0 && depth < 10; } @Override @@ -40,8 +47,15 @@ public Optional next(Path path, MediaType type) { return Optional.empty(); } - String filename = FilenameUtils.getBaseName(path.toFile().getParentFile().getName()); + String filename = FilenameUtils.getBaseName(getFolderName(path, this.depth)); String id = prefix ? MediaType.generateId(type, filename) : filename; return Optional.of(id); } + + private String getFolderName(Path path, int depth) { + if (depth == 0) { + return path.toFile().getParentFile().getName(); + } + return getFolderName(path.getParent(), depth - 1); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8.java index c4f17366d..cf0a07ff1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorGrid8.java @@ -29,8 +29,8 @@ public AverageColorGrid8() { super("features_AverageColorGrid8", 12595f / 4f, 192); } - protected AverageColorGrid8(String tableName, float maxDist) { - super(tableName, maxDist, 192); + protected AverageColorGrid8(String entityName, float maxDist) { + super(entityName, maxDist, 192); } protected static Pair partition(MultiImage img) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java index 7868311da..9a2475ed6 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AverageColorRaster.java @@ -299,7 +299,7 @@ public List getSimilar(String segmentId, ReadableQueryConfig qc) { @Override public void initalizePersistentLayer(Supplier supply) { - supply.get().createFeatureEntity(this.tableName, true, new AttributeDefinition("hist", AttributeType.VECTOR, 15), new AttributeDefinition("raster", AttributeType.VECTOR, 64)); + supply.get().createFeatureEntity(this.entityName, true, new AttributeDefinition("hist", AttributeType.VECTOR, 15), new AttributeDefinition("raster", AttributeType.VECTOR, 64)); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS.java index ce13f7ff9..09de75299 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CENS.java @@ -62,12 +62,12 @@ public abstract class CENS extends StagedFeatureModule { /** * Default constructor. * - * @param tableName Name of the entity (for persistence writer). + * @param entityName Name of the entity (for persistence writer). * @param minFrequency Minimum frequency to consider during HPCP analysis. * @param maxFrequency Maximum frequency to consider during HPCP analysis. */ - public CENS(String tableName, float minFrequency, float maxFrequency) { - super(tableName, 2.0f, SHINGLE_SIZE * HPCP.Resolution.FULLSEMITONE.bins); + public CENS(String entityName, float minFrequency, float maxFrequency) { + super(entityName, 2.0f, SHINGLE_SIZE * HPCP.Resolution.FULLSEMITONE.bins); /* Apply fields. */ this.minFrequency = minFrequency; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPText.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPText.java index cda37297b..f80ea6a90 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPText.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/CLIPText.java @@ -7,7 +7,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.function.Supplier; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -17,7 +16,6 @@ import org.tensorflow.ndarray.NdArrays; import org.tensorflow.ndarray.Shape; import org.tensorflow.ndarray.buffer.DataBuffers; -import org.tensorflow.ndarray.buffer.FloatDataBuffer; import org.tensorflow.types.TFloat16; import org.tensorflow.types.TInt64; import org.vitrivr.cineast.core.config.QueryConfig; @@ -127,7 +125,7 @@ private static float[] exec(HashMap inputMap) { @Override public List getSimilar(String segmentId, ReadableQueryConfig qc) { - List list = this.selector.getFeatureVectorsGeneric(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER, qc); + List list = this.selector.getFeatures(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER, qc); if (list.isEmpty()) { LOGGER.warn("No feature vector for shotId {} found, returning empty result-list", segmentId); return Collections.emptyList(); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java index ab21668b7..5bb45d7b5 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/DCTImageHash.java @@ -125,7 +125,7 @@ public void processSegment(SegmentContainer shot) { @Override public void init(PersistencyWriterSupplier phandlerSupply) { this.phandler = phandlerSupply.get(); - this.writer = new SimpleBitSetWriter(this.phandler, this.tableName); + this.writer = new SimpleBitSetWriter(this.phandler, this.entityName); } @Override @@ -145,7 +145,7 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc @Override public void initalizePersistentLayer(Supplier supply) { - supply.get().createEntity(this.tableName, + supply.get().createEntity(this.entityName, new AttributeDefinition(GENERIC_ID_COLUMN_QUALIFIER, AttributeDefinition.AttributeType.STRING), new AttributeDefinition(FEATURE_COLUMN_QUALIFIER, AttributeType.BITSET, 64) ); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java new file mode 100644 index 000000000..d144bdea8 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalOpenClipText.java @@ -0,0 +1,364 @@ +package org.vitrivr.cineast.core.features; + +import com.fasterxml.jackson.databind.json.JsonMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.tensorflow.Result; +import org.tensorflow.Tensor; +import org.tensorflow.ndarray.Shape; +import org.tensorflow.ndarray.buffer.DataBuffers; +import org.tensorflow.types.TFloat32; +import org.vitrivr.cineast.core.config.QueryConfig; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.CorrespondenceFunction; +import org.vitrivr.cineast.core.data.FloatVectorImpl; +import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.distance.DistanceElement; +import org.vitrivr.cineast.core.data.distance.SegmentDistanceElement; +import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; +import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; +import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.DBSelector; +import org.vitrivr.cineast.core.db.DBSelectorSupplier; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; +import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderJob; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderWorker; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.LightingOptions; +import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; +import org.vitrivr.cineast.core.util.KMeansPP; +import org.vitrivr.cineast.core.util.math.MathHelper; +import org.vitrivr.cineast.core.util.texturemodel.Viewpoint.ViewpointHelper; +import org.vitrivr.cineast.core.util.texturemodel.Viewpoint.ViewpointStrategy; +import org.vitrivr.cineast.core.util.web.ImageParser; + +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpClient.Version; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.List; +import java.util.function.Supplier; + +import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; + +public class ExternalOpenClipText extends AbstractFeatureModule { + + private static final Logger LOGGER = LogManager.getLogger(); + private static final String TABLE_NAME = "features_openclip"; + + private static final int EMBEDDING_SIZE = 512; + private static final ReadableQueryConfig.Distance DISTANCE = ReadableQueryConfig.Distance.cosine; + private static final CorrespondenceFunction CORRESPONDENCE = CorrespondenceFunction.linear(1); + + private static final String DEFAULT_API_ENDPOINT = "http://localhost:8888"; + + private static final String API_ENDPOINT_KEY = "api"; + + private final String externalApi; + + private final HttpClient httpClient = HttpClient.newBuilder() + .version(Version.HTTP_1_1) + .build(); + + private final JsonMapper mapper = new JsonMapper(); + private DBSelector selector; + + + public ExternalOpenClipText() { + super(TABLE_NAME, 1f, EMBEDDING_SIZE); + this.externalApi = DEFAULT_API_ENDPOINT; + } + + public ExternalOpenClipText(Map properties) { + super(TABLE_NAME, 1f, EMBEDDING_SIZE); + this.externalApi = properties.getOrDefault(API_ENDPOINT_KEY, DEFAULT_API_ENDPOINT); + } + + @Override + public void processSegment(SegmentContainer sc) { + // Return if already processed + if (phandler.idExists(sc.getId())) { + return; + } + + // Case: segment contains video frames + if (!sc.getVideoFrames().isEmpty() && sc.getVideoFrames().get(0) != VideoFrame.EMPTY_VIDEO_FRAME) { + var frame = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); + + float[] embeddingArray = new float[0]; + embeddingArray = apiRequest(frame); + LOGGER.debug("ML Clip MostRepresentativeFrame from external API url: {}", externalApi); + this.persist(sc.getId(), new FloatVectorImpl(embeddingArray)); + + return; + } + + // Case: segment contains image + if (sc.getMostRepresentativeFrame() != VideoFrame.EMPTY_VIDEO_FRAME) { + var image = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); + + if (image != null) { + float[] embeddingArray = new float[0]; + embeddingArray = apiRequest(image); + LOGGER.debug("ML Clip image from external API url: {}", externalApi); + this.persist(sc.getId(), new FloatVectorImpl(embeddingArray)); + } + return; + } + + // Case: segment contains model + var model = sc.getModel(); + if (model != null) { + float[] embeddingArray = embedModel(model); + this.persist(sc.getId(), new FloatVectorImpl(embeddingArray)); + System.gc(); + return; + } + } + + + @Override + public void initalizePersistentLayer(Supplier supply) { + supply.get().createFeatureEntity(TABLE_NAME, true, EMBEDDING_SIZE); + } + + @Override + public void dropPersistentLayer(Supplier supply) { + supply.get().dropEntity(TABLE_NAME); + } + + @Override + public void init(DBSelectorSupplier selectorSupply) { + this.selector = selectorSupply.get(); + this.selector.open(TABLE_NAME); + } + + private float[] apiRequest(BufferedImage image) { + // Image encode to base64 + var imageData = ImageParser.bufferedImageToDataURL(image, "png"); + + var builder = new StringBuilder() + .append(URLEncoder.encode("image", StandardCharsets.UTF_8)) + .append("=") + .append(URLEncoder.encode(imageData, StandardCharsets.UTF_8)); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(externalApi + "/image")) + .header("Content-Type", "application/x-www-form-urlencoded") + .POST(HttpRequest.BodyPublishers.ofString(builder.toString())) + .build(); + + HttpResponse response = null; + try { + response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + throw new IllegalStateException("received response code " + response.statusCode()); + } + return mapper.readValue(response.body(), float[].class); + } catch (IOException e) { + LOGGER.error("Error during CLIPImage execution. Check if the external API is running", e); + throw new RuntimeException(e); + } catch (InterruptedException e) { + LOGGER.error("Error during CLIPImage execution. Check if the external API is running", e); + throw new RuntimeException(e); + } + + } + + private float[] apiRequest(String query) throws IOException, InterruptedException { + + String builder = URLEncoder.encode("query", StandardCharsets.UTF_8) + + "=" + + URLEncoder.encode(query, StandardCharsets.UTF_8); + + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(builder)) + .uri(URI.create(externalApi)) + .header("Content-Type", "application/x-www-form-urlencoded") + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + throw new IllegalStateException("received response code " + response.statusCode()); + } + + return mapper.readValue(response.body(), float[].class); + } + + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + String text = sc.getText(); + if (text == null || text.isBlank()) { + return Collections.emptyList(); + } + + try { + float[] arr = apiRequest(text); + return getSimilar(new FloatArrayTypeProvider(arr), qc); + } catch (Exception e) { + LOGGER.error("error during CLIPText execution", e); + return new ArrayList<>(); + } + } + + public List getSimilar(String segmentId, ReadableQueryConfig qc) { + List list = this.selector.getFeatures(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER, qc); + if (list.isEmpty()) { + LOGGER.warn("No feature vector for shotId {} found, returning empty result-list", segmentId); + return Collections.emptyList(); + } + return getSimilar(list.get(0), qc); + } + + public List getSimilar(PrimitiveTypeProvider queryProvider, ReadableQueryConfig qc) { + ReadableQueryConfig qcc = QueryConfig.clone(qc).setDistance(DISTANCE); + List distances = this.selector.getNearestNeighboursGeneric(qc.getResultsPerModule(), queryProvider, FEATURE_COLUMN_QUALIFIER, SegmentDistanceElement.class, qcc); + CorrespondenceFunction function = qcc.getCorrespondenceFunction().orElse(CORRESPONDENCE); + return DistanceElement.toScore(distances, function); + } + + + @Override + public void finish() { + if (this.selector != null) { + this.selector.close(); + this.selector = null; + } + } + + + /** + * This method takes a list of images and determines, based on {@link ViewpointStrategy}, the embed vector which describes the images most precise. This method can be simplified, once the best strategy is determined. + * + * @param images the list of Images to embed + * @param viewpointStrategy the strategy to find the vector + * @return the embedding vector + */ + private float[] embedMostRepresentativeImages(List images, ViewpointStrategy viewpointStrategy) { + + var retVal = new float[EMBEDDING_SIZE]; + + switch (viewpointStrategy) { + // Strategy for embedding multiple images. Choose mean of the most contained cluster. Project mean to unit hypersphere. + case MULTI_IMAGE_KMEANS -> { + var floatvectors = new ArrayList(); + var vectors = embedMultipleImages(images); + vectors.forEach(v -> floatvectors.add(new FloatVectorImpl(v))); + var kmeans = KMeansPP.bestOfkMeansPP(floatvectors, new FloatVectorImpl(new float[EMBEDDING_SIZE]), 3, -1f, 5); + // Find the index of thr cluster with the most elements + int maxIndex = 0; + for (var ic = 0; ic < kmeans.getPoints().size(); ++ic) { + if (kmeans.getPoints().get(ic).size() > kmeans.getPoints().get(maxIndex).size()) { + maxIndex = ic; + } + if (kmeans.getPoints().get(ic).size() == kmeans.getPoints().get(maxIndex).size()) { + if (kmeans.getDistance(ic) < kmeans.getDistance(maxIndex)) { + maxIndex = ic; + } + } + } + ReadableFloatVector.toArray(kmeans.getCenters().get(maxIndex), retVal); + return MathHelper.normalizeL2InPlace(retVal); + } + // Strategy for embedding multiple images. Calculate mean over all. Project mean to unit hypersphere. + case MULTI_IMAGE_PROJECTEDMEAN -> { + var vectors = embedMultipleImages(images); + var vectorsMean = new float[EMBEDDING_SIZE]; + for (var vector : vectors) { + for (var ic = 0; ic < vector.length; ++ic) { + vectorsMean[ic] += vector[ic] / vectors.size(); + } + } + return MathHelper.normalizeL2InPlace(vectorsMean); + } + // Strategy for embedding an image consisting out of four sub images. + case MULTI_IMAGE_2_2 -> { + assert images.size() == 4; + var sz = images.get(0).getWidth(); + var size = sz * 2; + // Combine the images into a single image. + var canvas = new BufferedImage( + size, + size, + BufferedImage.TYPE_INT_RGB); + var graphics = canvas.getGraphics(); + graphics.setColor(Color.BLACK); + // ic: image counter, idx: x-axis-index, idy: yaxis-index + var ic = 0; + for (var partialImage : images) { + int idx = ic % 2; + int idy = ic < 2 ? 0 : 1; + graphics.drawImage(partialImage, idx * sz, idy * sz, null); + ++ic; + } + retVal = apiRequest(canvas); + } + } + return retVal; + } + + + /** + * Embeds a list of images + * + * @param images the list of images to embed + * @return the list of embedding vectors + */ + private List embedMultipleImages(List images) { + var vectors = new ArrayList(); + for (BufferedImage image : images) { + float[] embeddingArray = apiRequest(image); + vectors.add(embeddingArray); + } + return vectors; + } + + /** + * This method embeds a 3D model and returns the feature vector. + */ + private float[] embedModel(IModel model) { + //Options for window + var windowOptions = new WindowOptions() {{ + this.hideWindow = true; + this.width = 600; + this.height = 600; + }}; + // Options for renderer + var renderOptions = new RenderOptions() {{ + this.showTextures = true; + //this.lightingOptions.hasNonDefaultTexture = model.usesNonDefaultTexture(); + this.lightingOptions = LightingOptions.STATIC; + }}; + // Select the strategy which will be used for model embedding + var viewpointStrategy = ViewpointStrategy.MULTI_IMAGE_KMEANS; + // Get camera viewpoint for chosen strategy + var cameraPositions = ViewpointHelper.getCameraPositions(viewpointStrategy, model); + // Render an image for each camera position + var images = RenderJob.performStandardRenderJob(RenderWorker.getRenderJobQueue(), + model, cameraPositions, windowOptions, renderOptions); + + // Embedding based on strategy return value. Empty if an error occurred + if (images.isEmpty()) { + return new float[EMBEDDING_SIZE]; + } + if (images.size() == 1) { + return apiRequest(images.get(0)); + } + return embedMostRepresentativeImages(images, viewpointStrategy); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalTextFeature.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalTextFeature.java new file mode 100644 index 000000000..833ba7315 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalTextFeature.java @@ -0,0 +1,49 @@ +package org.vitrivr.cineast.core.features; + +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; +import org.vitrivr.cineast.core.data.entities.SimplePrimitiveTypeProviderFeatureDescriptor; +import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.abstracts.AbstractTextRetriever; +import org.vitrivr.cineast.core.util.CineastConstants; + +public class ExternalTextFeature extends AbstractTextRetriever { + + public static final String DEFAULT_TABLE_NAME = "features_externalText"; + + final FeatureClient client; + + public ExternalTextFeature() { + super(DEFAULT_TABLE_NAME); + throw new IllegalArgumentException("no properties specified"); + } + + public ExternalTextFeature(Map properties) throws IOException { + super(properties.getOrDefault(CineastConstants.ENTITY_NAME_KEY, DEFAULT_TABLE_NAME), properties); + + this.client = FeatureClient.build(properties); + } + + private static final Logger LOGGER = LogManager.getLogger(); + + @Override + public void processSegment(SegmentContainer sc) { + if (sc.getMostRepresentativeFrame() == null || sc.getMostRepresentativeFrame() == VideoFrame.EMPTY_VIDEO_FRAME) { + return; + } + try { + Map extractions = client.extract(sc); + String feature = (String) extractions.getOrDefault("raw_text", ""); + var descriptor = new SimpleFulltextFeatureDescriptor(sc.getId(), feature); + this.writer.write(descriptor); + } catch (Exception e) { + LOGGER.error("Error during extraction", e); + } + + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FESVectorFeature.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FESVectorFeature.java new file mode 100644 index 000000000..c50cfa983 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FESVectorFeature.java @@ -0,0 +1,438 @@ +package org.vitrivr.cineast.core.features; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.QueryConfig; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.config.ReadableQueryConfig.Distance; +import org.vitrivr.cineast.core.data.FloatVectorImpl; +import org.vitrivr.cineast.core.data.ReadableFloatVector; +import org.vitrivr.cineast.core.data.frames.VideoFrame; +import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; +import org.vitrivr.cineast.core.data.raw.CachedDataFactory; +import org.vitrivr.cineast.core.data.raw.images.MultiImage; +import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.abstracts.AbstractFeatureModule; +import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderJob; +import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderWorker; +import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; +import org.vitrivr.cineast.core.util.CineastConstants; +import org.vitrivr.cineast.core.util.KMeansPP; +import org.vitrivr.cineast.core.util.math.MathHelper; +import org.vitrivr.cineast.core.util.texturemodel.Viewpoint.ViewpointHelper; +import org.vitrivr.cineast.core.util.texturemodel.Viewpoint.ViewpointStrategy; +import org.vitrivr.cineast.core.util.web.ImageParser; +import org.vitrivr.cineast.core.util.web.WebClient; + +public class FESVectorFeature extends AbstractFeatureModule { + + public static final String DEFAULT_TABLE_NAME = "features_externalText"; + public static final float DEFAULT_MAX_DIST = 2.0f; + public static final int DEFAULT_EMBEDDING_SIZE = 512; + + public static final Distance DEFAULT_DISTANCE = ReadableQueryConfig.Distance.euclidean; + + public final int embedding_size; + + public final Distance distance; + + public final String model; + + + private static final Logger LOGGER = LogManager.getLogger(); + + final WebClient client; + + public FESVectorFeature() { + super(DEFAULT_TABLE_NAME, DEFAULT_MAX_DIST, DEFAULT_EMBEDDING_SIZE); + embedding_size = DEFAULT_EMBEDDING_SIZE; + distance = DEFAULT_DISTANCE; + model = null; + throw new IllegalArgumentException("no properties specified"); + } + + public FESVectorFeature(Map properties) throws IOException { + super(properties.getOrDefault( + CineastConstants.ENTITY_NAME_KEY, DEFAULT_TABLE_NAME), + Float.parseFloat(properties.getOrDefault("max_dist", String.valueOf(DEFAULT_MAX_DIST))), + Integer.parseInt(properties.getOrDefault("embedding_size", String.valueOf(DEFAULT_EMBEDDING_SIZE))) + ); + embedding_size = Integer.parseInt(properties.getOrDefault("embedding_size", String.valueOf(DEFAULT_EMBEDDING_SIZE))); + distance = ReadableQueryConfig.Distance.valueOf(properties.getOrDefault("distance", DEFAULT_DISTANCE.name())); + if(!properties.containsKey("endpoint")){ + throw new IllegalArgumentException("No endpoint specified for external client"); + } + String endpoint = properties.get("endpoint"); + if(properties.containsKey("model")){ + this.model = properties.get("model"); + }else{ + this.model = null; + } + this.client = new WebClient(endpoint); + } + @Override + public void processSegment(SegmentContainer sc) { + // Return if already processed + if (phandler.idExists(sc.getId())) { + return; + } + + // Case: segment contains video frames + if (!sc.getVideoFrames().isEmpty() && sc.getVideoFrames().get(0) != VideoFrame.EMPTY_VIDEO_FRAME) { + List frames = sc.getVideoFrames().stream().map(VideoFrame::getImage).collect(Collectors.toList()); + + float[] embeddingArray = new float[0]; + try { + embeddingArray = embedVideo(frames); + } catch (Exception e) { + LOGGER.error("Error during extraction", e); + } + this.persist(sc.getId(), new FloatVectorImpl(embeddingArray)); + + return; + } + + // Case: segment contains image + if (sc.getMostRepresentativeFrame() != VideoFrame.EMPTY_VIDEO_FRAME) { + BufferedImage image = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); + + if (image != null) { + float[] embeddingArray = new float[0]; + try { + embeddingArray = embedImage(image); + } catch (Exception e) { + LOGGER.error("Error during extraction", e); + } + this.persist(sc.getId(), new FloatVectorImpl(embeddingArray)); + } + + return; + } + + // Case: segment contains model + var model = sc.getModel(); + if (model != null) { + float[] embeddingArray = new float[0]; + try { + embeddingArray = embedModel(model); + } catch (Exception e) { + LOGGER.error("Error during extraction", e); + } + this.persist(sc.getId(), new FloatVectorImpl(embeddingArray)); + System.gc(); + return; + } + } + + @Override + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + // Ensure the correct distance function is used + QueryConfig queryConfig = QueryConfig.clone(qc); + queryConfig.setDistance(distance); + + // Case: segment contains text + if (!sc.getText().isEmpty()) { + String text = sc.getText(); + LOGGER.debug("Retrieving with TEXT: " + text); + float[] embeddingArray = new float[0]; + try { + embeddingArray = embedText(text); + } catch (Exception e) { + LOGGER.error("Error during extraction", e); + } + + return getSimilar(embeddingArray, queryConfig); + } + + // Case: segment contains image + if (sc.getMostRepresentativeFrame() != VideoFrame.EMPTY_VIDEO_FRAME) { + LOGGER.debug("Retrieving with IMAGE."); + BufferedImage image = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); + + if (image != null) { + float[] embeddingArray = new float[0]; + try { + embeddingArray = embedImage(image); + } catch (Exception e) { + LOGGER.error("Error during extraction", e); + } + return getSimilar(embeddingArray, queryConfig); + } + + LOGGER.error("Image was provided, but could not be decoded!"); + } + + // Case: segment contains model + var model = sc.getModel(); + if (model != null) { + LOGGER.debug("Retrieving with MODEL."); + float[] embeddingArray = new float[0]; + try { + embeddingArray = embedModel(model); + } catch (Exception e) { + LOGGER.error("Error during extraction", e); + } + return getSimilar(embeddingArray, queryConfig); + } + + LOGGER.error("Could not get similar because no acceptable modality was provided."); + return new ArrayList<>(); + } + + @Override + public List getSimilar(String segmentId, ReadableQueryConfig qc) { + // Ensure the correct distance function is used + QueryConfig queryConfig = QueryConfig.clone(qc); + queryConfig.setDistance(distance); + + return super.getSimilar(segmentId, queryConfig); + } + + + private float[] embedText(String text) throws IOException, InterruptedException { + String request; + if (model == null){ + String template = """ + {"task":"text_embedding", "text":"%s"} + """; + request = String.format(template, text); + }else{ + String template = """ + {"task":"text_embedding", "model":"%s", "text":"%s"} + """; + request = String.format(template, model, text); + } + return parseEmbedding(client.postJsonString(request)); + } + + + private float[] embedImage(BufferedImage image) throws IOException, InterruptedException { + String request; + if (model == null){ + String template = """ + {"task":"image_embedding", "image":"%s"} + """; + request = String.format(template, ImageParser.bufferedImageToDataURL(image, "png")); + }else{ + String template = """ + {"task":"image_embedding", "model":"%s", "image":"%s"} + """; + request = String.format(template, model, ImageParser.bufferedImageToDataURL(image, "png")); + } + return parseEmbedding(client.postJsonString(request)); + + } + + private float[] parseEmbedding(String json) throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.readValue(json, float[].class); + } + + + private List parseEmbeddingList(String json) throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + + // Parse JSON to List> + List> lists = objectMapper.readValue(json, new TypeReference>>() {}); + + // Convert List> to List + List result = new ArrayList<>(); + for (List list : lists) { + float[] arr = new float[list.size()]; + for (int i = 0; i < list.size(); i++) { + arr[i] = list.get(i).floatValue(); + } + result.add(arr); + } + + return result; + } + + /** + * Helper to convert images to frames + * + * @param images are converted to frames + * @return List for video embedding + */ + private List framesFromImages(List images) { + var frames = new ArrayList(); + for (BufferedImage image : images) { + var factory = CachedDataFactory.getDefault(); + frames.add(factory.newInMemoryMultiImage(image)); + } + return frames; + } + + /** + * This method takes a list of images and determines, based on {@link ViewpointStrategy}, the embed vector which describes the images most precise. This method can be simplified, once the best strategy is determined. + * + * @param images the list of Images to embed + * @param viewpointStrategy the strategy to find the vector + * @return the embedding vector + */ + private float[] embedMostRepresentativeImages(List images, ViewpointStrategy viewpointStrategy) throws IOException, InterruptedException { + + var retVal = new float[embedding_size]; + + switch (viewpointStrategy) { + // Strategy for embedding multiple images. Choose mean of the most contained cluster. Project mean to unit hypersphere. + case MULTI_IMAGE_KMEANS -> { + var floatvectors = new ArrayList(); + var vectors = embedMultipleImages(images); + vectors.forEach(v -> floatvectors.add(new FloatVectorImpl(v))); + var kmeans = KMeansPP.bestOfkMeansPP(floatvectors, new FloatVectorImpl(new float[embedding_size]), 3, -1f, 5); + // Find the index of thr cluster with the most elements + int maxIndex = 0; + for (var ic = 0; ic < kmeans.getPoints().size(); ++ic) { + if (kmeans.getPoints().get(ic).size() > kmeans.getPoints().get(maxIndex).size()) { + maxIndex = ic; + } + if (kmeans.getPoints().get(ic).size() == kmeans.getPoints().get(maxIndex).size()) { + if (kmeans.getDistance(ic) < kmeans.getDistance(maxIndex)) { + maxIndex = ic; + } + } + } + ReadableFloatVector.toArray(kmeans.getCenters().get(maxIndex), retVal); + return MathHelper.normalizeL2InPlace(retVal); + } + // Strategy for embedding multiple images. Calculate mean over all. Project mean to unit hypersphere. + case MULTI_IMAGE_PROJECTEDMEAN -> { + var vectors = embedMultipleImages(images); + var vectorsMean = new float[embedding_size]; + for (var vector : vectors) { + for (var ic = 0; ic < vector.length; ++ic) { + vectorsMean[ic] += vector[ic] / vectors.size(); + } + } + return MathHelper.normalizeL2InPlace(vectorsMean); + } + // Strategy for embedding multiple images. Create a video from the images and embed the video. + case MULTI_IMAGE_FRAME -> { + var frames = framesFromImages(images); + return embedVideo(frames); + } + // Strategy for embedding an image consisting out of four sub images. + case MULTI_IMAGE_2_2 -> { + assert images.size() == 4; + var sz = images.get(0).getWidth(); + var size = sz * 2; + // Combine the images into a single image. + var canvas = new BufferedImage( + size, + size, + BufferedImage.TYPE_INT_RGB); + var graphics = canvas.getGraphics(); + graphics.setColor(Color.BLACK); + // ic: image counter, idx: x-axis-index, idy: yaxis-index + var ic = 0; + for (var partialImage : images) { + int idx = ic % 2; + int idy = ic < 2 ? 0 : 1; + graphics.drawImage(partialImage, idx * sz, idy * sz, null); + ++ic; + } + retVal = embedImage(canvas); + } + } + return retVal; + } + + /** + * Embeds a list of images + * + * @param images the list of images to embed + * @return the list of embedding vectors + */ + private List embedMultipleImages(List images) throws IOException, InterruptedException { + String request = "{\"task\":\"image_embedding\", "; + if (model!=null){ + request += "\"model\":\"" + model + "\", "; + } + request += "\"image\":["; + if (!images.isEmpty()) { + for (var image : images) { + request += "\"" + ImageParser.bufferedImageToDataURL(image, "png") + "\","; + } + request = request.substring(0, request.length() - 1); // Remove trailing comma + } + request += "]"; + request += "}"; + String json = client.postJsonString(request); + return parseEmbeddingList(json); + + } + + /** + * This method embeds a 3D model and returns the feature vector. + */ + private float[] embedModel(IModel model) throws IOException, InterruptedException { + //Options for window + var windowOptions = new WindowOptions() {{ + this.hideWindow = true; + this.width = 600; + this.height = 600; + }}; + // Options for renderer + var renderOptions = new RenderOptions() {{ + this.showTextures = true; + }}; + // Select the strategy which will be used for model embedding + var viewpointStrategy = ViewpointStrategy.MULTI_IMAGE_KMEANS; + // Get camera viewpoint for chosen strategy + var cameraPositions = ViewpointHelper.getCameraPositions(viewpointStrategy, model); + // Render an image for each camera position + var images = RenderJob.performStandardRenderJob(RenderWorker.getRenderJobQueue(), + model, cameraPositions, windowOptions, renderOptions); + + // Embedding based on strategy return value. Empty if an error occurred + if (images.isEmpty()) { + return new float[embedding_size]; + } + if (images.size() == 1) { + return embedImage(images.get(0)); + } + return embedMostRepresentativeImages(images, viewpointStrategy); + } + + + private float[] embedVideo(List frames) throws IOException, InterruptedException { + var images = frames.stream().map(image -> image.getBufferedImage()).collect(Collectors.toList()); + images = images.stream().map(img -> new BufferedImage(img.getWidth(), img.getHeight(), BufferedImage.TYPE_INT_RGB)).collect(Collectors.toList()); + var encodings = embedMultipleImages(images); + + // Sum + float[] meanEncoding = encodings.stream().reduce(new float[embedding_size], (encoding0, encoding1) -> { + float[] tempSum = new float[embedding_size]; + + for (int i = 0; i < embedding_size; i++) { + tempSum[i] = encoding0[i] + encoding1[i]; + } + + return tempSum; + }); + + // Calculate mean + for (int i = 0; i < embedding_size; i++) { + meanEncoding[i] /= encodings.size(); + } + + return meanEncoding; + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClient.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClient.java new file mode 100644 index 000000000..a65bf874e --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClient.java @@ -0,0 +1,72 @@ +package org.vitrivr.cineast.core.features; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.exporter.AudioSegmentExporter; +import org.vitrivr.cineast.core.util.web.ImageParser; +import org.vitrivr.cineast.core.util.web.WebClient; +import org.vitrivr.cineast.core.util.web.MessageTemplate; + +public class FeatureClient { + protected final WebClient client; + protected final FeatureClientRequest request; + protected final MessageTemplate responseTemplate; + + protected static final Logger LOGGER = LogManager.getLogger(); + + public static FeatureClient build(Map properties) throws IOException { + if(!properties.containsKey("endpoint")){ + throw new IllegalArgumentException("No endpoint specified for external client"); + } + String endpoint = properties.get("endpoint"); + if(!properties.containsKey("request")){ + throw new IllegalArgumentException("No request specified for external client"); + } + String requestString = properties.get("request"); + if(!properties.containsKey("response")){ + throw new IllegalArgumentException("No response template specified for external client"); + } + String responseString = properties.get("response"); + FeatureClientRequest request; + if (new File(requestString).exists()) { + request = new FeatureClientJsonRequest(requestString); + }else { + request = new FeatureClientBinaryRequest(requestString); + } + return new FeatureClient(endpoint, request, responseString); + } + + protected FeatureClient(String endpoint, FeatureClientRequest req, String responseTemplatePath) throws IOException { + this.client = new WebClient(endpoint); + this.request = req; + this.responseTemplate = new MessageTemplate(responseTemplatePath); + } + + public Map extract(SegmentContainer sc) throws IOException, InterruptedException { + + String response = this.request.execute(sc, this.client); + + Map responseKeys = this.responseTemplate.parseString(response); + + HashMap result = new HashMap<>(); + for (var key : responseKeys.keySet()) { + switch (key) { + case "raw_text": + result.put(key, responseKeys.get(key)); + break; + default: + LOGGER.warn("Unknown key: {}", key); + break; + } + } + return result; + } + + + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientBinaryRequest.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientBinaryRequest.java new file mode 100644 index 000000000..4d167a180 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientBinaryRequest.java @@ -0,0 +1,28 @@ +package org.vitrivr.cineast.core.features; + +import java.io.IOException; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.abstracts.AbstractSegmentExporter; +import org.vitrivr.cineast.core.features.exporter.AudioSegmentExporter; +import org.vitrivr.cineast.core.util.web.WebClient; + +public class FeatureClientBinaryRequest extends FeatureClientRequest { + + private final String key; + public FeatureClientBinaryRequest(String requestString) { + super(); + this.key = requestString; + } + + @Override + public String execute(SegmentContainer sc, WebClient client) throws IOException, InterruptedException { + switch (this.key) { + case "wav_binary": + var wavExport = new AudioSegmentExporter(); + return client.postRawBinary(wavExport.exportToBinary(sc)); + default: + LOGGER.warn("Unknown key: {}", this.key); + return ""; + } + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientJsonRequest.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientJsonRequest.java new file mode 100644 index 000000000..1a9236cf9 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientJsonRequest.java @@ -0,0 +1,40 @@ +package org.vitrivr.cineast.core.features; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.exporter.AudioSegmentExporter; +import org.vitrivr.cineast.core.util.web.ImageParser; +import org.vitrivr.cineast.core.util.web.MessageTemplate; +import org.vitrivr.cineast.core.util.web.WebClient; + +public class FeatureClientJsonRequest extends FeatureClientRequest { + private final MessageTemplate requestTemplate; + public FeatureClientJsonRequest(String RequestTemplatePath) throws IOException { + super(); + this.requestTemplate = new MessageTemplate(RequestTemplatePath); + } + @Override + public String execute(SegmentContainer sc, WebClient client) throws IOException, InterruptedException { + Map keys = new HashMap<>(); + //iterate through keys + for (var key : this.requestTemplate.getKeys()) { + switch (key) { + case "png_image": + var bufImg = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); + keys.put(key, ImageParser.bufferedImageToDataURL(bufImg, "png")); + break; + case "wav_dataurl": + var wavExport = new AudioSegmentExporter(); + keys.put(key, wavExport.exportToDataUrl(sc)); + break; + default: + LOGGER.warn("Unknown key: {}", key); + break; + } + } + String request = this.requestTemplate.formatString(keys); + return client.postJsonString(request); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientRequest.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientRequest.java new file mode 100644 index 000000000..04b009daf --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientRequest.java @@ -0,0 +1,14 @@ +package org.vitrivr.cineast.core.features; + +import java.io.IOException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.util.web.WebClient; + +abstract class FeatureClientRequest { + + protected static final Logger LOGGER = LogManager.getLogger(); + + abstract public String execute(SegmentContainer sc, WebClient client) throws IOException, InterruptedException; +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOG.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOG.java index 57699e8e7..028c8c783 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOG.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/HOG.java @@ -27,8 +27,8 @@ public abstract class HOG extends AbstractCodebookFeatureModule { /** * */ - protected HOG(String tableName, float maxDist, int vectorLength) { - super(tableName, maxDist, vectorLength); + protected HOG(String entityName, float maxDist, int vectorLength) { + super(entityName, maxDist, vectorLength); } /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java new file mode 100644 index 000000000..33cce07cc --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/LevenshteinScoringTextRetriever.java @@ -0,0 +1,97 @@ +package org.vitrivr.cineast.core.features; + +import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; +import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.stream.Collectors; +import org.apache.commons.text.similarity.LevenshteinDistance; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.config.ReadableQueryConfig; +import org.vitrivr.cineast.core.data.entities.SimpleFulltextFeatureDescriptor; +import org.vitrivr.cineast.core.data.providers.primitive.PrimitiveTypeProvider; +import org.vitrivr.cineast.core.data.score.ScoreElement; +import org.vitrivr.cineast.core.data.score.SegmentScoreElement; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.abstracts.AbstractTextRetriever; + +public class LevenshteinScoringTextRetriever extends AbstractTextRetriever { + + private static final Logger LOGGER = LogManager.getLogger(); + + public LevenshteinScoringTextRetriever(Map properties) { + super(ProvidedOcrSearch.PROVIDED_OCR_SEARCH_TABLE_NAME, properties); + } + + private static final LevenshteinDistance distance = new LevenshteinDistance(); + + public static float similarity(String query, String candidate) { + + if (query.isBlank() || candidate.isBlank()) { + return 0f; + } + + int levDist = distance.apply(query, candidate); + + if (query.length() < candidate.length()) { + levDist -= (candidate.length() - query.length()); //don't penalize matching substrings + } + + return 1f - ((float)levDist / (float)query.length()); + + } + + public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { + + String text = sc.getText(); + if (text == null || text.isBlank()) { + return Collections.emptyList(); + } + return this.getSimilar(qc, text); + } + + protected List getSimilar(ReadableQueryConfig qc, String... terms) { + + if (terms.length == 0) { + return Collections.emptyList(); + } + + final Map scoreMap = new HashMap<>(); + + for (String term : terms) { + + if (term.isBlank()) { + continue; + } + + String stripped = term.strip(); + + final List> resultList = this.selector.getFulltextRows(qc.getResultsPerModule(), SimpleFulltextFeatureDescriptor.FIELDNAMES[1], qc, generateQuery(stripped)); + LOGGER.trace("Retrieved {} results for term '{}'", resultList.size(), term); + + for (Map result : resultList) { + String id = result.get(GENERIC_ID_COLUMN_QUALIFIER).getString(); + String text = result.get(FEATURE_COLUMN_QUALIFIER).getString(); + + float score = similarity(stripped, text); + + float bestScore = scoreMap.getOrDefault(id, 0f); + + if (score > bestScore) { + scoreMap.put(id, score); + } + } + + } + + return scoreMap.entrySet().stream().map(entry -> new SegmentScoreElement(entry.getKey(), entry.getValue())).sorted(SegmentScoreElement.SCORE_COMPARATOR).collect(Collectors.toList()); + + } + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java index d415551d9..d35299eb3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/Lightfield.java @@ -70,8 +70,8 @@ public abstract class Lightfield extends StagedFeatureModule { /** * Offscreen rendering environment used to create Lightfield images. */ - protected Lightfield(String tableName, float maxDist, int vectorLength, double[][] camerapositions) { - super(tableName, maxDist, vectorLength); + protected Lightfield(String entityName, float maxDist, int vectorLength, double[][] camerapositions) { + super(entityName, maxDist, vectorLength); if (camerapositions.length == 0) { throw new IllegalArgumentException("You must specify at least one camera position!"); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8.java index e3bf86129..fc97671ce 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/MedianColorGrid8.java @@ -30,8 +30,8 @@ public MedianColorGrid8() { super("features_MedianColorGrid8", 12595f / 4f, 192); } - protected MedianColorGrid8(String tableName, float maxDist) { - super(tableName, maxDist, 192); + protected MedianColorGrid8(String entityName, float maxDist) { + super(entityName, maxDist, 192); } protected static Pair partition(MultiImage img) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java index 962e58e69..b41604c52 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SURF.java @@ -26,8 +26,8 @@ public abstract class SURF extends AbstractCodebookFeatureModule { private static QueryConfig.Distance DEFAULT_DISTANCE = QueryConfig.Distance.chisquared; - protected SURF(String tableName, int vectorLength) { - super(tableName, 2.0f, vectorLength); + protected SURF(String entityName, int vectorLength) { + super(entityName, 2.0f, vectorLength); } @Override diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java index 9bff551c3..17b8c236a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SegmentTags.java @@ -46,7 +46,7 @@ public SegmentTags() { } @Override - public List getTableNames() { + public List getEntityNames() { return Collections.singletonList(SEGMENT_TAGS_TABLE_NAME); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java index 0d0962418..a60131cf3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SkeletonPose.java @@ -8,11 +8,9 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import georegression.struct.point.Point2D_F32; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.function.Supplier; @@ -112,7 +110,7 @@ public void init(PersistencyWriterSupplier phandlerSupply) { @Override public void initalizePersistentLayer(Supplier supply) { - supply.get().createFeatureEntity(this.tableName, false, + supply.get().createFeatureEntity(this.entityName, false, new AttributeDefinition(PERSON_ID_COL, AttributeDefinition.AttributeType.INT), new AttributeDefinition(FEATURE_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength), new AttributeDefinition(WEIGHT_COL, AttributeDefinition.AttributeType.VECTOR, this.vectorLength) @@ -364,7 +362,7 @@ private CottontailGrpc.QueryMessage buildQuery(float[] query, float[] weights, i return QueryMessage.newBuilder().setQuery( Query.newBuilder().setFrom( From.newBuilder().setScan(Scan.newBuilder().setEntity(EntityName.newBuilder() - .setName(this.tableName).setSchema(SchemaName.newBuilder().setName("cineast")))) + .setName(this.entityName).setSchema(SchemaName.newBuilder().setName("cineast")))) ).setProjection( Projection.newBuilder() .addElements(projectionElement(GENERIC_ID_COLUMN_QUALIFIER)) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java index e11f05271..bac200ae0 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/VisualTextCoEmbedding.java @@ -366,13 +366,14 @@ private List embedMultipleImages(List images) { private float[] embedModel(IModel model) { //Options for window var windowOptions = new WindowOptions() {{ - this.hideWindow = true; + this.hideWindow = false; this.width = 600; this.height = 600; }}; // Options for renderer var renderOptions = new RenderOptions() {{ this.showTextures = true; + this.lightingOptions.hasNonDefaultTexture = model.usesNonDefaultTexture(); }}; // Select the strategy which will be used for model embedding var viewpointStrategy = ViewpointStrategy.MULTI_IMAGE_KMEANS; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java index bc5bb5ed2..1853db4b3 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractCodebookFeatureModule.java @@ -33,8 +33,8 @@ public abstract class AbstractCodebookFeatureModule extends StagedFeatureModule */ private AssignCluster assignment; - protected AbstractCodebookFeatureModule(String tableName, float maxDist, int vectorLength) { - super(tableName, maxDist, vectorLength); + protected AbstractCodebookFeatureModule(String entityName, float maxDist, int vectorLength) { + super(entityName, maxDist, vectorLength); } /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractFeatureModule.java index 6886e3354..f08d6ce42 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractFeatureModule.java @@ -38,39 +38,39 @@ public abstract class AbstractFeatureModule implements Extractor, Retriever { private static final Logger LOGGER = LogManager.getLogger(); protected final float maxDist; protected final int vectorLength; - protected final String tableName; + protected final String entityName; protected SimpleFeatureDescriptorWriter writer; protected PrimitiveTypeProviderFeatureDescriptorWriter primitiveWriter; protected DBSelector selector; protected PersistencyWriter phandler; protected CorrespondenceFunction correspondence; - protected AbstractFeatureModule(String tableName, float maxDist, int vectorLength) { - this.tableName = tableName; + protected AbstractFeatureModule(String entityName, float maxDist, int vectorLength) { + this.entityName = entityName; this.maxDist = maxDist; this.vectorLength = vectorLength; this.correspondence = CorrespondenceFunction.linear(maxDist); } @Override - public List getTableNames() { - if (this.tableName == null) { + public List getEntityNames() { + if (this.entityName == null) { return new ArrayList<>(); } - return Collections.singletonList(this.tableName); + return Collections.singletonList(this.entityName); } @Override public void init(PersistencyWriterSupplier phandlerSupply) { this.phandler = phandlerSupply.get(); - this.writer = new SimpleFeatureDescriptorWriter(this.phandler, this.tableName); - this.primitiveWriter = new PrimitiveTypeProviderFeatureDescriptorWriter(this.phandler, this.tableName); + this.writer = new SimpleFeatureDescriptorWriter(this.phandler, this.entityName); + this.primitiveWriter = new PrimitiveTypeProviderFeatureDescriptorWriter(this.phandler, this.entityName); } @Override public void init(DBSelectorSupplier selectorSupply) { this.selector = selectorSupply.get(); - this.selector.open(this.tableName); + this.selector.open(this.entityName); } protected void persist(String shotId, ReadableFloatVector fv) { @@ -94,7 +94,7 @@ protected ReadableQueryConfig setQueryConfig(ReadableQueryConfig qc) { @Override public List getSimilar(String segmentId, ReadableQueryConfig qc) { - List list = this.selector.getFeatureVectorsGeneric(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER, qc); + List list = this.selector.getFeatures(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER, qc); if (list.isEmpty()) { LOGGER.warn("No feature vector for shotId {} found, returning empty result-list", segmentId); return new ArrayList<>(0); @@ -165,11 +165,11 @@ public void finish() { @Override public void initalizePersistentLayer(Supplier supply) { - supply.get().createFeatureEntity(this.tableName, true, this.vectorLength); + supply.get().createFeatureEntity(this.entityName, true, this.vectorLength); } @Override public void dropPersistentLayer(Supplier supply) { - supply.get().dropEntity(this.tableName); + supply.get().dropEntity(this.entityName); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractSegmentExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractSegmentExporter.java new file mode 100644 index 000000000..577c0e652 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractSegmentExporter.java @@ -0,0 +1,149 @@ +package org.vitrivr.cineast.core.features.abstracts; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.HashMap; +import java.util.function.Supplier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; +import org.vitrivr.cineast.core.db.setup.EntityCreator; +import org.vitrivr.cineast.core.features.extractor.Extractor; +import org.vitrivr.cineast.core.util.LogHelper; + +abstract public class AbstractSegmentExporter implements Extractor { + + protected static final Logger LOGGER = LogManager.getLogger(); + + private static final String PROPERTY_NAME_DESTINATION = "destination"; + + /** + * Destination path for the audio-segment. + */ + private final Path destination; + + /** + * Returns the file extension for the exported data. + * + * @return String containing the file extension without the dot. + */ + protected abstract String getFileExtension(); + + /** + * Returns the data-url prefix for the exported data. + * + * @return String containing the data-url prefix. + */ + protected abstract String getDataUrlPrefix(); + + /** + * Default constructor + */ + public AbstractSegmentExporter() { + this(new HashMap<>()); + } + + /** + * Default constructor. A segment exporter can be configured via named properties in the provided HashMap. + *

+ * Supported parameters: + * + *

    + *
  1. destination: Path where files should be stored.
    2. + *
+ * + * @param properties HashMap containing named properties + */ + public AbstractSegmentExporter(HashMap properties) { + this.destination = Path.of(properties.getOrDefault(PROPERTY_NAME_DESTINATION, "./export")); + } + + + /** + * Exports the data of a segment container to an output stream. + * + * @param sc SegmentContainer to export. + * @param stream OutputStream to write to. + */ + abstract public void exportToStream(SegmentContainer sc, OutputStream stream); + + /** + * Determines whether a segment can be exported or not, i.e. if there is enough data to create an export. For example, a segment without audio cannot be exported as audio. Does not check if the segment is already exported. + * + * @param sc The segment to be checked. + */ + abstract public boolean isExportable(SegmentContainer sc); + + @Override + public void processSegment(SegmentContainer shot) { + if (!isExportable(shot)) { + return; + } + try { + /* Prepare folder and OutputStream. */ + Path folder = this.destination.resolve(shot.getSuperId()); + if (!folder.toFile().exists()) { + folder.toFile().mkdirs(); + } + Path path = folder.resolve(shot.getId() +'.' + this.getFileExtension()); + OutputStream os = Files.newOutputStream(path); + + exportToStream(shot, os); + + /* Close OutputStream. */ + os.close(); + } catch (Exception e) { + LOGGER.error("Could not write data to file for segment {} due to {}.", shot.getId(), LogHelper.getStackTrace(e)); + } + } + + /** + * Exports a segment to a data-url. + * + * @param shot The segment to be exported. + * @return A String containing the data-url. + */ + public String exportToDataUrl(SegmentContainer shot) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + exportToStream(shot, baos); + byte[] bytes = baos.toByteArray(); + baos.close(); + String base64 = Base64.getEncoder().encodeToString(bytes); + return this.getDataUrlPrefix() + base64; + } + + /** + * Exports a segment to a byte array. + * + * @param shot The segment to be exported. + * @return A byte array containing the exported data. + */ + public byte[] exportToBinary(SegmentContainer shot) { + // create a ByteArrayOutputStream + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + // call the exportToStream method with the ByteArrayOutputStream + exportToStream(shot, baos); + + // convert the ByteArrayOutputStream's data to a byte array + return baos.toByteArray(); + } + + @Override + public void init(PersistencyWriterSupplier phandlerSupply) { /* Nothing to init. */ } + + @Override + public void finish() { /* Nothing to finish. */} + + @Override + public void initalizePersistentLayer(Supplier supply) { /* Nothing to init. */ } + + @Override + public void dropPersistentLayer(Supplier supply) { /* Nothing to drop. */} + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java index acdbdb741..5fe5634ba 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetriever.java @@ -1,6 +1,7 @@ package org.vitrivr.cineast.core.features.abstracts; import static org.vitrivr.cineast.core.util.CineastConstants.DB_DISTANCE_VALUE_QUALIFIER; +import static org.vitrivr.cineast.core.util.CineastConstants.ENTITY_NAME_KEY; import static org.vitrivr.cineast.core.util.CineastConstants.FEATURE_COLUMN_QUALIFIER; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; @@ -42,11 +43,11 @@ public abstract class AbstractTextRetriever implements Retriever, Extractor { /** * Generate a query term which will then be used for retrieval. */ - private static final Pattern regex = Pattern.compile("([^\"]\\S*|\".+?\")\\s*"); + protected static final Pattern regex = Pattern.compile("([^\"]\\S*|\".+?\")\\s*"); /** * Name of the table/entity used to store the data. */ - private final String tableName; + private final String entityName; /** * decorator for lucene queries @@ -65,27 +66,27 @@ public abstract class AbstractTextRetriever implements Retriever, Extractor { /** * Constructor for {@link AbstractTextRetriever} * - * @param tableName Name of the table/entity used to store the data + * @param entityName Name of the table/entity used to store the data */ - public AbstractTextRetriever(String tableName) { - this(tableName, new HashMap<>()); + public AbstractTextRetriever(String entityName) { + this(entityName, new HashMap<>()); } - public AbstractTextRetriever(String defaultTableName, Map properties) { - if (defaultTableName == null) { + public AbstractTextRetriever(String defaultEntityName, Map properties) { + if (defaultEntityName == null) { throw new IllegalStateException("If no entity is provided by the underlying feature, it needs to be specified in properties"); } - this.tableName = properties.getOrDefault("entity", defaultTableName); + this.entityName = properties.getOrDefault(ENTITY_NAME_KEY, defaultEntityName); this.decorator = properties.getOrDefault("decorator", ""); } public AbstractTextRetriever(Map properties) { - this(properties.get("entity"), properties); + this(properties.get(ENTITY_NAME_KEY), properties); } @Override - public List getTableNames() { - return Collections.singletonList(tableName); + public List getEntityNames() { + return Collections.singletonList(entityName); } @Override @@ -96,7 +97,7 @@ public void init(DBSelectorSupplier selectorSupply) { @Override public void init(PersistencyWriterSupplier phandlerSupply) { - this.writer = new SimpleFulltextFeatureDescriptorWriter(phandlerSupply.get(), this.tableName); + this.writer = new SimpleFulltextFeatureDescriptorWriter(phandlerSupply.get(), this.entityName); writer.init(); } @@ -119,12 +120,12 @@ public void initalizePersistentLayer(Supplier supply) { AttributeDefinition.AttributeType.STRING, hints); fields[1] = new AttributeDefinition(SimpleFulltextFeatureDescriptor.FIELDNAMES[1], AttributeDefinition.AttributeType.TEXT, hints); - supply.get().createEntity(this.tableName, fields); + supply.get().createEntity(this.entityName, fields); } @Override public void dropPersistentLayer(Supplier supply) { - supply.get().dropEntity(this.tableName); + supply.get().dropEntity(this.entityName); } /** @@ -133,7 +134,7 @@ public void dropPersistentLayer(Supplier supply) { * @return Name of the entity. */ public String getEntityName() { - return this.tableName; + return this.entityName; } @Override @@ -166,13 +167,13 @@ public List getSimilar(List segmentIds, ReadableQueryConfi */ @Override public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc) { - final String[] terms = generateQuery(sc, qc); + final String[] terms = generateQuery(sc.getText()); return this.getSimilar(qc, terms); } - protected String[] generateQuery(SegmentContainer sc, ReadableQueryConfig qc) { + protected String[] generateQuery(String text) { - Matcher m = regex.matcher(sc.getText()); + Matcher m = regex.matcher(text); ArrayList matches = new ArrayList<>(); while (m.find()) { @@ -182,7 +183,7 @@ protected String[] generateQuery(SegmentContainer sc, ReadableQueryConfig qc) { } } - return matches.toArray(new String[matches.size()]); + return matches.toArray(new String[0]); } /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java index 10f0899f3..77c226464 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/BooleanRetriever.java @@ -1,5 +1,6 @@ package org.vitrivr.cineast.core.features.abstracts; +import static org.vitrivr.cineast.core.util.CineastConstants.ENTITY_NAME_KEY; import static org.vitrivr.cineast.core.util.CineastConstants.GENERIC_ID_COLUMN_QUALIFIER; import java.util.Arrays; @@ -29,7 +30,7 @@ import org.vitrivr.cineast.core.features.retriever.MultipleInstantiatableRetriever; /** - * BooleanRetrievers operate on tables which are created externally. Therefore, both initializing and dropping the entity are managed externally. + * BooleanRetrievers operate on entities which are created externally. Therefore, both initializing and dropping the entity are managed externally. */ public abstract class BooleanRetriever implements MultipleInstantiatableRetriever { @@ -46,10 +47,10 @@ protected BooleanRetriever(String entity, Collection attributes) { } protected BooleanRetriever(Map properties) { - if (!properties.containsKey("entity")) { - throw new RuntimeException("no entity specified in properties map of BooleanRetriever"); + if (!properties.containsKey(ENTITY_NAME_KEY)) { + throw new RuntimeException("no entity specified in properties map"); } - this.entity = properties.get("entity"); + this.entity = properties.get(ENTITY_NAME_KEY); if (properties.containsKey("attribute")) { List attrs = Arrays.stream(properties.get("attribute").split(",")).map(String::trim) @@ -62,7 +63,7 @@ protected BooleanRetriever(Map properties) { } @Override - public List getTableNames() { + public List getEntityNames() { return Collections.singletonList(entity); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java index 66509b99e..c34bdfc60 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/MetadataFeatureModule.java @@ -22,6 +22,7 @@ import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; import org.vitrivr.cineast.core.data.entities.MediaSegmentDescriptor; import org.vitrivr.cineast.core.data.entities.SimpleFeatureDescriptor; +import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayProvider; import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; @@ -77,7 +78,7 @@ protected MetadataFeatureModule(int vectorLength, Map properties public abstract CorrespondenceFunction defaultCorrespondence(); @Override - public List getTableNames() { + public List getEntityNames() { return Collections.singletonList(featureEntityName()); } @@ -169,8 +170,9 @@ public List getSimilar(SegmentContainer sc, ReadableQueryConfig qc public List getSimilar(String segmentId, ReadableQueryConfig rqc) { return this.mediaSegmentReader.lookUpSegment(segmentId) .map(MediaSegmentDescriptor::getObjectId) - .map(objId -> this.dbSelector.getFeatureVectors(ID_COLUMN_NAME, new StringTypeProvider(objId), FEATURE_COLUMN_NAME, rqc)) + .map(objId -> this.dbSelector.getFeatures(ID_COLUMN_NAME, new StringTypeProvider(objId), FEATURE_COLUMN_NAME, rqc)) .flatMap(features -> features.stream().findFirst()) // Feature vectors are unique per id + .map(FloatArrayProvider::getFloatArray) .map(feature -> this.getSimilar(feature, rqc)) .orElse(Collections.emptyList()); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/StagedFeatureModule.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/StagedFeatureModule.java index 16fb27a90..bed80f8ca 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/StagedFeatureModule.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/abstracts/StagedFeatureModule.java @@ -5,10 +5,12 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.vitrivr.cineast.core.config.QueryConfig; import org.vitrivr.cineast.core.config.ReadableQueryConfig; import org.vitrivr.cineast.core.data.distance.SegmentDistanceElement; +import org.vitrivr.cineast.core.data.providers.primitive.FloatArrayProvider; import org.vitrivr.cineast.core.data.providers.primitive.StringTypeProvider; import org.vitrivr.cineast.core.data.score.ScoreElement; import org.vitrivr.cineast.core.data.segments.SegmentContainer; @@ -25,12 +27,12 @@ public abstract class StagedFeatureModule extends AbstractFeatureModule { /** * Constructor * - * @param tableName Name of the entity / table to persist data with and read data from. + * @param entityName Name of the entity / table to persist data with and read data from. * @param maxDist Maximum distance value (for normalization). * @param vectorLength Dimensionality of the feature vector. */ - protected StagedFeatureModule(String tableName, float maxDist, int vectorLength) { - super(tableName, maxDist, vectorLength); + protected StagedFeatureModule(String entityName, float maxDist, int vectorLength) { + super(entityName, maxDist, vectorLength); } /** @@ -95,7 +97,7 @@ public List getSimilar(String segmentId, ReadableQueryConfig qc) { QueryConfig qcc = this.defaultQueryConfig(qc); /* Lookup features. */ - List features = this.selector.getFeatureVectors(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER, qc); + List features = this.selector.getFeatures(GENERIC_ID_COLUMN_QUALIFIER, new StringTypeProvider(segmentId), FEATURE_COLUMN_QUALIFIER, qc).stream().map(FloatArrayProvider::getFloatArray).collect(Collectors.toList()); if (features.isEmpty()) { LOGGER.warn("No features could be fetched for the provided segmentId '{}'. Aborting query execution...", segmentId); return new ArrayList<>(0); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java index 9c08ed0dd..e89782956 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSegmentExporter.java @@ -1,41 +1,30 @@ package org.vitrivr.cineast.core.features.exporter; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; - import java.io.IOException; import java.io.OutputStream; import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.HashMap; -import java.util.function.Supplier; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.frames.AudioFrame; import org.vitrivr.cineast.core.data.segments.SegmentContainer; -import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; -import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.features.extractor.Extractor; +import org.vitrivr.cineast.core.features.abstracts.AbstractSegmentExporter; import org.vitrivr.cineast.core.util.LogHelper; /** * Exports the audio in a given segment as mono WAV file. */ -public class AudioSegmentExporter implements Extractor { - - - private static final Logger LOGGER = LogManager.getLogger(); +public class AudioSegmentExporter extends AbstractSegmentExporter { - private static final String PROPERTY_NAME_DESTINATION = "destination"; + @Override + protected String getFileExtension() { + return "wav"; + } - /** - * Destination path for the audio-segment. - */ - private Path destination; + @Override + protected String getDataUrlPrefix() { + return "data:audio/wav;base64,"; + } /** * Default constructor @@ -56,21 +45,19 @@ public AudioSegmentExporter() { * @param properties HashMap containing named properties */ public AudioSegmentExporter(HashMap properties) { - this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); + super(properties); } + + /** * Processes a SegmentContainer: Extract audio-data and writes to a WAVE file. * * @param shot SegmentContainer to process. */ @Override - public void processSegment(SegmentContainer shot) { + public void exportToStream(SegmentContainer shot, OutputStream stream) { try { - /* Prepare folder and OutputStream. */ - Path directory = this.destination.resolve(shot.getSuperId()); - Files.createDirectories(directory); - OutputStream stream = Files.newOutputStream(directory.resolve(shot.getId() + ".wav"), CREATE, TRUNCATE_EXISTING); /* Extract mean samples and perpare byte buffer. */ short[] data = shot.getMeanSamplesAsShort(); @@ -85,12 +72,16 @@ public void processSegment(SegmentContainer shot) { } stream.write(buffer.array()); - stream.close(); } catch (IOException | BufferOverflowException e) { LOGGER.fatal("Could not export audio segment {} due to a serious IO error ({}).", shot.getId(), LogHelper.getStackTrace(e)); } } + @Override + public boolean isExportable(SegmentContainer sc) { + return sc.getNumberOfSamples() > 0; + } + /** * Writes the WAV header to the ByteBuffer (1 channel). * @@ -123,16 +114,4 @@ private void writeWaveHeader(ByteBuffer buffer, float samplingrate, short channe buffer.putInt(subChunk2Length); /* Length of the data chunk. */ } - - @Override - public void init(PersistencyWriterSupplier phandlerSupply) { /* Nothing to init. */ } - - @Override - public void finish() { /* Nothing to finish. */} - - @Override - public void initalizePersistentLayer(Supplier supply) { /* Nothing to init. */ } - - @Override - public void dropPersistentLayer(Supplier supply) { /* Nothing to drop. */} } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java index 7185576af..59e72b79a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/AudioSpectogramExporter.java @@ -2,19 +2,14 @@ import java.awt.image.BufferedImage; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.io.OutputStream; import java.util.HashMap; import java.util.List; -import java.util.function.Supplier; import javax.imageio.ImageIO; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.vitrivr.cineast.core.data.segments.SegmentContainer; -import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; -import org.vitrivr.cineast.core.db.setup.EntityCreator; -import org.vitrivr.cineast.core.features.extractor.Extractor; +import org.vitrivr.cineast.core.features.abstracts.AbstractSegmentExporter; import org.vitrivr.cineast.core.util.LogHelper; import org.vitrivr.cineast.core.util.dsp.fft.STFT; import org.vitrivr.cineast.core.util.dsp.fft.Spectrum; @@ -24,7 +19,7 @@ /** * Visualizes and exporst the power spectogram (time vs. frequency vs. power) of the provided AudioSegment. */ -public class AudioSpectogramExporter implements Extractor { +public class AudioSpectogramExporter extends AbstractSegmentExporter { private static final Logger LOGGER = LogManager.getLogger(); @@ -32,15 +27,10 @@ public class AudioSpectogramExporter implements Extractor { /** * Property names that can be used in the configuration hash map. */ - private static final String PROPERTY_NAME_DESTINATION = "destination"; private static final String PROPERTY_NAME_WIDTH = "width"; private static final String PROPERTY_NAME_HEIGHT = "height"; private static final String PROPERTY_NAME_FORMAT = "format"; - /** - * Destination path; can be set in the AudioWaveformExporter properties. - */ - private final Path destination; /** * Width of the resulting image in pixels. @@ -53,10 +43,20 @@ public class AudioSpectogramExporter implements Extractor { private final int height; /** - * Output format for thumbnails. Defaults to PNG. + * Output format for thumbnails. Defaults to JPG. */ private final String format; + @Override + protected String getFileExtension() { + return this.format.toLowerCase(); + } + + @Override + protected String getDataUrlPrefix() { + return "data:image/" + format.toLowerCase() + ";base64,"; + } + /** * Default constructor */ @@ -78,21 +78,15 @@ public AudioSpectogramExporter() { * @param properties HashMap containing named properties */ public AudioSpectogramExporter(HashMap properties) { - this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); + super(properties); this.width = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_WIDTH, "800")); this.height = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_HEIGHT, "600")); this.format = properties.getOrDefault(PROPERTY_NAME_FORMAT, "JPG"); } @Override - public void processSegment(SegmentContainer shot) { - /* If shot has no samples, this step is skipped. */ - if (shot.getNumberOfSamples() == 0) { - return; - } - + public void exportToStream(SegmentContainer shot, OutputStream stream) { /* Prepare STFT and Spectrum for the segment. */ - final Path directory = this.destination.resolve(shot.getSuperId()); final STFT stft = shot.getSTFT(2048, 512, new HanningWindow()); final List spectrums = stft.getPowerSpectrum(); @@ -100,8 +94,7 @@ public void processSegment(SegmentContainer shot) { try { BufferedImage image = AudioSignalVisualizer.visualizeSpectogram(spectrums, this.width, this.height); if (image != null) { - Files.createDirectories(directory); - ImageIO.write(image, format, directory.resolve(shot.getId() + "." + format.toLowerCase()).toFile()); + ImageIO.write(image, format, stream); } else { LOGGER.warn("Spectrum could not be visualized!"); } @@ -111,14 +104,8 @@ public void processSegment(SegmentContainer shot) { } @Override - public void init(PersistencyWriterSupplier phandlerSupplier) { /* Noting to init. */} - - @Override - public void finish() { /* Nothing to finish. */} - - @Override - public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} + public boolean isExportable(SegmentContainer sc) { + return sc.getNumberOfSamples() > 0; + } - @Override - public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java index 67d2cbfad..70a4f5c2a 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/Model3DThumbnailExporter.java @@ -10,9 +10,12 @@ import java.util.Map; import java.util.function.Supplier; import javax.imageio.ImageIO; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.bytedeco.javacpp.presets.opencv_core; import org.joml.Vector3f; +import org.tensorflow.op.DtypesOps; import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; import org.vitrivr.cineast.core.data.segments.SegmentContainer; import org.vitrivr.cineast.core.db.PersistencyWriterSupplier; @@ -31,129 +34,149 @@ public class Model3DThumbnailExporter implements Extractor { - private static final Logger LOGGER = LogManager.getLogger(); - - /** - * Property names that can be used in the configuration hash map. - */ - private static final String PROPERTY_NAME_DESTINATION = "destination"; - private static final String PROPERTY_NAME_SIZE = "size"; - - /** - * Distance of camera from object. - */ - private static final float DISTANCE = 1f; //(float) Math.sqrt(3); - - /** - * Destination path; can be set in the AudioWaveformExporter properties. - */ - private final Path destination; - - /** - * Size of the resulting image in pixels (image will have dimension size x size). - */ - private final int size; - - - /** - * Background color of the resulting image. - */ - private final Color backgroundColor = Color.lightGray; - - /** - * Default constructor. The AudioWaveformExporter can be configured via named properties in the provided HashMap. Supported parameters: - * - *
    - *
  1. destination: Path where images should be stored.
    2. - *
  2. size: Width of the resulting image in pixels (size x size).
    3. - *
- * - * @param properties HashMap containing named properties - */ - public Model3DThumbnailExporter(Map properties) { - this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); - this.size = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_SIZE, "800")); - } - - /** - * Processes a SegmentContainer: Extracts audio-data and visualizes its waveform. - * - * @param shot SegmentContainer to process. - */ - @Override - public void processSegment(SegmentContainer shot) { - Path directory = this.destination.resolve(shot.getSuperId()); - try { - Files.createDirectories(directory); - // Get the model to generate a thumbnail for. - IModel model = shot.getModel(); - if (model.getMaterials().size() > 0) { - // Set options for the renderer. - var windowOptions = new WindowOptions(400, 400) {{ - this.hideWindow = true; - }}; - var renderOptions = new RenderOptions() {{ - this.showTextures = true; - }}; - // Set options for the entropy optimizer. - var opts = new OptimizerOptions() {{ - this.iterations = 100; - this.initialViewVector = new Vector3f(0, 0, 1); - this.method = EntopyCalculationMethod.RELATIVE_TO_TOTAL_AREA_WEIGHTED; - this.optimizer = EntropyOptimizerStrategy.RANDOMIZED; - this.yNegWeight = 0.7f; - this.yPosWeight = 0.8f; - }}; - - // Add a Random View, a front View an Upper Left View and an Entropy Optimized View - var cameraPositions = new LinkedList() {{ - add(new Vector3f( - (float) (Math.random() - 0.5) * 2f, - (float) (Math.random() - 0.5) * 2f, - (float) (Math.random() - 0.5) * 2f) - .normalize().mul(DISTANCE)); - add(new Vector3f(0f, 0f, 1f).normalize().mul(DISTANCE)); - add(new Vector3f(-1f, 1f, 1f).normalize().mul(DISTANCE)); - add(ModelEntropyOptimizer.getViewVectorWithMaximizedEntropy(model, opts)); - }}; - - // Render the model. - var images = RenderJob.performStandardRenderJob(RenderWorker.getRenderJobQueue(), model, cameraPositions, windowOptions, renderOptions); - assert images.size() == 4; - // Combine the images into a single image. - var canvas = new BufferedImage(this.size, this.size, BufferedImage.TYPE_INT_RGB); - var graphics = canvas.getGraphics(); - graphics.setColor(this.backgroundColor); - int sz = this.size / 2; - // ic: sub-image counter, idx: x-axis-index, idy: y-axis-index - var ic = 0; - for (var partialImage : images) { - int idx = ic % 2; - int idy = ic < 2 ? 0 : 1; - graphics.drawImage(partialImage, idx * sz, idy * sz, null); - ++ic; + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * Property names that can be used in the configuration hash map. + */ + private static final String PROPERTY_NAME_DESTINATION = "destination"; + private static final String PROPERTY_NAME_SIZE = "size"; + + /** + * Distance of camera from object. + */ + private static final float DISTANCE = 1f; //(float) Math.sqrt(3); + + /** + * Destination path; can be set in the AudioWaveformExporter properties. + */ + private final Path destination; + + /** + * Size of the resulting image in pixels (image will have dimension size x size). + */ + private final int size; + + + /** + * Background color of the resulting image. + */ + private final Color backgroundColor = Color.lightGray; + + /** + * Default constructor. The AudioWaveformExporter can be configured via named properties in the provided HashMap. Supported parameters: + * + *
    + *
  1. destination: Path where images should be stored.
    2. + *
  2. size: Width of the resulting image in pixels (size x size).
    3. + *
+ * + * @param properties HashMap containing named properties + */ + public Model3DThumbnailExporter(Map properties) { + + + this.destination = Paths.get(properties.getOrDefault(PROPERTY_NAME_DESTINATION, ".")); + this.size = Integer.parseInt(properties.getOrDefault(PROPERTY_NAME_SIZE, "1200")); + } + + + /** + * Processes a SegmentContainer: Extracts audio-data and visualizes its waveform. + * + * @param shot SegmentContainer to process. + */ + @Override + public void processSegment(SegmentContainer shot) { + Path directory = this.destination.resolve(shot.getSuperId()); + try { + Files.createDirectories(directory); + // Get the model to generate a thumbnail for. + IModel model = shot.getModel(); + if (model.getMaterials().size() > 0) { + // Set options for the renderer. + + var windowOptions = new WindowOptions(1200, 1200) {{ + this.hideWindow = false; + + }}; + var renderOptions = new RenderOptions() {{ + this.showTextures = true; + + this.lightingOptions.hasNonDefaultTexture = model.usesNonDefaultTexture(); + }}; + // Set options for the entropy optimizer. + var opts = new OptimizerOptions() {{ + this.iterations = 100; + this.initialViewVector = new Vector3f(0, 0, 1); + this.method = EntopyCalculationMethod.RELATIVE_TO_TOTAL_AREA_WEIGHTED; + this.optimizer = EntropyOptimizerStrategy.RANDOMIZED; + this.yNegWeight = 0.7f; + this.yPosWeight = 0.8f; + }}; + + // Add a Random View, a front View an Upper Left View and an Entropy Optimized View + + var cameraPositions = new LinkedList() {{ + add(new Vector3f( + (float) (Math.random() - 0.5) * 2f, + (float) (Math.random() - 0.5) * 2f, + (float) (Math.random() - 0.5) * 2f) + .normalize().mul(DISTANCE)); + add(new Vector3f(0f, 0f, 1f).normalize().mul(DISTANCE)); + add(new Vector3f(-1f, 1f, 1f).normalize().mul(DISTANCE)); + add(ModelEntropyOptimizer.getViewVectorWithMaximizedEntropy(model, opts)); + }}; + + var images = RenderJob.performStandardRenderJob(RenderWorker.getRenderJobQueue(), model, cameraPositions, windowOptions, renderOptions); + assert images.size() == 4; + + // Render the model. + // Combine the images into a single image. + var canvas = new BufferedImage(this.size, this.size, BufferedImage.TYPE_INT_RGB); + var graphics = canvas.getGraphics(); + + + var print = true; + if (print == true){ + graphics.setColor(Color.black); + graphics.drawImage(images.get(2), 0, 0, null); + ImageIO.write(canvas, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); + }else { + graphics.setColor(this.backgroundColor); + // Render the model. + // Combine the images into a single image. + int sz = this.size / 2; + // ic: sub-image counter, idx: x-axis-index, idy: y-axis-index + var ic = 0; + for (var partialImage : images) { + int idx = ic % 2; + int idy = ic < 2 ? 0 : 1; + graphics.drawImage(partialImage, idx * sz, idy * sz, null); + ++ic; + } + ImageIO.write(canvas, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); + } + } + } catch (IOException exception) { + LOGGER.fatal("Could not export thumbnail image for model {} due to a serious IO error ({}).", shot.getId(), LogHelper.getStackTrace(exception)); + } catch (Exception exception) { + LOGGER.error("Could not export thumbnail image for model {} because an unknown exception occurred ({}).", shot.getId(), LogHelper.getStackTrace(exception)); + } finally { + LOGGER.trace("Finished processing thumbnail {}.", shot.getId()); } - ImageIO.write(canvas, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile()); - } - } catch (IOException exception) { - LOGGER.fatal("Could not export thumbnail image for model {} due to a serious IO error ({}).", shot.getId(), LogHelper.getStackTrace(exception)); - } catch (Exception exception) { - LOGGER.error("Could not export thumbnail image for model {} because an unknown exception occurred ({}).", shot.getId(), LogHelper.getStackTrace(exception)); - } finally { - LOGGER.trace("Finished processing thumbnail {}.", shot.getId()); } - } - @Override - public void init(PersistencyWriterSupplier phandlerSupply) { /* Noting to init. */} + @Override + public void init(PersistencyWriterSupplier phandlerSupply) { /* Noting to init. */} - @Override - public void finish() { /* Nothing to finish. */} + @Override + public void finish() { /* Nothing to finish. */} - @Override - public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} + @Override + public void initalizePersistentLayer(Supplier supply) {/* Nothing to initialize. */} - @Override - public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} + @Override + public void dropPersistentLayer(Supplier supply) {/* Nothing to drop. */} } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/QueryImageExporter.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/QueryImageExporter.java index 30f6b4643..baa5e4a97 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/QueryImageExporter.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/exporter/QueryImageExporter.java @@ -68,7 +68,7 @@ public void finish() { } @Override - public List getTableNames() { + public List getEntityNames() { return new ArrayList<>(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java index d448a391f..cf16fb6c9 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/Renderer.java @@ -150,7 +150,7 @@ default void positionCameraPolar(float r, float theta, float phi, float cx, floa * * @return BufferedImage containing a snapshot of the current render-buffer. */ - BufferedImage obtain(); + BufferedImage obtain() throws InterruptedException; /** * Clears buffers to preset-values and applies a user-defined background colour. diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/Engine.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/Engine.java index 2899bf57f..61bf826dd 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/Engine.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/engine/Engine.java @@ -1,10 +1,12 @@ package org.vitrivr.cineast.core.render.lwjgl.engine; + import org.vitrivr.cineast.core.render.lwjgl.render.Render; import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; import org.vitrivr.cineast.core.render.lwjgl.scene.Camera; import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLScene; import org.vitrivr.cineast.core.render.lwjgl.scene.Scene; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.LightingOptions; import org.vitrivr.cineast.core.render.lwjgl.window.Window; import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; @@ -73,6 +75,9 @@ public void setRenderOptions(RenderOptions options){ this.render.setOptions(options); } + public void setLightingOptions(LightingOptions options){ + this.scene.setSceneLights(options); + } /** * Refreshes the engine. diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMaterial.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMaterial.java index dd66fe99a..08c538892 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMaterial.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMaterial.java @@ -29,6 +29,12 @@ public class GLMaterial { * The contained texture in gl context */ private final GLTexture texture; + + /** + * The contained texture in gl context + */ + private final GLTexture normalMapTexture; + /** * The material that is wrapped by this gl material. */ @@ -44,6 +50,11 @@ public GLMaterial(Material material) { this.material = material; this.material.getMeshes().forEach(mesh -> this.meshes.add(new GLMesh(mesh))); this.texture = new GLTexture(this.material.getTexture()); + if (this.material.hasNormalMapTexture()){ + this.normalMapTexture = new GLTexture(this.material.getNormalMapTexture()); + } else { + this.normalMapTexture = null; + } } /** @@ -76,6 +87,19 @@ public GLTexture getTexture() { return this.texture; } + /** + * Returns the gl texture of this gl material. + * + * @return The gl texture of this gl material. + */ + public GLTexture getNormalMapTexture() { + return this.normalMapTexture; + } + + public boolean hasNormalMapTexture() { + return this.normalMapTexture != null; + } + /** * Returns the color from wrapped generic material. * @@ -84,4 +108,28 @@ public GLTexture getTexture() { public Vector4f getDiffuseColor() { return this.material.getDiffuseColor(); } + /** + * Returns the color from wrapped generic material. + * + * @return The color from wrapped generic material. (r,g,b,opacity) + */ + public Vector4f getAmbientColor() { + return this.material.getAmbientColor(); + } + /** + * Returns the color from wrapped generic material. + * + * @return The color from wrapped generic material. (r,g,b,opacity) + */ + public Vector4f getSpecularColor() { + return this.material.getSpecularColor(); + } + /** + * Returns the color from wrapped generic material. + * + * @return The color from wrapped generic material. (r,g,b,opacity) + */ + public float getReflectance() { + return this.material.getReflectance(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMesh.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMesh.java index 3d175e5bf..b34b6af26 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMesh.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLMesh.java @@ -44,7 +44,8 @@ public class GLMesh { *
  • Generate, allocate and initialize Texture Coordinates Buffer
    • *
  • Generate, allocate and initialize Index Buffer
    • *
  • Unbind Vertex Array Object
    • - * + * + * * @param mesh The mesh that is wrapped by this gl mesh. */ public GLMesh(Mesh mesh) { @@ -66,6 +67,36 @@ public GLMesh(Mesh mesh) { GL30.glEnableVertexAttribArray(0); GL30.glVertexAttribPointer(0, 3, GL30.GL_FLOAT, false, 0, 0); + // Normals VBO + vboId = GL30.glGenBuffers(); + this.vboIdList.add(vboId); + var normalsBuffer = memoryStack.callocFloat(this.mesh.getVerticesNormals().length); + normalsBuffer.put(0, this.mesh.getVerticesNormals()); + GL30.glBindBuffer(GL30.GL_ARRAY_BUFFER, vboId); + GL30.glBufferData(GL30.GL_ARRAY_BUFFER, normalsBuffer, GL30.GL_STATIC_DRAW); + GL30.glEnableVertexAttribArray(1); + GL30.glVertexAttribPointer(1, 3, GL30.GL_FLOAT, false, 0, 0); + + // Tangents VBO + vboId = GL30.glGenBuffers(); + this.vboIdList.add(vboId); + var tangentsBuffer = memoryStack.callocFloat(this.mesh.getTangents().length); + tangentsBuffer.put(0, this.mesh.getTangents()); + GL30.glBindBuffer(GL30.GL_ARRAY_BUFFER, vboId); + GL30.glBufferData(GL30.GL_ARRAY_BUFFER, tangentsBuffer, GL30.GL_STATIC_DRAW); + GL30.glEnableVertexAttribArray(2); + GL30.glVertexAttribPointer(2, 3, GL30.GL_FLOAT, false, 0, 0); + + // Bitangents VBO + vboId = GL30.glGenBuffers(); + this.vboIdList.add(vboId); + var bitangentsBuffer = memoryStack.callocFloat(this.mesh.getBitangents().length); + bitangentsBuffer.put(0, this.mesh.getBitangents()); + GL30.glBindBuffer(GL30.GL_ARRAY_BUFFER, vboId); + GL30.glBufferData(GL30.GL_ARRAY_BUFFER, bitangentsBuffer, GL30.GL_STATIC_DRAW); + GL30.glEnableVertexAttribArray(3); + GL30.glVertexAttribPointer(3, 3, GL30.GL_FLOAT, false, 0, 0); + // Textures VBO (Vertex Buffer Object) vboId = GL30.glGenBuffers(); this.vboIdList.add(vboId); @@ -73,8 +104,8 @@ public GLMesh(Mesh mesh) { textureCoordinatesBuffer.put(0, this.mesh.getTextureCoords()); GL30.glBindBuffer(GL30.GL_ARRAY_BUFFER, vboId); GL30.glBufferData(GL30.GL_ARRAY_BUFFER, textureCoordinatesBuffer, GL30.GL_STATIC_DRAW); - GL30.glEnableVertexAttribArray(1); - GL30.glVertexAttribPointer(1, 2, GL30.GL_FLOAT, false, 0, 0); + GL30.glEnableVertexAttribArray(4); + GL30.glVertexAttribPointer(4, 2, GL30.GL_FLOAT, false, 0, 0); // Index VBO (Vertex Buffer Object) vboId = GL30.glGenBuffers(); @@ -90,8 +121,7 @@ public GLMesh(Mesh mesh) { } /** - * Cleans up the gl mesh and calls all underlying cleanup methods. - * Removes only the references to VBOs and VAOs. + * Cleans up the gl mesh and calls all underlying cleanup methods. Removes only the references to VBOs and VAOs. * Removes the Vertex Array Object (VAO) and all Vertex Buffer Object (VBO) ids. */ public void cleanup() { @@ -103,6 +133,7 @@ public void cleanup() { /** * Returns the number of vertices of the wrapped generic mesh. + * * @return The number of vertices of the wrapped generic mesh. */ public int getNumVertices() { @@ -111,6 +142,7 @@ public int getNumVertices() { /** * Returns the Vertex Array Object (VAO) id. + * * @return The Vertex Array Object (VAO) id. */ public final int getVaoId() { @@ -119,6 +151,7 @@ public final int getVaoId() { /** * Returns the ID of the wrapped generic mesh. + * * @return The ID of the wrapped generic mesh. */ public String getId() { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLModel.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLModel.java index c0ad58cca..409a3311e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLModel.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLModel.java @@ -18,6 +18,7 @@ * {@link IModel} -> {@link GLModel} */ public class GLModel implements IGLModel { + private static final Logger LOGGER = LogManager.getLogger(); /** @@ -83,4 +84,8 @@ public List getMaterials() { return Collections.unmodifiableList(this.materials); } + @Override + public boolean usesNonDefaultTexture() { + return this.model.usesNonDefaultTexture(); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLScene.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLScene.java index 38e82c2dc..e5ceec972 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLScene.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLScene.java @@ -7,12 +7,13 @@ import org.vitrivr.cineast.core.render.lwjgl.scene.Camera; import org.vitrivr.cineast.core.render.lwjgl.scene.Projection; import org.vitrivr.cineast.core.render.lwjgl.scene.Scene; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.LightingOptions; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.SceneLights; /** - * The GLScene class ist the top most class of the gl model hierarchy. - * The gl model hierarchy is used as a wrapper for the model hierarchy. - * Therefore, each gl class has a corresponding model class. - * The generic class has to be provided in the constructor. + * The GLScene class ist the top most class of the gl model hierarchy. The gl model hierarchy is used as a wrapper for + * the model hierarchy. Therefore, each gl class has a corresponding model class. The generic class has to be provided + * in the constructor. *
      *
    • Scene -> GLScene( Scene )
      • *
    • Model -> GlModel( IModel )
      • @@ -20,7 +21,7 @@ *
    • Mesh -> GLMesh( Mesh )
      • *
    • Texture -> GLTexture( Texture )
      • *
    - * + *

    * The purpose is to bring the generic model in an OpenGl context * {@link Scene} -> {@link GLScene} */ @@ -36,9 +37,8 @@ public class GLScene { private final Map models; /** - * The texture cache that is used by this gl scene. - * Textures are cached to avoid loading the same texture multiple times. - * Has no corresponding generic class. + * The texture cache that is used by this gl scene. Textures are cached to avoid loading the same texture multiple + * times. Has no corresponding generic class. */ private final GLTextureCache textureCache; @@ -65,18 +65,23 @@ public void addModel(IModel model) { } /** - * Updates the gl scene from the scene. - * It updates the gl scene content to match the scene content. + * Updates the gl scene from the scene. It updates the gl scene content to match the scene content. */ private void updateGlSceneFromScene() { this.scene.getModels().forEach((k, v) -> this.models.putIfAbsent(k, new GLModel(v))); this.models.forEach( (k, v) -> this.models.get(k).getMaterials() - .forEach(mat -> this.textureCache.addTextureIfAbsent(mat.getTexture()))); + .forEach(mat -> { + this.textureCache.addTextureIfAbsent(mat.getTexture()); + if (mat.hasNormalMapTexture()) { + this.textureCache.addTextureIfAbsent(mat.getNormalMapTexture()); + } + })); } /** * Adds an entity to the corresponding model. + * * @param entity The entity that is added to the model. */ public void addEntity(Entity entity) { @@ -90,6 +95,7 @@ public void addEntity(Entity entity) { /** * Returns the gl models of the gl scene. + * * @return The gl models of the gl scene. */ public Map getModels() { @@ -98,6 +104,7 @@ public Map getModels() { /** * Returns the texture cache of the gl scene. + * * @return The texture cache of the gl scene. */ public GLTextureCache getTextureCache() { @@ -106,6 +113,7 @@ public GLTextureCache getTextureCache() { /** * Returns the projection of the wrapped generic scene. + * * @return The projection of the wrapped generic scene. */ public Projection getProjection() { @@ -114,6 +122,7 @@ public Projection getProjection() { /** * Returns the camera of the wrapped generic scene. + * * @return The camera of the wrapped generic scene. */ public Camera getCamera() { @@ -121,10 +130,9 @@ public Camera getCamera() { } /** - * Clears the models of the gl scene but not containing resources. - * Removes the references to the wrapped generic models and textures. - * Hence, the models could be used by another extraction task this method does not close the models or textures. - * Can be used to only remove Models temporarily from gl scene. + * Clears the models of the gl scene but not containing resources. Removes the references to the wrapped generic + * models and textures. Hence, the models could be used by another extraction task this method does not close the + * models or textures. Can be used to only remove Models temporarily from gl scene. */ @SuppressWarnings("unused") public void clearModels() { @@ -133,9 +141,9 @@ public void clearModels() { } /** - * Cleans up the gl scene and calls all underlying cleanup methods. - * Removes only the references to wrapped generic models and textures. - * Hence, the model could be used by another extraction task this method does not close the generic models or textures. + * Cleans up the gl scene and calls all underlying cleanup methods. Removes only the references to wrapped generic + * models and textures. Hence, the model could be used by another extraction task this method does not close the + * generic models or textures. */ public void cleanup() { this.models.values().forEach(IGLModel::cleanup); @@ -145,11 +153,32 @@ public void cleanup() { /** * Resizes the projection of the wrapped generic scene. - * @param width The new width of the projection. + * + * @param width The new width of the projection. * @param height The new height of the projection. */ public void resize(int width, int height) { this.scene.getProjection().updateProjMatrix(width, height); } + /** + * Get the scene lights. + */ + public SceneLights getSceneLights() { + return this.scene.getSceneLights(); + } + + /** + * Get the scene lights. + */ + public void setSceneLights(SceneLights sceneLights) { + this.scene.setSceneLights(sceneLights); + } + + /** + * Get the scene lights. + */ + public void setSceneLights(LightingOptions options) { + this.scene.setSceneLights(options.getSceneLigths(this.scene)); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTexture.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTexture.java index db0fb4409..4e2f1b97b 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTexture.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTexture.java @@ -1,6 +1,7 @@ package org.vitrivr.cineast.core.render.lwjgl.glmodel; import java.nio.ByteBuffer; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Texture; import org.lwjgl.opengl.GL30; import org.lwjgl.stb.STBImage; import org.lwjgl.system.MemoryStack; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTextureCache.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTextureCache.java index d38a3b363..14c45190d 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTextureCache.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/GLTextureCache.java @@ -33,6 +33,7 @@ public GLTextureCache() { public void cleanup() { this.textures.values().forEach(GLTexture::cleanup); this.textures.clear(); + this.textures.put("default", new GLTexture(new Texture())); } /** diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/IGLModel.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/IGLModel.java index e4bbf177d..0ed05e4c7 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/IGLModel.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/glmodel/IGLModel.java @@ -44,4 +44,6 @@ public interface IGLModel{ * @return The gl materials of the gl model. */ List getMaterials(); + + boolean usesNonDefaultTexture(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/RenderOptions.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/RenderOptions.java index 3e5f88d4d..e565819c2 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/RenderOptions.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/RenderOptions.java @@ -3,6 +3,7 @@ import java.util.function.Function; import org.joml.Vector4f; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.LightingOptions; /** * RenderOptions @@ -26,6 +27,9 @@ public class RenderOptions { @SuppressWarnings("unused") public boolean showColor = false; + + public LightingOptions lightingOptions = LightingOptions.STATIC; + /** * Returns the color for the given value Can be used to colorize the model custom */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/SceneRender.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/SceneRender.java index f6d8e0700..554ffd649 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/SceneRender.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/SceneRender.java @@ -1,10 +1,19 @@ package org.vitrivr.cineast.core.render.lwjgl.render; import java.util.ArrayList; +import java.util.Vector; + +import org.joml.Matrix4f; +import org.joml.Vector3f; +import org.joml.Vector4f; import org.lwjgl.opengl.GL30; import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLScene; import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLTexture; import org.vitrivr.cineast.core.render.lwjgl.render.ShaderProgram.ShaderModuleData; +import org.vitrivr.cineast.core.render.lwjgl.scene.Scene; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.Attenuation; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.PointLight; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.SpotLight; /** * SceneRender @@ -18,131 +27,273 @@ */ public class SceneRender { - /** - * Resource path to the scene shader program - */ - private static final String VERTEX_SHADER_PATH = "./resources/renderer/lwjgl/shaders/scene.vert"; - /** - * Resource path to the fragment shader program - */ - private static final String FRAGMENT_SHADER_PATH = "./resources/renderer/lwjgl/shaders/scene.frag"; - /** - * Instance of the scene shader program - */ - private final ShaderProgram shaderProgram; - /** - * Uniforms for the scene shader - */ - private UniformsMap uniformsMap; - - /** - * SceneRender. During construction: Loads the scene shader from the resources - */ - public SceneRender() { - var shaderModuleDataList = new ArrayList(); - shaderModuleDataList.add(new ShaderModuleData(VERTEX_SHADER_PATH, GL30.GL_VERTEX_SHADER)); - shaderModuleDataList.add(new ShaderModuleData(FRAGMENT_SHADER_PATH, GL30.GL_FRAGMENT_SHADER)); - this.shaderProgram = new ShaderProgram(shaderModuleDataList); - this.createUniforms(); - } - - /** - * Creates the uniforms for the scene shader creates the following uniforms: - *

      - *
    • projectionMatrix
      • - *
    • modelMatrix
      • - *
    • viewMatrix
      • - *
    • txtSampler
      • - *
    • material.diffuse
      • - *
    - */ - private void createUniforms() { - this.uniformsMap = new UniformsMap(this.shaderProgram.getProgramId()); - this.uniformsMap.createUniform("projectionMatrix"); - this.uniformsMap.createUniform("modelMatrix"); - this.uniformsMap.createUniform("viewMatrix"); - this.uniformsMap.createUniform("txtSampler"); - this.uniformsMap.createUniform("material.diffuse"); - } - - /** - * Releases all resources - *
      - *
    • Releases the shader program
      • - *
    • Releases the uniforms
      • - *
    - */ - public void cleanup() { - this.shaderProgram.cleanup(); - this.uniformsMap.cleanup(); - this.uniformsMap = null; - } - - /** - * Renders the Models in the scene - * Creates standard render options - * @param scene Scene to render - */ - public void render(GLScene scene) { - this.render(scene, new RenderOptions()); - } - - /** - * Renders the Models in the scene - *
      - *
    • Binds projection matrix
      • - *
    • Binds view matrix
      • - *
    • Binds texture sampler
      • - *
    - * Further, iterate over all models in the scene - *
      - *
    • Iterate over all materials in the model
      • - *
    • Sets texture or color function
      • - *
    • Iterate over all meshes in the material
      • - *
    • Binds the mesh
      • - *
    • Iterate over all entities to draw the mesh
      • - *
    • Binds the model matrix
      • - *
    • Draws the mesh
      • - *
    • Unbinds
      • - *
    - * @param scene Scene to render - * @param opt Render options - */ - public void render(GLScene scene, RenderOptions opt) { - this.shaderProgram.bind(); - - this.uniformsMap.setUniform("projectionMatrix", scene.getProjection().getProjMatrix()); - this.uniformsMap.setUniform("viewMatrix", scene.getCamera().getViewMatrix()); - this.uniformsMap.setUniform("txtSampler", 0); - - var models = scene.getModels().values(); - var textures = scene.getTextureCache(); - - for (var model : models) { - var entities = model.getEntities(); - for (var material : model.getMaterials()) { - GLTexture texture; - // Either draw texture or use color function - if (opt.showTextures) { - this.uniformsMap.setUniform("material.diffuse", material.getDiffuseColor()); - texture = textures.getTexture(material.getTexture().getTexturePath()); - } else { - this.uniformsMap.setUniform("material.diffuse", opt.colorfunction.apply(1f)); - texture = textures.getTexture("default"); + private static final int MAX_POINT_LIGHTS = 5; + private static final int MAX_SPOT_LIGHTS = 5; + + /** + * Resource path to the scene shader program + */ + private static final String VERTEX_SHADER_PATH = "./resources/renderer/lwjgl/shaders/scene.vert"; + /** + * Resource path to the fragment shader program + */ + private static final String FRAGMENT_SHADER_PATH = "./resources/renderer/lwjgl/shaders/scene.frag"; + /** + * Instance of the scene shader program + */ + private final ShaderProgram shaderProgram; + /** + * Uniforms for the scene shader + */ + private UniformsMap uniformsMap; + + /** + * SceneRender. During construction: Loads the scene shader from the resources + */ + public SceneRender() { + var shaderModuleDataList = new ArrayList(); + shaderModuleDataList.add(new ShaderModuleData(VERTEX_SHADER_PATH, GL30.GL_VERTEX_SHADER)); + shaderModuleDataList.add(new ShaderModuleData(FRAGMENT_SHADER_PATH, GL30.GL_FRAGMENT_SHADER)); + this.shaderProgram = new ShaderProgram(shaderModuleDataList); + this.createUniforms(); + } + + /** + * Creates the uniforms for the scene shader creates the following uniforms: + *
      + *
    • projectionMatrix
      • + *
    • modelMatrix
      • + *
    • viewMatrix
      • + *
    • txtSampler
      • + *
    • material.diffuse
      • + *
    + */ + private void createUniforms() { + this.uniformsMap = new UniformsMap(this.shaderProgram.getProgramId()); + this.uniformsMap.createUniform("projectionMatrix"); + this.uniformsMap.createUniform("modelMatrix"); + this.uniformsMap.createUniform("viewMatrix"); + this.uniformsMap.createUniform("normalMapSampler"); + this.uniformsMap.createUniform("txtSampler"); + + this.uniformsMap.createUniform("material.ambient"); + this.uniformsMap.createUniform("material.diffuse"); + this.uniformsMap.createUniform("material.specular"); + this.uniformsMap.createUniform("material.reflectance"); + this.uniformsMap.createUniform("material.hasNormalMap"); + this.uniformsMap.createUniform("ambientLight.factor"); + this.uniformsMap.createUniform("ambientLight.color"); + + for (var ic = 0; ic < MAX_POINT_LIGHTS; ic++) { + var name = "pointLights[" + ic + "]."; + this.uniformsMap.createUniform(name + "position"); + this.uniformsMap.createUniform(name + "color"); + this.uniformsMap.createUniform(name + "intensity"); + this.uniformsMap.createUniform(name + "attenuation.constant"); + this.uniformsMap.createUniform(name + "attenuation.linear"); + this.uniformsMap.createUniform(name + "attenuation.exponent"); + } + for (var ic = 0; ic < MAX_SPOT_LIGHTS; ic++) { + var name = "spotLights[" + ic + "]."; + this.uniformsMap.createUniform(name + "pointLight.position"); + this.uniformsMap.createUniform(name + "pointLight.color"); + this.uniformsMap.createUniform(name + "pointLight.intensity"); + this.uniformsMap.createUniform(name + "pointLight.attenuation.constant"); + this.uniformsMap.createUniform(name + "pointLight.attenuation.linear"); + this.uniformsMap.createUniform(name + "pointLight.attenuation.exponent"); + this.uniformsMap.createUniform(name + "coneDirection"); + this.uniformsMap.createUniform(name + "cutOff"); } - GL30.glActiveTexture(GL30.GL_TEXTURE0); - texture.bind(); - for (var mesh : material.getMeshes()) { - GL30.glBindVertexArray(mesh.getVaoId()); - for (var entity : entities) { - this.uniformsMap.setUniform("modelMatrix", entity.getModelMatrix()); - GL30.glDrawElements(GL30.GL_TRIANGLES, mesh.getNumVertices(), GL30.GL_UNSIGNED_INT, 0); - } + + this.uniformsMap.createUniform("directionalLight.color"); + this.uniformsMap.createUniform("directionalLight.direction"); + this.uniformsMap.createUniform("directionalLight.intensity"); + } + + /** + * Releases all resources + *
      + *
    • Releases the shader program
      • + *
    • Releases the uniforms
      • + *
    + */ + public void cleanup() { + this.shaderProgram.cleanup(); + this.uniformsMap.cleanup(); + this.uniformsMap = null; + } + + /** + * Renders the Models in the scene Creates standard render options + * + * @param scene Scene to render + */ + public void render(GLScene scene) { + this.render(scene, new RenderOptions()); + } + + /** + * Renders the Models in the scene + *
      + *
    • Binds projection matrix
      • + *
    • Binds view matrix
      • + *
    • Binds texture sampler
      • + *
    + * Further, iterate over all models in the scene + *
      + *
    • Iterate over all materials in the model
      • + *
    • Sets texture or color function
      • + *
    • Iterate over all meshes in the material
      • + *
    • Binds the mesh
      • + *
    • Iterate over all entities to draw the mesh
      • + *
    • Binds the model matrix
      • + *
    • Draws the mesh
      • + *
    • Unbinds
      • + *
    + * + * @param scene Scene to render + * @param opt Render options + */ + public void render(GLScene scene, RenderOptions opt) { + this.shaderProgram.bind(); + + this.uniformsMap.setUniform("projectionMatrix", scene.getProjection().getProjMatrix()); + this.uniformsMap.setUniform("viewMatrix", scene.getCamera().getViewMatrix()); + this.uniformsMap.setUniform("normalMapSampler", 1); + this.uniformsMap.setUniform("txtSampler", 0); + + this.updateLights(scene); + + var models = scene.getModels().values(); + var textures = scene.getTextureCache(); + + for (var model : models) { + var entities = model.getEntities(); + + for (var material : model.getMaterials()) { + GLTexture texture; + + // Either draw texture or use color function + if (opt.showTextures) { + this.uniformsMap.setUniform("material.ambient", material.getAmbientColor()); + this.uniformsMap.setUniform("material.diffuse", material.getDiffuseColor()); + this.uniformsMap.setUniform("material.specular", material.getSpecularColor()); + this.uniformsMap.setUniform("material.reflectance", material.getReflectance()); + texture = textures.getTexture(material.getTexture().getTexturePath()); + + // Check if there is a normal map, and load it if there is. Further set flag for shader program. + if (material.hasNormalMapTexture()) { + this.uniformsMap.setUniform("normalMapSampler", 1); + var normalMapTexture = textures.getTexture(material.getTexture().getTexturePath()); + GL30.glActiveTexture(GL30.GL_TEXTURE1); + normalMapTexture.bind(); + } else { + this.uniformsMap.setUniform("normalMapSampler", 0); + } + } else { + this.uniformsMap.setUniform("material.diffuse", opt.colorfunction.apply(1f)); + texture = textures.getTexture("default"); + } + GL30.glActiveTexture(GL30.GL_TEXTURE0); + texture.bind(); + for (var mesh : material.getMeshes()) { + + GL30.glBindVertexArray(mesh.getVaoId()); + for (var entity : entities) { + this.uniformsMap.setUniform("modelMatrix", entity.getModelMatrix()); + GL30.glDrawElements(GL30.GL_TRIANGLES, mesh.getNumVertices(), GL30.GL_UNSIGNED_INT, 0); + } + } + } + } + GL30.glBindVertexArray(0); + this.shaderProgram.unbind(); + } + + private void updateLights(GLScene scene) { + var viewMatrix = scene.getCamera().getViewMatrix(); + var sceneLights = scene.getSceneLights(); + + var ambientLight = sceneLights.getAmbientLight(); + this.uniformsMap.setUniform("ambientLight.factor", ambientLight.getIntensity()); + this.uniformsMap.setUniform("ambientLight.color", ambientLight.getColor()); + + var directionalLight = sceneLights.getDirectionalLight(); + var auxDir = new Vector4f(directionalLight.getDirection(), 0); + auxDir.mul(viewMatrix); + var direction = new Vector3f(auxDir.x, auxDir.y, auxDir.z); + this.uniformsMap.setUniform("directionalLight.color", directionalLight.getColor()); + this.uniformsMap.setUniform("directionalLight.direction", direction); + this.uniformsMap.setUniform("directionalLight.intensity", directionalLight.getIntensity()); + + var pointLights = sceneLights.getPointLights(); + var numPointLight = pointLights.size(); + PointLight pointLight; + for (var ic = 0; ic < MAX_POINT_LIGHTS; ic++) { + if (ic < numPointLight) { + pointLight = pointLights.get(ic); + } else { + pointLight = null; + } + var name = "pointLights[" + ic + "]."; + this.updatePointLight(pointLight, name, viewMatrix); + } + + var spotLights = sceneLights.getSpotLights(); + var numSpotLight = spotLights.size(); + SpotLight spotLight; + for (var ic = 0; ic < MAX_SPOT_LIGHTS; ic++) { + if (ic < numSpotLight) { + spotLight = spotLights.get(ic); + } else { + spotLight = null; + } + var name = "spotLights[" + ic + "]."; + this.updateSpotLight(spotLight, name, viewMatrix); + } + } + + private void updatePointLight(PointLight pointLight, String name, Matrix4f viewMatrix) { + + var aux = new Vector4f(); + var lightPos = new Vector3f(); + var color = new Vector3f(); + var intensity = 0.0f; + var constant = 0.0f; + var linear = 0.0f; + var exponent = 0.0f; + if (pointLight != null) { + aux.set(pointLight.getPosition(), 1); + aux.mul(viewMatrix); + lightPos.set(aux.x, aux.y, aux.z); + color.set(pointLight.getColor()); + intensity = pointLight.getIntensity(); + var attenuation = pointLight.getAttenuation(); + constant = attenuation.getConstant(); + linear = attenuation.getLinear(); + exponent = attenuation.getExponent(); + } + + this.uniformsMap.setUniform(name + "position", lightPos); + this.uniformsMap.setUniform(name + "color", color); + this.uniformsMap.setUniform(name + "intensity", intensity); + this.uniformsMap.setUniform(name + "attenuation.constant", constant); + this.uniformsMap.setUniform(name + "attenuation.linear", linear); + this.uniformsMap.setUniform(name + "attenuation.exponent", exponent); + } + + private void updateSpotLight(SpotLight spotLight, String name, Matrix4f viewMatrix) { + PointLight pointLight = null; + var coneDirection = new Vector3f(); + float cutoff = 0.0f; + if (spotLight != null) { + coneDirection = spotLight.getConeDirection(); + cutoff = spotLight.getCutOff(); + pointLight = spotLight.getPointLight(); } - } + this.uniformsMap.setUniform(name + "coneDirection", coneDirection); + this.uniformsMap.setUniform(name + "cutOff", cutoff); + this.updatePointLight(pointLight, name + "pointLight.", viewMatrix); } - GL30.glBindVertexArray(0); - this.shaderProgram.unbind(); - } -} +} \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/ShaderProgram.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/ShaderProgram.java index 7e1acc729..7d4cab841 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/ShaderProgram.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/ShaderProgram.java @@ -134,7 +134,7 @@ public static String readShaderFile(String filePath) { try { str = new String(Files.readAllBytes(Paths.get(filePath))); } catch (IOException ex) { - throw new RuntimeException("Error reading file"); + throw new RuntimeException("Error reading file: " + filePath, ex); } return str; } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/UniformsMap.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/UniformsMap.java index 798710d4b..d5a422a77 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/UniformsMap.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/render/UniformsMap.java @@ -1,7 +1,10 @@ package org.vitrivr.cineast.core.render.lwjgl.render; import java.util.HashMap; +import java.util.Vector; import org.joml.Matrix4f; +import org.joml.Vector3d; +import org.joml.Vector3f; import org.joml.Vector4f; import org.lwjgl.opengl.GL30; import org.lwjgl.system.MemoryStack; @@ -98,6 +101,13 @@ public void setUniform(String uniformName, Matrix4f value) { } } + public void setUniform(String uniformName, float value) { + GL30.glUniform1f(this.getUniformLocation(uniformName), value); + } + public void setUniform(String uniformName, Vector3f value) { + GL30.glUniform3f(this.getUniformLocation(uniformName), value.x, value.y, value.z); + } + /** * Cleans up the uniforms */ diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/LWJGLOffscreenRenderer.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/LWJGLOffscreenRenderer.java index 46026f5b7..f2937bff1 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/LWJGLOffscreenRenderer.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/LWJGLOffscreenRenderer.java @@ -9,6 +9,9 @@ import org.vitrivr.cineast.core.render.lwjgl.engine.EngineLogic; import org.vitrivr.cineast.core.render.lwjgl.glmodel.GLScene; import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.LightColor; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.LightingOptions; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.SceneLights; import org.vitrivr.cineast.core.render.lwjgl.window.Window; import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; import org.vitrivr.cineast.core.data.m3d.texturemodel.Entity; @@ -21,276 +24,306 @@ import org.vitrivr.cineast.core.render.Renderer; /** - * This is the top most class of the LWJGL for Java 3D renderer. Its main function is to provide an interface between Engine and the outside world. It sets up the {@link Engine} and provides the interface to the outside world. {@link Renderer} It extends the abstract class {@link EngineLogic} which allows the instanced engine to call methods depending on the engine state. + * This is the top most class of the LWJGL for Java 3D renderer. Its main function is to provide an interface between + * Engine and the outside world. It sets up the {@link Engine} and provides the interface to the outside world. + * {@link Renderer} It extends the abstract class {@link EngineLogic} which allows the instanced engine to call methods + * depending on the engine state. */ @SuppressWarnings("deprecation") public class LWJGLOffscreenRenderer extends EngineLogic implements Renderer { - private static final Logger LOGGER = LogManager.getLogger(); - - /** - * The (offscreen) window options for the engine. - */ - private WindowOptions windowOptions; - /** - * The engine instance. - */ - private Engine engine; - - /** - * The model queue. From this queue the renderer takes the next model to render. - */ - private final LinkedTransferQueue modelQueue; - - /** - * The image queue. In this queue the renderer puts the rendered images. - */ - private final LinkedTransferQueue imageQueue; - - - /** - * Constructor for the LWJGLOffscreenRenderer. Initializes the model queue and the image queue. - */ - public LWJGLOffscreenRenderer() { - this.modelQueue = new LinkedTransferQueue<>(); - this.imageQueue = new LinkedTransferQueue<>(); - LOGGER.trace("LWJGLOffscreenRenderer created"); + private static final Logger LOGGER = LogManager.getLogger(); + + /** + * The (offscreen) window options for the engine. + */ + private WindowOptions windowOptions; + /** + * The engine instance. + */ + private Engine engine; + + /** + * The model queue. From this queue the renderer takes the next model to render. + */ + private final LinkedTransferQueue modelQueue; + + /** + * The image queue. In this queue the renderer puts the rendered images. + */ + private final LinkedTransferQueue imageQueue; + + + /** + * Constructor for the LWJGLOffscreenRenderer. Initializes the model queue and the image queue. + */ + public LWJGLOffscreenRenderer() { + this.modelQueue = new LinkedTransferQueue<>(); + this.imageQueue = new LinkedTransferQueue<>(); + LOGGER.trace("LWJGLOffscreenRenderer created"); + } + + /** + * Sets the window options for the engine. + * + * @param opts The window options. + */ + public void setWindowOptions(WindowOptions opts) { + this.windowOptions = opts; + } + + /** + * Sets the render options for the engine. + * + * @param opts The render options. + */ + public void setRenderOptions(RenderOptions opts) { + this.engine.setRenderOptions(opts); + } + + /** + * Sets the render options for the engine. + * + * @param opts The render options. + */ + public void setLighting(LightingOptions opts) { + this.engine.setLightingOptions(opts); + } + + + /** + * Starts the engine with given window options. Registers the LWJGLOffscreenRenderer as the engine logic. + */ + public void startEngine() { + var name = "LWJGLOffscreenRenderer"; + this.engine = new Engine(name, this.windowOptions, this); + } + + /** + * Starts the rendering process. + */ + @Override + public void render() { + this.engine.runOnce(); + LOGGER.trace("LWJGLOffscreenRenderer rendered"); + } + + + /** + * Is called once at the initialization of the engine. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS + * METHOD FROM THIS CLASS + */ + @Override + protected void init(Window window, GLScene scene, Render render) { + var sceneLights = new SceneLights(); + //sceneLights.addLight(LightColor.WHITE, new Vector3f(0.0f, 1.0f, 1.0f), 0.5f); + + scene.setSceneLights(sceneLights); + scene.getCamera().setPosition(0.0f, 0.0f, 1.0f); + } + + /** + * Is called from the engine before the render method. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS + * METHOD FROM THIS CLASS + */ + @Override + protected void beforeRender(Window window, GLScene scene, Render render) { + this.loadNextModelFromQueueToScene(window, scene); + } + + /** + * Is called from the engine after the render method. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS + * METHOD FROM THIS CLASS + */ + @Override + protected void afterRender(Window window, GLScene scene, Render render) { + var lfc = new LightfieldCamera(this.windowOptions); + this.imageQueue.add(lfc.takeLightfieldImage()); + } + + /** + * Is called from the engine as first step during refresh and cleanup DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT + * CALL THIS METHOD FROM THIS CLASS + */ + @Override + protected void cleanup() { + LOGGER.trace("LWJGLOffscreenRenderer cleaned"); + } + + + /** + * This method is called every frame. This is only used in continuous rendering. The purpose is to do some input + * handling. Could be use for optimize view angles on a fast manner. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT + * CALL THIS METHOD FROM THIS CLASS + */ + @Override + protected void input(Window window, GLScene scene, long diffTimeMillis) { + scene.getModels().forEach((k, v) -> v.getEntities().forEach(Entity::updateModelMatrix)); + } + + /** + * After Engine run This method is called every frame. This is only used in continuous rendering. The purpose is to + * process some life output. Could be use for optimize view angles on a fast manner. DO NOT CALL ENGINE METHODS IN + * THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS + */ + @Override + protected void update(Window window, GLScene scene, long diffTimeMillis) { + } + + /** + * This method is called to load the next model into the provided scene. + * + * @param scene The scene to put the model in. + */ + @SuppressWarnings("unused") + private void loadNextModelFromQueueToScene(Window window, GLScene scene) { + if (!this.modelQueue.isEmpty()) { + var model = (Model) this.modelQueue.poll(); + if (model.getEntities().size() == 0) { + var entity = new Entity("cube", model.getId()); + model.addEntityNorm(entity); + } + //cleans all current models from the scene + scene.cleanup(); + //adds the new model to the scene + scene.addModel(model); } - - /** - * Sets the window options for the engine. - * - * @param opts The window options. - */ - public void setWindowOptions(WindowOptions opts) { - this.windowOptions = opts; - } - - /** - * Sets the render options for the engine. - * - * @param opts The render options. - */ - public void setRenderOptions(RenderOptions opts) { - this.engine.setRenderOptions(opts); - } - - /** - * Starts the engine with given window options. Registers the LWJGLOffscreenRenderer as the engine logic. - */ - public void startEngine() { - var name = "LWJGLOffscreenRenderer"; - this.engine = new Engine(name, this.windowOptions, this); - } - - /** - * Starts the rendering process. - */ - @Override - public void render() { - this.engine.runOnce(); - LOGGER.trace("LWJGLOffscreenRenderer rendered"); - } - - - /** - * Is called once at the initialization of the engine. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS - */ - @Override - protected void init(Window window, GLScene scene, Render render) { - scene.getCamera().setPosition(0, 0, 1); - } - - /** - * Is called from the engine before the render method. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS - */ - @Override - protected void beforeRender(Window window, GLScene scene, Render render) { - this.loadNextModelFromQueueToScene(window, scene); - } - - /** - * Is called from the engine after the render method. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS - */ - @Override - protected void afterRender(Window window, GLScene scene, Render render) { - var lfc = new LightfieldCamera(this.windowOptions); - this.imageQueue.add(lfc.takeLightfieldImage()); - } - - /** - * Is called from the engine as first step during refresh and cleanup DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS - */ - @Override - protected void cleanup() { - LOGGER.trace("LWJGLOffscreenRenderer cleaned"); - } - - - /** - * This method is called every frame. This is only used in continuous rendering. The purpose is to do some input handling. Could be use for optimize view angles on a fast manner. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS - */ - @Override - protected void input(Window window, GLScene scene, long diffTimeMillis) { - scene.getModels().forEach((k, v) -> v.getEntities().forEach(Entity::updateModelMatrix)); - } - - /** - * After Engine run This method is called every frame. This is only used in continuous rendering. The purpose is to process some life output. Could be use for optimize view angles on a fast manner. DO NOT CALL ENGINE METHODS IN THIS METHOD DO NOT CALL THIS METHOD FROM THIS CLASS - */ - @Override - protected void update(Window window, GLScene scene, long diffTimeMillis) { - } - - /** - * This method is called to load the next model into the provided scene. - * - * @param scene The scene to put the model in. - */ - @SuppressWarnings("unused") - private void loadNextModelFromQueueToScene(Window window, GLScene scene) { - if (!this.modelQueue.isEmpty()) { - var model = (Model) this.modelQueue.poll(); - if (model.getEntities().size() == 0) { - var entity = new Entity("cube", model.getId()); - model.addEntityNorm(entity); - } - //cleans all current models from the scene - scene.cleanup(); - //adds the new model to the scene - scene.addModel(model); - } - scene.getModels().forEach((k, v) -> v.getEntities().forEach(Entity::updateModelMatrix)); - } - - /** - * Moves the camera in the scene with given deltas in cartesian coordinates. Look at the origin. - */ - public void moveCameraOrbit(float dx, float dy, float dz) { - this.engine.getCamera().moveOrbit(dx, dy, dz); - } - - /** - * Sets the camera in the scene to cartesian coordinates. Look at the origin. - */ - public void setCameraOrbit(float x, float y, float z) { - this.engine.getCamera().setOrbit(x, y, z); - } - - /** - * Moves the camera in the scene with given deltas in cartesian coordinates. Keep the orientation. - */ - @SuppressWarnings("unused") - public void setCameraPosition(float x, float y, float z) { - this.engine.getCamera().setPosition(x, y, z); - } - - /** - * Set position of the camera and look at the origin. Camera will stay aligned to the y plane. - */ - public void lookFromAtO(float x, float y, float z) { - var lookFrom = new Vector3f(x, y, z); - var lookAt = new Vector3f(0, 0, 0); - - this.engine.getCamera().setPositionAndOrientation(lookFrom, lookAt); - - } - - /** - * Set position and orientation of the camera. - * - * @deprecated Old renderer implementation. Not needed. Use quaternion instead. - */ - @Override - @Deprecated - public void positionCamera(double ex, double ey, double ez, - double cx, double cy, double cz, - double upx, double upy, double upz) { - this.engine.getCamera().setPositionAndOrientation( - new Vector3f((float) ex, (float) ey, (float) ez), - new Vector3f((float) cx, (float) cy, (float) cz), - new Vector3f((float) upx, (float) upy, (float) upz)); - } - - - /** - * Returns the aspect ratio of the window. - */ - @SuppressWarnings("unused") - public float getAspect() { - return (float) this.windowOptions.width / (float) this.windowOptions.height; - } - - /** - * Interface to outside to add a model to the scene. - */ - @Override - public void assemble(IModel model) { - this.modelQueue.add(model); - } - - /** - * Interface to outside to get a rendered image. - */ - @Override - public BufferedImage obtain() { - return this.imageQueue.poll(); - } - - /** - * This method disposes the engine. Window is destroyed and all resources are freed. - */ - @Override - public void clear(Color color) { - this.clear(); - } - - /** - * This method disposes the engine. Window is destroyed and all resources are freed. - */ - @Override - public void clear() { - this.engine.clear(); - this.engine = null; - } - - /** - * Retains control of the Renderer. While a Thread retains a renderer, no other thread should be allowed to use it! - * - * @deprecated Old renderer implementation. Indicates that the renderer should be retained. - */ - @Override - @Deprecated - public boolean retain() { - return true; - } - - /** - * Releases control of the Renderer, making it usable by other Threads again. - * - * @deprecated Old renderer implementation. Indicates that the renderer should be retained. - */ - @Override - @Deprecated - public void release() { - this.engine.clear(); - } - - /** - * Returns the width of the window. - * - * @return The width of the window. (in pixels) - */ - public int getWidth() { - return this.windowOptions.width; - } - - /** - * Returns the height of the window. - * - * @return The height of the window. (in pixels) - */ - public int getHeight() { - return this.windowOptions.height; + scene.getModels().forEach((k, v) -> v.getEntities().forEach(Entity::updateModelMatrix)); + } + + /** + * Moves the camera in the scene with given deltas in cartesian coordinates. Look at the origin. + */ + public void moveCameraOrbit(float dx, float dy, float dz) { + this.engine.getCamera().moveOrbit(dx, dy, dz); + } + + /** + * Sets the camera in the scene to cartesian coordinates. Look at the origin. + */ + public void setCameraOrbit(float x, float y, float z) { + this.engine.getCamera().setOrbit(x, y, z); + } + + /** + * Moves the camera in the scene with given deltas in cartesian coordinates. Keep the orientation. + */ + @SuppressWarnings("unused") + public void setCameraPosition(float x, float y, float z) { + this.engine.getCamera().setPosition(x, y, z); + } + + /** + * Set position of the camera and look at the origin. Camera will stay aligned to the y plane. + */ + public void lookFromAtO(float x, float y, float z) { + var lookFrom = new Vector3f(x, y, z); + var lookAt = new Vector3f(0, 0, 0); + + this.engine.getCamera().setPositionAndOrientation(lookFrom, lookAt); + + } + + /** + * Set position and orientation of the camera. + * + * @deprecated Old renderer implementation. Not needed. Use quaternion instead. + */ + @Override + @Deprecated + public void positionCamera(double ex, double ey, double ez, + double cx, double cy, double cz, + double upx, double upy, double upz) { + this.engine.getCamera().setPositionAndOrientation( + new Vector3f((float) ex, (float) ey, (float) ez), + new Vector3f((float) cx, (float) cy, (float) cz), + new Vector3f((float) upx, (float) upy, (float) upz)); + } + + + /** + * Returns the aspect ratio of the window. + */ + @SuppressWarnings("unused") + public float getAspect() { + return (float) this.windowOptions.width / (float) this.windowOptions.height; + } + + /** + * Interface to outside to add a model to the scene. + */ + @Override + public void assemble(IModel model) { + this.modelQueue.add(model); + } + + /** + * Interface to outside to get a rendered image. + */ + @Override + public BufferedImage obtain() { + try { + return this.imageQueue.poll(60, java.util.concurrent.TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOGGER.error("Timeout on rendering", e); + throw new RuntimeException("Timeout on rendering", e); } + } + + /** + * This method disposes the engine. Window is destroyed and all resources are freed. + */ + @Override + public void clear(Color color) { + this.clear(); + } + + /** + * This method disposes the engine. Window is destroyed and all resources are freed. + */ + @Override + public void clear() { + this.engine.clear(); + this.engine = null; + } + + /** + * Retains control of the Renderer. While a Thread retains a renderer, no other thread should be allowed to use it! + * + * @deprecated Old renderer implementation. Indicates that the renderer should be retained. + */ + @Override + @Deprecated + public boolean retain() { + return true; + } + + /** + * Releases control of the Renderer, making it usable by other Threads again. + * + * @deprecated Old renderer implementation. Indicates that the renderer should be retained. + */ + @Override + @Deprecated + public void release() { + this.engine.clear(); + } + + /** + * Returns the width of the window. + * + * @return The width of the window. (in pixels) + */ + public int getWidth() { + return this.windowOptions.width; + } + + /** + * Returns the height of the window. + * + * @return The height of the window. (in pixels) + */ + public int getHeight() { + return this.windowOptions.height; + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderWorker.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderWorker.java index c931d3a64..177c40dce 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderWorker.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/renderer/RenderWorker.java @@ -4,12 +4,15 @@ import java.util.Hashtable; import java.util.LinkedList; import java.util.concurrent.BlockingDeque; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.joml.Vector3f; import org.lwjgl.system.Configuration; import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; +import org.vitrivr.cineast.core.render.lwjgl.glmodel.IGLModel; import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.LightingOptions; import org.vitrivr.cineast.core.render.lwjgl.util.datatype.Variant; import org.vitrivr.cineast.core.render.lwjgl.util.fsm.abstractworker.JobControlCommand; import org.vitrivr.cineast.core.render.lwjgl.util.fsm.abstractworker.StateEnter; @@ -29,8 +32,8 @@ *

    * It constructs a Graph which describes the states and transitions which a render worker can do. *

    - * If a job throws an exception the worker will send a JobControlCommand. ERROR to the caller. - * Furthermore, the worker will unload the model. + * If a job throws an exception the worker will send a JobControlCommand. ERROR to the caller. Furthermore, the worker + * will unload the model. *

    * Each rendered image will be sent to the caller. *

    @@ -43,57 +46,58 @@ @StateProvider public class RenderWorker extends Worker { - private static final Logger LOGGER = LogManager.getLogger(); + private static final Logger LOGGER = LogManager.getLogger(); - /** - * The Lightweight Java Game Library (LWJGL) Offscreen Renderer instance. - */ - private LWJGLOffscreenRenderer renderer; + /** + * The Lightweight Java Game Library (LWJGL) Offscreen Renderer instance. + */ + private LWJGLOffscreenRenderer renderer; - /** - * Since only one OpenGl renderer can be instanced - * use Static queue - * The queue can be accessed by the caller wit {@link RenderWorker#getRenderJobQueue()} - * On this queue the caller will provide new render jobs. - */ - private static BlockingDeque renderJobQueue; + /** + * Since only one OpenGl renderer can be instanced + * use Static queue + * The queue can be accessed by the caller wit {@link RenderWorker#getRenderJobQueue()} + * On this queue the caller will provide new render jobs. + */ + private static BlockingDeque renderJobQueue; - /** - * Instantiates a new RenderWorker. - *

    - * The queue is stores in a static variable. - */ - public RenderWorker(BlockingDeque jobs) { - super(jobs); - RenderWorker.renderJobQueue = jobs; - LOGGER.trace("Initialized RenderWorker"); - } + /** + * Instantiates a new RenderWorker. + *

    + * The queue is stores in a static variable. + */ + public RenderWorker(BlockingDeque jobs) { + super(jobs); + RenderWorker.renderJobQueue = jobs; + LOGGER.trace("Initialized RenderWorker"); + } - /** - * static getter for the renderJobQueue - * A caller can use get the queue submit new jobs to the render worker. - * @return the render job queue - */ - public static BlockingDeque getRenderJobQueue() { - return renderJobQueue; - } + /** + * static getter for the renderJobQueue + * A caller can use get the queue submit new jobs to the render worker. + * + * @return the render job queue + */ + public static BlockingDeque getRenderJobQueue() { + return renderJobQueue; + } - /** - * The render worker main thread. - */ - public void run() { - Configuration.STACK_SIZE.set((int) java.lang.Math.pow(2, 17)); - this.renderer = new LWJGLOffscreenRenderer(); - var defaultOptions = new WindowOptions(); - renderer.setWindowOptions(defaultOptions); - renderer.startEngine(); - super.run(); - LOGGER.trace("Running RenderWorker"); - } + /** + * The render worker main thread. + */ + public void run() { + Configuration.STACK_SIZE.set((int) java.lang.Math.pow(2, 19)); + this.renderer = new LWJGLOffscreenRenderer(); + var defaultOptions = new WindowOptions(); + renderer.setWindowOptions(defaultOptions); + renderer.startEngine(); + super.run(); + LOGGER.trace("Running RenderWorker"); + } - // @formatter:off + // @formatter:off /** * Creates the graph for the RenderWorker. @@ -129,119 +133,127 @@ protected Graph createGraph() { } // @formatter:on - /** - * Handler for render exceptions. - * Unloads the model and sends a JobControlCommand. ERROR to the caller. - * @param ex The exception that was thrown. - * @return The handler message. - */ - @Override - protected String onJobException(Exception ex){ - this.unload(); - this.currentJob.putResultQueue(new RenderJob(JobControlCommand.JOB_FAILURE)); - return "Job failed"; - } + /** + * Handler for render exceptions. + * Unloads the model and sends a JobControlCommand. ERROR to the caller. + * + * @param ex The exception that was thrown. + * @return The handler message. + */ + @Override + protected String onJobException(Exception ex) { + this.unload(); + this.currentJob.putResultQueue(new RenderJob(JobControlCommand.JOB_FAILURE)); + return "Job failed"; + } - /** - * Initializes the renderer. - * Sets the window options and starts the engine. - */ - @StateEnter(state = RenderStates.INIT_WINDOW, data = RenderData.WINDOWS_OPTIONS) - public void setWindowOptions(WindowOptions opt) { - LOGGER.trace("INIT_WINDOW RenderWorker"); - this.renderer = new LWJGLOffscreenRenderer(); + /** + * Initializes the renderer. Sets the window options and starts the engine. + */ + @StateEnter(state = RenderStates.INIT_WINDOW, data = RenderData.WINDOWS_OPTIONS) + public void setWindowOptions(WindowOptions opt) { + LOGGER.trace("INIT_WINDOW RenderWorker"); + this.renderer = new LWJGLOffscreenRenderer(); - renderer.setWindowOptions(opt); - renderer.startEngine(); - } + renderer.setWindowOptions(opt); + renderer.startEngine(); + } - /** - * Sets specific render options. - */ - @StateEnter(state = RenderStates.INIT_RENDERER, data = RenderData.RENDER_OPTIONS) - public void setRendererOptions(RenderOptions opt) { - LOGGER.trace("INIT_RENDERER RenderWorker"); - this.renderer.setRenderOptions(opt); - } + /** + * Sets specific render options. + */ + @StateEnter(state = RenderStates.INIT_RENDERER, data = RenderData.RENDER_OPTIONS) + public void setRendererOptions(RenderOptions opt) { + LOGGER.trace("INIT_RENDERER RenderWorker"); + this.renderer.setRenderOptions(opt); + if (opt.lightingOptions != null) { + this.renderer.setLighting(opt.lightingOptions); + } + } - /** - * State to wait for new jobs. - */ - @StateEnter(state = RenderStates.IDLE) - public void idle() { - LOGGER.trace("IDLE RenderWorker"); - } + /** + * State to wait for new jobs. + */ + @StateEnter(state = RenderStates.IDLE) + public void idle() { + LOGGER.trace("IDLE RenderWorker"); + } - /** - * Register a model to the renderer. - * @param model The model to register and to be rendered. - */ - @StateEnter(state = RenderStates.LOAD_MODEL, data = RenderData.MODEL) - public void registerModel(IModel model) { - LOGGER.trace("LOAD_MODEL RenderWorker"); - this.renderer.assemble(model); - } + /** + * Register a model to the renderer. + * + * @param model The model to register and to be rendered. + */ + @StateEnter(state = RenderStates.LOAD_MODEL, data = RenderData.MODEL) + public void registerModel(IModel model) { + LOGGER.trace("LOAD_MODEL RenderWorker"); + this.renderer.assemble(model); + } - /** - * Renders the model. - * Sends the rendered image to the caller. - */ - @StateEnter(state = RenderStates.RENDER) - public void renderModel() { - LOGGER.trace("RENDER RenderWorker"); - this.renderer.render(); - var pic = this.renderer.obtain(); - var data = new Variant().set(RenderData.IMAGE, pic); - var responseJob = new RenderJob(data); - this.currentJob.putResultQueue(responseJob); - } + /** + * Renders the model. Sends the rendered image to the caller. + */ + @StateEnter(state = RenderStates.RENDER) + public void renderModel() { + try { + LOGGER.trace("RENDER RenderWorker"); + this.renderer.render(); + var pic = this.renderer.obtain(); + var data = new Variant().set(RenderData.IMAGE, pic); + var responseJob = new RenderJob(data); + this.currentJob.putResultQueue(responseJob); + } catch (Exception e) { + this.onJobException(e); + } + } - /** - * Rotates the camera. - * @param rotation The rotation vector (x,y,z) - */ - @StateEnter(state = RenderStates.ROTATE, data = RenderData.VECTOR) - public void rotate(Vector3f rotation) { - LOGGER.trace("ROTATE RenderWorker"); - this.renderer.moveCameraOrbit(rotation.x, rotation.y, rotation.z); - } + /** + * Rotates the camera. + * + * @param rotation The rotation vector (x,y,z) + */ + @StateEnter(state = RenderStates.ROTATE, data = RenderData.VECTOR) + public void rotate(Vector3f rotation) { + LOGGER.trace("ROTATE RenderWorker"); + this.renderer.moveCameraOrbit(rotation.x, rotation.y, rotation.z); + } - /** - * Looks at the origin from a specific position. - * The rotation is not affected. - * Removes the processed position vector from the list. - * @param vectors The list of position vectors - */ - @StateEnter(state = RenderStates.LOOKAT, data = RenderData.VECTORS) - public void lookAt(LinkedList vectors) { - LOGGER.trace("Look at RenderWorker"); - var vec = vectors.poll(); - assert vec != null; - this.renderer.setCameraOrbit(vec.x, vec.y, vec.z); - } + /** + * Looks at the origin from a specific position. The rotation is not affected. Removes the processed position vector + * from the list. + * + * @param vectors The list of position vectors + */ + @StateEnter(state = RenderStates.LOOKAT, data = RenderData.VECTORS) + public void lookAt(LinkedList vectors) { + LOGGER.trace("Look at RenderWorker"); + var vec = vectors.poll(); + assert vec != null; + this.renderer.setCameraOrbit(vec.x, vec.y, vec.z); + } - /** - * Looks from a specific position at the origin. - * Removes the processed position vector from the list. - * @param vectors The list of position vectors - */ - @StateEnter(state = RenderStates.LOOK_FROM_AT_O, data = RenderData.VECTORS) - public void lookFromAtO(LinkedList vectors) { - LOGGER.trace("LOOK_FROM_AT_O RenderWorker"); - var vec = vectors.poll(); - assert vec != null; - this.renderer.lookFromAtO(vec.x, vec.y, vec.z); - } + /** + * Looks from a specific position at the origin. Removes the processed position vector from the list. + * + * @param vectors The list of position vectors + */ + @StateEnter(state = RenderStates.LOOK_FROM_AT_O, data = RenderData.VECTORS) + public void lookFromAtO(LinkedList vectors) { + LOGGER.trace("LOOK_FROM_AT_O RenderWorker"); + var vec = vectors.poll(); + assert vec != null; + this.renderer.lookFromAtO(vec.x, vec.y, vec.z); + } - /** - * Unloads the model and sends a JobControlCommand.JOB_DONE to the caller. - */ - @StateEnter(state = RenderStates.UNLOAD_MODEL) - public void unload() { - LOGGER.trace("UNLOAD_MODEL RenderWorker"); - this.renderer.clear(); - this.renderer = null; - var responseJob = new RenderJob(JobControlCommand.JOB_DONE); - this.currentJob.putResultQueue(responseJob); - } + /** + * Unloads the model and sends a JobControlCommand.JOB_DONE to the caller. + */ + @StateEnter(state = RenderStates.UNLOAD_MODEL) + public void unload() { + LOGGER.trace("UNLOAD_MODEL RenderWorker"); + this.renderer.clear(); + this.renderer = null; + var responseJob = new RenderJob(JobControlCommand.JOB_DONE); + this.currentJob.putResultQueue(responseJob); + } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Camera.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Camera.java index bb3245deb..30fe6e44f 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Camera.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Camera.java @@ -10,7 +10,7 @@ *

    * This class is responsible for the camera position and orientation. It is used to calculate the view matrix for the renderer. It provides methods to move and rotate the camera. */ -public class Camera { +public class Camera implements ILocateable { /** * Helper Vector for X-Axis translation. diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/ILocateable.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/ILocateable.java new file mode 100644 index 000000000..1742a3b3e --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/ILocateable.java @@ -0,0 +1,7 @@ +package org.vitrivr.cineast.core.render.lwjgl.scene; + +import org.joml.Vector3f; + +public interface ILocateable { + Vector3f getPosition(); +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/LightfieldCamera.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/LightfieldCamera.java index 1a6dd194c..c7a639c46 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/LightfieldCamera.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/LightfieldCamera.java @@ -2,6 +2,9 @@ import java.awt.image.BufferedImage; import java.nio.FloatBuffer; +import java.nio.IntBuffer; + +import jdk.jfr.Unsigned; import org.lwjgl.BufferUtils; import org.lwjgl.opengl.GL30; import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; @@ -37,6 +40,7 @@ public LightfieldCamera(WindowOptions opts) { this.opts = opts; this.lightfieldImage = new BufferedImage(opts.width, opts.height, BufferedImage.TYPE_INT_RGB); this.imageData = BufferUtils.createFloatBuffer(opts.width * opts.height * 3); + GL30.glReadPixels(0, 0, opts.width, opts.height, GL30.GL_RGB, GL30.GL_FLOAT, this.imageData); this.imageData.rewind(); } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Scene.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Scene.java index f372df753..0d7a18914 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Scene.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/Scene.java @@ -4,10 +4,11 @@ import java.util.Map; import org.vitrivr.cineast.core.data.m3d.texturemodel.Entity; import org.vitrivr.cineast.core.data.m3d.texturemodel.IModel; +import org.vitrivr.cineast.core.render.lwjgl.scene.lights.SceneLights; /** - * The Scene class holds generic 3D scene elements (models, etc.). - * A scene consists of a model, a camera and a projection. + * The Scene class holds generic 3D scene elements (models, etc.). A scene consists of a model, a camera and a + * projection. */ public class Scene { @@ -24,10 +25,15 @@ public class Scene { */ private final Camera camera; + /** + * SceneLights Class holds all lights of the scene. + */ + private SceneLights sceneLights; + /** * Creates a new Scene. * - * @param width Width of the scene. + * @param width Width of the scene. * @param height Height of the scene. */ public Scene(int width, int height) { @@ -37,8 +43,7 @@ public Scene(int width, int height) { } /** - * Add an entity to a contained corresponding model. - * Can be used to resize the scene before gl context is created. + * Add an entity to a contained corresponding model. Can be used to resize the scene before gl context is created. */ @SuppressWarnings("unused") public void addEntity(Entity entity) { @@ -79,9 +84,23 @@ public Camera getCamera() { } /** - * Resizes the scene. - * Can be used to resize the scene before gl context is created. - * @param width Width of the scene. + * Get the scene lights. + */ + public SceneLights getSceneLights() { + return this.sceneLights; + } + + /** + * Set the scene lights. + */ + public void setSceneLights(SceneLights sceneLights) { + this.sceneLights = sceneLights; + } + + /** + * Resizes the scene. Can be used to resize the scene before gl context is created. + * + * @param width Width of the scene. * @param height Height of the scene. */ @SuppressWarnings("unused") diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/AbstractLight.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/AbstractLight.java new file mode 100644 index 000000000..0f60e673e --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/AbstractLight.java @@ -0,0 +1,36 @@ +package org.vitrivr.cineast.core.render.lwjgl.scene.lights; + +import org.joml.Vector3f; + +public abstract class AbstractLight { + + private final Vector3f color; + private float intensity; + + public AbstractLight(Vector3f color, float intensity) { + this.color = color; + this.intensity = intensity; + } + public AbstractLight(LightColor color, float intensity) { + this(color.getUnitRGB(), intensity); + } + + public Vector3f getColor() { + return this.color; + } + + public float getIntensity() { + return this.intensity; + } + public void setColor(Vector3f color) { + this.color.set(color); + } + public void setColor(float r, float g, float b) { + this.color.set(r, g, b); + } + + public void setIntensity(float intensity) { + this.intensity = intensity; + } + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/AmbientLight.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/AmbientLight.java new file mode 100644 index 000000000..0acd93b95 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/AmbientLight.java @@ -0,0 +1,18 @@ +package org.vitrivr.cineast.core.render.lwjgl.scene.lights; + +import org.joml.Vector3f; + +public class AmbientLight extends AbstractLight { + + public AmbientLight(Vector3f color, float intensity) { + super(color, intensity); + } + + public AmbientLight(LightColor lightColor, float intensity) { + super(lightColor, intensity); + } + + public AmbientLight() { + super(LightColor.WHITE.getUnitRGB(), 0.0f); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/Attenuation.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/Attenuation.java new file mode 100644 index 000000000..30617c883 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/Attenuation.java @@ -0,0 +1,39 @@ +package org.vitrivr.cineast.core.render.lwjgl.scene.lights; + +public class Attenuation { + + private float constant; + private float exponent; + private float linear; + + public Attenuation(float constant, float linear, float exponent) { + this.constant = constant; + this.exponent = exponent; + this.linear = linear; + } + + public float getConstant() { + return this.constant; + } + + public float getLinear() { + return this.linear; + } + + public float getExponent() { + return this.exponent; + } + + public void setConstant(float constant) { + this.constant = constant; + } + + public void setLinear(float linear) { + this.linear = linear; + } + + public void setExponent(float exponent) { + this.exponent = exponent; + } + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/DirectionalLight.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/DirectionalLight.java new file mode 100644 index 000000000..c141423ba --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/DirectionalLight.java @@ -0,0 +1,33 @@ +package org.vitrivr.cineast.core.render.lwjgl.scene.lights; + +import java.util.Vector; +import java.util.function.Supplier; +import org.joml.Vector3f; + +public class DirectionalLight extends AbstractLight { + + private Supplier direction; + + public DirectionalLight() { + super(LightColor.WHITE, 0.0f); + this.direction = () -> new Vector3f(0.0f, 1f, 0.0f); + } + + public DirectionalLight(LightColor color, Vector3f direction, float intensity) { + super(color, intensity); + this.direction = () -> direction; + } + + public DirectionalLight(LightColor color, Supplier direction, float intensity) { + super(color, intensity); + this.direction = direction; + } + + public Vector3f getDirection() { + return this.direction.get(); + } + + public void setDirection(Vector3f direction) { + this.direction = () -> direction; + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/LightColor.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/LightColor.java new file mode 100644 index 000000000..2789a9359 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/LightColor.java @@ -0,0 +1,173 @@ +package org.vitrivr.cineast.core.render.lwjgl.scene.lights; + +import org.joml.Vector3f; + +public enum LightColor { + RED(255.0f, 0.0f, 0.0f), + GREEN(0.0f, 255.0F, 0.0f), + BLUE(0.0f, 0.0f, 255.0F), + WHITE(255.0F, 255.0F, 255.0F), + BLACK(0.0f, 0.0f, 0.0f), + YELLOW(255.0F, 255.0F, 255.0F), + MAGENTA(255.0F, 255.0F, 255.0F), + CYAN(0.0f, 1.0f, 1.0f), + T1000(255.0F, 56.0F, 0.0F), + T1100(255.0F, 71.0F, 0.0F), + T1200(255.0F, 83.0F, 0.0F), + T1300(255.0F, 93.0F, 0.0F), + T1400(255.0F, 101.0F, 0.0F), + T1500(255.0F, 109.0F, 0.0F), + T1600(255.0F, 115.0F, 0.0F), + T1700(255.0F, 121.0F, 0.0F), + T1800(255.0F, 126.0F, 0.0F), + T1900(255.0F, 131.0F, 0.0F), + T2000(255.0F, 138.0F, 18.0F), + T2100(255.0F, 142.0F, 33.0F), + T2200(255.0F, 147.0F, 44.0F), + T2300(255.0F, 152.0F, 54.0F), + T2400(255.0F, 157.0F, 63.0F), + T2500(255.0F, 161.0F, 72.0F), + T2600(255.0F, 165.0F, 79.0F), + T2700(255.0F, 169.0F, 87.0F), + T2800(255.0F, 173.0F, 94.0F), + T2900(255.0F, 177.0F, 101.0F), + T3000(255.0F, 180.0F, 107.0F), + T3100(255.0F, 184.0F, 114.0F), + T3200(255.0F, 187.0F, 120.0F), + T3300(255.0F, 190.0F, 126.0F), + T3400(255.0F, 193.0F, 132.0F), + T3500(255.0F, 196.0F, 137.0F), + T3600(255.0F, 199.0F, 143.0F), + T3700(255.0F, 201.0F, 148.0F), + T3800(255.0F, 204.0F, 153.0F), + T3900(255.0F, 206.0F, 159.0F), + T4000(255.0F, 209.0F, 163.0F), + T4100(255.0F, 211.0F, 168.0F), + T4200(255.0F, 213.0F, 173.0F), + T4300(255.0F, 215.0F, 177.0F), + T4400(255.0F, 217.0F, 182.0F), + T4500(255.0F, 219.0F, 186.0F), + T4600(255.0F, 221.0F, 190.0F), + T4700(255.0F, 223.0F, 194.0F), + /** + * 4800K Direct Sunlight + */ + T4800(255.0F, 225.0F, 198.0F), + T4900(255.0F, 227.0F, 202.0F), + T5000(255.0F, 228.0F, 206.0F), + T5100(255.0F, 230.0F, 210.0F), + T5200(255.0F, 232.0F, 213.0F), + T5300(255.0F, 233.0F, 217.0F), + T5400(255.0F, 235.0F, 220.0F), + T5500(255.0F, 236.0F, 224.0F), + T5600(255.0F, 238.0F, 227.0F), + T5700(255.0F, 239.0F, 230.0F), + T5800(255.0F, 240.0F, 233.0F), + T5900(255.0F, 242.0F, 236.0F), + T6000(255.0F, 243.0F, 239.0F), + T6100(255.0F, 244.0F, 242.0F), + T6200(255.0F, 245.0F, 245.0F), + T6300(255.0F, 246.0F, 247.0F), + T6400(255.0F, 248.0F, 251.0F), + T6500(255.0F, 249.0F, 253.0F), + T6600(254.0F, 249.0F, 255.0F), + T6700(252.0F, 247.0F, 255.0F), + T6800(249.0F, 246.0F, 255.0F), + T6900(247.0F, 245.0F, 255.0F), + T7000(245.0F, 243.0F, 255.0F), + T7100(243.0F, 242.0F, 255.0F), + T7200(240.0F, 241.0F, 255.0F), + T7300(239.0F, 240.0F, 255.0F), + T7400(237.0F, 239.0F, 255.0F), + T7500(235.0F, 238.0F, 255.0F), + T7600(233.0F, 237.0F, 255.0F), + T7700(231.0F, 236.0F, 255.0F), + T7800(230.0F, 235.0F, 255.0F), + T7900(228.0F, 234.0F, 255.0F), + T8000(227.0F, 233.0F, 255.0F), + T8100(225.0F, 232.0F, 255.0F), + T8200(224.0F, 231.0F, 255.0F), + T8300(222.0F, 230.0F, 255.0F), + T8400(221.0F, 230.0F, 255.0F), + T8500(220.0F, 229.0F, 255.0F), + T8600(218.0F, 229.0F, 255.0F), + T8700(217.0F, 227.0F, 255.0F), + T8800(216.0F, 227.0F, 255.0F), + T8900(215.0F, 226.0F, 255.0F), + T9000(214.0F, 225.0F, 255.0F), + T9100(212.0F, 225.0F, 255.0F), + T9200(211.0F, 224.0F, 255.0F), + T9300(210.0F, 223.0F, 255.0F), + T9400(209.0F, 223.0F, 255.0F), + T9500(208.0F, 222.0F, 255.0F), + T9600(207.0F, 221.0F, 255.0F), + T9700(207.0F, 221.0F, 255.0F), + T9800(206.0F, 220.0F, 255.0F), + T9900(205.0F, 220.0F, 255.0F), + T10000(207.0F, 218.0F, 255.0F), + T10100(207.0F, 218.0F, 255.0F), + T10200(206.0F, 217.0F, 255.0F), + T10300(205.0F, 217.0F, 255.0F), + T10400(204.0F, 216.0F, 255.0F), + T10500(204.0F, 216.0F, 255.0F), + T10600(203.0F, 215.0F, 255.0F), + T10700(202.0F, 215.0F, 255.0F), + T10800(202.0F, 214.0F, 255.0F), + T10900(201.0F, 214.0F, 255.0F), + T11000(200.0F, 213.0F, 255.0F), + T11100(200.0F, 213.0F, 255.0F), + T11200(199.0F, 212.0F, 255.0F), + T11300(198.0F, 212.0F, 255.0F), + T11400(198.0F, 212.0F, 255.0F), + T11500(197.0F, 211.0F, 255.0F), + T11600(197.0F, 211.0F, 255.0F), + T11700(197.0F, 210.0F, 255.0F), + T11800(196.0F, 210.0F, 255.0F), + T11900(195.0F, 210.0F, 255.0F), + T12000(195.0F, 209.0F, 255.0F); + + + private final float r; + private final float g; + private final float b; + + LightColor(float r, float g, float b) { + this.r = r; + this.g = g; + this.b = b; + } + + public float getR() { + return this.r; + } + + public float getG() { + return this.g; + } + + public float getB() { + return this.b; + } + + public float getUnitR() { + return this.r / 255.0F; + } + + public float getUnitG() { + return this.g / 255.0F; + } + + public float getUnitB() { + return this.b / 255.0F; + } + + + public Vector3f getRGB() { + return new Vector3f(this.getR(), this.getG(), this.getB()); + } + + public Vector3f getUnitRGB() { + return new Vector3f(this.getUnitR(), this.getUnitG(), this.getUnitB()); + } + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/LightingOptions.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/LightingOptions.java new file mode 100644 index 000000000..4e3a6b0cb --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/LightingOptions.java @@ -0,0 +1,47 @@ +package org.vitrivr.cineast.core.render.lwjgl.scene.lights; + +import org.joml.Vector3f; +import org.vitrivr.cineast.core.render.lwjgl.glmodel.IGLModel; +import org.vitrivr.cineast.core.render.lwjgl.scene.Camera; +import org.vitrivr.cineast.core.render.lwjgl.scene.ILocateable; +import org.vitrivr.cineast.core.render.lwjgl.scene.Scene; + +public enum LightingOptions { + STATIC, + DIRECTIONAL_LIGHTING_ON_NO_TEXTURE, + DIRECTIONAL_LIGHTING, + + POINT_LIGHTING_ON_NO_TEXTURE; + + + public boolean hasNonDefaultTexture = true; + + public SceneLights getSceneLigths(Scene scene) { + switch (this) { + case STATIC -> { + return new SceneLights(); + } + case DIRECTIONAL_LIGHTING -> { + return new SceneLights(scene.getCamera()::getPosition); + } + case DIRECTIONAL_LIGHTING_ON_NO_TEXTURE -> { + if (this.hasNonDefaultTexture) { + return new SceneLights(); + } else { + return new SceneLights(scene.getCamera()::getPosition); + } + } + case POINT_LIGHTING_ON_NO_TEXTURE -> { + if (this.hasNonDefaultTexture) { + return new SceneLights(); + } else { + return new SceneLights(scene.getCamera()::getPosition, 3); + } + } + default -> { + return new SceneLights(); + } + } + } + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/PointLight.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/PointLight.java new file mode 100644 index 000000000..53bcd76fb --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/PointLight.java @@ -0,0 +1,46 @@ +package org.vitrivr.cineast.core.render.lwjgl.scene.lights; + +import java.util.Vector; +import java.util.function.Supplier; +import org.joml.Vector3f; + +public class PointLight extends AbstractLight { + + private Attenuation attenuation; + private Supplier position; + + public PointLight(Vector3f color, Vector3f position , float intensity) { + this(color, () -> position, intensity); + } + + public PointLight(Vector3f color, Supplier position , float intensity) { + super(color, intensity); + this.attenuation = new Attenuation(0, 0, 1); + this.position = position; + } + + public Attenuation getAttenuation() { + return this.attenuation; + } + + public Vector3f getPosition() { + return this.position.get(); + } + + public void setAttenuation(Attenuation attenuation) { + this.attenuation = attenuation; + } + + public void setPosition(Supplier position) { + this.position = position; + } + + public void setPosition(Vector3f position) { + this.position = () -> position; + } + + public void setPosition(float x, float y, float z) { + this.position = () -> new Vector3f(x, y, z); + } + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/SceneLights.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/SceneLights.java new file mode 100644 index 000000000..fcd59ade1 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/SceneLights.java @@ -0,0 +1,88 @@ +package org.vitrivr.cineast.core.render.lwjgl.scene.lights; + +import java.security.Provider; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +import org.joml.Vector3f; + +public class SceneLights { + + private final AmbientLight ambientLight; + private final DirectionalLight directionalLight; + private final List pointLights; + private final List spotLights; + + public SceneLights() { + // Default settings as no lights are set + this.ambientLight = new AmbientLight(LightColor.WHITE, 1.0F); + this.directionalLight = new DirectionalLight(LightColor.WHITE, new Vector3f(0.0F, 0.0F, 0.0F), 0.0f); + this.pointLights = new ArrayList<>(); + this.spotLights = new ArrayList<>(); + } + + public SceneLights(Supplier position) { + // Default settings as no lights are set + this.ambientLight = new AmbientLight(LightColor.WHITE, 0.0F); + this.directionalLight = new DirectionalLight(LightColor.WHITE, position, 0.5f); + this.pointLights = new ArrayList<>(); + this.spotLights = new ArrayList<>(); + } + + public SceneLights(Supplier position, int point) { + // Default settings as no lights are set + this.ambientLight = new AmbientLight(LightColor.WHITE, 0.0F); + + this.directionalLight = new DirectionalLight(LightColor.WHITE, position, 0.1f); + + this.pointLights = new ArrayList<>(); + this.spotLights = new ArrayList<>(); + for (var ic = 0; ic < point; ++ic) { + this.pointLights.add(new PointLight(LightColor.RED.getRGB(), position.get().rotateY((float) ic * (float) Math.PI / (float) point ), 0.1f / (point))); + } + } + + public AmbientLight getAmbientLight() { + return this.ambientLight; + } + + public DirectionalLight getDirectionalLight() { + return this.directionalLight; + } + + public List getPointLights() { + return Collections.unmodifiableList(this.pointLights); + } + + public List getSpotLights() { + return Collections.unmodifiableList(this.spotLights); + } + + public void setupAmbientLight(LightColor lightColor) { + this.ambientLight.setColor(lightColor.getUnitRGB()); + } + + public void setupAmbientLight(float intensity) { + this.ambientLight.setIntensity(intensity); + } + + public void setupAmbientLight(LightColor lightColor, float intensity) { + this.setupAmbientLight(lightColor); + this.setupAmbientLight(intensity); + } + + + public SceneLights addLight(LightColor lightColor, Vector3f position, float intensity) { + this.pointLights.add(new PointLight(lightColor.getUnitRGB(), position, intensity)); + return this; + } + + public SceneLights addLight(LightColor lightColor, Vector3f position, float intensity, Vector3f coneDirection, + float cutOff) { + this.spotLights.add(new SpotLight(lightColor.getUnitRGB(), position, intensity, coneDirection, cutOff)); + return this; + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/SpotLight.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/SpotLight.java new file mode 100644 index 000000000..e3de780b8 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/scene/lights/SpotLight.java @@ -0,0 +1,49 @@ +package org.vitrivr.cineast.core.render.lwjgl.scene.lights; + +import org.joml.Vector3f; + +public class SpotLight extends PointLight{ + private final Vector3f coneDirection; + private float cutOff; + private float cutOffAngle; + + public SpotLight(Vector3f color, Vector3f position, float intensity, Vector3f coneDirection, float cutOffAngle) { + super(color, position, intensity); + this.coneDirection = coneDirection; + this.cutOffAngle = cutOffAngle; + + } + public SpotLight(PointLight pointLight, Vector3f coneDirection, float cutOffAngle) { + this(pointLight.getColor(), pointLight.getPosition(), pointLight.getIntensity(), coneDirection, cutOffAngle); + } + + public Vector3f getConeDirection() { + return this.coneDirection; + } + public float getCutOff() { + return this.cutOff; + } + public float getCutOffAngle() { + return this.cutOffAngle; + } + + public PointLight getPointLight() { + return (PointLight) this; + } + public void setConeDirection(Vector3f coneDirection) { + this.coneDirection.set(coneDirection); + } + public void setConeDirection(float x, float y, float z) { + this.coneDirection.set(x, y, z); + } + + public void setCutOffAngle(float cutOffAngle) { + this.cutOffAngle = cutOffAngle; + } + + public void setPointLight(PointLight pointLight) { + this.setColor(pointLight.getColor()); + this.setPosition(pointLight.getPosition()); + this.setIntensity(pointLight.getIntensity()); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/JobControlCommand.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/JobControlCommand.java index 2ffdeba6a..275705f89 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/JobControlCommand.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/JobControlCommand.java @@ -17,7 +17,7 @@ public enum JobControlCommand { /** - * Is used to shut down the worker + * Is used to shutdown the worker */ SHUTDOWN_WORKER, } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/Worker.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/Worker.java index 63ee7e992..8197c9ada 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/Worker.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/render/lwjgl/util/fsm/abstractworker/Worker.java @@ -197,7 +197,13 @@ private void performJob(Job job) { this.onJobException(ex); LOGGER.error("Error in concrete Worker. Abort: ", ex); performed = true; - } finally { + } catch (Exception ex) { + // This exception is thrown if an exception is thrown during the job + this.onJobException(ex); + LOGGER.error("Error in job. Abort: ", ex); + performed = true; + } + finally { LOGGER.trace("Job Sequence ended"); } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/CineastConstants.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/CineastConstants.java index e4499afe6..9a0d4d84e 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/CineastConstants.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/CineastConstants.java @@ -28,6 +28,7 @@ public class CineastConstants { public static final String DB_DISTANCE_VALUE_QUALIFIER = "distance"; + public static final String ENTITY_NAME_KEY = "entity"; public static final String DOMAIN_COL_NAME = "domain"; public static final String KEY_COL_NAME = "key"; public static final String VAL_COL_NAME = "value"; diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java index 053e385e0..0c85fb780 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/ReflectionHelper.java @@ -145,7 +145,37 @@ public static Extractor newExtractor(String name) { } return instantiate(c); } catch (ClassNotFoundException | InstantiationException e) { - LOGGER.fatal("Failed to create Exporter. Could not find class with name {} ({}).", name, LogHelper.getStackTrace(e)); + LOGGER.fatal("Failed to create Extractor. Could not find class with name {} ({}).", name, LogHelper.getStackTrace(e)); + return null; + } + } + + + /** + * Tries to instantiate a new, named Extractor object. If the methods succeeds to do so, that instance is returned by the method. + *

    + * If the name contains dots (.), that name is treated as FQN. Otherwise, the FEATURE_MODULE_PACKAGE is assumed and the name is treated as simple name. + * + * @param name Name of the Exporter. + * @param configuration Configuration + * @return Instance of Exporter or null, if instantiation fails. + */ + @SuppressWarnings("unchecked") + public static Extractor newExtractor(String name, Map configuration) { + Class c = null; + try { + if (name.contains(".")) { + c = (Class) Class.forName(name); + } else { + c = getClassFromName(name, Extractor.class, FEATURE_MODULE_PACKAGE); + } + if (configuration == null || configuration.isEmpty()) { + return instantiate(c); + } else { + return instantiate(c, configuration); + } + } catch (ClassNotFoundException | InstantiationException e) { + LOGGER.fatal("Failed to create Extractor. Could not find class with name {} ({}).", name, LogHelper.getStackTrace(e)); return null; } } @@ -174,7 +204,7 @@ public static Extractor newExporter(String name, Map configurati return instantiate(c, configuration); } } catch (ClassNotFoundException | InstantiationException e) { - LOGGER.fatal("Failed to create Exporter. Could not find or access class with name {} ({}).", name, LogHelper.getStackTrace(e)); + LOGGER.fatal("Failed to create Extractor. Could not find or access class with name {} ({}).", name, LogHelper.getStackTrace(e)); return null; } } diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java index 345bd3794..ce5dd2229 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/ImageParser.java @@ -38,7 +38,7 @@ public static BufferedImage dataURLtoBufferedImage(String dataUrl) { return bimg; } - public static String BufferedImageToDataURL(BufferedImage img, String format) { + public static String bufferedImageToDataURL(BufferedImage img, String format) { ByteArrayOutputStream bouts = new ByteArrayOutputStream(); try { ImageIO.write(img, format, bouts); diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MessageTemplate.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MessageTemplate.java new file mode 100644 index 000000000..8c8d7157b --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MessageTemplate.java @@ -0,0 +1,105 @@ +package org.vitrivr.cineast.core.util.web; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import java.util.*; +import java.nio.file.*; +import java.io.IOException; +import java.util.regex.*; + +public class MessageTemplate { + + private final String template; + private static final Pattern KEY_PATTERN = Pattern.compile("\\$\\{(.*?)\\}"); + + public MessageTemplate(String path) throws IOException { + this.template = Files.readString(Paths.get(path)).strip(); + } + + public List getKeys() { + List keys = new ArrayList<>(); + Matcher matcher = KEY_PATTERN.matcher(template); + while (matcher.find()) { + keys.add(matcher.group(1)); + } + return keys; + } + + public String formatString(Map values) { + if (!getKeys().stream().allMatch(values::containsKey)) { + throw new IllegalArgumentException("Not all keys are present in the provided map."); + } + String result = template; + for (String key : values.keySet()) { + result = result.replace("${" + key + "}", values.get(key)); + } + return result; + } + + private String getNextPlaceholder(int start) { + int nextMatchStart = template.indexOf("${", start); + if (nextMatchStart != -1) { + return template.substring(nextMatchStart, template.indexOf("}", nextMatchStart) + 1); + } + return null; + } + + public Map parseString(String input) { + input = input.strip(); + Map values = new HashMap<>(); + List keys = getKeys(); + + Matcher matcher = KEY_PATTERN.matcher(template); + int lastEnd = 0; + int i = 0; + + while (matcher.find()) { + String beforeKey = template.substring(lastEnd, matcher.start()); + lastEnd = matcher.end(); + + if (!input.startsWith(beforeKey)) { + throw new IllegalArgumentException("Input does not match template"); + } + + input = input.substring(beforeKey.length()); + String nextPlaceholder = getNextPlaceholder(lastEnd); + + String value; + if (nextPlaceholder != null && input.contains(nextPlaceholder)) { + value = input.substring(0, input.indexOf(nextPlaceholder)); + input = input.substring(value.length()); + } else { + if (lastEnd < template.length()) { + String trailingTemplate = template.substring(lastEnd); + if (input.endsWith(trailingTemplate)) { + value = input.substring(0, input.length() - trailingTemplate.length()); + } else { + throw new IllegalArgumentException("Input does not match template"); + } + } else { + value = input; + } + } + + if (i < keys.size()) { + values.put(keys.get(i), value); + i++; + } + } + + if (keys.size() != values.size()) { + throw new IllegalArgumentException("Keys and values count do not match"); + } + + return values; + } + +} \ No newline at end of file diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/WebClient.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/WebClient.java new file mode 100644 index 000000000..563317dcb --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/WebClient.java @@ -0,0 +1,51 @@ +package org.vitrivr.cineast.core.util.web; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpClient.Version; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; + +public class WebClient { + + private final String endpoint; + + public WebClient(String endpoint) { + this.endpoint = endpoint; + } + + private final HttpClient httpClient = HttpClient.newBuilder() + .version(Version.HTTP_1_1) + .build(); + + public String postJsonString(String body) throws IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(body)) + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + throw new IllegalStateException("received response code " + response.statusCode()); + } + return response.body(); + } + + public String postRawBinary(byte[] body) throws IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofByteArray(body)) + .uri(URI.create(endpoint)) + .header("Content-Type", "application/octet-stream") + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + throw new IllegalStateException("received response code " + response.statusCode()); + } + return response.body(); + } +} diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java index 1808cb8dd..2c1571fee 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/db/DBIntegrationTest.java @@ -235,7 +235,7 @@ void entriesExistById() { @DisplayName("get multiple feature vectors") void getFeatureVectors() { this.selector.open(testVectorTableName); - final List vectors = this.selector.getFeatureVectorsGeneric(ID_COL_NAME, new StringTypeProvider(toId(0)), FEATURE_VECTOR_COL_NAME, queryConfig); + final List vectors = this.selector.getFeatures(ID_COL_NAME, new StringTypeProvider(toId(0)), FEATURE_VECTOR_COL_NAME, queryConfig); Assertions.assertTrue((Arrays.equals(PrimitiveTypeProvider.getSafeFloatArray(vectors.get(0)), new float[]{0, 0, 0}) | Arrays.equals(PrimitiveTypeProvider.getSafeFloatArray(vectors.get(0)), new float[]{0, 1, 0}))); Assertions.assertTrue((Arrays.equals(PrimitiveTypeProvider.getSafeFloatArray(vectors.get(1)), new float[]{0, 0, 0}) | Arrays.equals(PrimitiveTypeProvider.getSafeFloatArray(vectors.get(1)), new float[]{0, 1, 0}))); } diff --git a/cineast-core/src/test/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetrieverTest.java b/cineast-core/src/test/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetrieverTest.java index aba92bf63..08cd5b7cf 100644 --- a/cineast-core/src/test/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetrieverTest.java +++ b/cineast-core/src/test/java/org/vitrivr/cineast/core/features/abstracts/AbstractTextRetrieverTest.java @@ -30,7 +30,7 @@ public void testNonQuotedStringSplit() { } public void testMatch(String input, String... output) { - org.junit.jupiter.api.Assertions.assertArrayEquals(output, retriever.generateQuery(new TextQueryTermContainer(input), new QueryConfig(null))); + org.junit.jupiter.api.Assertions.assertArrayEquals(output, retriever.generateQuery(input)); } public SegmentContainer generateSegmentContainerFromText(String text) { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java index 9ba9ce747..15ee64111 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/CliUtils.java @@ -69,7 +69,7 @@ public static void printInfoForSegment(String segmentId, DBSelector selector, St } retrievalRuntimeConfig.getRetrieversByCategory(cat).forEach((ObjectDoubleProcedure) (retriever, weight) -> { System.out.println("= Retrieving for feature: " + retriever.getClass().getSimpleName() + " ="); - retriever.getTableNames().forEach(tableName -> { + retriever.getEntityNames().forEach(tableName -> { selector.open(tableName); List> rows = selector.getRows("id", new StringTypeProvider(segmentId)); if (retriever.getClass() == RangeBooleanRetriever.class) { @@ -91,7 +91,7 @@ public static void retrieveAndLog(List retrievers, ContinuousRetrieva DBSelector selector = Config.sharedConfig().getDatabase().getSelectorSupplier().get(); retrievers.forEach(retriever -> { AtomicBoolean entityExists = new AtomicBoolean(true); - retriever.getTableNames().forEach(table -> { + retriever.getEntityNames().forEach(table -> { if (!selector.existsEntity(table)) { System.out.println("Entity " + table + " does not exist"); entityExists.set(false); diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ThreeDeeTextureTestCommand.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ThreeDeeTextureTestCommand.java index 03f0694fe..25e3a867a 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ThreeDeeTextureTestCommand.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/cli/ThreeDeeTextureTestCommand.java @@ -2,50 +2,60 @@ import com.github.rvesse.airline.annotations.Command; import org.joml.Vector3f; +import org.vitrivr.cineast.core.data.m3d.texturemodel.Model; import org.vitrivr.cineast.core.data.m3d.texturemodel.ModelLoader; +import org.vitrivr.cineast.core.data.m3d.texturemodel.util.TextureLoadException; import org.vitrivr.cineast.core.render.lwjgl.render.RenderOptions; import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderJob; import org.vitrivr.cineast.core.render.lwjgl.renderer.RenderWorker; import org.vitrivr.cineast.core.render.lwjgl.window.WindowOptions; + import javax.imageio.ImageIO; import java.io.File; import java.io.IOException; import java.util.LinkedList; +import java.util.concurrent.TimeoutException; @Command(name = "3dtexttest", description = "Starts a 3D rendering test to check availability of an LWJGL OpenGL renderer.") public class ThreeDeeTextureTestCommand extends AbstractCineastCommand { - @Override - public void execute() { - System.out.println("Performing 3D test on texture model..."); - - var model = ModelLoader.loadModel("unit-cube", - "./resources/renderer/lwjgl/models/unit-cube/Cube_Text.gltf"); + @Override + public void execute() { + System.out.println("Performing 3D test on texture model..."); + Model model = null; + try { + model = ModelLoader.loadModel("unit-cube", + "./resources/renderer/lwjgl/models/unit-cube/Cube_Text.gltf"); + } catch (TextureLoadException e) { + throw new RuntimeException(e); + } catch (TimeoutException e) { + throw new RuntimeException(e); + } - // Options for window - var windowOptions = new WindowOptions() {{ - this.hideWindow = true; - this.width = 600; - this.height = 600; - }}; - // Options for renderer - var renderOptions = new RenderOptions() {{ - this.showTextures = true; - }}; - // Get camera viewpoint for chosen strategy - var cameraPositions = new LinkedList() {{ - add(new Vector3f(-1, 1, 1).normalize()); - }}; - // Render an image for each camera position - var images = RenderJob.performStandardRenderJob(RenderWorker.getRenderJobQueue(), - model, cameraPositions, windowOptions, renderOptions); + // Options for window + var windowOptions = new WindowOptions() {{ + this.hideWindow = true; + this.width = 600; + this.height = 600; + }}; + // Options for renderer + var renderOptions = new RenderOptions() {{ + this.showTextures = true; + }}; + // Get camera viewpoint for chosen strategy + var cameraPositions = new LinkedList() {{ + add(new Vector3f(-1, 1, 1).normalize()); + }}; + // Render an image for each camera position + var images = RenderJob.performStandardRenderJob(RenderWorker.getRenderJobQueue(), + model, cameraPositions, windowOptions, renderOptions); - try { - ImageIO.write(images.get(0), "PNG", new File("cineast-3dtexttest.png")); - System.out.println("3D test complete. Check for cineast-3dtexttest.png"); - } catch (IOException | NullPointerException e) { - System.err.println("Could not save rendered image due to an IO error."); + try { + ImageIO.write(images.get(0), "PNG", new File("cineast-3dtexttest.png")); + System.out.println("3D test complete. Check for cineast-3dtexttest.png"); + } catch (IOException | NullPointerException e) { + System.err.println("Could not save rendered image due to an IO error."); + } } - } } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java index e535ff32e..5a3b73ac7 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/APIConfig.java @@ -13,286 +13,302 @@ @JsonIgnoreProperties(ignoreUnknown = true) public final class APIConfig { - private boolean enableWebsocket = true; - private boolean enableWebsocketSecure = true; - private boolean enableExtractionServer = true; - private boolean enableRest = false; - private boolean enableRestSecure = false; - - private boolean enableGRPC = true; - private String keystore; - private String keystorePassword; - - private boolean enableRestLiveDoc = false; // Defaults to same result as enableRest - private String apiAddress = "http://localhost:4567/"; - - private int httpPort = 4567; - private int httpsPort = 4568; - - - private int grpcPort = 4570; - private int maxMessageSize = 5120 * 1000; /* Maximum size of a single WebSocket message (binary or text). */ - - private boolean allowExtraction = true; - - private boolean enableCLI = false; - private int threadPoolSize = 8; - - private boolean serveContent = false; - - /** - * A hack-flag to prevent the object serving using {@link MediaObjectDescriptor#getPath()} to find the actual media file. If this is true, the {@link MediaObjectDescriptor#getPath()}'s extension is appendet to {@link MediaObjectDescriptor#getObjectId()} and this is resolved to be directly under {@link #objectLocation}. - */ - private boolean objectsFilesAreIDed = false; - /** - * A hack to use mp4 for object servings - */ - private String videoExtension = "mp4"; - private boolean serveUI = false; - private String sessionExtractionConfigLocation = "extraction_config.json"; - private String thumbnailLocation = ""; - private String objectLocation = ""; - private String uiLocation = ""; - - @JsonCreator - public APIConfig() { - } - - @JsonProperty - public String getVideoExtension() { - return videoExtension; - } - - @JsonProperty - public void setVideoExtension(String videoExtension) { - this.videoExtension = videoExtension; - } - - @JsonProperty - public boolean getEnableWebsocket() { - return this.enableWebsocket; - } - - public void setEnableWebsocket(boolean enableWebsocket) { - this.enableWebsocket = enableWebsocket; - } - - @JsonProperty - public boolean getEnableWebsocketSecure() { - return this.enableWebsocketSecure; - } - - public void setEnableWebsocketSecure(boolean enableWebsocket) { - this.enableWebsocketSecure = enableWebsocket; - } - - @JsonProperty - public boolean getEnableRest() { - return this.enableRest; - } - - public void setEnableRest(boolean enableRest) { - this.enableRest = enableRest; - } - - @JsonProperty - public boolean getEnableRestSecure() { - return this.enableRestSecure; - } - - public void setEnableRestSecure(boolean enableRest) { - this.enableRestSecure = enableRest; - } - - @JsonProperty - public boolean getEnableLiveDoc() { - return this.enableRestLiveDoc; - } - - public void setEnableRestLiveDoc(boolean enableRestLiveDoc) { - this.enableRestLiveDoc = enableRestLiveDoc; - } - - @JsonProperty - public String getKeystore() { - return keystore; - } - - public void setKeystore(String keystore) { - this.keystore = keystore; - } - - @JsonProperty - public String getKeystorePassword() { - return keystorePassword; - } - - public void setKeystorePassword(String keystorePassword) { - this.keystorePassword = keystorePassword; - } - - @JsonProperty - public String getApiAddress() { - return apiAddress; - } - - public void setApiAddress(String apiAddress) { - this.apiAddress = apiAddress; - } - - @JsonProperty - public int getHttpPort() { - return httpPort; - } - - public void setHttpPort(int httpPort) { - if (httpPort < 1) { - throw new IllegalArgumentException("httpPort must be > 0"); - } - this.httpPort = httpPort; - } - - @JsonProperty - public int getHttpsPort() { - return httpsPort; - } - - public void setHttpsPort(int httpsPort) { - if (httpsPort < 1) { - throw new IllegalArgumentException("httpPort must be > 0"); - } - this.httpsPort = httpsPort; - } - - @JsonProperty - public int getMaxMessageSize() { - return this.maxMessageSize; - } - - public void setMaxMessageSize(int maxTextMessageSize) { - this.maxMessageSize = maxTextMessageSize; - } - - @JsonProperty - public boolean getAllowExtraction() { - return this.allowExtraction; - } - - public void setAllowExtraction(boolean allowExtraction) { - this.allowExtraction = allowExtraction; - } - - @JsonProperty - public boolean getEnableCli() { - return this.enableCLI; - } - - public void setEnableCLI(boolean enableCLI) { - this.enableCLI = enableCLI; - } - - @JsonProperty - public int getThreadPoolSize() { - return threadPoolSize; - } - - public void setThreadPoolSize(int threadPoolSize) { - this.threadPoolSize = threadPoolSize; - } - - @JsonProperty - public String getThumbnailLocation() { - return thumbnailLocation; - } - - public void setThumbnailLocation(String thumbnailLocation) { - this.thumbnailLocation = thumbnailLocation; - } - - @JsonProperty - public String getObjectLocation() { - return objectLocation; - } - - public void setObjectLocation(String objectLocation) { - this.objectLocation = objectLocation; - } - - @JsonProperty - public String getUiLocation() { - return uiLocation; - } - - public void setUiLocation(String uiLocation) { - this.uiLocation = uiLocation; - } - - @JsonProperty - public boolean getServeContent() { - return this.serveContent; - } - - public void setServeContent(boolean serveContent) { - this.serveContent = serveContent; - } - - @JsonProperty - public boolean getServeUI() { - return this.serveUI; - } - - public void setServeUI(boolean serveUI) { - this.serveUI = serveUI; - } - - @JsonProperty - public String getSessionExtractionConfigLocation() { - return this.sessionExtractionConfigLocation; - } - - public void setSessionExtractionConfigLocation(String sessionExtractionConfigLocation) { - this.sessionExtractionConfigLocation = sessionExtractionConfigLocation; - } - - @JsonProperty - public boolean getEnableExtractionServer() { - return enableExtractionServer; - } - - public void setEnableExtractionServer(boolean enableExtractionServer) { - this.enableExtractionServer = enableExtractionServer; - } - - @JsonProperty - public boolean getEnableGRPC() { - return enableGRPC; - } - - @JsonProperty - public void setEnableGRPC(boolean enableGRPC) { - this.enableGRPC = enableGRPC; - } - - @JsonProperty - public int getGrpcPort() { - return grpcPort; - } - - @JsonProperty - public void setGrpcPort(int grpcPort) { - this.grpcPort = grpcPort; - } - - /** - * A hack-flag to prevent the object serving using {@link MediaObjectDescriptor#getPath()} to find the actual media file. If this is true, the {@link MediaObjectDescriptor#getPath()}'s extension is appendet to {@link MediaObjectDescriptor#getObjectId()} and this is resolved to be directly under {@link #objectLocation}. - */ - @JsonProperty - public boolean isObjectsFilesAreIDed() { - return objectsFilesAreIDed; - } - - @JsonProperty - public void setObjectsFilesAreIDed(boolean b) { - objectsFilesAreIDed = b; - } + private boolean enableWebsocket = true; + private boolean enableWebsocketSecure = true; + private boolean enableExtractionServer = true; + private boolean enableRest = false; + private boolean enableRestSecure = false; + + private boolean enableCLI = false; + private boolean enableExternalClip = false; + + private boolean enableGRPC = true; + private String keystore; + private String keystorePassword; + + private boolean enableRestLiveDoc = false; // Defaults to same result as enableRest + private String apiAddress = "http://localhost:4567/"; + + private int httpPort = 4567; + private int httpsPort = 4568; + + + private int grpcPort = 4570; + private int maxMessageSize = 5120 * 1000; /* Maximum size of a single WebSocket message (binary or text). */ + + private boolean allowExtraction = true; + + private int threadPoolSize = 8; + + private boolean serveContent = false; + + /** + * A hack-flag to prevent the object serving using {@link MediaObjectDescriptor#getPath()} to find the actual media file. If this is true, the {@link MediaObjectDescriptor#getPath()}'s extension is appendet to {@link MediaObjectDescriptor#getObjectId()} and this is resolved to be directly under {@link #objectLocation}. + */ + private boolean objectsFilesAreIDed = false; + /** + * A hack to use mp4 for object servings + */ + private String videoExtension = "mp4"; + private boolean serveUI = false; + private String sessionExtractionConfigLocation = "extraction_config.json"; + private String thumbnailLocation = ""; + private String objectLocation = ""; + private String uiLocation = ""; + + + + + @JsonCreator + public APIConfig() { + } + + @JsonProperty + public String getVideoExtension() { + return videoExtension; + } + + @JsonProperty + public void setVideoExtension(String videoExtension) { + this.videoExtension = videoExtension; + } + + @JsonProperty + public boolean getEnableWebsocket() { + return this.enableWebsocket; + } + + public void setEnableWebsocket(boolean enableWebsocket) { + this.enableWebsocket = enableWebsocket; + } + + @JsonProperty + public boolean getEnableWebsocketSecure() { + return this.enableWebsocketSecure; + } + + public void setEnableWebsocketSecure(boolean enableWebsocket) { + this.enableWebsocketSecure = enableWebsocket; + } + + @JsonProperty + public boolean getEnableRest() { + return this.enableRest; + } + + public void setEnableRest(boolean enableRest) { + this.enableRest = enableRest; + } + + @JsonProperty + public boolean getEnableRestSecure() { + return this.enableRestSecure; + } + + public void setEnableRestSecure(boolean enableRest) { + this.enableRestSecure = enableRest; + } + + @JsonProperty + public boolean getEnableLiveDoc() { + return this.enableRestLiveDoc; + } + + public void setEnableRestLiveDoc(boolean enableRestLiveDoc) { + this.enableRestLiveDoc = enableRestLiveDoc; + } + + @JsonProperty + public String getKeystore() { + return keystore; + } + + public void setKeystore(String keystore) { + this.keystore = keystore; + } + + @JsonProperty + public String getKeystorePassword() { + return keystorePassword; + } + + public void setKeystorePassword(String keystorePassword) { + this.keystorePassword = keystorePassword; + } + + @JsonProperty + public String getApiAddress() { + return apiAddress; + } + + public void setApiAddress(String apiAddress) { + this.apiAddress = apiAddress; + } + + @JsonProperty + public int getHttpPort() { + return httpPort; + } + + public void setHttpPort(int httpPort) { + if (httpPort < 1) { + throw new IllegalArgumentException("httpPort must be > 0"); + } + this.httpPort = httpPort; + } + + @JsonProperty + public int getHttpsPort() { + return httpsPort; + } + + public void setHttpsPort(int httpsPort) { + if (httpsPort < 1) { + throw new IllegalArgumentException("httpPort must be > 0"); + } + this.httpsPort = httpsPort; + } + + @JsonProperty + public int getMaxMessageSize() { + return this.maxMessageSize; + } + + public void setMaxMessageSize(int maxTextMessageSize) { + this.maxMessageSize = maxTextMessageSize; + } + + @JsonProperty + public boolean getAllowExtraction() { + return this.allowExtraction; + } + + public void setAllowExtraction(boolean allowExtraction) { + this.allowExtraction = allowExtraction; + } + + @JsonProperty + public boolean getEnableCli() { + return this.enableCLI; + } + + public void setEnableCLI(boolean enableCLI) { + this.enableCLI = enableCLI; + } + + @JsonProperty + public int getThreadPoolSize() { + return threadPoolSize; + } + + public void setThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = threadPoolSize; + } + + @JsonProperty + public String getThumbnailLocation() { + return thumbnailLocation; + } + + public void setThumbnailLocation(String thumbnailLocation) { + this.thumbnailLocation = thumbnailLocation; + } + + @JsonProperty + public String getObjectLocation() { + return objectLocation; + } + + public void setObjectLocation(String objectLocation) { + this.objectLocation = objectLocation; + } + + @JsonProperty + public String getUiLocation() { + return uiLocation; + } + + public void setUiLocation(String uiLocation) { + this.uiLocation = uiLocation; + } + + @JsonProperty + public boolean getServeContent() { + return this.serveContent; + } + + public void setServeContent(boolean serveContent) { + this.serveContent = serveContent; + } + + @JsonProperty + public boolean getServeUI() { + return this.serveUI; + } + + public void setServeUI(boolean serveUI) { + this.serveUI = serveUI; + } + + @JsonProperty + public String getSessionExtractionConfigLocation() { + return this.sessionExtractionConfigLocation; + } + + public void setSessionExtractionConfigLocation(String sessionExtractionConfigLocation) { + this.sessionExtractionConfigLocation = sessionExtractionConfigLocation; + } + + @JsonProperty + public boolean getEnableExtractionServer() { + return enableExtractionServer; + } + + public void setEnableExtractionServer(boolean enableExtractionServer) { + this.enableExtractionServer = enableExtractionServer; + } + + @JsonProperty + public boolean getEnableGRPC() { + return enableGRPC; + } + + @JsonProperty + public void setEnableGRPC(boolean enableGRPC) { + this.enableGRPC = enableGRPC; + } + + @JsonProperty + public int getGrpcPort() { + return grpcPort; + } + + @JsonProperty + public void setGrpcPort(int grpcPort) { + this.grpcPort = grpcPort; + } + + /** + * A hack-flag to prevent the object serving using {@link MediaObjectDescriptor#getPath()} to find the actual media file. If this is true, the {@link MediaObjectDescriptor#getPath()}'s extension is appendet to {@link MediaObjectDescriptor#getObjectId()} and this is resolved to be directly under {@link #objectLocation}. + */ + @JsonProperty + public boolean isObjectsFilesAreIDed() { + return objectsFilesAreIDed; + } + + @JsonProperty + public void setObjectsFilesAreIDed(boolean b) { + objectsFilesAreIDed = b; + } + + @JsonProperty + public boolean getEnableExternalClip() { + return this.enableExternalClip; + } + + @JsonProperty + public boolean setEnableExternalClip(boolean enableExternalClip) { + return this.enableExternalClip = enableExternalClip; + } + } diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java index b1ebafd3e..81c27c63e 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractionPipelineConfig.java @@ -69,6 +69,11 @@ public void setTaskQueueSize(int taskQueueSize) { public boolean getEnableRenderWorker() { return this.enableRenderWorker; } + @JsonProperty + public void setEnableRenderWorker(boolean enableRenderWorker){ + this.enableRenderWorker = enableRenderWorker; + } + @JsonProperty public File getOutputLocation() { diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractorConfig.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractorConfig.java index 8db8d9fff..933fb0d00 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractorConfig.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/config/ExtractorConfig.java @@ -43,7 +43,7 @@ public void setProperties(HashMap properties) { @JsonIgnore public Extractor getExtractor() { - return ReflectionHelper.newExtractor(this.name); + return ReflectionHelper.newExtractor(this.name, this.properties); } @JsonIgnore diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java index 67f861fb2..f4a7fb78f 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/ExtractionItemContainer.java @@ -3,12 +3,14 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.File; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.StringUtils; import org.vitrivr.cineast.core.data.entities.MediaObjectDescriptor; import org.vitrivr.cineast.core.data.entities.MediaObjectMetadataDescriptor; @@ -35,7 +37,27 @@ public ExtractionItemContainer(@JsonProperty("object") MediaObjectDescriptor obj @JsonProperty("metadata") List metadata, @JsonProperty("uri") String uri) throws URISyntaxException { - this(object, metadata, Paths.get(new URI(uri))); + + this(object, metadata, convertUriToPathWindowsSafe(uri)); + } + + /** + * Annoying hack to also handle windows file URIs with a drive letter + */ + private static Path convertUriToPathWindowsSafe(String uri) throws URISyntaxException { + Path path; + try{ + path = new File(new URI(uri)).toPath(); + }catch(IllegalArgumentException ex){ + final URI parsedURI = new URI(uri); + if(StringUtils.isNotBlank(parsedURI.getAuthority())){ + path = new File(parsedURI.getAuthority()+parsedURI.getPath()).toPath(); + }else{ + // If for some unknown reasons we land here with a unix path, this should do the trick + path = new File(parsedURI.getPath()).toPath(); + } + } + return path; } @JsonIgnore diff --git a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java index fac707ab6..9647cf4a7 100644 --- a/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java +++ b/cineast-runtime/src/main/java/org/vitrivr/cineast/standalone/run/GenericExtractionItemHandler.java @@ -460,8 +460,8 @@ private Pair nextItem() { /* for ImageSequenceDecoders, it is expected that there might be images arriving which are not supported. The decoder reads folders and hands the images within to the extractors. */ continue; } - /* if not, log an error and move on */ - LOGGER.error("Media Type {} does not support file type {} for file {}", mediaType, type, item.getPathForExtraction().toString()); + /* if not, log an warn and move on */ + LOGGER.warn("Media Type {} does not support file type {} for file {}", mediaType, type, item.getPathForExtraction().toString()); continue; } diff --git a/cineast-runtime/src/main/resources/log4j2.xml b/cineast-runtime/src/main/resources/log4j2.xml index bd4e1fffe..5ea2b6ce1 100644 --- a/cineast-runtime/src/main/resources/log4j2.xml +++ b/cineast-runtime/src/main/resources/log4j2.xml @@ -24,14 +24,14 @@ - + extract --extraction ../Class/bsc-raphael-waltenspuel/setup/black01/3dm/config/extraction.json - + - + diff --git a/cineast.json b/cineast.json index 5f58c5b0f..851d8ead7 100644 --- a/cineast.json +++ b/cineast.json @@ -40,6 +40,12 @@ "visualtextcoembedding": [ {"feature": "VisualTextCoEmbedding", "weight": 1.0} ], + "clip": [ + {"feature": "CLIPText", "weight": 1.0} + ], + "mlclip": [ + {"feature": "ExternalOpenClipText", "weight": 1.0} + ], "boolean": [ { "feature": "CollectionBooleanRetriever", "weight": 1.0, diff --git a/external_extraction.json b/external_extraction.json new file mode 100644 index 000000000..ba6a8f095 --- /dev/null +++ b/external_extraction.json @@ -0,0 +1,57 @@ +{ + "input": { + "path": "data/", + "depth": 3, + "skip": 0, + "id": { + "name": "SequentialObjectIdGenerator", + "properties": {} + } + }, + "extractors": [ + { + "name": "ExternalAPI", + "properties": { + "tablename": "test_external", + "type": "vector", + "correspondence": "linear", + "correspondencep1": 5, + "distance": "cosine", + "vectorlen": 512, + "endpoint": "localhost:8999" + } + }, + { + "name": "ExternalAPI", + "properties": { + "tablename": "test_external2", + "type": "vector", + "correspondence": "hyperbolic", + "correspondencep1": 1, + "distance": "manhattan", + "vectorlen": 512, + "endpoint": "localhost:8999/" + } + } + ], + "metadata": [ + { + "name": "TechnicalVideoMetadataExtractor" + }, + { + "name": "EXIFMetadataExtractor" + } + ], + "exporters": [ + { + "name": "ShotThumbnailsExporter", + "properties": { + "destination": "thumbnails/" + } + } + ], + "database": { + "writer": "COTTONTAIL", + "selector": "COTTONTAIL" + } +} \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index bb38cb62b..d1c729356 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,7 +9,7 @@ version_commonslang3=3.12.0 version_commonsmath3=3.6.1 version_commonstext=1.9 version_cottontaildb_proto=0.14.2 -version_ffmpeg=5.1.2-1.5.8 +version_ffmpeg=6.0-1.5.9 version_gson=2.9.0 version_guava=31.1-jre vershion_hppc=0.9.1 diff --git a/resources/renderer/lwjgl/models/default/default.png b/resources/renderer/lwjgl/models/default/default.png index 909c66db1..13c0d359f 100644 Binary files a/resources/renderer/lwjgl/models/default/default.png and b/resources/renderer/lwjgl/models/default/default.png differ diff --git a/resources/renderer/lwjgl/shaders/scene.frag b/resources/renderer/lwjgl/shaders/scene.frag index 19061a687..381530850 100644 --- a/resources/renderer/lwjgl/shaders/scene.frag +++ b/resources/renderer/lwjgl/shaders/scene.frag @@ -1,18 +1,151 @@ #version 330 +const int MAX_POINT_LIGHTS = 5; +const int MAX_SPOT_LIGHTS = 5; +const float SPECULAR_COLOR = 10; + +in vec3 outPosition; +in vec3 outNormal; +in vec3 outTangent; +in vec3 outBitangent; in vec2 outTextCoord; out vec4 fragColor; +struct Attenuation +{ + float constant; + float linear; + float exponent; +}; + struct Material { + vec4 ambient; vec4 diffuse; + vec4 specular; + float reflectance; + int hasNormalMap; +}; + +struct AmbientLight +{ + float factor; + vec3 color; +}; + +struct PointLight +{ + vec3 position; + vec3 color; + float intensity; + Attenuation attenuation; +}; + +struct SpotLight +{ + PointLight pointLight; + vec3 coneDirection; + float cutOff; +}; + +struct DirectionalLight +{ + vec3 color; + vec3 direction; + float intensity; }; uniform sampler2D txtSampler; +uniform sampler2D normalMapSampler; uniform Material material; +uniform AmbientLight ambientLight; +uniform PointLight pointLights[MAX_POINT_LIGHTS]; +uniform SpotLight spotLights[MAX_SPOT_LIGHTS]; +uniform DirectionalLight directionalLight; -void main() -{ - fragColor = texture(txtSampler, outTextCoord) + material.diffuse; +vec3 calcNormal(vec3 normal, vec3 tangent, vec3 bitangent, vec2 textCoord) { + mat3 TBN = mat3(tangent, bitangent, normal); + vec3 newNormal = texture(normalMapSampler, textCoord).rgb; + newNormal = normalize(newNormal * 2.0 - 1.0); + newNormal = normalize(TBN * newNormal); + return newNormal; +} + +vec4 calcAmbient(AmbientLight ambientLight, vec4 ambient) { + return vec4(ambientLight.factor * ambientLight.color, 1) * ambient; +} + +vec4 calcLightColor(vec4 diffuse, vec4 specular, vec3 lightColor, float lightIntensity, vec3 position, vec3 toLightDirection, vec3 normal) { + vec4 diffuseColor = vec4(0, 0, 0, 1); + vec4 specColor = vec4(0, 0, 0, 1); + + // Diffuse Light + float diffuseFactor = max(dot(normal, toLightDirection), 0.0); + diffuseColor = diffuse * vec4(lightColor, 1) * lightIntensity * diffuseFactor; + + // Specular Light + vec3 cameraDirection = normalize(-position); + vec3 fromLightDirection = -toLightDirection; + vec3 reflectedLight = normalize(reflect(fromLightDirection, normal)); + float specularFactor = max(dot(cameraDirection, reflectedLight), 0.0); + specularFactor = pow(specularFactor, SPECULAR_COLOR); + specColor = specular * lightIntensity * specularFactor * material.reflectance * vec4(lightColor, 1); + + return (diffuseColor + specColor); +} + +vec4 calcPointLight(vec4 diffuse, vec4 specular, PointLight light, vec3 position, vec3 normal) { + vec3 lightDirection = light.position - position; + vec3 toLightDirection = normalize(lightDirection); + vec4 lightColor = calcLightColor(diffuse, specular, light.color, light.intensity, position, toLightDirection, normal); + + // Attenuation + float distance = length(lightDirection); + float attenuationInv = light.attenuation.constant + light.attenuation.linear * distance + light.attenuation.exponent * distance * distance; + return lightColor / attenuationInv; +} + +vec4 calcSpotLight(vec4 diffuse, vec4 specular, SpotLight light, vec3 position, vec3 normal) { + vec3 lightDirection = light.pointLight.position - position; + vec3 toLightDirection = normalize(lightDirection); + vec3 fromLightDirection = -toLightDirection; + float spotAlpha = dot(fromLightDirection, normalize(light.coneDirection)); + + vec4 color = vec4(0, 0, 0, 0); + + if (spotAlpha > light.cutOff) { + color = calcPointLight(diffuse, specular, light.pointLight, position, normal); + color *= (1.0 - (1.0 - spotAlpha) / (1.0 - light.cutOff)); + } + return color; +} + +vec4 calcDirectionalLight(vec4 diffuse, vec4 specular, DirectionalLight light, vec3 position, vec3 normal) { + return calcLightColor(diffuse, specular, light.color, light.intensity, position, normalize(light.direction), normal); +} + +void main() { + vec4 textColor = texture(txtSampler, outTextCoord); + vec4 ambient = calcAmbient(ambientLight, textColor + material.ambient); + vec4 diffuse = textColor + material.diffuse; + vec4 specular = textColor + material.specular; + + vec3 normal = outNormal; + if (material.hasNormalMap > 0) { + normal = calcNormal(outNormal, outTangent, outBitangent, outTextCoord); + } + + vec4 diffuseSpecularComp = calcDirectionalLight(diffuse, specular, directionalLight, outPosition, normal); + for (int ic = 0; ic < MAX_POINT_LIGHTS; ic++) { + if (pointLights[ic].intensity > 0) { + diffuseSpecularComp += calcPointLight(diffuse, specular, pointLights[ic], outPosition, normal); + } + } + for (int ic = 0; ic < MAX_SPOT_LIGHTS; ic++) { + if (spotLights[ic].pointLight.intensity > 0) { + diffuseSpecularComp += calcSpotLight(diffuse, specular, spotLights[ic], outPosition, normal); + } + } + fragColor = ambient + diffuseSpecularComp; } \ No newline at end of file diff --git a/resources/renderer/lwjgl/shaders/scene.vert b/resources/renderer/lwjgl/shaders/scene.vert index 338f34f89..d262824c8 100644 --- a/resources/renderer/lwjgl/shaders/scene.vert +++ b/resources/renderer/lwjgl/shaders/scene.vert @@ -1,8 +1,15 @@ #version 330 layout (location=0) in vec3 position; -layout (location=1) in vec2 texCoord; +layout (location=1) in vec3 normal; +layout (location=2) in vec3 tangent; +layout (location=3) in vec3 bitangent; +layout (location=4) in vec2 texCoord; +out vec3 outPosition; +out vec3 outNormal; +out vec3 outTangent; +out vec3 outBitangent; out vec2 outTextCoord; uniform mat4 projectionMatrix; @@ -11,6 +18,12 @@ uniform mat4 modelMatrix; void main() { - gl_Position = projectionMatrix * viewMatrix * modelMatrix * vec4(position, 1.0); + mat4 modelViewMatrix = viewMatrix * modelMatrix; + vec4 mvPosition = modelViewMatrix * vec4(position, 1.0); + gl_Position = projectionMatrix * mvPosition; + outPosition = mvPosition.xyz; + outNormal = normalize(modelViewMatrix * vec4(normal, 0.0)).xyz; + outTangent = normalize(modelViewMatrix * vec4(tangent, 0)).xyz; + outBitangent = normalize(modelViewMatrix * vec4(bitangent, 0)).xyz; outTextCoord = texCoord; } \ No newline at end of file