From 7d27038ac460ad402f1d02e90da2d20f8586e168 Mon Sep 17 00:00:00 2001 From: faberf Date: Thu, 8 Jun 2023 17:17:24 +0200 Subject: [PATCH] working commit for rewrite of external-api-extraction --- .../AbstractFeatureClientWrapper.java | 56 --------------- .../core/features/ExternalTextFeature.java | 12 ++-- .../cineast/core/features/FeatureClient.java | 72 +++++++++++++++++++ .../features/FeatureClientBinaryRequest.java | 28 ++++++++ .../features/FeatureClientJsonRequest.java | 40 +++++++++++ .../core/features/FeatureClientRequest.java | 14 ++++ .../core/features/SimpleFESWrapper.java | 58 --------------- .../core/util/web/MessageTemplate.java | 66 +++++++++++++++++ .../{FeatureClient.java => WebClient.java} | 22 ++++-- 9 files changed, 245 insertions(+), 123 deletions(-) delete mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/AbstractFeatureClientWrapper.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClient.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientBinaryRequest.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientJsonRequest.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientRequest.java delete mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/features/SimpleFESWrapper.java create mode 100644 cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MessageTemplate.java rename cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/{FeatureClient.java => WebClient.java} (53%) diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AbstractFeatureClientWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AbstractFeatureClientWrapper.java deleted file mode 100644 index 9ac2ccf53..000000000 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/AbstractFeatureClientWrapper.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.vitrivr.cineast.core.features; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.awt.image.BufferedImage; -import java.io.File; -import java.io.IOException; -import java.util.Map; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.vitrivr.cineast.core.util.web.FeatureClient; - -public abstract class AbstractFeatureClientWrapper { - protected final FeatureClient client; - protected static final Logger LOGGER = LogManager.getLogger(); - protected AbstractFeatureClientWrapper(FeatureClient client) { - this.client = client; - } - - private static FeatureClient buildFeatureClient(Map properties){ - if(!properties.containsKey("endpoint")){ - throw new IllegalArgumentException("No endpoint specified for external client"); //throw better exception - } - String endpoint = properties.get("endpoint"); - return new FeatureClient(endpoint); - } - - private static JsonNode getConfig(Map properties) throws IOException { - ObjectMapper mapper = new ObjectMapper(); - if(!properties.containsKey("config_path")){ - LOGGER.warn("No config path specified for external client, using empty config."); - return mapper.createObjectNode(); - }else { - return mapper.readTree(new File(properties.get("config_path"))); - } - } - - public static AbstractFeatureClientWrapper build(Map properties) throws Exception { - FeatureClient client = buildFeatureClient(properties); - JsonNode config = getConfig(properties); - switch (properties.getOrDefault("api","simpleFES")){ - case ("simpleFES"): - return new SimpleFESWrapper(client, config); - default: - throw new IllegalStateException("Unexpected value: " + properties.getOrDefault("api", "simpleFES")); - } - } - - - abstract public String extractTextFromImage(BufferedImage bufImg) throws IOException, InterruptedException; - - abstract public float[] extractVectorFromImage(BufferedImage bufImg) throws IOException, InterruptedException; - - abstract public float[] extractVectorFromText(String text) throws IOException, InterruptedException; - -} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalTextFeature.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalTextFeature.java index a020c263c..833ba7315 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalTextFeature.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/ExternalTextFeature.java @@ -1,6 +1,7 @@ package org.vitrivr.cineast.core.features; import java.awt.image.BufferedImage; +import java.io.IOException; import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -15,16 +16,17 @@ public class ExternalTextFeature extends AbstractTextRetriever { public static final String DEFAULT_TABLE_NAME = "features_externalText"; - final AbstractFeatureClientWrapper client; + final FeatureClient client; public ExternalTextFeature() { super(DEFAULT_TABLE_NAME); throw new IllegalArgumentException("no properties specified"); } - public ExternalTextFeature(Map properties) throws Exception { + public ExternalTextFeature(Map properties) throws IOException { super(properties.getOrDefault(CineastConstants.ENTITY_NAME_KEY, DEFAULT_TABLE_NAME), properties); - this.client = AbstractFeatureClientWrapper.build(properties); + + this.client = FeatureClient.build(properties); } private static final Logger LOGGER = LogManager.getLogger(); @@ -35,8 +37,8 @@ public void processSegment(SegmentContainer sc) { return; } try { - BufferedImage img = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); - String feature = client.extractTextFromImage(img); + Map extractions = client.extract(sc); + String feature = (String) extractions.getOrDefault("raw_text", ""); var descriptor = new SimpleFulltextFeatureDescriptor(sc.getId(), feature); this.writer.write(descriptor); } catch (Exception e) { diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClient.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClient.java new file mode 100644 index 000000000..a65bf874e --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClient.java @@ -0,0 +1,72 @@ +package org.vitrivr.cineast.core.features; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.exporter.AudioSegmentExporter; +import org.vitrivr.cineast.core.util.web.ImageParser; +import org.vitrivr.cineast.core.util.web.WebClient; +import org.vitrivr.cineast.core.util.web.MessageTemplate; + +public class FeatureClient { + protected final WebClient client; + protected final FeatureClientRequest request; + protected final MessageTemplate responseTemplate; + + protected static final Logger LOGGER = LogManager.getLogger(); + + public static FeatureClient build(Map properties) throws IOException { + if(!properties.containsKey("endpoint")){ + throw new IllegalArgumentException("No endpoint specified for external client"); + } + String endpoint = properties.get("endpoint"); + if(!properties.containsKey("request")){ + throw new IllegalArgumentException("No request specified for external client"); + } + String requestString = properties.get("request"); + if(!properties.containsKey("response")){ + throw new IllegalArgumentException("No response template specified for external client"); + } + String responseString = properties.get("response"); + FeatureClientRequest request; + if (new File(requestString).exists()) { + request = new FeatureClientJsonRequest(requestString); + }else { + request = new FeatureClientBinaryRequest(requestString); + } + return new FeatureClient(endpoint, request, responseString); + } + + protected FeatureClient(String endpoint, FeatureClientRequest req, String responseTemplatePath) throws IOException { + this.client = new WebClient(endpoint); + this.request = req; + this.responseTemplate = new MessageTemplate(responseTemplatePath); + } + + public Map extract(SegmentContainer sc) throws IOException, InterruptedException { + + String response = this.request.execute(sc, this.client); + + Map responseKeys = this.responseTemplate.parseString(response); + + HashMap result = new HashMap<>(); + for (var key : responseKeys.keySet()) { + switch (key) { + case "raw_text": + result.put(key, responseKeys.get(key)); + break; + default: + LOGGER.warn("Unknown key: {}", key); + break; + } + } + return result; + } + + + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientBinaryRequest.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientBinaryRequest.java new file mode 100644 index 000000000..4d167a180 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientBinaryRequest.java @@ -0,0 +1,28 @@ +package org.vitrivr.cineast.core.features; + +import java.io.IOException; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.abstracts.AbstractSegmentExporter; +import org.vitrivr.cineast.core.features.exporter.AudioSegmentExporter; +import org.vitrivr.cineast.core.util.web.WebClient; + +public class FeatureClientBinaryRequest extends FeatureClientRequest { + + private final String key; + public FeatureClientBinaryRequest(String requestString) { + super(); + this.key = requestString; + } + + @Override + public String execute(SegmentContainer sc, WebClient client) throws IOException, InterruptedException { + switch (this.key) { + case "wav_binary": + var wavExport = new AudioSegmentExporter(); + return client.postRawBinary(wavExport.exportToBinary(sc)); + default: + LOGGER.warn("Unknown key: {}", this.key); + return ""; + } + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientJsonRequest.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientJsonRequest.java new file mode 100644 index 000000000..1a9236cf9 --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientJsonRequest.java @@ -0,0 +1,40 @@ +package org.vitrivr.cineast.core.features; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.features.exporter.AudioSegmentExporter; +import org.vitrivr.cineast.core.util.web.ImageParser; +import org.vitrivr.cineast.core.util.web.MessageTemplate; +import org.vitrivr.cineast.core.util.web.WebClient; + +public class FeatureClientJsonRequest extends FeatureClientRequest { + private final MessageTemplate requestTemplate; + public FeatureClientJsonRequest(String RequestTemplatePath) throws IOException { + super(); + this.requestTemplate = new MessageTemplate(RequestTemplatePath); + } + @Override + public String execute(SegmentContainer sc, WebClient client) throws IOException, InterruptedException { + Map keys = new HashMap<>(); + //iterate through keys + for (var key : this.requestTemplate.getKeys()) { + switch (key) { + case "png_image": + var bufImg = sc.getMostRepresentativeFrame().getImage().getBufferedImage(); + keys.put(key, ImageParser.bufferedImageToDataURL(bufImg, "png")); + break; + case "wav_dataurl": + var wavExport = new AudioSegmentExporter(); + keys.put(key, wavExport.exportToDataUrl(sc)); + break; + default: + LOGGER.warn("Unknown key: {}", key); + break; + } + } + String request = this.requestTemplate.formatString(keys); + return client.postJsonString(request); + } +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientRequest.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientRequest.java new file mode 100644 index 000000000..04b009daf --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/FeatureClientRequest.java @@ -0,0 +1,14 @@ +package org.vitrivr.cineast.core.features; + +import java.io.IOException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.vitrivr.cineast.core.data.segments.SegmentContainer; +import org.vitrivr.cineast.core.util.web.WebClient; + +abstract class FeatureClientRequest { + + protected static final Logger LOGGER = LogManager.getLogger(); + + abstract public String execute(SegmentContainer sc, WebClient client) throws IOException, InterruptedException; +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SimpleFESWrapper.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SimpleFESWrapper.java deleted file mode 100644 index dc08ef66e..000000000 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/features/SimpleFESWrapper.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.vitrivr.cineast.core.features; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.json.JsonMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.awt.image.BufferedImage; -import java.io.IOException; -import org.vitrivr.cineast.core.util.web.FeatureClient; -import org.vitrivr.cineast.core.util.web.ImageParser; - -public class SimpleFESWrapper extends AbstractFeatureClientWrapper { - - private final JsonMapper mapper = new JsonMapper(); - private final JsonNode config; - - public SimpleFESWrapper(FeatureClient client, JsonNode config) throws Exception { - super(client); - this.config = config; - } - - @Override - public String extractTextFromImage(BufferedImage bufImg) throws IOException, InterruptedException { - var dataURL = ImageParser.bufferedImageToDataURL(bufImg, "png"); - // Make a deep copy of the node - JsonNode copyJsonNode = this.config.deepCopy(); - - // Cast the copied JsonNode as an ObjectNode - ObjectNode objectNode = (ObjectNode) copyJsonNode; - - // Add some properties - objectNode.put("image", dataURL); - - // Convert back to string and print - String requestBody = mapper.writeValueAsString(objectNode); - -// Gson gson = new Gson(); -// -// Map map = new HashMap<>(); -// map.put("image", query); -// -// String body = gson.toJson(map); - - String responseBody = this.client.getResponse(requestBody); - return mapper.readValue(responseBody, String[].class)[0]; // change this - } - - @Override - public float[] extractVectorFromImage(BufferedImage bufImg) throws IOException, InterruptedException { - return new float[0]; - } - - @Override - public float[] extractVectorFromText(String text) throws IOException, InterruptedException { - return new float[0]; - } - - -} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MessageTemplate.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MessageTemplate.java new file mode 100644 index 000000000..42a0951fb --- /dev/null +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/MessageTemplate.java @@ -0,0 +1,66 @@ +package org.vitrivr.cineast.core.util.web; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class MessageTemplate { + + private final String template; + + public MessageTemplate(String path) throws IOException { + this.template = Files.readString(new File(path).toPath()); + } + + public List getKeys(){ + List keys = new ArrayList<>(); + Matcher matcher = Pattern.compile("\\$\\{(.*?)\\}").matcher(template); + while(matcher.find()){ + keys.add(matcher.group(1)); + } + return keys; + } + + + public String formatString(Map values){ + // if not all keys are present, throw exception + for(String key : getKeys()){ + if(!values.containsKey(key)){ + throw new IllegalArgumentException("Missing key " + key); + } + } + String result = template; + for(String key : values.keySet()){ + result = result.replace("${" + key + "}", values.get(key)); + } + return result; + } + + public Map parseString(String input){ + Map values = new HashMap<>(); + + String[] parts = template.split("\\$\\{.*?\\}"); + String[] valueParts = new String[parts.length - 1]; + + for (int i = 0; i < parts.length - 1; i++) { + input = input.substring(parts[i].length()); + valueParts[i] = input.substring(0, input.indexOf(parts[i+1])); + } + + List keys = getKeys(); + for (int i = 0; i < keys.size(); i++) { + values.put(keys.get(i), valueParts[i]); + } + + return values; + } + + + +} diff --git a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/FeatureClient.java b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/WebClient.java similarity index 53% rename from cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/FeatureClient.java rename to cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/WebClient.java index 0dc630171..563317dcb 100644 --- a/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/FeatureClient.java +++ b/cineast-core/src/main/java/org/vitrivr/cineast/core/util/web/WebClient.java @@ -7,11 +7,11 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; -public class FeatureClient { +public class WebClient { private final String endpoint; - public FeatureClient(String endpoint) { + public WebClient(String endpoint) { this.endpoint = endpoint; } @@ -19,10 +19,10 @@ public FeatureClient(String endpoint) { .version(Version.HTTP_1_1) .build(); - public String getResponse(String body) throws IOException, InterruptedException { + public String postJsonString(String body) throws IOException, InterruptedException { HttpRequest request = HttpRequest.newBuilder() .POST(HttpRequest.BodyPublishers.ofString(body)) - .uri(URI.create(endpoint + "/extract")) + .uri(URI.create(endpoint)) .header("Content-Type", "application/json") .build(); @@ -34,4 +34,18 @@ public String getResponse(String body) throws IOException, InterruptedException return response.body(); } + public String postRawBinary(byte[] body) throws IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofByteArray(body)) + .uri(URI.create(endpoint)) + .header("Content-Type", "application/octet-stream") + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) { + throw new IllegalStateException("received response code " + response.statusCode()); + } + return response.body(); + } }