From 9f8d25f7ed95c6a1ac673c8c432c932a489e64a1 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Wed, 29 Jan 2025 18:37:34 -0700 Subject: [PATCH 01/13] WIP --- src/main/java/org/ecocean/Annotation.java | 4 +- .../java/org/ecocean/MarkedIndividual.java | 79 ++++++++++++++++++- src/main/resources/org/ecocean/package.jdo | 5 ++ src/main/webapp/appadmin/opensearchSync.jsp | 5 ++ 4 files changed, 89 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/ecocean/Annotation.java b/src/main/java/org/ecocean/Annotation.java index 932495754e..0506611820 100644 --- a/src/main/java/org/ecocean/Annotation.java +++ b/src/main/java/org/ecocean/Annotation.java @@ -134,7 +134,7 @@ public long setVersion() { public JSONObject opensearchMapping() { JSONObject map = super.opensearchMapping(); - JSONObject keywordType = new org.json.JSONObject("{\"type\": \"keyword\"}"); + JSONObject keywordType = new JSONObject("{\"type\": \"keyword\"}"); /* JSONObject keywordNormalType = new org.json.JSONObject( @@ -1373,7 +1373,7 @@ public static Base createFromApi(JSONObject payload, List files, Shepherd public static Object validateFieldValue(String fieldName, JSONObject data) throws ApiException { if (data == null) throw new ApiException("empty payload"); - org.json.JSONObject error = new org.json.JSONObject(); + JSONObject error = new JSONObject(); error.put("fieldName", fieldName); String exMessage = "invalid value for " + fieldName; Object returnValue = null; diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index bb14181e0c..87bb1a40c8 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -1,5 +1,7 @@ package org.ecocean; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; import java.net.URL; import java.text.SimpleDateFormat; @@ -94,6 +96,7 @@ public class MarkedIndividual extends Base implements java.io.Serializable { private Vector interestedResearchers = new Vector(); private String dateTimeCreated; + private long version = System.currentTimeMillis(); private String dateTimeLatestSighting; @@ -2595,10 +2598,77 @@ public void mergeAndThrowawayIndividual(MarkedIndividual other, String username, myShepherd.throwAwayMarkedIndividual(other); } +/* + all name context:value pairs [flexible list] [needs custom values exposed through site settings] + alternate ID + birth date + death date + sex + taxonomy + number of encounters + number of sightings + list of gps locations + list of collaborating researchers + number if images + all social groups [flexible list] [needs custom values exposed through site settings] + social group name + social role name + social group members (names and IDs) + membership start + membership end + all social relationships [flexible list] [needs custom values exposed through site settings] + relationship partner (name and ID) + relationship role + relationship start + relationship end + all co-occurrences + individual name (name and ID) + number of occurrences with that individual + */ + public org.json.JSONObject opensearchMapping() { + org.json.JSONObject map = super.opensearchMapping(); + org.json.JSONObject keywordType = new org.json.JSONObject("{\"type\": \"keyword\"}"); + +/* + JSONObject keywordNormalType = new org.json.JSONObject( + "{\"type\": \"keyword\", \"normalizer\": \"wildbook_keyword_normalizer\"}"); + */ + + // "id" is done in Base + map.put("sex", keywordType); + map.put("taxonomy", keywordType); + + // all case-insensitive keyword-ish types + // map.put("fubar", keywordNormalType); + + return map; + } + + public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd) + throws IOException, JsonProcessingException { + super.opensearchDocumentSerializer(jgen, myShepherd); + + jgen.writeStringField("sex", this.getSex()); + jgen.writeStringField("taxonomy", this.getTaxonomyString()); + if (this.getNumEncounters() > 0) { + jgen.writeNumberField("numberEncounters", this.getNumEncounters()); + Set occIds = new HashSet(); + for (Encounter enc : this.encounters) { + Occurrence occ = enc.getOccurrence(myShepherd); + if (occ != null) occIds.add(occ.getId()); + } + jgen.writeNumberField("numberOccurrences", occIds.size()); + } else { + jgen.writeNumberField("numberEncounters", 0); + jgen.writeNumberField("numberOccurrences", 0); + } + } + public void opensearchIndexDeep() throws IOException { this.opensearchIndex(); +/* final String indivId = this.getId(); ExecutorService executor = Executors.newFixedThreadPool(4); Runnable rn = new Runnable() { @@ -2644,6 +2714,7 @@ public void run() { executor.execute(rn); System.out.println("opensearchIndexDeep() [foreground] finished for MarkedIndividual " + indivId); + */ } public String toString() { @@ -2662,8 +2733,12 @@ public String toString() { } @Override public long getVersion() { - // Returning 0 for now since the class does not have a 'modified' attribute to compute this value, to be fixed in future. - return 0; + return version; + } + + public long setVersion() { + version = System.currentTimeMillis(); + return version; } @Override public String getAllVersionsSql() { diff --git a/src/main/resources/org/ecocean/package.jdo b/src/main/resources/org/ecocean/package.jdo index 7f63c2312a..9a14f57756 100755 --- a/src/main/resources/org/ecocean/package.jdo +++ b/src/main/resources/org/ecocean/package.jdo @@ -101,6 +101,11 @@ + + + + + diff --git a/src/main/webapp/appadmin/opensearchSync.jsp b/src/main/webapp/appadmin/opensearchSync.jsp index f965e45b20..d97aa91d89 100644 --- a/src/main/webapp/appadmin/opensearchSync.jsp +++ b/src/main/webapp/appadmin/opensearchSync.jsp @@ -21,6 +21,9 @@ if (indexName.equals("encounter")) { } else if (indexName.equals("annotation")) { cls = Annotation.class; obj = new Annotation(); +} else if (indexName.equals("individual")) { + cls = MarkedIndividual.class; + obj = new MarkedIndividual(); } System.out.println("opensearchSync.jsp begun..."); @@ -86,6 +89,8 @@ if (endNum > 0) { itr = myShepherd.getAllEncounters("catalogNumber"); } else if (indexName.equals("annotation")) { itr = myShepherd.getAllAnnotations("id"); + } else if (indexName.equals("individual")) { + itr = myShepherd.getAllMarkedIndividuals(); } while (itr.hasNext()) { Base iObj = (Base)itr.next(); From 63f83541d20d38dd43440d6a99e7c46b3b8e687e Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 30 Jan 2025 13:38:54 -0700 Subject: [PATCH 02/13] few more fields; fix getAllVersions --- .../java/org/ecocean/MarkedIndividual.java | 73 +++++++++++++++---- 1 file changed, 59 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 87bb1a40c8..c8ee7dacb1 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -8,6 +8,7 @@ import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.joda.time.DateTime; import java.util.concurrent.Executors; import java.util.concurrent.ExecutorService; @@ -2601,15 +2602,6 @@ public void mergeAndThrowawayIndividual(MarkedIndividual other, String username, /* all name context:value pairs [flexible list] [needs custom values exposed through site settings] alternate ID - birth date - death date - sex - taxonomy - number of encounters - number of sightings - list of gps locations - list of collaborating researchers - number if images all social groups [flexible list] [needs custom values exposed through site settings] social group name social role name @@ -2629,18 +2621,22 @@ public org.json.JSONObject opensearchMapping() { org.json.JSONObject map = super.opensearchMapping(); org.json.JSONObject keywordType = new org.json.JSONObject("{\"type\": \"keyword\"}"); -/* - JSONObject keywordNormalType = new org.json.JSONObject( + org.json.JSONObject keywordNormalType = new org.json.JSONObject( "{\"type\": \"keyword\", \"normalizer\": \"wildbook_keyword_normalizer\"}"); - */ // "id" is done in Base map.put("sex", keywordType); map.put("taxonomy", keywordType); + map.put("users", keywordType); // all case-insensitive keyword-ish types - // map.put("fubar", keywordNormalType); + map.put("names", keywordNormalType); + + map.put("timeOfBirth", new org.json.JSONObject("{\"type\": \"date\"}")); + map.put("timeOfDeath", new org.json.JSONObject("{\"type\": \"date\"}")); + map.put("locationsGeoPoint", new org.json.JSONObject("{\"type\": \"geo_point\"}")); + map.put("nameMap", new org.json.JSONObject("{\"type\": \"nested\"}")); return map; } @@ -2650,17 +2646,65 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd jgen.writeStringField("sex", this.getSex()); jgen.writeStringField("taxonomy", this.getTaxonomyString()); + if (this.getTimeOfBirth() > 0) { + String birthTime = Util.getISO8601Date(new DateTime(this.getTimeOfBirth()).toString()); + jgen.writeStringField("timeOfBirth", birthTime); + } + if (this.getTimeofDeath() > 0) { + String deathTime = Util.getISO8601Date(new DateTime(this.getTimeofDeath()).toString()); + jgen.writeStringField("timeOfDeath", deathTime); + } + if (this.getNames() != null) { + jgen.writeArrayFieldStart("names"); + Set names = this.getAllNamesList(); + if (names != null) + for (String name : names) { + jgen.writeString(name); + } + jgen.writeEndArray(); + jgen.writeFieldName("nameMap"); + jgen.writeRawValue(this.getNames().getValues().toString()); + } if (this.getNumEncounters() > 0) { + Set users = new HashSet(); jgen.writeNumberField("numberEncounters", this.getNumEncounters()); Set occIds = new HashSet(); + List dlats = new ArrayList(); + List dlons = new ArrayList(); + int numMAs = 0; for (Encounter enc : this.encounters) { + numMAs += enc.numAnnotations(); + users.addAll(enc.getAllSubmitterIds(myShepherd)); Occurrence occ = enc.getOccurrence(myShepherd); if (occ != null) occIds.add(occ.getId()); + Double dlat = enc.getDecimalLatitudeAsDouble(); + Double dlon = enc.getDecimalLongitudeAsDouble(); + if (Util.isValidDecimalLatitude(dlat) && Util.isValidDecimalLongitude(dlon)) { + dlats.add(dlat); + dlons.add(dlon); + } + } + jgen.writeNumberField("numberMediaAssets", numMAs); + + jgen.writeArrayFieldStart("locationGeoPoints"); + for (int i = 0; i < dlats.size(); i++) { + jgen.writeStartObject(); + jgen.writeNumberField("lat", dlats.get(i)); + jgen.writeNumberField("lon", dlons.get(i)); + jgen.writeEndObject(); } + jgen.writeEndArray(); + jgen.writeNumberField("numberOccurrences", occIds.size()); + jgen.writeArrayFieldStart("users"); + for (String uid : users) { + jgen.writeString(uid); + } + jgen.writeEndArray(); } else { jgen.writeNumberField("numberEncounters", 0); jgen.writeNumberField("numberOccurrences", 0); + jgen.writeNumberField("numberMediaAssets", 0); } } @@ -2742,6 +2786,7 @@ public long setVersion() { } @Override public String getAllVersionsSql() { - return "SELECT \"INDIVIDUALID\", CAST(0 AS BIGINT) FROM \"MARKEDINDIVIDUAL\""; + return + "SELECT \"INDIVIDUALID\", \"VERSION\" AS version FROM \"MARKEDINDIVIDUAL\" ORDER BY version"; } } From 6c3ac87c5b42ed18a8874e812774768696332721 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 30 Jan 2025 18:19:10 -0700 Subject: [PATCH 03/13] cooccurring helpers --- src/main/java/org/ecocean/Occurrence.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/main/java/org/ecocean/Occurrence.java b/src/main/java/org/ecocean/Occurrence.java index 01e9dd66f7..59a3bb6635 100644 --- a/src/main/java/org/ecocean/Occurrence.java +++ b/src/main/java/org/ecocean/Occurrence.java @@ -385,6 +385,24 @@ public ArrayList getMarkedIndividualNamesForThisOccurrence() { return names; } + public Set getMarkedIndividuals() { + return getMarkedIndividuals(null); + } + + public Set getMarkedIndividuals(MarkedIndividual skip) { + Set indivs = new HashSet(); + + if (this.encounters == null) return indivs; + String skipId = null; + if (skip != null) skipId = skip.getId(); + for (Encounter enc : this.encounters) { + MarkedIndividual indiv = enc.getIndividual(); + if ((indiv == null) || indiv.getId().equals(skipId)) continue; + indivs.add(indiv); + } + return indivs; + } + // TODO: validate and remove if ##DEPRECATED #509 - Base class setId() method public void setID(String id) { occurrenceID = id; From 1b80a6903094fb1320c9c3a26fc3a4309328608b Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 30 Jan 2025 18:19:47 -0700 Subject: [PATCH 04/13] yeah lets background individuals too --- src/main/java/org/ecocean/OpenSearch.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index 0b799c001d..cbe772b98b 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -157,6 +157,8 @@ public void run() { BACKGROUND_SLICE_SIZE); Base.opensearchSyncIndex(myShepherd, Annotation.class, BACKGROUND_SLICE_SIZE); + Base.opensearchSyncIndex(myShepherd, MarkedIndividual.class, + BACKGROUND_SLICE_SIZE); System.out.println("OpenSearch background indexing finished."); myShepherd.rollbackAndClose(); } catch (Exception ex) { From 0962e47be444aedd2b29c72af4fb1d1e2b1ef43f Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 30 Jan 2025 18:21:21 -0700 Subject: [PATCH 05/13] omgwtf apparently you cant have empty-string keys in nested opensearch docs. thats 3 hrs of my life i wont get back. --- .../java/org/ecocean/MarkedIndividual.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index c8ee7dacb1..5db33ae07b 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2634,7 +2634,7 @@ public org.json.JSONObject opensearchMapping() { map.put("timeOfBirth", new org.json.JSONObject("{\"type\": \"date\"}")); map.put("timeOfDeath", new org.json.JSONObject("{\"type\": \"date\"}")); - map.put("locationsGeoPoint", new org.json.JSONObject("{\"type\": \"geo_point\"}")); + map.put("locationGeoPoints", new org.json.JSONObject("{\"type\": \"geo_point\"}")); map.put("nameMap", new org.json.JSONObject("{\"type\": \"nested\"}")); return map; @@ -2659,11 +2659,22 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd Set names = this.getAllNamesList(); if (names != null) for (String name : names) { - jgen.writeString(name); + if (!Util.stringIsEmptyOrNull(name)) jgen.writeString(name); } jgen.writeEndArray(); - jgen.writeFieldName("nameMap"); - jgen.writeRawValue(this.getNames().getValues().toString()); + jgen.writeObjectFieldStart("nameMap"); + for (String key : this.getNames().getKeys()) { + if (Util.stringIsEmptyOrNull(key)) continue; + jgen.writeArrayFieldStart(key); + Set uniq = new HashSet(this.getNames().getValuesByKey(key)); + uniq.remove(""); + uniq.remove(null); + for (String nm : uniq) { + jgen.writeString(nm); + } + jgen.writeEndArray(); + } + jgen.writeEndObject(); } if (this.getNumEncounters() > 0) { Set users = new HashSet(); From 72124f5fd0b0c5af2e6e6356542caa808f183044 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 30 Jan 2025 19:25:21 -0700 Subject: [PATCH 06/13] give me a break --- .../java/org/ecocean/MarkedIndividual.java | 59 ++++++++++++++++++- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 5db33ae07b..b9b2534762 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2600,7 +2600,6 @@ public void mergeAndThrowawayIndividual(MarkedIndividual other, String username, } /* - all name context:value pairs [flexible list] [needs custom values exposed through site settings] alternate ID all social groups [flexible list] [needs custom values exposed through site settings] social group name @@ -2628,6 +2627,9 @@ public org.json.JSONObject opensearchMapping() { map.put("sex", keywordType); map.put("taxonomy", keywordType); map.put("users", keywordType); + map.put("socialUnits", keywordType); + map.put("relationshipRoles", keywordType); + map.put("cooccurrenceIndividualIds", keywordType); // all case-insensitive keyword-ish types map.put("names", keywordNormalType); @@ -2636,7 +2638,11 @@ public org.json.JSONObject opensearchMapping() { map.put("timeOfDeath", new org.json.JSONObject("{\"type\": \"date\"}")); map.put("locationGeoPoints", new org.json.JSONObject("{\"type\": \"geo_point\"}")); - map.put("nameMap", new org.json.JSONObject("{\"type\": \"nested\"}")); + // map.put("nameMap", new org.json.JSONObject("{\"type\": \"nested\"}")); + map.put("nameMap", new org.json.JSONObject("{\"type\": \"nested\", \"dynamic\": false}")); + // map.put("cooccurrenceIndividualMap", new org.json.JSONObject("{\"type\": \"nested\"}")); + map.put("cooccurrenceIndividualMap", + new org.json.JSONObject("{\"type\": \"nested\", \"dynamic\": false}")); return map; } @@ -2676,18 +2682,52 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd } jgen.writeEndObject(); } +/* + social group name + social role name + social group members (names and IDs) + membership start + membership end + all social relationships [flexible list] [needs custom values exposed through site settings] + relationship partner (name and ID) + relationship role + relationship start + relationship end + */ + jgen.writeArrayFieldStart("socialUnits"); + for (SocialUnit su : myShepherd.getAllSocialUnitsForMarkedIndividual(this)) { + Membership mem = su.getMembershipForMarkedIndividual(this); + if (mem != null) jgen.writeString(su.getSocialUnitName()); + } + jgen.writeEndArray(); + jgen.writeArrayFieldStart("relationshipRoles"); + for (String relRole : myShepherd.getAllRoleNamesForMarkedIndividual(this.getId())) { + jgen.writeString(relRole); + } + jgen.writeEndArray(); if (this.getNumEncounters() > 0) { Set users = new HashSet(); jgen.writeNumberField("numberEncounters", this.getNumEncounters()); Set occIds = new HashSet(); List dlats = new ArrayList(); List dlons = new ArrayList(); + Map coMap = new HashMap(); int numMAs = 0; for (Encounter enc : this.encounters) { numMAs += enc.numAnnotations(); users.addAll(enc.getAllSubmitterIds(myShepherd)); Occurrence occ = enc.getOccurrence(myShepherd); - if (occ != null) occIds.add(occ.getId()); + if (occ != null) { + occIds.add(occ.getId()); + Set coIndivs = occ.getMarkedIndividuals(this); + for (MarkedIndividual coInd : coIndivs) { + if (!coMap.containsKey(coInd)) { + coMap.put(coInd, 1); + } else { + coMap.put(coInd, coMap.get(coInd) + 1); + } + } + } Double dlat = enc.getDecimalLatitudeAsDouble(); Double dlon = enc.getDecimalLongitudeAsDouble(); if (Util.isValidDecimalLatitude(dlat) && Util.isValidDecimalLongitude(dlon)) { @@ -2695,6 +2735,19 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd dlons.add(dlon); } } + if (coMap.size() > 0) { + // json is a quick hacky way to write out using writeRawValue() + // JSONArray coNamesArr = new JSONArray(); TODO unsure how names should be used in index??? + org.json.JSONObject coMapJ = new org.json.JSONObject(); + jgen.writeArrayFieldStart("cooccurrenceIndividualIds"); + for (MarkedIndividual ind : coMap.keySet()) { + jgen.writeString(ind.getId()); + coMapJ.put(ind.getId(), coMap.get(ind)); + } + jgen.writeEndArray(); + jgen.writeFieldName("cooccurrenceIndividualMap"); + jgen.writeRawValue(coMapJ.toString()); + } jgen.writeNumberField("numberMediaAssets", numMAs); jgen.writeArrayFieldStart("locationGeoPoints"); From dee41490c89752ddd65b8946f051b4751e4a5c8b Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Thu, 30 Jan 2025 19:29:19 -0700 Subject: [PATCH 07/13] drop empty check --- .../java/org/ecocean/MarkedIndividual.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index b9b2534762..1c00ac9901 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2638,9 +2638,7 @@ public org.json.JSONObject opensearchMapping() { map.put("timeOfDeath", new org.json.JSONObject("{\"type\": \"date\"}")); map.put("locationGeoPoints", new org.json.JSONObject("{\"type\": \"geo_point\"}")); - // map.put("nameMap", new org.json.JSONObject("{\"type\": \"nested\"}")); map.put("nameMap", new org.json.JSONObject("{\"type\": \"nested\", \"dynamic\": false}")); - // map.put("cooccurrenceIndividualMap", new org.json.JSONObject("{\"type\": \"nested\"}")); map.put("cooccurrenceIndividualMap", new org.json.JSONObject("{\"type\": \"nested\", \"dynamic\": false}")); return map; @@ -2735,19 +2733,17 @@ relationship partner (name and ID) dlons.add(dlon); } } - if (coMap.size() > 0) { - // json is a quick hacky way to write out using writeRawValue() - // JSONArray coNamesArr = new JSONArray(); TODO unsure how names should be used in index??? - org.json.JSONObject coMapJ = new org.json.JSONObject(); - jgen.writeArrayFieldStart("cooccurrenceIndividualIds"); - for (MarkedIndividual ind : coMap.keySet()) { - jgen.writeString(ind.getId()); - coMapJ.put(ind.getId(), coMap.get(ind)); - } - jgen.writeEndArray(); - jgen.writeFieldName("cooccurrenceIndividualMap"); - jgen.writeRawValue(coMapJ.toString()); + // json is a quick hacky way to write out using writeRawValue() + // JSONArray coNamesArr = new JSONArray(); TODO unsure how names should be used in index??? + org.json.JSONObject coMapJ = new org.json.JSONObject(); + jgen.writeArrayFieldStart("cooccurrenceIndividualIds"); + for (MarkedIndividual ind : coMap.keySet()) { + jgen.writeString(ind.getId()); + coMapJ.put(ind.getId(), coMap.get(ind)); } + jgen.writeEndArray(); + jgen.writeFieldName("cooccurrenceIndividualMap"); + jgen.writeRawValue(coMapJ.toString()); jgen.writeNumberField("numberMediaAssets", numMAs); jgen.writeArrayFieldStart("locationGeoPoints"); From a2b7300dca766dfc750a4d6c18ea37d1a949780b Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Fri, 31 Jan 2025 11:23:53 -0700 Subject: [PATCH 08/13] let thru some indexNames --- src/main/java/org/ecocean/OpenSearch.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ecocean/OpenSearch.java b/src/main/java/org/ecocean/OpenSearch.java index cbe772b98b..4d2efbb72e 100644 --- a/src/main/java/org/ecocean/OpenSearch.java +++ b/src/main/java/org/ecocean/OpenSearch.java @@ -670,7 +670,9 @@ public static JSONObject sanitizeDoc(final JSONObject sourceDoc, String indexNam throws IOException { if ((user == null) || (sourceDoc == null)) throw new IOException("null user or sourceDoc"); JSONObject clean = new JSONObject(); - // this is just punting future classes to later development (should never happen) + // these classes we let anyone see as-is + if ("annotation".equals(indexName) || "individual".equals(indexName)) return sourceDoc; + // this is just punting future classes to later development if (!"encounter".equals(indexName)) return clean; boolean hasAccess = Encounter.opensearchAccess(sourceDoc, user, myShepherd); if (hasAccess) { From bf89aa4474f58809c0e77b0319b6d78716b40fa9 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Fri, 31 Jan 2025 11:24:13 -0700 Subject: [PATCH 09/13] socialUnits support --- .../java/org/ecocean/MarkedIndividual.java | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 1c00ac9901..5d1bcf6d3b 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -25,6 +25,7 @@ import org.ecocean.social.Membership; import org.ecocean.social.Relationship; import org.ecocean.social.SocialUnit; +import org.json.JSONArray; import java.text.DecimalFormat; @@ -2627,7 +2628,6 @@ public org.json.JSONObject opensearchMapping() { map.put("sex", keywordType); map.put("taxonomy", keywordType); map.put("users", keywordType); - map.put("socialUnits", keywordType); map.put("relationshipRoles", keywordType); map.put("cooccurrenceIndividualIds", keywordType); @@ -2639,6 +2639,7 @@ public org.json.JSONObject opensearchMapping() { map.put("locationGeoPoints", new org.json.JSONObject("{\"type\": \"geo_point\"}")); map.put("nameMap", new org.json.JSONObject("{\"type\": \"nested\", \"dynamic\": false}")); + map.put("socialUnits", new org.json.JSONObject("{\"type\": \"nested\"}")); map.put("cooccurrenceIndividualMap", new org.json.JSONObject("{\"type\": \"nested\", \"dynamic\": false}")); return map; @@ -2649,6 +2650,7 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd super.opensearchDocumentSerializer(jgen, myShepherd); jgen.writeStringField("sex", this.getSex()); + jgen.writeStringField("displayName", this.getDisplayName()); jgen.writeStringField("taxonomy", this.getTaxonomyString()); if (this.getTimeOfBirth() > 0) { String birthTime = Util.getISO8601Date(new DateTime(this.getTimeOfBirth()).toString()); @@ -2681,12 +2683,7 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd jgen.writeEndObject(); } /* - social group name - social role name - social group members (names and IDs) - membership start - membership end - all social relationships [flexible list] [needs custom values exposed through site settings] + SOCIAL RELATIONSHIPS relationship partner (name and ID) relationship role relationship start @@ -2694,8 +2691,27 @@ relationship partner (name and ID) */ jgen.writeArrayFieldStart("socialUnits"); for (SocialUnit su : myShepherd.getAllSocialUnitsForMarkedIndividual(this)) { + jgen.writeStartObject(); + jgen.writeStringField("name", su.getSocialUnitName()); + JSONArray memIds = new JSONArray(); + JSONArray memNames = new JSONArray(); + for (Membership mem : su.getAllMembers()) { + memIds.put(mem.getMarkedIndividual().getId()); + memNames.put(mem.getMarkedIndividual().getDisplayName()); + } + jgen.writeArrayFieldStart("memberIds"); + jgen.writeRawValue(memIds.toString()); + jgen.writeEndArray(); + jgen.writeArrayFieldStart("memberNames"); + jgen.writeRawValue(memNames.toString()); + jgen.writeEndArray(); Membership mem = su.getMembershipForMarkedIndividual(this); - if (mem != null) jgen.writeString(su.getSocialUnitName()); + if (mem != null) { + jgen.writeStringField("role", mem.getRole()); + jgen.writeStringField("startDate", mem.getStartDate()); + jgen.writeStringField("endDate", mem.getEndDate()); + } + jgen.writeStartObject(); } jgen.writeEndArray(); jgen.writeArrayFieldStart("relationshipRoles"); @@ -2772,7 +2788,6 @@ public void opensearchIndexDeep() throws IOException { this.opensearchIndex(); -/* final String indivId = this.getId(); ExecutorService executor = Executors.newFixedThreadPool(4); Runnable rn = new Runnable() { @@ -2818,7 +2833,6 @@ public void run() { executor.execute(rn); System.out.println("opensearchIndexDeep() [foreground] finished for MarkedIndividual " + indivId); - */ } public String toString() { From 69b48ae6db0bb8fec7c96d75029db3796c62d3f0 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Fri, 31 Jan 2025 11:58:52 -0700 Subject: [PATCH 10/13] useful methods --- src/main/java/org/ecocean/social/Relationship.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/org/ecocean/social/Relationship.java b/src/main/java/org/ecocean/social/Relationship.java index d735c67eeb..4e80343d15 100644 --- a/src/main/java/org/ecocean/social/Relationship.java +++ b/src/main/java/org/ecocean/social/Relationship.java @@ -91,6 +91,20 @@ public void setMarkedIndividualName1(String name) { public MarkedIndividual getMarkedIndividual1() { return individual1; } public MarkedIndividual getMarkedIndividual2() { return individual2; } + public MarkedIndividual getOtherMarkedIndividual(MarkedIndividual indiv) { + if ((indiv == null) || (individual1 == null) || (individual2 == null)) return null; + if (indiv.getId().equals(individual1.getId())) return individual2; + if (indiv.getId().equals(individual2.getId())) return individual1; + return null; + } + + public String getRoleFor(MarkedIndividual indiv) { + if ((indiv == null) || (individual1 == null) || (individual2 == null)) return null; + if (indiv.getId().equals(individual1.getId())) return markedIndividualRole1; + if (indiv.getId().equals(individual2.getId())) return markedIndividualRole2; + return null; + } + public void setIndividual1(MarkedIndividual indy) { this.individual1 = indy; } From 1ec20041ac79a0269e57e826f0846944b57d8e19 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Fri, 31 Jan 2025 12:02:29 -0700 Subject: [PATCH 11/13] relationships indexing --- .../java/org/ecocean/MarkedIndividual.java | 48 ++++++++----------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 5d1bcf6d3b..8c7375df6b 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2600,23 +2600,6 @@ public void mergeAndThrowawayIndividual(MarkedIndividual other, String username, myShepherd.throwAwayMarkedIndividual(other); } -/* - alternate ID - all social groups [flexible list] [needs custom values exposed through site settings] - social group name - social role name - social group members (names and IDs) - membership start - membership end - all social relationships [flexible list] [needs custom values exposed through site settings] - relationship partner (name and ID) - relationship role - relationship start - relationship end - all co-occurrences - individual name (name and ID) - number of occurrences with that individual - */ public org.json.JSONObject opensearchMapping() { org.json.JSONObject map = super.opensearchMapping(); org.json.JSONObject keywordType = new org.json.JSONObject("{\"type\": \"keyword\"}"); @@ -2628,7 +2611,6 @@ public org.json.JSONObject opensearchMapping() { map.put("sex", keywordType); map.put("taxonomy", keywordType); map.put("users", keywordType); - map.put("relationshipRoles", keywordType); map.put("cooccurrenceIndividualIds", keywordType); // all case-insensitive keyword-ish types @@ -2640,6 +2622,7 @@ public org.json.JSONObject opensearchMapping() { map.put("nameMap", new org.json.JSONObject("{\"type\": \"nested\", \"dynamic\": false}")); map.put("socialUnits", new org.json.JSONObject("{\"type\": \"nested\"}")); + map.put("relationships", new org.json.JSONObject("{\"type\": \"nested\"}")); map.put("cooccurrenceIndividualMap", new org.json.JSONObject("{\"type\": \"nested\", \"dynamic\": false}")); return map; @@ -2682,13 +2665,6 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd } jgen.writeEndObject(); } -/* - SOCIAL RELATIONSHIPS - relationship partner (name and ID) - relationship role - relationship start - relationship end - */ jgen.writeArrayFieldStart("socialUnits"); for (SocialUnit su : myShepherd.getAllSocialUnitsForMarkedIndividual(this)) { jgen.writeStartObject(); @@ -2711,12 +2687,26 @@ relationship partner (name and ID) jgen.writeStringField("startDate", mem.getStartDate()); jgen.writeStringField("endDate", mem.getEndDate()); } - jgen.writeStartObject(); + jgen.writeEndObject(); } jgen.writeEndArray(); - jgen.writeArrayFieldStart("relationshipRoles"); - for (String relRole : myShepherd.getAllRoleNamesForMarkedIndividual(this.getId())) { - jgen.writeString(relRole); + jgen.writeArrayFieldStart("relationships"); + for (Relationship rel : myShepherd.getAllRelationshipsForMarkedIndividual(this.getId())) { + jgen.writeStartObject(); + MarkedIndividual partner = rel.getOtherMarkedIndividual(this); + if (partner != null) { + jgen.writeStringField("partnerName", partner.getDisplayName()); + jgen.writeStringField("partnerId", partner.getId()); + jgen.writeStringField("partnerRole", rel.getRoleFor(partner)); + } + if (rel.getStartTime() > 0) + jgen.writeStringField("startTime", + Util.getISO8601Date(new DateTime(rel.getStartTime()).toString())); + if (rel.getEndTime() > 0) + jgen.writeStringField("endTime", + Util.getISO8601Date(new DateTime(rel.getEndTime()).toString())); + jgen.writeStringField("role", rel.getRoleFor(this)); + jgen.writeEndObject(); } jgen.writeEndArray(); if (this.getNumEncounters() > 0) { From 1639c8f6b09e200388de1d64e22050a5c0218e28 Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Fri, 31 Jan 2025 12:18:03 -0700 Subject: [PATCH 12/13] update version on changes --- .../java/org/ecocean/MarkedIndividual.java | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 8c7375df6b..5e0900931b 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -157,6 +157,7 @@ public MarkedIndividual(Encounter enc) { // Sets the Individual Id. @Override public void setId(String id) { individualID = id; + setVersion(); } // this is "something to show" (by default)... it falls back to the id, @@ -262,6 +263,7 @@ public static String getDisplayNameForID(Shepherd myShepherd, String indId) { public void setNames(MultiValue mv) { names = mv; refreshNamesCache(); + setVersion(); } public MultiValue getNames() { @@ -301,6 +303,7 @@ public void unsetNames() { this.names = new MultiValue(); if (Util.stringExists(legacy)) addName(NAMES_KEY_LEGACYINDIVIDUALID, legacy); + setVersion(); } // this adds to the default @@ -308,12 +311,14 @@ public void addName(String name) { if (names == null) names = new MultiValue(); names.addValuesDefault(name); refreshNamesCache(); + setVersion(); } public void addName(Object keyHint, String name) { if (names == null) names = new MultiValue(); names.addValues(keyHint, name); refreshNamesCache(); + setVersion(); } // adds a name and inserts a comment describing who, when, and (optionally) why that was done @@ -338,6 +343,7 @@ public void addNameByKey(String key, String value) { if (names == null) names = new MultiValue(); names.addValuesByKey(key, value); refreshNamesCache(); + setVersion(); } public boolean hasName(String value) { @@ -398,6 +404,7 @@ public MultiValue setNamesFromLegacy() { names.addValuesByKey(NAMES_KEY_ALTERNATEID, part[i]); } } + setVersion(); return names; } @@ -447,6 +454,7 @@ public void addIncrementalProjectId(Project project) { ". Individual already has an Id from project " + project.getProjectIdPrefix() + "."); } + setVersion(); } else { System.out.println( "[WARN]: Passed a null project to MarkedIndividual.addIncrementalProjectId() on " + @@ -471,6 +479,7 @@ public boolean addEncounter(Encounter newEncounter) { encounters.add(newEncounter); numberEncounters++; refreshDependentProperties(); + setVersion(); } setTaxonomyFromEncounters(); // will only set if has no value setSexFromEncounters(); // likewise @@ -493,6 +502,7 @@ public boolean addEncounterNoCommit(Encounter newEncounter) { encounters.add(newEncounter); numberEncounters++; refreshDependentProperties(); + setVersion(); } return isNew; } @@ -515,6 +525,7 @@ public boolean removeEncounter(Encounter getRidOfMe) { localHaplotypeReflection = null; getHaplotype(); + setVersion(); return changed; } @@ -542,6 +553,7 @@ public String refreshDateFirstIdentified() { String d = new Integer(first.getYear()).toString(); if (first.getMonth() > 0) d = new Integer(first.getMonth()).toString() + "/" + d; this.dateFirstIdentified = d; + setVersion(); return d; } @@ -552,6 +564,7 @@ public String refreshDateLastestSighting() { Encounter last = sorted[0]; if (last.getYear() < 1) return null; this.dateTimeLatestSighting = last.getDate(); + setVersion(); return last.getDate(); } @@ -852,12 +865,14 @@ public String getLegacyIndividualID() { // TODO: evaluate and remove if deprecated: ##DEPRECATED #509 - Base class getId() method public void setIndividualID(String id) { individualID = id; + setVersion(); } public void setAlternateID(String alt) { System.out.println( "WARNING: indiv.setAlternateID() is depricated, please consider modifying .names according to a hint/context"); this.addNameByKey(NAMES_KEY_ALTERNATEID, alt); + setVersion(); } // Returns the specified encounter, where the encounter numbers range from 0 to n-1, where n is the total number of encounters stored for this MarkedIndividual. @@ -956,11 +971,13 @@ public String getWebUrl(HttpServletRequest req) { } else { comments = newComments; } + setVersion(); } // Sets any additional, general comments recorded for this MarkedIndividual as a whole. @Override public void setComments(String comments) { this.comments = comments; + setVersion(); } // Returns the complete Vector of stored satellite tag data files for this MarkedIndividual. @@ -976,6 +993,7 @@ public String getSex() { // Sets the sex of this MarkedIndividual. public void setSex(String newSex) { if (newSex != null) { sex = newSex; } else { sex = null; } + setVersion(); } public boolean hasSex() { @@ -990,6 +1008,7 @@ public String getGenus() { public void setGenus(String newGenus) { genus = newGenus; + setVersion(); } public String getSpecificEpithet() { @@ -998,6 +1017,7 @@ public String getSpecificEpithet() { public void setSpecificEpithet(String newEpithet) { specificEpithet = newEpithet; + setVersion(); } public void setTaxonomyString(String tax) { @@ -1029,6 +1049,7 @@ public String setTaxonomyFromEncounters(boolean force) { if ((enc.getGenus() != null) && (enc.getSpecificEpithet() != null)) { genus = enc.getGenus(); specificEpithet = enc.getSpecificEpithet(); + setVersion(); return getTaxonomyString(); } } @@ -1046,6 +1067,7 @@ public String setSexFromEncounters(boolean force) { for (Encounter enc : encounters) { if (enc.getSex() != null && !enc.getSex().equals("unknown")) { sex = enc.getSex(); + setVersion(); return getSex(); } } @@ -1185,6 +1207,7 @@ public Vector getInterestedResearchers() { public void addInterestedResearcher(String email) { if (interestedResearchers == null) { interestedResearchers = new Vector(); } interestedResearchers.add(email); + setVersion(); } public void removeInterestedResearcher(String email) { @@ -1196,16 +1219,19 @@ public void removeInterestedResearcher(String email) { } } } + setVersion(); } public void setSeriesCode(String newCode) { seriesCode = newCode; + setVersion(); } // Adds a satellite tag data file for this MarkedIndividual. public void addDataFile(String dataFile) { if (dataFiles == null) { dataFiles = new Vector(); } dataFiles.add(dataFile); + setVersion(); } // Removes a satellite tag data file for this MarkedIndividual. @@ -1213,6 +1239,7 @@ public void removeDataFile(String dataFile) { if (dataFiles != null) { dataFiles.remove(dataFile); } + setVersion(); } public int getNumberTrainableEncounters() { @@ -1465,10 +1492,12 @@ public String getDateLatestSighting() { public void setDateTimeCreated(String time) { dateTimeCreated = time; + setVersion(); } public void setDateTimeLatestSighting(String time) { dateTimeLatestSighting = time; + setVersion(); } public String getDynamicProperties() { @@ -1477,6 +1506,7 @@ public String getDynamicProperties() { public void setDynamicProperties(String dprop) { this.dynamicProperties = dprop; + setVersion(); } public void setDynamicProperty(String name, String value) { @@ -1506,6 +1536,7 @@ public void setDynamicProperty(String name, String value) { dynamicProperties = dynamicProperties + name + "=" + value + ";"; } } + setVersion(); } public String getDynamicPropertyValue(String name) { @@ -1547,6 +1578,7 @@ public void removeDynamicProperty(String name) { ";") + ";"; } } + setVersion(); } public ArrayList getAllAppliedKeywordNames(Shepherd myShepherd) { @@ -1631,7 +1663,10 @@ public String getPatterningCode() { } // Sets the patterning type evident on this MarkedIndividual instance. - public void setPatterningCode(String newCode) { this.patterningCode = newCode; } + public void setPatterningCode(String newCode) { + this.patterningCode = newCode; + setVersion(); + } public void resetMaxNumYearsBetweenSightings() { int maxYears = 0; @@ -1649,6 +1684,7 @@ public void resetMaxNumYearsBetweenSightings() { } } maxYearsBetweenResightings = maxYears; + setVersion(); } public String sidesSightedInPeriod(int m_startYear, int m_startMonth, int m_startDay, @@ -1952,8 +1988,15 @@ public void doNotSetLocalHaplotypeReflection(String myHaplo) { public long getTimeOfBirth() { return timeOfBirth; } public long getTimeofDeath() { return timeOfDeath; } - public void setTimeOfBirth(long newTime) { timeOfBirth = newTime; } - public void setTimeOfDeath(long newTime) { timeOfDeath = newTime; } + public void setTimeOfBirth(long newTime) { + timeOfBirth = newTime; + setVersion(); + } + + public void setTimeOfDeath(long newTime) { + timeOfDeath = newTime; + setVersion(); + } public List getAllRelationships(Shepherd myShepherd) { return myShepherd.getAllRelationshipsForMarkedIndividual(individualID); @@ -2347,6 +2390,7 @@ public void refreshDependentProperties() { this.resetMaxNumYearsBetweenSightings(); this.refreshDateFirstIdentified(); this.refreshDateLastestSighting(); + this.setVersion(); } // to find an *exact match* on a name, you can use: regex = "(^|.*;)NAME(;.*|$)"; From 925e233a95f266ef76ddf7c3e32a966da478a45d Mon Sep 17 00:00:00 2001 From: Jon Van Oast Date: Mon, 3 Feb 2025 15:00:44 -0700 Subject: [PATCH 13/13] add deep index on encounter to trigger cascade index on individual --- src/main/java/org/ecocean/Encounter.java | 45 +++++++++++++++++++ .../java/org/ecocean/MarkedIndividual.java | 4 +- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ecocean/Encounter.java b/src/main/java/org/ecocean/Encounter.java index f361ef97e1..7ef92e211e 100644 --- a/src/main/java/org/ecocean/Encounter.java +++ b/src/main/java/org/ecocean/Encounter.java @@ -4311,6 +4311,51 @@ public void opensearchDocumentSerializer(JsonGenerator jgen, Shepherd myShepherd } } + public void opensearchIndexDeep() + throws IOException { + this.opensearchIndex(); + + final String encId = this.getId(); + ExecutorService executor = Executors.newFixedThreadPool(4); + Runnable rn = new Runnable() { + public void run() { + Shepherd bgShepherd = new Shepherd("context0"); + bgShepherd.setAction("Encounter.opensearchIndexDeep_" + encId); + bgShepherd.beginDBTransaction(); + try { + Encounter enc = bgShepherd.getEncounter(encId); + if ((enc == null) || !enc.hasMarkedIndividual()) { + // bgShepherd.rollbackAndClose(); + executor.shutdown(); + return; + } + MarkedIndividual indiv = enc.getIndividual(); + System.out.println("opensearchIndexDeep() background indexing indiv " + + indiv.getId() + " via enc " + encId); + try { + indiv.opensearchIndex(); + } catch (Exception ex) { + System.out.println("opensearchIndexDeep() background indexing " + + indiv.getId() + " FAILED: " + ex.toString()); + ex.printStackTrace(); + } + } catch (Exception e) { + System.out.println("opensearchIndexDeep() backgrounding Encounter " + encId + + " hit an exception."); + e.printStackTrace(); + } finally { + bgShepherd.rollbackAndClose(); + } + System.out.println("opensearchIndexDeep() backgrounding Encounter " + encId + + " finished."); + executor.shutdown(); + } + }; + System.out.println("opensearchIndexDeep() begin backgrounding indiv for " + this); + executor.execute(rn); + System.out.println("opensearchIndexDeep() [foreground] finished for Encounter " + encId); + } + // given a doc from opensearch, can user access it? public static boolean opensearchAccess(org.json.JSONObject doc, User user, Shepherd myShepherd) { diff --git a/src/main/java/org/ecocean/MarkedIndividual.java b/src/main/java/org/ecocean/MarkedIndividual.java index 5e0900931b..45e6a59df1 100644 --- a/src/main/java/org/ecocean/MarkedIndividual.java +++ b/src/main/java/org/ecocean/MarkedIndividual.java @@ -2840,8 +2840,8 @@ public void run() { int ct = 0; for (Encounter enc : indiv.getEncounters()) { ct++; - System.out.println("opensearchIndexDeep() background indexing " + - enc.getId() + " via " + indivId + " [" + ct + "/" + total + "]"); + System.out.println("opensearchIndexDeep() background indexing enc " + + enc.getId() + " via indiv " + indivId + " [" + ct + "/" + total + "]"); try { enc.opensearchIndex(); } catch (Exception ex) {