From f30e99ddbfc3d8edb052dcad4ab55e2510978cee Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Mon, 30 Sep 2024 15:30:59 +1000 Subject: [PATCH 1/9] SciStarter import job is now configurable --- grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy b/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy index eec02b3c1..b861010cd 100644 --- a/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy +++ b/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy @@ -10,7 +10,7 @@ class ImportSciStarterJob { static triggers = { Boolean enabled = grailsApplication.config.getProperty("sciStarter.importEnabled", Boolean, true) if (enabled) { - cron name: "every sunday", cronExpression: "0 0 0 ? * 1/7 *" + cron name: "every sunday", cronExpression: grailsApplication.config.getProperty("scistarter.cronSchedule", "0 0 0 ? * 1/7 *") } } From c3e98cc40576b7e312980d1a18960d4a6d470778 Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Tue, 1 Oct 2024 17:14:03 +1000 Subject: [PATCH 2/9] Removed worldExtent project site default for sciStarter projects without a project area --- grails-app/conf/data/worldExtent.json | 97 ------------------- .../au/org/ala/ecodata/ProjectService.groovy | 20 +--- 2 files changed, 1 insertion(+), 116 deletions(-) delete mode 100644 grails-app/conf/data/worldExtent.json diff --git a/grails-app/conf/data/worldExtent.json b/grails-app/conf/data/worldExtent.json deleted file mode 100644 index 126a98969..000000000 --- a/grails-app/conf/data/worldExtent.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "projects" : [ - ], - "isSciStarter":true, - "status" : "active", - "poi" : [ ], - "geoIndex" : { - "type" : "Polygon", - "coordinates" : [ - [ - [ - -180, - -90 - ], - [ - -180, - 90 - ], - [ - 180, - 90 - ], - [ - 180, - -90 - ], - [ - -180, - -90 - ] - ] - ] - }, - "extent" : { - "source" : "drawn", - "geometry" : { - "fid" : "", - "mvs" : null, - "precision" : "", - "centre" : [ - "11.601562499999986", - "-0.7859697100237464" - ], - "state" : null, - "layerName" : "", - "pid" : null, - "datum" : "", - "type" : "Polygon", - "decimalLatitude" : -0.7859697100237497, - "areaKmSq" : 509700727.0660377, - "nrm" : null, - "bbox" : "", - "mvg" : null, - "name" : "", - "decimalLongitude" : 11.601562499999986, - "locality" : "Lolo-Bouenguidi, Gabon", - "lga" : null, - "uncertainty" : "", - "coordinates" : [ - [ - [ - -180, - -90 - ], - [ - -180, - 90 - ], - [ - 180, - 90 - ], - [ - 180, - -90 - ], - [ - -180, - -90 - ] - ] - ], - "ibra" : null, - "imcra4_pb" : null, - "elect" : null, - "cmz" : null, - "other" : [ ], - "gerSubRegion" : [ ] - } - }, - "externalId" : "", - "area" : "5.097007270660377E8", - "description" : "Default map for SciStarter projects without a predefined region.", - "name" : "SciStarter Project Area", - "notes" : "", - "type" : "projectArea" -} \ No newline at end of file diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 47af6127f..3ebe19209 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -881,7 +881,7 @@ class ProjectService { * @return */ Map createSciStarterSites(Map project) { - Map result = [siteIds: null] + Map result = [siteIds: []] List sites = [] if (project.regions) { // convert region to site @@ -895,12 +895,6 @@ class ProjectService { } result.siteIds = sites - } else { - // if no region, then create world extent. - String siteId = getWorldExtent() - if (siteId) { - result.siteIds = [siteId] - } } result @@ -977,18 +971,6 @@ class ProjectService { } } - /** - * World extent is used as project area of all SciStarter Projects without project area. This function - * creates a new world extent every time it is called. - * @return - siteId - 'abcd-sds' - */ - String getWorldExtent() { - // use JSON.parse since JSONSlurper converts numbers to BigDecimal which throws error on serialization. - Object world = JSON.parse(getClass().getResourceAsStream("/data/worldExtent.json")?.getText()) - Map site = siteService.create(world) - return site.siteId - } - /** * Get a String that describes a project. This used for indexing purposes currently. * The motivaion for this method was because there was no single field to distinguish between projects but had multiple From 80d510a81b514005c976a87675a865661c601f4e Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Wed, 2 Oct 2024 10:07:17 +1000 Subject: [PATCH 3/9] Temporarily disabled SciStarter site indexing --- .../au/org/ala/ecodata/ProjectService.groovy | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 3ebe19209..24b262983 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -814,13 +814,13 @@ class ProjectService { Map organisation // create project extent - Map sites = createSciStarterSites(rawProp) - String projectSiteId - if (sites?.siteIds?.size()) { - projectSiteId = sites.siteIds[0] - } - - transformedProp.projectSiteId = projectSiteId +// Map sites = createSciStarterSites(rawProp) +// String projectSiteId +// if (sites?.siteIds?.size()) { +// projectSiteId = sites.siteIds[0] +// } +// +// transformedProp.projectSiteId = projectSiteId // create organisation if (transformedProp.organisationName) { @@ -842,9 +842,9 @@ class ProjectService { // use the projectId to associate site with project if (projectId) { - sites?.siteIds?.each { siteId -> - siteService.addProject(siteId, projectId) - } +// sites?.siteIds?.each { siteId -> +// siteService.addProject(siteId, projectId) +// } // create project logo. createSciStarterLogo(imageUrl, attribution, projectId) From 20cc0efe1a3b6bddbe84e78c97b6d0b6ac257710 Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Wed, 2 Oct 2024 13:42:59 +1000 Subject: [PATCH 4/9] Revert "Temporarily disabled SciStarter site indexing" This reverts commit 80d510a81b514005c976a87675a865661c601f4e. --- .../au/org/ala/ecodata/ProjectService.groovy | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 24b262983..3ebe19209 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -814,13 +814,13 @@ class ProjectService { Map organisation // create project extent -// Map sites = createSciStarterSites(rawProp) -// String projectSiteId -// if (sites?.siteIds?.size()) { -// projectSiteId = sites.siteIds[0] -// } -// -// transformedProp.projectSiteId = projectSiteId + Map sites = createSciStarterSites(rawProp) + String projectSiteId + if (sites?.siteIds?.size()) { + projectSiteId = sites.siteIds[0] + } + + transformedProp.projectSiteId = projectSiteId // create organisation if (transformedProp.organisationName) { @@ -842,9 +842,9 @@ class ProjectService { // use the projectId to associate site with project if (projectId) { -// sites?.siteIds?.each { siteId -> -// siteService.addProject(siteId, projectId) -// } + sites?.siteIds?.each { siteId -> + siteService.addProject(siteId, projectId) + } // create project logo. createSciStarterLogo(imageUrl, attribution, projectId) From 9ccce4064c0cb643f46735963ff9b7eba4c09489 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 2 Oct 2024 14:12:37 +1000 Subject: [PATCH 5/9] Revert "Removed worldExtent project site default for sciStarter projects without a project area" This reverts commit c3e98cc40576b7e312980d1a18960d4a6d470778. --- grails-app/conf/data/worldExtent.json | 97 +++++++++++++++++++ .../au/org/ala/ecodata/ProjectService.groovy | 20 +++- 2 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 grails-app/conf/data/worldExtent.json diff --git a/grails-app/conf/data/worldExtent.json b/grails-app/conf/data/worldExtent.json new file mode 100644 index 000000000..126a98969 --- /dev/null +++ b/grails-app/conf/data/worldExtent.json @@ -0,0 +1,97 @@ +{ + "projects" : [ + ], + "isSciStarter":true, + "status" : "active", + "poi" : [ ], + "geoIndex" : { + "type" : "Polygon", + "coordinates" : [ + [ + [ + -180, + -90 + ], + [ + -180, + 90 + ], + [ + 180, + 90 + ], + [ + 180, + -90 + ], + [ + -180, + -90 + ] + ] + ] + }, + "extent" : { + "source" : "drawn", + "geometry" : { + "fid" : "", + "mvs" : null, + "precision" : "", + "centre" : [ + "11.601562499999986", + "-0.7859697100237464" + ], + "state" : null, + "layerName" : "", + "pid" : null, + "datum" : "", + "type" : "Polygon", + "decimalLatitude" : -0.7859697100237497, + "areaKmSq" : 509700727.0660377, + "nrm" : null, + "bbox" : "", + "mvg" : null, + "name" : "", + "decimalLongitude" : 11.601562499999986, + "locality" : "Lolo-Bouenguidi, Gabon", + "lga" : null, + "uncertainty" : "", + "coordinates" : [ + [ + [ + -180, + -90 + ], + [ + -180, + 90 + ], + [ + 180, + 90 + ], + [ + 180, + -90 + ], + [ + -180, + -90 + ] + ] + ], + "ibra" : null, + "imcra4_pb" : null, + "elect" : null, + "cmz" : null, + "other" : [ ], + "gerSubRegion" : [ ] + } + }, + "externalId" : "", + "area" : "5.097007270660377E8", + "description" : "Default map for SciStarter projects without a predefined region.", + "name" : "SciStarter Project Area", + "notes" : "", + "type" : "projectArea" +} \ No newline at end of file diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index 3ebe19209..47af6127f 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -881,7 +881,7 @@ class ProjectService { * @return */ Map createSciStarterSites(Map project) { - Map result = [siteIds: []] + Map result = [siteIds: null] List sites = [] if (project.regions) { // convert region to site @@ -895,6 +895,12 @@ class ProjectService { } result.siteIds = sites + } else { + // if no region, then create world extent. + String siteId = getWorldExtent() + if (siteId) { + result.siteIds = [siteId] + } } result @@ -971,6 +977,18 @@ class ProjectService { } } + /** + * World extent is used as project area of all SciStarter Projects without project area. This function + * creates a new world extent every time it is called. + * @return - siteId - 'abcd-sds' + */ + String getWorldExtent() { + // use JSON.parse since JSONSlurper converts numbers to BigDecimal which throws error on serialization. + Object world = JSON.parse(getClass().getResourceAsStream("/data/worldExtent.json")?.getText()) + Map site = siteService.create(world) + return site.siteId + } + /** * Get a String that describes a project. This used for indexing purposes currently. * The motivaion for this method was because there was no single field to distinguish between projects but had multiple From a0a9f340645e3b8a64386fc9a5c703aa169df8a4 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 2 Oct 2024 14:14:23 +1000 Subject: [PATCH 6/9] fix for total field index limit on ES --- grails-app/services/au/org/ala/ecodata/SiteService.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy index 06cc655fe..ee25d59ed 100644 --- a/grails-app/services/au/org/ala/ecodata/SiteService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SiteService.groovy @@ -278,6 +278,8 @@ class SiteService { log.info("Unable to get simplified project area geometry (site ${site.siteId}") } + // remove extent to not avoid total fields limit in ES + site?.remove('extent') site } From e212fbf8d9e15e4c22074de74436af4503eaa14a Mon Sep 17 00:00:00 2001 From: Jack Brinkman Date: Wed, 2 Oct 2024 17:14:30 +1000 Subject: [PATCH 7/9] Improved logging of getSimpleProjectArea errors --- grails-app/services/au/org/ala/ecodata/SiteService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy index ee25d59ed..7e656c58c 100644 --- a/grails-app/services/au/org/ala/ecodata/SiteService.groovy +++ b/grails-app/services/au/org/ala/ecodata/SiteService.groovy @@ -275,7 +275,7 @@ class SiteService { } } } catch (Exception e) { - log.info("Unable to get simplified project area geometry (site ${site.siteId}") + log.info("Unable to get simplified project area geometry (site ${site.siteId})", e) } // remove extent to not avoid total fields limit in ES From 31a60695e3370283df6429a06bebbdc263d46b38 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 3 Oct 2024 07:05:06 +1000 Subject: [PATCH 8/9] fix for long string --- src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy index 0d73edeee..c194aea87 100644 --- a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy +++ b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy @@ -29,7 +29,8 @@ class GeometryUtils { static Map wktToGeoJson(String wkt, int decimals = 20) { WKTReader wktReader = new WKTReader() - Geometry geom = wktReader.read(wkt) + def reader = new StringReader(wkt) + Geometry geom = wktReader.read(reader) String geoJSON = new GeometryJSON(decimals).toString(geom) ObjectMapper mapper = new ObjectMapper() return mapper.readValue(geoJSON, Map) From ab4abdeefb9b4683d76cb323d9cc5d859b9be6bc Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 3 Oct 2024 09:44:24 +1000 Subject: [PATCH 9/9] fix for toString exception --- src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy index c194aea87..bda514e80 100644 --- a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy +++ b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy @@ -18,6 +18,8 @@ import org.opengis.referencing.crs.CoordinateReferenceSystem import org.opengis.referencing.operation.MathTransform import java.awt.geom.Point2D +import java.nio.charset.StandardCharsets + /** * Helper class for working with site geometry. */ @@ -29,11 +31,13 @@ class GeometryUtils { static Map wktToGeoJson(String wkt, int decimals = 20) { WKTReader wktReader = new WKTReader() + ByteArrayOutputStream outputStream = new ByteArrayOutputStream() + OutputStreamWriter writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8) def reader = new StringReader(wkt) Geometry geom = wktReader.read(reader) - String geoJSON = new GeometryJSON(decimals).toString(geom) + new GeometryJSON(decimals).write(geom, writer) ObjectMapper mapper = new ObjectMapper() - return mapper.readValue(geoJSON, Map) + return mapper.readValue(outputStream.toByteArray(), Map) } static String wktToMultiPolygonWkt(String wkt) {