Merge remote-tracking branch 'origin/dev' into feature/issue3292

AtlasOfLivingAustralia · Sep 27, 2024 · 632a810 · 632a810
2 parents fc1094d + ea6ef23
commit 632a810
Show file tree

Hide file tree

Showing 13 changed files with 842 additions and 622 deletions.
diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy
@@ -1042,6 +1042,8 @@ license.default = "https://creativecommons.org/licenses/by-nc/3.0/au/"
 projectActivity.notifyOnChange=true
 biocollect.baseURL="https://biocollect.ala.org.au"
 biocollect.projectActivityDataURL="${biocollect.baseURL}/bioActivity/projectRecords"
+biocollect.projectArea.simplificationThreshold=10000
+biocollect.projectArea.simplificationTolerance=0.0001
 
 // elasticsearch cluster setting
 // can transport layer connection be made from apps outside JVM

diff --git a/grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy b/grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy
@@ -374,8 +374,8 @@ class ProjectController {
     }
 
     def importProjectsFromSciStarter(){
-        Integer count = projectService.importProjectsFromSciStarter()?:0
-        render(text: [count: count] as JSON, contentType: 'application/json');
+        Map counts = projectService.importProjectsFromSciStarter()
+        render(counts as JSON, contentType: 'application/json');
     }
 
     /**

diff --git a/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy b/grails-app/jobs/au/org/ala/ecodata/ImportSciStarterJob.groovy
@@ -0,0 +1,20 @@
+package au.org.ala.ecodata
+
+import grails.core.GrailsApplication
+
+class ImportSciStarterJob {
+
+    GrailsApplication grailsApplication
+    ProjectService projectService
+
+    static triggers = {
+        Boolean enabled = grailsApplication.config.getProperty("sciStarter.importEnabled", Boolean, true)
+        if (enabled) {
+            cron name: "every sunday", cronExpression: "0 0 0 ? * 1/7 *"
+        }
+    }
+
+    def execute() {
+        projectService.importProjectsFromSciStarter()
+    }
+}
diff --git a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy
@@ -1164,7 +1164,7 @@ class ElasticSearchService {
             // GeoServer requires a single attribute with project area. Cannot use `sites` property (above) since it has
             // all sites associated with project.
             // todo: Check if BioCollect requires all sites in `sites` property. If no, merge `projectArea` with `sites`.
-            projectMap.projectArea = siteService.get(project.projectSiteId, [SiteService.FLAT, SiteService.INDEXING])
+            projectMap.projectArea = siteService.getSimpleProjectArea(projectMap.projectSiteId)
             projectMap.containsActivity = activityService.searchAndListActivityDomainObjects([projectId: projectMap.projectId], null, null, null, [max: 1, offset: 0])?.totalCount > 0
         }
         projectMap.sites?.each { site ->

diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy
@@ -4,7 +4,7 @@ import au.org.ala.ecodata.converter.SciStarterConverter
 import grails.converters.JSON
 import grails.core.GrailsApplication
 import groovy.json.JsonSlurper
-import org.codehaus.jackson.map.ObjectMapper
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.context.MessageSource
 import org.springframework.web.servlet.i18n.SessionLocaleResolver
 
@@ -728,6 +728,17 @@ class ProjectService {
         }
     }
 
+    Map getSciStarterProjectsPage(JsonSlurper slurper, int page = 1) {
+        String baseUrl = grailsApplication.config.getProperty("scistarter.baseUrl")
+        String finderUrl = grailsApplication.config.getProperty("scistarter.finderUrl")
+        String apiKey = grailsApplication.config.getProperty("scistarter.apiKey")
+        String url = "${baseUrl}${finderUrl}?format=json&key=${apiKey}&page=${page}"
+
+        String data = webService.get(url, false)
+
+        return slurper.parseText(data)
+    }
+
     /**
      * Import SciStarter projects to Biocollect. Import script does the following.
      * 1. gets the list of projects and contacts SciStarter for more details on a project
@@ -736,50 +747,51 @@ class ProjectService {
      *      And link artifacts to the project. TODO: creating project extent.
      * @return
      */
-    Integer importProjectsFromSciStarter() {
-        int ignoredProjects = 0, createdProjects = 0, updatedProjects = 0
+    Map importProjectsFromSciStarter() {
+        int ignoredProjects = 0, createdProjects = 0, updatedProjects = 0, page = 1
+        JsonSlurper slurper = new JsonSlurper()
+
         log.info("Starting SciStarter import")
         try {
-            JsonSlurper jsonSlurper = new JsonSlurper()
-            String sciStarterProjectUrl
-            // list all SciStarter projects
-            List projects = getSciStarterProjectsFromFinder()
-            projects?.eachWithIndex { pProperties, index ->
-                Map transformedProject
-                Map project = pProperties
-                if (project && project.title && project.id) {
-                    Project importedSciStarterProject = Project.findByExternalIdAndIsSciStarter(project.id?.toString(), true)
-                    // get more details about the project
+
+            while(true) {
+                // list SciStarter projects for the current page
+                Map data = getSciStarterProjectsPage(slurper, page)
+                log.info("-- PAGE ${page}/${Math.round(data.total / 10)} SCISTARTER --")
+
+                // Break the loop if there are no more projects left to import
+                if (data.entities.size() == 0) break
+
+                data.entities.eachWithIndex { project, index ->
                     try {
-                        sciStarterProjectUrl = "${grailsApplication.config.getProperty('scistarter.baseUrl')}${grailsApplication.config.getProperty('scistarter.projectUrl')}/${project.id}?key=${grailsApplication.config.getProperty('scistarter.apiKey')}"
-                        String text = webService.get(sciStarterProjectUrl, false);
-                        if (text instanceof String) {
-                            Map projectDetails = jsonSlurper.parseText(text)
-                            if (projectDetails.origin && projectDetails.origin == 'atlasoflivingaustralia') {
-                                // ignore projects SciStarter imported from Biocollect
-                                log.warn("Ignoring ${projectDetails.title} - ${projectDetails.id} - This is an ALA project.")
-                                ignoredProjects++
+                        Project existingProject = Project.findByExternalIdAndIsSciStarter(project.legacy_id.toString(), true)
+
+                        if (project.origin == 'atlasoflivingaustralia') {
+                            // ignore projects SciStarter imported from BioCollect
+                            log.info("Ignoring ALA project ${project.name} - ${project.id}")
+                            ignoredProjects++
+                        } else {
+                            // map properties from SciStarter to Biocollect
+                            Map transformedProject = SciStarterConverter.convert(project)
+                            if (!existingProject) {
+                                // create project & document & site & organisation
+                                createSciStarterProject(transformedProject, project)
+                                log.info("Creating ${project.name} in ecodata")
+
+                                createdProjects++
                             } else {
-                                projectDetails << project
-                                // map properties from SciStarter to Biocollect
-                                transformedProject = SciStarterConverter.convert(projectDetails)
-                                if (!importedSciStarterProject) {
-                                    // create project & document & site & organisation
-                                    createSciStarterProject(transformedProject, projectDetails)
-                                    createdProjects++
-                                } else {
-                                    // update a project just in case something has changed.
-                                    updateSciStarterProject(transformedProject, importedSciStarterProject)
-                                    log.info("Updating ${importedSciStarterProject.name} ${importedSciStarterProject.projectId}.")
-                                    updatedProjects++
-                                }
+                                // update a project just in case something has changed.
+                                updateSciStarterProject(transformedProject, existingProject)
+
+                                log.info("Updating ${existingProject.name} ${existingProject.projectId}.")
+                                updatedProjects++
                             }
                         }
-                    } catch (Exception e) {
-                        log.error("Error processing project - ${sciStarterProjectUrl}. Ignoring it. ${e.message}", e);
-                        ignoredProjects++
+                    }  catch (Exception e) {
+                        log.error("Error processing project - ${project.name}. Ignoring it. ${e.message}", e);
                     }
                 }
+                page++
             }
 
             log.info("Number of created projects ${createdProjects}. Number of ignored projects ${ignoredProjects}. Number of projects updated ${updatedProjects}.")
@@ -790,23 +802,7 @@ class ProjectService {
         }
 
         log.info("Completed SciStarter import")
-        createdProjects
-    }
-
-    /**
-     * Get the entire project list from SciStarter
-     * @return
-     * @throws SocketTimeoutException
-     * @throws Exception
-     */
-    List getSciStarterProjectsFromFinder() throws SocketTimeoutException, Exception {
-        String scistarterFinderUrl = "${grailsApplication.config.getProperty('scistarter.baseUrl')}${grailsApplication.config.getProperty('scistarter.finderUrl')}?format=json&q="
-        String responseText = webService.get(scistarterFinderUrl, false)
-        if (responseText instanceof String) {
-            ObjectMapper mapper = new ObjectMapper()
-            Map response = mapper.readValue(responseText, Map.class)
-            return response.results
-        }
+        [created: createdProjects, updated: updatedProjects, ignored: ignoredProjects]
     }
 
     /**
@@ -887,16 +883,14 @@ class ProjectService {
     Map createSciStarterSites(Map project) {
         Map result = [siteIds: null]
         List sites = []
-        if (project.regions?.size()) {
+        if (project.regions) {
             // convert region to site
-            project.regions.each { region ->
-                Map site = SciStarterConverter.siteMapping(region)
-                // only add valid geojson objects
-                if (site?.extent?.geometry && siteService.isGeoJsonValid((site?.extent?.geometry as JSON).toString())) {
-                    Map createdSite = siteService.create(site)
-                    if (createdSite.siteId) {
-                        sites.push(createdSite.siteId)
-                    }
+            Map site = SciStarterConverter.siteMapping(project)
+            // only add valid geojson objects
+            if (site?.extent?.geometry && siteService.isGeoJsonValid((site?.extent?.geometry as JSON).toString())) {
+                Map createdSite = siteService.create(site)
+                if (createdSite.siteId) {
+                    sites.push(createdSite.siteId)
                 }
             }
 

diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy
@@ -255,6 +255,32 @@ class SiteService {
         site
     }
 
+    def getSimpleProjectArea(projectSiteId) {
+        def threshold = grailsApplication.config.getProperty('biocollect.projectArea.simplificationThreshold', Integer, 10000)
+        def tolerance = grailsApplication.config.getProperty('biocollect.projectArea.simplificationTolerance', Double, 0.0001)
+
+        def site = get(projectSiteId, [SiteService.FLAT, SiteService.INDEXING])
+
+        try {
+            if (site != null) {
+                def projectArea = geometryAsGeoJson(site)
+
+                if (projectArea?.coordinates != null) {
+                    def coordsSize = projectArea.coordinates.flatten().size()
+                    if (coordsSize > threshold) {
+                        site.geoIndex = GeometryUtils.simplify(projectArea, tolerance)
+                    } else {
+                        site.geoIndex = projectArea
+                    }
+                }
+            }
+        } catch (Exception e) {
+            log.info("Unable to get simplified project area geometry (site ${site.siteId}")
+        }
+
+        site
+    }
+
     def create(props) {
       //  assert getCommonService()
         def site = new Site(siteId: Identifiers.getNew(true,''))