From bcb3aef1b899836edcf82cd675bd6bbc516c7a2a Mon Sep 17 00:00:00 2001 From: Julien Perret Date: Mon, 23 Sep 2024 16:38:19 +0200 Subject: [PATCH] better DataStore handling (keep the outputdatastore open in case of reuse) --- .../h24/tools/ExtractRelevantData.scala | 50 +++++++++---------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/src/main/scala/eighties/h24/tools/ExtractRelevantData.scala b/src/main/scala/eighties/h24/tools/ExtractRelevantData.scala index 287cee9..138394c 100644 --- a/src/main/scala/eighties/h24/tools/ExtractRelevantData.scala +++ b/src/main/scala/eighties/h24/tools/ExtractRelevantData.scala @@ -5,7 +5,7 @@ import java.util.NoSuchElementException import com.github.tototoshi.csv.CSVWriter import org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream import org.apache.poi.ss.usermodel.{Cell, CellType, Row, Sheet} -import org.geotools.data.{DataStoreFinder, Query, Transaction} +import org.geotools.data.{DataStore, DataStoreFinder, Query, Transaction} import org.geotools.data.shapefile.{ShapefileDataStore, ShapefileDataStoreFactory} import org.geotools.factory.CommonFactoryFinder import org.geotools.geometry.jts.JTS @@ -58,14 +58,7 @@ import java.util .text("output directory") ) } - def filterShape(file: File, filter: SimpleFeature => Boolean, outputFile: File): Unit = { - val params = new util.HashMap[String, Serializable]() - params.put("url", file.toURI.toURL) - val inputDataStore = DataStoreFinder.getDataStore(params) - val factory = new ShapefileDataStoreFactory - if outputFile.exists then outputFile.delete() - val outputDataStore = factory.createDataStore(outputFile.toURI.toURL) - val typeName = outputDataStore.getTypeNames()(0) + def filterShape(inputDataStore: DataStore, typeName: String, filter: SimpleFeature => Boolean, outputDataStore: DataStore): Unit = { outputDataStore.createSchema(inputDataStore.getSchema(typeName)) val writer = outputDataStore.getFeatureWriterAppend(typeName, Transaction.AUTO_COMMIT) try @@ -78,20 +71,8 @@ import java.util if filter(feature) then writer.next.setAttributes(feature.getAttributes) writer.write() - /* - val featureReader = Iterator.continually(reader.next).takeWhile(_ => reader.hasNext) - Log.log("Feature Reader is ready ...") - featureReader.filter(feature=>filter(feature)).foreach{ - feature=> -// Log.log("Write ...") - writer.next.setAttributes(feature.getAttributes) - writer.write() - } - */ writer.close() Log.log("Write close") - outputDataStore.dispose() - Log.log("dataStore dispose") catch case e: IOException => println("Had an IOException trying reading this file: " + e) case e: FileNotFoundException => println("File Not Found: " + e) @@ -139,19 +120,35 @@ import java.util // extract the relevant data from the input files and put them in the output directory val outputContourFile = new java.io.File(outputDirectory, config.contour.get.getName) Log.log("outputContourFile = " + outputContourFile) - filterShape(config.contour.get, (feature:SimpleFeature)=>if (config.deps.isEmpty) true else config.deps.get.contains(feature.getAttribute("DEPCOM").toString.trim.substring(0,2)), outputContourFile) - val store = new ShapefileDataStore(outputContourFile.toURI.toURL) + val factory = new ShapefileDataStoreFactory + def getInputDataStore(file: File): (DataStore, String) = + val params = new util.HashMap[String, Serializable]() + params.put("url", file.toURI.toURL) + val store = DataStoreFinder.getDataStore(params) + (store, store.getTypeNames()(0)) + def getOutputDataStore(file: File): DataStore = + if file.exists then file.delete() + factory.createDataStore(file.toURI.toURL) + val (contourStore, contourTypeName) = getInputDataStore(config.contour.get) + val outputContourStore = getOutputDataStore(outputContourFile) + def featureInDep(feature:SimpleFeature) = if (config.deps.isEmpty) true else config.deps.get.contains(feature.getAttribute("DEPCOM").toString.trim.substring(0,2)) + filterShape(contourStore, contourTypeName, featureInDep, outputContourStore) + // we keep the outputContourStore open to filter grid cells val ff = CommonFactoryFinder.getFilterFactory2() - val featureSource = store.getFeatureSource + val featureSource = outputContourStore.getFeatureSource(contourTypeName) val l2eCRS = CRS.decode("EPSG:27572") val l3CRS = CRS.decode("EPSG:2154") val transform = CRS.findMathTransform(l2eCRS, l3CRS, true) val outputGridFile = new java.io.File(outputDirectory, config.grid.get.getName) Log.log("outputGridFile = " + outputGridFile) Log.log("filtershape") - filterShape(config.grid.get, (feature:SimpleFeature)=>{!featureSource.getFeatures(ff.intersects(ff.property(store.getSchema.getGeometryDescriptor.getLocalName), ff.literal(JTS.transform(feature.getDefaultGeometry.asInstanceOf[Geometry], transform)))).isEmpty}, outputGridFile) + val (gridStore, gridTypeName) = getInputDataStore(config.grid.get) + val outputGridStore = getOutputDataStore(outputGridFile) + def featureIntersectsContours(feature:SimpleFeature) = {!featureSource.getFeatures(ff.intersects(ff.property(featureSource.getSchema.getGeometryDescriptor.getLocalName), ff.literal(JTS.transform(feature.getDefaultGeometry.asInstanceOf[Geometry], transform)))).isEmpty} + filterShape(gridStore, gridTypeName, featureIntersectsContours, outputGridStore) Log.log("store.dispose start") - store.dispose() + outputContourStore.dispose() + outputGridStore.dispose() Log.log("store.dispose end") val outputInfraPopulationFile = new java.io.File(outputDirectory, config.infraPopulation.get.getName.substring(0, config.infraPopulation.get.getName.lastIndexOf("."))+".csv.lzma") Log.log("outputInfraPopulationFile = "+outputInfraPopulationFile) @@ -162,5 +159,4 @@ import java.util Log.log("done") case _ => } - }