Merge pull request #106 from azavea/feature/dpd/vector-datasets

Add basic support for vector datasets
azavea · Nov 11, 2022 · af2cd07 · af2cd07
2 parents ff0a086 + 1aff650
commit af2cd07
Show file tree

Hide file tree

Showing 20 changed files with 480 additions and 75 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,4 +1,7 @@
 ## UPCOMING
+- Add support for opening vector datasets
+- Add GDALDataset.layerCount() to wrap GDALDatasetGetLayerCount()
+- Add GDALDataset.vectorConvert() to wrap GDALVectorTranslate()
 
 ## 1.1.2 (2022-10-12)
 - Allow users to specify a valid absolute URL when the default URL is invalid

diff --git a/README.md b/README.md
@@ -65,10 +65,12 @@ A promise that resolves when Loam is initialized. All of the functions described
 
 <br />
 
-### `loam.open(file)`
+### `loam.open(file, sidecars)`
 Creates a new GDAL Dataset.
 #### Parameters
-- `file`: A Blob or File object that should be opened with GDAL. GDAL is compiled with TIFF, PNG, and JPEG support.
+- `file`: A Blob or File object that should be opened with GDAL. GDAL is compiled with TIFF, PNG, and JPEG support. If you have a Blob, you may also control the name of the file that is shown to GDAL on the virtual filesystem by passing an object with the shape `{name: string, data: Blob}`. This can be useful if you are relying on GDAL behavior that uses file extensions to determine formats.
+- `sidecars`: An array of additional files that will be made present in the virtual file system when opening `file`. Some data formats are composed of multiple files (for example, Shapefiles have `.shp`, `.shx`, and `.prj` files, among others). If you need to include multiple files in order to open a dataset, pass the "main" file as `file`, and pass the others to `sidecars`. For a Shapefile, this would mean passing the `.shp` file as `file` and the `.shx`, `.prj`, and friends to `sidecars`. If `file` is a File, then `sidecars` must be an Array\<File>. If `file` is a Blob or Object (see above), then `sidecars` must be an Array\<Object> where each element has the shape `{name: string, data: Blob}`.
+
 #### Return value
 A promise that resolves with an instance of `GDALDataset`.
 
@@ -116,9 +118,16 @@ A promise that resolves immediately with an empty list (for historical reasons).
 <br />
 
 ### `GDALDataset.count()`
-Get the number of bands in the dataset.
+Get the number of raster bands in the dataset.
+#### Return value
+A promise which resolves to the number of raster bands in the dataset.
+
+<br />
+
+### `GDALDataset.layerCount()`
+Get the number of vector layers in the dataset.
 #### Return value
-A promise which resolves to the number of bands in the dataset.
+A promise which resolves to the number of vector layers in the dataset.
 
 <br />
 
@@ -168,6 +177,17 @@ A promise that resolves to a new `GDALDataset`.
 
 <br />
 
+### `GDALDataset.vectorConvert(args)`
+Converts vector data between different formats. This is the equivalent of the [ogr2ogr](https://gdal.org/programs/ogr2ogr.html) command.
+
+**Note**: This returns a new `GDALDataset` object but does not perform any immediate calculation. Instead, calls to `.vectorConvert()` are evaluated lazily. Each successive call to `.vectorConvert()` is stored in a list of operations on the dataset object. These operations are only evaluated when necessary in order to access some property of the dataset, such as its size, bytes, or layer count.
+#### Parameters
+- `args`: An array of strings, each representing a single command-line argument accepted by the `ogr2ogr` command. The `dst_datasource_name` and `src_datasource_name` parameters should be omitted; these are handled by `GDALDataset`. Example: `ds.vectorConvert(['-f', 'GeoJSON'])`.
+#### Return value
+A promise that resolves to a new `GDALDataset`.
+
+<br />
+
 ### `GDALDataset.warp(args)`
 Image reprojection and warping utility. This is the equivalent of the [gdalwarp](https://gdal.org/programs/gdalwarp.html) command.
 

diff --git a/demo/index.html b/demo/index.html
@@ -4,14 +4,25 @@
         <title>Loam Test</title>
         <script type="text/javascript" src="/loam.js"></script>
     </head>
+
     <body>
-        <form>
-            <input type="file" id="geotiff-file" />
-        </form>
         <p>
-            Select a GeoTIFF using the Browse... button. Information about the file will be
-            displayed below.
+            Select a file using the Source file input and then click "Display metadata". Metadata
+            about the file will be displayed below. GeoTIFFs work best, but GeoJSON and Shapefiles
+            will work to some extent as well. Make sure to select sidecar files when relevant (e.g.
+            .prj, .dbf, .shx, etc. for Shapefiles).
         </p>
+        <form>
+            <label>
+                Source file:
+                <input type="file" id="source-file" />
+            </label>
+            <label>
+                Additional files:
+                <input type="file" id="sidecar-files" multiple />
+            </label>
+            <button type="button" id="display-metadata-button">Display metadata</button>
+        </form>
 
         <div id="gdalinfo"></div>
         <script src="index.js"></script>

diff --git a/demo/index.js b/demo/index.js
@@ -1,60 +1,67 @@
 /* global loam */
 
 // Use the locally built version of loam, with a CDN copy of GDAL from unpkg.
-loam.initialize('/', 'https://unpkg.com/gdal-js@2.0.0/');
+loam.initialize('/', 'https://unpkg.com/gdal-js@2.1.0/');
 
 const EPSG4326 =
     'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.01745329251994328,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]';
 
 function displayInfo() {
-    const file = document.querySelector('#geotiff-file').files[0];
+    const sourceFile = document.querySelector('#source-file').files[0];
+    const sidecars = Array.from(document.querySelector('#sidecar-files').files);
+
     const displayElem = document.getElementById('gdalinfo');
 
     // Clear display text
     displayElem.innerText = '';
     // Use Loam to get GeoTIFF metadata
-    loam.open(file).then((ds) => {
-        return Promise.all([ds.width(), ds.height(), ds.count(), ds.wkt(), ds.transform()]).then(
-            ([width, height, count, wkt, geoTransform]) => {
-                displayElem.innerText +=
-                    'Size: ' + width.toString() + ', ' + height.toString() + '\n';
-                displayElem.innerText += 'Band count: ' + count.toString() + '\n';
-                displayElem.innerText += 'Coordinate system:\n' + wkt + '\n';
-
-                const cornersPx = [
-                    [0, 0],
-                    [width, 0],
-                    [width, height],
-                    [0, height],
+    loam.open(sourceFile, sidecars).then((ds) => {
+        return Promise.all([
+            ds.width(),
+            ds.height(),
+            ds.count(),
+            ds.layerCount(),
+            ds.wkt(),
+            ds.transform(),
+        ]).then(([width, height, count, layerCount, wkt, geoTransform]) => {
+            displayElem.innerText += 'Size: ' + width.toString() + ', ' + height.toString() + '\n';
+            displayElem.innerText += 'Raster band count: ' + count.toString() + '\n';
+            displayElem.innerText += 'Vector layer count: ' + layerCount.toString() + '\n';
+            displayElem.innerText += 'Coordinate system:\n' + wkt + '\n';
+
+            const cornersPx = [
+                [0, 0],
+                [width, 0],
+                [width, height],
+                [0, height],
+            ];
+            const cornersGeo = cornersPx.map(([x, y]) => {
+                return [
+                    // http://www.gdal.org/gdal_datamodel.html
+                    geoTransform[0] + geoTransform[1] * x + geoTransform[2] * y,
+                    geoTransform[3] + geoTransform[4] * x + geoTransform[5] * y,
                 ];
-                const cornersGeo = cornersPx.map(([x, y]) => {
-                    return [
-                        // http://www.gdal.org/gdal_datamodel.html
-                        geoTransform[0] + geoTransform[1] * x + geoTransform[2] * y,
-                        geoTransform[3] + geoTransform[4] * x + geoTransform[5] * y,
-                    ];
-                });
+            });
 
-                loam.reproject(wkt, EPSG4326, cornersGeo).then((cornersLngLat) => {
-                    displayElem.innerText += 'Corner Coordinates:\n';
-                    cornersLngLat.forEach(([lng, lat], i) => {
-                        displayElem.innerText +=
-                            '(' +
-                            cornersGeo[i][0].toString() +
-                            ', ' +
-                            cornersGeo[i][1].toString() +
-                            ') (' +
-                            lng.toString() +
-                            ', ' +
-                            lat.toString() +
-                            ')\n';
-                    });
+            loam.reproject(wkt, EPSG4326, cornersGeo).then((cornersLngLat) => {
+                displayElem.innerText += 'Corner Coordinates:\n';
+                cornersLngLat.forEach(([lng, lat], i) => {
+                    displayElem.innerText +=
+                        '(' +
+                        cornersGeo[i][0].toString() +
+                        ', ' +
+                        cornersGeo[i][1].toString() +
+                        ') (' +
+                        lng.toString() +
+                        ', ' +
+                        lat.toString() +
+                        ')\n';
                 });
-            }
-        );
+            });
+        });
     });
 }
 
-document.getElementById('geotiff-file').onchange = function () {
+document.getElementById('display-metadata-button').onclick = function () {
     displayInfo();
 };
diff --git a/package.json b/package.json
@@ -7,7 +7,7 @@
         "build": "webpack --config=webpack.dev.js && webpack --config=webpack.prod.js",
         "dev": "webpack --progress --color --watch --config=webpack.dev.js",
         "demo": "webpack serve --config=webpack.dev.js",
-        "format": "prettier --write ./src",
+        "format": "prettier --write ./src ./test ./demo",
         "test": "karma start --single-run --browser ChromeHeadless karma.conf.js",
         "test:watch": "karma start --auto-watch --browser ChromeHeadless karma.conf.js",
         "test:ci": "prettier --check src/**/*.js && webpack --config=webpack.dev.js && webpack --config=webpack.prod.js && karma start --single-run --browser ChromeHeadless karma.conf.js"
@@ -58,7 +58,7 @@
         "yargs": "^17.5.1"
     },
     "dependencies": {
-        "gdal-js": "2.0.0"
+        "gdal-js": "2.1.0"
     },
     "packageManager": "[email protected]"
 }
diff --git a/src/api.js b/src/api.js
@@ -1,9 +1,9 @@
 import { initWorker, clearWorker, runOnWorker } from './workerCommunication.js';
-import { GDALDataset } from './gdalDataset.js';
+import { DatasetSource, GDALDataset } from './gdalDataset.js';
 
-function open(file) {
+function open(file, sidecars = []) {
     return new Promise((resolve, reject) => {
-        const ds = new GDALDataset({ func: 'GDALOpen', src: file, args: [] });
+        const ds = new GDALDataset(new DatasetSource('GDALOpen', file, sidecars));
 
         return ds.open().then(
             () => resolve(ds),
@@ -14,7 +14,7 @@ function open(file) {
 
 function rasterize(geojson, args) {
     return new Promise((resolve, reject) => {
-        resolve(new GDALDataset({ func: 'GDALRasterize', src: geojson, args: args }));
+        resolve(new GDALDataset(new DatasetSource('GDALRasterize', geojson, [], args)));
     });
 }
 

diff --git a/src/gdalDataset.js b/src/gdalDataset.js
@@ -8,6 +8,22 @@ export class DatasetOperation {
     }
 }
 
+// The starting point of a dataset within a GDAL webworker context. Outputs a dataset.
+// - functionName is the name of a GDAL function that can generate a dataset
+//   Currently, this is either GDALOpen or GDALRasterize
+// - file is a File, Blob, or {name: string, data: Blob}
+// - sidecars are any other files that need to get loaded into the worker filesystem alongside file
+//   Must match the type of file
+// - args are any additional arguments that should be passed to functionName.
+export class DatasetSource {
+    constructor(functionName, file, sidecars = [], args = []) {
+        this.func = functionName;
+        this.src = file;
+        this.sidecars = sidecars;
+        this.args = args;
+    }
+}
+
 export class GDALDataset {
     constructor(source, operations) {
         this.source = source;
@@ -32,6 +48,10 @@ export class GDALDataset {
         return accessFromDataset('GDALGetRasterCount', this);
     }
 
+    layerCount() {
+        return accessFromDataset('GDALDatasetGetLayerCount', this);
+    }
+
     width() {
         return accessFromDataset('GDALGetRasterXSize', this);
     }
@@ -59,6 +79,17 @@ export class GDALDataset {
         });
     }
 
+    vectorConvert(args) {
+        return new Promise((resolve, reject) => {
+            resolve(
+                new GDALDataset(
+                    this.source,
+                    this.operations.concat(new DatasetOperation('GDALVectorTranslate', args))
+                )
+            );
+        });
+    }
+
     warp(args) {
         return new Promise((resolve, reject) => {
             resolve(

diff --git a/src/worker.js b/src/worker.js
@@ -6,13 +6,15 @@
 import wGDALOpen from './wrappers/gdalOpen.js';
 import wGDALRasterize from './wrappers/gdalRasterize.js';
 import wGDALClose from './wrappers/gdalClose.js';
+import wGDALDatasetGetLayerCount from './wrappers/gdalDatasetGetLayerCount.js';
 import wGDALDEMProcessing from './wrappers/gdalDemProcessing.js';
 import wGDALGetRasterCount from './wrappers/gdalGetRasterCount.js';
 import wGDALGetRasterXSize from './wrappers/gdalGetRasterXSize.js';
 import wGDALGetRasterYSize from './wrappers/gdalGetRasterYSize.js';
 import wGDALGetProjectionRef from './wrappers/gdalGetProjectionRef.js';
 import wGDALGetGeoTransform from './wrappers/gdalGetGeoTransform.js';
 import wGDALTranslate from './wrappers/gdalTranslate.js';
+import wGDALVectorTranslate from './wrappers/gdalVectorTranslate.js';
 import wGDALWarp from './wrappers/gdalWarp.js';
 import wReproject from './wrappers/reproject.js';
 
@@ -74,7 +76,13 @@ self.Module = {
         // C returns a pointer to a GDALDataset, we need to use 'number'.
         //
         registry.GDALOpen = wGDALOpen(
-            self.Module.cwrap('GDALOpen', 'number', ['string']),
+            self.Module.cwrap('GDALOpenEx', 'number', [
+                'string', // Filename
+                'number', // nOpenFlags
+                'number', // NULL-terminated list of drivers to limit to when opening file
+                'number', // NULL-terminated list of option flags passed to drivers
+                'number', // Paths to sibling files to avoid file system searches
+            ]),
             errorHandling,
             DATASETPATH
         );
@@ -97,6 +105,10 @@ self.Module = {
             self.Module.cwrap('GDALGetRasterCount', 'number', ['number']),
             errorHandling
         );
+        registry.GDALDatasetGetLayerCount = wGDALDatasetGetLayerCount(
+            self.Module.cwrap('GDALDatasetGetLayerCount', 'number', ['number']),
+            errorHandling
+        );
         registry.GDALGetRasterXSize = wGDALGetRasterXSize(
             self.Module.cwrap('GDALGetRasterXSize', 'number', ['number']),
             errorHandling
@@ -123,6 +135,19 @@ self.Module = {
             errorHandling,
             DATASETPATH
         );
+        // Equivalent to ogr2ogr
+        registry.GDALVectorTranslate = wGDALVectorTranslate(
+            self.Module.cwrap('GDALVectorTranslate', 'number', [
+                'string', // Output path or NULL
+                'number', // Destination dataset or NULL
+                'number', // Number of input datasets (only 1 is supported)
+                'number', // GDALDatasetH * list of source datasets
+                'number', // GDALVectorTranslateOptions *
+                'number', // int * to use for error reporting
+            ]),
+            errorHandling,
+            DATASETPATH
+        );
         registry.GDALWarp = wGDALWarp(
             self.Module.cwrap('GDALWarp', 'number', [
                 'string', // Destination dataset path or NULL
@@ -170,7 +195,11 @@ self.Module = {
 
 function handleDatasetAccess(accessor, dataset) {
     // 1: Open the source.
-    let srcDs = registry[dataset.source.func](dataset.source.src, dataset.source.args);
+    let srcDs = registry[dataset.source.func](
+        dataset.source.src,
+        dataset.source.args,
+        dataset.source.sidecars
+    );
 
     let resultDs = srcDs;
 

diff --git a/src/wrappers/gdalDatasetGetLayerCount.js b/src/wrappers/gdalDatasetGetLayerCount.js
@@ -0,0 +1,19 @@
+export default function (GDALDatasetGetLayerCount, errorHandling) {
+    return function (datasetPtr) {
+        const result = GDALDatasetGetLayerCount(datasetPtr);
+
+        const errorType = errorHandling.CPLGetLastErrorType();
+
+        // Check for errors; clean up and throw if error is detected
+        if (
+            errorType === errorHandling.CPLErr.CEFailure ||
+            errorType === errorHandling.CPLErr.CEFatal
+        ) {
+            const message = errorHandling.CPLGetLastErrorMsg();
+
+            throw new Error('Error in GDALDatasetGetLayerCount: ' + message);
+        } else {
+            return result;
+        }
+    };
+}