KDStore added; faster point-indexer. organize better. more tests

Open-S2 · Oct 31, 2024 · b55642b · b55642b
1 parent 28e3d03
commit b55642b
Show file tree

Hide file tree

Showing 123 changed files with 130,345 additions and 2,293 deletions.
diff --git a/LICENSE b/LICENSE
@@ -4,7 +4,7 @@ Copyright 2024 - Open S2
 
 All rights reserved.
 
-If you use any code from the s2maps-gpu engine, whether by forking the repository or incorporating snippets into your own project, you must maintain the code that includes the watermark and attribution to the OpenS2 initiative as is. You can learn more, including exceptions to this rule found at the attribution section <https://www.opens2.com/legal/attribution#when-attribute> and watermark section <https://www.opens2.com/legal/attribution#when-watermark>.
+If you use any code from the s2-tools engine, whether by forking the repository or incorporating snippets into your own project, you must maintain the code that includes the watermark and attribution to the OpenS2 initiative as is. You can learn more, including exceptions to this rule found at the attribution section <https://www.opens2.com/legal/attribution#when-attribute> and watermark section <https://www.opens2.com/legal/attribution#when-watermark>.
 
 By use, copy, modify, merge, publish, distribute, contribute, sublicense, and/or sell copies of the Software, you confirm that you are not employed at MapTiler AG, you are in no way affiliated to MapTiler AG, your work is not on behalf of MapTiler AG or associated with MapTiler AG. To the best of your knowledge, information and/or belief your work will not make it's way into the hands of MapTiler AG.
 
@@ -426,3 +426,63 @@ express Statement of Purpose.
  d. Affirmer understands and acknowledges that Creative Commons is not a
     party to this document and has no duty or obligation with respect to
     this CC0 or use of the Work.
+
+---
+
+Some code has been repurposed from <https://github.com/mourner/kdbush>
+
+ISC License
+
+Copyright (c) 2018, Vladimir Agafonkin
+
+Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.
+
+---
+
+Some code has been repurposed from <https://github.com/mourner/geokdbush>
+
+ISC License
+
+Copyright (c) 2017, Vladimir Agafonkin
+
+Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.
+
+---
+
+Some code has been repurposed from <https://github.com/mourner/tinyqueue>
+
+ISC License
+
+Copyright (c) 2017, Vladimir Agafonkin
+
+Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.
diff --git a/README.md b/README.md
@@ -35,6 +35,40 @@ yarn add s2-tools
 bun add s2-tools
 ```
 
+## Components
+
+> 💡 **NOTE:** The sizes are estimates and can change based on how you use them. Click the module link for documentation and more precise guides on file cost.
+
+### Converters
+
+| Main Modules             | Size                          | Description                                                      |
+| ------------------------ | ----------------------------- | ---------------------------------------------------------------- |
+| [toJSON]                 | ![To JSON Badge][toJSONBadge] | Convert any Reader to JSON data.                                 |
+| [toTiles]                | ![FT Badge][toTilesBadge]     | Convert any Reader to vector and/or raster tiles.                |
+
+[toJSON]: https://github.com/Open-S2/s2-tools/tree/master/docs-ts/converters/toJSON.md
+[toJSONBadge]: https://deno.bundlejs.com/badge?q=s2-tools&treeshake=[{+toJSON,toJSONLD,BufferReader+}]
+[toTiles]: https://github.com/Open-S2/s2-tools/tree/master/docs-ts/converters/toTiles.md
+[toTilesBadge]: https://deno.bundlejs.com/badge?q=s2-tools&treeshake=[{+toTiles,BufferReader+}]
+
+### Data Stores
+
+| Main Modules             | Size                          | Description                                                      |
+| ------------------------ | ----------------------------- | ---------------------------------------------------------------- |
+| [externalSort]           | ![ES Badge][esBadge]          | Sort large files with uint64 keys                                |
+| [kv]                     | ![KV Badge][kvBadge]          | Collection of tools using the filesystem to read and write data. |
+| [multiMap]               | ![MM Badge][mmBadge]          | Collection of tools using the filesystem to read and write data. |
+| [vector]                 | ![Vec Badge][vecBadge]        | Collection of tools using the filesystem to read and write data. |
+
+[externalSort]: https://github.com/Open-S2/s2-tools/tree/master/docs-ts/dataStores/externalSort.md
+[esBadge]: https://deno.bundlejs.com/badge?q=s2-tools/file&treeshake=[{externalSort}]
+[kv]: https://github.com/Open-S2/s2-tools/tree/master/docs-ts/dataStores/kv.md
+[kvBadge]: https://deno.bundlejs.com/badge?q=s2-tools&treeshake=[{KV}]
+[multiMap]: https://github.com/Open-S2/s2-tools/tree/master/docs-ts/dataStores/multimap.md
+[mmBadge]: https://deno.bundlejs.com/badge?q=s2-tools&treeshake=[{MultiMap}]
+[vector]: https://github.com/Open-S2/s2-tools/tree/master/docs-ts/dataStores/vector.md
+[vecBadge]: https://deno.bundlejs.com/badge?q=s2-tools&treeshake=[{Vector}]
+
 ---
 
 ## Development

diff --git a/TODO b/TODO
@@ -1,3 +1,4 @@
+[ ] - reimagine the pointIndex system, its too slow
 [ ] - bin -> toS2JSON, toRasterTiles, & toVectorTiles
 [ ] - toVectorTiles
 [ ] - - INPUTS: all S2JSON types as `s2jsonld` files
@@ -8,3 +9,5 @@
 [ ] - - KML, KMZ, GML, & GPX
 [ ] - - raster tile reader including DEM for z-data
 [ ] - - LAS/LAZ
+[ ] - geometry tools (lookup readme in geometry folder)
+[ ] - support missing projections in proj4 found in c++ implementation
diff --git a/assets/code-coverage.svg b/assets/code-coverage.svg
diff --git a/benchmarks/pointIndex.ts b/benchmarks/pointIndex.ts
@@ -1,6 +1,6 @@
-// import KDBush from 'kdbush';
-import PointIndex from '../src/dataStructures/pointIndex';
+import KDBush from 'kdbush';
 import { fromLonLat } from '../src/geometry/s2/point';
+import { PointIndex, PointIndexFast } from '../src/dataStructures';
 
 const TOTAL_SIZE = 1_000_000;
 
@@ -16,21 +16,21 @@ for (let i = 0; i < TOTAL_SIZE; i++) {
 
 // ---------------------------------------------- BUSH
 
-// const bushBuildStart = Bun.nanoseconds();
-// const bush = new KDBush(TOTAL_SIZE);
-// for (let i = 0; i < TOTAL_SIZE; i++) bush.add(lls[i].lon, lls[i].lat);
-// bush.finish();
-// const bushBuildEnd = Bun.nanoseconds();
-// const bushBuildSeconds = (bushBuildEnd - bushBuildStart) / 1_000_000_000;
-// console.info('bush Build time: ', bushBuildSeconds);
+const bushBuildStart = Bun.nanoseconds();
+const bush = new KDBush(TOTAL_SIZE);
+for (let i = 0; i < TOTAL_SIZE; i++) bush.add(lls[i].lon, lls[i].lat);
+bush.finish();
+const bushBuildEnd = Bun.nanoseconds();
+const bushBuildSeconds = (bushBuildEnd - bushBuildStart) / 1_000_000_000;
+console.info('bush Build time: ', bushBuildSeconds);
 
-// const bushSearchTime = Bun.nanoseconds();
-// const _withinSearch = bush.within(lls[0].lon, lls[0].lat, 1);
-// const withinSearchEnd = Bun.nanoseconds();
-// const withinSearchSeconds = (withinSearchEnd - bushSearchTime) / 1_000_000_000;
-// console.info('bush Search time: ', withinSearchSeconds);
+const bushSearchTime = Bun.nanoseconds();
+const _withinSearch = bush.within(lls[0].lon, lls[0].lat, 1);
+const withinSearchEnd = Bun.nanoseconds();
+const withinSearchSeconds = (withinSearchEnd - bushSearchTime) / 1_000_000_000;
+console.info('bush Search time: ', withinSearchSeconds);
 
-// console.info('bush total time: ', bushBuildSeconds + withinSearchSeconds);
+console.info('bush total time: ', bushBuildSeconds + withinSearchSeconds);
 
 // ---------------------------------------------- POINT INDEX
 console.info('\n\n');
@@ -56,6 +56,32 @@ console.info('index Search time: ', withinSearchSeconds2);
 
 console.info('index total time: ', indexBuildSeconds + withinSearchSeconds2);
 
+// ---------------------------------------------- POINT INDEX FAST
+console.info('\n\n');
+
+const indexBuildStartFast = Bun.nanoseconds();
+const indexFast = new PointIndexFast<{ a: number }>();
+for (let i = 0; i < TOTAL_SIZE; i++) {
+  indexFast.insertLonLat(lls[i].lon, lls[i].lat, { a: i });
+}
+const indexSortTimeFast = Bun.nanoseconds();
+await indexFast.sort();
+const indexBuildEndFast = Bun.nanoseconds();
+const indexSortSecondsFast = (indexBuildEndFast - indexSortTimeFast) / 1_000_000_000;
+const indexBuildSecondsFast = (indexBuildEndFast - indexBuildStartFast) / 1_000_000_000;
+console.info('index Build time: ', indexBuildSecondsFast);
+console.info('index Sort time: ', indexSortSecondsFast);
+
+const indexSearchTimeFast = Bun.nanoseconds();
+const _withinSearch2Fast = indexFast.searchRadiusSphere(lls[0].lon, lls[0].lat, 100);
+const withinSearchEnd2Fast = Bun.nanoseconds();
+const withinSearchSeconds2Fast = (withinSearchEnd2Fast - indexSearchTimeFast) / 1_000_000_000;
+console.info('index Search time: ', withinSearchSeconds2Fast);
+
+console.info('index total time: ', indexBuildSecondsFast + withinSearchSeconds2Fast);
+
+// ---------------------------------------------- UTILS
+
 /**
  * Generate a random whole number between two given values.
  * @param a - The lower bound (inclusive).

diff --git a/bun.lockb b/bun.lockb
diff --git a/docs-ts/converters/toJSON.md b/docs-ts/converters/toJSON.md
@@ -0,0 +1 @@
+# To JSON
diff --git a/docs-ts/converters/toTiles.md b/docs-ts/converters/toTiles.md
@@ -0,0 +1 @@
+# To Tiles
diff --git a/docs-ts/dataStore/externalSort.md b/docs-ts/dataStore/externalSort.md
@@ -0,0 +1 @@
+# External Sort
diff --git a/docs-ts/dataStore/kv.md b/docs-ts/dataStore/kv.md
@@ -0,0 +1 @@
+# KV
diff --git a/docs-ts/dataStore/multiMap.md b/docs-ts/dataStore/multiMap.md
@@ -0,0 +1 @@
+# Multi Map
diff --git a/docs-ts/dataStore/vector.md b/docs-ts/dataStore/vector.md
@@ -0,0 +1 @@
+# Vector
diff --git a/docs-ts/dataStructures/cache.md b/docs-ts/dataStructures/cache.md
@@ -0,0 +1 @@
+# Cache
diff --git a/docs-ts/dataStructures/pointCluster.md b/docs-ts/dataStructures/pointCluster.md
@@ -0,0 +1 @@
+# Point Cluster
diff --git a/docs-ts/dataStructures/pointIndex.md b/docs-ts/dataStructures/pointIndex.md
@@ -0,0 +1 @@
+# Point Index
diff --git a/docs-ts/dataStructures/priorityQueue.md b/docs-ts/dataStructures/priorityQueue.md
@@ -0,0 +1 @@
+# Priority Queue
diff --git a/docs-ts/dataStructures/tile.md b/docs-ts/dataStructures/tile.md
@@ -0,0 +1 @@
+# Tile
diff --git a/docs-ts/geometry/angles.md b/docs-ts/geometry/angles.md
diff --git a/docs-ts/geometry/bbox.md b/docs-ts/geometry/bbox.md
diff --git a/docs-ts/geometry/id.md b/docs-ts/geometry/id.md
diff --git a/docs-ts/geometry/lonlat.md b/docs-ts/geometry/lonlat.md
@@ -0,0 +1 @@
+# Lon Lat
diff --git a/docs-ts/geometry/planets.md b/docs-ts/geometry/planets.md
@@ -0,0 +1 @@
+# Planets
diff --git a/docs-ts/geometry/predicates.md b/docs-ts/geometry/predicates.md
diff --git a/docs-ts/geometry/s2.md b/docs-ts/geometry/s2.md
diff --git a/docs-ts/geometry/tools.md b/docs-ts/geometry/tools.md
diff --git a/docs-ts/geometry/wm.md b/docs-ts/geometry/wm.md
diff --git a/docs-ts/proj4/datums.md b/docs-ts/proj4/datums.md
@@ -0,0 +1 @@
+# PROJ4 Datums
diff --git a/docs-ts/proj4/mgrs.md b/docs-ts/proj4/mgrs.md
@@ -0,0 +1 @@
+# PROJ4 MGRS
diff --git a/docs-ts/proj4/projections.md b/docs-ts/proj4/projections.md
@@ -0,0 +1 @@
+# PROJ4 Projections
diff --git a/docs-ts/proj4/transformer.md b/docs-ts/proj4/transformer.md
@@ -0,0 +1 @@
+# PROJ4 Transformer
diff --git a/docs-ts/readers/csv.md b/docs-ts/readers/csv.md
@@ -0,0 +1 @@
+# CSV Reader
diff --git a/docs-ts/readers/geotiff.md b/docs-ts/readers/geotiff.md
@@ -0,0 +1 @@
+# GeoTIFF Reader
diff --git a/docs-ts/readers/json.md b/docs-ts/readers/json.md
@@ -0,0 +1 @@
+# JSON Reader
diff --git a/docs-ts/readers/nadgrid.md b/docs-ts/readers/nadgrid.md
@@ -0,0 +1 @@
+# NadGrid Reader
diff --git a/docs-ts/readers/osm.md b/docs-ts/readers/osm.md
@@ -0,0 +1 @@
+# Open Street Map Reader
diff --git a/docs-ts/readers/pmtiles.md b/docs-ts/readers/pmtiles.md
@@ -0,0 +1 @@
+# (S2) PM Tiles Reader
diff --git a/docs-ts/readers/protobuf.md b/docs-ts/readers/protobuf.md
@@ -0,0 +1 @@
+# Protobuf Reader
diff --git a/docs-ts/readers/shapefile.md b/docs-ts/readers/shapefile.md
@@ -0,0 +1 @@
+# ShapeFile Reader
diff --git a/docs-ts/readers/wkt.md b/docs-ts/readers/wkt.md
@@ -0,0 +1 @@
+# WKT Reader
diff --git a/docs-ts/readers/xml.md b/docs-ts/readers/xml.md
@@ -0,0 +1 @@
+# XML Reader
diff --git a/docs-ts/space/satellite.md b/docs-ts/space/satellite.md
@@ -0,0 +1 @@
+# Satellite
diff --git a/docs-ts/tools/delaunator.md b/docs-ts/tools/delaunator.md
@@ -0,0 +1 @@
+# Delaunator
diff --git a/docs-ts/tools/orthodrome.md b/docs-ts/tools/orthodrome.md
@@ -0,0 +1 @@
+# Orthodrome
diff --git a/docs-ts/tools/polylabel.md b/docs-ts/tools/polylabel.md
@@ -0,0 +1 @@
+# Polylabel
diff --git a/docs-ts/writers/pmtiles.md b/docs-ts/writers/pmtiles.md
@@ -0,0 +1 @@
+# PM Tiles Writer
diff --git a/docs-ts/writers/tile.md b/docs-ts/writers/tile.md
@@ -0,0 +1 @@
+# Tiles Writer
diff --git a/package.json b/package.json
@@ -10,15 +10,19 @@
     "S2"
   ],
   "exports": {
-    ".": "./dist/index.js"
+    ".": "./dist/index.js",
+    "./file": "./dist/file.js",
+    "./mmap": "./dist/mmap.js"
   },
   "directories": {
     "lib": "dist"
   },
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "typescript": {
-    "main": "src/index.ts"
+    "main": "src/index.ts",
+    "file": "src/file.ts",
+    "mmap": "src/mmap.ts"
   },
   "files": [
     "dist"
@@ -63,10 +67,12 @@
   "homepage": "https://github.com/Open-S2/s2-tools#readme",
   "devDependencies": {
     "@skypack/package-check": "^0.2.2",
+    "@types/all-the-cities": "^3.1.3",
     "@types/bun": "^1.1.10",
     "@types/node": "^22.5.5",
     "@types/tmp": "^0.2.6",
     "@webgpu/types": "^0.1.48",
+    "all-the-cities": "^3.1.0",
     "eslint": "^9.11.1",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-jsdoc": "^50.3.0",
@@ -79,6 +85,7 @@
     "nextafter": "^1.0.0",
     "prettier": "^3.3.3",
     "robust-orientation": "^1.2.1",
+    "satellite.js": "^5.0.0",
     "tmp": "^0.2.3",
     "typedoc": "^0.26.7",
     "typedoc-plugin-coverage": "^3.3.0",

diff --git a/src/converters/README.md b/src/converters/README.md
@@ -0,0 +1 @@
+# Converters
diff --git a/src/converters/toJSON/index.ts b/src/converters/toJSON/index.ts
@@ -1,4 +1,4 @@
-import { convert } from '../../geometry/convert';
+import { convert } from '../../geometry/tools/convert';
 import { mergeBBoxes } from '../../geometry';
 
 import type { FeatureIterator } from '../../readers';