setup lanczos; toTiles makes some progress

README.md

@@ -3,6 +3,9 @@
 </h1>
 
 <p align="center">
+  <a href="https://img.shields.io/github/actions/workflow/status/Open-S2/s2-tools/test.yml?logo=github">
+    <img src="https://img.shields.io/github/actions/workflow/status/Open-S2/s2-tools/test.yml?logo=github" alt="GitHub Actions Workflow Status">
+  </a>
   <a href="https://npmjs.org/package/s2-tools">
     <img src="https://img.shields.io/npm/v/s2-tools.svg?logo=npm&logoColor=white" alt="npm">
   </a>
@@ -122,6 +125,8 @@ To generate the coverage report, use the following command:
 
 ```bash
 cargo tarpaulin
+# faster
+cargo tarpaulin --color always --skip-clean
 # bacon
 bacon coverage # or type `l` inside the tool
 ```

bacon.toml

@@ -42,7 +42,7 @@ need_stdout = true
 command = [
     "sh",
     "-c",
-    "cargo tarpaulin --out xml > /dev/null 2>&1 && pycobertura show cobertura.xml",
+    "cargo tarpaulin --skip-clean --color always --out xml > /dev/null 2>&1 && pycobertura show cobertura.xml",
 ]
 need_stdout = true
 

src/converters/toTiles/index.ts

@@ -1,11 +1,12 @@
 import { MetadataBuilder } from 's2-tilejson';
 
 import type { Reader } from '../../readers';
+import type { TileWriter } from '../../writers';
 import type { VectorFeatures } from '../../geometry';
 import type { Attribution, Encoding, LayerMetaData, Scheme } from 's2-tilejson';
-import type { TileWriter, Writer } from '../../writers';
 
 import type { ClusterOptions } from '../../dataStructures/pointCluster';
+import type { TileStoreOptions } from '../../dataStructures/tile';
 
 /** A layer defines the exact mechanics of what data to parse and how the data is stored */
 export interface SourceLayer {
@@ -15,16 +16,69 @@ export interface SourceLayer {
   metadata: LayerMetaData;
 }
 
+/**
+ * Before tiling the data, you can mutate it here. It can also act as a filter if you return undefined
+ */
+export type OnFeature = (feature: VectorFeatures) => VectorFeatures | undefined;
+
 /** Sources are the blueprints of what data to fetch and how to store it */
-export interface Source {
+export interface BaseSource {
+  /** The name of the source */
+  name: string;
   /** The reader to parse the data from */
   data: Reader;
+}
+
+/** Vector source */
+export interface VectorSource extends BaseSource {
+  /** The layers to construct and organize the data around for this source */
+  layers: SourceLayer[];
   /** If options are provided, the assumption is the point data is clustered */
   cluster?: ClusterOptions;
+  /** tile buffer on each side so lines and polygons don't get clipped */
+  buffer?: number;
+  /** whether to build the bounding box for each tile feature */
+  buildBBox?: boolean;
   /** Before tiling the data, you can mutate it here. It can also act as a filter if you return undefined */
-  onFeature?: (feature: VectorFeatures) => VectorFeatures | undefined;
-  /** The layers to construct and organize the data around for this source */
-  layers: SourceLayer[];
+  onFeature?: OnFeature;
+}
+
+/** Raster source */
+export interface RasterSource extends BaseSource {
+  description?: string;
+}
+
+/** A layerguide is a minified version of a layer used by workers to build tiles */
+export interface LayerGuide {
+  /** Components of how the layer is built and stored */
+  metadata: LayerMetaData;
+  /** Stringified version of the onFeature used by the source so it can be shipped to a worker. */
+  onFeature?: string;
+  /** Guide on how to splice the data into vector tiles */
+  tileGuide: TileStoreOptions;
+}
+
+/** A parsed LayerGuide where onFeature is parsed back into a function */
+export interface ParsedLayerGuide extends Omit<LayerGuide, 'onFeature'> {
+  onFeature?: OnFeature;
+}
+
+/** A Source Guide is a minified version of a source used by workers to build tiles */
+export interface SourceGuide {
+  /** The name of the source */
+  [sourceName: string]: {
+    /** Name of the associated layer */
+    [layerName: string]: LayerGuide;
+  };
+}
+
+/** A parsed SourceGuide where onFeature is parsed back into a function */
+export interface ParsedSourceGuide {
+  /** The name of the source */
+  [sourceName: string]: {
+    /** Name of the associated layer */
+    [layerName: string]: ParsedLayerGuide;
+  };
 }
 
 /** A user defined guide on building the vector tiles */
@@ -42,45 +96,52 @@ export interface BuildGuide {
   scheme?: Scheme;
   /** The encoding format. Can be either 'gz', 'br', 'zstd' or 'none' [Default: 'gz'] */
   encoding?: Encoding;
-  /** The sources that the tile is built from and how the layers are to be stored */
-  sources: Source[];
-  /** The attribution of the data. Store as { key: presentation name }. [value: href link] */
+  /** The vector sources that the tile is built from and how the layers are to be stored */
+  vectorSources?: VectorSource[];
+  /** The raster sources that will be conjoined into a single rgba pixel index for tile extraction */
+  rasterSources?: RasterSource[];
+  /** The attribution of the data. Store as { key: 'presentation name' }. [value: href link] */
   attribution?: Attribution;
   /**
    * The vector format if applicable helps define how the vector data is stored.
    * - The more modern vector format is the 'open-v2' which supports things like m-values
    * and 3D geometries.
    * - The legacy vector format is the 'open-v1' which only supports 2D geometries and works on
    * older map engines like Mapbox-gl-js.
+   * [Default: 'open-v2']
    */
-  vectorFormat?: 'open-v2' | 'open-v1';
+  vectorFormat?: 'open-v1' | 'open-v2';
   /**
    * The data created will be stored in either a folder structure or a pmtiles file
    * Folder structure is either '{face}/{zoom}/{x}/{y}.pbf' or '{zoom}/{x}/{y}.pbf'.
    * PMTiles store all data in a single data file.
    */
   tileWriter: TileWriter;
-  /** Explain to the module what kind of writer to use (A buffer or file writer) */
-  writer: Writer;
+  /** Set the number of threads to use. [Default: 1] */
+  threads?: number;
 }
 
 /**
  * Build vector tiles give a guide on what sources to parse data from and how to store it
  * @param buildGuide - the user defined guide on building the vector tiles
  */
-export async function toTiles(buildGuide: BuildGuide): Promise<void> {
+export async function toVectorTiles(buildGuide: BuildGuide): Promise<void> {
   const { tileWriter } = buildGuide;
 
-  // first setup our metadata builder
-  const metaBuilder = new MetadataBuilder();
-  updateBuilder(metaBuilder, buildGuide);
+  // TODO: Prepare init messages for workers
 
   // TODO: iterate features and have workers split/store them
 
+  // TODO: externalSort on features at this point.
+
   // TODO: have workers build tiles
 
-  // FINISH:
+  // build metadata based on the guide
+  const metaBuilder = new MetadataBuilder();
+  updateBuilder(metaBuilder, buildGuide);
   const metadata = metaBuilder.commit();
+
+  // FINISH
   await tileWriter.commit(metadata);
 }
 
@@ -89,7 +150,7 @@ export async function toTiles(buildGuide: BuildGuide): Promise<void> {
  * @param buildGuide - the user defined guide on building the vector tiles
  */
 function updateBuilder(metaBuilder: MetadataBuilder, buildGuide: BuildGuide): void {
-  const { name, description, version, scheme, encoding, attribution, sources } = buildGuide;
+  const { name, description, version, scheme, encoding, attribution, vectorSources } = buildGuide;
 
   metaBuilder.setName(name);
   metaBuilder.setExtension('pbf');
@@ -103,14 +164,13 @@ function updateBuilder(metaBuilder: MetadataBuilder, buildGuide: BuildGuide): vo
       metaBuilder.addAttribution(displayName, href);
     }
   }
-  for (const { layers } of sources) {
-    for (const layer of layers) {
-      metaBuilder.addLayer(layer.name, layer.metadata);
-    }
+  for (const { layers } of vectorSources ?? []) {
+    for (const layer of layers) metaBuilder.addLayer(layer.name, layer.metadata);
   }
 }
 
 // TODO:
+// VECTOR:
 // - step 1: ship individual features to workers
 // - - step 1a: splite the features into tiles, requesting all tiles from range min->max of the layer
 // - - step 1b: store all those features into a multimap where the key is the tile-id and the value is the features
@@ -119,3 +179,18 @@ function updateBuilder(metaBuilder: MetadataBuilder, buildGuide: BuildGuide): vo
 // - - step 2b: build the tile from the features, gzip, etc. then ship the buffer and metadata to main thread
 // - - step 2c: store metadata into metaBuilder and the buffer into the store
 // finish
+//
+// CLUSTER:
+// - step 1: for each point, just add.
+// - step 2: cluster.
+// - step 3: request tiles as needed.
+//
+// RASTER:
+// - step 1: for every pixel, store to a point index where the m-value is the rgba value
+// - step 2: build tiles from the point index
+// - - step 2a: start at max zoom - for each pixel in the tile search the range and find average
+//              of all the points in that range (remember color is logarithmic so multiply each pixel r-g-b-a by itself first)
+//              If too far zoomed in, find the closest pixel within a resonable radius.
+// - - step 2b: after finishing max zoom, use https://github.com/rgba-image/lanczos as I move towards minzoom.
+// - - sidenote: I think the easiest way to build tiles is start at 0 zoom and dive down everytime we find at least one point, then move back up
+// finish

src/converters/toTiles/tileWorkers/buildTile.ts

@@ -0,0 +1,16 @@
+declare let self: Worker;
+
+// TWO WAYS TO BUILD TILES:
+// * Vector Tiles
+// * Raster Tiles
+
+/**
+ * A worker that sorts a chunk of a file and sends it to an output directory
+ * @param event - the sort chunk message
+ * @param _event
+ */
+self.onmessage = (_event: Bun.MessageEvent<unknown>): void => {
+  // void sortChunk(event.data as SortChunk).then((outFile): void => {
+  //   postMessage(outFile);
+  // });
+};

src/converters/toTiles/vectorWorker/file.ts

@@ -0,0 +1,14 @@
+declare let self: Worker;
+
+import { FileMultiMap } from '../../../dataStore/multimap/file';
+import VectorTileWorker from './vectorTileWorker';
+
+import type { VectorFeature } from '../../../geometry';
+
+/** Convert a vector feature to a collection of tiles and store each tile feature */
+class FileVectorTileWorker extends VectorTileWorker {
+  writer = new FileMultiMap<VectorFeature>();
+}
+
+const vecWorker = new FileVectorTileWorker();
+self.onmessage = vecWorker.onmessage.bind(vecWorker);

src/converters/toTiles/vectorWorker/index.ts

@@ -0,0 +1,6 @@
+declare let self: Worker;
+
+import VectorTileWorker from './vectorTileWorker';
+
+const vecWorker = new VectorTileWorker();
+self.onmessage = vecWorker.onmessage.bind(vecWorker);

src/converters/toTiles/vectorWorker/vectorTileWorker.ts

@@ -0,0 +1,71 @@
+import { MultiMap } from '../../../dataStore';
+
+import type { MultiMapStore } from '../../../dataStore';
+import type { VectorFeature } from '../../../geometry';
+import type { OnFeature, ParsedSourceGuide, SourceGuide } from '../..';
+
+/** Take in options that will be used to create a tiled data correctly */
+export interface InitMessage {
+  /** Message type */
+  type: 'init';
+  /** The sources that will be used to create the tile */
+  sources: SourceGuide;
+}
+
+/** Take in a feature that will be added to the tile */
+export interface FeatureMessage {
+  /** Message type */
+  type: 'feature';
+  /** The feature to add to the tile */
+  feature: VectorFeature;
+}
+
+/** Convert a vector feature to a collection of tiles and store each tile feature */
+export default class VectorTileWorker {
+  sources: ParsedSourceGuide = {};
+  writer: MultiMapStore<VectorFeature> = new MultiMap<VectorFeature>();
+  /**
+   * Tile-ize input vector features and store them
+   * @param event - the init message or a feature message
+   */
+  onmessage(event: Bun.MessageEvent<InitMessage | FeatureMessage>): void {
+    this.handleMessage(event.data);
+    self.postMessage({ type: 'ready' });
+  }
+
+  /**
+   * Tile-ize input vector features and store them
+   * @param message - the init message or a feature message
+   */
+  handleMessage(message: InitMessage | FeatureMessage): void {
+    const { type } = message;
+    if (type === 'init') {
+      this.sources = parseSourceGuide(message.sources);
+    } else {
+      // TODO:
+    }
+  }
+}
+
+/**
+ * Convert a source guide to a parsed source guide (where onFeature is parsed back into a function)
+ * @param sourceGuide - the source guide to parse
+ * @returns the parsed source guide
+ */
+function parseSourceGuide(sourceGuide: SourceGuide): ParsedSourceGuide {
+  const res: ParsedSourceGuide = {};
+
+  for (const [sourceName, source] of Object.entries(sourceGuide)) {
+    for (const [layerName, layer] of Object.entries(source)) {
+      res[sourceName][layerName] = {
+        ...layer,
+        onFeature:
+          layer.onFeature !== undefined
+            ? (new Function(layer.onFeature)() as OnFeature)
+            : undefined,
+      };
+    }
+  }
+
+  return res;
+}

src/dataStructures/tile.ts

@@ -95,7 +95,7 @@ export class Tile {
  * @param ti - tile i
  * @param tj - tile j
  */
-function _transform(geometry: VectorGeometry, zoom: number, ti: number, tj: number) {
+function _transform(geometry: VectorGeometry, zoom: number, ti: number, tj: number): void {
   const { type, coordinates } = geometry;
   zoom = 1 << zoom;
 
@@ -141,7 +141,7 @@ export interface TileStoreOptions {
   minzoom?: number;
   /** max zoom level to cluster the points on */
   maxzoom?: number;
-  /** tile buffer on each side in pixels */
+  /** max zoom to index data on construction */
   indexMaxzoom?: number;
   /** simplification tolerance (higher means simpler) */
   tolerance?: number;

src/file.ts

@@ -6,3 +6,4 @@ export * from './dataStore/vector/file';
 export * from './readers/shapefile/file';
 export * from './readers/file';
 export * from './writers/file';
+export * from './writers/tile';