feat: support fpzip and kempressed codecs

.github/workflows/build.yml

@@ -227,5 +227,6 @@ jobs:
     - run: ./src/neuroglancer/mesh/draco/build.sh
     - run: ./src/neuroglancer/sliceview/compresso/build.sh
     - run: ./src/neuroglancer/sliceview/png/build.sh
+    - run: ./src/neuroglancer/sliceview/fpzip/build.sh
     # Check that there are no differences.
     - run: git diff --exit-code

config/bundle-config.js

@@ -54,6 +54,7 @@ const DEFAULT_DATA_SOURCES = exports.DEFAULT_DATA_SOURCES = [
       'neuroglancer/async_computation/decode_gzip',
       'neuroglancer/async_computation/decode_compresso',
       'neuroglancer/async_computation/decode_png',
+      'neuroglancer/async_computation/decode_fpzip',
     ],
   },
   {

src/neuroglancer/async_computation/decode_fpzip.ts

@@ -0,0 +1,49 @@
+/**
+ * @license
+ * Copyright 2021 William Silversmith
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {decompressFpzip, decompressKempressed} from 'neuroglancer/sliceview/fpzip';
+import {decodeFpzip, decodeKempressed} from 'neuroglancer/async_computation/decode_fpzip_request';
+import {registerAsyncComputation} from 'neuroglancer/async_computation/handler';
+
+registerAsyncComputation(
+    decodeFpzip,
+    async function(
+      data: Uint8Array, 
+      width: number, height: number, depth: number,
+      numComponents: number, bytesPerPixel:number, 
+    ) {   
+      const result = await decompressFpzip(
+        data, 
+        width, height, depth, 
+        numComponents, bytesPerPixel
+      );
+      return { value: result, transfer: [result.buffer] };
+    });
+
+registerAsyncComputation(
+    decodeKempressed,
+    async function(
+      data: Uint8Array, 
+      width: number, height: number, depth: number,
+      numComponents: number, bytesPerPixel:number, 
+    ) {   
+      const result = await decompressKempressed(
+        data, 
+        width, height, depth, 
+        numComponents, bytesPerPixel
+      );
+      return { value: result, transfer: [result.buffer] };
+    });

src/neuroglancer/async_computation/decode_fpzip_request.ts

@@ -0,0 +1,29 @@
+/**
+ * @license
+ * Copyright 2022 William Silversmith
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {asyncComputation} from 'neuroglancer/async_computation';
+
+export const decodeFpzip = asyncComputation<(
+    data: Uint8Array, 
+    width: number, height: number, depth: number,
+    numComponents: number, bytesPerPixel:number
+) => Float32Array>('decodeFpzip');
+
+export const decodeKempressed = asyncComputation<(
+    data: Uint8Array, 
+    width: number, height: number, depth: number,
+    numComponents: number, bytesPerPixel:number
+) => Float32Array>('decodeKempressed');

src/neuroglancer/datasource/precomputed/backend.ts

@@ -33,6 +33,7 @@ import {decodeCompressoChunk} from 'neuroglancer/sliceview/backend_chunk_decoder
 import {decodeJpegChunk} from 'neuroglancer/sliceview/backend_chunk_decoders/jpeg';
 import {decodePngChunk} from 'neuroglancer/sliceview/backend_chunk_decoders/png';
 import {decodeRawChunk} from 'neuroglancer/sliceview/backend_chunk_decoders/raw';
+import {decodeFpzipChunk, decodeKempressedChunk} from 'neuroglancer/sliceview/backend_chunk_decoders/fpzip';
 import {VolumeChunk, VolumeChunkSource} from 'neuroglancer/sliceview/volume/backend';
 import {fetchSpecialHttpByteRange} from 'neuroglancer/util/byte_range_http_requests';
 import {CancellationToken} from 'neuroglancer/util/cancellation';
@@ -257,6 +258,8 @@ chunkDecoders.set(VolumeChunkEncoding.JPEG, decodeJpegChunk);
 chunkDecoders.set(VolumeChunkEncoding.COMPRESSED_SEGMENTATION, decodeCompressedSegmentationChunk);
 chunkDecoders.set(VolumeChunkEncoding.COMPRESSO, decodeCompressoChunk);
 chunkDecoders.set(VolumeChunkEncoding.PNG, decodePngChunk);
+chunkDecoders.set(VolumeChunkEncoding.FPZIP, decodeFpzipChunk);
+chunkDecoders.set(VolumeChunkEncoding.KEMPRESSED, decodeKempressedChunk);
 
 @registerSharedObject() export class PrecomputedVolumeChunkSource extends
 (WithParameters(WithSharedCredentialsProviderCounterpart<SpecialProtocolCredentials>()(VolumeChunkSource), VolumeChunkSourceParameters)) {

src/neuroglancer/datasource/precomputed/base.ts

@@ -23,7 +23,9 @@ export enum VolumeChunkEncoding {
   JPEG,
   COMPRESSED_SEGMENTATION,
   COMPRESSO,
-  PNG
+  PNG,
+  FPZIP,
+  KEMPRESSED
 }
 
 export class VolumeChunkSourceParameters {

src/neuroglancer/sliceview/backend_chunk_decoders/fpzip.ts

@@ -0,0 +1,59 @@
+/**
+ * @license
+ * Copyright 2022 William Silvermsith.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {TypedArray} from 'neuroglancer/util/array';
+import {DATA_TYPE_BYTES} from 'neuroglancer/util/data_type';
+import {decodeRawChunk} from 'neuroglancer/sliceview/backend_chunk_decoders/raw';
+import {VolumeChunk} from 'neuroglancer/sliceview/volume/backend';
+import {CancellationToken} from 'neuroglancer/util/cancellation';
+import {decodeFpzip, decodeKempressed} from 'neuroglancer/async_computation/decode_fpzip_request';
+import {requestAsyncComputation} from 'neuroglancer/async_computation/request';
+
+export async function decodeFpzipChunk(
+    chunk: VolumeChunk, cancellationToken: CancellationToken, response: ArrayBuffer) {
+
+  const chunkDataSize = chunk.chunkDataSize!;
+  const dataType = chunk.source!.spec.dataType;
+  let image : TypedArray = await requestAsyncComputation(
+    decodeFpzip, cancellationToken, [response],
+    /*buffer=*/(new Uint8Array(response)),
+    /*width=*/chunkDataSize[0],
+    /*height=*/chunkDataSize[1],
+    /*depth=*/chunkDataSize[2],
+    /*numComponents=*/chunkDataSize[3] || 1,
+    /*bytesPerPixel=*/DATA_TYPE_BYTES[dataType],
+  );
+
+  await decodeRawChunk(chunk, cancellationToken, image.buffer);
+}
+
+export async function decodeKempressedChunk(
+    chunk: VolumeChunk, cancellationToken: CancellationToken, response: ArrayBuffer) {
+
+  const chunkDataSize = chunk.chunkDataSize!;
+  const dataType = chunk.source!.spec.dataType;
+  let image : TypedArray = await requestAsyncComputation(
+    decodeKempressed, cancellationToken, [response],
+    /*buffer=*/(new Uint8Array(response)),
+    /*width=*/chunkDataSize[0],
+    /*height=*/chunkDataSize[1],
+    /*depth=*/chunkDataSize[2],
+    /*numComponents=*/chunkDataSize[3] || 1,
+    /*bytesPerPixel=*/DATA_TYPE_BYTES[dataType],
+  );
+
+  await decodeRawChunk(chunk, cancellationToken, image.buffer);
+}

src/neuroglancer/sliceview/fpzip/Dockerfile

@@ -0,0 +1,15 @@
+# 2.0.24 (2021-06-10)
+FROM emscripten/emsdk@sha256:81ec54b7a096d28f24d906955dbf98ff336cca47658d980c243baa36f6484f9f
+
+ENV FPZIP_1_3_0_SHA256 248df7d84259e3feaa4c4797956b2a77c3fcd734e8f8fdc51ce171dcf4f0136c
+
+RUN mkdir -p /usr/src/fpzip \
+    && curl -SL -o /usr/src/fpzip.tar.gz https://github.com/LLNL/fpzip/releases/download/1.3.0/fpzip-1.3.0.tar.gz
+
+RUN echo "${FPZIP_1_3_0_SHA256} /usr/src/fpzip.tar.gz" | sha256sum --check --status
+RUN tar -xzC /usr/src/fpzip -f /usr/src/fpzip.tar.gz --strip-components=1 \
+    && rm /usr/src/fpzip.tar.gz
+
+# RUN mkdir -p /usr/src/fpzip/build && cd /usr/src/fpzip/build && cmake -DSHARED_LIBS_SUPPORTED=OFF .. && make
+
+

src/neuroglancer/sliceview/fpzip/build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -xve
+
+# This script builds `fpzip.wasm` using emsdk in a docker container.
+
+cd "$(dirname "$0")"
+
+docker build .
+docker run \
+       --rm \
+       -v ${PWD}:/src \
+       -u $(id -u):$(id -g) \
+       $(docker build -q .) \
+       ./build_wasm.sh

src/neuroglancer/sliceview/fpzip/build_wasm.sh

@@ -0,0 +1,31 @@
+#!/bin/bash -xve
+
+FPZIP=/usr/src/fpzip
+SRC=$FPZIP/src
+
+compile_options=(
+    -Oz # O2 + drops vectorization, but WASM doesn't support vectorization anyway
+    -I$FPZIP/include 
+
+    $SRC/error.cpp $SRC/rcdecoder.cpp $SRC/rcencoder.cpp 
+    $SRC/rcqsmodel.cpp $SRC/write.cpp $SRC/read.cpp 
+
+    -DFPZIP_FP=FPZIP_FP_FAST -DFPZIP_BLOCK_SIZE=0x1000
+    -DWITH_UNION
+
+    fpzip_wasm.cpp
+     -DNDEBUG
+     --no-entry
+     -s FILESYSTEM=0
+     -s ALLOW_MEMORY_GROWTH=1 
+     -s TOTAL_STACK=32768
+     -s TOTAL_MEMORY=64kb
+     -s EXPORTED_FUNCTIONS='["_fpzip_decompress","_fpzip_dekempress","_check_valid","_malloc","_free"]'
+     -s MALLOC=emmalloc
+     -s ENVIRONMENT=worker
+     -s STANDALONE_WASM=1
+     # -s ASSERTIONS=1 
+     # -s SAFE_HEAP=1
+     -o libfpzip.wasm
+)
+em++ ${compile_options[@]}