fix!: delta index fragment bitmaps contained previous index coverage (#…

…3377)

BREAKING CHANGE: delta index fragment bitmaps will now only contain the
fragment ids covered by the delta, not the full index. To get the full
bitmap, make sure to union with all index segments with the same name.
Old datasets will still show previous fragment ids, until a write is
done, which forces a migration. **If corrupted fragment ids are present
in a dataset, then the `dataset.index_statistics` will return an error.
Before using `dataset.index_statistics()`, call `dataset.validate()` to
check the integrity and use `dataset.delete("false")` to force a
migration.**

Fixes #3374

Cargo.toml

@@ -21,7 +21,7 @@ exclude = ["python"]
 resolver = "2"
 
 [workspace.package]
-version = "0.22.1"
+version = "0.23.0"
 edition = "2021"
 authors = ["Lance Devs <[email protected]>"]
 license = "Apache-2.0"
@@ -44,21 +44,21 @@ categories = [
 rust-version = "1.80.1"
 
 [workspace.dependencies]
-lance = { version = "=0.22.1", path = "./rust/lance" }
-lance-arrow = { version = "=0.22.1", path = "./rust/lance-arrow" }
-lance-core = { version = "=0.22.1", path = "./rust/lance-core" }
-lance-datafusion = { version = "=0.22.1", path = "./rust/lance-datafusion" }
-lance-datagen = { version = "=0.22.1", path = "./rust/lance-datagen" }
-lance-encoding = { version = "=0.22.1", path = "./rust/lance-encoding" }
-lance-encoding-datafusion = { version = "=0.22.1", path = "./rust/lance-encoding-datafusion" }
-lance-file = { version = "=0.22.1", path = "./rust/lance-file" }
-lance-index = { version = "=0.22.1", path = "./rust/lance-index" }
-lance-io = { version = "=0.22.1", path = "./rust/lance-io" }
-lance-jni = { version = "=0.22.1", path = "./java/core/lance-jni" }
-lance-linalg = { version = "=0.22.1", path = "./rust/lance-linalg" }
-lance-table = { version = "=0.22.1", path = "./rust/lance-table" }
-lance-test-macros = { version = "=0.22.1", path = "./rust/lance-test-macros" }
-lance-testing = { version = "=0.22.1", path = "./rust/lance-testing" }
+lance = { version = "=0.23.0", path = "./rust/lance" }
+lance-arrow = { version = "=0.23.0", path = "./rust/lance-arrow" }
+lance-core = { version = "=0.23.0", path = "./rust/lance-core" }
+lance-datafusion = { version = "=0.23.0", path = "./rust/lance-datafusion" }
+lance-datagen = { version = "=0.23.0", path = "./rust/lance-datagen" }
+lance-encoding = { version = "=0.23.0", path = "./rust/lance-encoding" }
+lance-encoding-datafusion = { version = "=0.23.0", path = "./rust/lance-encoding-datafusion" }
+lance-file = { version = "=0.23.0", path = "./rust/lance-file" }
+lance-index = { version = "=0.23.0", path = "./rust/lance-index" }
+lance-io = { version = "=0.23.0", path = "./rust/lance-io" }
+lance-jni = { version = "=0.23.0", path = "./java/core/lance-jni" }
+lance-linalg = { version = "=0.23.0", path = "./rust/lance-linalg" }
+lance-table = { version = "=0.23.0", path = "./rust/lance-table" }
+lance-test-macros = { version = "=0.23.0", path = "./rust/lance-test-macros" }
+lance-testing = { version = "=0.23.0", path = "./rust/lance-testing" }
 approx = "0.5.1"
 # Note that this one does not include pyarrow
 arrow = { version = "53.2", optional = false, features = ["prettyprint"] }
@@ -114,7 +114,7 @@ datafusion-physical-expr = { version = "44.0" }
 deepsize = "0.2.0"
 dirs = "5.0.0"
 either = "1.0"
-fsst = { version = "=0.22.1", path = "./rust/lance-encoding/src/compression_algo/fsst" }
+fsst = { version = "=0.23.0", path = "./rust/lance-encoding/src/compression_algo/fsst" }
 futures = "0.3"
 http = "1.1.0"
 hyperloglogplus = { version = "0.4.1", features = ["const-loop"] }

java/core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lance-parent</artifactId>
-        <version>0.21.2</version>
+        <version>0.23.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lance-parent</artifactId>
-    <version>0.21.2</version>
+    <version>0.23.0</version>
     <packaging>pom</packaging>
 
     <name>Lance Parent</name>

java/spark/pom.xml

@@ -8,7 +8,7 @@
     <parent>
         <groupId>com.lancedb</groupId>
         <artifactId>lance-parent</artifactId>
-        <version>0.21.2</version>
+        <version>0.23.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -112,7 +112,7 @@
         <dependency>
             <groupId>com.lancedb</groupId>
             <artifactId>lance-core</artifactId>
-            <version>0.21.2</version>
+            <version>0.23.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pylance"
-version = "0.22.1"
+version = "0.23.0"
 edition = "2021"
 authors = ["Lance Devs <[email protected]>"]
 rust-version = "1.65"