Merge pull request #11787 from rouault/zarr_v3_update

Zarr V3: update with latest Zarr V3 specification
OSGeo · Feb 10, 2025 · 6884b62 · 6884b62
2 parents e1c8701 + 99b1c0e
commit 6884b62
Show file tree

Hide file tree

Showing 17 changed files with 407 additions and 201 deletions.
diff --git a/autotest/gdrivers/data/zarr/v3/test.zr3/ar/zarr.json b/autotest/gdrivers/data/zarr/v3/test.zr3/ar/zarr.json
@@ -19,5 +19,6 @@
       "separator":"\/"
     }
   },
-  "fill_value": 255
+  "fill_value": 255,
+  "codecs": [{"name":"bytes"}]
 }
diff --git a/autotest/gdrivers/data/zarr/v3/test.zr3/marvin/android/zarr.json b/autotest/gdrivers/data/zarr/v3/test.zr3/marvin/android/zarr.json
@@ -18,5 +18,6 @@
   "chunk_key_encoding":{
     "name":"v2"
   },
-  "fill_value":255
+  "fill_value":255,
+  "codecs": [{"name":"bytes"}]
 }
diff --git a/autotest/gdrivers/data/zarr/v3/test.zr3/marvin/paranoid/zarr.json b/autotest/gdrivers/data/zarr/v3/test.zr3/marvin/paranoid/zarr.json
@@ -0,0 +1,4 @@
+{
+  "zarr_format":3,
+  "node_type":"group"
+}
diff --git a/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/ar/c/0 b/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/ar/c/0
@@ -0,0 +1 @@
+
diff --git a/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/ar/zarr.json b/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/ar/zarr.json
@@ -0,0 +1,23 @@
+{
+  "zarr_format":3,
+  "node_type":"array",
+  "shape":[
+    2
+  ],
+  "data_type":"uint8",
+  "chunk_grid":{
+    "name":"regular",
+    "configuration":{
+      "chunk_shape":[
+        2
+      ]
+    }
+  },
+  "chunk_key_encoding":{
+    "name":"default",
+    "configuration":{
+      "separator":"\/"
+    }
+  },
+  "fill_value": 255
+}
diff --git a/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/marvin/android/0.0 b/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/marvin/android/0.0
@@ -0,0 +1 @@
+
diff --git a/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/marvin/android/zarr.json b/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/marvin/android/zarr.json
@@ -0,0 +1,22 @@
+{
+  "zarr_format":3,
+  "node_type":"array",
+  "shape":[
+    5,
+    4
+  ],
+  "data_type":"uint8",
+  "chunk_grid":{
+    "name":"regular",
+    "configuration":{
+      "chunk_shape":[
+        5,
+        4
+      ]
+    }
+  },
+  "chunk_key_encoding":{
+    "name":"v2"
+  },
+  "fill_value":255
+}
diff --git a/...test.zr3/marvin/paranoid/DO_NOT_REMOVE_ME → ...decs.zr3/marvin/paranoid/DO_NOT_REMOVE_ME b/...test.zr3/marvin/paranoid/DO_NOT_REMOVE_ME → ...decs.zr3/marvin/paranoid/DO_NOT_REMOVE_ME
diff --git a/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/marvin/zarr.json b/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/marvin/zarr.json
@@ -0,0 +1,7 @@
+{
+  "zarr_format":3,
+  "node_type":"group",
+  "attributes":{
+    "foo":"bar"
+  }
+}
diff --git a/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/zarr.json b/autotest/gdrivers/data/zarr/v3/test_deprecated_no_codecs.zr3/zarr.json
@@ -0,0 +1,7 @@
+{
+  "zarr_format":3,
+  "node_type":"group",
+  "attributes":{
+    "root_foo":"bar"
+  }
+}
diff --git a/autotest/gdrivers/zarr_driver.py b/autotest/gdrivers/zarr_driver.py
@@ -982,11 +982,13 @@ def test_zarr_read_ARRAY_DIMENSIONS(use_zmetadata, filename):
     assert len(rg.GetDimensions()) == 2
 
 
+@pytest.mark.parametrize(
+    "ds_name", ["data/zarr/v3/test_deprecated_no_codecs.zr3", "data/zarr/v3/test.zr3"]
+)
 @pytest.mark.parametrize("use_get_names", [True, False])
-def test_zarr_read_v3(use_get_names):
+def test_zarr_read_v3(ds_name, use_get_names):
 
-    filename = "data/zarr/v3/test.zr3"
-    ds = gdal.OpenEx(filename, gdal.OF_MULTIDIM_RASTER)
+    ds = gdal.OpenEx(ds_name, gdal.OF_MULTIDIM_RASTER)
     assert ds is not None
     rg = ds.GetRootGroup()
     assert rg.GetName() == "/"
@@ -1930,21 +1932,28 @@ def create():
 @pytest.mark.parametrize(
     "compressor,options,expected_json",
     [
-        ["NONE", [], None],
+        ["NONE", [], [{"name": "bytes", "configuration": {"endian": "little"}}]],
         [
             "gzip",
             [],
-            [{"name": "gzip", "configuration": {"level": 6}}],
+            [
+                {"name": "bytes", "configuration": {"endian": "little"}},
+                {"name": "gzip", "configuration": {"level": 6}},
+            ],
         ],
         [
             "gzip",
             ["GZIP_LEVEL=1"],
-            [{"name": "gzip", "configuration": {"level": 1}}],
+            [
+                {"name": "bytes", "configuration": {"endian": "little"}},
+                {"name": "gzip", "configuration": {"level": 1}},
+            ],
         ],
         [
             "blosc",
             [],
             [
+                {"name": "bytes", "configuration": {"endian": "little"}},
                 {
                     "name": "blosc",
                     "configuration": {
@@ -1954,7 +1963,7 @@ def create():
                         "typesize": 1,
                         "blocksize": 0,
                     },
-                }
+                },
             ],
         ],
         [
@@ -1966,6 +1975,7 @@ def create():
                 "BLOSC_BLOCKSIZE=2",
             ],
             [
+                {"name": "bytes", "configuration": {"endian": "little"}},
                 {
                     "name": "blosc",
                     "configuration": {
@@ -1974,7 +1984,23 @@ def create():
                         "shuffle": "noshuffle",
                         "blocksize": 2,
                     },
-                }
+                },
+            ],
+        ],
+        [
+            "zstd",
+            ["ZSTD_LEVEL=20"],
+            [
+                {"name": "bytes", "configuration": {"endian": "little"}},
+                {"name": "zstd", "configuration": {"level": 20, "checksum": False}},
+            ],
+        ],
+        [
+            "zstd",
+            ["ZSTD_CHECKSUM=YES"],
+            [
+                {"name": "bytes", "configuration": {"endian": "little"}},
+                {"name": "zstd", "configuration": {"level": 13, "checksum": True}},
             ],
         ],
     ],
@@ -2032,28 +2058,28 @@ def read():
     [
         [
             ["@ENDIAN=little"],
-            [{"configuration": {"endian": "little"}, "name": "endian"}],
+            [{"configuration": {"endian": "little"}, "name": "bytes"}],
         ],
-        [["@ENDIAN=big"], [{"configuration": {"endian": "big"}, "name": "endian"}]],
+        [["@ENDIAN=big"], [{"configuration": {"endian": "big"}, "name": "bytes"}]],
         [
             ["@ENDIAN=little", "CHUNK_MEMORY_LAYOUT=F"],
             [
                 {"name": "transpose", "configuration": {"order": "F"}},
-                {"configuration": {"endian": "little"}, "name": "endian"},
+                {"configuration": {"endian": "little"}, "name": "bytes"},
             ],
         ],
         [
             ["@ENDIAN=big", "CHUNK_MEMORY_LAYOUT=F"],
             [
                 {"name": "transpose", "configuration": {"order": "F"}},
-                {"configuration": {"endian": "big"}, "name": "endian"},
+                {"configuration": {"endian": "big"}, "name": "bytes"},
             ],
         ],
         [
             ["@ENDIAN=big", "CHUNK_MEMORY_LAYOUT=F", "COMPRESS=GZIP"],
             [
                 {"name": "transpose", "configuration": {"order": "F"}},
-                {"name": "endian", "configuration": {"endian": "big"}},
+                {"name": "bytes", "configuration": {"endian": "big"}},
                 {"name": "gzip", "configuration": {"level": 6}},
             ],
         ],
@@ -3180,6 +3206,7 @@ def create():
             assert "codecs" in j
             assert j["codecs"] == [
                 {"name": "transpose", "configuration": {"order": "F"}},
+                {"name": "bytes", "configuration": {"endian": "little"}},
                 {"name": "gzip", "configuration": {"level": 6}},
             ]
 
@@ -5512,7 +5539,7 @@ def test_zarr_driver_copy_files(format):
         assert gdal.Open(filename)
 
         assert gdal.VSIStatL(newfilename)
-        print(gdal.ReadDirRecursive(newfilename))
+        # print(gdal.ReadDirRecursive(newfilename))
         assert gdal.Open(newfilename)
 
     finally:

diff --git a/doc/source/drivers/raster/zarr.rst b/doc/source/drivers/raster/zarr.rst
@@ -348,6 +348,9 @@ with ``ARRAY:`` using :program:`gdalmdimtranslate`):
 
       Compression method.
 
+      For FORMAT=ZARR_V3, only ``NONE``, ``BLOSC``, ``GZIP`` and ``ZSTD`` are
+      supported.
+
 -  .. co:: FILTER
       :choices: NONE, DELTA
       :default: NONE