[Bug]: validation error on NWB Zarr file

[CodyCBakerPhD]

What happened?

Ran across this while debugging Zarr support for NWB Inspector NeurodataWithoutBorders/nwbinspector#513

An actual error this time, for a relatively simple file

Only detectable by running validation on the file (io.write and io.read both work fine)

Opening here since it only happens for NWBZarrIO, but might be a HDMF or PyNWB issue so feel free to transfer this

Steps to Reproduce

import datetime
import uuid

import hdmf_zarr
import numpy
import pynwb

nwbfile = pynwb.NWBFile(
    session_description="",
    identifier=str(uuid.uuid4()),
    session_start_time=datetime.datetime.now().astimezone()
)
regular_timestamps = numpy.arange(1.2, 11.2, 2)
timestamps_length = len(regular_timestamps)
time_series = pynwb.TimeSeries(
    name="test_time_series",
    data=numpy.zeros(shape=(timestamps_length, timestamps_length - 1)),
    timestamps=regular_timestamps,
    unit="",
)
nwbfile.add_acquisition(time_series)

nwbfile_path = "../test_validation_time_series.nwb.zarr"
with hdmf_zarr.NWBZarrIO(path=nwbfile_path, mode="w") as io:
    io.write(nwbfile)

with hdmf_zarr.NWBZarrIO(path=nwbfile_path, mode="r") as io:
    invalidations = pynwb.validate(io=io)

Traceback

Traceback (most recent call last):
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\IPython\core\interactiveshell.py", line 3577, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-5-43139d2216bf>", line 28, in <module>
    invalidations = pynwb.validate(io=io)
                    ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\utils.py", line 672, in func_call
    return func(**pargs)
           ^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\pynwb\validate.py", line 145, in validate
    validation_errors = _validate_helper(io=io, namespace=namespace or CORE_NAMESPACE)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\pynwb\validate.py", line 29, in _validate_helper
    return validator.validate(builder)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\utils.py", line 668, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 299, in validate
    return validator.validate(builder)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\utils.py", line 668, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 479, in validate
    errors.extend(self.__validate_children(builder))
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 505, in __validate_children
    yield from self.__validate_child_builder(child_spec, child_builder, parent_builder)
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 555, in __validate_child_builder
    yield from child_validator.validate(child_builder)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\utils.py", line 668, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 479, in validate
    errors.extend(self.__validate_children(builder))
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 505, in __validate_children
    yield from self.__validate_child_builder(child_spec, child_builder, parent_builder)
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 555, in __validate_child_builder
    yield from child_validator.validate(child_builder)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\utils.py", line 668, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 479, in validate
    errors.extend(self.__validate_children(builder))
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 505, in __validate_children
    yield from self.__validate_child_builder(child_spec, child_builder, parent_builder)
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 555, in __validate_child_builder
    yield from child_validator.validate(child_builder)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\utils.py", line 668, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 437, in validate
    dtype, string_format = get_type(data, builder.dtype)
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\theac\anaconda3\envs\nwbinspector_created_9_25_24\Lib\site-packages\hdmf\validate\validator.py", line 157, in get_type
    dtype, string_format = get_type(data[0][i])
                                    ~~~~~~~^^^
IndexError: invalid index to scalar variable.

Operating System

Windows

Python Executable

Conda

Python Version

3.12

Package Versions

No response

Code of Conduct

• I agree to follow this project's Code of Conduct
• Have you checked the Contributing document?
• Have you ensured this bug was not already reported?

[mavaylon1]

Thanks for raising this issue. We will take a look after SFN.

[mavaylon1]

Traceback (most recent call last):
  File "/Users/mavaylon/Research/NWB/hdmf-zarr/tests/unit/reproduce.py", line 28, in <module>
    invalidations = pynwb.validate(io=io)
                    ^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mavaylon/Research/NWB/hdmf/src/hdmf/utils.py", line 672, in func_call
    return func(**pargs)
           ^^^^^^^^^^^^^
  File "/Users/mavaylon/Research/NWB/pynwb/src/pynwb/validate.py", line 145, in validate
    validation_errors = _validate_helper(io=io, namespace=namespace or CORE_NAMESPACE)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mavaylon/Research/NWB/pynwb/src/pynwb/validate.py", line 29, in _validate_helper
    return validator.validate(builder)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mavaylon/Research/NWB/hdmf/src/hdmf/utils.py", line 668, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mavaylon/Research/NWB/hdmf/src/hdmf/validate/validator.py", line 299, in validate
    return validator.validate(builder)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mavaylon/Research/NWB/hdmf/src/hdmf/utils.py", line 668, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mavaylon/Research/NWB/hdmf/src/hdmf/validate/validator.py", line 479, in validate
    errors.extend(self.__validate_children(builder))
  File "/Users/mavaylon/Research/NWB/hdmf/src/hdmf/validate/validator.py", line 505, in __validate_children
    yield from self.__validate_child_builder(child_spec, child_builder, parent_builder)
  File "/Users/mavaylon/Research/NWB/hdmf/src/hdmf/validate/validator.py", line 555, in __validate_child_builder
    yield from child_validator.validate(child_builder)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mavaylon/Research/NWB/hdmf/src/hdmf/utils.py", line 668, in func_call
    return func(args[0], **pargs)
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mavaylon/Research/NWB/hdmf/src/hdmf/validate/validator.py", line 438, in validate
    if not check_type(self.spec.dtype, dtype, string_format):
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/mavaylon/Research/NWB/hdmf/src/hdmf/validate/validator.py", line 67, in check_type
    if 'vlen' in received.metadata:
       ^^^^^^^^^^^^^^^^^^^^^^^^^^^
TypeError: argument of type 'NoneType' is not iterable

[mavaylon1]

The example gives me a different trace. PyNWB did have a release since this was posted (possible reason). Either way, currently looking into this.