added a json converter class in json_encoder.py to pyscan/general. Th…

pyscan/drivers/testing/driver_test_logs/test_instrument_driver_test_log.txt

@@ -1,8 +1,9 @@
-Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.4.0 at 2024-06-05 15:07:38
-Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.4.0 at 2024-06-05 15:07:38
-Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.4.0 at 2024-06-05 15:07:38
-Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.4.0 at 2024-06-05 15:07:38
-Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.4.0 at 2024-06-05 15:06:58
-Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.4.0 at 2024-06-05 15:06:57
-Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.4.0 at 2024-06-05 15:06:57
-Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.3.0 at 2024-05-30 10:39:46
+Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 10:06:04
+Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 10:06:04
+Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 10:06:04
+Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 10:06:04
+Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 10:00:57
+Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 10:00:57
+Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 10:00:57
+Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 10:00:57
+Passed with test_instrument_driver version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 09:51:39

pyscan/drivers/testing/driver_test_logs/test_voltage_test_log.txt

@@ -1,3 +1,3 @@
-Passed with test_voltage version v0.1.0 tested on pyscan version v0.4.0 at 2024-06-05 15:07:38
-Passed with test_voltage version v0.1.0 tested on pyscan version v0.4.0 at 2024-06-05 15:06:58
-Passed with test_voltage version v0.1.0 tested on pyscan version v0.3.0 at 2024-05-30 10:39:46
+Passed with test_voltage version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 10:06:04
+Passed with test_voltage version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 10:00:57
+Passed with test_voltage version v0.1.0 tested on pyscan version v0.5.1 at 2024-06-17 09:51:39

pyscan/general/__init__.py

@@ -1,16 +1,15 @@
+# objects
+from .item_attribute import ItemAttribute
+from .json_encoder import CustomJSONEncoder
 
 # methods
 from .d_range import drange
 from .first_string import first_string
 from .is_list_type import is_list_type
 from .is_numeric_type import is_numeric_type
 from .quadrature_sum import quadrature_sum
-from .recursive_to_dict import recursive_to_dict
-from .recursive_to_item_attribute import recursive_to_itemattribute
 from .same_length import same_length
 from .set_difference import set_difference
 from .stack_or_append import stack_or_append
 from .get_pyscan_version import get_pyscan_version
-
-# objects
-from .item_attribute import ItemAttribute
+from .json_decoder import item_attribute_object_hook

pyscan/general/json_decoder.py

@@ -0,0 +1,26 @@
+from pyscan.general import ItemAttribute
+
+
+def item_attribute_object_hook(data):
+    '''
+    Function to be used as the object_hook in json.loads to convert dictionaries
+    into ItemAttribute objects.
+
+    Parameters
+    ----------
+    data : dict
+        The dictionary to convert.
+
+    Returns
+    -------
+    ItemAttribute
+    '''
+    new_data = ItemAttribute()
+
+    for key, value in data.items():
+        if isinstance(value, dict):
+            # Recursively convert nested dictionaries
+            value = item_attribute_object_hook(value)
+        new_data[key] = value
+
+    return new_data

pyscan/general/json_encoder.py

@@ -0,0 +1,117 @@
+import json
+import numpy as np
+
+
+class CustomJSONEncoder(json.JSONEncoder):
+    """
+    A custom JSON encoder subclassing json.JSONEncoder to handle additional data types.
+    Used in pyscan to properly format data for json.dumps when saving metadata.
+
+    This encoder extends the default JSONEncoder to support serialization of several
+    additional Python data types, including NumPy data types and objects with a __dict__
+    attribute, making it suitable for serializing more complex Python objects into JSON format.
+
+    The encoder handles the following data types specifically:
+    - None: Serialized as the string 'None'.
+    - Boolean, String, Integer, Float: Serialized using the default JSON serialization.
+    - Callable objects: Serialized by converting their __dict__ attribute using recursive_to_dict.
+    - Dictionaries: Serialized using recursive_to_dict to handle nested dictionaries.
+    - Classes (type objects): Serialized as is (may require custom handling).
+    - NumPy integers and floats: Serialized as standard Python integers and floats.
+    - NumPy arrays: Serialized as lists using the tolist() method.
+    - Objects with a __dict__ attribute: Serialized by converting their __dict__ attribute using recursive_to_dict.
+    - Iterators (excluding strings, bytes, and byte arrays): Serialized as lists.
+
+    For any other data types not explicitly handled, the encoder falls back to the default
+    JSONEncoder handling, which may raise a TypeError if the type is not serializable.
+
+    Methods
+    -------
+    default(obj):
+        Overridden method from json.JSONEncoder that specifies how to serialize the supported data types.
+
+    recursive_to_dict(obj_dict):
+        Helper method to recursively serialize objects and dictionaries, particularly useful for
+        objects with nested dictionaries or __dict__ attributes.
+
+    Example
+    -------
+    >>> import numpy as np
+    >>> data = {
+    ...     "numpy_int": np.int32(10),
+    ...     "numpy_float": np.float64(3.14),
+    ...     "numpy_array": np.array([1, 2, 3]),
+    ...     "custom_object": CustomObject()
+    ... }
+    >>> encoded_str = json.dumps(data, cls=CustomJSONEncoder)
+    """
+
+    def default(self, obj):
+        if obj is None:
+            return 'None'
+        elif isinstance(obj, (bool, str, int, float)):
+            return obj
+        elif hasattr(obj, '__call__'):
+            return self.recursive_to_dict(obj.__dict__)
+        elif isinstance(obj, dict):
+            return self.recursive_to_dict(obj)
+        elif isinstance(obj, type):
+            return obj
+        # Handle numpy integers
+        elif isinstance(obj, (np.integer, np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64,
+                              np.uint8, np.uint16, np.uint32, np.uint64)):
+            return int(obj)
+        # Handle numpy floating values
+        elif isinstance(obj, (np.floating, np.float16, np.float32, np.float64)):
+            return float(obj)
+        # Handle numpy arrays
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        # Handle objects with a __dict__ attribute
+        elif hasattr(obj, "__dict__"):
+            return self.recursive_to_dict(obj.__dict__)
+        # Handle iterators (excluding strings, bytes, and byte arrays)
+        elif hasattr(obj, "__iter__"):
+            return list(obj)
+        # Fallback: use the base class handling, which handles strings and types serializable by default json encoder
+        else:
+            return super().default(obj)
+
+    def recursive_to_dict(self, obj_dict):
+        new_dict = {}
+
+        for key, value in obj_dict.items():
+            # print(key, value)
+            # is method/function
+            if key in ['logger', 'expt_thread', 'data_path',
+                       'instrument', 'module_id_string', 'spec']:
+                pass
+            elif hasattr(value, '__call__'):
+                new_dict[key] = value.__name__
+            elif isinstance(value, str):
+                new_dict[key] = value
+            # is a dict
+            elif isinstance(value, dict):
+                new_dict[key] = self.recursive_to_dict(value)
+            # if it is a np integer
+            elif isinstance(value, (np.integer, np.int_, np.intc, np.intp, np.int8, np.int16, np.int32, np.int64,
+                                    np.uint8, np.uint16, np.uint32, np.uint64)):
+                new_dict[key] = int(value)
+            # if it is a np floating value
+            elif isinstance(value, (np.floating, np.float16, np.float32, np.float64)):
+                new_dict[key] = float(value)
+            # if it is an np array
+            elif isinstance(value, np.ndarray):
+                new_dict[key] = value.tolist()
+            # is an iterator
+            elif hasattr(value, "__iter__"):
+                new_dict[key] = list(value)
+            # is an object
+            elif hasattr(value, "__dict__"):
+                new_dict[key] = self.recursive_to_dict(value.__dict__)
+            # anything else
+            else:
+                new_dict[key] = value
+                # maybe pass this, but test first
+
+        return new_dict

pyscan/measurement/abstract_experiment.py

@@ -6,9 +6,9 @@
 from threading import Thread as thread
 from time import strftime
 from pyscan.general import (ItemAttribute,
-                            recursive_to_dict,
                             is_list_type)
 from pyscan.measurement.scans import PropertyScan, RepeatScan
+from pyscan.general.json_encoder import CustomJSONEncoder
 
 
 class AbstractExperiment(ItemAttribute):
@@ -226,11 +226,9 @@ def save_metadata(self):
         save_path = self.runinfo.data_path / '{}.hdf5'.format(self.runinfo.long_name)
         save_name = str(save_path.absolute())
 
-        data = recursive_to_dict(self.__dict__)
-
         with h5py.File(save_name, 'a') as f:
-            f.attrs['runinfo'] = json.dumps(data['runinfo'])
-            f.attrs['devices'] = json.dumps(data['devices'])
+            f.attrs['runinfo'] = json.dumps(self.runinfo, cls=CustomJSONEncoder)
+            f.attrs['devices'] = json.dumps(self.devices, cls=CustomJSONEncoder)
 
     def start_thread(self):
         '''Starts experiment as a background thread, this works in conjunction with live plot

pyscan/measurement/load_experiment.py

@@ -3,7 +3,8 @@
 import pickle
 import json
 from pathlib import Path
-from pyscan.general import ItemAttribute, recursive_to_itemattribute
+from pyscan.general import ItemAttribute
+from pyscan.general.json_decoder import item_attribute_object_hook
 
 
 def load_experiment(file_name):
@@ -35,7 +36,7 @@ def load_experiment(file_name):
             open('{}.pkl'.format(file_name), "rb"))
 
         expt = ItemAttribute()
-        expt.runinfo = ItemAttribute()
+        # expt.runinfo = ItemAttribute()
         expt.devices = ItemAttribute()
 
         for key, value in meta_data['runinfo'].items():
@@ -53,15 +54,11 @@ def load_experiment(file_name):
 
     elif data_version == 0.2:
         expt = ItemAttribute()
-        expt.runinfo = ItemAttribute()
-        expt.devices = ItemAttribute()
 
         f = h5py.File('{}'.format(file_name), 'r')
-        runinfo = json.loads(f.attrs['runinfo'])
-        expt.runinfo = recursive_to_itemattribute(runinfo)
+        expt.runinfo = json.loads(f.attrs['runinfo'], object_hook=item_attribute_object_hook)
 
-        devices = json.loads(f.attrs['devices'])
-        expt.devices = recursive_to_itemattribute(devices)
+        expt.devices = json.loads(f.attrs['devices'], object_hook=item_attribute_object_hook)
 
         for key, value in f.items():
             expt[key] = (f[key][:]).astype('float64')

test/measurement/test_abstract_experiment.py

@@ -290,3 +290,6 @@ def test_ms_diff_inputs(data_dir=None, measure_function=measure_point, allocate=
     test_ms_diff_inputs(data_dir=None, measure_function=measure_up_to_3D)
     test_ms_diff_inputs(data_dir='./backup', measure_function=measure_up_to_3D)
     test_ms_diff_inputs(data_dir='./backup', measure_function=measure_up_to_3D, allocate='preallocate_line')
+
+
+test_abstract_experiment()