Skip to content

Commit

Permalink
Merge pull request #7 from francisco-dlp/nion15update_francisco
Browse files Browse the repository at this point in the history 
Nion15update Part-2
  • Loading branch information
@francisco-dlp
francisco-dlp authored Oct 25, 2022
2 parents 9f0feca + f806a8f commit 9b8ac1e
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 131 deletions.
59 changes: 31 additions & 28 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,34 +18,37 @@ pip install --upgrade https://github.com/hyperspy/hyperspy_swift_library/archive

```python
>>> from hyperspy_swift_library import SwiftLibraryReader
>>> import pandas as pd
>>> lib = SwiftLibraryReader("Nion Swift Library 20200606/")
>>> lib.list_data_items(signal_type="image")
0
Title: HADF
Created: 2020-06-06T11:04:36.368747
Shape: [2048, 2048]
Datum dimension: 2
1
Title: LowMag2_TEM
Created: 2020-06-06T11:05:57.354602
Shape: [520, 696]
Datum dimension: 2
2
Title: LowMag1_TEM
Created: 2020-06-06T11:05:57.405164
Shape: [520, 696]
Datum dimension: 2

>>> lib.load_data(2, lazy=True)
<LazySignal2D, title: , dimensions: (|512, 512)>

>>> df = lib.get_data_items()
>>> df["title"]
0 HADF
1 LowMag2_TEM
2 LowMag1_TEM
>>> project = SwiftLibraryReader("Nion Swift Project 20221025.nsproj")
```

If [pandas]() is installed, the `get_data_items_properties` returns a Pandas `DataFrame`.

Using Pandas syntax, it is very easy to select the information that you want to display.
For example, to display only the title and data_shape columns of the `DataFrame`:

```python
>>> df = project.get_data_items_properties()
>>> df[["title", "data_shape"]]
title data_shape
0 EELS [128, 1024]
1 Orsay Scan (BF) [512, 512]
2 EELS [1024]
3 Orsay Scan (eels_spim) [256, 256, 1024]
4 Pick Sum of Orsay Scan (eels_spim) <Not data>
```

To filter data based on its title:

```python
>>> df[df['title'].str.endswith("(BF)")]['title']
1 Orsay Scan (BF)
Name: title, dtype: object
```

To load data as a HyperSpy signal:

>>> df[df["title"].str.endswith("_TEM")] # Can be used for filtering based on "title"
```python
>>> s = project.load_data(0)
>>> s
<LazySignal2D, title: , dimensions: (|1024, 128)>
```
173 changes: 71 additions & 102 deletions hyperspy_swift_library/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,56 +1,18 @@
import pathlib

from nion.swift.model import DocumentModel
from nion.swift.model import DataItem
from nion.swift.model import FileStorageSystem
from nion.swift.model import Persistence
from nion.swift.model import Profile
from nion.swift.model import Project
from nion.swift.model import HardwareSource
import pathlib
import typing
import dask.array as da
from hyperspy.misc.utils import DictionaryTreeBrowser
from hyperspy._signals.signal1d import Signal1D, LazySignal1D
from hyperspy._signals.signal2d import Signal2D, LazySignal2D
import os

__version__ = "0.2"


__version__ = "0.1"


def get_storage_system(workspace_dir):
# This function is adapted from Swift's profile
workspace_dir = pathlib.Path(workspace_dir)
library_path = Profile._migrate_library(workspace_dir, do_logging=True)
this_storage_version = DataItem.DataItem.storage_version
auto_migrations = list()
auto_migrations.append(
Profile.AutoMigration(
pathlib.Path(workspace_dir) /
"Nion Swift Workspace.nslib",
[
pathlib.Path(workspace_dir) /
"Nion Swift Data"]))
auto_migrations.append(
Profile.AutoMigration(
pathlib.Path(workspace_dir) /
"Nion Swift Workspace.nslib",
[
pathlib.Path(workspace_dir) /
"Nion Swift Data 10"]))
auto_migrations.append(
Profile.AutoMigration(
pathlib.Path(workspace_dir) /
"Nion Swift Workspace.nslib",
[
pathlib.Path(workspace_dir) /
"Nion Swift Data 11"]))
# Attemp at being future proof
if this_storage_version > 12:
for storage_version in range(12, this_storage_version):
auto_migrations.append(
Profile.AutoMigration(pathlib.Path(workspace_dir) / f"Nion Swift Library {storage_version}.nslib",
[pathlib.Path(workspace_dir) / f"Nion Swift Data {storage_version}"]))

# NOTE: when adding an AutoMigration here, also add the corresponding file
# copy in _migrate_library
storage_system = FileStorageSystem.FileStorageSystem(library_path, [pathlib.Path(
workspace_dir) / f"Nion Swift Data {this_storage_version}"], auto_migrations=auto_migrations)
return storage_system


def axes_swift2hspy(axes, shape):
Expand All @@ -60,77 +22,84 @@ def axes_swift2hspy(axes, shape):


class SwiftLibraryReader:
def __init__(self, workspace_dir):
self._storage_system = get_storage_system(workspace_dir)
self._data_items = sorted(
self._storage_system.find_data_items(),
key=lambda x: x.read_properties()["created"])
self._data_items_properties = [
di.read_properties() for di in self._data_items]

def get_data_items(self):
"""Creates a DataFrame containing data_items properties in a NionSwift library
"""Load items in Nion Swift project with HyperSpy"""

def __init__(self, project_file_path):
"""Parse Nion Swift project file
Parameters
----------
project_file_path: str
The path of the Nion Swift project file with extension `.nsproj`.
"""
current_path = os.getcwd()
file_path = pathlib.Path(project_file_path)

if not os.path.isfile(file_path):
raise ValueError(f"The file {project_file_path} does not exist")
if file_path.suffix == ".nsproj":
r = Profile.IndexProjectReference()
r.project_path = file_path
else:
raise ValueError("Please provide a file with extension `.nsproj`")
r.persistent_object_context = Persistence.PersistentObjectContext()
r.load_project(None, pathlib.Path("."), cache_factory=None)
# r.project._raw_properties["version"] = 3
r.project.read_project()
self._idp = r

def get_data_items_properties(self):
"""Data items properties
Returns
----------
DataFrame : Pandas DataFrame containing data_items properties if Pandas is installed; otherwise a dictionary containing the same data.
Examples
--------
properties : pandas.DataFrame or dictionary
Pandas DataFrame containing data_items properties if Pandas is installed; otherwise a dictionary containing the same data.
>>> df["title"]
0 HADF
1 LowMag2_TEM
2 LowMag1_TEM
Name: title, dtype: object
"""

>>> df[df["title"].str.endswith("_TEM")] # can be used for filtering based on "title".
p = self._idp.project
properties = {}
for n, data_item in enumerate(p.data_items):
if data_item.xdata:
for key in data_item.properties.keys():
if key in properties.keys():
properties[key].append(data_item.properties[key])
else:
properties[key] = [data_item.properties[key]]
else:
for item in properties.items():
if item[0] == "title":
item[1].append(data_item.properties["title"])
else:
item[1].append("<Not data>")

"""
try:
import pandas as pd
df = pd.DataFrame(self._data_items_properties)
df = pd.DataFrame(properties)
return df
except ImportError as e:
properties = self._data_items_properties
return properties

def list_data_items(self, signal_type=None):
for i, md in enumerate(self._data_items_properties):
if signal_type == "ndspectrum":
if md["datum_dimension_count"] != 1 or md["data_shape"][0] < 2:
continue
elif signal_type == "spectrum":
if md["datum_dimension_count"] != 1 or md["data_shape"][0] > 1:
continue
elif signal_type == "image":
if md["datum_dimension_count"] != 2 or len(
md["data_shape"]) != 2:
continue
elif signal_type == "ndimage":
if md["datum_dimension_count"] != 2 or len(
md["data_shape"]) < 3:
continue
elif signal_type is not None:
raise ValueError(
"signal_type must be one of: ndspectrum, spectrum, ndimage, image, not %s" %
signal_type)
datum_dimension_count = 1
print(f"{i}")
print(f'\tTitle: {md["title"]}')
print(f'\tCreated: {md["created"]}')
print(f'\tShape: {md["data_shape"]}')
print(f'\tDatum dimension: {md["datum_dimension_count"]}')

def load_data(self, num, lazy=True):
handler = self._data_items[num]
md = self._data_items_properties[num]
"""The data item in a HyperSpy signal
Parameters:
----------
num: int
The index of the data item.
lazy: bool
If True, the data is not loaded into memory.
"""
project = self._idp.project
handler = project.data_items[num]
md = handler.properties
if md["datum_dimension_count"] == 1:
Signal = LazySignal1D if lazy else Signal1D
elif md["datum_dimension_count"] == 2:
Signal = LazySignal2D if lazy else Signal2D
data = handler.read_data()
data = handler.data
if lazy:
data = da.from_array(data)
signal = Signal(
Expand All @@ -139,4 +108,4 @@ def load_data(self, num, lazy=True):
md["dimensional_calibrations"],
shape=md["data_shape"]))
signal.original_metadata.add_dictionary(md)
return signal
return signal
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
long_description_content_type="text/markdown",
url="https://github.com/hyperspy/hyperspy_swift_libray",
packages=setuptools.find_packages(),
install_requires=["nionswift>0.14", "hyperspy"],
install_requires=["nionswift>=0.15", "hyperspy"],
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License",
Expand Down

0 comments on commit 9b8ac1e

Please sign in to comment.