python: rewrite Python high level API in python

.gitignore

@@ -40,3 +40,7 @@ docs/pyvenv.cfg
 # Visual Studio
 .vs/
 CMakeSettings.json
+
+# Python wheels stuff
+
+*.egg-info/

CMakeLists.txt

@@ -326,6 +326,8 @@ if(ADIOS2_BUILD_EXAMPLES)
   add_subdirectory(examples)
 endif()
 
+add_subdirectory(python)
+
 #------------------------------------------------------------------------------#
 # Testing
 #------------------------------------------------------------------------------#

bindings/Python/CMakeLists.txt

@@ -10,7 +10,7 @@ Python_add_library(adios2_py MODULE
   py11File.cpp py11File.tcc
   py11glue.cpp
 )
-target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2${ADIOS2_LIBRARY_SUFFIX}")
+target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2_bindings${ADIOS2_LIBRARY_SUFFIX}")
 if(ADIOS2_HAVE_MPI)
   target_sources(adios2_py PRIVATE
     py11ADIOSMPI.cpp
@@ -29,36 +29,36 @@ target_link_libraries(adios2_py PRIVATE
   ${maybe_adios2_cxx11_mpi} adios2_cxx11
   ${maybe_adios2_core_mpi} adios2_core
   adios2::thirdparty::pybind11
-  ${maybe_mpi4py} Python::NumPy
+  ${maybe_mpi4py}
   )
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py.in
-  ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py
+  ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py
   @ONLY
 )
 
 set_target_properties(adios2_py PROPERTIES
   CXX_VISIBILITY_PRESET hidden
-  OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}
-  ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
+  OUTPUT_NAME adios2_bindings${ADIOS2_LIBRARY_SUFFIX}
+  ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
 )
 
-string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2")
+string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2/bindings")
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")
   set_target_properties(adios2_py PROPERTIES
     INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}"
   )
 endif()
 
 install(TARGETS adios2_py
-  DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2
+  DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/bindings
   COMPONENT adios2_python-python
 )
-install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py
-  DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2
+install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py
+  DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/bindings
   COMPONENT adios2_python-python
 )

bindings/Python/__init__.py.in

@@ -1,3 +1,3 @@
-from .adios2@ADIOS2_LIBRARY_SUFFIX@ import *
+from .adios2_bindings@ADIOS2_LIBRARY_SUFFIX@ import *
 
 __version__ = "@ADIOS2_VERSION@"

bindings/Python/py11Variable.cpp

@@ -189,6 +189,12 @@ size_t Variable::BlockID() const
     return m_VariableBase->m_BlockID;
 }
 
+size_t Variable::SingleValue() const
+{
+    helper::CheckForNullptr(m_VariableBase, "in call to Variable::SingleValue");
+    return m_VariableBase->m_SingleValue;
+}
+
 // size_t Variable::AddOperation(const Operator op, const Params &parameters) {}
 
 // std::vector<Operation> Variable::Operations() const {}

bindings/Python/py11Variable.h

@@ -95,6 +95,8 @@ class Variable
 
     size_t BlockID() const;
 
+    size_t SingleValue() const;
+
     /**
      * EXPERIMENTAL: Adds operation and parameters to current Variable object
      * @param op operator to be added

bindings/Python/py11glue.cpp

@@ -360,6 +360,23 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
         .def("GetResult", &adios2::py11::Query::GetResult)
         .def("GetBlockIDs", &adios2::py11::Query::GetBlockIDs);
 
+    pybind11::class_<adios2::py11::Operator>(m, "Operator")
+        // Python 2
+        .def("__nonzero__",
+             [](const adios2::py11::Operator &op) {
+                 const bool opBool = op ? true : false;
+                 return opBool;
+             })
+        // Python 3
+        .def("__bool__",
+             [](const adios2::py11::Operator &op) {
+                 const bool opBool = op ? true : false;
+                 return opBool;
+             })
+        .def("Type", &adios2::py11::Operator::Type)
+        .def("SetParameter", &adios2::py11::Operator::SetParameter)
+        .def("Parameters", &adios2::py11::Operator::Parameters);
+
     pybind11::class_<adios2::py11::Variable>(m, "Variable")
         // Python 2
         .def("__nonzero__",
@@ -389,6 +406,7 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
         .def("Steps", &adios2::py11::Variable::Steps)
         .def("StepsStart", &adios2::py11::Variable::StepsStart)
         .def("BlockID", &adios2::py11::Variable::BlockID)
+        .def("SingleValue", &adios2::py11::Variable::SingleValue)
         .def("AddOperation", &adios2::py11::Variable::AddOperation)
         .def("Operations", &adios2::py11::Variable::Operations);
 
@@ -482,23 +500,6 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
 
         .def("BlocksInfo", &adios2::py11::Engine::BlocksInfo);
 
-    pybind11::class_<adios2::py11::Operator>(m, "Operator")
-        // Python 2
-        .def("__nonzero__",
-             [](const adios2::py11::Operator &op) {
-                 const bool opBool = op ? true : false;
-                 return opBool;
-             })
-        // Python 3
-        .def("__bool__",
-             [](const adios2::py11::Operator &op) {
-                 const bool opBool = op ? true : false;
-                 return opBool;
-             })
-        .def("Type", &adios2::py11::Operator::Type)
-        .def("SetParameter", &adios2::py11::Operator::SetParameter)
-        .def("Parameters", &adios2::py11::Operator::Parameters);
-
     pybind11::class_<adios2::py11::File>(m, "File")
         .def("__repr__",
              [](const adios2::py11::File &stream) {

cmake/DetectOptions.cmake

@@ -396,12 +396,12 @@ if(NOT SHARED_LIBS_SUPPORTED)
 endif()
 
 if(ADIOS2_USE_Python STREQUAL AUTO)
-  find_package(Python 3 COMPONENTS Interpreter Development NumPy)
+  find_package(Python 3 COMPONENTS Interpreter Development)
   if(Python_FOUND AND ADIOS2_HAVE_MPI)
     find_package(PythonModule COMPONENTS mpi4py mpi4py/mpi4py.h)
   endif()
 elseif(ADIOS2_USE_Python)
-  find_package(Python 3 REQUIRED COMPONENTS Interpreter Development NumPy)
+  find_package(Python 3 REQUIRED COMPONENTS Interpreter Development)
   if(ADIOS2_HAVE_MPI)
     find_package(PythonModule REQUIRED COMPONENTS mpi4py mpi4py/mpi4py.h)
   endif()

pyproject.toml

@@ -0,0 +1,43 @@
+[project]
+name="adios2"
+version="0.0.1"
+description="The Adaptable IO System"
+authors = [
+  {name = "Vicente Adolfo Bolea Sanchez", email = "[email protected]"},
+]
+
+keywords = [
+    "Python",
+    "Web",
+    "Application",
+    "Framework",
+]
+classifiers = [
+    "License :: Other/Proprietary License",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Software Development :: Libraries :: Application Frameworks",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+dependencies = [
+    "numpy",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pip>=21.3",
+  "pytest",
+  "setuptools",
+  'black',
+]
+
+[tool.setuptools]
+packages = [ "adios2"]
+
+
+[tool.black]
+line-length = 99
+target-version = ['py38', 'py39', 'py310']
+include = 'python/adios2/.*.py|testing/adios2/python/.*.py'

python/CMakeLists.txt

@@ -0,0 +1,13 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+add_custom_target(python_api ALL COMMAND ${CMAKE_COMMAND} -E copy_directory
+                  ${CMAKE_CURRENT_SOURCE_DIR}/adios2
+                  ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2)
+
+install(DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/
+  DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/
+  COMPONENT adios2_python-python
+)

python/adios2/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/python3
+
+import adios2.bindings

python/adios2/adios.py

@@ -0,0 +1,36 @@
+from adios2.io import IO
+from adios2.operator import Operator
+
+import adios2.bindings as bindings
+
+
+class ADIOS:
+    def __init__(self):
+        self._impl = bindings.ADIOS()
+        self._operators = {}
+        self._ios = {}
+
+    def DeclareIO(self, name):
+        self._ios[name] = IO(self._impl, name)
+        return self._ios[name]
+
+    def AtIO(self, name):
+        return self._ios[name]
+
+    def RemoveIO(self, name):
+        del self._ios[name]
+        self._impl.RemoveIO(name)
+
+    def RemoveAllIOs(self):
+        self._ios = {}
+        self._impl.RemoveAllIOs()
+
+    def DefineOperator(self, name, type, parameters={}):
+        self._operators[name] = Operator(self._impl, name, type, parameters)
+        return self._operators[name]
+
+    def InquireOperator(self, name):
+        return self._operators[name]
+
+    def FlushAll(self):
+        self._impl.FlushAll()

python/adios2/attribute.py

@@ -0,0 +1,20 @@
+import adios2.bindings
+
+
+class Attribute:
+    def __init__(self, io, name, *args, **kwargs):
+        self._impl = io.DefineAttribute(name, *args, **kwargs)
+
+    def __eq__(self, other):
+        if isinstance(other, Attribute):
+            return self.Name() == other.Name()
+        return False
+
+    def Name(self):
+        return self._impl.Name()
+
+    def Data(self):
+        return self._impl.Data()
+
+    def DataString(self):
+        return self._impl.DataString()

python/adios2/engine.py

@@ -0,0 +1,66 @@
+import numpy as np
+import adios2.bindings as bindings
+
+from adios2.attribute import Attribute
+from adios2.variable import Variable
+
+
+class Engine:
+    def __init__(self, io, name, mode):
+        self._impl = io.Open(name, mode)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        self.Close()
+
+    def Close(self, transportIndex=-1):
+        self._impl.Close(transportIndex)
+
+    def Steps(self):
+        return self._impl.Steps()
+
+    def CurrentStep(self):
+        return self._impl.CurrentStep()
+
+    def BeginStep(self, *args, **kwargs):
+        return self._impl.BeginStep(*args, **kwargs)
+
+    def EndStep(self):
+        self._impl.EndStep()
+
+    def BlocksInfo(self, name, step):
+        return self._impl.BlocksInfo(name, step)
+
+    def Put(self, variable, content, mode=bindings.Mode.Sync):
+        if isinstance(content, np.ndarray):
+            self._impl.Put(variable._impl, content, mode)
+        else:
+            self._impl.Put(variable._impl, content)
+        self.PerformDataWrite()
+
+    def PerformPuts(self):
+        self._impl.PerformPuts()
+
+    def PerformDataWrite(self):
+        self._impl.PerformDataWrite()
+
+    def Get(self, variable, content=None, mode=bindings.Mode.Sync):
+        if isinstance(content, np.ndarray):
+            self._impl.Get(variable._impl, content, mode)
+            return None
+        else:
+            return self._impl.Get(variable._impl, mode)
+
+    def PerformGets(self):
+        self._impl.PerformGets()
+
+    def LockReaderSelections(self):
+        self._impl.LockReaderSelections()
+
+    def LockWriterDefinitions(self):
+        self._impl.LockWriterDefinitions()
+
+    def Flush(self, transportIndex=-1):
+        self._impl.Flush(transportIndex)