Merge branch 'main' of https://github.com/microsoft/onnxruntime-exten…

…sions into negx

.pipelines/ci.yml

@@ -186,6 +186,23 @@ stages:
         ./build.sh -DOCOS_ENABLE_CPP_EXCEPTIONS=OFF -DOCOS_ENABLE_SELECTED_OPLIST=ON -DOCOS_ENABLE_CTEST=OFF
       displayName: Build ort-extensions with only one operator was selected
 
+  ##############################
+  # Linux for pre-processing API
+  ##############################
+  - job: Linux_PPApiBuild
+    pool:
+      name: 'onnxruntime-extensions-Linux-CPU'
+
+    steps:
+    # compiled as only one operator selected.
+    - bash: |
+        set -e -x -u
+        ./build.sh -DOCOS_ENABLE_C_API=ON
+        cd out/Linux
+        ctest -C RelWithDebInfo --output-on-failure
+      displayName: Build ort-extensions with API enabled and run tests
+
+
 - stage: MacOSBuilds
   dependsOn: []
   jobs:
@@ -246,6 +263,22 @@ stages:
         ctest -C RelWithDebInfo --output-on-failure
       displayName: Run C++ native tests
 
+  ##############################
+  # MacOS for pre-processing API
+  ##############################
+  - job: MacOS_PPApiBuild
+    pool:
+      vmImage: 'macOS-13'
+
+    steps:
+    # compiled as only one operator selected.
+    - bash: |
+        set -e -x -u
+        ./build.sh -DOCOS_ENABLE_C_API=ON
+        cd out/Darwin/RelWithDebInfo
+        ctest -C RelWithDebInfo --output-on-failure
+      displayName: Build ort-extensions with API enabled and run tests
+
   #############
   # macOS Python
   #############
@@ -388,6 +421,17 @@ stages:
         ctest -C RelWithDebInfo --output-on-failure
       displayName: build and test ort-extensions with VC static runtime.
 
+  - job: Windows_PPApiBuild
+    pool:
+      name: 'onnxruntime-extensions-Windows-CPU'
+
+    steps:
+    - script: |
+        call .\build.bat -DOCOS_ENABLE_C_API=ON
+        cd out\Windows
+        ctest -C RelWithDebInfo --output-on-failure
+      displayName: Build ort-extensions with API enabled and run tests
+
   ################
   # Windows Python
   ################

.pyproject/cmdclass.py

@@ -148,6 +148,7 @@ def initialize_options(self):
         self.no_opencv = None
         self.cc_debug = None
         self.cuda_archs = None
+        self.ort_pkg_dir = None
 
     def _parse_options(self, options):
         for segment in options.split(','):
@@ -199,6 +200,9 @@ def build_cmake(self, extension):
             '-DCMAKE_BUILD_TYPE=' + config
         ]
 
+        if self.ort_pkg_dir:
+            cmake_args += ['-DONNXRUNTIME_PKG_DIR=' + self.ort_pkg_dir]
+
         if self.no_opencv:
             # Disabling openCV can drastically reduce the build time.
             cmake_args += [

CMakeLists.txt

@@ -714,12 +714,13 @@ list(APPEND ocos_libraries noexcep_operators)
 target_compile_definitions(ocos_operators PRIVATE ${OCOS_COMPILE_DEFINITIONS})
 target_link_libraries(ocos_operators PRIVATE ${ocos_libraries})
 
-file(GLOB shared_TARGET_LIB_SRC "shared/lib/*.cc" "shared/lib/*.h")
-if (NOT OCOS_ENABLE_C_API)
-  file(GLOB shared_TARGET_LIB_C_API_SRC "shared/lib/*tokenizer*")
-  list(REMOVE_ITEM shared_TARGET_LIB_SRC ${shared_TARGET_LIB_C_API_SRC})
+set (file_patterns "shared/lib/*.cc")
+if (OCOS_ENABLE_C_API)
+  list(APPEND file_patterns "shared/api/*.h*" "shared/api/*.cc")
 endif()
 
+file(GLOB shared_TARGET_LIB_SRC ${file_patterns})
+
 if(NOT OCOS_ENABLE_STATIC_LIB AND CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
   add_executable(ortcustomops ${shared_TARGET_LIB_SRC})
   set_target_properties(ortcustomops PROPERTIES LINK_FLAGS "                                  \
@@ -887,18 +888,18 @@ if(OCOS_BUILD_APPLE_FRAMEWORK)
 endif()
 
 if (_ORTX_STANDALONE_PROJECT)
-# clean up the requirements.txt files from 3rd party project folder to suppress the code security false alarms
-file(GLOB_RECURSE NO_USE_FILES ${CMAKE_BINARY_DIR}/_deps/*requirements.txt)
-message(STATUS "Found the following requirements.txt: ${NO_USE_FILES}")
+  # clean up the requirements.txt files from 3rd party project folder to suppress the code security false alarms
+  file(GLOB_RECURSE NO_USE_FILES ${CMAKE_BINARY_DIR}/_deps/*requirements.txt)
+  message(STATUS "Found the following requirements.txt: ${NO_USE_FILES}")
 
-foreach(nf ${NO_USE_FILES})
-  execute_process(COMMAND ${CMAKE_COMMAND} -E remove ${nf})
-endforeach()
+  foreach(nf ${NO_USE_FILES})
+    execute_process(COMMAND ${CMAKE_COMMAND} -E remove ${nf})
+  endforeach()
 
-# Run CPack to generate the NuGet package
-include(CPack)
-endif()
+  # Run CPack to generate the NuGet package
+  include(CPack)
 
-if(OCOS_ENABLE_CTEST)
-  include(ext_tests)
+  if(OCOS_ENABLE_CTEST)
+    include(ext_tests)
+  endif()
 endif()

base/file_sys.h

@@ -0,0 +1,83 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#ifdef _WIN32
+#include <Windows.h>
+#endif  // _WIN32
+
+#include <sys/stat.h>
+
+#include <string>
+#include <fstream>
+
+namespace ort_extensions {
+
+class path {
+ public:
+  path() = default;
+  path(const std::string& path) : path_(path){};
+
+  static constexpr char separator =
+#ifdef _WIN32
+      '\\';
+#else
+      '/';
+#endif
+
+  using ios_base = std::ios_base;
+  std::ifstream open(ios_base::openmode mode = ios_base::in) const {
+    // if Windows, need to convert the string to UTF-16
+#ifdef _WIN32
+    return std::ifstream(to_wstring(), mode);
+#else
+    return std::ifstream(path_, mode);
+#endif  // _WIN32
+  }
+
+  const std::string& string() const {
+    return path_;
+  }
+
+  path join(const std::string& path) const {
+    return path_ + separator + path;
+  }
+
+  path operator/(const std::string& path) const {
+    return join(path);
+  }
+
+  path operator/(const path& path) {
+    return join(path.path_);
+  }
+
+  bool is_directory() const {
+#ifdef _WIN32
+    struct _stat64 info;
+    if (_wstat64(to_wstring().c_str(), &info) != 0) {
+      return false;
+    }
+#else
+    struct stat info;
+    if (stat(path_.c_str(), &info) != 0) {
+      return false;
+    }
+#endif  // _WIN32
+    return (info.st_mode & S_IFDIR) != 0;
+  }
+
+ private:
+  std::string path_;
+
+#ifdef _WIN32
+  std::wstring to_wstring() const {
+    int size_needed = MultiByteToWideChar(CP_UTF8, 0, path_.c_str(), -1, nullptr, 0);
+    std::wstring utf16_str(size_needed, 0);
+    MultiByteToWideChar(CP_UTF8, 0, path_.c_str(), -1, &utf16_str[0], size_needed);
+    return utf16_str;
+  }
+#endif  // _WIN32
+};
+
+}  // namespace ort_extensions

base/ocos.cc

@@ -4,6 +4,15 @@
 #include "ocos.h"
 #include "narrow.h"
 
+OrtxStatus::operator OrtStatus*() const noexcept {
+  if (IsOk()) {
+    return nullptr;
+  }
+
+  OrtStatus* status = OrtW::CreateStatus(Message(), OrtErrorCode::ORT_RUNTIME_EXCEPTION);
+  return status;
+}
+
 OrtErrorCode BaseKernel::GetErrorCodeAndRelease(OrtStatusPtr status) const noexcept {
   if (status == nullptr) {
     return ORT_OK;

base/ortx_common.h

@@ -5,7 +5,7 @@
 #include <optional>
 #include <string>
 #include <sstream>
-#include "status.h"
+
 #include "string_utils.h"
 #ifdef _WIN32
 #include <Windows.h>