Library autoloader

include/clang/Interpreter/CppInterOp.h

@@ -757,6 +757,14 @@ namespace Cpp {
                                    unsigned complete_line = 1U,
                                    unsigned complete_column = 1U);
 
+  /// Set libraries autoload.
+  ///\param[in] autoload - true to enable library autoload.
+  CPPINTEROP_API void SetLibrariesAutoload(bool autoload = true);
+
+  /// Get libraries autoload status.
+  ///\returns LibraryAutoLoad state (true = libraries autoload is on).
+  CPPINTEROP_API bool GetLibrariesAutoload();
+
 } // end namespace Cpp
 
 #endif // CPPINTEROP_CPPINTEROP_H

lib/Interpreter/CppInterOp.cpp

@@ -32,14 +32,17 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Demangle/Demangle.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_os_ostream.h"
 
 #include <map>
 #include <set>
 #include <sstream>
 #include <string>
+#include <utility>
 
 // Stream redirect.
 #ifdef _WIN32
@@ -64,6 +67,10 @@ namespace Cpp {
   using namespace llvm;
   using namespace std;
 
+  // Last assigned Autoload SearchGenerator
+  // TODO: Test fot thread safe.
+  class AutoLoadLibrarySearchGenerator;
+  static AutoLoadLibrarySearchGenerator *sAutoSG = nullptr;
   // Flag to indicate ownership when an external interpreter instance is used.
   static bool OwningSInterpreter = true;
   static compat::Interpreter* sInterpreter = nullptr;
@@ -1137,7 +1144,6 @@ namespace Cpp {
       llvm::consumeError(std::move(Err)); // nullptr if missing
     else
       return llvm::jitTargetAddressToPointer<void*>(*FDAorErr);
-
     return nullptr;
   }
 
@@ -2728,7 +2734,7 @@ namespace Cpp {
     std::string BinaryPath = GetExecutablePath(/*Argv0=*/nullptr, MainAddr);
 
     // build/tools/clang/unittests/Interpreter/Executable -> build/
-    StringRef Dir = sys::path::parent_path(BinaryPath);
+    Dir = sys::path::parent_path(BinaryPath);
 
     Dir = sys::path::parent_path(Dir);
     Dir = sys::path::parent_path(Dir);
@@ -3546,4 +3552,199 @@ namespace Cpp {
                          complete_column);
   }
 
+#define DEBUG_TYPE "autoload"
+
+  static inline std::string DemangleNameForDlsym(const std::string& name) {
+    std::string nameForDlsym = name;
+
+    static bool is_demangle_active = false;
+    static bool demangle = false;
+    if (!is_demangle_active) {
+      auto& I = getInterp();
+      llvm::orc::LLJIT& EE = *compat::getExecutionEngine(I);
+      auto t = EE.getTargetTriple();
+      demangle = t.isOSDarwin() || t.isOSWindows();
+      is_demangle_active = true;
+    }
+
+    // The JIT gives us a mangled name which has an additional leading underscore
+    // on macOS and Windows, for instance __ZN8TRandom34RndmEv. However, dlsym
+    // requires us to remove it.
+    // FIXME: get this information from the DataLayout via getGlobalPrefix()!
+    if (demangle && nameForDlsym[0] == '_')
+      nameForDlsym.erase(0, 1);
+    return nameForDlsym;
+  }
+
+  class AutoLoadLibrarySearchGenerator : public llvm::orc::DefinitionGenerator {
+    bool Enabled = false;
+  public:
+    bool isEnabled() {
+      return Enabled;
+    }
+
+    void setEnabled(bool enabled) {
+      Enabled = enabled;
+    }
+
+    // Lazy materialization unit class helper
+    class AutoloadLibraryMU : public llvm::orc::MaterializationUnit {
+      std::string lib;
+      std::string fLibrary;
+      llvm::orc::SymbolNameVector fSymbols;
+    public:
+      AutoloadLibraryMU(const std::string &Library, const llvm::orc::SymbolNameVector &Symbols)
+        : MaterializationUnit({getSymbolFlagsMap(Symbols), nullptr}), fLibrary(Library), fSymbols(Symbols) {}
+
+      StringRef getName() const override {
+        return "<Symbols from Autoloaded Library>";
+      }
+
+      void materialize(std::unique_ptr<llvm::orc::MaterializationResponsibility> R) override {
+        //if (!sAutoSG || !sAutoSG->isEnabled()) {
+        //  R->failMaterialization();
+        //  return;
+        //}
+
+        LLVM_DEBUG(dbgs() << "Materialize " << lib << " syms=" << fSymbols);
+
+        auto& I = getInterp();
+        auto *DLM = I.getDynamicLibraryManager();
+
+        llvm::orc::SymbolMap loadedSymbols;
+        llvm::orc::SymbolNameSet failedSymbols;
+        bool loadedLibrary = false;
+
+        for (const auto &symbol : fSymbols) {
+          std::string symbolStr = (*symbol).str();
+          std::string nameForDlsym = DemangleNameForDlsym(symbolStr);
+
+          // Check if the symbol is available without loading the library.
+          void *addr = llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(nameForDlsym);
+
+          if (!addr && !loadedLibrary) {
+            // Try to load the library which should provide the symbol definition.
+            if (DLM->loadLibrary(lib, false) != DynamicLibraryManager::LoadLibResult::kLoadLibSuccess) {
+              LLVM_DEBUG(dbgs() << "MU: Failed to load library " << lib);
+              string err = "MU: Failed to load library! " + lib;
+              perror(err.c_str());
+            } else {
+              LLVM_DEBUG(dbgs() << "MU: Autoload library " << lib);
+            }
+
+            // Only try loading the library once.
+            loadedLibrary = true;
+
+            addr = llvm::sys::DynamicLibrary::SearchForAddressOfSymbol(nameForDlsym);
+          }
+
+          if (addr) {
+            loadedSymbols[symbol] =
+#if CLANG_VERSION_MAJOR < 17
+              llvm::JITEvaluatedSymbol::fromPointer(addr, JITSymbolFlags::Exported);
+#else
+              llvm::orc::ExecutorSymbolDef(llvm::orc::ExecutorAddr::fromPtr(addr), JITSymbolFlags::Exported);
+#endif // CLANG_VERSION_MAJOR < 17
+          } else {
+            // Collect all failing symbols, delegate their responsibility and then
+            // fail their materialization. R->defineNonExistent() sounds like it
+            // should do that, but it's not implemented?!
+            failedSymbols.insert(symbol);
+          }
+        }
+
+        if (!failedSymbols.empty()) {
+          auto failingMR = R->delegate(failedSymbols);
+          if (failingMR) {
+            (*failingMR)->failMaterialization();
+          }
+        }
+
+        if (!loadedSymbols.empty()) {
+          llvm::cantFail(R->notifyResolved(loadedSymbols));
+
+#if CLANG_VERSION_MAJOR > 18
+          llvm::orc::SymbolDependenceGroup DepGroup;
+          llvm::cantFail(R->notifyEmitted({DepGroup}));
+#else
+	  llvm::cantFail(R->notifyEmitted());
+#endif
+        }
+      }
+
+      void discard(const llvm::orc::JITDylib &JD, const llvm::orc::SymbolStringPtr &Name) override {}
+
+      private:
+        static llvm::orc::SymbolFlagsMap getSymbolFlagsMap(const llvm::orc::SymbolNameVector &Symbols) {
+          llvm::orc::SymbolFlagsMap map;
+          for (const auto &symbolName : Symbols)
+            map[symbolName] = llvm::JITSymbolFlags::Exported;
+          return map;
+        }
+    };
+
+    llvm::Error tryToGenerate(llvm::orc::LookupState &LS, llvm::orc::LookupKind K, llvm::orc::JITDylib &JD,
+                              llvm::orc::JITDylibLookupFlags JDLookupFlags, const llvm::orc::SymbolLookupSet &Symbols) override {
+      if (!isEnabled())
+	return llvm::Error::success();
+      LLVM_DEBUG(dbgs() << "tryToGenerate");
+
+      auto& I = getInterp();
+      auto *DLM = I.getDynamicLibraryManager();
+
+      std::unordered_map<std::string, llvm::orc::SymbolNameVector> found;
+      llvm::orc::SymbolMap NewSymbols;
+      for (const auto &KV : Symbols) {
+	const auto &Name = KV.first;
+	if ((*Name).empty())
+	  continue;
+
+	auto lib = DLM->searchLibrariesForSymbol(*Name, /*searchSystem=*/true); // false?
+	if (lib.empty())
+	  continue;
+
+	found[lib].push_back(Name);
+
+	// Workaround: This getAddressOfGlobal call make first symbol search
+	// to work, immediatelly after library auto load. This approach do not
+	// use MU
+	//DLM->loadLibrary(lib, true);
+	//I.getAddressOfGlobal(*Name);
+      }
+
+      for (auto &&KV : found) {
+	auto MU = std::make_unique<AutoloadLibraryMU>(KV.first, std::move(KV.second));
+	if (auto Err = JD.define(MU))
+	  return Err;
+      }
+
+      return llvm::Error::success();
+    }
+
+  };
+
+  void SetLibrariesAutoload(bool autoload /* = true */) {
+    auto& I = getInterp();
+    llvm::orc::LLJIT& EE = *compat::getExecutionEngine(I);
+#if CLANG_VERSION_MAJOR < 17
+    llvm::orc::JITDylib& DyLib = EE.getMainJITDylib();
+#else
+    llvm::orc::JITDylib& DyLib = *EE.getProcessSymbolsJITDylib().get();
+#endif // CLANG_VERSION_MAJOR
+
+    if (!sAutoSG) {
+      sAutoSG = &DyLib.addGenerator(std::make_unique<AutoLoadLibrarySearchGenerator>());
+    }
+    sAutoSG->setEnabled(autoload);
+
+    LLVM_DEBUG(dbgs() << "Autoload=" << (sAutoSG->isEnabled() ? "ON" : "OFF"));
+  }
+
+  bool GetLibrariesAutoload() {
+    LLVM_DEBUG(dbgs() << "Autoload is " << (sAutoSG && sAutoSG->isEnabled() ? "ON" : "OFF"));
+    return sAutoSG && sAutoSG->isEnabled();
+  }
+
+#undef DEBUG_TYPE
+
   } // end namespace Cpp

lib/Interpreter/DynamicLibraryManager.cpp

@@ -243,7 +243,7 @@ namespace Cpp {
     // get canonical path name and check if already loaded
     const std::string Path = platform::NormalizePath(foundDyLib);
     if (Path.empty()) {
-      LLVM_DEBUG(dbgs() << "cling::DynamicLibraryManager::lookupLibMaybeAddExt(): "
+      LLVM_DEBUG(dbgs() << "DynamicLibraryManager::lookupLibMaybeAddExt(): "
                         << "error getting real (canonical) path of library " << foundDyLib << '\n');
       return foundDyLib;
     }
@@ -392,10 +392,9 @@ namespace Cpp {
       return;
 
     DyLibHandle dyLibHandle = nullptr;
-    for (DyLibs::const_iterator I = m_DyLibs.begin(), E = m_DyLibs.end();
-         I != E; ++I) {
-      if (I->second == canonicalLoadedLib) {
-        dyLibHandle = I->first;
+    for (const auto& dylib : m_DyLibs) {
+      if (dylib.second == canonicalLoadedLib) {
+        dyLibHandle = dylib.first;
         break;
       }
     }
@@ -405,7 +404,7 @@ namespace Cpp {
     std::string errMsg;
     platform::DLClose(dyLibHandle, &errMsg);
     if (!errMsg.empty()) {
-      LLVM_DEBUG(dbgs() << "cling::DynamicLibraryManager::unloadLibrary(): "
+      LLVM_DEBUG(dbgs() << "DynamicLibraryManager::unloadLibrary(): "
                         << errMsg << '\n');
     }
 

lib/Interpreter/DynamicLibraryManagerSymbol.cpp

@@ -1260,7 +1260,7 @@ namespace Cpp {
 
   void DynamicLibraryManager::initializeDyld(
                  std::function<bool(llvm::StringRef)> shouldPermanentlyIgnore) {
-     //assert(!m_Dyld && "Already initialized!");
+    assert(!m_Dyld && "Already initialized!");
     if (m_Dyld)
       delete m_Dyld;
 

unittests/CMakeLists.txt

@@ -38,6 +38,7 @@ endif()
                        DEPENDS)
   # FIXME: Just call gtest_add_tests this function is available.
   #gtest_add_tests(${name} "${Arguments}" AUTO)
+  llvm_update_compile_flags(${name})
 endfunction()
 
 add_subdirectory(CppInterOp)

unittests/CppInterOp/CMakeLists.txt

@@ -20,8 +20,8 @@ target_link_libraries(CppInterOpTests
 )
 
 set_output_directory(CppInterOpTests
-  BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/CppInterOpTests/unittests/bin/$<CONFIG>/
-  LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/CppInterOpTests/unittests/bin/$<CONFIG>/
+  BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/bin/$<CONFIG>/
+  LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/bin/$<CONFIG>/
 )
 
 if(NOT WIN32)
@@ -43,10 +43,27 @@ target_link_libraries(DynamicLibraryManagerTests
   clangCppInterOp
 )
 
+set_source_files_properties(DynamicLibraryManagerTest.cpp PROPERTIES COMPILE_DEFINITIONS
+  "LLVM_BINARY_DIR=\"${LLVM_BINARY_DIR}\""
+  )
+set_source_files_properties(DynamicLibraryManagerTest.cpp PROPERTIES COMPILE_DEFINITIONS
+  "CPPINTEROP_VERSION=\"${CPPINTEROP_VERSION}\""
+  )
+
+set_output_directory(DynamicLibraryManagerTests BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/TestSharedLib/unittests/bin/$<CONFIG>/)
+
 set_output_directory(DynamicLibraryManagerTests
   BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/TestSharedLib/unittests/bin/$<CONFIG>/
   LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/TestSharedLib/unittests/bin/$<CONFIG>/
 )
 add_dependencies(DynamicLibraryManagerTests TestSharedLib)
 #export_executable_symbols_for_plugins(TestSharedLib)
 add_subdirectory(TestSharedLib)
+
+add_dependencies(DynamicLibraryManagerTests TestSharedLib1)
+#export_executable_symbols_for_plugins(TestSharedLib1)
+add_subdirectory(TestSharedLib1)
+
+add_dependencies(DynamicLibraryManagerTests TestSharedLib2)
+#export_executable_symbols_for_plugins(TestSharedLib2)
+add_subdirectory(TestSharedLib2)