diff --git a/CMakeLists.txt b/CMakeLists.txt index 04ded5f2..042d4432 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,8 +87,8 @@ option(XEUS_CPP_BUILD_STATIC "Build xeus-cpp static library" ON) OPTION(XEUS_CPP_BUILD_SHARED "Split xcpp build into executable and library" ON) OPTION(XEUS_CPP_BUILD_EXECUTABLE "Build the xcpp executable" ON) -OPTION(XEUS_CPP_USE_SHARED_XEUS "Link xcpp with the xeus shared library (instead of the static library)" ON) -OPTION(XEUS_CPP_USE_SHARED_XEUS_CPP "Link xcpp with the xeus shared library (instead of the static library)" ON) +OPTION(XEUS_CPP_USE_SHARED_XEUS "Link xcpp with the xeus shared library (instead of the static library)" ON) +OPTION(XEUS_CPP_USE_SHARED_XEUS_CPP "Link xcpp with the xeus shared library (instead of the static library)" ON) OPTION(XEUS_CPP_EMSCRIPTEN_WASM_BUILD "Build for wasm with emscripten" OFF) @@ -129,15 +129,18 @@ else() add_compile_options(-fexceptions) endif () -if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Intel") +if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR + CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR + CMAKE_CXX_COMPILER_ID MATCHES "Intel") + if(NOT XEUS_CPP_EMSCRIPTEN_WASM_BUILD) add_compile_options(-Wunused-parameter -Wextra -Wreorder) endif() - CHECK_CXX_COMPILER_FLAG("-std=c++17" HAS_CPP_17_FLAG) if (HAS_CPP_17_FLAG) - add_compile_options(-std=c++17) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") + set(CMAKE_CXX_STANDARD 17) else () message(FATAL_ERROR "Unsupported compiler -- xeus requires C++17 support!") endif () @@ -151,8 +154,8 @@ endif() find_package(CppInterOp REQUIRED CONFIG PATHS "${CPPINTEROP_DIR}" "${CPPINTEROP_DIR}/lib") find_package(argparse REQUIRED) find_package(pugixml REQUIRED) -##set(Python_FIND_VIRTUALENV ONLY) -##find_package(Python COMPONENTS Interpreter Development) +set(Python_FIND_VIRTUALENV ONLY) +find_package(Python COMPONENTS Interpreter Development) # Source files # ============ @@ -168,6 +171,7 @@ set(XEUS_CPP_SRC src/xinterpreter.cpp src/xoptions.cpp src/xparser.cpp + src/xmagics/pythonexec.cpp ) set(XEUS_CPP_MAIN_SRC @@ -327,23 +331,30 @@ if (XEUS_CPP_BUILD_EXECUTABLE) xeus_cpp_set_common_options(xcpp) xeus_cpp_set_kernel_options(xcpp) target_link_libraries(xcpp PRIVATE xeus-zmq) + set_target_properties(xcpp PROPERTIES + ENABLE_EXPORTS 1 + CXX_STANDARD ${CMAKE_CXX_STANDARD}) + target_link_libraries(xcpp PUBLIC xeus-cpp pthread Python::Python) + + ##TODO: We may be need sse RPATH + set_target_properties(xcpp clangCppInterOp PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) + if(APPLE) + target_link_libraries(xcpp PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup) + elseif(NOT MSVC) + target_link_libraries(xcpp PUBLIC -Wl,--unresolved-symbols=ignore-in-object-files) + endif() -##set_target_properties(xcpp PROPERTIES -## ENABLE_EXPORTS 1 -## CXX_STANDARD ${CMAKE_CXX_STANDARD} -##) -##target_link_libraries(xcpp PUBLIC xeus-cpp pthread Python::Python) -## -###TODO: We may be need sse RPATH -###set_target_properties(xcpp clangCppInterOp PROPERTIES -### INSTALL_RPATH_USE_LINK_PATH TRUE -###) -##if(APPLE) -## target_link_libraries(xcpp PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup) -##elseif(NOT MSVC) -## target_link_libraries(xcpp PUBLIC -Wl,--unresolved-symbols=ignore-in-object-files) -##endif() - + target_include_directories(xeus-cpp PUBLIC ${Python_INCLUDE_DIRS}) + target_link_libraries(xeus-cpp PUBLIC ${PYTHON_LIBRARIES}) + target_link_libraries(xeus-cpp ${PYTHON_LIBRARIES_Development_Main}) + set_target_properties(xeus-cpp PROPERTIES + PUBLIC_HEADER "${XEUS_CPP_HEADERS}" + COMPILE_DEFINITIONS "XEUS_CPP_EXPORTS" + PREFIX "" + VERSION ${${PROJECT_NAME}_VERSION} + SOVERSION ${XEUS_CPP_VERSION_MAJOR} + OUTPUT_NAME "libxeus-cpp" + CXX_STANDARD ${CMAKE_CXX_STANDARD}) endif() if(XEUS_CPP_EMSCRIPTEN_WASM_BUILD) diff --git a/Dockerfile b/Dockerfile index bf8833a7..8dee1d1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # https://hub.docker.com/r/jupyter/base-notebook/tags ARG BASE_CONTAINER=jupyter/base-notebook ARG BASE_TAG=latest -ARG BUILD_TYPE=Release +ARG BUILD_TYPE=Debug FROM $BASE_CONTAINER:$BASE_TAG @@ -80,7 +80,7 @@ USER ${NB_UID} ENV NB_PYTHON_PREFIX=${CONDA_DIR} \ KERNEL_PYTHON_PREFIX=${CONDA_DIR} \ # CPLUS_INCLUDE_PATH="${CONDA_DIR}/include:/home/${NB_USER}/include:/home/runner/work/xeus-clang-repl/xeus-clang-repl/clang-dev/clang/include:/home/jovyan/clad/include:/home/jovyan/CppInterOp/include" - CPLUS_INCLUDE_PATH="${CONDA_DIR}/include:/home/${NB_USER}/include:/home/jovyan/clad/include:/home/jovyan/CppInterOp/include" + CPLUS_INCLUDE_PATH="${CONDA_DIR}/include:/home/jovyan/clad/include:/home/jovyan/CppInterOp/include" WORKDIR "${HOME}" @@ -131,6 +131,7 @@ RUN \ cppzmq \ xtl \ 'clangdev>=17' \ + 'llvm-openmp' \ pugixml \ cpp-argparse \ zlib \ @@ -173,7 +174,8 @@ RUN \ # Build CppInterOp # sys_incs=$(LC_ALL=C c++ -xc++ -E -v /dev/null 2>&1 | LC_ALL=C sed -ne '/starts here/,/End of/p' | LC_ALL=C sed '/^ /!d' | cut -c2- | tr '\n' ':') && \ - export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/include/llvm:${PATH_TO_LLVM_BUILD}/include/clange:$CPLUS_INCLUDE_PATH:${sys_incs%:}" && \ + #/usr/include/x86_64-linux-gnu:/usr/include: + export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/include/llvm:${PATH_TO_LLVM_BUILD}/include/clang:$CPLUS_INCLUDE_PATH:${sys_incs%:}" && \ git clone https://github.com/compiler-research/CppInterOp.git && \ export CB_PYTHON_DIR="$PWD/cppyy-backend/python" && \ export CPPINTEROP_DIR="$CB_PYTHON_DIR/cppyy_backend" && \ diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp index 4ce850e1..86ca75dc 100644 --- a/src/xinterpreter.cpp +++ b/src/xinterpreter.cpp @@ -23,6 +23,18 @@ #include "xeus-cpp/xbuffer.hpp" #include "xeus-cpp/xeus_cpp_config.hpp" +#include "xeus-cpp/xinterpreter.hpp" +#include "xeus-cpp/xmagics.hpp" + +#include "xinput.hpp" +// #include "xinspect.hpp" +// #include "xmagics/executable.hpp" +// #include "xmagics/execution.hpp" +#include "xmagics/os.hpp" +#include "xmagics/pythonexec.hpp" +#include "xparser.hpp" +#include "xsystem.hpp" + using Args = std::vector; void* createInterpreter(const Args &ExtraArgs = {}) { @@ -329,6 +341,7 @@ namespace xcpp // preamble_manager["magics"].get_cast().register_magic("executable", executable(m_interpreter)); // preamble_manager["magics"].get_cast().register_magic("file", writefile()); // preamble_manager["magics"].get_cast().register_magic("timeit", timeit(&m_interpreter)); + preamble_manager["magics"].get_cast().register_magic("python", pythonexec()); } std::string interpreter::get_stdopt(int argc, const char* const* argv) diff --git a/src/xmagics/os.cpp b/src/xmagics/os.cpp index 089f8f68..35894725 100644 --- a/src/xmagics/os.cpp +++ b/src/xmagics/os.cpp @@ -15,7 +15,7 @@ #include "os.hpp" #include "../xparser.hpp" -#include "xeus-cpp/xoptions.hpp" +//#include "xeus-cpp/xoptions.hpp" namespace xcpp { diff --git a/src/xmagics/pythonexec.cpp b/src/xmagics/pythonexec.cpp new file mode 100644 index 00000000..ccd308c0 --- /dev/null +++ b/src/xmagics/pythonexec.cpp @@ -0,0 +1,210 @@ +//===------------ pythonexec.hpp - Python/C++ Interoperability ------------===// +// +// Licensed under the Apache License v2.0. +// SPDX-License-Identifier: Apache-2.0 +// +// The full license is in the file LICENSE, distributed with this software. +// +//===----------------------------------------------------------------------===// +// +// This file defines the Python/C++ interoperability in which two cells within +// the same notebook can be in a either language. +// +//===----------------------------------------------------------------------===// + +#include "Python.h" + +#include "pythonexec.hpp" +#include "../xparser.hpp" + +#include +#include +#include +#include +#include +#include + +namespace xcpp { +static PyObject *gMainDict = 0; + +void pythonexec::startup() { + static bool isInitialized = false; + if (isInitialized) + return; + + Py_Initialize(); + + PyRun_SimpleString("import sys\nprint(sys.path)"); + + // Import cppyy module + PyObject* cppyyModule = PyImport_ImportModule("cppyy"); + if (!cppyyModule) { + PyErr_Print(); + Py_Finalize(); + return; // Handle import error as needed + } + + PyObject* mainModule = PyImport_AddModule("__main__"); + PyObject_SetAttrString(mainModule, "cppyy", cppyyModule); + Py_XDECREF(cppyyModule); + isInitialized = true; + // gMainDict = PyModule_GetDict(mainModule); + // Py_INCREF(gMainDict); + // if (!gMainDict) + // printf("Could not add module __main__"); + + + // // Retrieve the dictionary of cppyy module + // PyObject* cppyyDict = PyModule_GetDict(cppyyModule); + // Py_DECREF(cppyyModule); + // if (!cppyyDict) { + // PyErr_Print(); + // Py_Finalize(); + // return; // Handle retrieval error as needed + // } + + // // Add cppyyDict to gMainDict (if needed for further usage) + // PyDict_Update(gMainDict, cppyyDict); + + // Py_DECREF(cppyyDict); + // PyRun_SimpleString("import cppyy"); +} + +argparser pythonexec::get_options() { + argparser argpars{"python", "Start executing Python Cell"}; + return argpars; +} + +void pythonexec::execute(std::string &line, std::string &cell) { + // std::istringstream iss(line); + // std::vector results((std::istream_iterator(iss)), + // std::istream_iterator()); + startup(); + +// auto argpars = get_options(); +// argpars.parse(line); + + std::string code; + + code += cell; + if (trim(code).empty()) + return; + // PyRun_SimpleString( + // "globals_copy_lists = " + // "globals().copy()\nfirst_dict_ints={k:globals_copy_lists[k] for k in " + // "set(globals_copy_lists) if type(globals_copy_lists[k]) == " + // "int}\nfirst_dict_lists={k:globals_copy_lists[k] for k in " + // "set(globals_copy_lists) if type(globals_copy_lists[k]) == list}"); + + // PyRun_SimpleString("tmp = globals().copy()\nvars = [f'int {k} = {v};' for + // k,v in tmp.items() if type(v) == int and not k.startswith('_') and k!='tmp' + // and k!='In' and k!='Out' and k!='sys' and not hasattr(v, + // '__call__')]\nprint(vars)"); PyRun_SimpleString("b = + // globals().copy()\nnew_ints = ' '.join([f'int {k} = {b[k]};' for k in set(b) + // - set(first_dict) if type(b[k]) == int])\nprint('new_ints: ', new_ints)"); + + Cpp::BeginStdStreamCapture(Cpp::kStdErr); + Cpp::BeginStdStreamCapture(Cpp::kStdOut); + + PyRun_SimpleString(code.c_str()); + + std::cout << Cpp::EndStdStreamCapture(); + std::cerr << Cpp::EndStdStreamCapture(); + + // PyObject* objectsRepresentation = PyObject_Repr(gMainDict); + // const char* s = PyUnicode_AsUTF8(objectsRepresentation); + // printf("REPR of global dict: %s\n", s); +} + +void pythonexec::update_python_dict_var(const char *name, int value) { + if (!gMainDict) + startup(); + PyObject *s; + s = PyLong_FromLong(value); + PyDict_SetItemString(gMainDict, name, s); + Py_DECREF(s); +} + +void pythonexec::update_python_dict_var_vector(const char *name, + std::vector &data) { + if (!gMainDict) + startup(); + PyObject *listObj = PyList_New(data.size()); + if (!listObj) + throw std::logic_error("Unable to allocate memory for Python list"); + for (unsigned int i = 0; i < data.size(); i++) { + PyObject *num = PyLong_FromLong((int)data[i]); + if (!num) { + Py_DECREF(listObj); + throw std::logic_error("Unable to allocate memory for Python list"); + } + PyList_SET_ITEM(listObj, i, num); + } + PyDict_SetItemString(gMainDict, name, listObj); +} + +// check python globals +void pythonexec::check_python_globals() { + if (!Py_IsInitialized()) + return; + // execute the command + PyRun_SimpleString("print(globals())"); +} + +// execute a python comand +void pythonexec::exec_python_simple_command(const std::string code) { + if (!Py_IsInitialized()) + return; + // execute the command + PyRun_SimpleString(code.c_str()); +} + +std::string pythonexec::transfer_python_ints_utility() { + if (!Py_IsInitialized()) + return " "; + // transfer ints utility + PyRun_SimpleString( + "def getNewInts():\n glob_ints_utils = globals().copy()\n new_ints " + "= ' '.join([f'int {k} = {glob_ints_utils[k]};' for k in " + "set(glob_ints_utils) - set(first_dict_ints) if type(glob_ints_utils[k]) " + "== int])\n return new_ints"); + PyObject *ints_result = + PyObject_CallFunction(PyDict_GetItemString(gMainDict, "getNewInts"), 0); + if (!ints_result) { + printf("Could not retrieve Python integers!\n"); + return " "; + } else { + std::string newPythonInts = PyUnicode_AsUTF8(ints_result); + // printf("new ints %s\n", PyUnicode_AsUTF8(ints_result)); + Py_DECREF(ints_result); + return newPythonInts; + } +} + +std::string pythonexec::transfer_python_lists_utility() { + if (!Py_IsInitialized()) + return " "; + // transfer lists utility + PyRun_SimpleString("def getNewLists():\n l = globals().copy()\n " + "new_lists = ' '.join([f'int {k}() = {l[k]};' for k in " + "set(l) - set(first_dict_lists) if type(l[k]) == " + "list]).replace('[','{').replace(']','}').replace('(','[')" + ".replace(')',']')\n return new_lists"); + PyObject *lists_result = + PyObject_CallFunction(PyDict_GetItemString(gMainDict, "getNewLists"), 0); + if (!lists_result) { + printf("Could not retrieve Python lists!\n"); + return " "; + } else { + std::string newPythonLists = PyUnicode_AsUTF8(lists_result); + Py_DECREF(lists_result); + return newPythonLists; + } +} + +bool pythonexec::python_check_for_initialisation() { + if (!Py_IsInitialized()) + return false; + return true; +} +} // namespace xcpp diff --git a/src/xmagics/pythonexec.hpp b/src/xmagics/pythonexec.hpp new file mode 100644 index 00000000..7b56d9a3 --- /dev/null +++ b/src/xmagics/pythonexec.hpp @@ -0,0 +1,55 @@ +//===------- pythonexec.hpp - Python/C++ Interoperability -------*- C++ -*-===// +// +// Licensed under the Apache License v2.0. +// SPDX-License-Identifier: Apache-2.0 +// +// The full license is in the file LICENSE, distributed with this software. +// +//===----------------------------------------------------------------------===// +// +// This file defines the Python/C++ interoperability in which two cells within +// the same notebook can be in a either language. +// +//===----------------------------------------------------------------------===// + +#ifndef XMAGICS_PYTHONEXEC_HPP +#define XMAGICS_PYTHONEXEC_HPP + +#include "xeus-cpp/xbuffer.hpp" +#include "xeus-cpp/xinterpreter.hpp" +#include "xeus-cpp/xmagics.hpp" +#include "xeus-cpp/xoptions.hpp" + +#include +#include + +namespace xcpp { + + class pythonexec: public xmagic_cell + { + public: + + argparser get_options(); + virtual void operator()(const std::string& line, const std::string& cell) override { + std::string cline = line; + std::string ccell = cell; + execute(cline, ccell); + }; + + static void update_python_dict_var(const char *name, int value); + static void update_python_dict_var_vector(const char *name, + std::vector &data); + void check_python_globals(); + void exec_python_simple_command(const std::string code); + static std::string transfer_python_ints_utility(); + static std::string transfer_python_lists_utility(); + static bool python_check_for_initialisation(); + + private: + + static void startup(); + void execute(std::string &line, std::string &cell); + }; + +} // namespace xcpp +#endif