Skip to content

Commit

Permalink
Add python magic
Browse files Browse the repository at this point in the history
  • Loading branch information
alexander-penev committed Oct 30, 2023
1 parent 0ff8b19 commit 0de88f2
Show file tree
Hide file tree
Showing 6 changed files with 318 additions and 27 deletions.
57 changes: 34 additions & 23 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ option(XEUS_CPP_BUILD_STATIC "Build xeus-cpp static library" ON)
OPTION(XEUS_CPP_BUILD_SHARED "Split xcpp build into executable and library" ON)
OPTION(XEUS_CPP_BUILD_EXECUTABLE "Build the xcpp executable" ON)

OPTION(XEUS_CPP_USE_SHARED_XEUS "Link xcpp with the xeus shared library (instead of the static library)" ON)
OPTION(XEUS_CPP_USE_SHARED_XEUS_CPP "Link xcpp with the xeus shared library (instead of the static library)" ON)
OPTION(XEUS_CPP_USE_SHARED_XEUS "Link xcpp with the xeus shared library (instead of the static library)" ON)
OPTION(XEUS_CPP_USE_SHARED_XEUS_CPP "Link xcpp with the xeus shared library (instead of the static library)" ON)

OPTION(XEUS_CPP_EMSCRIPTEN_WASM_BUILD "Build for wasm with emscripten" OFF)

Expand Down Expand Up @@ -129,15 +129,18 @@ else()
add_compile_options(-fexceptions)
endif ()

if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Intel")
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR
CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR
CMAKE_CXX_COMPILER_ID MATCHES "Intel")

if(NOT XEUS_CPP_EMSCRIPTEN_WASM_BUILD)
add_compile_options(-Wunused-parameter -Wextra -Wreorder)
endif()


CHECK_CXX_COMPILER_FLAG("-std=c++17" HAS_CPP_17_FLAG)
if (HAS_CPP_17_FLAG)
add_compile_options(-std=c++17)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
set(CMAKE_CXX_STANDARD 17)
else ()
message(FATAL_ERROR "Unsupported compiler -- xeus requires C++17 support!")
endif ()
Expand All @@ -151,8 +154,8 @@ endif()
find_package(CppInterOp REQUIRED CONFIG PATHS "${CPPINTEROP_DIR}" "${CPPINTEROP_DIR}/lib")
find_package(argparse REQUIRED)
find_package(pugixml REQUIRED)
##set(Python_FIND_VIRTUALENV ONLY)
##find_package(Python COMPONENTS Interpreter Development)
set(Python_FIND_VIRTUALENV ONLY)
find_package(Python COMPONENTS Interpreter Development)

# Source files
# ============
Expand All @@ -168,6 +171,7 @@ set(XEUS_CPP_SRC
src/xinterpreter.cpp
src/xoptions.cpp
src/xparser.cpp
src/xmagics/pythonexec.cpp
)

set(XEUS_CPP_MAIN_SRC
Expand Down Expand Up @@ -327,23 +331,30 @@ if (XEUS_CPP_BUILD_EXECUTABLE)
xeus_cpp_set_common_options(xcpp)
xeus_cpp_set_kernel_options(xcpp)
target_link_libraries(xcpp PRIVATE xeus-zmq)
set_target_properties(xcpp PROPERTIES
ENABLE_EXPORTS 1
CXX_STANDARD ${CMAKE_CXX_STANDARD})
target_link_libraries(xcpp PUBLIC xeus-cpp pthread Python::Python)

##TODO: We may be need sse RPATH
set_target_properties(xcpp clangCppInterOp PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE)
if(APPLE)
target_link_libraries(xcpp PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup)
elseif(NOT MSVC)
target_link_libraries(xcpp PUBLIC -Wl,--unresolved-symbols=ignore-in-object-files)
endif()

##set_target_properties(xcpp PROPERTIES
## ENABLE_EXPORTS 1
## CXX_STANDARD ${CMAKE_CXX_STANDARD}
##)
##target_link_libraries(xcpp PUBLIC xeus-cpp pthread Python::Python)
##
###TODO: We may be need sse RPATH
###set_target_properties(xcpp clangCppInterOp PROPERTIES
### INSTALL_RPATH_USE_LINK_PATH TRUE
###)
##if(APPLE)
## target_link_libraries(xcpp PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup)
##elseif(NOT MSVC)
## target_link_libraries(xcpp PUBLIC -Wl,--unresolved-symbols=ignore-in-object-files)
##endif()

target_include_directories(xeus-cpp PUBLIC ${Python_INCLUDE_DIRS})
target_link_libraries(xeus-cpp PUBLIC ${PYTHON_LIBRARIES})
target_link_libraries(xeus-cpp ${PYTHON_LIBRARIES_Development_Main})
set_target_properties(xeus-cpp PROPERTIES
PUBLIC_HEADER "${XEUS_CPP_HEADERS}"
COMPILE_DEFINITIONS "XEUS_CPP_EXPORTS"
PREFIX ""
VERSION ${${PROJECT_NAME}_VERSION}
SOVERSION ${XEUS_CPP_VERSION_MAJOR}
OUTPUT_NAME "libxeus-cpp"
CXX_STANDARD ${CMAKE_CXX_STANDARD})
endif()

if(XEUS_CPP_EMSCRIPTEN_WASM_BUILD)
Expand Down
8 changes: 5 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
# https://hub.docker.com/r/jupyter/base-notebook/tags
ARG BASE_CONTAINER=jupyter/base-notebook
ARG BASE_TAG=latest
ARG BUILD_TYPE=Release
ARG BUILD_TYPE=Debug

FROM $BASE_CONTAINER:$BASE_TAG

Expand Down Expand Up @@ -80,7 +80,7 @@ USER ${NB_UID}
ENV NB_PYTHON_PREFIX=${CONDA_DIR} \
KERNEL_PYTHON_PREFIX=${CONDA_DIR} \
# CPLUS_INCLUDE_PATH="${CONDA_DIR}/include:/home/${NB_USER}/include:/home/runner/work/xeus-clang-repl/xeus-clang-repl/clang-dev/clang/include:/home/jovyan/clad/include:/home/jovyan/CppInterOp/include"
CPLUS_INCLUDE_PATH="${CONDA_DIR}/include:/home/${NB_USER}/include:/home/jovyan/clad/include:/home/jovyan/CppInterOp/include"
CPLUS_INCLUDE_PATH="${CONDA_DIR}/include:/home/jovyan/clad/include:/home/jovyan/CppInterOp/include"

WORKDIR "${HOME}"

Expand Down Expand Up @@ -131,6 +131,7 @@ RUN \
cppzmq \
xtl \
'clangdev>=17' \
'llvm-openmp' \
pugixml \
cpp-argparse \
zlib \
Expand Down Expand Up @@ -173,7 +174,8 @@ RUN \
# Build CppInterOp
#
sys_incs=$(LC_ALL=C c++ -xc++ -E -v /dev/null 2>&1 | LC_ALL=C sed -ne '/starts here/,/End of/p' | LC_ALL=C sed '/^ /!d' | cut -c2- | tr '\n' ':') && \
export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/include/llvm:${PATH_TO_LLVM_BUILD}/include/clange:$CPLUS_INCLUDE_PATH:${sys_incs%:}" && \
#/usr/include/x86_64-linux-gnu:/usr/include:
export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/include/llvm:${PATH_TO_LLVM_BUILD}/include/clang:$CPLUS_INCLUDE_PATH:${sys_incs%:}" && \
git clone https://github.com/compiler-research/CppInterOp.git && \
export CB_PYTHON_DIR="$PWD/cppyy-backend/python" && \
export CPPINTEROP_DIR="$CB_PYTHON_DIR/cppyy_backend" && \
Expand Down
13 changes: 13 additions & 0 deletions src/xinterpreter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,18 @@
#include "xeus-cpp/xbuffer.hpp"
#include "xeus-cpp/xeus_cpp_config.hpp"

#include "xeus-cpp/xinterpreter.hpp"
#include "xeus-cpp/xmagics.hpp"

#include "xinput.hpp"
// #include "xinspect.hpp"
// #include "xmagics/executable.hpp"
// #include "xmagics/execution.hpp"
#include "xmagics/os.hpp"
#include "xmagics/pythonexec.hpp"
#include "xparser.hpp"
#include "xsystem.hpp"

using Args = std::vector<const char*>;

void* createInterpreter(const Args &ExtraArgs = {}) {
Expand Down Expand Up @@ -329,6 +341,7 @@ namespace xcpp
// preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("executable", executable(m_interpreter));
// preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("file", writefile());
// preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("timeit", timeit(&m_interpreter));
preamble_manager["magics"].get_cast<xmagics_manager>().register_magic("python", pythonexec());
}

std::string interpreter::get_stdopt(int argc, const char* const* argv)
Expand Down
2 changes: 1 addition & 1 deletion src/xmagics/os.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
#include "os.hpp"
#include "../xparser.hpp"

#include "xeus-cpp/xoptions.hpp"
//#include "xeus-cpp/xoptions.hpp"

namespace xcpp
{
Expand Down
210 changes: 210 additions & 0 deletions src/xmagics/pythonexec.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
//===------------ pythonexec.hpp - Python/C++ Interoperability ------------===//
//
// Licensed under the Apache License v2.0.
// SPDX-License-Identifier: Apache-2.0
//
// The full license is in the file LICENSE, distributed with this software.
//
//===----------------------------------------------------------------------===//
//
// This file defines the Python/C++ interoperability in which two cells within
// the same notebook can be in a either language.
//
//===----------------------------------------------------------------------===//

#include "Python.h"

#include "pythonexec.hpp"
#include "../xparser.hpp"

#include <cstddef>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>

namespace xcpp {
static PyObject *gMainDict = 0;

void pythonexec::startup() {
static bool isInitialized = false;
if (isInitialized)
return;

Py_Initialize();

PyRun_SimpleString("import sys\nprint(sys.path)");

// Import cppyy module
PyObject* cppyyModule = PyImport_ImportModule("cppyy");
if (!cppyyModule) {
PyErr_Print();
Py_Finalize();
return; // Handle import error as needed
}

PyObject* mainModule = PyImport_AddModule("__main__");
PyObject_SetAttrString(mainModule, "cppyy", cppyyModule);
Py_XDECREF(cppyyModule);
isInitialized = true;
// gMainDict = PyModule_GetDict(mainModule);
// Py_INCREF(gMainDict);
// if (!gMainDict)
// printf("Could not add module __main__");


// // Retrieve the dictionary of cppyy module
// PyObject* cppyyDict = PyModule_GetDict(cppyyModule);
// Py_DECREF(cppyyModule);
// if (!cppyyDict) {
// PyErr_Print();
// Py_Finalize();
// return; // Handle retrieval error as needed
// }

// // Add cppyyDict to gMainDict (if needed for further usage)
// PyDict_Update(gMainDict, cppyyDict);

// Py_DECREF(cppyyDict);
// PyRun_SimpleString("import cppyy");
}

argparser pythonexec::get_options() {
argparser argpars{"python", "Start executing Python Cell"};
return argpars;
}

void pythonexec::execute(std::string &line, std::string &cell) {
// std::istringstream iss(line);
// std::vector<std::string> results((std::istream_iterator<std::string>(iss)),
// std::istream_iterator<std::string>());
startup();

// auto argpars = get_options();
// argpars.parse(line);

std::string code;

code += cell;
if (trim(code).empty())
return;
// PyRun_SimpleString(
// "globals_copy_lists = "
// "globals().copy()\nfirst_dict_ints={k:globals_copy_lists[k] for k in "
// "set(globals_copy_lists) if type(globals_copy_lists[k]) == "
// "int}\nfirst_dict_lists={k:globals_copy_lists[k] for k in "
// "set(globals_copy_lists) if type(globals_copy_lists[k]) == list}");

// PyRun_SimpleString("tmp = globals().copy()\nvars = [f'int {k} = {v};' for
// k,v in tmp.items() if type(v) == int and not k.startswith('_') and k!='tmp'
// and k!='In' and k!='Out' and k!='sys' and not hasattr(v,
// '__call__')]\nprint(vars)"); PyRun_SimpleString("b =
// globals().copy()\nnew_ints = ' '.join([f'int {k} = {b[k]};' for k in set(b)
// - set(first_dict) if type(b[k]) == int])\nprint('new_ints: ', new_ints)");

Cpp::BeginStdStreamCapture(Cpp::kStdErr);
Cpp::BeginStdStreamCapture(Cpp::kStdOut);

PyRun_SimpleString(code.c_str());

std::cout << Cpp::EndStdStreamCapture();
std::cerr << Cpp::EndStdStreamCapture();

// PyObject* objectsRepresentation = PyObject_Repr(gMainDict);
// const char* s = PyUnicode_AsUTF8(objectsRepresentation);
// printf("REPR of global dict: %s\n", s);
}

void pythonexec::update_python_dict_var(const char *name, int value) {
if (!gMainDict)
startup();
PyObject *s;
s = PyLong_FromLong(value);
PyDict_SetItemString(gMainDict, name, s);
Py_DECREF(s);
}

void pythonexec::update_python_dict_var_vector(const char *name,
std::vector<int> &data) {
if (!gMainDict)
startup();
PyObject *listObj = PyList_New(data.size());
if (!listObj)
throw std::logic_error("Unable to allocate memory for Python list");
for (unsigned int i = 0; i < data.size(); i++) {
PyObject *num = PyLong_FromLong((int)data[i]);
if (!num) {
Py_DECREF(listObj);
throw std::logic_error("Unable to allocate memory for Python list");
}
PyList_SET_ITEM(listObj, i, num);
}
PyDict_SetItemString(gMainDict, name, listObj);
}

// check python globals
void pythonexec::check_python_globals() {
if (!Py_IsInitialized())
return;
// execute the command
PyRun_SimpleString("print(globals())");
}

// execute a python comand
void pythonexec::exec_python_simple_command(const std::string code) {
if (!Py_IsInitialized())
return;
// execute the command
PyRun_SimpleString(code.c_str());
}

std::string pythonexec::transfer_python_ints_utility() {
if (!Py_IsInitialized())
return " ";
// transfer ints utility
PyRun_SimpleString(
"def getNewInts():\n glob_ints_utils = globals().copy()\n new_ints "
"= ' '.join([f'int {k} = {glob_ints_utils[k]};' for k in "
"set(glob_ints_utils) - set(first_dict_ints) if type(glob_ints_utils[k]) "
"== int])\n return new_ints");
PyObject *ints_result =
PyObject_CallFunction(PyDict_GetItemString(gMainDict, "getNewInts"), 0);
if (!ints_result) {
printf("Could not retrieve Python integers!\n");
return " ";
} else {
std::string newPythonInts = PyUnicode_AsUTF8(ints_result);
// printf("new ints %s\n", PyUnicode_AsUTF8(ints_result));
Py_DECREF(ints_result);
return newPythonInts;
}
}

std::string pythonexec::transfer_python_lists_utility() {
if (!Py_IsInitialized())
return " ";
// transfer lists utility
PyRun_SimpleString("def getNewLists():\n l = globals().copy()\n "
"new_lists = ' '.join([f'int {k}() = {l[k]};' for k in "
"set(l) - set(first_dict_lists) if type(l[k]) == "
"list]).replace('[','{').replace(']','}').replace('(','[')"
".replace(')',']')\n return new_lists");
PyObject *lists_result =
PyObject_CallFunction(PyDict_GetItemString(gMainDict, "getNewLists"), 0);
if (!lists_result) {
printf("Could not retrieve Python lists!\n");
return " ";
} else {
std::string newPythonLists = PyUnicode_AsUTF8(lists_result);
Py_DECREF(lists_result);
return newPythonLists;
}
}

bool pythonexec::python_check_for_initialisation() {
if (!Py_IsInitialized())
return false;
return true;
}
} // namespace xcpp
Loading

0 comments on commit 0de88f2

Please sign in to comment.