diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 7d6215364..185d49d5a 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -11,9 +11,13 @@ jobs: build: runs-on: ubuntu-latest + env: + USE_BAZEL_VERSION=7.2.1 + steps: - uses: actions/checkout@v4 + - name: Search for duplicated error codes run: bash find_duplicate_error_codes.sh @@ -27,12 +31,14 @@ jobs: sudo apt-get install -y openjdk-11-jdk libzmq3-dev - name: Java Tests run: | - export USE_BAZEL_VERSION=7.2.1 bazel test //base/javacontainer/test/... working-directory: ./exaudfclient/ - name: ExaudfLib Tests run: | - export USE_BAZEL_VERSION=7.2.1 bazel test //base/exaudflib/test/... working-directory: ./exaudfclient/ + - name: Script Options Parser Tests + run: | + bazel test //base/script_options_parser/test/... + working-directory: ./exaudfclient/ diff --git a/exaudfclient/README.md b/exaudfclient/README.md index 848c6212a..c15b86918 100644 --- a/exaudfclient/README.md +++ b/exaudfclient/README.md @@ -70,7 +70,7 @@ The usage of multiple linker namespace requires some precautions in the build pr ## Precautions in the build process -In the build process you need to be cautious which libraries you link together and that no link leaks symbols from a library in one namespace to a library in the other namespace. Furthermore, you have to build a shared library with all dependency linked to it as output target. In our case, we have to main output targets //:exaudfclient and //:libexaudflib.so. Both get loaded into different linker namespaces. The language container live in the same namespace as //:exaudfclient. This namespace must not know anyhing about protobuf and zeromq, because it is possible that a language container may load protobuf or zeromq in a different version. Protobuf and zeromq are only known in the namespace of //:libexaudflib.so. The target //:libexaudflib.so depends on //base/exaudflib:exaudflib which contains the logic of the exaudflib. You must not depend on //base/exaudflib:exaudflib in //:exaudfclient or the langauge container, because this would leak zeromq and protobuf. If you need to depend on the other dependency of //base/exaudflib:exaudflib which not depend on protobuf or zeromq them self, such as //base/exaudflib:script_data_transfer_objects, //base/exaudflib:script_data_transfer_objects_wrapper, //base/exaudflib:scriptoptionlines, use either their target as self, the collection of libraries //base/exaudflib:exaudflib-deps or the collection of headers //base/exaudflib:header. +In the build process you need to be cautious which libraries you link together and that no link leaks symbols from a library in one namespace to a library in the other namespace. Furthermore, you have to build a shared library with all dependency linked to it as output target. In our case, we have to main output targets //:exaudfclient and //:libexaudflib.so. Both get loaded into different linker namespaces. The language container live in the same namespace as //:exaudfclient. This namespace must not know anyhing about protobuf and zeromq, because it is possible that a language container may load protobuf or zeromq in a different version. Protobuf and zeromq are only known in the namespace of //:libexaudflib.so. The target //:libexaudflib.so depends on //base/exaudflib:exaudflib which contains the logic of the exaudflib. You must not depend on //base/exaudflib:exaudflib in //:exaudfclient or the langauge container, because this would leak zeromq and protobuf. If you need to depend on the other dependency of //base/exaudflib:exaudflib which not depend on protobuf or zeromq them self, such as //base/exaudflib:script_data_transfer_objects, //base/exaudflib:script_data_transfer_objects_wrapper, //base/script_options_parser:scriptoptionlines, use either their target as self, the collection of libraries //base/exaudflib:exaudflib-deps or the collection of headers //base/exaudflib:header. ## Precautions in the implementations diff --git a/exaudfclient/base/benchmark_container/BUILD b/exaudfclient/base/benchmark_container/BUILD index dcdffdcdd..33b5be8cd 100644 --- a/exaudfclient/base/benchmark_container/BUILD +++ b/exaudfclient/base/benchmark_container/BUILD @@ -4,5 +4,5 @@ cc_library( name = "benchmark_container", srcs = ["benchmark_container.cc","benchmark_container.h"], hdrs = ["benchmark_container.h"], - deps = ["//base/exaudflib:exaudflib-deps","//base/exaudflib:header", "//base:debug_message_h"]+["//base/exaudflib:scriptoptionlines"] + deps = ["//base/exaudflib:exaudflib-deps","//base/exaudflib:header", "//base:debug_message_h"]+["//base/script_options_parser:scriptoptionlines"] ) \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/BUILD b/exaudfclient/base/exaudflib/BUILD index d82726c02..ed07bfde9 100644 --- a/exaudfclient/base/exaudflib/BUILD +++ b/exaudfclient/base/exaudflib/BUILD @@ -3,15 +3,6 @@ exports_files(["exascript.i"]) load("//:variables.bzl", "VM_ENABLED_DEFINES") -cc_library( - name = "scriptoptionlines", - hdrs = ["vm/scriptoptionlines.h"], - srcs = ["vm/scriptoptionlines.cc","vm/scriptoptionlines.h"], - copts= ["-fno-lto"], - # We deactivate link time optimization, because otherwise we get the following linker/compiler error while linking/compiling exaudfclient - # function ExecutionGraph::extractOptionLine(std::__cxx11::basic_string, std::allocator >&, std::__cxx11::basic_string, std::allocator >, std::__cxx11::basic_string, std::allocator >, std::__cxx11::basic_string, std::allocator >, unsigned long&, std::function): error: undefined reference to 'std::__cxx11::basic_string, std::allocator > std::operator+, std::allocator >(std::__cxx11::basic_string, std::allocator > const&, char const*)' -) - cc_library( name = "script_data_transfer_objects", srcs = ["swig/script_data_transfer_objects.cc","swig/script_data_transfer_objects.h"], @@ -82,7 +73,7 @@ cc_library( # The only target which must depend on it is //:libexaudflib-complete.so. # If you depend on this target you will load protobuf into your linker namespace # which might cause problems with tensorflow. If you need -# "//base/exaudflib:scriptoptionlines","//base/exaudflib:script_data_transfer_objects" or +# "//base/exaudflib:script_data_transfer_objects" or # "//base/exaudflib:script_data_transfer_objects_wrapper" in the language container # depend directly on them or depend on exaudflib-deps cc_library( diff --git a/exaudfclient/base/exaudflib/swig/swig_meta_data.h b/exaudfclient/base/exaudflib/swig/swig_meta_data.h index cad380dc2..28f310ea6 100644 --- a/exaudfclient/base/exaudflib/swig/swig_meta_data.h +++ b/exaudfclient/base/exaudflib/swig/swig_meta_data.h @@ -27,7 +27,7 @@ class SWIGMetadata { if (impl!=nullptr) { delete impl; } - } + } virtual const char* databaseName() { return impl->databaseName(); } virtual const char* databaseVersion() { return impl->databaseVersion(); } virtual const char* scriptName() { return impl->scriptName(); } diff --git a/exaudfclient/base/exaudflib/test/BUILD b/exaudfclient/base/exaudflib/test/BUILD index 231096441..51f6e75a4 100644 --- a/exaudfclient/base/exaudflib/test/BUILD +++ b/exaudfclient/base/exaudflib/test/BUILD @@ -1,9 +1,8 @@ cc_test( name = "exaudflib-test", - srcs = ["script_data_transfer_objects_test.cpp", "script_option_lines_test.cpp"], + srcs = ["script_data_transfer_objects_test.cpp"], deps = [ "//base/exaudflib:script_data_transfer_objects", - "//base/exaudflib:scriptoptionlines", "@googletest//:gtest_main", ], ) \ No newline at end of file diff --git a/exaudfclient/base/javacontainer/BUILD b/exaudfclient/base/javacontainer/BUILD index b5224f1ba..6b5feb3e1 100644 --- a/exaudfclient/base/javacontainer/BUILD +++ b/exaudfclient/base/javacontainer/BUILD @@ -108,7 +108,7 @@ cc_library( name = "javacontainer", srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h", ":dummy"], hdrs = [":filter_swig_code_exascript_java_h", "exascript_java_jni_decl.h"], - deps = ["@ssl//:ssl","@java//:java", ":exascript_java", "//base/exaudflib:header", "//base:debug_message_h","//base/exaudflib:scriptoptionlines"], + deps = ["@ssl//:ssl","@java//:java", ":exascript_java", "//base/exaudflib:header", "//base:debug_message_h","//base/javacontainer/script_options:java_scriptoptionlines"], # copts= ["-O0","-fno-lto"], alwayslink=True, ) diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.cc b/exaudfclient/base/javacontainer/javacontainer_impl.cc index 35786801d..780bed0ef 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.cc +++ b/exaudfclient/base/javacontainer/javacontainer_impl.cc @@ -1,19 +1,18 @@ #include #include #include -#include #include #include #include #include -#include "base/exaudflib/swig/swig_meta_data.h" #include "exascript_java_jni_decl.h" #include "base/debug_message.h" #include "base/javacontainer/javacontainer.h" #include "base/javacontainer/javacontainer_impl.h" -#include "base/exaudflib/vm/scriptoptionlines.h" +#include "base/javacontainer/script_options/converter.h" + using namespace SWIGVMContainers; using namespace std; @@ -23,11 +22,23 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI): m_checkOnly(checkOnly), m_ex stringstream ss; m_exaJavaPath = "/exaudf/base/javacontainer"; // TODO hardcoded path - DBG_FUNC_CALL(cerr,getScriptClassName()); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used - DBG_FUNC_CALL(cerr,importScripts()); - DBG_FUNC_CALL(cerr,getExternalJvmOptions()); + + + + JavaScriptOptions::ScriptOptionsConverter optionsConverter([&](const std::string &msg){throwException(msg);}, + m_jvmOptions); + + DBG_FUNC_CALL(cerr,optionsConverter.getScriptClassName(m_scriptCode)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used + DBG_FUNC_CALL(cerr,optionsConverter.convertImportScripts(m_scriptCode)); + DBG_FUNC_CALL(cerr,optionsConverter.getExternalJvmOptions(m_scriptCode)); DBG_FUNC_CALL(cerr,setClasspath()); - DBG_FUNC_CALL(cerr,addExternalJarPaths()); + DBG_FUNC_CALL(cerr,optionsConverter.getExternalJarPaths(m_scriptCode)); + + for (set::iterator it = optionsConverter.getJarPaths().begin(); it != optionsConverter.getJarPaths().end(); + ++it) { + addJarToClasspath(*it); + } + m_needsCompilation = checkNeedsCompilation(); if (m_needsCompilation) { DBG_FUNC_CALL(cerr,addPackageToScript()); @@ -233,94 +244,6 @@ void JavaVMImpl::compileScript() { check("F-UDF-CL-SL-JAVA-1037",calledUndefinedSingleCall); } -void JavaVMImpl::addExternalJarPaths() { - const string jarKeyword = "%jar"; - const string whitespace = " \t\f\v"; - const string lineEnd = ";"; - size_t pos; - while (true) { - string jarPath = ExecutionGraph::extractOptionLine(m_scriptCode, jarKeyword, whitespace, lineEnd, pos, [&](const char* msg){throwException(msg);}); - if (jarPath == "") - break; - for (size_t start = 0, delim = 0; ; start = delim + 1) { - delim = jarPath.find(":", start); - if (delim != string::npos) { - string jar = jarPath.substr(start, delim - start); - if (m_jarPaths.find(jar) == m_jarPaths.end()) - m_jarPaths.insert(jar); - } - else { - string jar = jarPath.substr(start); - if (m_jarPaths.find(jar) == m_jarPaths.end()) - m_jarPaths.insert(jar); - break; - } - } - } - for (set::iterator it = m_jarPaths.begin(); it != m_jarPaths.end(); ++it) { - addJarToClasspath(*it); - } -} - -void JavaVMImpl::getScriptClassName() { - const string scriptClassKeyword = "%scriptclass"; - const string whitespace = " \t\f\v"; - const string lineEnd = ";"; - size_t pos; - string scriptClass = - ExecutionGraph::extractOptionLine( - m_scriptCode, - scriptClassKeyword, - whitespace, - lineEnd, - pos, - [&](const char* msg){throwException("F-UDF-CL-SL-JAVA-1038: "+std::string(msg));} - ); - if (scriptClass != "") { - m_jvmOptions.push_back("-Dexasol.scriptclass=" + scriptClass); - } -} - -void JavaVMImpl::importScripts() { - SWIGMetadata *meta = NULL; - const string importKeyword = "%import"; - const string whitespace = " \t\f\v"; - const string lineEnd = ";"; - size_t pos; - // Attention: We must hash the parent script before modifying it (adding the - // package definition). Otherwise we don't recognize if the script imports its self - m_importedScriptChecksums.insert(scriptToMd5(m_scriptCode.c_str())); - while (true) { - string scriptName = - ExecutionGraph::extractOptionLine( - m_scriptCode, - importKeyword, - whitespace, - lineEnd, - pos, - [&](const char* msg){throwException("F-UDF-CL-SL-JAVA-1039: "+std::string(msg));} - ); - if (scriptName == "") - break; - if (!meta) { - meta = new SWIGMetadata(); - if (!meta) - throwException("F-UDF-CL-SL-JAVA-1040: Failure while importing scripts"); - } - const char *scriptCode = meta->moduleContent(scriptName.c_str()); - const char *exception = meta->checkException(); - if (exception) - throwException("F-UDF-CL-SL-JAVA-1041: "+std::string(exception)); - if (m_importedScriptChecksums.insert(scriptToMd5(scriptCode)).second) { - // Script has not been imported yet - // If this imported script contains %import statements - // they will be resolved in this while loop. - m_scriptCode.insert(pos, scriptCode); - } - } - if (meta) - delete meta; -} bool JavaVMImpl::check(const string& errorCode, string& calledUndefinedSingleCall) { jthrowable ex = m_env->ExceptionOccurred(); @@ -448,15 +371,6 @@ bool JavaVMImpl::checkNeedsCompilation() { return false == trimmedScriptCode.empty(); } -vector JavaVMImpl::scriptToMd5(const char *script) { - MD5_CTX ctx; - unsigned char md5[MD5_DIGEST_LENGTH]; - MD5_Init(&ctx); - MD5_Update(&ctx, script, strlen(script)); - MD5_Final(md5, &ctx); - return vector(md5, md5 + sizeof(md5)); -} - void JavaVMImpl::addJarToClasspath(const string& path) { string jarPath = path; // m_exaJavaPath + "/jars/" + path; @@ -488,38 +402,6 @@ void JavaVMImpl::addJarToClasspath(const string& path) { m_classpath += ":" + jarPath; } -void JavaVMImpl::getExternalJvmOptions() { - const string jvmOption = "%jvmoption"; - const string whitespace = " \t\f\v"; - const string lineEnd = ";"; - size_t pos; - while (true) { - string options = - ExecutionGraph::extractOptionLine( - m_scriptCode, - jvmOption, - whitespace, - lineEnd, - pos, - [&](const char* msg){throwException("F-UDF-CL-SL-JAVA-1064: "+std::string(msg));} - ); - if (options == "") - break; - for (size_t start = 0, delim = 0; ; start = delim + 1) { - start = options.find_first_not_of(whitespace, start); - if (start == string::npos) - break; - delim = options.find_first_of(whitespace, start); - if (delim != string::npos) { - m_jvmOptions.push_back(options.substr(start, delim - start)); - } - else { - m_jvmOptions.push_back(options.substr(start)); - break; - } - } - } -} void JavaVMImpl::setJvmOptions() { bool minHeap = false; diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.h b/exaudfclient/base/javacontainer/javacontainer_impl.h index 79f67318f..97ce92f63 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.h +++ b/exaudfclient/base/javacontainer/javacontainer_impl.h @@ -35,22 +35,14 @@ class JavaVMImpl { void throwException(const char *message); void throwException(const std::exception& ex); void throwException(const std::string& ex); - //void throwException(swig_undefined_single_call_exception& ex); - void importScripts(); - void addExternalJarPaths(); - void getExternalJvmOptions(); - void getScriptClassName(); void setJvmOptions(); void addJarToClasspath(const std::string& path); - std::vector scriptToMd5(const char *script); bool m_checkOnly; std::string m_exaJavaPath; std::string m_localClasspath; std::string m_scriptCode; std::string m_exaJarPath; std::string m_classpath; - std::set > m_importedScriptChecksums; - std::set m_jarPaths; bool m_exceptionThrown; std::vector m_jvmOptions; JavaVM *m_jvm; diff --git a/exaudfclient/base/javacontainer/script_options/BUILD b/exaudfclient/base/javacontainer/script_options/BUILD new file mode 100644 index 000000000..0b1db87b3 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/BUILD @@ -0,0 +1,9 @@ +package(default_visibility = ["//visibility:public"]) + + +cc_library( + name = "java_script_option_lines", + hdrs = [":converter.h"], + srcs = [":parser.h", ":converter.h", ":converter.cc", ":parser_legacy.h", ":parser_legacy.cc"], + deps = ["//base/script_options_parser:script_option_lines_parser", "//base/exaudflib:header", "//base/exaudflib:exaudflib-deps"], +) diff --git a/exaudfclient/base/javacontainer/script_options/converter.cc b/exaudfclient/base/javacontainer/script_options/converter.cc new file mode 100644 index 000000000..dc9f2f35d --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/converter.cc @@ -0,0 +1,124 @@ +#include "base/javacontainer/script_options/converter.h" +#include "base/javacontainer/script_options/parser_legacy.h" +#include "base/exaudflib/swig/swig_meta_data.h" +#include +#include + + + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +inline std::vector scriptToMd5(const char *script) { + MD5_CTX ctx; + unsigned char md5[MD5_DIGEST_LENGTH]; + MD5_Init(&ctx); + MD5_Update(&ctx, script, strlen(script)); + MD5_Final(md5, &ctx); + return std::vector(md5, md5 + sizeof(md5)); +} + + +ScriptOptionsConverter::ScriptOptionsConverter(std::function throwException, + std::vector& jvmOptions): +m_scriptOptionsParser(std::make_unique()), +m_throwException(throwException), +m_jvmOptions(jvmOptions), +m_jarPaths() +{} + +void ScriptOptionsConverter::getExternalJarPaths(std::string & src_scriptCode) { + std::vector jarPaths; + m_scriptOptionsParser->findExternalJarPaths(src_scriptCode, jarPaths, + [&](const std::string& msg){m_throwException("F-UDF-CL-SL-JAVA-1600" + msg);}); + for (const std::string& jarPath : jarPaths) { + for (size_t start = 0, delim = 0; ; start = delim + 1) { + delim = jarPath.find(":", start); + if (delim != std::string::npos) { + std::string jar = jarPath.substr(start, delim - start); + if (m_jarPaths.find(jar) == m_jarPaths.end()) + m_jarPaths.insert(jar); + } + else { + std::string jar = jarPath.substr(start); + if (m_jarPaths.find(jar) == m_jarPaths.end()) + m_jarPaths.insert(jar); + break; + } + } + } +} + +void ScriptOptionsConverter::getScriptClassName(std::string & src_scriptCode) { + std::string scriptClass; + + m_scriptOptionsParser->getScriptClassName(src_scriptCode, scriptClass, + [&](const std::string& msg){m_throwException("F-UDF-CL-SL-JAVA-1601: " + msg);}); + + if (scriptClass != "") { + m_jvmOptions.push_back("-Dexasol.scriptclass=" + scriptClass); + } +} + +void ScriptOptionsConverter::getExternalJvmOptions(std::string & src_scriptCode) { + std::vector jvmOptions; + const std::string whitespace = " \t\f\v"; + m_scriptOptionsParser->getExternalJvmOptions(src_scriptCode, jvmOptions, + [&](const std::string& msg){m_throwException("F-UDF-CL-SL-JAVA-1602" + msg);}); + + for (const std::string& jvmOption: jvmOptions) { + for (size_t start = 0, delim = 0; ; start = delim + 1) { + start = jvmOption.find_first_not_of(whitespace, start); + if (start == std::string::npos) + break; + delim = jvmOption.find_first_of(whitespace, start); + if (delim != std::string::npos) { + m_jvmOptions.push_back(jvmOption.substr(start, delim - start)); + } + else { + m_jvmOptions.push_back(jvmOption.substr(start)); + break; + } + } + } +} + +void ScriptOptionsConverter::convertImportScripts(std::string & src_scriptCode) { + SWIGMetadata *meta = NULL; + // Attention: We must hash the parent script before modifying it (adding the + // package definition). Otherwise we don't recognize if the script imports its self + m_importedScriptChecksums.insert(scriptToMd5(src_scriptCode.c_str())); + while (true) { + std::pair nextImportStatement; + m_scriptOptionsParser->getNextImportScript(src_scriptCode, nextImportStatement, + [&](const std::string& msg){m_throwException("F-UDF-CL-SL-JAVA-1604" + msg);}); + if (nextImportStatement.first == "") + break; + if (!meta) { + meta = new SWIGMetadata(); + if (!meta) + m_throwException("F-UDF-CL-SL-JAVA-1603: Failure while importing scripts"); + } + const char *scriptCode = meta->moduleContent(nextImportStatement.first.c_str()); + const char *exception = meta->checkException(); + if (exception) + m_throwException("F-UDF-CL-SL-JAVA-1605: "+std::string(exception)); + if (m_importedScriptChecksums.insert(scriptToMd5(scriptCode)).second) { + // Script has not been imported yet + // If this imported script contains %import statements + // they will be resolved in this while loop. + src_scriptCode.insert(nextImportStatement.second, scriptCode); + } + } + if (meta) + delete meta; +} + +const std::set & ScriptOptionsConverter::getJarPaths() { + return m_jarPaths; +} + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/script_options/converter.h b/exaudfclient/base/javacontainer/script_options/converter.h new file mode 100644 index 000000000..cf2bce0ab --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/converter.h @@ -0,0 +1,54 @@ +#ifndef SCRIPTOPTIONLINEPARSERCONVERTER_H +#define SCRIPTOPTIONLINEPARSERCONVERTER_H 1 + +#include +#include +#include +#include +#include + +#include "base/javacontainer/script_options/parser.h" + + + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +class ScriptOptionsConverter { + + public: + ScriptOptionsConverter(std::function throwException, + std::vector& jvmOptions); + + + void getExternalJarPaths(std::string & src_scriptCode); + + void getScriptClassName(std::string & src_scriptCode); + + void convertImportScripts(std::string & src_scriptCode); + + void getExternalJvmOptions(std::string & src_scriptCode); + + const std::set & getJarPaths(); + private: + + std::unique_ptr m_scriptOptionsParser; + + std::function m_throwException; + + std::vector& m_jvmOptions; + + std::set m_jarPaths; + + std::set > m_importedScriptChecksums; +}; + + + + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + +#endif //SCRIPTOPTIONLINEPARSERCONVERTER_H \ No newline at end of file diff --git a/exaudfclient/base/javacontainer/script_options/parser.h b/exaudfclient/base/javacontainer/script_options/parser.h new file mode 100644 index 000000000..bfc2ed8ee --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/parser.h @@ -0,0 +1,35 @@ +#ifndef SCRIPTOPTIONLINEPARSER_H +#define SCRIPTOPTIONLINEPARSER_H 1 + +#include +#include +#include + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +struct ScriptOptionsParser { + + virtual void findExternalJarPaths(std::string & src_scriptCode, + std::vector& jarPaths, + std::function throwException) = 0; + + virtual void getScriptClassName(std::string & src_scriptCode, std::string &scriptClassName, + std::function throwException) = 0; + + virtual void getNextImportScript(std::string & src_scriptCode, + std::pair & result, + std::function throwException) = 0; + + virtual void getExternalJvmOptions(std::string & src_scriptCode, + std::vector& jvmOptions, + std::function throwException) = 0; +}; + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + + +#endif //SCRIPTOPTIONLINEPARSER_H \ No newline at end of file diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc new file mode 100644 index 000000000..cd73abb78 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg.cc @@ -0,0 +1,10 @@ +#include "base/javacontainer/script_options/parser_ctpg.h" + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/script_options/parser_ctpg.h b/exaudfclient/base/javacontainer/script_options/parser_ctpg.h new file mode 100644 index 000000000..3b71a5eeb --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/parser_ctpg.h @@ -0,0 +1,13 @@ +#ifndef SCRIPTOPTIONLINEPARSERCTPGY_H +#define SCRIPTOPTIONLINEPARSERCTPGY_H 1 + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + +#endif //SCRIPTOPTIONLINEPARSERCTPGY_H diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.cc b/exaudfclient/base/javacontainer/script_options/parser_legacy.cc new file mode 100644 index 000000000..f236ffb63 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.cc @@ -0,0 +1,86 @@ +#include "base/javacontainer/script_options/parser_legacy.h" +#include "base/script_options_parser/script_option_lines.h" + + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +ScriptOptionLinesParserLegacy::ScriptOptionLinesParserLegacy() +: m_whitespace(" \t\f\v") +, m_lineend(";") +, m_jarKeyword("%jar") +, m_scriptClassKeyword("%scriptclass") +, m_importKeyword("%import") +, m_jvmOptionKeyword("%jvmoption") {} + +void ScriptOptionLinesParserLegacy::findExternalJarPaths(std::string & src_scriptCode, + std::vector& jarPaths, + std::function throwException) { + callParserForManyValues(src_scriptCode, m_jarKeyword, jarPaths, throwException); +} + + +void ScriptOptionLinesParserLegacy::getScriptClassName(std::string & src_scriptCode, std::string &scriptClassName, + std::function throwException) { + size_t pos; + scriptClassName = callParserForSingleValue(src_scriptCode, pos, m_scriptClassKeyword, throwException); +} + + +void ScriptOptionLinesParserLegacy::getNextImportScript(std::string & src_scriptCode, + std::pair & result, + std::function throwException) { + size_t pos; + const std::string scriptName = callParserForSingleValue(src_scriptCode, pos, m_importKeyword, throwException); + result.first = scriptName; + result.second = pos; +} + + +void ScriptOptionLinesParserLegacy::getExternalJvmOptions(std::string & src_scriptCode, + std::vector& jvmOptions, + std::function throwException) { + callParserForManyValues(src_scriptCode, m_jvmOptionKeyword, jvmOptions, throwException); +} + + +std::string ScriptOptionLinesParserLegacy::callParserForSingleValue(std::string & src_scriptCode, size_t & pos, + const std::string & keyword, + std::function throwException) { + return ExecutionGraph::extractOptionLine( + src_scriptCode, + keyword, + m_whitespace, + m_lineend, + pos, + [&](const char* msg){throwException(std::string("F-UDF-CL-SL-JAVA-1606: ") + msg);} + ); +} + +void ScriptOptionLinesParserLegacy::callParserForManyValues(std::string & src_scriptCode, + const std::string & keyword, + std::vector& result, + std::function throwException) { + size_t pos; + while (true) { + const std::string options = + ExecutionGraph::extractOptionLine( + src_scriptCode, + keyword, + m_whitespace, + m_lineend, + pos, + [&](const char* msg){throwException(std::string("F-UDF-CL-SL-JAVA-1607: ") + msg);} + ); + if (options == "") + break; + result.push_back(options); + } +} + + + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.h b/exaudfclient/base/javacontainer/script_options/parser_legacy.h new file mode 100644 index 000000000..5aac0f2b4 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.h @@ -0,0 +1,53 @@ +#ifndef SCRIPTOPTIONLINEPARSERLEGACY_H +#define SCRIPTOPTIONLINEPARSERLEGACY_H 1 + + +#include "base/javacontainer/script_options/parser.h" + + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { + + public: + ScriptOptionLinesParserLegacy(); + + void findExternalJarPaths(std::string & src_scriptCode, + std::vector& jarPaths, + std::function throwException); + + void getScriptClassName(std::string & src_scriptCode, std::string &scriptClassName, + std::function throwException); + + void getNextImportScript(std::string & src_scriptCode, + std::pair & result, + std::function throwException); + + + void getExternalJvmOptions(std::string & src_scriptCode, + std::vector& jvmOptions, + std::function throwException); + + private: + std::string callParserForSingleValue(std::string & src_scriptCode, size_t &pos, const std::string & keyword, + std::function throwException); + void callParserForManyValues(std::string & src_scriptCode, const std::string & keyword, + std::vector& result, + std::function throwException); + private: + const std::string m_whitespace; + const std::string m_lineend; + const std::string m_jarKeyword; + const std::string m_scriptClassKeyword; + const std::string m_importKeyword; + const std::string m_jvmOptionKeyword; +}; + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + + +#endif //SCRIPTOPTIONLINEPARSERLEGACY_H \ No newline at end of file diff --git a/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc b/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc index 387cfda6d..8b098a3f3 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javacontainer_test.cc @@ -13,8 +13,6 @@ TEST(JavaContainer, basic_jar) { EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, "\n"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/base/javacontainer/exaudf_deploy.jar"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/exaudf/base/javacontainer/exaudf_deploy.jar:base/javacontainer/test/test.jar"); - const std::vector expectedJarPaths = {"base/javacontainer/test/test.jar"}; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jarPaths, expectedJarPaths); EXPECT_FALSE(vm.getJavaVMInternalStatus().m_needsCompilation); const std::vector expectedJVMOptions = { "-Dexasol.scriptclass=com.exasol.udf_profiling.UdfProfiler", "-Xms128m", "-Xmx128m", "-Xss512k", @@ -44,7 +42,6 @@ TEST(JavaContainer, basic_inline) { EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/base/javacontainer/exaudf_deploy.jar"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar"); - EXPECT_TRUE(vm.getJavaVMInternalStatus().m_jarPaths.empty()); EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); const std::vector expectedJVMOptions = { "-Xms128m", "-Xmx128m", "-Xss512k", "-XX:ErrorFile=/tmp/hs_err_pid%p.log", @@ -75,8 +72,6 @@ TEST(JavaContainer, combined_inline_jar) { EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/base/javacontainer/exaudf_deploy.jar"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar:base/javacontainer/test/test.jar"); - const std::vector expectedJarPaths = {"base/javacontainer/test/test.jar"}; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jarPaths, expectedJarPaths); EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); const std::vector expectedJVMOptions = { "-Xms128m", "-Xmx128m", "-Xss512k", "-XX:ErrorFile=/tmp/hs_err_pid%p.log", @@ -106,8 +101,6 @@ TEST(JavaContainer, quoted_jvm_option) { EXPECT_EQ(vm.getJavaVMInternalStatus().m_scriptCode, expected_script_code); EXPECT_EQ(vm.getJavaVMInternalStatus().m_exaJarPath, "/exaudf/base/javacontainer/exaudf_deploy.jar"); EXPECT_EQ(vm.getJavaVMInternalStatus().m_classpath, "/tmp:/exaudf/base/javacontainer/exaudf_deploy.jar"); - const std::vector expectedJarPaths = {}; - EXPECT_EQ(vm.getJavaVMInternalStatus().m_jarPaths, expectedJarPaths); EXPECT_TRUE(vm.getJavaVMInternalStatus().m_needsCompilation); /* * Note: The option "DEF" is wrong and causes UDF's to crash! diff --git a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc index f4c0310c4..c4ceab9c3 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc +++ b/exaudfclient/base/javacontainer/test/cpp/javavm_test.cc @@ -12,7 +12,6 @@ JavaVMTest::JavaVMTest(std::string scriptCode) : javaVMInternalStatus() { javaVMInternalStatus.m_scriptCode = javaVMImpl.m_scriptCode; javaVMInternalStatus.m_exaJarPath = javaVMImpl.m_exaJarPath; javaVMInternalStatus.m_classpath = javaVMImpl.m_classpath; - javaVMInternalStatus.m_jarPaths.assign(javaVMImpl.m_jarPaths.begin(), javaVMImpl.m_jarPaths.end()); javaVMInternalStatus.m_jvmOptions = javaVMImpl.m_jvmOptions; javaVMInternalStatus.m_needsCompilation = javaVMImpl.m_needsCompilation; delete script_code; diff --git a/exaudfclient/base/javacontainer/test/cpp/javavm_test.h b/exaudfclient/base/javacontainer/test/cpp/javavm_test.h index 2c580e460..5de601e70 100644 --- a/exaudfclient/base/javacontainer/test/cpp/javavm_test.h +++ b/exaudfclient/base/javacontainer/test/cpp/javavm_test.h @@ -12,7 +12,6 @@ struct JavaVMInternalStatus { std::string m_exaJarPath; std::string m_classpath; bool m_needsCompilation; - std::vector m_jarPaths; std::vector m_jvmOptions; }; diff --git a/exaudfclient/base/python/python3/BUILD b/exaudfclient/base/python/python3/BUILD index 422693eb2..ccde16665 100644 --- a/exaudfclient/base/python/python3/BUILD +++ b/exaudfclient/base/python/python3/BUILD @@ -92,7 +92,7 @@ cc_library( hdrs = [":exascript_python_int", ":filter_swig_code_exascript_python_h"], include_prefix = ".", deps = ["@python3//:python3",":exascript_python","//base/exaudflib:header", - "//base:debug_message_h","//base/exaudflib:scriptoptionlines", "//base/python:pythoncontainer_header"], + "//base:debug_message_h", "//base/python:pythoncontainer_header"], alwayslink=True, ) diff --git a/exaudfclient/base/python/pythoncontainer.cc b/exaudfclient/base/python/pythoncontainer.cc index d87d373e1..cc258113b 100644 --- a/exaudfclient/base/python/pythoncontainer.cc +++ b/exaudfclient/base/python/pythoncontainer.cc @@ -1,5 +1,4 @@ #include "pythoncontainer.h" -#include "base/exaudflib/swig/swig_meta_data.h" #include #ifdef _POSIX_C_SOURCE #undef _POSIX_C_SOURCE @@ -11,7 +10,6 @@ #include "exascript_python_int.h" #include "exascript_python.h" #include "base/debug_message.h" -#include "base/exaudflib/vm/scriptoptionlines.h" #include "base/exaudflib/swig/script_data_transfer_objects.h" diff --git a/exaudfclient/base/script_options_parser/BUILD b/exaudfclient/base/script_options_parser/BUILD new file mode 100644 index 000000000..39d46a95a --- /dev/null +++ b/exaudfclient/base/script_options_parser/BUILD @@ -0,0 +1,8 @@ +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "script_option_lines_parser", + hdrs = ["script_option_lines.h"], + srcs = ["script_option_lines.cc","script_option_lines.h"], + copts= ["-fno-lto"], +) diff --git a/exaudfclient/base/exaudflib/vm/scriptoptionlines.cc b/exaudfclient/base/script_options_parser/script_option_lines.cc similarity index 99% rename from exaudfclient/base/exaudflib/vm/scriptoptionlines.cc rename to exaudfclient/base/script_options_parser/script_option_lines.cc index dbc521658..4baf5776b 100644 --- a/exaudfclient/base/exaudflib/vm/scriptoptionlines.cc +++ b/exaudfclient/base/script_options_parser/script_option_lines.cc @@ -2,6 +2,8 @@ #include #include + + namespace ExecutionGraph { @@ -100,4 +102,5 @@ std::string extractOptionLine(std::string& code, const std::string option, const return result; } + } // namespace ExecutionGraph diff --git a/exaudfclient/base/exaudflib/vm/scriptoptionlines.h b/exaudfclient/base/script_options_parser/script_option_lines.h similarity index 97% rename from exaudfclient/base/exaudflib/vm/scriptoptionlines.h rename to exaudfclient/base/script_options_parser/script_option_lines.h index 065ae8a37..e84fbdc53 100644 --- a/exaudfclient/base/exaudflib/vm/scriptoptionlines.h +++ b/exaudfclient/base/script_options_parser/script_option_lines.h @@ -23,4 +23,4 @@ std::string extractOptionLine(std::string& code, const std::string option, const } // namespace ExecutionGraph -#endif // SCRIPTOPTIONLINES_H +#endif // SCRIPTOPTIONLINES_H \ No newline at end of file diff --git a/exaudfclient/base/script_options_parser/test/BUILD b/exaudfclient/base/script_options_parser/test/BUILD new file mode 100644 index 000000000..b9543cf89 --- /dev/null +++ b/exaudfclient/base/script_options_parser/test/BUILD @@ -0,0 +1,8 @@ +cc_test( + name = "script-options-parser-test", + srcs = ["script_option_lines_test.cpp"], + deps = [ + "//base/script_options_parser:scriptoptionlinesparser", + "@googletest//:gtest_main", + ], +) \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/test/script_option_lines_test.cpp b/exaudfclient/base/script_options_parser/test/script_option_lines_test.cpp similarity index 88% rename from exaudfclient/base/exaudflib/test/script_option_lines_test.cpp rename to exaudfclient/base/script_options_parser/test/script_option_lines_test.cpp index 5866169a7..4a13e9369 100644 --- a/exaudfclient/base/exaudflib/test/script_option_lines_test.cpp +++ b/exaudfclient/base/script_options_parser/test/script_option_lines_test.cpp @@ -1,4 +1,4 @@ -#include "base/exaudflib/vm/scriptoptionlines.h" +#include "base/script_options_parser/scriptoptionlines.h" #include #include #include @@ -183,3 +183,28 @@ TEST(ScriptOptionLinesTest, test_values_must_not_contain_spaces) { EXPECT_EQ(code, expected_result_code); } +TEST(ScriptOptionLinesTest, test_multiple_lines_with_code) { + /** + Verify that the parser can read options coming after some code. + */ + size_t pos; + const std::string original_code = + "%jvmoption -Dhttp.agent=\"ABC DEF\"; class Abc{};\n\n" + "%jar /buckets/bucketfs1/jars/exajdbc.jar; class DEF{};\n"; + std::string code = original_code; + + std::string res = extractOptionLine(code, "%jvmoption", whitespace, lineEnd, pos, throwException); + EXPECT_EQ(res, "-Dhttp.agent=\"ABC DEF\""); + std::string expected_result_code = + " class Abc{};\n\n" + "%jar /buckets/bucketfs1/jars/exajdbc.jar; class DEF{};\n"; + EXPECT_EQ(code, expected_result_code); + + res = extractOptionLine(code, "%jar", whitespace, lineEnd, pos, throwException); + EXPECT_EQ(res, "/buckets/bucketfs1/jars/exajdbc.jar"); + expected_result_code = + " class Abc{};\n\n" + " class DEF{};\n"; + + EXPECT_EQ(code, expected_result_code); +}