diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 185d49d5..04632466 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -12,8 +12,7 @@ jobs: runs-on: ubuntu-latest env: - USE_BAZEL_VERSION=7.2.1 - + USE_BAZEL_VERSION: 7.2.1 steps: - uses: actions/checkout@v4 diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc b/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc index 67f6d374..067b91b7 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc +++ b/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc @@ -4,7 +4,7 @@ extern "C" { -SWIGVMContainers::SWIGMetadata* create_SWIGMetaData() { +SWIGVMContainers::SWIGMetadataIf* create_SWIGMetaData() { return new SWIGVMContainers::SWIGMetadata_Impl(); } diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h b/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h index bebb5612..d666b74e 100644 --- a/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h +++ b/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h @@ -13,10 +13,9 @@ namespace SWIGVMContainers { -class SWIGMetadata_Impl : public SWIGMetadata { +class SWIGMetadata_Impl : public SWIGMetadataIf { public: SWIGMetadata_Impl(): - SWIGMetadata(false), m_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id), m_socket(*(exaudflib::global.sock)), m_exch(&exaudflib::global.exchandler), diff --git a/exaudfclient/base/exaudflib/swig/swig_meta_data.h b/exaudfclient/base/exaudflib/swig/swig_meta_data.h index 28f310ea..875f3d64 100644 --- a/exaudfclient/base/exaudflib/swig/swig_meta_data.h +++ b/exaudfclient/base/exaudflib/swig/swig_meta_data.h @@ -7,9 +7,56 @@ namespace SWIGVMContainers { -class SWIGMetadata { - SWIGMetadata* impl=nullptr; - typedef SWIGVMContainers::SWIGMetadata* (*CREATE_METADATA_FUN)(); +struct SWIGMetadataIf { + + virtual ~SWIGMetadataIf() {}; + virtual const char* databaseName() = 0; + virtual const char* databaseVersion() = 0; + virtual const char* scriptName() = 0; + virtual const char* scriptSchema() = 0; + virtual const char* currentUser() = 0; + virtual const char* scopeUser() = 0; + virtual const char* currentSchema() = 0; + virtual const char* scriptCode() = 0; + virtual const unsigned long long sessionID() = 0; + virtual const char *sessionID_S() = 0; + virtual const unsigned long statementID() = 0; + virtual const unsigned int nodeCount() = 0; + virtual const unsigned int nodeID() = 0; + virtual const unsigned long long vmID() = 0; + virtual const unsigned long long memoryLimit() = 0; + virtual const VMTYPE vmType() = 0; + virtual const char *vmID_S() = 0; + virtual const ExecutionGraph::ConnectionInformationWrapper* connectionInformation(const char* connection_name) = 0; + virtual const char* moduleContent(const char* name) = 0; + virtual const unsigned int inputColumnCount() = 0; + virtual const char *inputColumnName(unsigned int col) = 0; + virtual const SWIGVM_datatype_e inputColumnType(unsigned int col) = 0; + virtual const char *inputColumnTypeName(unsigned int col) = 0; + virtual const unsigned int inputColumnSize(unsigned int col) = 0; + virtual const unsigned int inputColumnPrecision(unsigned int col) = 0; + virtual const unsigned int inputColumnScale(unsigned int col) = 0; + virtual const SWIGVM_itertype_e inputType() = 0; + virtual const unsigned int outputColumnCount() = 0; + virtual const char *outputColumnName(unsigned int col) = 0; + virtual const SWIGVM_datatype_e outputColumnType(unsigned int col) = 0; + virtual const char *outputColumnTypeName(unsigned int col) = 0; + virtual const unsigned int outputColumnSize(unsigned int col) = 0; + virtual const unsigned int outputColumnPrecision(unsigned int col) = 0; + virtual const unsigned int outputColumnScale(unsigned int col) = 0; + virtual const SWIGVM_itertype_e outputType() = 0; + virtual const bool isEmittedColumn(unsigned int col) = 0; + virtual const char* checkException() = 0; + virtual const char* pluginLanguageName() = 0; + virtual const char* pluginURI() = 0; + virtual const char* outputAddress() = 0; +}; + + + +class SWIGMetadata : public SWIGMetadataIf { + SWIGMetadataIf* impl=nullptr; + typedef SWIGVMContainers::SWIGMetadataIf* (*CREATE_METADATA_FUN)(); public: SWIGMetadata() { diff --git a/exaudfclient/base/exaudflib/udf_plugin_interface.h b/exaudfclient/base/exaudflib/udf_plugin_interface.h index 39454004..47e047c7 100644 --- a/exaudfclient/base/exaudflib/udf_plugin_interface.h +++ b/exaudfclient/base/exaudflib/udf_plugin_interface.h @@ -10,7 +10,7 @@ class SWIGTableIterator; } extern "C" { -SWIGVMContainers::SWIGMetadata* create_SWIGMetaData(); +SWIGVMContainers::SWIGMetadataIf* create_SWIGMetaData(); SWIGVMContainers::AbstractSWIGTableIterator* create_SWIGTableIterator(); SWIGVMContainers::SWIGRAbstractResultHandler* create_SWIGResultHandler(SWIGVMContainers::SWIGTableIterator* table_iterator); } diff --git a/exaudfclient/base/javacontainer/BUILD b/exaudfclient/base/javacontainer/BUILD index 6b5feb3e..724b5a91 100644 --- a/exaudfclient/base/javacontainer/BUILD +++ b/exaudfclient/base/javacontainer/BUILD @@ -108,7 +108,7 @@ cc_library( name = "javacontainer", srcs = [":javacontainer.cc", ":javacontainer.h", ":javacontainer_impl.cc", ":javacontainer_impl.h", ":dummy"], hdrs = [":filter_swig_code_exascript_java_h", "exascript_java_jni_decl.h"], - deps = ["@ssl//:ssl","@java//:java", ":exascript_java", "//base/exaudflib:header", "//base:debug_message_h","//base/javacontainer/script_options:java_scriptoptionlines"], + deps = ["@ssl//:ssl","@java//:java", ":exascript_java", "//base/exaudflib:header", "//base:debug_message_h","//base/javacontainer/script_options:java_script_option_lines"], # copts= ["-O0","-fno-lto"], alwayslink=True, ) diff --git a/exaudfclient/base/javacontainer/javacontainer_impl.cc b/exaudfclient/base/javacontainer/javacontainer_impl.cc index 780bed0e..882fb6dd 100644 --- a/exaudfclient/base/javacontainer/javacontainer_impl.cc +++ b/exaudfclient/base/javacontainer/javacontainer_impl.cc @@ -11,7 +11,8 @@ #include "base/debug_message.h" #include "base/javacontainer/javacontainer.h" #include "base/javacontainer/javacontainer_impl.h" -#include "base/javacontainer/script_options/converter.h" +#include "base/javacontainer/script_options/extractor.h" +#include "base/javacontainer/script_options/parser_legacy.h" using namespace SWIGVMContainers; @@ -23,18 +24,17 @@ JavaVMImpl::JavaVMImpl(bool checkOnly, bool noJNI): m_checkOnly(checkOnly), m_ex stringstream ss; m_exaJavaPath = "/exaudf/base/javacontainer"; // TODO hardcoded path + JavaScriptOptions::ScriptOptionLinesParserLegacy scriptOptionsParser; + JavaScriptOptions::Extractor extractor(scriptOptionsParser, + [&](const std::string &msg){throwException(msg);}); + DBG_FUNC_CALL(cerr,extractor.extract(m_scriptCode)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used - JavaScriptOptions::ScriptOptionsConverter optionsConverter([&](const std::string &msg){throwException(msg);}, - m_jvmOptions); - - DBG_FUNC_CALL(cerr,optionsConverter.getScriptClassName(m_scriptCode)); // To be called before scripts are imported. Otherwise, the script classname from an imported script could be used - DBG_FUNC_CALL(cerr,optionsConverter.convertImportScripts(m_scriptCode)); - DBG_FUNC_CALL(cerr,optionsConverter.getExternalJvmOptions(m_scriptCode)); DBG_FUNC_CALL(cerr,setClasspath()); - DBG_FUNC_CALL(cerr,optionsConverter.getExternalJarPaths(m_scriptCode)); - for (set::iterator it = optionsConverter.getJarPaths().begin(); it != optionsConverter.getJarPaths().end(); + m_jvmOptions = std::move(extractor.moveJvmOptions()); + + for (set::iterator it = extractor.getJarPaths().begin(); it != extractor.getJarPaths().end(); ++it) { addJarToClasspath(*it); } diff --git a/exaudfclient/base/javacontainer/script_options/BUILD b/exaudfclient/base/javacontainer/script_options/BUILD index 0b1db87b..a833102b 100644 --- a/exaudfclient/base/javacontainer/script_options/BUILD +++ b/exaudfclient/base/javacontainer/script_options/BUILD @@ -3,7 +3,9 @@ package(default_visibility = ["//visibility:public"]) cc_library( name = "java_script_option_lines", - hdrs = [":converter.h"], - srcs = [":parser.h", ":converter.h", ":converter.cc", ":parser_legacy.h", ":parser_legacy.cc"], - deps = ["//base/script_options_parser:script_option_lines_parser", "//base/exaudflib:header", "//base/exaudflib:exaudflib-deps"], + hdrs = [":extractor.h", ":parser_legacy.h"], + srcs = [":parser.h", ":converter.h", ":converter.cc", ":parser_legacy.cc", ":extractor.cc", + ":keywords.h", ":checksum.h", ":checksum.cc"], + deps = ["//base/script_options_parser:script_option_lines_parser", "//base:debug_message_h", + "//base/exaudflib:header", "//base/exaudflib:exaudflib-deps"], ) diff --git a/exaudfclient/base/javacontainer/script_options/checksum.cc b/exaudfclient/base/javacontainer/script_options/checksum.cc new file mode 100644 index 00000000..ae003c60 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/checksum.cc @@ -0,0 +1,26 @@ +#include "base/javacontainer/script_options/checksum.h" +#include +#include + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +inline std::vector scriptToMd5(const char *script) { + MD5_CTX ctx; + unsigned char md5[MD5_DIGEST_LENGTH]; + MD5_Init(&ctx); + MD5_Update(&ctx, script, strlen(script)); + MD5_Final(md5, &ctx); + return std::vector(md5, md5 + sizeof(md5)); +} + + +bool Checksum::addScript(const char *script) { + return m_importedScriptChecksums.insert(scriptToMd5(script)).second; +} + + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/script_options/checksum.h b/exaudfclient/base/javacontainer/script_options/checksum.h new file mode 100644 index 00000000..374b41b7 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/checksum.h @@ -0,0 +1,29 @@ +#ifndef SCRIPTOPTIONLINEPARSERCHECKSUM_H +#define SCRIPTOPTIONLINEPARSERCHECKSUM_H 1 + +#include +#include +#include + + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +class Checksum { + +public: + Checksum() = default; + + bool addScript(const char *script); + +private: + std::set > m_importedScriptChecksums; +}; + + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + +#endif //SCRIPTOPTIONLINEPARSERCHECKSUM_H \ No newline at end of file diff --git a/exaudfclient/base/javacontainer/script_options/converter.cc b/exaudfclient/base/javacontainer/script_options/converter.cc index dc9f2f35..e73cada5 100644 --- a/exaudfclient/base/javacontainer/script_options/converter.cc +++ b/exaudfclient/base/javacontainer/script_options/converter.cc @@ -1,123 +1,54 @@ #include "base/javacontainer/script_options/converter.h" -#include "base/javacontainer/script_options/parser_legacy.h" -#include "base/exaudflib/swig/swig_meta_data.h" -#include -#include - - +#include namespace SWIGVMContainers { namespace JavaScriptOptions { -inline std::vector scriptToMd5(const char *script) { - MD5_CTX ctx; - unsigned char md5[MD5_DIGEST_LENGTH]; - MD5_Init(&ctx); - MD5_Update(&ctx, script, strlen(script)); - MD5_Final(md5, &ctx); - return std::vector(md5, md5 + sizeof(md5)); -} - - -ScriptOptionsConverter::ScriptOptionsConverter(std::function throwException, - std::vector& jvmOptions): -m_scriptOptionsParser(std::make_unique()), -m_throwException(throwException), -m_jvmOptions(jvmOptions), -m_jarPaths() -{} - -void ScriptOptionsConverter::getExternalJarPaths(std::string & src_scriptCode) { - std::vector jarPaths; - m_scriptOptionsParser->findExternalJarPaths(src_scriptCode, jarPaths, - [&](const std::string& msg){m_throwException("F-UDF-CL-SL-JAVA-1600" + msg);}); - for (const std::string& jarPath : jarPaths) { - for (size_t start = 0, delim = 0; ; start = delim + 1) { - delim = jarPath.find(":", start); - if (delim != std::string::npos) { - std::string jar = jarPath.substr(start, delim - start); - if (m_jarPaths.find(jar) == m_jarPaths.end()) - m_jarPaths.insert(jar); - } - else { - std::string jar = jarPath.substr(start); - if (m_jarPaths.find(jar) == m_jarPaths.end()) - m_jarPaths.insert(jar); - break; - } +Converter::Converter() +: m_jvmOptions() +, m_jarPaths() +, m_whitespace(" \t\f\v") {} + +void Converter::convertExternalJar(const std::string & value) { + for (size_t start = 0, delim = 0; ; start = delim + 1) { + delim = value.find(":", start); + if (delim != std::string::npos) { + std::string jar = value.substr(start, delim - start); + if (m_jarPaths.find(jar) == m_jarPaths.end()) + m_jarPaths.insert(jar); + } + else { + std::string jar = value.substr(start); + if (m_jarPaths.find(jar) == m_jarPaths.end()) + m_jarPaths.insert(jar); + break; } } } -void ScriptOptionsConverter::getScriptClassName(std::string & src_scriptCode) { - std::string scriptClass; - - m_scriptOptionsParser->getScriptClassName(src_scriptCode, scriptClass, - [&](const std::string& msg){m_throwException("F-UDF-CL-SL-JAVA-1601: " + msg);}); - - if (scriptClass != "") { - m_jvmOptions.push_back("-Dexasol.scriptclass=" + scriptClass); - } -} - -void ScriptOptionsConverter::getExternalJvmOptions(std::string & src_scriptCode) { - std::vector jvmOptions; - const std::string whitespace = " \t\f\v"; - m_scriptOptionsParser->getExternalJvmOptions(src_scriptCode, jvmOptions, - [&](const std::string& msg){m_throwException("F-UDF-CL-SL-JAVA-1602" + msg);}); - - for (const std::string& jvmOption: jvmOptions) { - for (size_t start = 0, delim = 0; ; start = delim + 1) { - start = jvmOption.find_first_not_of(whitespace, start); - if (start == std::string::npos) - break; - delim = jvmOption.find_first_of(whitespace, start); - if (delim != std::string::npos) { - m_jvmOptions.push_back(jvmOption.substr(start, delim - start)); - } - else { - m_jvmOptions.push_back(jvmOption.substr(start)); - break; - } - } +void Converter::convertScriptClassName(const std::string & value) { + if (value != "") { + m_jvmOptions.push_back("-Dexasol.scriptclass=" + value); } } -void ScriptOptionsConverter::convertImportScripts(std::string & src_scriptCode) { - SWIGMetadata *meta = NULL; - // Attention: We must hash the parent script before modifying it (adding the - // package definition). Otherwise we don't recognize if the script imports its self - m_importedScriptChecksums.insert(scriptToMd5(src_scriptCode.c_str())); - while (true) { - std::pair nextImportStatement; - m_scriptOptionsParser->getNextImportScript(src_scriptCode, nextImportStatement, - [&](const std::string& msg){m_throwException("F-UDF-CL-SL-JAVA-1604" + msg);}); - if (nextImportStatement.first == "") +void Converter::convertJvmOption(const std::string & value) { + for (size_t start = 0, delim = 0; ; start = delim + 1) { + start = value.find_first_not_of(m_whitespace, start); + if (start == std::string::npos) break; - if (!meta) { - meta = new SWIGMetadata(); - if (!meta) - m_throwException("F-UDF-CL-SL-JAVA-1603: Failure while importing scripts"); + delim = value.find_first_of(m_whitespace, start); + if (delim != std::string::npos) { + m_jvmOptions.push_back(value.substr(start, delim - start)); } - const char *scriptCode = meta->moduleContent(nextImportStatement.first.c_str()); - const char *exception = meta->checkException(); - if (exception) - m_throwException("F-UDF-CL-SL-JAVA-1605: "+std::string(exception)); - if (m_importedScriptChecksums.insert(scriptToMd5(scriptCode)).second) { - // Script has not been imported yet - // If this imported script contains %import statements - // they will be resolved in this while loop. - src_scriptCode.insert(nextImportStatement.second, scriptCode); + else { + m_jvmOptions.push_back(value.substr(start)); + break; } } - if (meta) - delete meta; } -const std::set & ScriptOptionsConverter::getJarPaths() { - return m_jarPaths; -} } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/converter.h b/exaudfclient/base/javacontainer/script_options/converter.h index cf2bce0a..eda178db 100644 --- a/exaudfclient/base/javacontainer/script_options/converter.h +++ b/exaudfclient/base/javacontainer/script_options/converter.h @@ -15,33 +15,32 @@ namespace SWIGVMContainers { namespace JavaScriptOptions { -class ScriptOptionsConverter { +class Converter { public: - ScriptOptionsConverter(std::function throwException, - std::vector& jvmOptions); - + Converter(); - void getExternalJarPaths(std::string & src_scriptCode); + void convertExternalJar(const std::string & value); + + void convertScriptClassName(const std::string & value); - void getScriptClassName(std::string & src_scriptCode); + void convertJvmOption(const std::string & value); - void convertImportScripts(std::string & src_scriptCode); + const std::set & getJarPaths() const { + return m_jarPaths; + } - void getExternalJvmOptions(std::string & src_scriptCode); + std::vector&& moveJvmOptions() { + return std::move(m_jvmOptions); + } - const std::set & getJarPaths(); private: - - std::unique_ptr m_scriptOptionsParser; - - std::function m_throwException; - std::vector& m_jvmOptions; + std::vector m_jvmOptions; std::set m_jarPaths; - std::set > m_importedScriptChecksums; + const std::string m_whitespace; }; diff --git a/exaudfclient/base/javacontainer/script_options/extractor.cc b/exaudfclient/base/javacontainer/script_options/extractor.cc new file mode 100644 index 00000000..7f20c213 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/extractor.cc @@ -0,0 +1,36 @@ +#include "base/javacontainer/script_options/extractor.h" +#include "base/javacontainer/script_options/parser.h" + +#include "base/debug_message.h" +#include + +#define EXTR_DBG_FUNC_CALL(f) DBG_FUNC_CALL(std::cerr, f) + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +Extractor::Extractor(ScriptOptionsParser & parser, + std::function throwException) +: m_throwException(throwException) +, m_parser(parser) {} + + +void Extractor::extract(std::string & scriptCode) { + m_parser.prepareScriptCode(scriptCode); + EXTR_DBG_FUNC_CALL(m_parser.parseForScriptClass( [&](const std::string& value){ + EXTR_DBG_FUNC_CALL(m_converter.convertScriptClassName(value)); + }, m_throwException)); + EXTR_DBG_FUNC_CALL(m_parser.extractImportScripts(m_throwException)); + EXTR_DBG_FUNC_CALL(m_parser.parseForJvmOptions( [&](const std::string& value){ + EXTR_DBG_FUNC_CALL(m_converter.convertJvmOption(value)); + }, m_throwException)); + EXTR_DBG_FUNC_CALL(m_parser.parseForExternalJars( [&](const std::string& value){ + EXTR_DBG_FUNC_CALL(m_converter.convertExternalJar(value)); + }, m_throwException)); + scriptCode = std::move(m_parser.getScriptCode()); +} + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/script_options/extractor.h b/exaudfclient/base/javacontainer/script_options/extractor.h new file mode 100644 index 00000000..bebe5749 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/extractor.h @@ -0,0 +1,50 @@ +#ifndef SCRIPTOPTIONLINEPEXTRACTOR_H +#define SCRIPTOPTIONLINEPEXTRACTOR_H 1 + +#include +#include +#include +#include + + +#include "base/javacontainer/script_options/converter.h" + + + +namespace SWIGVMContainers { + +class SWIGMetadata; + +namespace JavaScriptOptions { + +class ScriptOptionsParser; + +class Extractor { + + public: + Extractor(ScriptOptionsParser & parser, + std::function throwException); + + const std::set & getJarPaths() const { + return m_converter.getJarPaths(); + } + + std::vector&& moveJvmOptions() { + return std::move(m_converter.moveJvmOptions()); + } + + void extract(std::string & scriptCode); + + private: + std::function m_throwException; + + Converter m_converter; + ScriptOptionsParser & m_parser; +}; + + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + +#endif //SCRIPTOPTIONLINEPEXTRACTOR_H \ No newline at end of file diff --git a/exaudfclient/base/javacontainer/script_options/keywords.h b/exaudfclient/base/javacontainer/script_options/keywords.h new file mode 100644 index 00000000..840509a9 --- /dev/null +++ b/exaudfclient/base/javacontainer/script_options/keywords.h @@ -0,0 +1,33 @@ +#ifndef SCRIPTOPTIONLINEKEYWORDS_H +#define SCRIPTOPTIONLINEKEYWORDS_H 1 + +#include + +namespace SWIGVMContainers { + +namespace JavaScriptOptions { + +class Keywords { + public: + Keywords() + : m_jarKeyword("%jar") + , m_scriptClassKeyword("%scriptclass") + , m_importKeyword("%import") + , m_jvmOptionKeyword("%jvmoption") {} + const std::string & jarKeyword() { return m_jarKeyword; } + const std::string & scriptClassKeyword() { return m_scriptClassKeyword; } + const std::string & importKeyword() { return m_importKeyword; } + const std::string & jvmOptionKeyword() { return m_jvmOptionKeyword; } + private: + std::string m_jarKeyword; + std::string m_scriptClassKeyword; + std::string m_importKeyword; + std::string m_jvmOptionKeyword; +}; + +} //namespace JavaScriptOptions + +} //namespace SWIGVMContainers + + +#endif // SCRIPTOPTIONLINEKEYWORDS_H \ No newline at end of file diff --git a/exaudfclient/base/javacontainer/script_options/parser.h b/exaudfclient/base/javacontainer/script_options/parser.h index bfc2ed8e..cc16e108 100644 --- a/exaudfclient/base/javacontainer/script_options/parser.h +++ b/exaudfclient/base/javacontainer/script_options/parser.h @@ -10,21 +10,44 @@ namespace SWIGVMContainers { namespace JavaScriptOptions { struct ScriptOptionsParser { + /* + Passes the script code for parsing to the parser. The parser might modify the script code, because it will remove + known options. + */ + virtual void prepareScriptCode(const std::string & scriptCode) = 0; + /* + Searches for script class option. + If the option is found, the function removes the option from "scriptCode" and calls "callback" with the option value and position + within "scriptCode". + */ + virtual void parseForScriptClass(std::function callback, + std::function throwException) = 0; + /* + Searches for JVM options. + If an option is found, the function removes the option from "scriptCode" and calls "callback" with the option value and position + within "scriptCode". + */ + virtual void parseForJvmOptions(std::function callback, + std::function throwException) = 0; - virtual void findExternalJarPaths(std::string & src_scriptCode, - std::vector& jarPaths, + /* + Searches for External Jar. + If an option is found, the function removes the option from "scriptCode" and calls "callback" with the option value and position + within "scriptCode". + */ + virtual void parseForExternalJars(std::function callback, std::function throwException) = 0; - virtual void getScriptClassName(std::string & src_scriptCode, std::string &scriptClassName, - std::function throwException) = 0; - - virtual void getNextImportScript(std::string & src_scriptCode, - std::pair & result, - std::function throwException) = 0; + /* + Searches for the "%import" options and embeds the respective imported script code at the same location as + the option in the script code. + */ + virtual void extractImportScripts(std::function throwException) = 0; - virtual void getExternalJvmOptions(std::string & src_scriptCode, - std::vector& jvmOptions, - std::function throwException) = 0; + /* + Returns the (eventually modified) script code. + */ + virtual std::string && getScriptCode() = 0; }; } //namespace JavaScriptOptions diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.cc b/exaudfclient/base/javacontainer/script_options/parser_legacy.cc index f236ffb6..7a8bee47 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_legacy.cc +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.cc @@ -1,6 +1,9 @@ #include "base/javacontainer/script_options/parser_legacy.h" +#include "base/javacontainer/script_options/checksum.h" #include "base/script_options_parser/script_option_lines.h" +#include "base/exaudflib/swig/swig_meta_data.h" +#include namespace SWIGVMContainers { @@ -9,78 +12,181 @@ namespace JavaScriptOptions { ScriptOptionLinesParserLegacy::ScriptOptionLinesParserLegacy() : m_whitespace(" \t\f\v") , m_lineend(";") -, m_jarKeyword("%jar") -, m_scriptClassKeyword("%scriptclass") -, m_importKeyword("%import") -, m_jvmOptionKeyword("%jvmoption") {} +, m_scriptCode() +, m_keywords() {} -void ScriptOptionLinesParserLegacy::findExternalJarPaths(std::string & src_scriptCode, - std::vector& jarPaths, - std::function throwException) { - callParserForManyValues(src_scriptCode, m_jarKeyword, jarPaths, throwException); +void ScriptOptionLinesParserLegacy::prepareScriptCode(const std::string & scriptCode) { + m_scriptCode = scriptCode; } - -void ScriptOptionLinesParserLegacy::getScriptClassName(std::string & src_scriptCode, std::string &scriptClassName, - std::function throwException) { - size_t pos; - scriptClassName = callParserForSingleValue(src_scriptCode, pos, m_scriptClassKeyword, throwException); +void ScriptOptionLinesParserLegacy::extractImportScripts(std::function throwException) { + std::unique_ptr metaData; + // Attention: We must hash the parent script before modifying it (adding the + // package definition). Otherwise we don't recognize if the script imports its self + Checksum importedScriptChecksums; + importedScriptChecksums.addScript(m_scriptCode.c_str()); + /* + The following while loop iteratively replaces import scripts in the script code. Each replacement is done in two steps: + 1. Remove the "%import ..." option in the script code + 2. Insert the new script code at the same location + It can happen that the imported scripts have again an "%import ..." option. + Those cases will be handled in the next iteration of the while loop, because the parser searches the options in + each iteration from the beginning of the (then modified) script code. + For example, lets assume the following Jave UDF, stored in member variable 'm_scriptCode': + %import other_script_A; + class MyJavaUdf { + static void run(ExaMetadata exa, ExaIterator ctx) throws Exception { + ctx.emit(\"Success!\");\n" + } + }; + and 'other_script_A' is defined as (which will be retrieved over SWIGMetadata.moduleContent()): + %import other_script_B; + class OtherClassA { + static void doSomething() { + } + }; + and other_script_B as: + %import other_script_A; + class OtherClassB { + static void doSomething() { + } + }; + The first iteration of the while loop would modify the member variable m_scriptCode to: + %import other_script_B; + class OtherClassA { + static void doSomething() { + } + }; + class MyJavaUdf { + static void run(ExaMetadata exa, ExaIterator ctx) throws Exception { + ctx.emit(\"Success!\");\n" + } + }; + The second iteration of the while loop would modify the member variable m_scriptCode to: + The first iteration of the while loop would modify the member variable m_scriptCode to: + %import other_script_A; + class OtherClassB { + static void doSomething() { + } + }; + class OtherClassA { + static void doSomething() { + } + }; + class MyJavaUdf { + static void run(ExaMetadata exa, ExaIterator ctx) throws Exception { + ctx.emit(\"Success!\");\n" + } + }; + The third iteration of the while loop would modify the member variable m_scriptCode to: + class OtherClassB { + static void doSomething() { + } + }; + class OtherClassA { + static void doSomething() { + } + }; + class MyJavaUdf { + static void run(ExaMetadata exa, ExaIterator ctx) throws Exception { + ctx.emit(\"Success!\");\n" + } + }; + because the content of "other_script_A" is already stored in the checksum, + and the parser only removes the script options keyword, but does not insert again the code of other_script_A. + The fourth iteration of the while loop would detect no further import option keywords and would break the loop. + */ + while (true) { + std::string newScript; + size_t scriptPos; + parseForSingleOption(m_keywords.importKeyword(), + [&](const std::string& value, size_t pos){scriptPos = pos; newScript = value;}, + [&](const std::string& msg){throwException("F-UDF-CL-SL-JAVA-1614" + msg);}); + if (!newScript.empty()) { + if (!metaData) { + metaData = std::make_unique(); + if (!metaData) + throwException("F-UDF-CL-SL-JAVA-1615: Failure while importing scripts"); + } + const char *importScriptCode = metaData->moduleContent(newScript.c_str()); + const char *exception = metaData->checkException(); + if (exception) + throwException("F-UDF-CL-SL-JAVA-1616: " + std::string(exception)); + if (importedScriptChecksums.addScript(importScriptCode)) { + // Script has not been imported yet + // If this imported script contains %import statements + // they will be resolved in the next iteration of the while loop. + m_scriptCode.insert(scriptPos, importScriptCode); + } + } else { + break; + } + } } - -void ScriptOptionLinesParserLegacy::getNextImportScript(std::string & src_scriptCode, - std::pair & result, - std::function throwException) { - size_t pos; - const std::string scriptName = callParserForSingleValue(src_scriptCode, pos, m_importKeyword, throwException); - result.first = scriptName; - result.second = pos; +void ScriptOptionLinesParserLegacy::parseForScriptClass(std::function callback, + std::function throwException) { + parseForSingleOption(m_keywords.scriptClassKeyword(), + [&](const std::string& value, size_t pos){callback(value);}, + [&](const std::string& msg){throwException("F-UDF-CL-SL-JAVA-1610" + msg);}); } +void ScriptOptionLinesParserLegacy::parseForJvmOptions(std::function callback, + std::function throwException) { + parseForMultipleOptions(m_keywords.jvmOptionKeyword(), + [&](const std::string& value, size_t pos){callback(value);}, + [&](const std::string& msg){throwException("F-UDF-CL-SL-JAVA-1612" + msg);}); +} -void ScriptOptionLinesParserLegacy::getExternalJvmOptions(std::string & src_scriptCode, - std::vector& jvmOptions, - std::function throwException) { - callParserForManyValues(src_scriptCode, m_jvmOptionKeyword, jvmOptions, throwException); +void ScriptOptionLinesParserLegacy::parseForExternalJars(std::function callback, + std::function throwException) { + parseForMultipleOptions(m_keywords.jarKeyword(), + [&](const std::string& value, size_t pos){callback(value);}, + [&](const std::string& msg){throwException("F-UDF-CL-SL-JAVA-1613" + msg);}); } +std::string && ScriptOptionLinesParserLegacy::getScriptCode() { + return std::move(m_scriptCode); +} -std::string ScriptOptionLinesParserLegacy::callParserForSingleValue(std::string & src_scriptCode, size_t & pos, - const std::string & keyword, - std::function throwException) { - return ExecutionGraph::extractOptionLine( - src_scriptCode, +void ScriptOptionLinesParserLegacy::parseForSingleOption(const std::string keyword, + std::function callback, + std::function throwException) { + size_t pos; + const std::string option = + ExecutionGraph::extractOptionLine( + m_scriptCode, keyword, m_whitespace, m_lineend, pos, [&](const char* msg){throwException(std::string("F-UDF-CL-SL-JAVA-1606: ") + msg);} ); + if (option != "") { + callback(option, pos); + } } -void ScriptOptionLinesParserLegacy::callParserForManyValues(std::string & src_scriptCode, - const std::string & keyword, - std::vector& result, - std::function throwException) { +void ScriptOptionLinesParserLegacy::parseForMultipleOptions(const std::string keyword, + std::function callback, + std::function throwException) { size_t pos; while (true) { - const std::string options = + const std::string option = ExecutionGraph::extractOptionLine( - src_scriptCode, + m_scriptCode, keyword, m_whitespace, m_lineend, pos, [&](const char* msg){throwException(std::string("F-UDF-CL-SL-JAVA-1607: ") + msg);} ); - if (options == "") + if (option == "") break; - result.push_back(options); + callback(option, pos); } } - - } //namespace JavaScriptOptions } //namespace SWIGVMContainers diff --git a/exaudfclient/base/javacontainer/script_options/parser_legacy.h b/exaudfclient/base/javacontainer/script_options/parser_legacy.h index 5aac0f2b..145fe74e 100644 --- a/exaudfclient/base/javacontainer/script_options/parser_legacy.h +++ b/exaudfclient/base/javacontainer/script_options/parser_legacy.h @@ -3,6 +3,7 @@ #include "base/javacontainer/script_options/parser.h" +#include "base/javacontainer/script_options/keywords.h" namespace SWIGVMContainers { @@ -14,35 +15,34 @@ class ScriptOptionLinesParserLegacy : public ScriptOptionsParser { public: ScriptOptionLinesParserLegacy(); - void findExternalJarPaths(std::string & src_scriptCode, - std::vector& jarPaths, - std::function throwException); + void prepareScriptCode(const std::string & scriptCode) override; - void getScriptClassName(std::string & src_scriptCode, std::string &scriptClassName, - std::function throwException); + void parseForScriptClass(std::function callback, + std::function throwException) override; - void getNextImportScript(std::string & src_scriptCode, - std::pair & result, - std::function throwException); + void parseForJvmOptions(std::function callback, + std::function throwException) override; + void parseForExternalJars(std::function callback, + std::function throwException) override; - void getExternalJvmOptions(std::string & src_scriptCode, - std::vector& jvmOptions, - std::function throwException); + void extractImportScripts(std::function throwException) override; + + std::string && getScriptCode() override; private: - std::string callParserForSingleValue(std::string & src_scriptCode, size_t &pos, const std::string & keyword, - std::function throwException); - void callParserForManyValues(std::string & src_scriptCode, const std::string & keyword, - std::vector& result, - std::function throwException); + void parseForSingleOption(const std::string key, + std::function callback, + std::function throwException); + void parseForMultipleOptions(const std::string key, + std::function callback, + std::function throwException); + private: const std::string m_whitespace; const std::string m_lineend; - const std::string m_jarKeyword; - const std::string m_scriptClassKeyword; - const std::string m_importKeyword; - const std::string m_jvmOptionKeyword; + std::string m_scriptCode; + Keywords m_keywords; }; } //namespace JavaScriptOptions diff --git a/exaudfclient/base/script_options_parser/script_option_lines.cc b/exaudfclient/base/script_options_parser/script_option_lines.cc index 4baf5776..4b3752e9 100644 --- a/exaudfclient/base/script_options_parser/script_option_lines.cc +++ b/exaudfclient/base/script_options_parser/script_option_lines.cc @@ -1,4 +1,4 @@ -#include "scriptoptionlines.h" +#include "script_option_lines.h" #include #include diff --git a/exaudfclient/base/script_options_parser/test/BUILD b/exaudfclient/base/script_options_parser/test/BUILD index b9543cf8..06c831be 100644 --- a/exaudfclient/base/script_options_parser/test/BUILD +++ b/exaudfclient/base/script_options_parser/test/BUILD @@ -2,7 +2,7 @@ cc_test( name = "script-options-parser-test", srcs = ["script_option_lines_test.cpp"], deps = [ - "//base/script_options_parser:scriptoptionlinesparser", + "//base/script_options_parser:script_option_lines_parser", "@googletest//:gtest_main", ], ) \ No newline at end of file diff --git a/exaudfclient/base/script_options_parser/test/script_option_lines_test.cpp b/exaudfclient/base/script_options_parser/test/script_option_lines_test.cpp index 4a13e936..fea8a04c 100644 --- a/exaudfclient/base/script_options_parser/test/script_option_lines_test.cpp +++ b/exaudfclient/base/script_options_parser/test/script_option_lines_test.cpp @@ -1,4 +1,4 @@ -#include "base/script_options_parser/scriptoptionlines.h" +#include "base/script_options_parser/script_option_lines.h" #include #include #include