From 24a35fd01ce9fbe5515fd4c1bc5c6684985e9515 Mon Sep 17 00:00:00 2001 From: Igor Gaponenko Date: Tue, 17 Sep 2024 18:51:51 -0700 Subject: [PATCH] A utility for parsing CSV dialect parameters of the ingest requests --- src/replica/ingest/CMakeLists.txt | 1 + src/replica/ingest/IngestHttpSvcMod.cc | 19 +-------- src/replica/ingest/IngestUtils.cc | 54 +++++++++++++++++++++++++ src/replica/ingest/IngestUtils.h | 55 ++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 17 deletions(-) create mode 100644 src/replica/ingest/IngestUtils.cc create mode 100644 src/replica/ingest/IngestUtils.h diff --git a/src/replica/ingest/CMakeLists.txt b/src/replica/ingest/CMakeLists.txt index 8e9b1bd15..5d6b99a97 100644 --- a/src/replica/ingest/CMakeLists.txt +++ b/src/replica/ingest/CMakeLists.txt @@ -13,6 +13,7 @@ target_sources(replica_ingest PRIVATE IngestResourceMgrT.cc IngestSvc.cc IngestSvcConn.cc + IngestUtils.cc TransactionContrib.cc ) target_link_libraries(replica_ingest PUBLIC diff --git a/src/replica/ingest/IngestHttpSvcMod.cc b/src/replica/ingest/IngestHttpSvcMod.cc index a922d89dd..718533a72 100644 --- a/src/replica/ingest/IngestHttpSvcMod.cc +++ b/src/replica/ingest/IngestHttpSvcMod.cc @@ -26,6 +26,7 @@ #include "http/Method.h" #include "replica/ingest/IngestRequest.h" #include "replica/ingest/IngestRequestMgr.h" +#include "replica/ingest/IngestUtils.h" #include "replica/services/ServiceProvider.h" #include "replica/util/Csv.h" @@ -183,23 +184,7 @@ shared_ptr IngestHttpSvcMod::_createRequest(bool async) const { string const url = body().required("url"); string const charsetName = body().optional("charset_name", config->get("worker", "ingest-charset-name")); - - csv::DialectInput dialectInput; - // Allow an empty string in the input. Simply replace the one (if present) with - // the corresponding default value of the parameter. - auto const getDialectParam = [&](string const& param, string const& defaultValue) -> string { - string val = body().optional(param, defaultValue); - if (val.empty()) val = defaultValue; - return val; - }; - dialectInput.fieldsTerminatedBy = - getDialectParam("fields_terminated_by", csv::Dialect::defaultFieldsTerminatedBy); - dialectInput.fieldsEnclosedBy = - getDialectParam("fields_enclosed_by", csv::Dialect::defaultFieldsEnclosedBy); - dialectInput.fieldsEscapedBy = getDialectParam("fields_escaped_by", csv::Dialect::defaultFieldsEscapedBy); - dialectInput.linesTerminatedBy = - getDialectParam("lines_terminated_by", csv::Dialect::defaultLinesTerminatedBy); - + auto const dialectInput = parseDialectInput(body()); auto const httpMethod = http::string2method(body().optional("http_method", "GET")); string const httpData = body().optional("http_data", string()); vector const httpHeaders = body().optionalColl("http_headers", vector()); diff --git a/src/replica/ingest/IngestUtils.cc b/src/replica/ingest/IngestUtils.cc new file mode 100644 index 000000000..d7c776f37 --- /dev/null +++ b/src/replica/ingest/IngestUtils.cc @@ -0,0 +1,54 @@ +/* + * LSST Data Management System + * + * This product includes software developed by the + * LSST Project (http://www.lsst.org/). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the LSST License Statement and + * the GNU General Public License along with this program. If not, + * see . + */ + +// Class header +#include "replica/ingest/IngestUtils.h" + +// Qserv headers +#include "http/RequestBodyJSON.h" +#include "replica/util/Csv.h" + +using namespace std; + +namespace lsst::qserv::replica { + +csv::DialectInput parseDialectInput(http::RequestBodyJSON const& body) { + csv::DialectInput dialectInput; + + // Allow an empty string in the input. Simply replace the one (if present) with + // the corresponding default value of the parameter. + auto const getDialectParam = [&](string const& param, string const& defaultValue) -> string { + string val = body.optional(param, defaultValue); + if (val.empty()) val = defaultValue; + return val; + }; + dialectInput.fieldsTerminatedBy = + getDialectParam("fields_terminated_by", csv::Dialect::defaultFieldsTerminatedBy); + dialectInput.fieldsEnclosedBy = + getDialectParam("fields_enclosed_by", csv::Dialect::defaultFieldsEnclosedBy); + dialectInput.fieldsEscapedBy = getDialectParam("fields_escaped_by", csv::Dialect::defaultFieldsEscapedBy); + dialectInput.linesTerminatedBy = + getDialectParam("lines_terminated_by", csv::Dialect::defaultLinesTerminatedBy); + + return dialectInput; +} + +} // namespace lsst::qserv::replica \ No newline at end of file diff --git a/src/replica/ingest/IngestUtils.h b/src/replica/ingest/IngestUtils.h new file mode 100644 index 000000000..3db98ec1f --- /dev/null +++ b/src/replica/ingest/IngestUtils.h @@ -0,0 +1,55 @@ +/* + * LSST Data Management System + * + * This product includes software developed by the + * LSST Project (http://www.lsst.org/). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the LSST License Statement and + * the GNU General Public License along with this program. If not, + * see . + */ +#ifndef LSST_QSERV_REPLICA_INGESTUTILS_H +#define LSST_QSERV_REPLICA_INGESTUTILS_H + +// System headers +#include + +// Third party headers +#include "nlohmann/json.hpp" + +// Qserv headers +#include "http/RequestBodyJSON.h" + +// Forward declarations + +namespace lsst::qserv::http { +class RequestBodyJSON; +} // namespace lsst::qserv::http + +namespace lsst::qserv::replica::csv { +class DialectInput; +} // namespace lsst::qserv::replica::csv + +// This header declarations +namespace lsst::qserv::replica { + +/** + * Parse the dialect input from the request body. + * @param body The request body. + * @return The parsed dialect input. + */ +csv::DialectInput parseDialectInput(http::RequestBodyJSON const& body); + +} // namespace lsst::qserv::replica + +#endif // LSST_QSERV_REPLICA_INGESTUTILS_H