Skip to content

Commit

Permalink
A utility for parsing CSV dialect parameters of the ingest requests
Browse files Browse the repository at this point in the history
  • Loading branch information
iagaponenko committed Oct 4, 2024
1 parent 21f2835 commit 24a35fd
Show file tree
Hide file tree
Showing 4 changed files with 112 additions and 17 deletions.
1 change: 1 addition & 0 deletions src/replica/ingest/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ target_sources(replica_ingest PRIVATE
IngestResourceMgrT.cc
IngestSvc.cc
IngestSvcConn.cc
IngestUtils.cc
TransactionContrib.cc
)
target_link_libraries(replica_ingest PUBLIC
Expand Down
19 changes: 2 additions & 17 deletions src/replica/ingest/IngestHttpSvcMod.cc
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include "http/Method.h"
#include "replica/ingest/IngestRequest.h"
#include "replica/ingest/IngestRequestMgr.h"
#include "replica/ingest/IngestUtils.h"
#include "replica/services/ServiceProvider.h"
#include "replica/util/Csv.h"

Expand Down Expand Up @@ -183,23 +184,7 @@ shared_ptr<IngestRequest> IngestHttpSvcMod::_createRequest(bool async) const {
string const url = body().required<string>("url");
string const charsetName =
body().optional<string>("charset_name", config->get<string>("worker", "ingest-charset-name"));

csv::DialectInput dialectInput;
// Allow an empty string in the input. Simply replace the one (if present) with
// the corresponding default value of the parameter.
auto const getDialectParam = [&](string const& param, string const& defaultValue) -> string {
string val = body().optional<string>(param, defaultValue);
if (val.empty()) val = defaultValue;
return val;
};
dialectInput.fieldsTerminatedBy =
getDialectParam("fields_terminated_by", csv::Dialect::defaultFieldsTerminatedBy);
dialectInput.fieldsEnclosedBy =
getDialectParam("fields_enclosed_by", csv::Dialect::defaultFieldsEnclosedBy);
dialectInput.fieldsEscapedBy = getDialectParam("fields_escaped_by", csv::Dialect::defaultFieldsEscapedBy);
dialectInput.linesTerminatedBy =
getDialectParam("lines_terminated_by", csv::Dialect::defaultLinesTerminatedBy);

auto const dialectInput = parseDialectInput(body());
auto const httpMethod = http::string2method(body().optional<string>("http_method", "GET"));
string const httpData = body().optional<string>("http_data", string());
vector<string> const httpHeaders = body().optionalColl<string>("http_headers", vector<string>());
Expand Down
54 changes: 54 additions & 0 deletions src/replica/ingest/IngestUtils.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* LSST Data Management System
*
* This product includes software developed by the
* LSST Project (http://www.lsst.org/).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the LSST License Statement and
* the GNU General Public License along with this program. If not,
* see <http://www.lsstcorp.org/LegalNotices/>.
*/

// Class header
#include "replica/ingest/IngestUtils.h"

// Qserv headers
#include "http/RequestBodyJSON.h"
#include "replica/util/Csv.h"

using namespace std;

namespace lsst::qserv::replica {

csv::DialectInput parseDialectInput(http::RequestBodyJSON const& body) {
csv::DialectInput dialectInput;

// Allow an empty string in the input. Simply replace the one (if present) with
// the corresponding default value of the parameter.
auto const getDialectParam = [&](string const& param, string const& defaultValue) -> string {
string val = body.optional<string>(param, defaultValue);
if (val.empty()) val = defaultValue;
return val;
};
dialectInput.fieldsTerminatedBy =
getDialectParam("fields_terminated_by", csv::Dialect::defaultFieldsTerminatedBy);
dialectInput.fieldsEnclosedBy =
getDialectParam("fields_enclosed_by", csv::Dialect::defaultFieldsEnclosedBy);
dialectInput.fieldsEscapedBy = getDialectParam("fields_escaped_by", csv::Dialect::defaultFieldsEscapedBy);
dialectInput.linesTerminatedBy =
getDialectParam("lines_terminated_by", csv::Dialect::defaultLinesTerminatedBy);

return dialectInput;
}

} // namespace lsst::qserv::replica
55 changes: 55 additions & 0 deletions src/replica/ingest/IngestUtils.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* LSST Data Management System
*
* This product includes software developed by the
* LSST Project (http://www.lsst.org/).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the LSST License Statement and
* the GNU General Public License along with this program. If not,
* see <http://www.lsstcorp.org/LegalNotices/>.
*/
#ifndef LSST_QSERV_REPLICA_INGESTUTILS_H
#define LSST_QSERV_REPLICA_INGESTUTILS_H

// System headers
#include <string>

// Third party headers
#include "nlohmann/json.hpp"

// Qserv headers
#include "http/RequestBodyJSON.h"

// Forward declarations

namespace lsst::qserv::http {
class RequestBodyJSON;
} // namespace lsst::qserv::http

namespace lsst::qserv::replica::csv {
class DialectInput;
} // namespace lsst::qserv::replica::csv

// This header declarations
namespace lsst::qserv::replica {

/**
* Parse the dialect input from the request body.
* @param body The request body.
* @return The parsed dialect input.
*/
csv::DialectInput parseDialectInput(http::RequestBodyJSON const& body);

} // namespace lsst::qserv::replica

#endif // LSST_QSERV_REPLICA_INGESTUTILS_H

0 comments on commit 24a35fd

Please sign in to comment.