Extended Replication system's framework

The new design of the Qserv managememt control plain now relies upon
the HTTP protocol to communicate with workers.

src/replica/AddReplicaQservHttpMgtRequest.cc

@@ -0,0 +1,75 @@
+/*
+ * LSST Data Management System
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program.  If not,
+ * see <http://www.lsstcorp.org/LegalNotices/>.
+ */
+
+// Class header
+#include "replica/AddReplicaQservHttpMgtRequest.h"
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+using namespace nlohmann;
+using namespace std;
+
+namespace {
+
+LOG_LOGGER _log = LOG_GET("lsst.qserv.replica.AddReplicaQservHttpMgtRequest");
+
+}  // namespace
+
+namespace lsst::qserv::replica {
+
+AddReplicaQservHttpMgtRequest::Ptr AddReplicaQservHttpMgtRequest::create(
+        shared_ptr<ServiceProvider> const& serviceProvider, string const& worker, unsigned int chunk,
+        vector<string> const& databases, AddReplicaQservHttpMgtRequest::CallbackType const& onFinish) {
+    return AddReplicaQservHttpMgtRequest::Ptr(
+            new AddReplicaQservHttpMgtRequest(serviceProvider, worker, chunk, databases, onFinish));
+}
+
+AddReplicaQservHttpMgtRequest::AddReplicaQservHttpMgtRequest(
+        shared_ptr<ServiceProvider> const& serviceProvider, string const& worker, unsigned int chunk,
+        vector<string> const& databases, AddReplicaQservHttpMgtRequest::CallbackType const& onFinish)
+        : QservHttpMgtRequest(serviceProvider, "QSERV_ADD_REPLICA", worker),
+          _chunk(chunk),
+          _databases(databases),
+          _onFinish(onFinish) {}
+
+list<pair<string, string>> AddReplicaQservHttpMgtRequest::extendedPersistentState() const {
+    list<pair<string, string>> result;
+    result.emplace_back("chunk", to_string(chunk()));
+    for (auto&& database : databases()) {
+        result.emplace_back("database", database);
+    }
+    return result;
+}
+
+void AddReplicaQservHttpMgtRequest::createHttpReqImpl(replica::Lock const& lock) {
+    string const method = "POST";
+    string const target = "/replica";
+    json const data = json::object({{"chunk", _chunk}, {"databases", _databases}});
+    createHttpReq(lock, method, target, data);
+}
+
+void AddReplicaQservHttpMgtRequest::notify(replica::Lock const& lock) {
+    LOGS(_log, LOG_LVL_TRACE, context() << __func__);
+    notifyDefaultImpl<AddReplicaQservHttpMgtRequest>(lock, _onFinish);
+}
+
+}  // namespace lsst::qserv::replica

src/replica/AddReplicaQservHttpMgtRequest.h

@@ -0,0 +1,110 @@
+/*
+ * LSST Data Management System
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program.  If not,
+ * see <http://www.lsstcorp.org/LegalNotices/>.
+ */
+#ifndef LSST_QSERV_REPLICA_ADDREPLICAQSERVHTTPMGTREQUEST_H
+#define LSST_QSERV_REPLICA_ADDREPLICAQSERVHTTPMGTREQUEST_H
+
+// System headers
+#include <list>
+#include <memory>
+#include <string>
+#include <vector>
+#include <utility>
+
+// Third party headers
+#include "nlohmann/json.hpp"
+
+// Qserv headers
+#include "replica/QservHttpMgtRequest.h"
+
+namespace lsst::qserv::replica {
+class ServiceProvider;
+}  // namespace lsst::qserv::replica
+
+// This header declarations
+
+namespace lsst::qserv::replica {
+
+/**
+ * Class AddReplicaQservHttpMgtRequest implements a request notifying Qserv workers
+ * on new chunks added to the database.
+ */
+class AddReplicaQservHttpMgtRequest : public QservHttpMgtRequest {
+public:
+    typedef std::shared_ptr<AddReplicaQservHttpMgtRequest> Ptr;
+
+    /// The function type for notifications on the completion of the request
+    typedef std::function<void(Ptr)> CallbackType;
+
+    AddReplicaQservHttpMgtRequest() = delete;
+    AddReplicaQservHttpMgtRequest(AddReplicaQservHttpMgtRequest const&) = delete;
+    AddReplicaQservHttpMgtRequest& operator=(AddReplicaQservHttpMgtRequest const&) = delete;
+
+    virtual ~AddReplicaQservHttpMgtRequest() final = default;
+
+    /**
+     * Static factory method is needed to prevent issues with the lifespan
+     * and memory management of instances created otherwise (as values or via
+     * low-level pointers).
+     *
+     * @param serviceProvider A reference to a provider of services for accessing
+     *   Configuration, saving the request's persistent state to the database.
+     * @param worker The name of a worker to send the request to.
+     * @param chunk The chunk number.
+     * @param databases The names of databases.
+     * @param onFinish (optional) callback function to be called upon request completion.
+     * @return A pointer to the created object.
+     */
+    static Ptr create(std::shared_ptr<ServiceProvider> const& serviceProvider, std::string const& worker,
+                      unsigned int chunk, std::vector<std::string> const& databases,
+                      CallbackType const& onFinish = nullptr);
+
+    /// @return the chunk number
+    unsigned int chunk() const { return _chunk; }
+
+    /// @return names of databases
+    std::vector<std::string> const& databases() const { return _databases; }
+
+    /// @see QservHttpMgtRequest::extendedPersistentState()
+    virtual std::list<std::pair<std::string, std::string>> extendedPersistentState() const final;
+
+protected:
+    /// @see QservHttpMgtRequest::createHttpReqImpl
+    virtual void createHttpReqImpl(replica::Lock const& lock) final;
+
+    /// @see QservHttpMgtRequest::notify
+    virtual void notify(replica::Lock const& lock) final;
+
+private:
+    /// @see AddReplicaQservHttpMgtRequest::create()
+    AddReplicaQservHttpMgtRequest(std::shared_ptr<ServiceProvider> const& serviceProvider,
+                                  std::string const& worker, unsigned int chunk,
+                                  std::vector<std::string> const& databases, CallbackType const& onFinish);
+
+    // Input parameters
+
+    unsigned int const _chunk;
+    std::vector<std::string> const _databases;
+    CallbackType _onFinish;  ///< The callback is reset when the request finishes.
+};
+
+}  // namespace lsst::qserv::replica
+
+#endif  // LSST_QSERV_REPLICA_ADDREPLICAQSERVHTTPMGTREQUEST_H

src/replica/CMakeLists.txt

@@ -11,6 +11,7 @@ target_sources(replica PRIVATE
     ${REPLICA_PB_HDRS}
     AbortTransactionApp.cc
     AbortTransactionJob.cc
+    AddReplicaQservHttpMgtRequest.cc
     AddReplicaQservMgtRequest.cc
     AdminApp.cc
     Application.cc
@@ -73,9 +74,13 @@ target_sources(replica PRIVATE
     FindRequest.cc
     FixUpApp.cc
     FixUpJob.cc
+    GetReplicasQservHttpMgtRequest.cc
     GetReplicasQservMgtRequest.cc
+    GetDbStatusQservHttpMgtRequest.cc
     GetDbStatusQservMgtRequest.cc
+    GetConfigQservHttpMgtRequest.cc
     GetConfigQservMgtRequest.cc
+    GetStatusQservHttpMgtRequest.cc
     GetStatusQservMgtRequest.cc
     HealthMonitorTask.cc
     HttpAsyncReqApp.cc
@@ -130,6 +135,7 @@ target_sources(replica PRIVATE
     PurgeJob.cc
     QhttpTestApp.cc
     QservGetReplicasJob.cc
+    QservHttpMgtRequest.cc
     QservMgtRequest.cc
     QservMgtServices.cc
     QservStatusJob.cc
@@ -143,6 +149,7 @@ target_sources(replica PRIVATE
     RegistryHttpSvc.cc
     RegistryHttpSvcMod.cc
     RegistryWorkers.cc
+    RemoveReplicaQservHttpMgtRequest.cc
     RemoveReplicaQservMgtRequest.cc
     ReplicaInfo.cc
     ReplicateApp.cc
@@ -156,6 +163,7 @@ target_sources(replica PRIVATE
     ServiceManagementRequest.cc
     ServiceManagementRequestBase.cc
     ServiceProvider.cc
+    SetReplicasQservHttpMgtRequest.cc
     SetReplicasQservMgtRequest.cc
     SqlAlterTablesJob.cc
     SqlAlterTablesRequest.cc
@@ -202,6 +210,7 @@ target_sources(replica PRIVATE
     SuccessRateGenerator.cc
     SyncApp.cc
     Task.cc
+    TestEchoQservHttpMgtRequest.cc
     TestEchoQservMgtRequest.cc
     TransactionContrib.cc
     TransactionsApp.cc

src/replica/GetConfigQservHttpMgtRequest.cc

@@ -0,0 +1,60 @@
+/*
+ * LSST Data Management System
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program.  If not,
+ * see <http://www.lsstcorp.org/LegalNotices/>.
+ */
+
+// Class header
+#include "replica/GetConfigQservHttpMgtRequest.h"
+
+// LSST headers
+#include "lsst/log/Log.h"
+
+using namespace std;
+
+namespace {
+
+LOG_LOGGER _log = LOG_GET("lsst.qserv.replica.GetConfigQservHttpMgtRequest");
+
+}  // namespace
+
+namespace lsst::qserv::replica {
+
+shared_ptr<GetConfigQservHttpMgtRequest> GetConfigQservHttpMgtRequest::create(
+        shared_ptr<ServiceProvider> const& serviceProvider, string const& worker,
+        GetConfigQservHttpMgtRequest::CallbackType const& onFinish) {
+    return shared_ptr<GetConfigQservHttpMgtRequest>(
+            new GetConfigQservHttpMgtRequest(serviceProvider, worker, onFinish));
+}
+
+GetConfigQservHttpMgtRequest::GetConfigQservHttpMgtRequest(
+        shared_ptr<ServiceProvider> const& serviceProvider, string const& worker,
+        GetConfigQservHttpMgtRequest::CallbackType const& onFinish)
+        : QservHttpMgtRequest(serviceProvider, "QSERV_GET_DATABASE_STATUS", worker), _onFinish(onFinish) {}
+
+void GetConfigQservHttpMgtRequest::createHttpReqImpl(replica::Lock const& lock) {
+    string const service = "/config";
+    createHttpReq(lock, service);
+}
+
+void GetConfigQservHttpMgtRequest::notify(replica::Lock const& lock) {
+    LOGS(_log, LOG_LVL_TRACE, context() << __func__);
+    notifyDefaultImpl<GetConfigQservHttpMgtRequest>(lock, _onFinish);
+}
+
+}  // namespace lsst::qserv::replica

src/replica/GetConfigQservHttpMgtRequest.h

@@ -0,0 +1,86 @@
+/*
+ * LSST Data Management System
+ *
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the LSST License Statement and
+ * the GNU General Public License along with this program.  If not,
+ * see <http://www.lsstcorp.org/LegalNotices/>.
+ */
+#ifndef LSST_QSERV_REPLICA_GETCONFIGQSERVHTTPMGTREQUEST_H
+#define LSST_QSERV_REPLICA_GETCONFIGQSERVHTTPMGTREQUEST_H
+
+// System headers
+#include <memory>
+#include <string>
+
+// Qserv headers
+#include "replica/QservHttpMgtRequest.h"
+
+namespace lsst::qserv::replica {
+class ServiceProvider;
+}  // namespace lsst::qserv::replica
+
+// This header declarations
+namespace lsst::qserv::replica {
+
+/**
+ * Class GetConfigQservHttpMgtRequest is a request for obtaining various info
+ * on the database service of the Qserv worker.
+ */
+class GetConfigQservHttpMgtRequest : public QservHttpMgtRequest {
+public:
+    /// The function type for notifications on the completion of the request
+    typedef std::function<void(std::shared_ptr<GetConfigQservHttpMgtRequest> const&)> CallbackType;
+
+    GetConfigQservHttpMgtRequest() = delete;
+    GetConfigQservHttpMgtRequest(GetConfigQservHttpMgtRequest const&) = delete;
+    GetConfigQservHttpMgtRequest& operator=(GetConfigQservHttpMgtRequest const&) = delete;
+
+    virtual ~GetConfigQservHttpMgtRequest() final = default;
+
+    /**
+     * Static factory method is needed to prevent issues with the lifespan
+     * and memory management of instances created otherwise (as values or via
+     * low-level pointers).
+     * @param serviceProvider A reference to a provider of services for accessing
+     *   Configuration, saving the request's persistent state to the database.
+     * @param worker The name of a worker to send the request to.
+     * @param onFinish (optional) callback function to be called upon request completion.
+     * @return A pointer to the created object.
+     */
+    static std::shared_ptr<GetConfigQservHttpMgtRequest> create(
+            std::shared_ptr<ServiceProvider> const& serviceProvider, std::string const& worker,
+            CallbackType const& onFinish = nullptr);
+
+protected:
+    /// @see QservHttpMgtRequest::createHttpReqImpl()
+    virtual void createHttpReqImpl(replica::Lock const& lock) final;
+
+    /// @see QservHttpMgtRequest::notify()
+    virtual void notify(replica::Lock const& lock) final;
+
+private:
+    /// @see GetConfigQservHttpMgtRequest::create()
+    GetConfigQservHttpMgtRequest(std::shared_ptr<ServiceProvider> const& serviceProvider,
+                                 std::string const& worker, CallbackType const& onFinish);
+
+    // Input parameters
+
+    CallbackType _onFinish;  ///< This callback is reset after finishing the request.
+};
+
+}  // namespace lsst::qserv::replica
+
+#endif  // LSST_QSERV_REPLICA_GETCONFIGQSERVHTTPMGTREQUEST_H