Remove DashboardBackendClient usage in heal command (#542)

rs/cli/src/clients.rs

@@ -1,9 +1,8 @@
 use async_trait::async_trait;
-use decentralization::HealResponse;
 use decentralization::SubnetChangeResponse;
 use ic_base_types::PrincipalId;
 use ic_management_types::{
-    requests::{HealRequest, MembershipReplaceRequest, NodesRemoveRequest, NodesRemoveResponse, SubnetCreateRequest, SubnetResizeRequest},
+    requests::{MembershipReplaceRequest, NodesRemoveRequest, NodesRemoveResponse, SubnetCreateRequest, SubnetResizeRequest},
     Artifact, Network, NetworkError, Release, TopologyChangeProposal,
 };
 use log::error;
@@ -98,14 +97,6 @@ impl DashboardBackendClient {
             .rest_send()
             .await
     }
-
-    pub(crate) async fn network_heal(&self, request: HealRequest) -> anyhow::Result<HealResponse> {
-        reqwest::Client::new()
-            .post(self.url.join("network/heal").map_err(|e| anyhow::anyhow!(e))?)
-            .json(&request)
-            .rest_send()
-            .await
-    }
 }
 
 #[async_trait]

rs/cli/src/runner.rs

@@ -4,17 +4,18 @@ use crate::operations::hostos_rollout::{HostosRollout, HostosRolloutResponse, No
 use crate::ops_subnet_node_replace;
 use crate::{ic_admin, local_unused_port};
 use actix_web::dev::ServerHandle;
-use decentralization::network::TopologyManager;
-use decentralization::network::{SubnetChange, SubnetQuerier, SubnetQueryBy};
+use decentralization::network::{AvailableNodesQuerier, SubnetChange, SubnetQuerier, SubnetQueryBy};
+use decentralization::network::{DecentralizedSubnet, NetworkHealRequest, NetworkHealSubnets, Node as DecentralizedNode, TopologyManager};
 use decentralization::SubnetChangeResponse;
 use futures::future::join_all;
 use ic_base_types::PrincipalId;
-use ic_management_backend::endpoints;
 use ic_management_backend::proposal::ProposalAgent;
 use ic_management_backend::public_dashboard::query_ic_dashboard_list;
 use ic_management_backend::registry::{self, RegistryState};
+use ic_management_backend::subnets::unhealthy_with_nodes;
+use ic_management_backend::{endpoints, health, health::HealthStatusQuerier};
 use ic_management_types::requests::NodesRemoveRequest;
-use ic_management_types::{Artifact, Network, Node, NodeFeature, NodeProvidersResponse, TopologyChangePayload};
+use ic_management_types::{Artifact, Network, NetworkError, Node, NodeFeature, NodeProvidersResponse, TopologyChangePayload};
 use itertools::Itertools;
 use log::{info, warn};
 use registry_canister::mutations::do_change_subnet_membership::ChangeSubnetMembershipPayload;
@@ -518,10 +519,28 @@ impl Runner {
         _verbose: bool,
         simulate: bool,
     ) -> Result<(), anyhow::Error> {
-        let change = self.get_backend_client().await?.network_heal(request).await?;
-        println!("{}", change);
+        let nodes_health = health::HealthClient::new(self.registry().await.network()).nodes().await?;
+        let subnets = self.registry().await.subnets();
+        let subnets_to_heal = unhealthy_with_nodes(&subnets, nodes_health)
+            .await
+            .iter()
+            .flat_map(|(id, unhealthy_nodes)| {
+                let unhealthy_nodes = unhealthy_nodes.iter().map(DecentralizedNode::from).collect::<Vec<_>>();
+                let unhealthy_subnet = subnets.get(id).ok_or(NetworkError::SubnetNotFound(*id))?;
+
+                Ok::<NetworkHealSubnets, NetworkError>(decentralization::network::NetworkHealSubnets {
+                    name: unhealthy_subnet.metadata.name.clone(),
+                    decentralized_subnet: DecentralizedSubnet::from(unhealthy_subnet),
+                    unhealthy_nodes,
+                })
+            })
+            .collect_vec();
+
+        let subnets_change_response: Vec<SubnetChangeResponse> = NetworkHealRequest::new(subnets_to_heal)
+            .heal_and_optimize(self.registry().await.available_nodes().await?, request.max_replaceable_nodes_per_sub)?;
 
-        let errors = join_all(change.subnets_change_response.iter().map(|subnet_change_response| async move {
+        subnets_change_response.iter().for_each(|change| println!("{}", change));
+        let errors = join_all(subnets_change_response.iter().map(|subnet_change_response| async move {
             self.run_membership_change(
                 subnet_change_response.clone(),
                 ops_subnet_node_replace::replace_proposal_options(subnet_change_response)?,
@@ -537,7 +556,6 @@ impl Runner {
         .into_iter()
         .filter_map(|f| f.err())
         .collect::<Vec<_>>();
-
         if !errors.is_empty() {
             anyhow::bail!("Errors: {:?}", errors);
         }

rs/decentralization/src/lib.rs

@@ -183,17 +183,3 @@ impl Display for SubnetChangeResponse {
         Ok(())
     }
 }
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct HealResponse {
-    pub subnets_change_response: Vec<SubnetChangeResponse>,
-}
-
-impl Display for HealResponse {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        for change in &self.subnets_change_response {
-            writeln!(f, "{}", change)?;
-        }
-        Ok(())
-    }
-}

rs/ic-management-backend/src/endpoints/mod.rs

@@ -1,5 +1,4 @@
 pub mod governance_canister;
-pub mod network;
 pub mod nodes_ops;
 pub mod query_decentralization;
 pub mod release;
@@ -87,7 +86,6 @@ pub async fn run_backend(
             .service(self::subnet::create_subnet)
             .service(self::subnet::resize)
             .service(self::subnet::change_preview)
-            .service(self::network::heal)
             .service(self::nodes_ops::remove)
             .service(self::query_decentralization::decentralization_subnet_query)
             .service(self::query_decentralization::decentralization_whatif_query)

rs/ic-management-backend/src/subnets.rs

@@ -4,6 +4,7 @@ use decentralization::{network::SubnetChange, SubnetChangeResponse};
 use ic_base_types::PrincipalId;
 use ic_management_types::{Node, Status, Subnet, TopologyChangeProposal};
 
+#[allow(dead_code)]
 pub async fn unhealthy_with_nodes(
     subnets: &BTreeMap<PrincipalId, Subnet>,
     nodes_health: BTreeMap<PrincipalId, Status>,

rs/ic-management-types/src/errors.rs

@@ -44,3 +44,15 @@ impl From<serde_json::Error> for NetworkError {
         NetworkError::DataRequestError(err.to_string())
     }
 }
+
+impl From<NetworkError> for anyhow::Error {
+    fn from(e: NetworkError) -> Self {
+        match e {
+            NetworkError::NodeNotFound(id) => anyhow::anyhow!("Node not found: {:?}", id),
+            NetworkError::SubnetNotFound(id) => anyhow::anyhow!("Subnet not found: {:?}", id),
+            NetworkError::ResizeFailed(msg) => anyhow::anyhow!("Resize Failed: {:?}", msg),
+            NetworkError::DataRequestError(msg) => anyhow::anyhow!("Data request error: {:?}", msg),
+            NetworkError::IllegalRequest(msg) => anyhow::anyhow!("Illegal request: {:?}", msg),
+        }
+    }
+}