Fix poll_retry and add max_num_poll_retry

README.md

@@ -216,6 +216,7 @@ You can also use environment variables with string interpolation in your configu
 
 * `poll_interval` (type: _integer_, allowed: seconds, default: `120`) — Interval for which to probe nodes in `poll` mode
 * `poll_retry` (type: _integer_, allowed: seconds, default: `2`) — Interval after which to try probe for a second time nodes in `poll` mode (only when the first check fails)
+* `max_num_poll_retry` (type: _integer_, allowed: any number, default: `3`) — Maximum number of times to retry to probe while previous checks have failed.
 * `poll_http_status_healthy_above` (type: _integer_, allowed: HTTP status code, default: `200`) — HTTP status above which `poll` checks to HTTP replicas reports as `healthy`
 * `poll_http_status_healthy_below` (type: _integer_, allowed: HTTP status code, default: `400`) — HTTP status under which `poll` checks to HTTP replicas reports as `healthy`
 * `poll_delay_dead` (type: _integer_, allowed: seconds, default: `10`) — Delay after which a node in `poll` mode is to be considered `dead` (ie. check response delay)

config.cfg

@@ -34,6 +34,7 @@ custom_html = ""
 
 poll_interval = 120
 poll_retry = 2
+max_num_poll_retry = 3
 
 poll_http_status_healthy_above = 200
 poll_http_status_healthy_below = 400

src/config/config.rs

@@ -69,6 +69,9 @@ pub struct ConfigMetrics {
     #[serde(default = "defaults::metrics_poll_retry")]
     pub poll_retry: u64,
 
+    #[serde(default = "defaults::metrics_max_num_poll_retry")]
+    pub max_num_poll_retry: u64,
+
     #[serde(default = "defaults::metrics_poll_http_status_healthy_above")]
     pub poll_http_status_healthy_above: u16,
 

src/config/defaults.rs

@@ -37,6 +37,10 @@ pub fn metrics_poll_retry() -> u64 {
     2
 }
 
+pub fn metrics_max_num_poll_retry() -> u64 {
+    3
+}
+
 pub fn metrics_poll_http_status_healthy_above() -> u16 {
     200
 }

src/prober/manager.rs

@@ -33,7 +33,6 @@ use crate::prober::manager::STORE as PROBER_STORE;
 use crate::prober::mode::Mode;
 use crate::APP_CONF;
 
-const PROBE_HOLD_MILLISECONDS: u64 = 500;
 const PROBE_ICMP_TIMEOUT_SECONDS: u64 = 1;
 
 lazy_static! {
@@ -190,13 +189,13 @@ fn proceed_replica_probe_poll_with_retry(
 ) -> (Status, Option<Duration>) {
     let (mut status, mut latency, mut retry_count) = (Status::Dead, None, 0);
 
-    while retry_count <= APP_CONF.metrics.poll_retry && status == Status::Dead {
+    while retry_count <= APP_CONF.metrics.max_num_poll_retry && status == Status::Dead {
         debug!(
             "will probe replica: {:?} with retry count: {}",
             replica_url, retry_count
         );
 
-        thread::sleep(Duration::from_millis(PROBE_HOLD_MILLISECONDS));
+        thread::sleep(Duration::from_secs(APP_CONF.metrics.poll_retry));
 
         let probe_results = proceed_replica_probe_poll(
             replica_url,