From 4eae590e99ef7e5a8cb232a5e91fc971801a8864 Mon Sep 17 00:00:00 2001 From: Tim Bruijnzeels Date: Thu, 9 Sep 2021 16:32:06 +0200 Subject: [PATCH] Keep child state (active/pending) in status and show in CLI and metrics. (#656) --- defaults/krill.conf | 1 + src/commons/api/ca.rs | 59 +++++++++++++++++------ src/daemon/ca/manager.rs | 15 +++++- src/daemon/ca/status.rs | 12 ++++- src/daemon/http/server.rs | 21 +++++++- test-resources/krill-init-multi-user.conf | 1 + test-resources/krill-init.conf | 1 + 7 files changed, 92 insertions(+), 18 deletions(-) diff --git a/defaults/krill.conf b/defaults/krill.conf index bb63bf84..af541d4a 100644 --- a/defaults/krill.conf +++ b/defaults/krill.conf @@ -160,6 +160,7 @@ # # krill_cas_children{ca="ca"} number of children for CA # krill_ca_child_success{ca="ca", child="child"} status of last child to CA connection (0=issue, 1=success) +# krill_ca_child_state{ca="ca", child="child"} child state (see 'suspend_child_after_inactive_hours' config) (0=suspended, 1=active) # krill_ca_child_last_connection{ca="ca", child="child"} unix timestamp in seconds of last child to CA connection # krill_ca_child_last_success{ca="ca", child="child"} unix timestamp in seconds of last successful child to CA connection # krill_ca_child_agent_total{ca="ca", user_agent="ua string"} total children per user agent based on their last connection diff --git a/src/commons/api/ca.rs b/src/commons/api/ca.rs index 1cf24dcd..0543fa7f 100644 --- a/src/commons/api/ca.rs +++ b/src/commons/api/ca.rs @@ -1637,10 +1637,10 @@ impl ChildrenConnectionStats { ChildrenConnectionStats { children } } - pub fn inactive_children(&self, threshold_hours: i64) -> Vec { + pub fn suspension_candidates(&self, threshold_hours: i64) -> Vec { self.children .iter() - .filter(|child| child.inactive(threshold_hours)) + .filter(|child| child.suspension_candidate(threshold_hours)) .map(|child| child.handle.clone()) .collect() } @@ -1649,22 +1649,23 @@ impl ChildrenConnectionStats { impl fmt::Display for ChildrenConnectionStats { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { if !self.children.is_empty() { - writeln!(f, "handle, user_agent, last_exchange, result")?; + writeln!(f, "handle,user_agent,last_exchange,result,state")?; for child in &self.children { match &child.last_exchange { None => { - writeln!(f, "{},n/a,never,n/a", child.handle)?; + writeln!(f, "{},n/a,never,n/a,{}", child.handle, child.state)?; } Some(exchange) => { let agent = exchange.user_agent.as_deref().unwrap_or(""); writeln!( f, - "{},{},{},{}", + "{},{},{},{},{}", child.handle, agent, exchange.timestamp.to_rfc3339(), - exchange.result + exchange.result, + child.state )?; } } @@ -1678,19 +1679,29 @@ impl fmt::Display for ChildrenConnectionStats { pub struct ChildConnectionStats { handle: ChildHandle, last_exchange: Option, + state: ChildState, } impl ChildConnectionStats { - pub fn new(handle: ChildHandle, last_exchange: Option) -> Self { - ChildConnectionStats { handle, last_exchange } + pub fn new(handle: ChildHandle, last_exchange: Option, state: ChildState) -> Self { + ChildConnectionStats { + handle, + last_exchange, + state, + } } - /// The child is considered 'inactive' if there was at least one exchange, and the - /// last exchange is longer ago than the specified threshold hours. - pub fn inactive(&self, threshold_hours: i64) -> bool { - match &self.last_exchange { - None => false, // if there has been no exchange at all, the child is not yet active, rather than inactive - Some(exchange) => exchange.timestamp < (Timestamp::now_minus_hours(threshold_hours)), + /// The child is considered a candidate for suspension if there was at least one exchange, + /// and the last exchange is longer ago than the specified threshold hours, and the child + /// is not already suspended. + pub fn suspension_candidate(&self, threshold_hours: i64) -> bool { + if self.state == ChildState::Suspended { + false + } else { + match &self.last_exchange { + None => false, // if there has been no exchange at all, the child is not yet active, rather than inactive + Some(exchange) => exchange.timestamp < (Timestamp::now_minus_hours(threshold_hours)), + } } } } @@ -1701,6 +1712,7 @@ impl ChildConnectionStats { pub struct ChildStatus { last_exchange: Option, last_success: Option, + suspended: Option, } impl ChildStatus { @@ -1712,6 +1724,7 @@ impl ChildStatus { user_agent, }); self.last_success = Some(timestamp); + self.suspended = None; } pub fn set_failure(&mut self, user_agent: Option, error_response: ErrorResponse) { @@ -1720,6 +1733,11 @@ impl ChildStatus { result: ExchangeResult::Failure(error_response), user_agent, }); + self.suspended = None; + } + + pub fn set_suspended(&mut self) { + self.suspended = Some(Timestamp::now()) } pub fn last_exchange(&self) -> Option<&ChildExchange> { @@ -1729,6 +1747,18 @@ impl ChildStatus { pub fn last_success(&self) -> Option { self.last_success } + + pub fn suspended(&self) -> Option { + self.suspended + } + + pub fn child_state(&self) -> ChildState { + if self.suspended.is_none() { + ChildState::Active + } else { + ChildState::Suspended + } + } } impl Default for ChildStatus { @@ -1736,6 +1766,7 @@ impl Default for ChildStatus { ChildStatus { last_exchange: None, last_success: None, + suspended: None, } } } diff --git a/src/daemon/ca/manager.rs b/src/daemon/ca/manager.rs index 132fb2a2..925090bd 100644 --- a/src/daemon/ca/manager.rs +++ b/src/daemon/ca/manager.rs @@ -765,11 +765,24 @@ impl CaManager { if let Some(threshold_hours) = self.config.suspend_child_after_inactive_hours { if let Ok(ca_status) = self.get_ca_status(&ca_handle).await { let connections = ca_status.get_children_connection_stats(); - for child in connections.inactive_children(threshold_hours) { + for child in connections.suspension_candidates(threshold_hours) { info!( "Child '{}' under CA '{}' was inactive for more than {} hours. Will suspend it.", child, ca_handle, threshold_hours ); + if let Err(e) = self + .status_store + .lock() + .await + .set_child_suspended(&ca_handle, &child) + .await + { + error!( + "Could not update status to 'suspended' for inactive child, error: {}", + e + ); + } + let req = UpdateChildRequest::suspend(); if let Err(e) = self.ca_child_update(&ca_handle, child, req, actor).await { error!("Could not suspend inactive child, error: {}", e); diff --git a/src/daemon/ca/status.rs b/src/daemon/ca/status.rs index ac246942..e0e8d905 100644 --- a/src/daemon/ca/status.rs +++ b/src/daemon/ca/status.rs @@ -30,7 +30,10 @@ impl CaStatus { .children .clone() .into_iter() - .map(|(handle, status)| ChildConnectionStats::new(handle, status.into())) + .map(|(handle, status)| { + let state = status.child_state(); + ChildConnectionStats::new(handle, status.into(), state) + }) .collect(); ChildrenConnectionStats::new(children) } @@ -167,6 +170,13 @@ impl StatusStore { .await } + /// Marks a child as suspended. Note that it will be implicitly unsuspended whenever a new success or + /// or failure is recorded for the child. + pub async fn set_child_suspended(&self, ca: &Handle, child: &ChildHandle) -> KrillResult<()> { + self.update_ca_child_status(ca, child, |status| status.set_suspended()) + .await + } + pub async fn remove_child(&self, ca: &Handle, child: &ChildHandle) -> KrillResult<()> { self.update_ca_status(ca, |status| { status.children.remove(child); diff --git a/src/daemon/http/server.rs b/src/daemon/http/server.rs index 9a440bf2..c3e9fd84 100644 --- a/src/daemon/http/server.rs +++ b/src/daemon/http/server.rs @@ -536,6 +536,7 @@ pub async fn metrics(req: Request) -> RoutingResult { // krill_cas_children{ca="parent"} 11 // nr of children // krill_ca_child_success{ca="parent", child="child"} 1 + // krill_ca_child_state{ca="parent", child="child"} 1 // krill_ca_child_last_connection{ca="parent", child="child"} 1630921599 // krill_ca_child_last_success{ca="parent", child="child"} 1630921599 // krill_ca_child_agent_total{ca="parent", ua="krill/0.9.2"} 11 @@ -572,8 +573,24 @@ pub async fn metrics(req: Request) -> RoutingResult { res.push('\n'); res.push_str( - "# HELP krill_ca_child_last_connection unix timestamp in seconds of last child to CA connection\n", - ); + "# HELP krill_ca_child_state child state (see 'suspend_child_after_inactive_hours' config) (0=suspended, 1=active)\n", + ); + res.push_str("# TYPE krill_ca_child_state gauge\n"); + for (ca, status) in ca_status_map.iter() { + // skip the ones for which we have no status yet, i.e it was really only just added + // and no attempt to connect has yet been made. + for (child, status) in status.children().iter() { + let value = if status.suspended().is_none() { 0 } else { 1 }; + + res.push_str(&format!( + "krill_ca_child_state{{ca=\"{}\", child=\"{}\"}} {}\n", + ca, child, value + )); + } + } + + res.push('\n'); + res.push_str("# HELP krill_ca_child_last_connection unix timestamp in seconds of last child to CA connection\n"); res.push_str("# TYPE krill_ca_child_last_connection gauge\n"); for (ca, status) in ca_status_map.iter() { // skip the ones for which we have no status yet, i.e it was really only just added diff --git a/test-resources/krill-init-multi-user.conf b/test-resources/krill-init-multi-user.conf index 9baeeeab..2712da53 100644 --- a/test-resources/krill-init-multi-user.conf +++ b/test-resources/krill-init-multi-user.conf @@ -160,6 +160,7 @@ service_uri = "https://localhost:3001/" # # krill_cas_children{ca="ca"} number of children for CA # krill_ca_child_success{ca="ca", child="child"} status of last child to CA connection (0=issue, 1=success) +# krill_ca_child_state{ca="ca", child="child"} child state (see 'suspend_child_after_inactive_hours' config) (0=suspended, 1=active) # krill_ca_child_last_connection{ca="ca", child="child"} unix timestamp in seconds of last child to CA connection # krill_ca_child_last_success{ca="ca", child="child"} unix timestamp in seconds of last successful child to CA connection # krill_ca_child_agent_total{ca="ca", user_agent="ua string"} total children per user agent based on their last connection diff --git a/test-resources/krill-init.conf b/test-resources/krill-init.conf index 17a59a5a..e10aba0a 100644 --- a/test-resources/krill-init.conf +++ b/test-resources/krill-init.conf @@ -160,6 +160,7 @@ service_uri = "https://localhost:3001/" # # krill_cas_children{ca="ca"} number of children for CA # krill_ca_child_success{ca="ca", child="child"} status of last child to CA connection (0=issue, 1=success) +# krill_ca_child_state{ca="ca", child="child"} child state (see 'suspend_child_after_inactive_hours' config) (0=suspended, 1=active) # krill_ca_child_last_connection{ca="ca", child="child"} unix timestamp in seconds of last child to CA connection # krill_ca_child_last_success{ca="ca", child="child"} unix timestamp in seconds of last successful child to CA connection # krill_ca_child_agent_total{ca="ca", user_agent="ua string"} total children per user agent based on their last connection