Automate (re)-publication. Closes #33.

This commit is contained in:
Tim Bruijnzeels
2019-06-05 17:15:27 +02:00
parent 448c7357f0
commit 16ce1db2ec
20 changed files with 188 additions and 91 deletions
+4
View File
@@ -14,6 +14,10 @@ rpki = "^0.4"
serde = { version = "^1.0", features = ["derive" ] }
serde_json = "^1.0"
# Logging
fern = "^0.5"
log = "^0.4"
[dependencies.krill_commons]
path = "../commons"
version = "0.2.0"
+17 -9
View File
@@ -96,16 +96,24 @@ impl<S: CaSigner> CaServer<S> {
}
}
pub fn publish_ta(&self) -> CaResult<(), S> {
let handle = ta_handle();
let ta = self.ta_store.get_latest(&handle)?;
let ta_publish_cmd = TrustAnchorCommandDetails::publish(
&handle,
self.signer.clone()
);
pub fn republish_all(&self) -> CaResult<(), S> {
self.publish_ta()
}
let events = ta.process_command(ta_publish_cmd)?;
self.ta_store.update(&handle, ta, events)?;
pub fn publish_ta(&self) -> CaResult<(), S> {
// if there is a TA, publish it
let ta_handle = ta_handle();
if let Ok(ta) = self.ta_store.get_latest(&ta_handle) {
let ta_republish = TrustAnchorCommandDetails::republish(
&ta_handle,
self.signer.clone()
);
let events = ta.process_command(ta_republish)?;
if ! events.is_empty() {
self.ta_store.update(&ta_handle, ta, events)?;
}
}
Ok(())
}
+1
View File
@@ -3,6 +3,7 @@ extern crate bytes;
extern crate core;
#[macro_use] extern crate derive_more;
extern crate hex;
#[macro_use] extern crate log;
extern crate rand;
#[macro_use] extern crate serde;
extern crate serde_json;
+32 -28
View File
@@ -1,6 +1,6 @@
//! Supports publishing signed objects.
use std::io;
use std::{io, thread};
use std::path::PathBuf;
use krill_commons::api::admin::{
@@ -77,7 +77,7 @@ impl CommandDetails for PubClientCommandDetails {
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct PubClient {
id: Handle,
handle: Handle,
version: u64,
server: PubServerInfo
}
@@ -89,12 +89,11 @@ impl Aggregate for PubClient {
type Error = Error;
fn init(event: Self::InitEvent) -> Result<Self, Self::Error> {
let (id, _version, details) = event.unwrap();
let id = Handle::from(id);
let (handle, _version, details) = event.unwrap();
let version = 1;
let server = details.0;
Ok (PubClient { id, version, server })
Ok (PubClient { handle, version, server })
}
fn version(&self) -> u64 {
@@ -105,7 +104,10 @@ impl Aggregate for PubClient {
unimplemented!() // no events to process, yet
}
fn process_command(&self, _command: Self::Command) -> Result<Vec<Self::Event>, Self::Error> {
fn process_command(
&self,
_command: Self::Command
) -> Result<Vec<Self::Event>, Self::Error> {
unimplemented!() // no commands to process, yet
}
}
@@ -139,30 +141,32 @@ impl PubClients {
match client.server_info() {
PubServerInfo::KrillServer(service_uri, token) => {
let service_uri = service_uri.as_str();
let uri = format!("{}publication/{}", service_uri, handle);
let service_uri = service_uri.clone();
let token = token.clone();
// Note, I could not think of a convenient way to pass down the test
// context, since there are different threads involved when testing.
// So, for now, just setting test mode whenever the publication is done
// at localhost.
if service_uri.starts_with("https://localhost") {
httpclient::TEST_MODE.with(|m| { *m.borrow_mut() = true; });
}
thread::spawn(move ||{
// Note, I could not think of a convenient way to pass down
// the test context, since there are different threads
// involved when testing. So, for now, just setting test
// mode whenever the publication is done at localhost.
if service_uri.as_str().starts_with("https://localhost") {
httpclient::TEST_MODE.with(|m| { *m.borrow_mut() = true; });
}
match httpclient::post_json(&uri, delta, Some(&token)) {
Err(httpclient::Error::ErrorWithJson(_code, err)) => {
if err.code() == 2007 || err.code() == 2008 {
// TODO, do full sync!
unimplemented!()
} else {
error!("{}", err)
}
},
Err(e) => error!("{}", e),
Ok(()) => {}
}
});
let uri = format!("{}publication/{}", service_uri, handle.as_str());
match httpclient::post_json(&uri, delta, Some(token)) {
Err(httpclient::Error::ErrorWithJson(_code, err)) => {
if err.code() == 2007 || err.code() == 2008 {
// TODO, do full sync!
unimplemented!()
} else {
panic!("{}", err)
}
},
Err(e) => panic!("{}", e),
Ok(()) => {}
}
}
}
}
+1 -1
View File
@@ -72,7 +72,7 @@ impl CaSignSupport {
// TODO for now only publish MFT and CRL, revoking old MFTs only
if let Some(mft) = old_mft {
let revocation = Revocation::from(mft);
revocations.add(revocation.clone());
revocations.add(revocation);
revocations_delta.add(revocation);
}
+16 -13
View File
@@ -119,7 +119,7 @@ impl TrustAnchorInitDetails {
cert.set_v4_resources(Some(resources.v4().deref().clone()));
cert.set_v6_resources(Some(resources.v6().deref().clone()));
cert.into_cert(signer.as_ref(), key).map_err(|e| Error::signer_error(e))
cert.into_cert(signer.as_ref(), key).map_err(Error::signer_error)
}
}
@@ -163,7 +163,7 @@ pub type TrustAnchorCommand<S> = SentCommand<TrustAnchorCommandDetails<S>>;
#[derive(Clone, Debug)]
pub enum TrustAnchorCommandDetails<S: CaSigner> {
Publish(Arc<S>)
Republish(Arc<S>)
}
impl<S: CaSigner> CommandDetails for TrustAnchorCommandDetails<S> {
@@ -171,11 +171,11 @@ impl<S: CaSigner> CommandDetails for TrustAnchorCommandDetails<S> {
}
impl<S: CaSigner> TrustAnchorCommandDetails<S> {
pub fn publish(handle: &Handle, signer: Arc<S>) -> TrustAnchorCommand<S> {
pub fn republish(handle: &Handle, signer: Arc<S>) -> TrustAnchorCommand<S> {
SentCommand::new(
&handle,
None,
TrustAnchorCommandDetails::Publish(signer)
TrustAnchorCommandDetails::Republish(signer)
)
}
}
@@ -190,7 +190,7 @@ type TaResult<R> = Result<R, Error>;
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct TrustAnchor<S: CaSigner> {
id: Handle,
handle: Handle,
version: u64,
repo_info: RepoInfo,
@@ -233,15 +233,19 @@ impl<S: CaSigner> TrustAnchor<S> {
impl<S: CaSigner> TrustAnchor<S> {
fn publish(&self, signer: Arc<S>) -> TaResult<Vec<TrustAnchorEvent>> {
fn republish(&self, signer: Arc<S>) -> TaResult<Vec<TrustAnchorEvent>> {
if !self.current_key.needs_publication() {
return Ok(vec![])
}
let delta = CaSignSupport::publish(
signer,
&self.current_key,
self.repo_info(),
""
).map_err(|e| Error::signer_error(e))?;
).map_err(Error::signer_error)?;
Ok(vec![TrustAnchorEventDetails::published(&self.id, self.version, delta)])
Ok(vec![TrustAnchorEventDetails::published(&self.handle, self.version, delta)])
}
}
@@ -252,8 +256,7 @@ impl<S: CaSigner> Aggregate for TrustAnchor<S> {
type Error = Error;
fn init(event: Self::InitEvent) -> Result<Self, Self::Error> {
let (id, _version, init) = event.unwrap();
let id = Handle::from(id);
let (handle, _version, init) = event.unwrap();
let version = 1; // after applying init
let repo_info = init.repo_info;
@@ -262,7 +265,7 @@ impl<S: CaSigner> Aggregate for TrustAnchor<S> {
Ok(
TrustAnchor {
id,
handle,
version,
repo_info,
current_key,
@@ -287,7 +290,7 @@ impl<S: CaSigner> Aggregate for TrustAnchor<S> {
fn process_command(&self, command: Self::Command) -> TaResult<Vec<TrustAnchorEvent>> {
match command.into_details() {
TrustAnchorCommandDetails::Publish(signer) => self.publish(signer)
TrustAnchorCommandDetails::Republish(signer) => self.republish(signer)
}
}
}
@@ -369,7 +372,7 @@ mod tests {
store.add(init).unwrap();
let ta = store.get_latest(&handle).unwrap();
let publish_cmd = TrustAnchorCommandDetails::publish(&handle, signer);
let publish_cmd = TrustAnchorCommandDetails::republish(&handle, signer);
let events = ta.process_command(publish_cmd).unwrap();
let _ta = store.update(&handle, ta, events).unwrap();
+1 -1
View File
@@ -94,7 +94,7 @@ impl KrillClient {
Ok(ApiResponse::TrustAnchorInfo(ta))
},
TrustAnchorCommand::Publish => {
let uri = self.resolve_uri("api/v1/publish/ta");
let uri = self.resolve_uri("api/v1/republish");
httpclient::post_empty(&uri, Some(&self.token))?;
Ok(ApiResponse::Empty)
}
+12 -3
View File
@@ -6,6 +6,7 @@ use std::fmt;
use std::str::FromStr;
use bytes::Bytes;
use chrono::Duration;
use rpki::cert::Cert;
use rpki::crypto::PublicKey;
@@ -38,6 +39,7 @@ use crate::rpki::manifest::{
};
//------------ TaCertificate -------------------------------------------------
/// Contains a CA Certificate that has been issued to this CA, for some key.
@@ -222,6 +224,11 @@ impl CaKey {
pub fn incoming_cert(&self) -> &IncomingCertificate { &self.incoming_cert }
pub fn current_set(&self) -> &CurrentObjectSet { &self.current_set }
pub fn needs_publication(&self) -> bool {
self.current_set.number == 1 ||
self.current_set.next_update < Time::now() + Duration::hours(8)
}
pub fn apply_delta(&mut self, delta: PublicationDelta) {
self.current_set.apply_delta(delta)
}
@@ -323,7 +330,7 @@ impl CurrentObjects {
pub fn mft_entries(&self) -> Vec<FileAndHash<Bytes, Bytes>> {
self.0.keys().filter(|k| !k.ends_with("mft")).map(|k| {
let name_bytes = Bytes::from(k.as_str());
let hash_bytes = self.0.get(k).unwrap().content.to_encoded_hash().into();
let hash_bytes = self.0[k].content.to_encoded_hash().into();
FileAndHash::new(name_bytes, hash_bytes)
}).collect()
}
@@ -365,7 +372,9 @@ pub struct Revocations(Vec<Revocation>);
impl Revocations {
pub fn to_crl_entries(&self) -> Vec<CrlEntry> {
self.0.iter().map(|r| CrlEntry::new(Serial::from(r.serial), r.revocation_date)).collect()
self.0.iter()
.map(|r| CrlEntry::new(r.serial, r.revocation_date))
.collect()
}
/// Purges all expired revocations, and returns them.
@@ -535,7 +544,7 @@ impl Into<publication::PublishDelta> for ObjectsDelta {
fn into(self) -> publication::PublishDelta {
let mut builder = publication::PublishDeltaBuilder::new();
fn resolve(uri: &uri::Rsync, name: &ObjectName) -> uri::Rsync {
fn resolve(uri: &uri::Rsync, name: &str) -> uri::Rsync {
let uri = format!("{}{}", uri.to_string(), name);
uri::Rsync::from_string(uri).unwrap()
}
+9 -9
View File
@@ -26,7 +26,7 @@ pub trait AggregateStore<A: Aggregate>: Send + Sync {
fn get_latest(&self, id: &Handle) -> StoreResult<Arc<A>>;
/// Adds a new aggregate instance based on the init event.
fn add(&self, init: A::InitEvent) -> StoreResult<()>;
fn add(&self, init: A::InitEvent) -> StoreResult<Arc<A>>;
/// Updates the aggregate instance in the store. Expects that the
/// Arc<A> retrieved using 'get_latest' is moved here, so clone on
@@ -115,7 +115,7 @@ impl<A: Aggregate> DiskAggregateStore<A> {
impl<A: Aggregate> AggregateStore<A> for DiskAggregateStore<A> {
fn get_latest(&self, handle: &Handle) -> StoreResult<Arc<A>> {
info!("Trying to load aggregate id: {}", handle);
debug!("Trying to load aggregate id: {}", handle);
match self.cache_get(handle) {
None => {
match self.store.get_aggregate(handle)? {
@@ -126,7 +126,7 @@ impl<A: Aggregate> AggregateStore<A> for DiskAggregateStore<A> {
Some(agg) => {
let arc: Arc<A> = Arc::new(agg);
self.cache_update(handle, arc.clone());
info!("Loaded aggregate id: {} from disk", handle);
debug!("Loaded aggregate id: {} from disk", handle);
Ok(arc)
}
}
@@ -136,7 +136,7 @@ impl<A: Aggregate> AggregateStore<A> for DiskAggregateStore<A> {
let agg = Arc::make_mut(&mut arc);
self.store.update_aggregate(handle, agg)?;
}
info!("Loaded aggregate id: {} from memory", handle);
debug!("Loaded aggregate id: {} from memory", handle);
Ok(arc)
}
}
@@ -145,7 +145,7 @@ impl<A: Aggregate> AggregateStore<A> for DiskAggregateStore<A> {
fn add(
&self,
init: A::InitEvent
) -> StoreResult<()> {
) -> StoreResult<Arc<A>> {
self.store.store_event(&init)?;
let handle = init.handle().clone();
@@ -154,9 +154,9 @@ impl<A: Aggregate> AggregateStore<A> for DiskAggregateStore<A> {
self.store.store_aggregate(&handle, &aggregate)?;
let arc = Arc::new(aggregate);
self.cache_update(&handle, arc);
self.cache_update(&handle, arc.clone());
Ok(())
Ok(arc)
}
@@ -168,7 +168,6 @@ impl<A: Aggregate> AggregateStore<A> for DiskAggregateStore<A> {
) -> StoreResult<Arc<A>> {
// Get the latest arc.
let mut latest = self.get_latest(handle)?;
{
// Verify whether there is a concurrency issue
if prev.version() != latest.version() {
@@ -181,9 +180,10 @@ impl<A: Aggregate> AggregateStore<A> for DiskAggregateStore<A> {
// make the arc mutable, hopefully forgetting prev will avoid the clone
let agg = Arc::make_mut(&mut latest);
// Using a lock on the hashmap here to ensure that all updates happen sequentially.
// It would be better to get a lock only for this specific aggregate. So it may be
// worth rethinking the stru
// worth rethinking the structure.
//
// That said.. saving and applying events is really quick, so this should not hurt
// performance much.
+4 -4
View File
@@ -190,12 +190,12 @@ impl KeyStore for DiskKeyStore {
Err(KeyStoreError::JsonError(e))
},
Ok(v) => {
info!("Deserialized json at: {}", path_str);
debug!("Deserialized json at: {}", path_str);
Ok(Some(v))
}
}
} else {
warn!("Could not find file at: {}", path_str);
debug!("Could not find file at: {}", path_str);
Ok(None)
}
}
@@ -217,12 +217,12 @@ impl KeyStore for DiskKeyStore {
Err(KeyStoreError::JsonError(e))
},
Ok(v) => {
info!("Deserialized event at: {}", path_str);
debug!("Deserialized event at: {}", path_str);
Ok(Some(v))
}
}
} else {
info!("No more events at: {}", path_str);
debug!("No more events at: {}", path_str);
Ok(None)
}
}
+1
View File
@@ -10,6 +10,7 @@ actix-web = { version = "0.7.19", features = ["alpn"] }
base64 = "^0.9"
bcder = "^0.3"
bytes = "^0.4"
clokwerk = "^0.1"
chrono = { version = "^0.4", features = ["serde"] }
clap = "^2.32"
derive_more = "^0.13"
+2 -2
View File
@@ -85,9 +85,9 @@ pub struct Authorizer {
}
impl Authorizer {
pub fn new(krill_auth_token: &str) -> Self {
pub fn new(krill_auth_token: &Token) -> Self {
Authorizer {
krill_auth_token: Token::from(krill_auth_token)
krill_auth_token: krill_auth_token.clone()
}
}
+5 -4
View File
@@ -13,6 +13,7 @@ use serde::{Deserialize, Deserializer};
use toml;
use krill_commons::util::ext_serde;
use crate::http::ssl;
use krill_commons::api::admin::Token;
const SERVER_NAME: &str = "Krill";
@@ -35,11 +36,11 @@ impl ConfigDefaults {
fn log_type() -> LogType { LogType::Stderr }
fn syslog_facility() -> Facility { Facility::LOG_DAEMON }
fn log_file() -> PathBuf { PathBuf::from("./krill.log")}
fn auth_token() -> String {
fn auth_token() -> Token {
use std::env;
match env::var("KRILL_AUTH_TOKEN") {
Ok(token) => token,
Ok(token) => Token::from(token),
Err(_) => {
eprintln!("You MUST provide a value for the master API key, either by setting \"auth_token\" in the config file, or by setting the KRILL_AUTH_TOKEN environment variable.");
::std::process::exit(1);
@@ -101,7 +102,7 @@ pub struct Config {
log_file: PathBuf,
#[serde(default = "ConfigDefaults::auth_token")]
pub auth_token: String
pub auth_token: Token
}
/// # Accessors
@@ -164,7 +165,7 @@ impl Config {
let mut log_file = data_dir.clone();
log_file.push("krill.log");
let syslog_facility = ConfigDefaults::syslog_facility();
let auth_token = "secret".to_string();
let auth_token = Token::from("secret");
Config {
ip,
+2 -2
View File
@@ -216,8 +216,8 @@ pub fn init_trust_anchor(req: &HttpRequest) -> HttpResponse {
render_empty_res(rw_server(req).init_trust_anchor())
}
pub fn publish_trust_anchor(req: &HttpRequest) -> HttpResponse {
render_empty_res(ro_server(req).publish_trust_anchor())
pub fn republish_all(req: &HttpRequest) -> HttpResponse {
render_empty_res(ro_server(req).republish_all())
}
pub fn tal(req: &HttpRequest) -> HttpResponse {
+6 -5
View File
@@ -16,7 +16,7 @@ use actix_web::http::{Method, StatusCode};
use bcder::decode;
use futures::Future;
use openssl::ssl::{SslMethod, SslAcceptor, SslAcceptorBuilder, SslFiletype};
use crate::auth::{self, Authorizer, CheckAuthorisation, Credentials};
use crate::auth::{self, CheckAuthorisation, Credentials};
use crate::config::Config;
use crate::endpoints;
use crate::http::ssl;
@@ -70,10 +70,12 @@ impl PubServerApp {
r.method(Method::POST).f(endpoints::init_trust_anchor);
})
.resource("/api/v1/publish/ta", |r| {
r.method(Method::POST).f(endpoints::publish_trust_anchor)
.resource("/api/v1/republish", |r| {
r.method(Method::POST).f(endpoints::republish_all)
})
.resource("/ta/ta.tal", |r| {
r.method(Method::GET).f(endpoints::tal);
})
@@ -135,14 +137,13 @@ impl PubServerApp {
pub fn create_server(
config: &Config
) -> Result<Arc<RwLock<KrillServer>>, Error> {
let authorizer = Authorizer::new(&config.auth_token);
let pub_server = KrillServer::build(
&config.data_dir,
&config.rsync_base,
config.service_uri(),
&config.rrdp_base_uri,
authorizer,
&config.auth_token,
)?;
Ok(Arc::new(RwLock::new(pub_server)))
+30 -6
View File
@@ -22,6 +22,8 @@ use krill_pubd::PubServer;
use krill_pubd::publishers::Publisher;
use crate::auth::Authorizer;
use republisher::Republisher;
//------------ KrillServer ---------------------------------------------------
@@ -58,7 +60,12 @@ pub struct KrillServer {
caserver: CaServer<OpenSslSigner>,
// CMS+XML proxy server for non-Krill clients
proxy_server: ProxyServer
proxy_server: ProxyServer,
// Responsible for republishing periodically
#[allow(dead_code)] // keep this in scope
republisher: Republisher
}
/// # Set up and initialisation
@@ -70,11 +77,13 @@ impl KrillServer {
base_uri: &uri::Rsync,
service_uri: uri::Https,
rrdp_base_uri: &uri::Https,
authorizer: Authorizer,
token: &Token,
) -> Result<Self, Error> {
let mut repo_dir = work_dir.clone();
repo_dir.push("repo");
let authorizer = Authorizer::new(token);
let pubserver = PubServer::build(
base_uri.clone(),
rrdp_base_uri.clone(),
@@ -91,6 +100,11 @@ impl KrillServer {
let pub_clients = Arc::new(PubClients::build(work_dir)?);
let caserver = CaServer::build(work_dir, pub_clients.clone(), signer)?;
let republisher = {
let publish_uri = format!("{}api/v1/republish", service_uri);
Republisher::new(publish_uri, token)
};
Ok(
KrillServer {
service_uri,
@@ -99,7 +113,8 @@ impl KrillServer {
pubserver,
pub_clients,
caserver,
proxy_server
proxy_server,
republisher
}
)
}
@@ -270,11 +285,20 @@ impl KrillServer {
self.pub_clients.add(req)?;
// Add TA
self.caserver.init_ta(repo_info, ta_aia, vec![ta_uri]).map_err(Error::CaServerError)
self.caserver.init_ta(
repo_info,
ta_aia,
vec![ta_uri]
).map_err(Error::CaServerError)?;
// Force initial publication
self.caserver.publish_ta()?;
Ok(())
}
pub fn publish_trust_anchor(&self) -> Result<(), Error> {
self.caserver.publish_ta()?;
pub fn republish_all(&self) -> Result<(), Error> {
self.caserver.republish_all()?;
Ok(())
}
}
+2
View File
@@ -5,6 +5,7 @@ extern crate bytes;
extern crate bcder;
extern crate chrono;
extern crate clap;
extern crate clokwerk;
extern crate core;
#[macro_use] extern crate derive_more;
extern crate futures;
@@ -34,6 +35,7 @@ pub mod config;
pub mod endpoints;
pub mod krillserver;
pub mod http;
mod republisher;
pub mod test;
+39
View File
@@ -0,0 +1,39 @@
use clokwerk::{Scheduler, ScheduleHandle, TimeUnits};
use std::time::Duration;
use krill_commons::util::httpclient;
use krill_commons::api::admin::Token;
/// This type is responsible for periodically calling the
/// API to republish all CAs. Only CAs that *need* to republish
/// will do so (i.e. if there are no changes, and the nextUpdate
/// is still comfortably far in the future, this is a no-op).
///
/// This is done by calling the actual HTTPS end-point. While
/// this may seem somewhat convoluted, this eliminates the need
/// for this type to share state with the main application.
pub struct Republisher {
// Responsible for background tasks, e.g. re-publishing
#[allow(dead_code)] // just need to keep this in scope
tasks_thread: ScheduleHandle
}
impl Republisher {
pub fn new(publish_trigger_uri: String, token: &Token) -> Self {
let token = token.clone();
let mut scheduler = Scheduler::new();
scheduler.every(5.seconds()).run(move || {
if let Err(e) = httpclient::post_empty(
&publish_trigger_uri,
Some(&token)
) {
error!("Could not publish: {}", e);
}
});
Republisher {
tasks_thread: scheduler.watch_thread(Duration::from_millis(100))
}
}
}
+2 -2
View File
@@ -15,7 +15,7 @@ fn embedded_trust_anchor() {
let command = Command::TrustAnchor(TrustAnchorCommand::Show);
execute_krillc_command(command);
let command = Command::TrustAnchor(TrustAnchorCommand::Publish);
let _res = execute_krillc_command(command);
// let command = Command::TrustAnchor(TrustAnchorCommand::Publish);
// let _res = execute_krillc_command(command);
});
}
+2 -2
View File
@@ -178,9 +178,9 @@ impl Publisher {
impl Publisher {
fn create(event: PublisherInit) -> Self {
let (id, _version, init) = event.unwrap();
let (handle, _version, init) = event.unwrap();
Publisher {
handle: Handle::from(id),
handle,
version: 1,
deactivated: false,
token: init.token,