diff --git a/src/api/mod.rs b/src/api/mod.rs index 80654eea..9d114ade 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,14 +1,143 @@ //! Data structures for the API, shared between client and server. -pub mod publishers; -pub mod publication; -pub mod rrdp_data; +pub mod publisher_data; +pub mod publication_data; +pub mod repo_data; + +use bytes::Bytes; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use crate::util::sha256; + + +//------------ Base64 -------------------------------------------------------- + +/// This type contains a base64 encoded structure. The publication protocol +/// deals with objects in their base64 encoded form. +/// +/// Note that we store this in a Bytes to make it cheap to clone this. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Base64(Bytes); + +impl Base64 { + pub fn from_content(content: &[u8]) -> Self { + Base64::from(base64::encode(content)) + } + + /// Decodes into bytes (e.g. for saving to disk for rcync) + pub fn to_bytes(&self) -> Bytes { + Bytes::from(base64::decode(&self.0).unwrap()) + } + + pub fn to_hex_hash(&self) -> String { + hex::encode(sha256(&self.to_bytes())) + } + + pub fn to_encoded_hash(&self) -> EncodedHash { + EncodedHash::from(self.to_hex_hash()) + } +} + +impl AsRef for Base64 { + fn as_ref(&self) -> &str { + use std::str; + str::from_utf8(&self.0).unwrap() + } +} + +impl From for Base64 { + fn from(s: String) -> Self { + Base64(Bytes::from(s)) + } +} + + +impl ToString for Base64 { + fn to_string(&self) -> String { + unsafe { + String::from_utf8_unchecked(self.0.to_vec()) + } + } +} + +impl Serialize for Base64 { + fn serialize( + &self, serializer: S + ) -> Result where S: Serializer { + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Base64 { + fn deserialize( + deserializer: D + ) -> Result where D: Deserializer<'de> { + let string = String::deserialize(deserializer)?; + Ok(Base64::from(string)) + } +} + + +//------------ EncodedHash --------------------------------------------------- + +/// This type contains a hex encoded sha256 hash. +/// +/// Note that we store this in a Bytes for cheap cloning. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct EncodedHash(Bytes); + +impl EncodedHash { + pub fn from_content(content: &[u8]) -> Self { + let sha256 = sha256(content); + let hex = hex::encode(sha256); + EncodedHash::from(hex) + } +} + +impl AsRef for EncodedHash { + fn as_ref(&self) -> &str { + use std::str; + str::from_utf8(&self.0).unwrap() + } +} + +impl From for EncodedHash { + fn from(s: String) -> Self { + EncodedHash(Bytes::from(s.to_lowercase())) + } +} + +impl ToString for EncodedHash { + fn to_string(&self) -> String { + unsafe { + String::from_utf8_unchecked(self.0.to_vec()) + } + } +} + +impl Serialize for EncodedHash { + fn serialize( + &self, serializer: S + ) -> Result where S: Serializer { + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for EncodedHash { + fn deserialize( + deserializer: D + ) -> Result where D: Deserializer<'de> { + let string = String::deserialize(deserializer)?; + Ok(EncodedHash::from(string)) + } +} + + //------------ Link ---------------------------------------------------------- /// Defines a link element to include as part of a links array in a Json /// response. #[derive(Clone, Debug, Serialize)] -pub struct Link<'a> { - rel: &'a str, +pub struct Link { + rel: String, link: String } diff --git a/src/api/publication.rs b/src/api/publication_data.rs similarity index 78% rename from src/api/publication.rs rename to src/api/publication_data.rs index 8836ec01..fb63eeb4 100644 --- a/src/api/publication.rs +++ b/src/api/publication_data.rs @@ -1,9 +1,8 @@ //! Support for requests sent to the Json API -use bytes::Bytes; use rpki::uri; +use crate::api::{ Base64, EncodedHash }; use crate::util::ext_serde; use crate::util::file::CurrentFile; -use crate::util::sha256; //------------ PublishRequest ------------------------------------------------ @@ -51,6 +50,10 @@ impl PublishDelta { } pub fn is_empty(&self) -> bool { self.len() == 0 } + + pub fn unwrap(self) -> (Vec, Vec, Vec) { + (self.publishes, self.updates, self.withdraws) + } } @@ -104,18 +107,15 @@ pub struct Publish { serialize_with = "ext_serde::ser_rsync_uri")] uri: uri::Rsync, - #[serde( - deserialize_with = "ext_serde::de_bytes", - serialize_with = "ext_serde::ser_bytes")] - content: Bytes + content: Base64 } impl Publish { - pub fn new(tag: Option, uri: uri::Rsync, content: Bytes) -> Self { + pub fn new(tag: Option, uri: uri::Rsync, content: Base64) -> Self { Publish { tag, uri, content } } - pub fn with_hash_tag(uri: uri::Rsync, content: Bytes) -> Self { - let tag = Some(hex::encode(sha256(&content))); + pub fn with_hash_tag(uri: uri::Rsync, content: Base64) -> Self { + let tag = Some(content.to_hex_hash()); Publish { tag, uri, content } } @@ -127,7 +127,11 @@ impl Publish { } } pub fn uri(&self) -> &uri::Rsync{ &self.uri} - pub fn content(&self) -> &Bytes{ &self.content } + pub fn content(&self) -> &Base64{ &self.content } + + pub fn unwrap(self) -> (Option, uri::Rsync, Base64) { + (self.tag, self.uri, self.content) + } } @@ -145,32 +149,26 @@ pub struct Update { serialize_with = "ext_serde::ser_rsync_uri")] uri: uri::Rsync, - #[serde( - deserialize_with = "ext_serde::de_bytes", - serialize_with = "ext_serde::ser_bytes")] - content: Bytes, + content: Base64, - #[serde( - deserialize_with = "ext_serde::de_bytes", - serialize_with = "ext_serde::ser_bytes")] - hash: Bytes, + hash: EncodedHash, } impl Update { pub fn new( tag: Option, uri: uri::Rsync, - content: Bytes, - old_hash: Bytes + content: Base64, + old_hash: EncodedHash ) -> Self { Update { tag, uri, content, hash: old_hash } } pub fn with_hash_tag( uri: uri::Rsync, - content: Bytes, - old_hash: Bytes + content: Base64, + old_hash: EncodedHash ) -> Self { - let tag = Some(hex::encode(sha256(&content))); + let tag = Some(content.to_hex_hash()); Update { tag, uri, content, hash: old_hash } } @@ -182,8 +180,12 @@ impl Update { } } pub fn uri(&self) -> &uri::Rsync { &self.uri} - pub fn content(&self) -> &Bytes { &self.content } - pub fn hash(&self) -> &Bytes { &self.hash } + pub fn content(&self) -> &Base64 { &self.content } + pub fn hash(&self) -> &EncodedHash { &self.hash } + + pub fn unwrap(self) -> (Option, uri::Rsync, Base64, EncodedHash) { + (self.tag, self.uri, self.content, self.hash) + } } @@ -201,19 +203,16 @@ pub struct Withdraw { serialize_with = "ext_serde::ser_rsync_uri")] uri: uri::Rsync, - #[serde( - deserialize_with = "ext_serde::de_bytes", - serialize_with = "ext_serde::ser_bytes")] - hash: Bytes, + hash: EncodedHash, } impl Withdraw { - pub fn new(tag: Option, uri: uri::Rsync, hash: Bytes) -> Self { + pub fn new(tag: Option, uri: uri::Rsync, hash: EncodedHash) -> Self { Withdraw { tag, uri, hash } } - pub fn with_hash_tag(uri: uri::Rsync, hash: Bytes) -> Self { - let tag = Some(hex::encode(&hash)); + pub fn with_hash_tag(uri: uri::Rsync, hash: EncodedHash) -> Self { + let tag = Some(hash.to_string()); Withdraw { tag, uri, hash } } @@ -233,7 +232,11 @@ impl Withdraw { } } pub fn uri(&self) -> &uri::Rsync { &self.uri} - pub fn hash(&self) -> &Bytes { &self.hash } + pub fn hash(&self) -> &EncodedHash { &self.hash } + + pub fn unwrap(self) -> (Option, uri::Rsync, EncodedHash) { + (self.tag, self.uri, self.hash) + } } //------------ PublishReply -------------------------------------------------- @@ -281,21 +284,14 @@ pub struct ListElement { serialize_with = "ext_serde::ser_rsync_uri")] uri: uri::Rsync, - #[serde( - deserialize_with = "ext_serde::de_bytes", - serialize_with = "ext_serde::ser_bytes")] - /// The sha-256 hash of the file (as is used on the RPKI manifests and - /// in the publication protocol for list, update and withdraw). Saving - /// this rather than calculating on demand seems a small price for some - /// performance gain. - hash: Bytes + hash: EncodedHash } impl ListElement { - pub fn new(uri: uri::Rsync, hash: Bytes) -> Self { + pub fn new(uri: uri::Rsync, hash: EncodedHash) -> Self { ListElement { uri, hash } } pub fn uri(&self) -> &uri::Rsync { &self.uri } - pub fn hash(&self) -> &Bytes { &self.hash} + pub fn hash(&self) -> &EncodedHash { &self.hash } } diff --git a/src/api/publishers.rs b/src/api/publisher_data.rs similarity index 74% rename from src/api/publishers.rs rename to src/api/publisher_data.rs index f8caffdd..def2d11f 100644 --- a/src/api/publishers.rs +++ b/src/api/publisher_data.rs @@ -2,13 +2,21 @@ //! //! i.e. this is stuff the the server needs to serialize only, so typically //! we can work with references here. -use std::sync::Arc; use rpki::uri; use crate::api::Link; +use crate::eventsourcing::AggregateId; +use crate::krilld::pubd::publishers::Publisher; use crate::remote::id::IdCert; use crate::util::ext_serde; +//------------ PublisherHandle ----------------------------------------------- + +/// A type for referring to publishers, both in the api as well as to the +/// aggregates. +pub type PublisherHandle = AggregateId; + + //------------ CmsAuthData --------------------------------------------------- /// This type contains the data needed for handling RFC8183 requests/responses, @@ -51,11 +59,12 @@ impl PartialEq for CmsAuthData { impl Eq for CmsAuthData {} -//------------ Publisher ----------------------------------------------------- +//------------ PublisherRequest ---------------------------------------------- -/// This type defines Publisher CAs that are allowed to publish. +/// This type defines request for a new Publisher (CA that is allowed to +/// publish). #[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Publisher { +pub struct PublisherRequest { handle: String, /// The token used by the API @@ -69,14 +78,14 @@ pub struct Publisher { cms_auth_data: Option } -impl Publisher { +impl PublisherRequest { pub fn new( handle: String, token: String, base_uri: uri::Rsync, rfc8181: Option ) -> Self { - Publisher { + PublisherRequest { handle, token, base_uri, @@ -85,7 +94,7 @@ impl Publisher { } } -impl Publisher { +impl PublisherRequest { pub fn handle(&self) -> &String { &self.handle } @@ -101,17 +110,22 @@ impl Publisher { pub fn cms_auth_data(&self) -> &Option { &self.cms_auth_data } + + /// Return all the values (handle, token, base_uri, rfc8181opt). + pub fn unwrap(self) -> (String, String, uri::Rsync, Option) { + (self.handle, self.token, self.base_uri, self.cms_auth_data) + } } -impl PartialEq for Publisher { - fn eq(&self, other: &Publisher) -> bool { +impl PartialEq for PublisherRequest { + fn eq(&self, other: &PublisherRequest) -> bool { self.handle == other.handle && self.base_uri == other.base_uri && self.cms_auth_data == other.cms_auth_data } } -impl Eq for Publisher {} +impl Eq for PublisherRequest {} //------------ PublisherSummaryInfo ------------------------------------------ @@ -119,33 +133,25 @@ impl Eq for Publisher {} /// Defines a summary of publisher information to be used in the publisher /// list. #[derive(Clone, Debug, Serialize)] -pub struct PublisherSummaryInfo<'a> { - id: &'a str, - links: Vec> +pub struct PublisherSummaryInfo { + id: String, + links: Vec } -impl<'a> PublisherSummaryInfo<'a> { +impl PublisherSummaryInfo { pub fn from( - publisher: &'a Publisher, - path_publishers: &'a str - ) -> PublisherSummaryInfo<'a> { - let id = publisher.handle().as_str(); + handle: &PublisherHandle, + path_publishers: &str + ) -> PublisherSummaryInfo { let mut links = Vec::new(); - - let response_link = Link { - rel: "response.xml", - link: format!("{}/{}/response.xml", path_publishers, id) - }; let self_link = Link { - rel: "self", - link: format!("{}/{}", path_publishers, id) + rel: "self".to_string(), + link: format!("{}/{}", path_publishers, handle.as_ref()) }; - - links.push(response_link); links.push(self_link); PublisherSummaryInfo { - id, + id: handle.to_string(), links } } @@ -156,15 +162,15 @@ impl<'a> PublisherSummaryInfo<'a> { /// This type represents a list of (all) current publishers to show in the API #[derive(Clone, Debug, Serialize)] -pub struct PublisherList<'a> { - publishers: Vec> +pub struct PublisherList { + publishers: Vec } -impl<'a> PublisherList<'a> { - pub fn from( - publishers: &'a[Arc], - path_publishers: &'a str - ) -> PublisherList<'a> { +impl PublisherList { + pub fn build( + publishers: &[PublisherHandle], + path_publishers: &str + ) -> PublisherList { let publishers: Vec = publishers.iter().map(|p| PublisherSummaryInfo::from(&p, path_publishers) ).collect(); @@ -194,24 +200,27 @@ pub struct Rfc8181Details<'a> { #[derive(Clone, Debug, Serialize)] pub struct PublisherDetails<'a> { - publisher_handle: &'a str, + handle: &'a str, + + retired: bool, #[serde(serialize_with = "ext_serde::ser_rsync_uri")] base_uri: &'a uri::Rsync, rfc8181: Option>, - links: Vec> + links: Vec } impl<'a> PublisherDetails<'a> { pub fn from( - publisher: &'a Arc, + publisher: &'a Publisher, path_publishers: &'a str, base_service_uri: &uri::Http ) -> PublisherDetails<'a> { - let handle = publisher.handle().as_str(); + let handle = publisher.id().as_ref(); let base_uri = publisher.base_uri(); + let retired = publisher.retired(); // Derive the RFC8181 service URI. let service_uri = format!("{}{}", base_service_uri, handle); @@ -229,12 +238,13 @@ impl<'a> PublisherDetails<'a> { let mut links = Vec::new(); links.push(Link { - rel: "response.xml", + rel: "response.xml".to_string(), link: format!("{}/{}/response.xml", path_publishers, handle) }); PublisherDetails { - publisher_handle: handle, + handle, + retired, base_uri, rfc8181, links diff --git a/src/api/repo_data.rs b/src/api/repo_data.rs new file mode 100644 index 00000000..115f8a65 --- /dev/null +++ b/src/api/repo_data.rs @@ -0,0 +1,671 @@ +//! Data objects used in the (RRDP) repository. I.e. the publish, update, and +//! withdraw elements, as well as the notification, snapshot and delta file +//! definitions. +use std::collections::HashMap; +use std::io; +use std::path::PathBuf; +use bytes::Bytes; +use rpki::uri; +use crate::api::publication_data; +use crate::api::Base64; +use crate::api::EncodedHash; +use crate::util::ext_serde; +use crate::util::file; +use crate::util::Time; +use crate::util::xml::XmlWriter; + + +const VERSION: &str = "1"; +const NS: &str = "http://www.ripe.net/rpki/rrdp"; + +//------------ PublishElement ------------------------------------------------ + +/// The publishes as used in the RRDP protocol. +/// +/// Note that the difference with the publication protocol is the absence of +/// the tag. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct PublishElement { + base64: Base64, + + #[serde( + deserialize_with = "ext_serde::de_rsync_uri", + serialize_with = "ext_serde::ser_rsync_uri")] + uri: uri::Rsync +} + +impl PublishElement { + pub fn new(base64: Base64, uri: uri::Rsync) -> Self { + PublishElement { base64, uri } + } + + pub fn base64(&self) -> &Base64 { &self.base64 } + pub fn uri(&self) -> &uri::Rsync { &self.uri } +} + +impl From for PublishElement { + fn from(p: publication_data::Publish) -> Self { + let (_tag, uri, base64) = p.unwrap(); + PublishElement { uri, base64 } + } +} + + +//------------ UpdateElement ------------------------------------------------- + +/// The updates as used in the RRDP protocol. +/// +/// Note that the difference with the publication protocol is the absence of +/// the tag. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct UpdateElement { + #[serde( + deserialize_with = "ext_serde::de_rsync_uri", + serialize_with = "ext_serde::ser_rsync_uri")] + uri: uri::Rsync, + hash: EncodedHash, + base64: Base64 +} + +impl UpdateElement { + pub fn uri(&self) -> &uri::Rsync { &self.uri } + pub fn hash(&self) -> &EncodedHash { &self.hash } + pub fn base64(&self) -> &Base64 { &self.base64 } +} + +impl From for UpdateElement { + fn from(u: publication_data::Update) -> Self { + let (_tag, uri, base64, hash) = u.unwrap(); + UpdateElement { uri, base64, hash } + } +} + +impl Into for UpdateElement { + fn into(self) -> PublishElement { + PublishElement { uri: self.uri, base64: self.base64 } + } +} + + +//------------ WithdrawElement ----------------------------------------------- + +/// The withdraws as used in the RRDP protocol. +/// +/// Note that the difference with the publication protocol is the absence of +/// the tag. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct WithdrawElement { + #[serde( + deserialize_with = "ext_serde::de_rsync_uri", + serialize_with = "ext_serde::ser_rsync_uri")] + uri: uri::Rsync, + hash: EncodedHash +} + +impl WithdrawElement { + pub fn uri(&self) -> &uri::Rsync { &self.uri } + pub fn hash(&self) -> &EncodedHash { &self.hash } +} + +impl From for WithdrawElement { + fn from(w: publication_data::Withdraw) -> Self { + let (_tag, uri, hash) = w.unwrap(); + WithdrawElement { uri, hash } + } +} + + + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Notification { + session: String, + serial: u64, + time: Time, + snapshot: SnapshotRef, + deltas: Vec, + old_refs: Vec<(Time, FileRef)> +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct NotificationUpdate { + time: Time, + session: Option, + snapshot: SnapshotRef, + delta: DeltaRef, + last_delta: u64 +} + +impl NotificationUpdate { + pub fn new( + time: Time, + session: Option, + snapshot: SnapshotRef, + delta: DeltaRef, + last_delta: u64 + ) -> Self { + NotificationUpdate { time, session, snapshot, delta, last_delta } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct NotificationCreate { + session: String, + snapshot: SnapshotRef +} + +impl NotificationUpdate { + pub fn unwrap(self) -> (Time, Option, SnapshotRef, DeltaRef, u64) { + (self.time, self.session, self.snapshot, self.delta, self.last_delta) + } +} + +impl Notification { + pub fn old_refs(&self) -> &Vec<(Time, FileRef)> { + &self.old_refs + } + + pub fn update(&mut self, update: NotificationUpdate) { + let (time, session_opt, snapshot, delta, last_delta) = update.unwrap(); + if let Some(session) = session_opt { + self.session = session; + } + + self.serial += 1; + self.time = time; + + let mut refs_to_retire = vec![]; + + refs_to_retire.push((Time::now(), self.snapshot.clone())); + self.snapshot = snapshot; + + for d in &self.deltas { + if d.serial < last_delta { + refs_to_retire.push((Time::now(), d.file_ref.clone())); + } + } + + self.deltas.insert(0, delta); + self.deltas.retain(|delta| delta.serial >= last_delta); + self.old_refs.append(&mut refs_to_retire); + } + + /// Cleans up all old references from before the given time. + pub fn clean_up(&mut self, t: Time) { + self.old_refs.retain(|old_ref| {! old_ref.0.on_or_before(&t)}) + } + + pub fn create(session: String, snapshot: SnapshotRef) -> Self { + Notification { + session, + serial: 0, + time: Time::now(), + snapshot, + deltas: vec![], + old_refs: vec![] + } + } + + pub fn write_xml(&self, path: &PathBuf) -> Result<(), io::Error> { + debug!("Writing notification file: {}", path.to_string_lossy()); + let mut file = file::create_file_with_path(&path)?; + + XmlWriter::encode_to_file(& mut file, |w| { + + let a = [ + ("xmlns", NS), + ("version", VERSION), + ("session_id", self.session.as_ref()), + ("serial", &format!("{}", self.serial)), + ]; + + w.put_element( + "notification", + Some(&a), + |w| { + { + // snapshot ref + let uri = self.snapshot.uri.to_string(); + let a = [ + ("uri", uri.as_str()), + ("hash", self.snapshot.hash.as_ref()) + ]; + w.put_element( + "snapshot", + Some(&a), + |w| { w.empty() } + )?; + } + + { + // delta refs + for delta in &self.deltas { + let serial = format!("{}", delta.serial); + let uri = delta.file_ref.uri.to_string(); + let a = [ + ("serial", serial.as_ref()), + ("uri", uri.as_str()), + ("hash", delta.file_ref.hash.as_ref()) + ]; + w.put_element( + "delta", + Some(&a), + |w| { w.empty() } + )?; + } + } + + Ok(()) + } + ) + })?; + + Ok(()) + + } + +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct FileRef { + #[serde( + deserialize_with = "ext_serde::de_http_uri", + serialize_with = "ext_serde::ser_http_uri")] + uri: uri::Http, + path: PathBuf, + hash: EncodedHash, +} + +impl FileRef { + pub fn new(uri: uri::Http, path: PathBuf, hash: EncodedHash) -> Self { + FileRef { uri, path, hash } + } + pub fn uri(&self) -> &uri::Http { &self.uri } + pub fn path(&self) -> &PathBuf { &self.path } + pub fn hash(&self) -> &EncodedHash { &self.hash } +} + +pub type SnapshotRef = FileRef; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct DeltaRef { + serial: u64, + file_ref: FileRef +} + +impl DeltaRef { + pub fn new(serial: u64, file_ref: FileRef) -> Self { + DeltaRef { serial, file_ref } + } + + pub fn serial(&self) -> u64 { self.serial } +} + +impl AsRef for DeltaRef { + fn as_ref(&self) -> &FileRef { + &self.file_ref + } +} + + +//------------ CurrentObjects ------------------------------------------------ + +/// Defines a current set of published elements. +/// +// Note this is mapped internally for speedy access, by hash, rather than uri +// for two reasons: +// a) URIs in RPKI may change in future +// b) The publish element as it appears in an RFC8182 snapshot.xml includes +// the uri and the base64, but not the hash. So keeping the actual elements +// around means we can be more efficient in producing that output. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct CurrentObjects(HashMap); + +impl Default for CurrentObjects { + fn default() -> Self { + CurrentObjects(HashMap::new()) + } +} + +impl CurrentObjects { + fn elements(&self) -> Vec<&PublishElement> { + let mut res = vec![]; + for el in self.0.values() { + res.push(el) + } + res + } +} + + +//------------ VerificationError --------------------------------------------- + +/// Issues with relation to verifying deltas. +#[derive(Clone, Debug, Display)] +pub enum VerificationError { + #[display(fmt="Publishing ({}) outside of jail URI ({}) is not allowed.", _0, _1)] + UriOutsideJail(uri::Rsync, uri::Rsync), + + #[display(fmt="File already exists for uri (use update!): {}", _0)] + ObjectAlreadyPresent(uri::Rsync), + + #[display(fmt="File does not match hash at uri: {}", _0)] + NoObjectForHashAndOrUri(uri::Rsync), +} + +impl VerificationError { + fn outside(jail: &uri::Rsync, uri: &uri::Rsync) -> Self { + VerificationError::UriOutsideJail(uri.clone(), jail.clone()) + } + + fn present(uri: &uri::Rsync) -> Self { + VerificationError::ObjectAlreadyPresent(uri.clone()) + } + + fn no_match(uri: &uri::Rsync) -> Self { + VerificationError::NoObjectForHashAndOrUri(uri.clone()) + } +} + +impl CurrentObjects { + + fn has_match( + &self, + hash: &EncodedHash, + uri: &uri::Rsync + ) -> bool { + match self.0.get(hash) { + Some(el) => el.uri() == uri, + None => false + } + } + + pub fn verify_delta( + &self, + delta: &DeltaElements, + jail: &uri::Rsync + ) -> Result<(), VerificationError> { + + for p in delta.publishes() { + if ! jail.is_parent_of(p.uri()) { + return Err(VerificationError::outside(jail, p.uri())) + } + let hash = p.base64().to_encoded_hash(); + if self.0.contains_key(&hash) { + return Err(VerificationError::present(p.uri())) + } + } + + for u in delta.updates() { + if ! self.has_match(u.hash(), u.uri()) { + return Err(VerificationError::no_match(u.uri())); + } + } + + for w in delta.withdraws() { + if ! self.has_match(w.hash(), w.uri()) { + return Err(VerificationError::no_match(w.uri())); + } + } + + Ok(()) + } + + /// Applies a delta to CurrentObjects. This will asume that the delta + /// contains only valid updates for this delta. + pub fn apply_delta(&mut self, delta: DeltaElements) { + let (publishes, updates, withdraws) = delta.unwrap(); + + for p in publishes { + let hash = p.base64().to_encoded_hash(); + self.0.insert(hash, p); + } + + for u in updates { + self.0.remove(u.hash()); + let p: PublishElement = u.into(); + let hash = p.base64().to_encoded_hash(); + self.0.insert(hash, p); + } + + for w in withdraws { + self.0.remove(w.hash()); + } + } + + pub fn len(&self) -> usize { self.0.len() } + + pub fn is_empty(&self) -> bool { self.0.is_empty() } + + pub fn to_list_reply(&self) -> publication_data::ListReply { + let elements = self.0.iter().map(|el| { + let hash = el.0.clone(); + let uri = el.1.uri().clone(); + publication_data::ListElement::new(uri, hash) + }).collect(); + + publication_data::ListReply::new(elements) + } +} + + +//------------ Snapshot ------------------------------------------------------ + +/// A structure to contain the RRDP snapshot data. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Snapshot { + session: String, + serial: u64, + current_objects: CurrentObjects +} + +impl Snapshot { + pub fn new(session: String) -> Self { + let current_objects = CurrentObjects::default(); + Snapshot { session, serial: 0, current_objects } + } + + pub fn apply_delta(&mut self, delta: Delta) { + let (session, serial, elements) = delta.unwrap(); + self.session = session; + self.serial = serial; + self.current_objects.apply_delta(elements) + } + + pub fn len(&self) -> usize { self.current_objects.len() } + + pub fn is_empty(&self) -> bool { self.current_objects.is_empty() } + + pub fn write_xml(&self, path: &PathBuf) -> Result { + let vec = XmlWriter::encode_vec(|w| { + let a = [ + ("xmlns", NS), + ("version", VERSION), + ("session_id", self.session.as_ref()), + ("serial", &format!("{}", self.serial)), + ]; + + w.put_element( + "snapshot", + Some(&a), + |w| { + for el in self.current_objects.elements() { + let uri = el.uri.to_string(); + let atr = [ ("uri", uri.as_ref())]; + w.put_element( + "publish", + Some(&atr), + |w| { + w.put_text(el.base64.as_ref()) + } + )?; + } + Ok(()) + } + ) + }); + let bytes = Bytes::from(vec); + + file::save(&bytes, path)?; + let hash = EncodedHash::from_content(&bytes); + + Ok(hash) + } +} + + +//------------ DeltaElements ------------------------------------------------- + +/// Defines the elements for an RRDP delta. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct DeltaElements { + publishes: Vec, + updates: Vec, + withdraws: Vec +} + +impl From for DeltaElements { + fn from(d: publication_data::PublishDelta) -> Self { + let (pbls, upds, wdrs) = d.unwrap(); + + let publishes = pbls.into_iter().map(PublishElement::from).collect(); + let updates = upds.into_iter().map(UpdateElement::from).collect(); + let withdraws = wdrs.into_iter().map(WithdrawElement::from).collect(); + + DeltaElements { publishes, updates, withdraws } + } +} + +impl DeltaElements { + pub fn unwrap( + self + ) -> (Vec, Vec, Vec) { + (self.publishes, self.updates, self.withdraws) + } + + pub fn len(&self) -> usize { + self.publishes.len() + self.updates.len() + self.withdraws.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn publishes(&self) -> &Vec { + &self.publishes + } + + pub fn updates(&self) -> &Vec { + &self.updates + } + + pub fn withdraws(&self) -> &Vec { + &self.withdraws + } +} + + +//------------ Delta --------------------------------------------------------- + +/// Defines an RRDP delta. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Delta { + session: String, + serial: u64, + time: Time, + elements: DeltaElements +} + +impl Delta { + pub fn new( + session: String, + serial: u64, + elements: DeltaElements + ) -> Self { + Delta { session, time: Time::now(), serial, elements } + } + + pub fn session(&self) -> &str { &self.session } + pub fn serial(&self) -> u64 { self.serial } + pub fn time(&self) -> &Time { &self.time } + pub fn elements(&self) -> &DeltaElements { &self.elements } + + /// Total number of elements + /// + /// This is a cheap approximation of the size of the delta that can help + /// in determining the choice of how many deltas to include in a + /// notification file. + pub fn len(&self) -> usize { self.elements.len() } + + pub fn is_empty(&self) -> bool { self.elements.is_empty() } + + pub fn unwrap(self) -> (String, u64, DeltaElements) { + (self.session, self.serial, self.elements) + } + + pub fn write_xml(&self, path: &PathBuf) -> Result { + + let vec = XmlWriter::encode_vec(|w| { + let a = [ + ("xmlns", NS), + ("version", VERSION), + ("session_id", self.session.as_ref()), + ("serial", &format!("{}", self.serial)), + ]; + + w.put_element( + "delta", + Some(&a), + |w| { + for el in &self.elements.publishes { + let uri = el.uri.to_string(); + let atr = [ ("uri", uri.as_ref())]; + w.put_element( + "publish", + Some(&atr), + |w| { + w.put_text(el.base64.as_ref()) + } + )?; + } + + for el in &self.elements.updates { + let uri = el.uri.to_string(); + let atr = [ + ("uri", uri.as_ref()), + ("hash", el.hash.as_ref()) + ]; + w.put_element( + "publish", + Some(&atr), + |w| { + w.put_text(el.base64.as_ref()) + } + )?; + } + + for el in &self.elements.withdraws { + let uri = el.uri.to_string(); + let atr = [ + ("uri", uri.as_ref()), + ("hash", el.hash.as_ref()) + ]; + w.put_element( + "withdraw", + Some(&atr), + |w| { w.empty() } + )?; + } + + Ok(()) + } + ) + }); + + let bytes = Bytes::from(vec); + file::save(&bytes, &path)?; + let hash = EncodedHash::from_content(&bytes); + + Ok(hash) + } + +} diff --git a/src/api/rrdp_data.rs b/src/api/rrdp_data.rs deleted file mode 100644 index 6c7283a9..00000000 --- a/src/api/rrdp_data.rs +++ /dev/null @@ -1,537 +0,0 @@ -use std::collections::HashMap; -use std::io; -use std::fs::File; -use std::num::ParseIntError; -use std::ops::Deref; -use std::path::PathBuf; -use std::str::FromStr; -use bytes::Bytes; -use rpki::uri; -use crate::util::xml::{AttributesError, XmlReader, XmlReaderErr, XmlWriter}; -use crate::util::file::{self, RecursorError}; -use crate::util::ext_serde; -use util::sha256; - -const VERSION: &str = "1"; -const NS: &str = "http://www.ripe.net/rpki/rrdp"; - -#[derive(Clone, Debug, Deserialize, Eq, Serialize, PartialEq)] -pub struct PublishedObject { - #[serde( - deserialize_with = "ext_serde::de_rsync_uri", - serialize_with = "ext_serde::ser_rsync_uri")] - uri: uri::Rsync, - - #[serde( - deserialize_with = "ext_serde::de_bytes", - serialize_with = "ext_serde::ser_bytes")] - content: Bytes, - - #[serde( - deserialize_with = "ext_serde::de_bytes", - serialize_with = "ext_serde::ser_bytes")] - hash: Bytes -} - -impl PublishedObject { - pub fn new(uri: uri::Rsync, content: Bytes) -> Self { - let hash = sha256(&content); - PublishedObject { uri , content, hash } - } - - pub fn uri(&self) -> &uri::Rsync{ &self.uri } - pub fn content(&self) -> &Bytes { &self.content } - pub fn hash(&self) -> &Bytes { &self.hash } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Snapshot { - session: String, - serial: usize, - objects: HashMap> -} - -impl Snapshot { - pub fn new( - session: String, - serial: usize, - objects: HashMap> - ) -> Self { - Snapshot { session, serial, objects } - } - - pub fn objects(&self) -> &HashMap> { - &self.objects - } - - pub fn to_xml(&self) -> Vec { - XmlWriter::encode_vec(|w| { - let a = [ - ("xmlns", NS), - ("version", VERSION), - ("session_id", self.session.as_ref()), - ("serial", &format!("{}", self.serial)), - ]; - - w.put_element( - "snapshot", - Some(&a), - |w| { - for uri in self.objects.keys() { - let objects = &self.objects[uri]; - for cf in objects { - let uri = cf.uri.to_string(); - let a = [ ("uri", uri.as_ref()) ]; - w.put_element( - "publish", - Some(&a), - |w| { - w.put_blob(&cf.content) - } - )?; - } - } - Ok(()) - } - ) - }) - } -} - - - -//------------ Notification -------------------------------------------------- - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Notification { - session_id: String, - serial: usize, - snapshot: SnapshotRef, - deltas: Vec -} - -/// # Accessors -/// -impl Notification { - pub fn serial(&self) -> usize { - self.serial - } - pub fn session_id(&self) -> &String { &self.session_id } - - pub fn deltas(&self) -> &Vec { - &self.deltas - } -} - -/// # Load and save -/// -impl Notification { - - /// Build up the current notification based on what's on disk. - /// Note that this will return None, if there is nothing on disk. - /// - /// Also note that in future we may want to cache things for - /// efficiency (but make it work with multi master). - pub fn build( - path: &PathBuf, - base_uri: &uri::Http, - rrdp_base: &PathBuf - ) -> Option { - let mut builder = NotificationBuilder::new(); - - match XmlReader::open(path, |r| -> Result<(), Error> { - r.take_named_element("notification", |mut a, r| { - { - // process attributes - builder.with_session_id(a.take_req("session_id")?); - let serial = usize::from_str(a.take_req("serial")?.as_ref())?; - builder.with_serial(serial); - // about NS - } - - { - // expect snapshot ref - r.take_named_element( - "snapshot", - |mut a, _r| -> Result<(), Error> { - let uri = uri::Http::from_string(a.take_req("uri")?)?; - let file_info = FileInfo::for_uri( - &uri, - base_uri, - rrdp_base - )?; - - builder.with_snapshot( - SnapshotRef { file_info } - ); - Ok(()) - })?; - } - - { - // deltas - loop { - let d = r.take_opt_element(|t, mut a, _r| { - match t.name.as_ref() { - "delta" => { - let uri = uri::Http::from_string( - a.take_req("uri")? - )?; - let serial = usize::from_str( - a.take_req("serial")?.as_ref() - )?; - let file_info = FileInfo::for_uri( - &uri, - base_uri, - rrdp_base - )?; - - Ok(Some(DeltaRef { - serial, - file_info - })) - }, - _ => Err(Error::NotificationFileError) - } - })?; - match d { - None => break, - Some(d) => builder.add_delta(d) - } - } - Ok(()) - } - }) - }).map_err(|_| Error::NotificationFileError) { - Ok(_) => Some(builder.build()), - Err(_) => None - } - } - - /// Saves a notification file as RFC8182 XML. - pub fn save(&self, path: &PathBuf) -> Result<(), Error> { - debug!("Writing notification file: {}", path.to_string_lossy()); - let mut file = file::create_file_with_path(&path)?; - - XmlWriter::encode_to_file(& mut file, |w| { - - let a = [ - ("xmlns", NS), - ("version", VERSION), - ("session_id", self.session_id.as_ref()), - ("serial", &format!("{}", self.serial)), - ]; - - w.put_element( - "notification", - Some(&a), - |w| { - { - // snapshot ref - let uri = self.snapshot.uri.to_string(); - let hash = &self.snapshot.hash; - let a = [ - ("uri", uri.as_str()), - ("hash", hash) - ]; - w.put_element( - "snapshot", - Some(&a), - |w| { w.empty() } - )?; - } - - { - // delta refs - for delta in &self.deltas { - let serial = format!("{}", delta.serial); - let uri = delta.uri.to_string(); - let hash = &delta.hash; - let a = [ - ("serial", serial.as_ref()), - ("uri", uri.as_str()), - ("hash", hash) - ]; - w.put_element( - "delta", - Some(&a), - |w| { w.empty() } - )?; - } - } - - Ok(()) - } - ) - })?; - - Ok(()) - } -} - - -//------------ SnapshotRef --------------------------------------------------- - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct SnapshotRef { - file_info: FileInfo -} - -impl SnapshotRef { - pub fn new(file_info: FileInfo) -> Self { - SnapshotRef { file_info } - } -} - -impl Deref for SnapshotRef { - type Target = FileInfo; - - fn deref(&self) -> &FileInfo { - &self.file_info - } -} - - -//------------ DeltaRef ------------------------------------------------------ - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct DeltaRef { - serial: usize, - file_info: FileInfo, -} - -impl DeltaRef { - pub fn new(serial: usize, file_info: FileInfo) -> Self { - DeltaRef { serial, file_info } - } - pub fn serial(&self) -> &usize { - &self.serial - } -} - - -impl Deref for DeltaRef { - type Target = FileInfo; - - fn deref(&self) -> &FileInfo { - &self.file_info - } -} - - -//------------ FileInfo ------------------------------------------------------ - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct FileInfo { - #[serde( - deserialize_with = "ext_serde::de_http_uri", - serialize_with = "ext_serde::ser_http_uri")] - uri: uri::Http, - hash: String, - size: usize -} - -impl FileInfo { - fn new(uri: uri::Http, hash: String, size: usize) -> FileInfo { - FileInfo { uri, hash, size} - } - - pub fn for_path_and_uri( - path: &PathBuf, - uri: uri::Http - ) -> Result { - let bytes = { - use std::io::Read; - - let mut f = File::open(path)?; - let mut bytes = Vec::new(); - f.read_to_end(&mut bytes)?; - bytes - }; - - let size = bytes.len(); - - let hash = { - use crate::util::sha256; - use bytes::Bytes; - - hex::encode(&sha256(&Bytes::from(bytes))) - }; - - Ok(FileInfo::new(uri, hash, size)) - } - - pub fn for_uri( - uri: &uri::Http, - base_uri: &uri::Http, - rrdp_base: &PathBuf - ) -> Result { - let base_string = base_uri.to_string(); - let uri_string = uri.to_string(); - - if ! uri_string.as_str().starts_with(base_string.as_str()) { - Err(Error::NotificationFileError) - } else { - let (_, rel) = uri_string.split_at(base_string.len()); - let mut path = rrdp_base.clone(); - path.push(rel); - - FileInfo::for_path_and_uri(&path, uri.clone()) - } - } - - pub fn for_path( - path: &PathBuf, - base_uri: &uri::Http, - rrdp_base: &PathBuf - ) -> Result { - let relative = path.strip_prefix(rrdp_base) - .map_err(|_| Error::UriConfigError)?.to_string_lossy(); - let base_uri = base_uri.to_string(); - let uri = uri::Http::from_string( - format!("{}{}", base_uri, relative) - ).map_err(|_| Error::UriConfigError)?; - - FileInfo::for_path_and_uri(path, uri) - } - - -} - - -//------------ NotificationBuilder ------------------------------------------- - -#[derive(Default)] -pub struct NotificationBuilder { - serial: Option, - session_id: Option, - snapshot: Option, - deltas: Vec -} - -impl NotificationBuilder { - pub fn new() -> Self { - Self::default() - } - - pub fn with_serial(&mut self, serial: usize) { - self.serial = Some(serial); - } - - pub fn with_session_id(&mut self, session_id: String) { - self.session_id = Some(session_id); - } - - pub fn with_snapshot(&mut self, snapshot: SnapshotRef) { - self.snapshot = Some(snapshot); - } - - pub fn with_deltas(&mut self, deltas: Vec) { - self.deltas = deltas; - } - - pub fn add_delta(&mut self, delta: DeltaRef) { - self.deltas.push(delta); - } - - pub fn add_delta_to_start(&mut self, delta: DeltaRef) { - self.deltas.insert(0, delta); - } - - /// Keeps at least two deltas, and beyond that only if the size is - /// smaller than the snapshot. - /// - /// Note we may add something to exclude old deltas later, if we find - /// that e.g. access to old deltas is very infrequent and excluding - /// them would shrink the notification file size. - fn curate_deltas(&mut self) { - let size_snapshot = match &self.snapshot { - Some(snapshot) => snapshot.size, - None => 0 - }; - let mut total_deltas = 0; - let mut count = 0; - - self.deltas.retain(|d| { - count += 1; - total_deltas += d.size; - count <= 2 || total_deltas < size_snapshot - }) - - } - - - /// Builds the notification, panics if any of the options are not set. - /// This can only happen if there is a bug. - pub fn build(mut self) -> Notification { - self.curate_deltas(); - Notification { - serial: self.serial.unwrap(), - session_id: self.session_id.unwrap(), - snapshot: self.snapshot.unwrap(), - deltas: self.deltas - } - } -} - - -//------------ Error --------------------------------------------------------- - -#[derive(Debug, Display)] -pub enum Error { - #[display(fmt="{}", _0)] - IoError(io::Error), - - #[display(fmt="{}", _0)] - RecursorError(RecursorError), - - #[display(fmt="{}", _0)] - UriError(uri::Error), - - #[display(fmt="{}", _0)] - XmlReaderErr(XmlReaderErr), - - #[display(fmt="{}", _0)] - AttributesError(AttributesError), - - #[display(fmt="{}", _0)] - ParseIntError(ParseIntError), - - #[display(fmt="Error with notification file.")] - NotificationFileError, - - #[display(fmt="Error with uri config.")] - UriConfigError -} - -impl From for Error { - fn from(e: io::Error) -> Self { - Error::IoError(e) - } -} - -impl From for Error { - fn from(e: RecursorError) -> Self { - Error::RecursorError(e) - } -} - -impl From for Error { - fn from(e: uri::Error) -> Self { Error::UriError(e) } -} - -impl From for Error { - fn from(e: XmlReaderErr) -> Self { - Error::XmlReaderErr(e) - } -} - -impl From for Error { - fn from(e: AttributesError) -> Self { - Error::AttributesError(e) - } -} - -impl From for Error { - fn from(e: ParseIntError) -> Self { Error::ParseIntError(e) } -} \ No newline at end of file diff --git a/src/eventsourcing/es_example.rs b/src/eventsourcing/es_example.rs new file mode 100644 index 00000000..b5a65b99 --- /dev/null +++ b/src/eventsourcing/es_example.rs @@ -0,0 +1,425 @@ +//! Example implementation using the eventsourcing module. +//! +//! Goal is two-fold: document using a simple domain, and test the module. +//! +use super::*; + + +//------------ InitPersonEvent ----------------------------------------------- + +/// Every aggregate defines their own initialisation event. This is the first +/// event stored for an instance. +/// +/// Here we define a type wrapping around the generic StoredEvent, so we only +/// need to define the unique initialisation details. +type InitPersonEvent = StoredEvent; + +impl InitPersonEvent { + pub fn init(id: &AggregateId, name: &str) -> Self { + StoredEvent::new(id, 0, InitPersonDetails { name: name.to_string()}) + } +} + +#[derive(Clone, Deserialize, Serialize)] +struct InitPersonDetails { + pub name: String +} + + +//------------ InitPersonEvent ----------------------------------------------- + +/// Every aggregate defines their own set of events - i.e. state changes. The +/// state of an aggregate can only change when events are applied. And events +/// cannot have side effects. If they did, then replaying events would become +/// problematic. +/// +/// Here we make a type alias wrapped around the generic StoredEvent and +/// include an enum with event details specific for Persons. Furthermore we +/// provide an implementation for this type alias so that we can have some +/// convenience functions for creating these events. +type PersonEvent = StoredEvent; + +#[derive(Clone, Deserialize, Serialize)] +enum PersonEventDetails { + NameChanged(String), + HadBirthday +} + +impl PersonEvent { + pub fn had_birthday(p: &Person) -> Self { + StoredEvent::new(p.id(), p.version, PersonEventDetails::HadBirthday) + } + + pub fn name_changed(p: &Person, name: String) -> Self { + StoredEvent::new( + p.id(), + p.version, + PersonEventDetails::NameChanged(name)) + } +} + + +//------------ PersonCommand ------------------------------------------------- + +/// In order to change an aggregate a command is sent to it. The aggregate +/// will then validate the command and if there are no issues, it will return +/// a list (vec) of events that may be applied. This process in itself does +/// not change any state, the state of the aggregate is only changed when +/// those events are applied. +/// +/// Commands are not recorded. Only the resulting events are. For this reason +/// commands may have side-effects: e.g. write something to disk, send an +/// email, etc. +/// +/// Here we define a type wrapping around the generic SentCommand, so we only +/// need to provide an enum with specific command details. We also have an +/// implementation for this type alias providing some convenience methods. +type PersonCommand = SentCommand; + +#[derive(Clone, Deserialize, Serialize)] +enum PersonCommandDetails { + ChangeName(String), + GoAroundTheSun +} + +impl CommandDetails for PersonCommandDetails { + type Event = PersonEvent; +} + +impl PersonCommand { + + pub fn go_around_sun(id: &AggregateId, version: Option) -> Self { + Self::new(id, version, PersonCommandDetails::GoAroundTheSun) + } + + pub fn change_name(id: &AggregateId, version: Option, s: &str) -> Self { + let details = PersonCommandDetails::ChangeName(s.to_string()); + Self::new(id, version, details) + } +} + +//------------ PersonError --------------------------------------------------- + +/// Errors specific to the Person aggregate, should only ever be returned when +/// applying a command that does not validate. +#[derive(Clone, Debug, Display)] +enum PersonError { + #[display(fmt = "No person can live longer than 255 years")] + TooOld +} + +impl std::error::Error for PersonError {} + + +//------------ PersonResult -------------------------------------------------- + +/// A shorthand for the result type returned by the process_command function +/// of the Person aggregate. +type PersonResult = Result, PersonError>; + + +//------------ Person ------------------------------------------------------ + +/// Defines a person object. Persons have a name and an age. +/// +#[derive(Clone, Deserialize, Serialize)] +struct Person { + /// The id is needed when generating events. + id: AggregateId, + + /// The version of for this particular Person. Versions + /// are incremented whenever events are applied. They are + /// used to store those and apply events in the correct + /// sequence, as well as to detect concurrency issues when + /// a command is sent. + version: u64, + + name: String, + age: u8 +} + +impl Person { + pub fn id(&self) -> &AggregateId { &self.id } + pub fn version(&self) -> u64 { self.version } + pub fn name(&self) -> &String { &self.name } + pub fn age(&self) -> u8 { self.age } +} + +impl Aggregate for Person { + type Command = PersonCommand; + type Event = PersonEvent; + type InitEvent = InitPersonEvent; + type Error = PersonError; + + fn init(event: InitPersonEvent) -> Result { + let (id, _version, init) = event.unwrap(); + Ok(Person { + id, version: 1, name: init.name, age: 0 + }) + } + + fn version(&self) -> u64 { + self.version + } + + fn apply(&mut self, event: PersonEvent) { + match event.into_details() { + PersonEventDetails::NameChanged(name) => { self.name = name }, + PersonEventDetails::HadBirthday => { self.age += 1 } + } + self.version += 1; + } + + fn process_command(&self, command: Self::Command) -> PersonResult { + match command.into_details() { + PersonCommandDetails::ChangeName(name) => { + let event = PersonEvent::name_changed(&self, name); + Ok(vec![event]) + }, + PersonCommandDetails::GoAroundTheSun => { + if self.age == 255 { + Err(PersonError::TooOld) + } else { + let event = PersonEvent::had_birthday(&self); + Ok(vec![event]) + } + } + } + } +} + + +/// This type is responsible for managing all persons. I.e. creating new +/// instances, returning a reference for reading, dispatching commands, and +/// storing them. +/// +/// It is generic over the keystore used. +/// +// Compiler does not see this is used in test +#[allow(dead_code)] +struct PersonManager { + /// Here we use a cache to make matters complicated^H interesting.. + /// Of course this may not always be the best idea.. + cache: RwLock>>, + + /// The keystore where snapshots and events may be retrieved and stored. + store: S +} + +impl PersonManager { + + // Compiler does not see this is used in test + #[allow(dead_code)] + pub fn new(store: S) -> Self { + let values = RwLock::new(HashMap::new()); + PersonManager { + cache: values, store + } + } + + // Compiler does not see this is used in test + #[allow(dead_code)] + fn update_cache( + &self, + id: &AggregateId, + mut force: bool + ) -> Result<(), PersonManagerError> { + + let mut cache = self.cache.write().unwrap(); + + let mut has_key = cache.contains_key(id); + + if ! has_key { + let init_key = S::key_for_event(0); + if let Some(init) = self.store.get(id, &init_key)? { + let mut agg = Person::init(init)?; + + cache.insert(id.clone(), Arc::new(agg)); + force = true; + has_key = true; + } + } + + if has_key && force { + // We MUST have an entry now + let arc = cache.get_mut(id).unwrap(); + let agg = Arc::make_mut(arc); + + loop { + let ver = agg.version(); + let key = S::key_for_event(ver); + if let Some(event) = self.store.get(id, &key)? { + agg.apply(event) + } else { + break + } + } + } + + Ok(()) + } + + + /// Get a reference to the latest version of the aggregate. + // Compiler does not see this is used in test + #[allow(dead_code)] + fn get_latest( + &self, + id: &AggregateId + ) -> Result>, PersonManagerError> { + self.update_cache(id, false)?; + Ok(self.cache.read().unwrap().get(id).cloned()) + } + + // Compiler does not see this is used in test + #[allow(dead_code)] + fn create( + &self, + id: &AggregateId, + event: InitPersonEvent + ) -> Result<(), PersonManagerError> { + self.update_cache(id, true)?; + + let mut cache = self.cache.write().unwrap(); + if cache.contains_key(id) { + Err(PersonManagerError::AggregateAlreadyExists) + } else { + let key = S::key_for_event(0); + self.store.store(id, &key, &event)?; + + let agg = Person::init(event)?; + + cache.insert(id.clone(), Arc::new(agg)); + Ok(()) + } + } + + /// Apply a command to the latest aggregate, save the events and return + /// the updated aggregate. + // Compiler does not see this is used in test + #[allow(dead_code)] + fn apply( + &self, + command: PersonCommand + ) -> Result<(), PersonManagerError> { + let id = command.id().clone(); + self.update_cache(&id, true)?; + + let mut cache = self.cache.write().unwrap(); + + match cache.get_mut(&id) { + None => Err(PersonManagerError::AggregateDoesNotExist), + Some(agg) => { + + let agg = Arc::make_mut(agg); + + if let Some(version) = command.version() { + if version != agg.version() { + // TODO check conflicts + return Err(PersonManagerError::ConcurrentModification) + } + } + + let events = agg.process_command(command)?; + + for e in events { + let key = S::key_for_event(e.version()); + self.store.store(&id, &key, &e)?; + + agg.apply(e); + } + + Ok(()) + } + } + } +} + + +//------------ PersonManagerError -------------------------------------------- + +// Compiler does not see this is used in test +#[allow(dead_code)] +#[derive(Debug, Display)] +enum PersonManagerError { + #[display(fmt = "Aggregate does not exist")] + AggregateDoesNotExist, + + #[display(fmt = "Aggregate already exists")] + AggregateAlreadyExists, + + #[display(fmt = "Concurrent modification. Command rejected")] + ConcurrentModification, + + #[display(fmt = "{}", _0)] + PersonError(PersonError), + + #[display(fmt = "{}", _0)] + KeyStoreError(KeyStoreError), +} + +impl From for PersonManagerError { + fn from(e: PersonError) -> Self { PersonManagerError::PersonError(e) } +} + +impl From for PersonManagerError { + fn from(e: KeyStoreError) -> Self { PersonManagerError::KeyStoreError(e) } +} + +//------------ Tests --------------------------------------------------------- + +#[cfg(test)] +mod tests { + + use super::*; + use crate::util::test; + + #[test] + fn test() { + test::test_with_tmp_dir(|d| { + + let storage = DiskKeyStore::new(d.clone()); + let manager = PersonManager::new(storage); + + let id_alice = AggregateId::from("alice"); + let alice_init = InitPersonEvent::init(&id_alice, "alice smith"); + + manager.create(&id_alice, alice_init).unwrap(); + + let alice = manager.get_latest(&id_alice).unwrap().unwrap(); + assert_eq!(alice.name(), "alice smith"); + assert_eq!(alice.age(), 0); + + let mut age = 0; + loop { + manager.apply( + PersonCommand::go_around_sun(&id_alice, None) + ).unwrap(); + age = age + 1; + if age == 21 { + break + } + } + + let alice = manager.get_latest(&id_alice).unwrap().unwrap(); + assert_eq!(alice.name(), "alice smith"); + assert_eq!(alice.age(), 21); + + manager.apply( + PersonCommand::change_name(&id_alice, Some(22), "alice smith-doe") + ).unwrap(); + + let alice = manager.get_latest(&id_alice).unwrap().unwrap(); + assert_eq!(alice.name(), "alice smith-doe"); + assert_eq!(alice.age(), 21); + + // Should read state from disk + let storage = DiskKeyStore::new(d); + let manager = PersonManager::new(storage); + + let alice = manager.get_latest(&id_alice).unwrap().unwrap(); + assert_eq!(alice.name(), "alice smith-doe"); + assert_eq!(alice.age(), 21); + }) + } +} \ No newline at end of file diff --git a/src/eventsourcing/mod.rs b/src/eventsourcing/mod.rs new file mode 100644 index 00000000..34938d1e --- /dev/null +++ b/src/eventsourcing/mod.rs @@ -0,0 +1,367 @@ +//! Event sourcing support for Krill + +mod es_example; // Example implementation and tests. + +use std::any::Any; +use std::collections::HashMap; +use std::fmt; +use std::fmt::Display; +use std::fs; +use std::fs::File; +use std::io; +use std::io::Write; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::RwLock; +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json; +use crate::util::file; + + +//------------ Storable ------------------------------------------------------ + +pub trait Storable: Clone + Serialize + DeserializeOwned + Sized {} +impl Storable for T { } + + +//------------ AggregateId --------------------------------------------------- + +#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +pub struct AggregateId(String); + +impl From<&str> for AggregateId { + fn from(s: &str) -> Self { + AggregateId(s.to_string()) + } +} + +impl From for AggregateId { + fn from(s: String) -> Self { + AggregateId(s) + } +} + +impl AsRef for AggregateId { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl Display for AggregateId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +//------------ Aggregate ----------------------------------------------------- + +pub trait Aggregate: Storable { + + type Command: Command; + type Event: Event; + type InitEvent: Event; + type Error: std::error::Error; + + /// Creates a new instance. Expects an event with data needed to + /// initialise the instance. Typically this means that a specific + /// 'create' event is passed, with all the needed data, or just an empty + /// marker if no data is needed. Implementations must return an error in + /// case the instance cannot be created. + fn init(event: Self::InitEvent) -> Result; + + /// Returns the current version of the aggregate. + fn version(&self) -> u64; + + /// Moves this, and applies the event to this. This MUST not result in + /// any errors. Applying the event just updates data and is side-effect + /// free. + /// + /// Note that both self and the event are moved. This is done because we + /// want to enable moving data into the new aggregate without the need for + /// additional allocations. + fn apply(&mut self, event: Self::Event); + + /// Processes a command. I.e. validate the command, and return a list of + /// events that will result in the desired new state, but do not apply + /// these event here. The command processing must be side-effect free. + /// + /// The command is moved, because we want to enable moving its data + /// without reallocating. + fn process_command(&self, command: Self::Command) -> Result, Self::Error>; +} + + +//------------ Event -------------------------------------------------------- + +pub trait Event: Storable { + /// Identifies the aggregate, useful when storing and retrieving the event. + fn id(&self) -> &AggregateId; + + /// The version of the aggregate that this event updates. + /// In other words, an aggregate that is currently at version x, will get + /// version x + 1, when the event for version x is applied. + fn version(&self) -> u64; +} + +#[derive(Clone, Deserialize, Serialize)] +pub struct StoredEvent { + id: AggregateId, + version: u64, + #[serde(deserialize_with = "E::deserialize")] + details: E +} + +impl StoredEvent { + pub fn new(id: &AggregateId, version: u64, event: E) -> Self { + StoredEvent { id: id.clone(), version, details: event } + } + + pub fn details(&self) -> &E { & self.details } + + pub fn into_details(self) -> E { self.details } + + /// Return the parts of this event. + pub fn unwrap(self) -> (AggregateId, u64, E) { + (self.id, self.version, self.details) + } +} + +impl Event for StoredEvent { + fn id(&self) -> &AggregateId { + &self.id + } + + fn version(&self) -> u64 { + self.version + } +} + +//------------ Command ------------------------------------------------------- + +/// Commands are used to send an intent to change an aggregate. +/// +/// Think of this as the data container for your update API, plus some +/// meta-data to ensure that the command is sent to the right instance of an +/// Aggregate, and that concurrency issues are handled. +pub trait Command: Storable { + /// Identify the type of event returned by the aggregate that uses this + /// command. This is needed because we may need to check whether a + /// command conflicts with recent events. + type Event: Event; + + /// Identifies the aggregate, useful when storing and retrieving the event. + fn id(&self) -> &AggregateId; + + /// The version of the aggregate that this command updates. If this + /// command should update whatever the latest version happens to be, then + /// use None here. + fn version(&self) -> Option; + + /// In case of concurrent processing of commands, the aggregate may be + /// outdated when a command is applied. In such cases this method expects + /// the list of events that happened since the ['affected_version'] and + /// will return whether there is a conflict. If there is no conflict that + /// the command may be applied again. + /// + /// Note that this defaults to true, which is the safe choice when in + /// doubt. If you choose to implement this, then you will also need to + /// implement the ['set_affected_version'] function. + fn conflicts(&self, _events: &[Self::Event]) -> bool { true } +} + + +//------------ SentCommand --------------------------------------------------- + +/// Convenience wrapper so that implementations can just implement +/// ['CommandDetails'] and leave the id and version boilerplate. +#[derive(Clone, Deserialize, Serialize)] +pub struct SentCommand { + id: AggregateId, + version: Option, + #[serde(deserialize_with = "C::deserialize")] + details: C +} + +impl Command for SentCommand { + type Event = C::Event; + + fn id(&self) -> &AggregateId { + &self.id + } + + fn version(&self) -> Option { + self.version + } +} + +impl SentCommand { + pub fn new(id: &AggregateId, version: Option, details: C) -> Self { + SentCommand { id: id.clone(), version, details } + } + + pub fn into_details(self) -> C { self.details } +} + + +//------------ CommandDetails ------------------------------------------------ + +/// Implement this for an enum with CommandDetails, so you you can reuse the +/// id and version boilerplate from ['SentCommand']. +pub trait CommandDetails: Storable { + type Event: Event; +} + + +//------------ KeyStore ------------------------------------------------------ + +/// Generic KeyStore for AggregateManager +pub trait KeyStore { + + type Key; + + fn key_for_snapshot() -> Self::Key; + fn key_for_event(version: u64) -> Self::Key; + + /// Returns whether a key already exists. + fn has_key(&self, id: &AggregateId, key: &Self::Key) -> bool; + + fn has_aggregate(&self, id: &AggregateId) -> bool; + + fn aggregates(&self) -> Vec; // Use Iterator? + + /// Throws an error if the key already exists. + fn store( + &self, + id: &AggregateId, + key: &Self::Key, + value: &V + ) -> Result<(), KeyStoreError>; + + /// Get the value for this key, if any exists. + fn get( + &self, + id: &AggregateId, + key: &Self::Key + ) -> Result, KeyStoreError>; +} + + +//------------ KeyStoreError ------------------------------------------------- + +/// This type defines possible Errors for KeyStore +#[derive(Debug, Display)] +pub enum KeyStoreError { + #[display(fmt = "{}", _0)] + IoError(io::Error), + + #[display(fmt = "{}", _0)] + JsonError(serde_json::Error), + + #[display(fmt = "Key already exists: {}", _0)] + KeyExists(String) +} + +impl From for KeyStoreError { + fn from(e: io::Error) -> Self { KeyStoreError::IoError(e) } +} + +impl From for KeyStoreError { + fn from(e: serde_json::Error) -> Self { KeyStoreError::JsonError(e) } +} + +impl std::error::Error for KeyStoreError { } + + +//------------ DiskKeyStore -------------------------------------------------- + +/// This type can store and retrieve values to/from disk, using json +/// serialization. +pub struct DiskKeyStore { + dir: PathBuf +} + +impl KeyStore for DiskKeyStore { + type Key = PathBuf; + + fn key_for_snapshot() -> Self::Key { + PathBuf::from("snapshot.json") + } + + fn key_for_event(version: u64) -> Self::Key { + PathBuf::from(format!("delta-{}.json", version)) + } + + fn has_key(&self, id: &AggregateId, key: &Self::Key) -> bool { + self.file_path(id, key).exists() + } + + fn has_aggregate(&self, _id: &AggregateId) -> bool { + unimplemented!() + } + + fn aggregates(&self) -> Vec { + let mut res: Vec = Vec::new(); + for d in fs::read_dir(&self.dir).unwrap() { + let full_path = d.unwrap().path(); + let path = full_path.file_name().unwrap(); + res.push(AggregateId::from(path.to_string_lossy().as_ref())); + } + res + } + + fn store( + &self, + id: &AggregateId, + key: &Self::Key, + value: &V + ) -> Result<(), KeyStoreError> { + if self.has_key(id, key) { + Err(KeyStoreError::KeyExists(key.to_string_lossy().to_string())) + } else { + let mut f = file::create_file_with_path(&self.file_path(id, key))?; + let json = serde_json::to_string(value)?; + f.write_all(json.as_ref())?; + Ok(()) + } + } + + fn get( + &self, + id: &AggregateId, + key: &Self::Key + ) -> Result, KeyStoreError> { + if self.has_key(id, key) { + let f = File::open(self.file_path(id, key))?; + let v: V = serde_json::from_reader(f)?; + Ok(Some(v)) + } else { + Ok(None) + } + } +} + +impl DiskKeyStore { + pub fn new(dir: PathBuf) -> Self { + DiskKeyStore { dir } + } + + pub fn under_work_dir( + work_dir: &PathBuf, + name_space: &str + ) -> Result { + let mut path = work_dir.clone(); + path.push(name_space); + if ! path.is_dir() { + fs::create_dir_all(&path)?; + } + Ok(Self::new(path)) + } + + fn file_path(&self, id: &AggregateId, key: &::Key) -> PathBuf { + let mut file_path = self.dir.clone(); + file_path.push(id.to_string()); + file_path.push(key); + file_path + } +} diff --git a/src/krillc/data.rs b/src/krillc/data.rs index 29fc5a16..79c897ff 100644 --- a/src/krillc/data.rs +++ b/src/krillc/data.rs @@ -190,7 +190,9 @@ pub struct CmsAuthData { /// /api/v1/publishers/{handle} #[derive(Clone, Debug, Deserialize, Serialize)] pub struct PublisherDetails { - publisher_handle: String, + handle: String, + + retired: bool, #[serde( deserialize_with = "ext_serde::de_rsync_uri", @@ -204,8 +206,8 @@ pub struct PublisherDetails { } impl PublisherDetails { - pub fn publisher_handle(&self) -> &str { - &self.publisher_handle + pub fn handle(&self) -> &str { + &self.handle } pub fn identity_cert(&self) -> Option<&IdCert> { match self.cms_auth { @@ -213,6 +215,7 @@ impl PublisherDetails { Some(ref details) => Some(&details.id_cert) } } + pub fn retired(&self) -> bool { self.retired } } impl PartialEq for PublisherDetails { @@ -237,7 +240,7 @@ impl Report for PublisherDetails { let mut res = String::new(); res.push_str("handle: "); - res.push_str(self.publisher_handle.as_str()); + res.push_str(self.handle.as_str()); res.push_str("\n"); res.push_str("base uri: "); diff --git a/src/krillc/mod.rs b/src/krillc/mod.rs index eda3d50a..370d8b5a 100644 --- a/src/krillc/mod.rs +++ b/src/krillc/mod.rs @@ -4,7 +4,7 @@ pub mod options; use std::io; use bytes::Bytes; use rpki::uri; -use crate::api::publishers::Publisher; +use crate::api::publisher_data::PublisherRequest; use crate::krillc::data::{ ApiResponse, PublisherDetails, @@ -77,7 +77,7 @@ impl KrillClient { Ok(ApiResponse::PublisherList(list)) }, PublishersCommand::Add(add) => { - let pbl = Publisher::new( + let pbl = PublisherRequest::new( add.handle, add.token, add.base_uri, @@ -149,7 +149,7 @@ impl KrillClient { } } - fn add_publisher(&self, pbl: Publisher) -> Result { + fn add_publisher(&self, pbl: PublisherRequest) -> Result { httpclient::post_json( &self.resolve_uri("api/v1/publishers"), pbl, diff --git a/src/krilld/auth.rs b/src/krilld/auth.rs index 0353a69e..c31bcad3 100644 --- a/src/krilld/auth.rs +++ b/src/krilld/auth.rs @@ -1,39 +1,38 @@ //! Authorization for the API - use std::sync::{Arc, RwLock, RwLockReadGuard}; -use actix_web::{HttpResponse, HttpRequest, Result}; +use actix_web::{Form, HttpResponse, HttpRequest, Result}; use actix_web::http::HeaderMap; use actix_web::middleware::{Middleware, Started}; use actix_web::middleware::identity::RequestIdentity; - use crate::krilld::krillserver::KrillServer; -use actix_web::Form; +use crate::eventsourcing::DiskKeyStore; + const ADMIN_API_PATH: &str = "/api/"; const PUBLICATION_API_PATH: &str = "/publication/"; pub struct CheckAuthorisation; -impl Middleware>> for CheckAuthorisation { +impl Middleware>>> for CheckAuthorisation { fn start( &self, - req: &HttpRequest>> + req: &HttpRequest>>> ) -> Result { if req.identity() == Some("admin".to_string()) { return Ok(Started::Done) } - let server: RwLockReadGuard = req.state().read().unwrap(); + let server: RwLockReadGuard> = req.state().read().unwrap(); let mut allowed = true; let token_opt = Self::extract_token(req.headers()); if req.path().starts_with(ADMIN_API_PATH) { - allowed = server.allow_api(token_opt) + allowed = server.is_api_allowed(token_opt) } else if req.path().starts_with(PUBLICATION_API_PATH) { let handle_opt = Self::extract_publication_handle(req.path()); - allowed = server.allow_publication_api(handle_opt, token_opt); + allowed = server.is_publication_api_allowed(handle_opt, token_opt); } if allowed { @@ -90,7 +89,7 @@ impl Authorizer { } } - pub fn api_allowed(&self, token_opt: Option) -> bool { + pub fn is_api_allowed(&self, token_opt: Option) -> bool { match token_opt { None => false, Some(secret) => self.krill_auth_token == secret @@ -105,11 +104,11 @@ pub struct Login { #[allow(clippy::needless_pass_by_value)] pub fn login_page( - req: HttpRequest>>, + req: HttpRequest>>>, form: Form ) -> HttpResponse { - let server: RwLockReadGuard = req.state().read().unwrap(); - if server.allow_api(Some(form.token.clone())) { + let server: RwLockReadGuard> = req.state().read().unwrap(); + if server.is_api_allowed(Some(form.token.clone())) { req.remember("admin".to_string()); HttpResponse::Found().header("location", "/api/v1/publishers").finish() } else { diff --git a/src/krilld/endpoints.rs b/src/krilld/endpoints.rs index 8bb76b18..5b12ce12 100644 --- a/src/krilld/endpoints.rs +++ b/src/krilld/endpoints.rs @@ -1,26 +1,30 @@ //! Process requests received, delegate, and wrap up the responses. -use std::error; use std::sync::{RwLockReadGuard, RwLockWriteGuard}; use actix_web::{HttpResponse, ResponseError}; use actix_web::http::StatusCode; use serde::Serialize; -use crate::api::publishers; -use crate::api::publication; -use crate::krilld::http::server::{HttpRequest, PublisherHandle}; +use crate::api::publisher_data; +use crate::api::publisher_data::PublisherHandle; +use crate::api::publication_data; +use crate::eventsourcing::DiskKeyStore; +use crate::krilld::http::server::HttpRequest; use crate::krilld::krillserver::{self, KrillServer}; use crate::krilld::pubd; +use crate::krilld::pubd::publishers::PublisherError; +use crate::krilld::pubd::repo::RrdpServerError; +use crate::remote::cmsproxy; use crate::remote::sigmsg::SignedMessage; //------------ Support Functions --------------------------------------------- /// Returns a server in a read lock -pub fn ro_server(req: &HttpRequest) -> RwLockReadGuard { +pub fn ro_server(req: &HttpRequest) -> RwLockReadGuard> { req.state().read().unwrap() } /// Returns a server in a write lock -pub fn rw_server(req: &HttpRequest) -> RwLockWriteGuard { +pub fn rw_server(req: &HttpRequest) -> RwLockWriteGuard> { req.state().write().unwrap() } @@ -67,7 +71,10 @@ pub fn publishers(req: &HttpRequest) -> HttpResponse { Err(e) => server_error(&Error::ServerError(e)), Ok(publishers) => { render_json( - publishers::PublisherList::from(&publishers, "/api/v1/publishers") + publisher_data::PublisherList::build( + &publishers, + "/api/v1/publishers" + ) ) } } @@ -78,7 +85,7 @@ pub fn publishers(req: &HttpRequest) -> HttpResponse { #[allow(clippy::needless_pass_by_value)] pub fn add_publisher( req: HttpRequest, - pbl: publishers::Publisher + pbl: publisher_data::PublisherRequest ) -> HttpResponse { let mut server = rw_server(&req); match server.add_publisher(pbl) { @@ -94,10 +101,8 @@ pub fn remove_publisher( req: HttpRequest, handle: PublisherHandle ) -> HttpResponse { - match rw_server(&req).remove_publisher(handle) { + match rw_server(&req).remove_publisher(&handle) { Ok(()) => api_ok(), - Err(krillserver::Error::PublisherStore( - pubd::Error::UnknownPublisher(_))) => api_ok(), Err(e) => server_error(&Error::ServerError(e)) } } @@ -109,11 +114,11 @@ pub fn publisher_details( handle: PublisherHandle ) -> HttpResponse { let server = ro_server(&req); - match server.publisher(handle) { + match server.publisher(&handle) { Ok(None) => api_not_found(), Ok(Some(publisher)) => { render_json( - publishers::PublisherDetails::from( + publisher_data::PublisherDetails::from( &publisher, "/api/v1/publishers", server.service_base_uri()) @@ -130,19 +135,14 @@ pub fn repository_response( req: HttpRequest, handle: PublisherHandle ) -> HttpResponse { - match ro_server(&req).repository_response(handle) { + match ro_server(&req).repository_response(&handle) { Ok(res) => { HttpResponse::Ok() .content_type("application/xml") .body(res.encode_vec()) }, - Err(krillserver::Error::PublisherStore - (pubd::Error::UnknownPublisher(_))) => { - api_not_found() - }, - Err(e) => { - server_error(&Error::ServerError(e)) - } + + Err(e) => server_error(&Error::ServerError(e)) } } @@ -156,8 +156,8 @@ pub fn handle_rfc8181_request( msg: SignedMessage, handle: PublisherHandle ) -> HttpResponse { - let mut server: RwLockWriteGuard = rw_server(&req); - match server.handle_rfc8181_request(&msg, handle.as_ref()) { + let mut server: RwLockWriteGuard> = rw_server(&req); + match server.handle_rfc8181_request(&msg, &handle) { Ok(captured) => { HttpResponse::build(StatusCode::OK) .content_type("application/rpki-publication") @@ -173,10 +173,10 @@ pub fn handle_rfc8181_request( #[allow(clippy::needless_pass_by_value)] pub fn handle_delta( req: HttpRequest, - delta: publication::PublishDelta, + delta: publication_data::PublishDelta, handle: PublisherHandle ) -> HttpResponse { - match rw_server(&req).handle_delta(delta, handle.as_ref()) { + match rw_server(&req).handle_delta(delta, &handle) { Ok(()) => api_ok(), Err(e) => server_error(&Error::ServerError(e)) } @@ -188,7 +188,7 @@ pub fn handle_list( req: HttpRequest, handle: PublisherHandle ) -> HttpResponse { - match ro_server(&req).handle_list(handle.as_ref()) { + match ro_server(&req).handle_list(&handle) { Ok(list) => render_json(list), Err(e) => server_error(&Error::ServerError(e)) } @@ -208,6 +208,7 @@ pub fn current_snapshot_json(req: &HttpRequest) -> HttpResponse { //------------ Error --------------------------------------------------------- #[derive(Debug, Display)] +#[allow(clippy::large_enum_variant)] pub enum Error { #[display(fmt = "{}", _0)] ServerError(krillserver::Error), @@ -229,7 +230,7 @@ trait ErrorToCode { fn code(&self) -> usize; } -impl error::Error for Error { +impl std::error::Error for Error { fn description(&self) -> &str { "Error happened" } @@ -240,17 +241,7 @@ impl ErrorToStatus for Error { match self { Error::ServerError(e) => e.status(), Error::JsonError(_) => StatusCode::BAD_REQUEST, - Error::PublisherRequestError => StatusCode::BAD_REQUEST, - } - } -} - -impl ErrorToCode for Error { - fn code(&self) -> usize { - match self { - Error::ServerError(e) => e.code(), - Error::JsonError(_) => 1001, - Error::PublisherRequestError => 1002, + Error::PublisherRequestError => StatusCode::BAD_REQUEST } } } @@ -258,33 +249,20 @@ impl ErrorToCode for Error { impl ErrorToStatus for krillserver::Error { fn status(&self) -> StatusCode { match self { - krillserver::Error::CmsProxy(_) => StatusCode::BAD_REQUEST, - krillserver::Error::PublisherStore(e) => e.status(), - krillserver::Error::Repository(_) => StatusCode::BAD_REQUEST, krillserver::Error::NoIdCert => StatusCode::FORBIDDEN, + krillserver::Error::CmsProxy(e) => e.status(), + krillserver::Error::IoError(_) => StatusCode::INTERNAL_SERVER_ERROR, + krillserver::Error::PubServer(e) => e.status() } } } -impl ErrorToCode for krillserver::Error { - fn code(&self) -> usize { +impl ErrorToStatus for cmsproxy::Error { + fn status(&self) -> StatusCode { match self { - krillserver::Error::PublisherStore(e) => e.code(), - krillserver::Error::Repository(_) => 3002, - krillserver::Error::CmsProxy(_) => 3003, - krillserver::Error::NoIdCert => 2001, - - } - } -} - -impl ErrorToCode for pubd::Error { - fn code(&self) -> usize { - match self { - pubd::Error::ForwardSlashInHandle(_) => 1004, - pubd::Error::DuplicatePublisher(_) => 1005, - pubd::Error::UnknownPublisher(_) => 1006, - _ => 3001 + cmsproxy::Error::ValidationError(_) => StatusCode::FORBIDDEN, + cmsproxy::Error::MessageError(_) => StatusCode::BAD_REQUEST, + cmsproxy::Error::ResponderError(_) => StatusCode::INTERNAL_SERVER_ERROR } } } @@ -292,17 +270,107 @@ impl ErrorToCode for pubd::Error { impl ErrorToStatus for pubd::Error { fn status(&self) -> StatusCode { match self { - pubd::Error::ForwardSlashInHandle(_) => - StatusCode::BAD_REQUEST, - pubd::Error::DuplicatePublisher(_) => - StatusCode::BAD_REQUEST, - pubd::Error::UnknownPublisher(_) => - StatusCode::BAD_REQUEST, - _ => StatusCode::INTERNAL_SERVER_ERROR + pubd::Error::IoError(_) => StatusCode::INTERNAL_SERVER_ERROR, + pubd::Error::InvalidBaseUri => StatusCode::BAD_REQUEST, + pubd::Error::InvalidHandle(_) => StatusCode::BAD_REQUEST, + pubd::Error::ReservedName(_) => StatusCode::BAD_REQUEST, + pubd::Error::DuplicatePublisher(_) => StatusCode::BAD_REQUEST, + pubd::Error::UnknownPublisher(_) => StatusCode::FORBIDDEN, + pubd::Error::ConcurrentModification(_, _) => StatusCode::BAD_REQUEST, + pubd::Error::PublisherError(e) => e.status(), + pubd::Error::RrdpServerError(e) => e.status(), + pubd::Error::KeyStoreError(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } + +} + +impl ErrorToStatus for PublisherError { + fn status(&self) -> StatusCode { + match self { + PublisherError::Deactivated => StatusCode::FORBIDDEN, + PublisherError::VerificationError(_) => StatusCode::FORBIDDEN, } } } +impl ErrorToStatus for RrdpServerError { + fn status(&self) -> StatusCode { + match self { + RrdpServerError::IoError(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + + + +impl ErrorToCode for Error { + fn code(&self) -> usize { + match self { + Error::ServerError(e) => e.code(), + Error::JsonError(_) => 1001, + Error::PublisherRequestError => 1002 + } + } +} + +impl ErrorToCode for krillserver::Error { + fn code(&self) -> usize { + match self { + krillserver::Error::NoIdCert => 2001, + krillserver::Error::CmsProxy(e) => e.code(), + krillserver::Error::IoError(_) => 3001, + krillserver::Error::PubServer(e) => e.code() + } + } +} + +impl ErrorToCode for cmsproxy::Error { + fn code(&self) -> usize { + match self { + cmsproxy::Error::ValidationError(_) => 2001, + cmsproxy::Error::MessageError(_) => 1003, + cmsproxy::Error::ResponderError(_) => 3003 + } + } +} + +impl ErrorToCode for pubd::Error { + fn code(&self) -> usize { + match self { + pubd::Error::IoError(_) => 3001, + pubd::Error::InvalidBaseUri => 2002, + pubd::Error::InvalidHandle(_) => 1004, + pubd::Error::ReservedName(_) => 1007, + pubd::Error::DuplicatePublisher(_) => 1005, + pubd::Error::UnknownPublisher(_) => 1006, + pubd::Error::ConcurrentModification(_, _) => 2003, + pubd::Error::PublisherError(e) => e.code(), + pubd::Error::RrdpServerError(e) => e.code(), + pubd::Error::KeyStoreError(_) => 3001, + } + } +} + +impl ErrorToCode for PublisherError { + fn code(&self) -> usize { + match self { + PublisherError::Deactivated => 2004, + PublisherError::VerificationError(_) => 2005, + } + } +} + +impl ErrorToCode for RrdpServerError { + fn code(&self) -> usize { + match self { + RrdpServerError::IoError(_) => 3001 + } + } +} + + + #[derive(Debug, Serialize)] struct ErrorResponse { code: usize, diff --git a/src/krilld/http/server.rs b/src/krilld/http/server.rs index 691ab18c..f4adf4d6 100644 --- a/src/krilld/http/server.rs +++ b/src/krilld/http/server.rs @@ -3,19 +3,24 @@ //! Here we deal with booting and setup, and once active deal with parsing //! arguments and routing of requests, typically handing off to the //! daemon::api::endpoints functions for processing and responding. -use std::error; +use std::io; use std::fs::File; use std::sync::{Arc, RwLock, RwLockReadGuard}; use actix_web::{pred, fs, server}; use actix_web::{App, FromRequest, HttpResponse}; use actix_web::dev::MessageBody; use actix_web::middleware; +use actix_web::middleware::identity::CookieIdentityPolicy; +use actix_web::middleware::identity::IdentityService; use actix_web::http::{Method, StatusCode}; use bcder::decode; use futures::Future; use openssl::ssl::{SslMethod, SslAcceptor, SslAcceptorBuilder, SslFiletype}; -use crate::api::publication; -use crate::api::publishers; +use crate::api::publication_data; +use crate::api::publisher_data; +use crate::api::publisher_data::PublisherHandle; +use crate::eventsourcing::DiskKeyStore; +use crate::krilld::auth; use crate::krilld::auth::{Authorizer, CheckAuthorisation}; use crate::krilld::config::Config; use crate::krilld::endpoints; @@ -24,22 +29,20 @@ use crate::krilld::krillserver; use crate::krilld::krillserver::KrillServer; use crate::remote::rfc8183; use crate::remote::sigmsg::SignedMessage; -use actix_web::middleware::identity::IdentityService; -use actix_web::middleware::identity::CookieIdentityPolicy; -use krilld::auth; const NOT_FOUND: &[u8] = include_bytes!("../../../ui/dev/html/404.html"); const LOGIN: &[u8] = include_bytes!("../../../ui/dev/html/login.html"); + //------------ PubServerApp -------------------------------------------------- -pub struct PubServerApp(App>>); +pub struct PubServerApp(App>>>); /// # Set up methods /// impl PubServerApp { - pub fn new(server: Arc>) -> Self { + pub fn new(server: Arc>>) -> Self { let mut app = App::with_state(server) .middleware(middleware::Logger::default()) .middleware(IdentityService::new( @@ -106,29 +109,36 @@ impl PubServerApp { PubServerApp(with_statics(app)) } - pub fn create_server(config: &Config) -> Arc> { + pub fn create_server( + config: &Config + ) -> Result>>, Error> { let authorizer = Authorizer::new(&config.auth_token); - let pub_server = match KrillServer::build( + + let pubserver_store = DiskKeyStore::under_work_dir(&config.data_dir, "pubsrv")?; + + let pub_server = KrillServer::build( &config.data_dir, &config.rsync_base, config.service_uri(), &config.rrdp_base_uri, - authorizer - ) { - Err(e) => { - error!("{}", e); - ::std::process::exit(1); - }, - Ok(server) => server - }; - Arc::new(RwLock::new(pub_server)) + authorizer, + pubserver_store + )?; + + Ok(Arc::new(RwLock::new(pub_server))) } /// Used to start the server with an existing executor (for tests) /// /// Note https is not supported in tests. pub fn start(config: &Config) { - let ps = PubServerApp::create_server(config); + let ps = match PubServerApp::create_server(config) { + Ok(server) => server, + Err(e) => { + eprintln!("{}", e); + ::std::process::exit(1); + } + }; server::new(move || PubServerApp::new(ps.clone())) .bind(config.socket_addr()) @@ -139,7 +149,13 @@ impl PubServerApp { /// Used to run the server in blocking mode, from the main method. pub fn run(config: &Config) { - let ps = PubServerApp::create_server(config); + let ps = match PubServerApp::create_server(config) { + Ok(server) => server, + Err(e) => { + eprintln!("{}", e); + ::std::process::exit(1); + } + }; let server = server::new(move || PubServerApp::new(ps.clone())); @@ -214,7 +230,8 @@ impl PubServerApp { // https://github.com/actix/actix-website/blob/master/content/docs/static-files.md // https://www.keycdn.com/blog/http-cache-headers fn serve_rrdp_files(req: &HttpRequest) -> HttpResponse { - let server: RwLockReadGuard = req.state().read().unwrap(); + let server: RwLockReadGuard> = req.state().read() + .unwrap(); match req.match_info().get("path") { Some(path) => { @@ -277,7 +294,7 @@ impl FromRequest for SignedMessage { /// PublisherRequestChoice, which contains either an /// rfc8183::PublisherRequest, or an API publisher request (no ID certs and /// CMS etc). -impl FromRequest for publishers::Publisher { +impl FromRequest for publisher_data::PublisherRequest { type Config = (); type Result = Box>; @@ -288,7 +305,7 @@ impl FromRequest for publishers::Publisher { Box::new(MessageBody::new(req) .from_err() .and_then(|bytes| { - let p: publishers::Publisher = + let p: publisher_data::PublisherRequest = serde_json::from_reader(bytes.as_ref()) .map_err(Error::JsonError)?; Ok(p) @@ -300,10 +317,6 @@ impl FromRequest for publishers::Publisher { //------------ PublisherHandle ----------------------------------------------- -/// Defines a publisher_handle in a path, then can be built with FromRequest, -/// so that we can easily use this as a parameter on server methods. -pub struct PublisherHandle(pub String); - impl FromRequest for PublisherHandle { type Config = (); type Result = Result; @@ -313,24 +326,17 @@ impl FromRequest for PublisherHandle { _cfg: &Self::Config ) -> Self::Result { if let Some(handle) = req.match_info().get("handle") { - let handle = handle.to_string(); - Ok(PublisherHandle(handle)) + Ok(PublisherHandle::from(handle)) } else { Err(Error::WrongPath.into()) } } } -impl AsRef for PublisherHandle { - fn as_ref(&self) -> &str { - &self.0 - } -} - //------------ PublishDelta -------------------------------------------------- /// Support converting request body into PublishDelta -impl FromRequest for publication::PublishDelta { +impl FromRequest for publication_data::PublishDelta { type Config = (); type Result = Box>; @@ -341,7 +347,7 @@ impl FromRequest for publication::PublishDelta { Box::new(MessageBody::new(req).limit(255 * 1024 * 1024) // up to 256MB .from_err() .and_then(|bytes| { - let delta: publication::PublishDelta = + let delta: publication_data::PublishDelta = serde_json::from_reader(bytes.as_ref())?; Ok(delta) }) @@ -353,7 +359,7 @@ impl FromRequest for publication::PublishDelta { //------------ IntoHttpHandler ----------------------------------------------- impl server::IntoHttpHandler for PubServerApp { - type Handler = >> as server::IntoHttpHandler>::Handler; + type Handler = >>> as server::IntoHttpHandler>::Handler; fn into_handler(self) -> Self::Handler { self.0.into_handler() @@ -374,12 +380,13 @@ fn with_statics(app: App) -> App { //------------ HttpRequest --------------------------------------------------- -pub type HttpRequest = actix_web::HttpRequest>>; +pub type HttpRequest = actix_web::HttpRequest>>>; //------------ Error --------------------------------------------------------- #[derive(Debug, Display)] +#[allow(clippy::large_enum_variant)] pub enum Error { #[display(fmt = "{}", _0)] ServerError(krillserver::Error), @@ -396,6 +403,9 @@ pub enum Error { #[display(fmt = "Wrong path")] WrongPath, + #[display(fmt = "{}", _0)] + IoError(io::Error), + #[display(fmt = "{}", _0)] Other(String), } @@ -404,7 +414,15 @@ impl From for Error { fn from(e: serde_json::Error) -> Self { Error::JsonError(e) } } -impl error::Error for Error { +impl From for Error { + fn from(e: io::Error) -> Self { Error::IoError(e) } +} + +impl From for Error { + fn from(e: krillserver::Error) -> Self { Error::ServerError(e) } +} + +impl std::error::Error for Error { fn description(&self) -> &str { "An error happened" } diff --git a/src/krilld/http/ssl.rs b/src/krilld/http/ssl.rs index 7f7100a8..96c9f354 100644 --- a/src/krilld/http/ssl.rs +++ b/src/krilld/http/ssl.rs @@ -23,6 +23,7 @@ use crate::util::file; use crate::util::softsigner::SignerKeyId; use crate::remote::builder; + const KEY_SIZE: u32 = 2048; pub const HTTPS_SUB_DIR: &str = "ssl"; pub const KEY_FILE: &str = "key.pem"; diff --git a/src/krilld/krillserver.rs b/src/krilld/krillserver.rs index d588d1e6..a7d7e9a5 100644 --- a/src/krilld/krillserver.rs +++ b/src/krilld/krillserver.rs @@ -1,20 +1,21 @@ //! An RPKI publication protocol server. +use std::io; use std::path::PathBuf; use std::sync::Arc; use bcder::Captured; use rpki::uri; -use crate::api::publication; -use crate::api::publishers; +use crate::api::publication_data; +use crate::api::publisher_data; +use crate::api::publisher_data::PublisherHandle; +use crate::eventsourcing::KeyStore; use crate::krilld::auth::Authorizer; -use crate::krilld::pubd::{self, PublisherStore}; -use crate::krilld::pubd::repo::{self, Repository}; +use crate::krilld::pubd::PubServer; +use crate::krilld::pubd; +use crate::krilld::pubd::publishers::Publisher; use crate::remote::cmsproxy::{self, CmsProxy}; use crate::remote::rfc8183; use crate::remote::sigmsg::SignedMessage; -/// # Naming things in the keystore. -const ACTOR: &str = "krill pubd"; - //------------ KrillServer --------------------------------------------------- @@ -30,8 +31,7 @@ const ACTOR: &str = "krill pubd"; /// * Process publish / list requests by known publishers /// * Updates the repository on disk /// * Updates the RRDP files -#[derive(Clone, Debug)] -pub struct KrillServer { +pub struct KrillServer { // The base URI for this service service_uri: uri::Http, @@ -45,14 +45,11 @@ pub struct KrillServer { cms_proxy: CmsProxy, // The configured publishers - publisher_store: PublisherStore, - - // The repository responsible for publishing rsync and rrdp - repository: Repository, + pubserver: PubServer } /// # Set up and initialisation -impl KrillServer { +impl KrillServer { /// Creates a new publication server. Note that state is preserved /// on disk in the work_dir provided. pub fn build( @@ -60,11 +57,20 @@ impl KrillServer { base_uri: &uri::Rsync, service_uri: uri::Http, rrdp_base_uri: &uri::Http, - authorizer: Authorizer + authorizer: Authorizer, + store: S ) -> Result { let cms_proxy = CmsProxy::build(work_dir)?; - let publisher_store = PublisherStore::build(work_dir, base_uri)?; - let repository = Repository::build(rrdp_base_uri, work_dir)?; + + let mut repo_dir = work_dir.clone(); + repo_dir.push("repo"); + + let pubserver = PubServer::build( + base_uri.clone(), + rrdp_base_uri.clone(), + repo_dir, + store + ).map_err(Error::PubServer)?; Ok( KrillServer { @@ -72,8 +78,7 @@ impl KrillServer { work_dir: work_dir.clone(), authorizer, cms_proxy, - publisher_store, - repository, + pubserver } ) } @@ -83,22 +88,23 @@ impl KrillServer { } } -impl KrillServer { - pub fn allow_api(&self, token_opt: Option) -> bool { - self.authorizer.api_allowed(token_opt) +impl KrillServer { + pub fn is_api_allowed(&self, token_opt: Option) -> bool { + self.authorizer.is_api_allowed(token_opt) } - pub fn allow_publication_api( + pub fn is_publication_api_allowed( &self, handle_opt: Option, token_opt: Option ) -> bool { match handle_opt { None => false, - Some(handle) => { + Some(handle_str) => { match token_opt { None => false, Some(token) => { + let handle = PublisherHandle::from(handle_str); if let Ok(Some(pbl)) = self.publisher(&handle) { pbl.token() == &token } else { @@ -113,46 +119,37 @@ impl KrillServer { } /// # Configure publishers -impl KrillServer { - /// Returns all currently configured publishers. - pub fn publishers(&self) -> Result>, Error> { - self.publisher_store - .publishers() - .map_err(|e| { Error::PublisherStore(e) }) +impl KrillServer { + + /// Returns all currently configured publishers. (excludes deactivated) + pub fn publishers( + &self + ) -> Result, Error> { + self.pubserver.list_publishers().map_err(Error::PubServer) } /// Adds the publishers, blows up if it already existed. pub fn add_publisher( &mut self, - pbl: publishers::Publisher + pbl_req: publisher_data::PublisherRequest ) -> Result<(), Error> { - - self.publisher_store.add_publisher( - pbl, - ACTOR - )?; - Ok(()) + self.pubserver.create_publisher(pbl_req).map_err(Error::PubServer) } /// Removes a publisher, blows up if it didn't exist. pub fn remove_publisher( &mut self, - name: impl AsRef + handle: &PublisherHandle ) -> Result<(), Error> { - self.publisher_store.remove_publisher( - name, - ACTOR - )?; - Ok(()) + self.pubserver.deactivate_publisher(handle).map_err(Error::PubServer) } /// Returns an option for a publisher. pub fn publisher( &self, - name: impl AsRef - ) -> Result>, Error> { - self.publisher_store.publisher(name) - .map_err(Error::PublisherStore) + handle: &PublisherHandle + ) -> Result>, Error> { + self.pubserver.get_publisher(handle).map_err(Error::PubServer) } /// Returns a repository response for the given publisher. @@ -160,16 +157,20 @@ impl KrillServer { /// Returns an error if the publisher is unknown. pub fn repository_response( &self, - name: impl AsRef + handle: &PublisherHandle ) -> Result { - let publisher = self.publisher_store.get_publisher(name)?; - let rrdp_notification = self.repository.rrdp_notification_uri(); - self.cms_proxy - .repository_response( - &publisher, - self.service_base_uri(), - rrdp_notification) - .map_err(Error::CmsProxy) + match self.pubserver.get_publisher(handle)? { + None => Err(Error::NoIdCert), + Some(publisher) => { + let rrdp_notify = self.pubserver.rrdp_notification(); + self.cms_proxy + .repository_response( + &publisher, + self.service_base_uri(), + rrdp_notify) + .map_err(Error::CmsProxy) + } + } } pub fn rrdp_base_path(&self) -> PathBuf { @@ -181,7 +182,7 @@ impl KrillServer { /// # Handle publication requests /// -impl KrillServer { +impl KrillServer { /// Handles an incoming SignedMessage, verifies it's validly signed by /// a known publisher and process the QueryMessage contained. Returns @@ -198,13 +199,17 @@ impl KrillServer { pub fn handle_rfc8181_request( &mut self, sigmsg: &SignedMessage, - handle: &str + handle: &PublisherHandle ) -> Result { - debug!("Handling request for: {}", handle); - let publisher = self.publisher_store.get_publisher(handle)?; + debug!("Handling request for: {}", handle.to_string()); + + let publisher = match self.pubserver.get_publisher(handle)? { + Some(publisher) => publisher, + None => return Err(Error::NoIdCert) + }; let id_cert = match publisher.cms_auth_data() { - Some(details) => details.id_cert(), + Some(data) => data.id_cert(), None => return Err(Error::NoIdCert) }; @@ -212,13 +217,13 @@ impl KrillServer { Err(e) => self.cms_proxy.wrap_error(&e).map_err(Error::CmsProxy), Ok(req) => { let reply = match req { - publication::PublishRequest::List => { + publication_data::PublishRequest::List => { self.handle_list(handle) - .map(publication::PublishReply::List) + .map(publication_data::PublishReply::List) }, - publication::PublishRequest::Delta(delta) => { + publication_data::PublishRequest::Delta(delta) => { self.handle_delta(delta, handle) - .map(|_| publication::PublishReply::Success) + .map(|_| publication_data::PublishReply::Success) } }; @@ -226,7 +231,7 @@ impl KrillServer { Ok(reply) => { self.cms_proxy.wrap_publish_reply(reply).map_err(Error::CmsProxy) }, - Err(Error::Repository(e)) => { + Err(Error::PubServer(e)) => { self.cms_proxy.wrap_error(&e).map_err(Error::CmsProxy) }, Err(e) => Err(e) @@ -240,75 +245,50 @@ impl KrillServer { #[allow(clippy::needless_pass_by_value)] pub fn handle_delta( &mut self, - delta: publication::PublishDelta, - handle: &str + delta: publication_data::PublishDelta, + handle: &PublisherHandle ) -> Result<(), Error> { - let publisher = self.publisher_store.get_publisher(handle)?; - let base_uri = publisher.base_uri(); - self.repository.publish(&delta, base_uri).map_err(Error::Repository) + self.pubserver.publish(handle, delta).map_err(Error::PubServer) } /// Handles a list request sent to the API, or.. through the CmsProxy. pub fn handle_list( &self, - handle: &str - ) -> Result { - let publisher = self.publisher_store.get_publisher(handle)?; - let base_uri = publisher.base_uri(); - self.repository.list(base_uri).map_err(Error::Repository) + handle: &PublisherHandle + ) -> Result { + self.pubserver.list(handle).map_err(Error::PubServer) } } -// /// # Serve RRDP files -// /// -//impl KrillServer { -// /// Gets the current notification -// pub fn current_notification(&self) -> Result { -// unimplemented!() -// } -// -// /// Gets the current snapshot -// pub fn current_snapshot(&self) -> Result { -// unimplemented!() -// } -// -//} - //------------ Error --------------------------------------------------------- #[derive(Debug, Display)] +#[allow(clippy::large_enum_variant)] pub enum Error { + #[display(fmt="{}", _0)] + IoError(io::Error), + #[display(fmt="{}", _0)] CmsProxy(cmsproxy::Error), #[display(fmt="{}", _0)] - PublisherStore(pubd::Error), - - #[display(fmt="{}", _0)] - Repository(repo::Error), + PubServer(pubd::Error), #[display(fmt="No IdCert known for this publisher")] NoIdCert } +impl From for Error { + fn from(e: io::Error) -> Self { Error::IoError(e) } +} + impl From for Error { - fn from(e: cmsproxy::Error) -> Self { - Error::CmsProxy(e) - } + fn from(e: cmsproxy::Error) -> Self { Error::CmsProxy(e) } } impl From for Error { - fn from(e: pubd::Error) -> Self { - Error::PublisherStore(e) - } + fn from(e: pubd::Error) -> Self { Error::PubServer(e) } } -impl From for Error { - fn from(e: repo::Error) -> Self { - Error::Repository(e) - } -} - - // Tested through integration tests \ No newline at end of file diff --git a/src/krilld/pubd/mod.rs b/src/krilld/pubd/mod.rs index bcb027f1..506f4e5b 100644 --- a/src/krilld/pubd/mod.rs +++ b/src/krilld/pubd/mod.rs @@ -1,57 +1,313 @@ +pub mod publishers; pub mod repo; -pub mod rrdpd; -pub mod rsyncd; -use std::fs; +use std::collections::HashMap; use std::io; use std::path::PathBuf; -use std::sync::Arc; +use std::sync::{ + Arc, + RwLock, + RwLockReadGuard, + RwLockWriteGuard +}; use rpki::uri; -use crate::api::publishers; -use crate::remote::rfc8183; -use crate::storage::keystore::{self, Info, Key, KeyStore}; -use crate::storage::caching_ks::CachingDiskKeyStore; +use crate::api::publication_data; +use crate::api::publisher_data::{ + PublisherHandle, + PublisherRequest +}; +use crate::api::repo_data::DeltaElements; +use crate::krilld::pubd::repo::{ + RetentionTime, + RrdpCommand, + RrdpEvent, + RrdpInit, + RrdpServer, + RrdpServerError, + RsyncdStore, + rrdp_id +}; +use crate::eventsourcing::{ + Aggregate, + AggregateId, + Command, + Event, + KeyStore, + KeyStoreError, +}; +use crate::krilld::pubd::publishers::{ + Publisher, + PublisherCommand, + PublisherError, + PublisherEvent, + PublisherEventDetails, + PublisherInit, +}; -pub const RSYNC_FOLDER: &str = "rsync"; -//------------ PublisherStore ------------------------------------------------- +//------------ PubServer ----------------------------------------------------- -/// This type contains all configured Publishers, allowed to publish at this -/// publication server. Essentially this wraps around the storage that -/// contains all current publishers, and keeps a full audit trail of changes -/// to this. -#[derive(Clone, Debug)] -pub struct PublisherStore { - store: CachingDiskKeyStore, - base_uri: uri::Rsync, +/// This server manages all publishers. I.e. finds them, adds them, dispatches +/// commands to them, stores them.. also publishes the combined snapshots and +/// deltas, and manages the files on disk for rsync. +pub struct PubServer { + cache: RwLock>>, + rrdp_server: RwLock, + rsyncd_store: RsyncdStore, + store: S, + base_uri: uri::Rsync // jail for the publishers } - -impl PublisherStore { +impl PubServer { pub fn build( - work_dir: &PathBuf, - base_uri: &uri::Rsync + base_uri: uri::Rsync, + base_http_uri: uri::Http, // for the RRDP files + repo_dir: PathBuf, // for the RRDP and rsync files + store: S ) -> Result { - let mut publisher_dir = PathBuf::from(work_dir); - publisher_dir.push("publishers"); - if ! publisher_dir.is_dir() { - fs::create_dir_all(&publisher_dir)?; - } + let rsyncd_store = RsyncdStore::build(&repo_dir)?; - Ok( - PublisherStore { - store: CachingDiskKeyStore::build(publisher_dir)?, - base_uri: base_uri.clone() - } - ) + let rrdp_server = Self::get_or_init_rrpd_server( + &store, + base_http_uri, + repo_dir + )?; + + let pubserver = PubServer { + cache: RwLock::new(HashMap::new()), + rrdp_server, + rsyncd_store, + store, + base_uri + }; + + // Warm up the caches + let _ = pubserver.list_publishers()?; + let _ = pubserver.rrdp_updated_writer()?; + + Ok(pubserver) } - fn verify_handle(&self, handle: &str) -> Result<(), Error> { - if handle.contains('/') { - return Err(Error::ForwardSlashInHandle(handle.to_string())) + /// Read an existing RRDP server, if that fails initialise a new one, and + /// if that also fails blow up miserably! + fn get_or_init_rrpd_server( + store: &S, + base_uri: uri::Http, + repo_dir: PathBuf + ) -> Result, Error> { + let id = rrdp_id(); + let key = S::key_for_event(0); + + // TODO Support snapshots. + let init = store.get::(&id, &key) + .map_err(Error::KeyStoreError)? + .unwrap_or( + { + let init = RrdpInit::init_new(base_uri, repo_dir); + + store.store(&id, &key, &init) + .map_err(Error::KeyStoreError)?; + + init + } + ); + + let rrdp = RrdpServer::init(init)?; + + Ok(RwLock::new(rrdp)) // Note server still needs to update. + } +} + +/// # Managing the cache, and reading/writing events. +/// +impl PubServer { + + fn cache_reader(&self) -> RwLockReadGuard>> { + self.cache.read().unwrap() + } + + fn cache_writer(&self) -> RwLockWriteGuard>> { + self.cache.write().unwrap() + } + + fn cache_contains(&self, id: &AggregateId) -> bool { + self.cache_reader().contains_key(id) + } + + fn cache_insert(&self, id: &AggregateId, pbl: Publisher) { + self.cache_writer().insert(id.clone(), Arc::new(pbl)); + } + + fn update_publisher(&self, id: &AggregateId, op: F) -> Result<(), Error> + where F: FnOnce(&mut Publisher) -> Result<(), Error> { + let mut writer = self.cache.write().unwrap(); + self.update_publisher_in_cache(id, &mut writer)?; + match writer.get_mut(id) { + None => Err(Error::UnknownPublisher(id.to_string())), + Some(arc) => { + let publisher = Arc::make_mut(arc); + op(publisher) + } + } + } + + fn update_publisher_in_cache( + &self, + id: &AggregateId, + writer: &mut RwLockWriteGuard>> + ) -> Result<(), Error> { + if ! writer.contains_key(id) { + if let Some(event) = self.store_get_init(id)? { + let mut pbl = Publisher::init(event)?; + writer.insert(id.clone(), Arc::new(pbl)); + } } - if self.has_publisher(handle) { + match writer.get_mut(id) { + None => Ok(()), // done updating + Some(arc) => { + let pbl = Arc::make_mut(arc); + while let Some(event) = self.store_get_event(id, pbl.version())? { + pbl.apply(event); + } + Ok(()) + } + } + } + + fn cache_update(&self, id: &AggregateId) -> Result<(), Error> { + if ! self.cache_contains(id) { + // Check if there is something in the store, and if so add the v1 + if let Some(event) = self.store_get_init(id)? { + let mut pbl = Publisher::init(event)?; + self.cache_insert(id, pbl); + } + } + + let needs_update = { + match self.cache_reader().get(id) { + None => false, + Some(pbl) => self.store_has_event(id, pbl.version()) + } + }; + + if needs_update { + let mut wrt = self.cache_writer(); + let arc = wrt.get_mut(id).unwrap(); + let pbl = Arc::make_mut(arc); + while let Some(event) = self.store_get_event(id, pbl.version())? { + pbl.apply(event); + } + } + + Ok(()) + } + + fn rrdp_reader(&self) -> RwLockReadGuard { + self.rrdp_server.read().unwrap() + } + + fn rrdp_updated_writer( + &self + ) -> Result, Error> { + let mut rrdp_writer = self.rrdp_server.write().unwrap(); + while let Some(event) = self.store_get_rrdp_event(rrdp_writer.version())? { + rrdp_writer.apply(event); + } + Ok(rrdp_writer) + } + + fn store_get_rrdp_event( + &self, + version: u64 + ) -> Result, Error> { + let key = S::key_for_event(version); + let id = rrdp_id(); + self.store.get::(&id, &key).map_err(Error::KeyStoreError) + } + + fn store_save_rrdp_event( + &self, + event: &RrdpEvent + ) -> Result<(), Error> { + let key = S::key_for_event(event.version()); + self.store.store(event.id(), &key, event).map_err(Error::KeyStoreError)?; + Ok(()) + } + + fn store_get_init( + &self, + id: &AggregateId + ) -> Result, Error> { + let key = S::key_for_event(0); + self.store.get::(id, &key).map_err(Error::KeyStoreError) + } + + fn store_save_init( + &self, + event: &PublisherInit + ) -> Result<(), Error> { + let key = S::key_for_event(event.version()); + self.store.store(event.id(), &key, event).map_err(Error::KeyStoreError)?; + Ok(()) + } + + fn store_has_event( + &self, + id: &AggregateId, + version: u64 + ) -> bool { + let key = S::key_for_event(version); + self.store.has_key(id, &key) + } + + fn store_get_event( + &self, + id: &AggregateId, + version: u64 + ) -> Result, Error> { + let key = S::key_for_event(version); + self.store.get::(id, &key).map_err(Error::KeyStoreError) + } + + fn store_save_event( + &self, + event: &PublisherEvent + ) -> Result<(), Error> { + let key = S::key_for_event(event.version()); + self.store.store(event.id(), &key, event).map_err(Error::KeyStoreError)?; + Ok(()) + } +} + +/// # Add / remove / list publishers +impl PubServer { + + fn has_publisher(&self, handle: &PublisherHandle) -> Result { + Ok(self.get_publisher(handle)?.is_some()) + } + + /// Gets an existing publisher. Note, will also return de-activated + /// publishers when asked. + pub fn get_publisher( + &self, + handle: &PublisherHandle + ) -> Result>, Error> { + self.cache_update(handle)?; + Ok(self.cache_reader().get(handle).cloned()) + } + + fn verify_handle(&self, handle: &PublisherHandle) -> Result<(), Error> { + + if handle.as_ref() == rrdp_id().as_ref() { + return Err(Error::ReservedName(handle.to_string())) + } + + if ! handle.as_ref().bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return Err(Error::InvalidHandle(handle.to_string())) + } + + if self.has_publisher(handle)? { return Err(Error::DuplicatePublisher(handle.to_string())) } @@ -59,180 +315,248 @@ impl PublisherStore { } fn verify_base_uri(&self, base_uri: &uri::Rsync) -> Result<(), Error> { - let base_uri = base_uri.to_string(); - if base_uri.starts_with(self.base_uri.to_string().as_str()) && - base_uri.ends_with('/') { + if self.base_uri.is_parent_of(base_uri) && base_uri.ends_with("/") { + // Note it's allowed for multiple publishers to share the same + // base_uri, and for publishers to have a base_uri under another. + // Maybe we will want to change this in future, but for the moment + // this freedom is given to the admin. There are use cases here, + // such as hierarchical rsync, and possibly migrations that appear + // as standard key rolls to an RP. Ok(()) } else { Err(Error::InvalidBaseUri) } } - /// Adds a Publisher based on a PublisherRequest (from the RFC 8183 xml). - /// - /// Will return an error if the publisher already exists! Use - /// update_publisher in case you want to update an existing publisher. - pub fn add_publisher( - &mut self, - pbl: publishers::Publisher, - actor: &str + /// Adds a publisher. Will complain if a publisher already exists for this + /// handle. Will also verify that the base_uri is allowed. + pub fn create_publisher( + &self, + req: PublisherRequest ) -> Result<(), Error> { - self.verify_handle(pbl.handle())?; - self.verify_base_uri(pbl.base_uri())?; + let (handle, token, uri, cms_data) = req.unwrap(); + let handle = PublisherHandle::from(handle); + self.verify_handle(&handle)?; + self.verify_base_uri(&uri)?; - let key = Key::new(pbl.handle()); - let info = Info::now( - actor, - &format!("Added publisher: {}", pbl.handle()) + let init = PublisherInit::init( + &handle, + token, + uri, + cms_data ); - info!("Added publisher: {}", pbl.handle()); - self.store.store(key, pbl, info)?; + let pbl = Publisher::init(init.clone())?; + self.store_save_init(&init)?; + self.cache_insert(&handle, pbl); Ok(()) } - /// Removes a publisher with a given name. - /// - /// Will return an error if ths publisher does not exist. - pub fn remove_publisher( - &mut self, - handle: impl AsRef, - actor: &str - ) -> Result<(), Error> { - let name = handle.as_ref(); - match self.publisher(name)? { - None => Err(Error::UnknownPublisher(name.to_string())), - Some(_p) => { - let key = Key::new(name); - - let info = Info::now( - actor, - &format!("Removed publisher: {}", name) - ); - - self.store.archive(&key, info)?; - Ok(()) + /// Returns a list of publisher handles, excludes retired publishers. + pub fn list_publishers(&self) -> Result, Error> { + let mut res = vec![]; + for handle in self.store.aggregates() { + if handle.as_ref() == rrdp_id().as_ref() { + continue; + } + if ! self.get_publisher(&handle)?.unwrap().retired() { + res.push(handle) } } - } - - /// Returns whether a publisher exists for this name. - pub fn has_publisher(&self, handle: &str) -> bool { - let key = Key::new(handle); - match self.store.version(&key) { - Ok(Some(version)) => { - if version > 0 { - true - } else { - debug!("Publisher {} was archived.", handle); - false - } - }, - _ => false - } - } - - /// Returns an Optional Arc to a publisher for this name. - pub fn publisher( - &self, - handle: impl AsRef - ) -> Result>, Error> { - let key = Key::new(handle.as_ref()); - self.store.get(&key).map_err(|e| { Error::KeyStoreError(e)}) - } - - /// Returns an Arc to a publisher, and returns an error if the publisher - /// does not exist. - pub fn get_publisher( - &self, - handle: impl AsRef - ) -> Result, Error> { - let name = handle.as_ref(); - match self.publisher(name)? { - None => Err(Error::UnknownPublisher(name.to_string())), - Some(p) => Ok(p) - } - } - - /// Returns all current publishers. Note: using a Vec here is - /// relatively expensive, however, it is easy to implement and debug, and - /// this method is rarely needed - mainly for tests and to rebuild/update - /// the publisher list at start up. - pub fn publishers(&self) -> Result>, Error> { - let mut res = Vec::new(); - - for k in self.store.keys() { - if let Some(arc) = self.store.get(&k)? { - res.push(arc); - } - } - Ok(res) } + /// Deactivates a publisher. For now this is irreversible, but we may add + /// re-activation in future. Reason is that we never forget the history + /// of the old publisher, and if handles are re-used by different + /// entities that would get confusing. + pub fn deactivate_publisher( + &self, + handle: &PublisherHandle + ) -> Result<(), Error> { + let cmd = PublisherCommand::deactivate(handle); + self.command_publisher(cmd)?; + Ok(()) + } + + /// Apply a command to a publisher. If this was a successful publication + /// command, then return the delta so that it can be published by the + /// RRDP server. + fn command_publisher( + &self, + command: PublisherCommand + ) -> Result, Error> { + let handle = command.id().clone(); + self.cache_update(&handle)?; + + let mut res = None; + + match self.cache_writer().get_mut(&handle) { + None => Err(Error::UnknownPublisher(handle.to_string())), + Some(arc) => { + let pbl = Arc::make_mut(arc); + + if let Some(version) = command.version() { + if version != pbl.version() { + return Err( + Error::ConcurrentModification(version, pbl.version()) + ) + } + } + + for event in pbl.process_command(command)? { + if let PublisherEventDetails::Published(delta) = event.details() { + res = Some(delta.clone()) + } + + self.store_save_event(&event)?; + pbl.apply(event); + } + + Ok(res) + } + } + } } + +/// # Publication Protocol support +/// +impl PubServer { + + pub fn publish( + &self, + handle: &PublisherHandle, + delta: publication_data::PublishDelta + ) -> Result<(), Error> { + + + self.update_publisher(handle, |publisher| { + let cmd = PublisherCommand::publish(handle, delta); + + let publisher_events = publisher.process_command(cmd)?; + + for event in &publisher_events { + if let PublisherEventDetails::Published(delta) = event.details() { + let mut rrdp = self.rrdp_updated_writer()?; + let delta = delta.clone(); + + self.rsyncd_store.publish(&delta)?; + + let add_cmd = RrdpCommand::add_delta(delta); + let rrdp_add_delta_events = rrdp.process_command(add_cmd)?; + + for e in &rrdp_add_delta_events { + rrdp.apply(e.clone()); + } + + let publish_cmd = RrdpCommand::publish(); + let rrdp_publish_events = rrdp.process_command(publish_cmd)?; + + for e in &rrdp_publish_events { + rrdp.apply(e.clone()) + } + + let retention = RetentionTime::from_secs(0); + let clean_cmd = RrdpCommand::clean_up(retention); + let rrdp_clean_events = rrdp.process_command(clean_cmd)?; + + for e in rrdp_add_delta_events { + self.store_save_rrdp_event(&e)?; + } + + for e in rrdp_publish_events { + self.store_save_rrdp_event(&e)?; + } + + for e in rrdp_clean_events { + self.store_save_rrdp_event(&e)?; + rrdp.apply(e); + } + } + } + + for e in publisher_events { + self.store_save_event(&e)?; + } + + Ok(()) + })?; + + Ok(()) + } + + pub fn list( + &self, + handle: &PublisherHandle + ) -> Result { + match self.get_publisher(handle)? { + Some(publisher) => Ok(publisher.list_current()), + None => Err(Error::UnknownPublisher(handle.to_string())) + } + } +} + + +/// # Publishing +/// +impl PubServer { + pub fn rrdp_notification(&self) -> uri::Http { + self.rrdp_reader().notification_uri() + } +} + + //------------ Error --------------------------------------------------------- #[derive(Debug, Display)] pub enum Error { - #[display(fmt ="{}", _0)] - KeyStoreError(keystore::Error), - - #[display(fmt ="{}", _0)] + #[display(fmt = "{}", _0)] IoError(io::Error), - #[display(fmt = - "The '/' in publisher_handle ({}) is not supported - because we \ - are deriving the base directory for a publisher from this. This \ - behaviour may be updated in future.", _0)] - ForwardSlashInHandle(String), + #[display(fmt = "The publisher handle may only contain a-ZA-Z0-9 and _. You sent: {}", _0)] + InvalidHandle(String), - #[display(fmt = "Duplicate publisher with name: {}.", _0)] + #[display(fmt = "The publisher handle may not use the reserved name: {}", _0)] + ReservedName(String), + + #[display(fmt = "Duplicate publisher with name: {} (note: might be de-activated).", _0)] DuplicatePublisher(String), #[display(fmt = "Unknown publisher with name: {}.", _0)] UnknownPublisher(String), - #[display(fmt = "Error in base URI: {}.", _0)] - UriError(uri::Error), + #[display(fmt = "Trying to update version: {}, publisher at: {}", _0, _1)] + ConcurrentModification(u64, u64), - #[display(fmt = "Invalid Publisher Request: {}.", _0)] - PublisherRequestError(rfc8183::PublisherRequestError), + #[display(fmt = "Base uri for publisher needs to be under server, and must end with a '/'.")] + InvalidBaseUri, - #[display(fmt = "Cannot override handle using path parameter for json api")] - HandleOverrideNotAllowed, + #[display(fmt = "{}", _0)] + PublisherError(PublisherError), - #[display(fmt = "Base uri for publisher needs to be folder under base uri\ - for server, and must end with a '/'.")] - InvalidBaseUri -} + #[display(fmt = "{}", _0)] + RrdpServerError(RrdpServerError), -impl From for Error { - fn from(e: keystore::Error) -> Self { - Error::KeyStoreError(e) - } + #[display(fmt = "{}", _0)] + KeyStoreError(KeyStoreError), } impl From for Error { - fn from(e: io::Error) -> Self { - Error::IoError(e) - } + fn from(e: io::Error) -> Self { Error::IoError(e) } } -impl From for Error { - fn from(e: uri::Error) -> Self { - Error::UriError(e) - } +impl From for Error { + fn from(e: PublisherError) -> Self { Error::PublisherError(e) } } -impl From for Error { - fn from(e: rfc8183::PublisherRequestError) -> Self { - Error::PublisherRequestError(e) - } +impl From for Error { + fn from(e: RrdpServerError) -> Self { Error::RrdpServerError(e) } } +impl std::error::Error for Error {} + //------------ Tests --------------------------------------------------------- @@ -240,29 +564,85 @@ impl From for Error { mod tests { use super::*; + use std::path::PathBuf; + use bytes::Bytes; + use crate::api::publication_data::PublishDeltaBuilder; + use crate::eventsourcing::DiskKeyStore; + use crate::util::file::CurrentFile; use crate::util::test; + use api::repo_data::VerificationError; - fn test_publisher_store(dir: &PathBuf) -> PublisherStore { - let uri = test::rsync_uri("rsync://host/module/"); - PublisherStore::build(dir, &uri).unwrap() + fn server_base_uri() -> uri::Rsync { + test::rsync_uri("rsync://localhost/repo/") + } + + fn server_base_http_uri() -> uri::Http { + test::http_uri("http://localhost/rrdp/") + } + + fn make_publisher_req( + name: &str, + uri: &str, + work_dir: &PathBuf + ) -> PublisherRequest { + let base_uri = test::rsync_uri(uri); + let token = "secret"; + let openssl_dir = test::create_sub_dir(work_dir); + + let pr = test::new_publisher_request(name, &openssl_dir); + pr.into_publisher(token.to_string(), base_uri.clone()) + } + + fn make_server(work_dir: &PathBuf) -> PubServer { + let mut store_dir = work_dir.clone(); + store_dir.push("pubserver"); + let store = DiskKeyStore::new(store_dir); + + let mut base_dir = work_dir.clone(); + base_dir.push("repo"); + + PubServer::build( + server_base_uri(), + server_base_http_uri(), + base_dir, + store + ).unwrap() } #[test] - fn should_refuse_slash_in_publisher_handle() { + fn should_add_publisher() { test::test_with_tmp_dir(|d| { - let mut ps = test_publisher_store(&d); + let publisher_req = make_publisher_req( + "alice", + "rsync://localhost/repo/alice/", + &d + ); - let name = "alice/bob"; - let base_uri = test::rsync_uri("rsync://host/module/alice/bob"); - let token = "secret"; + let server = make_server(&d); + server.create_publisher(publisher_req).unwrap(); - let pr = test::new_publisher_request(name, &d); + let handle = PublisherHandle::from("alice"); + let alice = server.get_publisher(&handle).unwrap().unwrap(); - let pbl = pr.into_publisher(token.to_string(), base_uri.clone()); + assert_eq!(alice.id(), &handle); + }) + } - match ps.add_publisher(pbl, "test") { - Err(Error::ForwardSlashInHandle(_)) => { }, // Ok - _ => panic!("Should have seen error.") + #[test] + fn should_refuse_invalid_publisher_handle() { + test::test_with_tmp_dir(|d| { + let publisher_req = make_publisher_req( + "alice&", + "rsync://localhost/repo/alice/", + &d + ); + + let server = make_server(&d); + match server.create_publisher(publisher_req) { + Err(Error::InvalidHandle(handle)) => { + assert_eq!(handle, "alice&".to_string()) + }, + _ => panic!("Expected error") } }) } @@ -270,19 +650,16 @@ mod tests { #[test] fn should_refuse_base_uri_not_ending_with_slash() { test::test_with_tmp_dir(|d| { - let mut ps = test_publisher_store(&d); + let publisher_req = make_publisher_req( + "alice", + "rsync://localhost/repo/alice", + &d + ); - let name = "alice"; - let base_uri = test::rsync_uri("rsync://host/module/alice"); - let token = "secret"; - - let pr = test::new_publisher_request(name, &d); - - let pbl = pr.into_publisher(token.to_string(), base_uri.clone()); - - match ps.add_publisher(pbl, "test") { - Err(Error::InvalidBaseUri) => { }, // Ok - _ => panic!("Should have seen error.") + let server = make_server(&d); + match server.create_publisher(publisher_req) { + Err(Error::InvalidBaseUri) => { }, + _ => panic!("Expected error") } }) } @@ -290,97 +667,257 @@ mod tests { #[test] fn should_refuse_base_uri_outside_of_server_base() { test::test_with_tmp_dir(|d| { - let mut ps = test_publisher_store(&d); + let publisher_req = make_publisher_req( + "alice", + "rsync://localhost/outside/alice/", + &d + ); - let name = "alice"; - let base_uri = test::rsync_uri("rsync://host/modu/alice/"); - let token = "secret"; - - let pr = test::new_publisher_request(name, &d); - - let pbl = pr.into_publisher(token.to_string(), base_uri.clone()); - - match ps.add_publisher(pbl, "test") { - Err(Error::InvalidBaseUri) => { }, // Ok - _ => panic!("Should have seen error.") + let server = make_server(&d); + match server.create_publisher(publisher_req) { + Err(Error::InvalidBaseUri) => { }, + _ => panic!("Expected error") } }) } - #[test] - fn should_add_publisher() { - test::test_with_tmp_dir(|d| { - let mut ps = test_publisher_store(&d); - - let name = "alice"; - let base_uri = test::rsync_uri("rsync://host/module/alice/"); - let token = "secret"; - - let pr = test::new_publisher_request(name, &d); - let id_cert = pr.id_cert().clone(); - - let pbl = pr.into_publisher(token.to_string(), base_uri.clone()); - - let actor = "test"; - ps.add_publisher(pbl, actor).unwrap(); - - assert!(ps.has_publisher(&name)); - - // Get the Arc out of the Result>, Error> - let publisher_found = ps.publisher(&name).unwrap().unwrap(); - - let expected_rfc8181 = publishers::CmsAuthData::new(None, id_cert); - - assert_eq!(publisher_found.handle(), "alice"); - assert_eq!(publisher_found.base_uri(), &base_uri); - assert_eq!(publisher_found.cms_auth_data(), &Some(expected_rfc8181)); - - }) - } - #[test] fn should_not_add_publisher_twice() { test::test_with_tmp_dir(|d| { - let mut ps = test_publisher_store(&d); + let publisher_req = make_publisher_req( + "alice", + "rsync://localhost/repo/alice/", + &d + ); - let name = "alice"; - let base_uri = test::rsync_uri("rsync://host/module/alice/"); - let token = "secret"; - - let pr = test::new_publisher_request(name, &d); - let pbl = pr.into_publisher(token.to_string(), base_uri.clone()); - - let actor = "test"; - ps.add_publisher(pbl.clone(), actor).unwrap(); - assert!(ps.has_publisher(&name)); - - match ps.add_publisher(pbl, actor) { - Err(Error::DuplicatePublisher(_)) => { }, // Ok - _ => panic!("Should have seen error.") + let server = make_server(&d); + server.create_publisher(publisher_req.clone()).unwrap(); + match server.create_publisher(publisher_req) { + Err(Error::DuplicatePublisher(name)) => { + assert_eq!(name, "alice".to_string()) + }, + _ => panic!("Expected error") } }) - } #[test] fn should_remove_publisher() { test::test_with_tmp_dir(|d| { - let mut ps = test_publisher_store(&d); + let publisher_req = make_publisher_req( + "alice", + "rsync://localhost/repo/alice/", + &d + ); + let handle = PublisherHandle::from("alice"); - let name = "alice"; - let base_uri = test::rsync_uri("rsync://host/module/alice/"); - let token = "secret"; + let server = make_server(&d); - let pr = test::new_publisher_request(name, &d); - let pbl = pr.into_publisher(token.to_string(), base_uri.clone()); + server.create_publisher(publisher_req).unwrap(); - let actor = "test"; - ps.add_publisher(pbl, actor).unwrap(); - assert_eq!(1, ps.publishers().unwrap(). len()); + let list = server.list_publishers().unwrap(); + assert_eq!(list, vec![handle.clone()]); - ps.remove_publisher(name, actor).unwrap(); - assert_eq!(0, ps.publishers().unwrap(). len()); + let retire = PublisherCommand::deactivate(&handle); + + server.command_publisher(retire).unwrap(); + let list = server.list_publishers().unwrap(); + assert!(list.is_empty()); + }) + } + + #[test] + fn should_list_files() { + test::test_with_tmp_dir(|d| { + let publisher_req = make_publisher_req( + "alice", + "rsync://localhost/repo/alice/", + &d + ); + let handle = PublisherHandle::from("alice"); + + let server = make_server(&d); + server.create_publisher(publisher_req).unwrap(); + + let alice = server.get_publisher(&handle).unwrap().unwrap(); + + let list_reply = alice.list_current(); + assert_eq!(0, list_reply.elements().len()); }); } -} + #[test] + fn should_publish_files() { + test::test_with_tmp_dir(|d| { + // get the file out of a list_reply + fn find_in_reply<'a>( + reply: &'a publication_data::ListReply, + uri: &uri::Rsync + ) -> Option<&'a publication_data::ListElement> { + reply.elements().iter().find(|e| e.uri() == uri) + } + + let publisher_req = make_publisher_req( + "alice", + "rsync://localhost/repo/alice/", + &d + ); + let handle = PublisherHandle::from("alice"); + + let server = make_server(&d); + server.create_publisher(publisher_req).unwrap(); + + // Publish a single file + let file1 = CurrentFile::new( + test::rsync_uri("rsync://localhost/repo/alice/file.txt"), + &Bytes::from("example content") + ); + + let file2 = CurrentFile::new( + test::rsync_uri("rsync://localhost/repo/alice/file2.txt"), + &Bytes::from("example content 2") + ); + + let mut builder = PublishDeltaBuilder::new(); + builder.add_publish(file1.as_publish()); + builder.add_publish(file2.as_publish()); + let delta = builder.finish(); + + server.publish(&handle, delta).unwrap(); + + // Two files should now appear in the list + let alice = server.get_publisher(&handle).unwrap().unwrap(); + let list_reply = alice.list_current(); + assert_eq!(2, list_reply.elements().len()); + assert!(find_in_reply( + &list_reply, + &test::rsync_uri("rsync://localhost/repo/alice/file.txt") + ).is_some()); + assert!(find_in_reply( + &list_reply, + &test::rsync_uri("rsync://localhost/repo/alice/file2.txt") + ).is_some()); + + // Update + // - update file + // - withdraw file2 + // - add file 3 + + let file1_update = CurrentFile::new( + test::rsync_uri("rsync://localhost/repo/alice/file.txt"), + &Bytes::from("example content - updated") + ); + + let file3 = CurrentFile::new( + test::rsync_uri("rsync://localhost/repo/alice/file3.txt"), + &Bytes::from("example content 3") + ); + + let mut builder = PublishDeltaBuilder::new(); + builder.add_update(file1_update.as_update(file1.hash())); + builder.add_withdraw(file2.as_withdraw()); + builder.add_publish(file3.as_publish()); + let delta = builder.finish(); + + server.publish(&handle, delta).unwrap(); + + // Two files should now appear in the list + let alice = server.get_publisher(&handle).unwrap().unwrap(); + let list_reply = alice.list_current(); + + assert_eq!(2, list_reply.elements().len()); + assert!(find_in_reply( + &list_reply, + &test::rsync_uri("rsync://localhost/repo/alice/file.txt") + ).is_some()); + assert_eq!( + find_in_reply( + &list_reply, + &test::rsync_uri("rsync://localhost/repo/alice/file.txt") + ).unwrap().hash(), + file1_update.hash() + ); + assert!(find_in_reply( + &list_reply, + &test::rsync_uri("rsync://localhost/repo/alice/file3.txt") + ).is_some()); + + // Should reject publish outside of base uri + let file_outside = CurrentFile::new( + test::rsync_uri("rsync://localhost/repo/bob/file.txt"), + &Bytes::from("irrelevant") + ); + let mut builder = PublishDeltaBuilder::new(); + builder.add_publish(file_outside.as_publish()); + let delta = builder.finish(); + + match server.publish(&handle, delta) { + Err( + Error::PublisherError( + PublisherError::VerificationError( + VerificationError::UriOutsideJail(_, _) + ) + ) + ) => {}, // ok + _ => panic!("Expected error publishing outside of base uri jail") + } + + // Should reject update of file that does not exist + let file2_update = CurrentFile::new( + test::rsync_uri("rsync://localhost/repo/alice/file2.txt"), + &Bytes::from("example content 2 updated") + ); // file2 was removed + let mut builder = PublishDeltaBuilder::new(); + builder.add_update(file2_update.as_update(file2.hash())); + let delta = builder.finish(); + + match server.publish(&handle, delta) { + Err( + Error::PublisherError( + PublisherError::VerificationError( + VerificationError::NoObjectForHashAndOrUri(_) + ) + ) + ) => {}, + // ok + _ => panic!("Expected error when file for update can't be found") + } + + // should reject withdraw for file that does not exist + let mut builder = PublishDeltaBuilder::new(); + builder.add_withdraw(file2.as_withdraw()); + let delta = builder.finish(); + let cmd = PublisherCommand::publish(&handle, delta); + + match server.command_publisher(cmd) { + Err( + Error::PublisherError( + PublisherError::VerificationError( + VerificationError::NoObjectForHashAndOrUri(_) + ) + ) + ) => {}, // ok + _ => panic!("Expected error withdrawing file that does not exist") + } + + // should reject publish for file that does exist + let mut builder = PublishDeltaBuilder::new(); + builder.add_publish(file3.as_publish()); + let delta = builder.finish(); + + match server.publish(&handle, delta) { + Err( + Error::PublisherError( + PublisherError::VerificationError + (VerificationError::ObjectAlreadyPresent(uri) + ) + ) + ) => { assert_eq!( + uri, + test::rsync_uri("rsync://localhost/repo/alice/file3.txt") + )}, + _ => panic!("Expected error publishing file that already exists") + } + }); + } +} \ No newline at end of file diff --git a/src/krilld/pubd/publishers.rs b/src/krilld/pubd/publishers.rs new file mode 100644 index 00000000..ea0d3b14 --- /dev/null +++ b/src/krilld/pubd/publishers.rs @@ -0,0 +1,266 @@ +use rpki::uri; +use crate::api::publication_data; +use crate::api::publisher_data::{ + CmsAuthData, + PublisherHandle, + PublisherRequest +}; +use crate::api::repo_data::{ + CurrentObjects, + DeltaElements, + VerificationError +}; +use crate::eventsourcing::{ + Aggregate, + CommandDetails, + StoredEvent, + SentCommand +}; +use crate::util::ext_serde; + + +//------------ PublisherInit ------------------------------------------------- + +pub type PublisherInit = StoredEvent; + +#[derive(Clone, Deserialize, Serialize)] +pub struct InitPublisherDetails { + token: String, + + #[serde( + deserialize_with = "ext_serde::de_rsync_uri", + serialize_with = "ext_serde::ser_rsync_uri")] + base_uri: uri::Rsync, + cms_auth_data: Option +} + +impl PublisherInit { + pub fn init( + id: &PublisherHandle, + token: String, + base_uri: uri::Rsync, + cms_auth_data: Option + ) -> Self { + StoredEvent::new( + id, + 0, + InitPublisherDetails { token, base_uri, cms_auth_data } + ) + } +} + +impl From for PublisherInit { + fn from(req: PublisherRequest) -> Self { + let (handle, token, base_uri, cms_auth_data) = req.unwrap(); // (self + let id = PublisherHandle::from(handle); + let details = InitPublisherDetails { token, base_uri, cms_auth_data }; + StoredEvent::new(&id, 0, details) + } +} + + +//------------ PublisherEvent ------------------------------------------------ + +pub type PublisherEvent = StoredEvent; + +#[derive(Clone, Deserialize, Serialize)] +pub enum PublisherEventDetails { + Deactivated, + Published(DeltaElements) +} + +impl PublisherEvent { + pub fn deactivated(id: &PublisherHandle, version: u64) -> Self { + PublisherEvent::new(id, version, PublisherEventDetails::Deactivated) + } + + pub fn published( + id: &PublisherHandle, + version: u64, + delta: DeltaElements + ) -> Self { + PublisherEvent::new(id, version, PublisherEventDetails::Published(delta)) + } +} + + +//------------ PublisherCommand ---------------------------------------------- + +pub type PublisherCommand = SentCommand; + +#[derive(Clone, Deserialize, Serialize)] +pub enum PublisherCommandDetails { + Deactivate, + Publish(publication_data::PublishDelta) +} + +impl CommandDetails for PublisherCommandDetails { + type Event = PublisherEvent; +} + +impl PublisherCommand { + pub fn deactivate(id: &PublisherHandle) -> Self { + PublisherCommand::new(id, None, PublisherCommandDetails::Deactivate) + } + + pub fn publish(id: &PublisherHandle, delta: publication_data::PublishDelta) -> Self { + PublisherCommand::new(id, None, PublisherCommandDetails::Publish(delta)) + } +} + + +//------------ PublisherError ------------------------------------------------ + +#[derive(Clone, Debug, Display)] +pub enum PublisherError { + #[display(fmt = "Publisher is (already) de-activated")] + Deactivated, + + #[display(fmt="{}", _0)] + VerificationError(VerificationError), +} + +impl From for PublisherError { + fn from(e: VerificationError) -> Self { + PublisherError::VerificationError(e) + } +} + +impl std::error::Error for PublisherError {} + + +//------------ Publisher ----------------------------------------------------- + +/// This type defines Publisher CAs that are allowed to publish. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Publisher { + /// Aggregate house keeping + id: PublisherHandle, + version: u64, + retired: bool, + + /// Publication jail for this publisher + #[serde( + deserialize_with = "ext_serde::de_rsync_uri", + serialize_with = "ext_serde::ser_rsync_uri")] + base_uri: uri::Rsync, + + /// The token used by the API + token: String, + + /// The optional RFC8181 identity, for the RFC8183 pub protocol. + cms_auth_data: Option, + + /// All objects currently published by this publisher, by hash + current_objects: CurrentObjects +} + +/// # Accessors +impl Publisher { + pub fn id(&self) -> &PublisherHandle { + &self.id + } + + pub fn retired(&self) -> bool { self.retired } + + pub fn token(&self) -> &String { + &self.token + } + + pub fn base_uri(&self) -> &uri::Rsync { + &self.base_uri + } + + pub fn cms_auth_data(&self) -> &Option { + &self.cms_auth_data + } +} + +/// # Life cycle +/// +impl Publisher { + + fn create(event: PublisherInit) -> Self { + let (id, _version, init) = event.unwrap(); + Publisher { + id, + version: 1, + retired: false, + token: init.token, + base_uri: init.base_uri, + cms_auth_data: init.cms_auth_data, + current_objects: CurrentObjects::default() + } + } + + fn deactivate(&self) -> Result, PublisherError> { + if self.retired { + Err(PublisherError::Deactivated) + } else { + let e = PublisherEvent::deactivated(&self.id, self.version); + Ok(vec![e]) + } + } +} + + +/// # Publication protocol +/// +impl Publisher { + /// Gets an owned list reply containing all objects for this publisher. + /// Note that cloning the uris and hashes is relatively cheap because of + /// the use of Bytes as the underlying structure. Still, it may be good + /// to change this implementation in future to return a structure that + /// takes references, and only lives long enough to compose a response. + pub fn list_current(&self) -> publication_data::ListReply { + self.current_objects.to_list_reply() + } + + /// Verifies a delta command and returns an event containing the delta, + /// provided that it's legitimate. + fn process_delta_cmd( + &self, + delta: publication_data::PublishDelta + ) -> Result, PublisherError> { + + let delta = DeltaElements::from(delta); + self.current_objects.verify_delta(&delta, &self.base_uri)?; + + Ok(vec![PublisherEvent::published(&self.id, self.version, delta)]) + } + + fn apply_delta(&mut self, delta: DeltaElements) { + self.current_objects.apply_delta(delta); + } +} + + +impl Aggregate for Publisher { + type Command = PublisherCommand; + type Event = PublisherEvent; + type InitEvent = PublisherInit; + type Error = PublisherError; + + fn init(event: Self::InitEvent) -> Result { + Ok(Self::create(event)) + } + + fn version(&self) -> u64 { + self.version + } + + fn apply(&mut self, event: Self::Event) { + match event.into_details() { + PublisherEventDetails::Deactivated => self.retired = true, + PublisherEventDetails::Published(delta) => self.apply_delta(delta) + } + self.version += 1; + } + + fn process_command(&self, command: Self::Command) -> Result, Self::Error> { + match command.into_details() { + PublisherCommandDetails::Deactivate => self.deactivate(), + PublisherCommandDetails::Publish(delta) => self.process_delta_cmd(delta) + } + } +} diff --git a/src/krilld/pubd/repo.rs b/src/krilld/pubd/repo.rs index 13ba3267..f5fc65c0 100644 --- a/src/krilld/pubd/repo.rs +++ b/src/krilld/pubd/repo.rs @@ -1,266 +1,470 @@ +use std::{io, fs}; use std::path::PathBuf; +use std::time::Duration; use rpki::uri; -use crate::api::publication; -use crate::krilld::pubd::rsyncd; -use crate::krilld::pubd::rrdpd; +use crate::api::repo_data::{ + Delta, + DeltaElements, + DeltaRef, + FileRef, + Notification, + NotificationUpdate, + Snapshot, + SnapshotRef, +}; +use crate::eventsourcing::{ + Aggregate, + AggregateId, + CommandDetails, + StoredEvent, + SentCommand, +}; +use crate::util::{ + ext_serde, + file, + Time +}; -//------------ Repository ---------------------------------------------------- -/// This type orchestrates publishing in both an RSYNC and RRDP -/// (RFC8182) format. +const RRDP_FOLDER: &str = "rrdp"; +const RSYNC_FOLDER: &str = "rsync"; + +// Todo: make a const fn once that is stable. +pub fn rrdp_id() -> AggregateId { AggregateId::from("rrdp_server")} + + + +//------------ RrdpInit ------------------------------------------------------ + +pub type RrdpInit = StoredEvent; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct RrdpInitDetails { + session: String, + + #[serde( + deserialize_with = "ext_serde::de_http_uri", + serialize_with = "ext_serde::ser_http_uri")] + base_uri: uri::Http, + + repo_dir: PathBuf +} + +impl RrdpInit { + pub fn init_new(base_uri: uri::Http, repo_dir: PathBuf) -> Self { + use rand::{thread_rng, Rng}; + let mut rng = thread_rng(); + let rnd: u32 = rng.gen(); + let session = format!("{}", rnd); + + StoredEvent::new( + &rrdp_id(), + 0, + RrdpInitDetails { session, base_uri, repo_dir } + ) + } +} + + +//------------ RrdpEvent ------------------------------------------------------ + +pub type RrdpEvent = StoredEvent; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[allow(clippy::large_enum_variant)] +pub enum RrdpEventDetails { + AddedDelta(Delta), + UpdatedNotification(NotificationUpdate), + CleanedUp(Time) +} + +impl RrdpEvent { + fn added_delta(id: &AggregateId, ver: u64, delta: Delta) -> Self { + StoredEvent::new(id, ver, RrdpEventDetails::AddedDelta(delta)) + } + + fn updated_notification( + id: &AggregateId, + ver: u64, + notif: NotificationUpdate + ) -> Self { + StoredEvent::new(id, ver, RrdpEventDetails::UpdatedNotification(notif)) + } + + fn cleaned_up( + id: &AggregateId, + ver: u64, + time: Time + ) -> Self { + StoredEvent::new(id, ver, RrdpEventDetails::CleanedUp(time)) + } +} + + +//------------ RrdpCommand --------------------------------------------------- + +pub type RrdpCommand = SentCommand; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum RrdpCommandDetails { + AddDelta(DeltaElements), + Publish, + Cleanup(RetentionTime) +} + +/// The retention time for snapshot and delta files no longer referenced. +pub type RetentionTime = Duration; + +impl CommandDetails for RrdpCommandDetails { + type Event = RrdpEvent; +} + +impl RrdpCommand { + pub fn add_delta(delta: DeltaElements) -> Self { + SentCommand::new(&rrdp_id(), None, RrdpCommandDetails::AddDelta(delta)) + } + + pub fn publish() -> Self { + SentCommand::new(&rrdp_id(), None, RrdpCommandDetails::Publish) + } + + pub fn clean_up(retention: RetentionTime) -> Self { + SentCommand::new(&rrdp_id(), None, RrdpCommandDetails::Cleanup(retention)) + } +} + + +//------------ RrdpServerError ----------------------------------------------- + +#[derive(Debug, Display)] +pub enum RrdpServerError { + + #[display(fmt = "{}", _0)] + IoError(io::Error), +} + +impl From for RrdpServerError { + fn from(e: io::Error) -> Self { RrdpServerError::IoError(e) } +} + +impl std::error::Error for RrdpServerError {} + + +//------------ RrdpResult ---------------------------------------------------- + +pub type RrdpResult = Result, RrdpServerError>; + + +//------------ RrdpServer ---------------------------------------------------- + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct RrdpServer { + // aggregate ID is fixed using a const str RRDP_ID + version: u64, + + session: String, + serial: u64, + + /// The base URI for notification, snapshot and delta files. + #[serde( + deserialize_with = "ext_serde::de_http_uri", + serialize_with = "ext_serde::ser_http_uri")] + base_uri: uri::Http, + + /// The base directory where notification, snapshot and deltas will be + /// published. + rrdp_base: PathBuf, + + notification: Notification, + snapshot: Snapshot, + deltas: Vec +} + +/// # Publishing +/// +impl RrdpServer { + + fn process_published_delta(&mut self, delta: Delta) { + self.snapshot.apply_delta(delta.clone()); + self.deltas.insert(0, delta); + + // Keep a minimum of 2 deltas, and a maximum for which the combined + // number of elements does not exceed the number of elements in the + // snapshot. + { + let size_snapshot = self.snapshot.len(); + let mut total_deltas = 0; + let mut count = 0; + + self.deltas.retain(|d| { + count += 1; + total_deltas += d.len(); + count <= 2 || total_deltas < size_snapshot + }); + } + + self.serial += 1; + } + + /// Creates a delta for the delta elements. Assumes (for now) that the + /// delta elements have been verified for the publisher. + fn add_delta(&self, elements: DeltaElements) -> RrdpResult { + let next = self.serial + 1; + let session = self.session.clone(); + let delta = Delta::new(session, next, elements); + + Ok(vec![RrdpEvent::added_delta(&rrdp_id(), self.version, delta)]) + } + + /// Publishes the latest notification, snapshot and delta file to disk. + /// Return event to move old files to clean-up list. + fn publish(&self) -> RrdpResult { + let snapshot_hash = self.snapshot.write_xml(&self.snapshot_path())?; + let snapshot_ref = SnapshotRef::new( + self.snapshot_uri(), + self.snapshot_path(), + snapshot_hash + ); + + // Note we always have at least 1 delta when publishing. + let last_delta = &self.deltas[0]; + let delta_hash = last_delta.write_xml(&self.delta_path(last_delta.serial()))?; + let delta_ref = DeltaRef::new( + last_delta.serial(), + FileRef::new( + self.delta_uri(last_delta.serial()), + self.delta_path(last_delta.serial()), + delta_hash + ) + ); + + let update = NotificationUpdate::new( + Time::now(), + None, + snapshot_ref, + delta_ref, + self.deltas.last().unwrap().serial() + ); + + let mut notification = self.notification.clone(); + notification.update(update.clone()); + notification.write_xml(&self.notification_path())?; + + Ok(vec![ + RrdpEvent::updated_notification( + &rrdp_id(), + self.version, + update + ) + ]) + } + + /// Cleans out old files on disk, returns event for cleaning up the state. + fn cleanup(&self, retention: RetentionTime) -> RrdpResult { + let cut_off = Time::before_now(retention); + for old in self.notification.old_refs() { + if old.0.on_or_before(&cut_off) { + // Don't care if it were already deleted. + let _ = file::clean_file_and_path(&old.1.path()); + } + } + + Ok(vec![ + RrdpEvent::cleaned_up( + &rrdp_id(), + self.version, + cut_off + ) + ]) + } +} + +/// rrdp paths and uris +/// +impl RrdpServer { + pub fn notification_uri(&self) -> uri::Http { + uri::Http::from_string( + format!("{}notifcation.xml", self.base_uri.to_string()) + ).unwrap() // Cannot fail. Config checked at startup. + } + + fn notification_path(&self) -> PathBuf { + let mut path = self.rrdp_base.clone(); + path.push("notification.xml"); + path + } + + fn snapshot_rel(session: &str, serial: u64) -> String { + format!("{}/{}/snapshot.xml", session, serial) + } + + fn new_snapshot_path(base: &PathBuf, session: &str, serial: u64) -> PathBuf { + let mut path = base.clone(); + path.push(Self::snapshot_rel(session, serial)); + path + } + + fn snapshot_path(&self) -> PathBuf { + Self::new_snapshot_path(&self.rrdp_base, &self.session, self.serial) + } + + fn new_snapshot_uri(base: &uri::Http, session: &str, serial: u64) -> uri::Http { + uri::Http::from_string( + format!("{}{}", + base.to_string(), + Self::snapshot_rel(session, serial) + ) + ).unwrap() // Cannot fail. Config checked at startup. + } + + fn snapshot_uri(&self) -> uri::Http { + Self::new_snapshot_uri(&self.base_uri, &self.session, self.serial) + } + + fn delta_rel(session: &str, serial: u64) -> String { + format!("{}/{}/delta.xml", session, serial) + } + + fn delta_uri(&self, serial: u64) -> uri::Http { + uri::Http::from_string( + format!("{}{}", + self.base_uri.to_string(), + Self::delta_rel(&self.session, serial) + ) + ).unwrap() // Cannot fail. Config checked at startup. + } + + fn delta_path(&self, serial: u64) -> PathBuf { + let mut path = self.rrdp_base.clone(); + path.push(Self::delta_rel(&self.session, serial)); + path + } +} + + + +impl Aggregate for RrdpServer { + type Command = RrdpCommand; + type Event = RrdpEvent; + type InitEvent = RrdpInit; + type Error = RrdpServerError; + + fn init(event: Self::InitEvent) -> Result { + let init = event.into_details(); + let version = 1; + let session = init.session; + let base_uri = init.base_uri; + let mut rrdp_base = init.repo_dir.clone(); + rrdp_base.push(RRDP_FOLDER); + + let serial = 0; + let snapshot = Snapshot::new(session.clone()); + + let snapshot_path = Self::new_snapshot_path(&rrdp_base, &session, 0); + let snapshot_uri = Self::new_snapshot_uri(&base_uri, &session, 0); + let snapshot_hash = snapshot.write_xml(&snapshot_path)?; + + let snapshot_ref = SnapshotRef::new( + snapshot_uri, + snapshot_path, + snapshot_hash + ); + + let notification = Notification::create(session.clone(), snapshot_ref); + let deltas = vec![]; + + Ok(RrdpServer { + version, + session, + base_uri, + rrdp_base, + serial, + notification, + snapshot, + deltas + }) + } + + fn version(&self) -> u64 { + self.version + } + + fn apply(&mut self, event: Self::Event) { + match event.into_details() { + RrdpEventDetails::AddedDelta(delta) => + self.process_published_delta(delta), + RrdpEventDetails::UpdatedNotification(notification) => + self.notification.update(notification), + RrdpEventDetails::CleanedUp(time) => + self.notification.clean_up(time) + } + self.version += 1; + + } + + fn process_command(&self, command: Self::Command) -> RrdpResult { + match command.into_details() { + RrdpCommandDetails::AddDelta(els) => self.add_delta(els), + RrdpCommandDetails::Publish => self.publish(), + RrdpCommandDetails::Cleanup(retention) => self.cleanup(retention), + } + } +} + + +//------------ RsyncdStore --------------------------------------------------- + +/// This type is responsible for publishing files on disk in a structure so +/// that an rscynd can be set up to serve this (RPKI) data. Note that the +/// rsync host name and module are part of the path, so make sure that the +/// rsyncd modules and paths are setup properly for each supported rsync +/// base uri used. #[derive(Clone, Debug)] -pub struct Repository { - // file_store - fs: rsyncd::FileStore, - - // RRDP - rrdp: rrdpd::RrdpServer +pub struct RsyncdStore { + rsync_dir: PathBuf } /// # Construct /// -impl Repository { - pub fn build( - rrdp_base_uri: &uri::Http, - work_dir: &PathBuf - ) -> Result - { - let fs = rsyncd::FileStore::build(work_dir)?; - let rrdp = rrdpd::RrdpServer::build(rrdp_base_uri, work_dir)?; - Ok( Repository { fs, rrdp } ) +impl RsyncdStore { + pub fn build(repo_dir: &PathBuf) -> Result { + let mut rsync_dir = PathBuf::from(repo_dir); + rsync_dir.push(RSYNC_FOLDER); + if ! rsync_dir.is_dir() { + fs::create_dir_all(&rsync_dir)?; + } + Ok ( RsyncdStore { rsync_dir } ) } } -/// # Access +/// # Publishing /// -impl Repository { - /// Returns the RRDP notification URI for inclusion in the - /// Repository Response - pub fn rrdp_notification_uri(&self) -> uri::Http { - self.rrdp.notification_uri().clone() - } -} +impl RsyncdStore { + /// Saves all the publishes and updates, deletes all the withdraws. + pub fn publish(&self, delta: &DeltaElements) -> Result<(), io::Error> { -/// # Publish / List -/// -impl Repository { - /// Publishes an publish query and returns a success reply embedded in - /// a message. Throws an error in case of issues. The PubServer needs - /// to wrap such errors in a response message to the publisher. - pub fn publish( - &mut self, - delta: &publication::PublishDelta, - base_uri: &uri::Rsync - ) -> Result<(), Error> { - debug!("Processing update with {} elements", delta.len()); - self.fs.publish(delta, base_uri)?; - self.rrdp.publish(delta, base_uri)?; + for p in delta.publishes() { + file::save_with_rsync_uri( + &p.base64().to_bytes(), + &self.rsync_dir, + p.uri() + )?; + } + + for u in delta.updates() { + file::save_with_rsync_uri( + &u.base64().to_bytes(), + &self.rsync_dir, + u.uri() + )?; + } + + for w in delta.withdraws() { + file::delete_with_rsync_uri( + &self.rsync_dir, + w.uri() + )?; + } Ok(()) } - - /// Lists the objects for a base_uri, presumably all for the same - /// publisher. - pub fn list( - &self, - base_uri: &uri::Rsync - ) -> Result { - debug!("Processing list query"); - let files = self.fs.list(base_uri)?; - Ok(publication::ListReply::from_files(files)) - } -} - - - - -//------------ Error --------------------------------------------------------- - -#[derive(Debug, Display)] -pub enum Error { - #[display(fmt="{}", _0)] - Rrdpd(rrdpd::Error), - - #[display(fmt="{}", _0)] - Rsyncd(rsyncd::Error), -} - -impl From for Error { - fn from(e: rrdpd::Error) -> Self { Error::Rrdpd(e) } -} - -impl From for Error { - fn from(e: rsyncd::Error) -> Self { Error::Rsyncd(e) } -} - - - -//------------ Tests --------------------------------------------------------- - -#[cfg(test)] -mod tests { - - use super::*; - use bytes::Bytes; - use crate::util::file::CurrentFile; - use crate::util::test; - use crate::api::publication::PublishDeltaBuilder; - use crate::api::rrdp_data; - - #[test] - fn should_publish() { - test::test_with_tmp_dir(|d| { - let rrdp_base_uri = test::http_uri("http://localhost:3000/repo/"); - let mut repo = Repository::build(&rrdp_base_uri, &d).unwrap() ; - - // Publish a file - let rsync_for_alice = - test::rsync_uri("rsync://host:10873/module/alice"); - let file = CurrentFile::new( - test::rsync_uri("rsync://host:10873/module/alice/file.txt"), - Bytes::from("example content") - ); - - let mut builder = PublishDeltaBuilder::new(); - builder.add_publish(file.as_publish()); - let delta = builder.finish(); - - repo.publish(&delta, &rsync_for_alice).unwrap(); - - // Now publish an update a bunch of times - // (overwrite file with same file, strictly speaking allowed, - // and convenient here) - - let file_update = file.clone(); - - let mut builder = PublishDeltaBuilder::new(); - builder.add_update(file_update.as_update(file.hash())); - let delta = builder.finish(); - - repo.publish(&delta, &rsync_for_alice).unwrap(); - repo.publish(&delta, &rsync_for_alice).unwrap(); - repo.publish(&delta, &rsync_for_alice).unwrap(); - repo.publish(&delta, &rsync_for_alice).unwrap(); - repo.publish(&delta, &rsync_for_alice).unwrap(); - - // Now we expect a notification file with serial 6, which only - // includes deltas for 5 and 6, because more deltas would - // exceed the size of the snapshot. - - let mut rrdp_disk_path = d.clone(); - rrdp_disk_path.push("rrdp"); - - let mut notification_disk_path = rrdp_disk_path.clone(); - notification_disk_path.push("notification.xml"); - - match rrdp_data::Notification::build( - ¬ification_disk_path, - &rrdp_base_uri, - &rrdp_disk_path - ) { - Some(notification) => { - let expected_serial: usize = 6; - let expected_prev: usize = 5; - assert_eq!(notification.serial(), expected_serial); - - let deltas = notification.deltas(); - assert_eq!(2, deltas.len()); - - assert!( - deltas.iter().find(|d| { - d.serial() == &expected_serial} - ).is_some() - ); - - assert!( - deltas.iter().find(|d| { - d.serial() == &expected_prev} - ).is_some() - ); - }, - None => panic!("Should have derived notification"), - } - }); - } - - #[test] - fn should_store_list_withdraw_files() { - test::test_with_tmp_dir(|d| { - let mut file_store = rsyncd::FileStore::build(&d).unwrap(); - - // Using a port here to make sure that it works in mapping - // the rsync URI to and from disk. - let base_uri = test::rsync_uri("rsync://host:10873/module/alice/"); - - // Publish a file - let file = CurrentFile::new( - test::rsync_uri("rsync://host:10873/module/alice/file.txt"), - Bytes::from("example content") - ); - - let mut builder = PublishDeltaBuilder::new(); - builder.add_publish(file.as_publish()); - let delta = builder.finish(); - - file_store.publish(&delta, &base_uri).unwrap(); - - // See that it's the only one listed - let files = file_store.list(&base_uri).unwrap(); - assert_eq!(1, files.len()); - assert!(files.contains(&file)); - - // Update a file - let file_update = CurrentFile::new( - file.uri().clone(), - Bytes::from("example updated content") - ); - - let mut builder = PublishDeltaBuilder::new(); - builder.add_update(file_update.as_update(file.hash())); - let delta = builder.finish(); - file_store.publish(&delta, &base_uri).unwrap(); - - // See that it's the only one listed - let files = file_store.list(&base_uri).unwrap(); - assert_eq!(1, files.len()); - assert!(files.contains(&file_update)); - - // Withdraw a file - let mut builder = PublishDeltaBuilder::new(); - builder.add_withdraw(file_update.as_withdraw()); - let delta = builder.finish(); - file_store.publish(&delta, &base_uri).unwrap(); - - // See that there are no files listed - let files = file_store.list(&base_uri).unwrap(); - assert_eq!(0, files.len()); - }); - } - - #[test] - fn should_not_allow_publishing_or_withdrawing_outside_of_base() { - test::test_with_tmp_dir(|d| { - let mut file_store = rsyncd::FileStore::build(&d).unwrap(); - - // Using a port here to make sure that it works in mapping - // the rsync URI to and from disk. - let base_uri = test::rsync_uri("rsync://host:10873/module/alice/"); - - // Publish a file - let file = CurrentFile::new( - test::rsync_uri("rsync://host:10873/module/bob/file.txt"), - Bytes::from("example content") - ); - - let mut builder = PublishDeltaBuilder::new(); - builder.add_publish(file.as_publish()); - let delta = builder.finish(); - - match file_store.publish(&delta, &base_uri) { - Err(rsyncd::Error::OutsideBaseUri) => {}, - _ => { panic!("Expected Error::OutsideBaseUri") } - } - }); - } -} - - +} \ No newline at end of file diff --git a/src/krilld/pubd/rrdpd.rs b/src/krilld/pubd/rrdpd.rs deleted file mode 100644 index 425298bf..00000000 --- a/src/krilld/pubd/rrdpd.rs +++ /dev/null @@ -1,489 +0,0 @@ -use std::io; -use std::fs; -use std::num::ParseIntError; -use std::path::PathBuf; -use bytes::Bytes; -use rpki::uri; -use crate::api::publication; -use crate::api::rrdp_data::{self, Notification}; -use crate::api::rrdp_data::Snapshot; -use crate::api::rrdp_data::PublishedObject; -use crate::util::file::{self, RecursorError}; -use crate::storage::keystore::{self, Key, KeyStore}; -use crate::storage::caching_ks::CachingDiskKeyStore; -use std::sync::Arc; -use std::collections::HashMap; -use storage::keystore::Info; -use api::rrdp_data::FileInfo; -use api::rrdp_data::DeltaRef; -use util::xml::XmlWriter; -use api::rrdp_data::NotificationBuilder; -use api::rrdp_data::SnapshotRef; - -//const VERSION: &'static str = "1"; -//const NS: &'static str = "http://www.ripe.net/rpki/rrdp"; -const RRDP_FOLDER: &str = "rrdp"; -const FS_FOLDER: &str = "rsync"; - -const VERSION: &str = "1"; -const NS: &str = "http://www.ripe.net/rpki/rrdp"; - - - -/// This type publishes RRDP notifications, snapshots and deltas so that they -/// can be served to relying parties. -#[derive(Clone, Debug)] -pub struct RrdpServer { - store: CachingDiskKeyStore, - - // The base URI path for notification, snapshot and delta files. - base_uri: uri::Http, - - // Dir for notification, snapshot and delta files. - rrdp_base: PathBuf, - - // Dir for file_store (so that snapshots can be derived) - fs_base: PathBuf -} - -/// # Setup and initialisation -impl RrdpServer { - - /// Creates a new RrdpServer. - /// - /// This will pick up the saved state from the notification.xml if - /// present, or initialise a new server with a random session_id, - /// starting at serial 1, and including a snapshot for everything - /// currently stored in the rsync file_store. - pub fn build( - base_uri: &uri::Http, - work_dir: &PathBuf - ) -> Result - { - if ! base_uri.to_string().ends_with('/') { - return Err(Error::UriConfigError) - } - - let mut rrdp_store_dir = PathBuf::from(work_dir); - rrdp_store_dir.push("rrdp_store"); - if ! rrdp_store_dir.is_dir() { - fs::create_dir_all(&rrdp_store_dir)?; - } - - let store = CachingDiskKeyStore::build(rrdp_store_dir)?; - - let rrdp_base = file::sub_dir(work_dir, RRDP_FOLDER)?; - let fs_base = file::sub_dir(work_dir, FS_FOLDER)?; - Ok(RrdpServer { store, base_uri: base_uri.clone(), rrdp_base, fs_base }) - } -} - -/// # Storing, Retrieving, and referencing Notification, Snapshot, Deltas -/// -impl RrdpServer { - -// const REL_NOTIFICATION: &'static str = "notification.xml"; - - fn key_notification() -> Key { - Key::new("notification") - } - - fn key_snapshot(session: &str, serial: usize) -> Key { - Key::new(&format!("{}-{}-snapshot", session, serial)) - } - - pub fn get_notification( - &self - ) -> Result>, Error> { - let key = Self::key_notification(); - self.store.get(&key).map_err(Error::Keystore) - } - - pub fn get_snapshot( - &self, - session: &str, - serial: usize - ) -> Result>, Error> { - let key = Self::key_snapshot(session, serial); - self.store.get(&key).map_err(Error::Keystore ) - } - - pub fn save_notification( - &mut self, - notification: Notification - ) -> Result<(), Error> { - let key = Self::key_notification(); - self.store.store( - key, - notification, - Info::now("server", "notification") - ).map_err(Error::Keystore) - } - - pub fn save_snapshot( - &mut self, - session: &str, - serial: usize, - snapshot: Snapshot - ) -> Result<(), Error> { - let key = Self::key_snapshot(session, serial); - self.store.store( - key, - snapshot, - Info::now("server", "notification") - ).map_err(Error::Keystore) - } -} - -/// # Publishing -/// -impl RrdpServer { - - fn verified_rel( - uri: &uri::Rsync, - base_uri: &uri::Rsync - ) -> Result { - match uri.relative_to(base_uri) { - Some(rel) => unsafe { // uri ensures characters are safe - Ok(std::str::from_utf8_unchecked(rel).to_string()) - }, - None => Err(Error::OutsideBaseUri) - } - } - - - /// Process an update PublishQuery and produce a new delta, snapshot - /// and notification file. Assumes that this is called *after* the - /// ['FileStore'] has published, so files should already be saved to - /// disk and the snapshots can be derived from this. - pub fn publish( - &mut self, - delta: &publication::PublishDelta, - base_uri: &uri::Rsync - ) -> Result<(), Error> { - - let (session, serial, deltas) = match self.get_notification()? { - Some(notification) => { - ( - notification.session_id().clone(), - notification.serial(), - notification.deltas().clone() - ) - }, - None => { - ( - { - use rand::{thread_rng, Rng}; - let mut rng = thread_rng(); - let rnd: u32 = rng.gen(); - format!("{}", rnd) - }, - 0, - Vec::new() - ) - } - }; - - let mut all_objects = match self.get_snapshot(&session, serial)? { - Some(snapshot) => snapshot.objects().clone(), - None => HashMap::new() - }; - - let mut objects = match all_objects.get(&base_uri.to_string()) { - Some(objects) => objects.clone(), - None => Vec::new() - }; - - for p in delta.publishes() { - let _rel = Self::verified_rel(p.uri(), base_uri)?; - let object = PublishedObject::new(p.uri().clone(), p.content().clone()); - if objects.contains(&object) { - return Err(Error::ObjectAlreadyPresent(p.uri().clone())) - } - objects.push(object); - } - - for u in delta.updates() { - let _rel = Self::verified_rel(u.uri(), base_uri)?; - - match objects.iter().position(|cur| {cur.uri() == u.uri()}) { - None => return Err(Error::NoObjectPresent(u.uri().clone())), - Some(pos) => { - if objects[pos].hash() != u.hash() { - return Err(Error::NoObjectMatchingHash) - } else { - objects.remove(pos); - } - } - } - - let object = PublishedObject::new(u.uri().clone(), u.content().clone()); - objects.push(object); - } - - for w in delta.withdraws() { - let _rel = Self::verified_rel(w.uri(), base_uri)?; - match objects.iter().position(|cur| {cur.uri() == w.uri()}) { - None => return Err(Error::NoObjectPresent(w.uri().clone())), - Some(pos) => { - if objects[pos].hash() != w.hash() { - return Err(Error::NoObjectMatchingHash) - } else { - objects.remove(pos); - } - } - } - } - - all_objects.insert(base_uri.to_string(), objects); - - let new_serial = serial + 1; - - // Create new snapshot - let snapshot = Snapshot::new( - session.clone(), - new_serial, - all_objects - ); - - // Save the xml to disk - let snapshot_ref = { - let xml = snapshot.to_xml(); - let path = self.snapshot_path(&session, new_serial); - file::save(&Bytes::from(xml), &path)?; - SnapshotRef::new( - FileInfo::for_path_and_uri( - &path, self.snapshot_uri(&session, new_serial) - )? - ) - }; - - // save to store - self.save_snapshot(&session, new_serial, snapshot)?; - - // Create new delta - let delta_ref = self.save_delta(&session, new_serial, delta)?; - - // Create new notification file - let new_notification = { - let mut builder = NotificationBuilder::new(); - builder.with_session_id(session); - builder.with_serial(new_serial); - builder.with_deltas(deltas); - builder.add_delta_to_start(delta_ref); - builder.with_snapshot(snapshot_ref); - builder.build() - }; - - - let path = self.notification_path(); - new_notification.save(&path)?; - - self.save_notification(new_notification)?; - - Ok(()) - } - - - /// Saves the RFC8181 PublishQuery as an RFC8182 delta file. - fn save_delta( - &mut self, - session_id: &str, - serial: usize, - delta: &publication::PublishDelta - ) -> Result - { - let path = self.delta_path(session_id, serial); - debug!("Writing delta: {}", path.to_string_lossy()); - let mut file = file::create_file_with_path(&path)?; - - XmlWriter::encode_to_file(& mut file, |w| { - - let a = [ - ("xmlns", NS), - ("version", VERSION), - ("session_id", session_id), - ("serial", &format!("{}", serial)), - ]; - - w.put_element( - "delta", - Some(&a), - |w| { - for publish in delta.publishes() { - let uri = publish.uri().to_string(); - let a = [ - ("uri", uri.as_ref()) - ]; - w.put_element( - "publish", - Some(&a), - |w| { - w.put_blob(publish.content()) - } - )? - } - - for update in delta.updates() { - let uri = update.uri().to_string(); - let hash = hex::encode(update.hash()); - let a = [ - ("uri", uri.as_ref()), - ("hash", hash.as_ref()) - ]; - w.put_element( - "publish", - Some(&a), - |w| { - w.put_blob(update.content()) - } - )? - } - - for withdraw in delta.withdraws() { - let uri = withdraw.uri().to_string(); - let hash = hex::encode(withdraw.hash()); - let a = [ - ("uri", uri.as_ref()), - ("hash", hash.as_ref()) - ]; - w.put_element( - "withdraw", - Some(&a), - |w| { - w.empty() - } - )? - } - Ok(()) - }) - })?; - - let file_info = FileInfo::for_path( - &path, - &self.base_uri, - &self.rrdp_base - )?; - Ok(DeltaRef::new(serial, file_info)) - } - - pub fn notification_uri(&self) -> uri::Http { - uri::Http::from_string( - format!("{}notification.xml", self.base_uri.to_string()) - ).unwrap() // Cannot fail. Config checked at startup. - } - - pub fn snapshot_uri(&self, session: &str, serial: usize) -> uri::Http { - uri::Http::from_string( - format!("{}{}/{}/snapshot.xml", - self.base_uri.to_string(), - session, - serial - ) - ).unwrap() // Cannot fail. Config checked at startup. - } - - pub fn delta_uri(&self, session: &str, serial: usize) -> uri::Http { - uri::Http::from_string( - format!("{}{}/{}/snapshot.xml", - self.base_uri.to_string(), - session, - serial - ) - ).unwrap() // Cannot fail. Config checked at startup. - } - - pub fn notification_path(&self) -> PathBuf { - let mut path = self.rrdp_base.clone(); - path.push("notification.xml"); - path - } - - pub fn delta_path(&self, session: &str, serial: usize) -> PathBuf { - let mut path = self.serial_path(session, serial); - path.push("delta.xml"); - path - } - - pub fn snapshot_path(&self, session: &str, serial: usize) -> PathBuf { - let mut path = self.serial_path(session, serial); - path.push("snapshot.xml"); - path - } - - fn serial_path(&self, session: &str, serial: usize) -> PathBuf { - let mut path = self.rrdp_base.clone(); - path.push(session); - path.push(format!("{}", serial)); - path - } -} - - - - -//------------ Error --------------------------------------------------------- - -#[derive(Debug, Display)] -pub enum Error { - #[display(fmt="{}", _0)] - IoError(io::Error), - - #[display(fmt="{}", _0)] - Keystore(keystore::Error), - - #[display(fmt="{}", _0)] - RecursorError(RecursorError), - - #[display(fmt="{}", _0)] - UriError(uri::Error), - - #[display(fmt="File already exists for uri (use update!): {}", _0)] - ObjectAlreadyPresent(uri::Rsync), - - #[display(fmt="Np file present for uri: {}", _0)] - NoObjectPresent(uri::Rsync), - - #[display(fmt="File does not match hash")] - NoObjectMatchingHash, - - #[display(fmt="Publishing outside of base URI is not allowed.")] - OutsideBaseUri, - - #[display(fmt="Issue deriving RRDP URI, check config. Base URI must end with a '/'!")] - UriConfigError, - - #[display(fmt="Error deserializing existing notification.xml")] - NotificationFileError, - - #[display(fmt="{}", _0)] - ParseIntError(ParseIntError), - - #[display(fmt="{}", _0)] - RrdpData(rrdp_data::Error) -} - -impl From for Error { - fn from(e: io::Error) -> Self { Error::IoError(e) } -} - -impl From for Error { - fn from(e: keystore::Error) -> Self { Error::Keystore(e) } -} - -impl From for Error { - fn from(e: uri::Error) -> Self { Error::UriError(e) } -} - -impl From for Error { - fn from(e: ParseIntError) -> Self { Error::ParseIntError(e) } -} - -impl From for Error { - fn from(e: rrdp_data::Error) -> Self { Error::RrdpData(e) } -} - -impl From for Error { - fn from(e: RecursorError) -> Self { Error::RecursorError(e) } -} - diff --git a/src/krilld/pubd/rsyncd.rs b/src/krilld/pubd/rsyncd.rs deleted file mode 100644 index 55cc75c7..00000000 --- a/src/krilld/pubd/rsyncd.rs +++ /dev/null @@ -1,207 +0,0 @@ -use std::{io, fs}; -use std::path::PathBuf; -use rpki::uri; -use crate::api::publication; -use crate::krilld::pubd::RSYNC_FOLDER; -use crate::util::file::{self, CurrentFile, RecursorError}; - - -//------------ FileStore ----------------------------------------------------- - -/// This type is responsible for publishing files on disk in a structure so -/// that an rscynd can be set up to serve this (RPKI) data. Note that the -/// rsync host name and module are part of the path, so make sure that the -/// rsyncd modules and paths are setup properly for each supported rsync -/// base uri used. -#[derive(Clone, Debug)] -pub struct FileStore { - base_dir: PathBuf -} - -/// # Construct -/// -impl FileStore { - pub fn build(work_dir: &PathBuf) -> Result { - let mut rsync_dir = PathBuf::from(work_dir); - rsync_dir.push(RSYNC_FOLDER); - if ! rsync_dir.is_dir() { - fs::create_dir_all(&rsync_dir)?; - } - Ok ( FileStore { base_dir: rsync_dir } ) - } -} - -/// # Publishing -/// -impl FileStore { - /// Process a PublishQuery update - pub fn publish( - &mut self, - delta: &publication::PublishDelta, - base_uri: &uri::Rsync - ) -> Result<(), Error> { - self.verify_query(delta, base_uri)?; - self.update_files(delta)?; - - Ok(()) - } - - pub fn list( - &self, - base_uri: &uri::Rsync - ) -> Result, Error> { - let path = self.path_for_publisher(base_uri); - - if !path.exists() { - Ok(Vec::new()) - } else { - file::crawl_incl_rsync_base(&path, base_uri) - .map_err(Error::RecursorError) - } - } - - /// Assert that all updates are confined to the given base_uri; i.e. do - /// not allow publishers to update things outside of their own jail. - /// - /// Note this is done as a separate check, because there is a requirement - /// that in case of any errors in an update, nothing is published. So, - /// checking this first is a good enough form of a poor-man's transaction. - fn verify_query( - &self, - delta: &publication::PublishDelta, - base_uri: &uri::Rsync - ) -> Result<(), Error> { - - for p in delta.publishes() { - Self::assert_uri(base_uri, p.uri())?; - if self.get_current_file_opt(p.uri()).is_some() { - return Err(Error::ObjectAlreadyPresent(p.uri().clone())) - } - } - - for u in delta.updates() { - Self::assert_uri(base_uri, u.uri())?; - if let Some(cur) = self.get_current_file_opt(u.uri()) { - if cur.hash() != u.hash() { - return Err(Error::NoObjectMatchingHash) - } - } else { - return Err(Error::NoObjectPresent(u.uri().clone())) - } - } - - for w in delta.withdraws() { - Self::assert_uri(base_uri, w.uri())?; - if let Some(cur) = self.get_current_file_opt(w.uri()) { - if cur.hash() != w.hash() { - return Err(Error::NoObjectMatchingHash) - } - } else { - return Err(Error::NoObjectPresent(w.uri().clone())) - } - } - - debug!("Update is consistent with current state"); - Ok(()) - } - - /// Perform the actual updates on disk. This assumes that the updates - /// have been verified. This can still blow up if there is an I/O issue - /// writing to disk. - fn update_files( - &self, - delta: &publication::PublishDelta - ) -> Result<(), Error> { - for p in delta.publishes() { - debug!("Saving file for uri: {}", p.uri().to_string()); - file::save_with_rsync_uri( - p.content(), - &self.base_dir, - p.uri() - )?; - } - - for u in delta.updates() { - debug!("Updating file for uri: {}", u.uri().to_string()); - file::save_with_rsync_uri( - u.content(), - &self.base_dir, - u.uri() - )?; - } - - for w in delta.withdraws() { - debug!("Withdrawing file for uri: {}", w.uri().to_string()); - file::delete_with_rsync_uri( - &self.base_dir, - w.uri() - )?; - } - - Ok(()) - } - - fn assert_uri(base: &uri::Rsync, file: &uri::Rsync) -> Result<(), Error> { - if base.module() == file.module() && - file.path().starts_with(base.path()) { - Ok(()) - } else { - Err(Error::OutsideBaseUri) - } - } - - fn get_current_file_opt( - &self, - file_uri: &uri::Rsync - ) -> Option { - match file::read_with_rsync_uri(&self.base_dir, file_uri) { - Ok(bytes) => Some(CurrentFile::new(file_uri.clone(), bytes)), - Err(_) => None - } - } - - // Returns the relative sub-dir that we should scan for this particular - // publisher. - fn path_for_publisher(&self, file_uri: &uri::Rsync) -> PathBuf { - let mut path = self.base_dir.clone(); - let module = file_uri.module(); - path.push(PathBuf::from(module.authority())); - path.push(PathBuf::from(module.module())); - path.push(PathBuf::from(file_uri.path())); - path - } -} - - -//------------ Error --------------------------------------------------------- - -#[derive(Debug, Display)] -pub enum Error { - #[display(fmt = "{}", _0)] - IoError(io::Error), - - #[display(fmt="{}", _0)] - RecursorError(RecursorError), - - #[display(fmt="File already exists for uri (use update!): {}", _0)] - ObjectAlreadyPresent(uri::Rsync), - - #[display(fmt="Np file present for uri: {}", _0)] - NoObjectPresent(uri::Rsync), - - #[display(fmt="File does not match hash")] - NoObjectMatchingHash, - - #[display(fmt="Publishing outside of base URI is not allowed.")] - OutsideBaseUri, -} - -impl From for Error { - fn from(e: io::Error) -> Self { - Error::IoError(e) - } -} - -impl From for Error { - fn from(e: RecursorError) -> Self { Error::RecursorError(e) } -} diff --git a/src/lib.rs b/src/lib.rs index 481dd2cd..0e495118 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,6 +28,7 @@ extern crate ring; extern crate untrusted; pub mod api; +pub mod eventsourcing; pub mod krillc; pub mod krilld; pub mod pubc; diff --git a/src/pubc/apiclient.rs b/src/pubc/apiclient.rs index 1739e08a..db08ba30 100644 --- a/src/pubc/apiclient.rs +++ b/src/pubc/apiclient.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use clap::{App, Arg, SubCommand}; use rpki::uri; -use crate::api::publication; +use crate::api::publication_data; use crate::pubc; use crate::util::httpclient; @@ -190,7 +190,7 @@ impl Format { #[derive(Clone, Debug, Eq, PartialEq)] pub enum ApiResponse { Success, - List(publication::ListReply), + List(publication_data::ListReply), } impl ApiResponse { @@ -211,7 +211,7 @@ impl ApiResponse { ApiResponse::List(list) => { for el in list.elements() { println!("{} {}", - hex::encode(el.hash()), + el.hash().to_string(), el.uri().to_string() ); } @@ -238,14 +238,14 @@ pub fn execute(options: Options) -> Result { } -fn list_query(connection: &Connection) -> Result { +fn list_query(connection: &Connection) -> Result { let uri = format!( "{}publication/{}", &connection.server_uri.to_string(), &connection.handle ); - match httpclient::get_json::( + match httpclient::get_json::( &uri, Some(&connection.token) ) { diff --git a/src/pubc/cmsclient.rs b/src/pubc/cmsclient.rs index ba925aeb..b4ca3e51 100644 --- a/src/pubc/cmsclient.rs +++ b/src/pubc/cmsclient.rs @@ -8,7 +8,8 @@ use clap::{App, Arg, SubCommand}; use rpki::x509::ValidationError; use rpki::crypto::{PublicKeyFormat, Signer}; use toml; -use crate::api::publication; +use crate::api::publication_data; +use crate::pubc; use crate::remote::builder; use crate::remote::builder::{IdCertBuilder, SignedMessageBuilder}; use crate::remote::id::{MyIdentity, MyRepoInfo, ParentInfo}; @@ -19,7 +20,6 @@ use crate::storage::caching_ks::CachingDiskKeyStore; use crate::storage::keystore::{self, Info, Key, KeyStore}; use crate::util::httpclient; use crate::util::softsigner::{self, OpenSslSigner}; -use pubc; /// # Some constants for naming resources in the keystore for clients. @@ -178,7 +178,7 @@ impl PubClient { /// validly signed and all. pub fn get_server_list( &mut self - ) -> Result { + ) -> Result { let query = rfc8181::Message::list_query(); let signed_request = self.sign_request(query)?; diff --git a/src/pubc/mod.rs b/src/pubc/mod.rs index 2ef0697b..97ab2392 100644 --- a/src/pubc/mod.rs +++ b/src/pubc/mod.rs @@ -3,15 +3,15 @@ pub mod cmsclient; use std::path::PathBuf; use rpki::uri; -use crate::api::publication; +use crate::api::publication_data; use crate::util::file; pub fn create_delta( - list_reply: &publication::ListReply, + list_reply: &publication_data::ListReply, dir: &PathBuf, base_rsync: &uri::Rsync -) -> Result { - let mut delta_builder = publication::PublishDeltaBuilder::new(); +) -> Result { + let mut delta_builder = publication_data::PublishDeltaBuilder::new(); let current = file::crawl_incl_rsync_base(dir, base_rsync)?; @@ -19,7 +19,7 @@ pub fn create_delta( for p in list_reply.elements() { if current.iter().find(|c| c.uri() == p.uri()).is_none() { delta_builder.add_withdraw( - publication::Withdraw::from_list_element(p) + publication_data::Withdraw::from_list_element(p) ); } } diff --git a/src/remote/cmsproxy.rs b/src/remote/cmsproxy.rs index eeb29061..b64e7bee 100644 --- a/src/remote/cmsproxy.rs +++ b/src/remote/cmsproxy.rs @@ -1,12 +1,10 @@ -use std::sync::Arc; use std::path::PathBuf; use bcder::Captured; use rpki::uri; use rpki::x509::ValidationError; -use crate::api::publishers; -use crate::api::publication; -use crate::krilld::pubd::repo; -use crate::krilld::pubd::rsyncd; +use crate::api::publication_data; +use crate::krilld::pubd; +use crate::krilld::pubd::publishers::Publisher; use crate::remote::id::IdCert; use crate::remote::responder; use crate::remote::responder::Responder; @@ -52,7 +50,7 @@ impl CmsProxy { &mut self, msg: &SignedMessage, id_cert: &IdCert, - ) -> Result { + ) -> Result { debug!("Validating Signed Message"); msg.validate(id_cert)?; let msg = rfc8181::Message::from_signed_message(&msg)?; @@ -64,13 +62,13 @@ impl CmsProxy { /// in signed CMS pub fn wrap_publish_reply( &mut self, - reply: publication::PublishReply + reply: publication_data::PublishReply ) -> Result { let msg = match reply { - publication::PublishReply::Success => { + publication_data::PublishReply::Success => { rfc8181::Message::success_reply() }, - publication::PublishReply::List(list) => { + publication_data::PublishReply::List(list) => { rfc8181::Message::list_reply(list) } }; @@ -98,7 +96,7 @@ impl CmsProxy { /// Returns an RFC8183 Repository Response pub fn repository_response( &self, - publisher: &Arc, + publisher: &Publisher, base_service_uri: &uri::Http, rrdp_notification_uri: uri::Http ) -> Result { @@ -106,22 +104,20 @@ impl CmsProxy { let service_uri = format!( "{}rfc8181/{}", base_service_uri.to_string(), - publisher.handle() + publisher.id() ); let service_uri = uri::Http::from_string(service_uri).unwrap(); self.responder .repository_response( - &publisher, + publisher, service_uri, rrdp_notification_uri) .map_err(Error::ResponderError) } - - - } + //------------ Error --------------------------------------------------------- #[derive(Debug, Display)] @@ -176,22 +172,8 @@ impl ToReportErrorCode for rfc8181::MessageError { } } -impl ToReportErrorCode for repo::Error { +impl ToReportErrorCode for pubd::Error { fn to_report_error_code(&self) -> rfc8181::ReportErrorCode { - match self { - repo::Error::Rsyncd( - rsyncd::Error::ObjectAlreadyPresent(_)) => - rfc8181::ReportErrorCode::ObjectAlreadyPresent, - repo::Error::Rsyncd( - rsyncd::Error::NoObjectPresent(_)) => - rfc8181::ReportErrorCode::NoObjectPresent, - repo::Error::Rsyncd( - rsyncd::Error::NoObjectMatchingHash) => - rfc8181::ReportErrorCode::NoObjectMatchingHash, - repo::Error::Rsyncd( - rsyncd::Error::OutsideBaseUri) => - rfc8181::ReportErrorCode::PermissionFailure, - _ => rfc8181::ReportErrorCode::OtherError - } + unimplemented!() } } diff --git a/src/remote/responder.rs b/src/remote/responder.rs index 092759a7..1c1d29b7 100644 --- a/src/remote/responder.rs +++ b/src/remote/responder.rs @@ -6,7 +6,7 @@ use bcder::{Captured, Mode}; use bcder::encode::Values; use rpki::crypto::{PublicKeyFormat, Signer}; use rpki::uri; -use crate::api::publishers::Publisher; +use crate::krilld::pubd::publishers::Publisher; use crate::remote::builder; use crate::remote::builder::{IdCertBuilder, SignedMessageBuilder}; use crate::remote::id::MyIdentity; @@ -95,7 +95,7 @@ impl Responder { impl Responder { pub fn repository_response( &self, - publisher: &Arc, + publisher: &Publisher, service_uri: uri::Http, rrdp_notification_uri: uri::Http ) -> Result { @@ -107,7 +107,7 @@ impl Responder { }; - let handle = publisher.handle(); + let handle = publisher.id(); let id_cert = my_id.id_cert().clone(); let sia_base = publisher.base_uri().clone(); @@ -115,7 +115,7 @@ impl Responder { Ok( RepositoryResponse::new( tag, - handle.clone(), + handle.to_string(), id_cert, service_uri, sia_base, @@ -196,8 +196,11 @@ impl From> for Error { #[cfg(test)] mod tests { use super::*; + use crate::api::publisher_data::CmsAuthData; + use crate::api::publisher_data::PublisherRequest; + use crate::eventsourcing::Aggregate; + use crate::krilld::pubd::publishers::PublisherInit; use crate::util::test; - use crate::api::publishers::CmsAuthData; #[test] fn should_have_response_for_publisher() { @@ -214,12 +217,17 @@ mod tests { let cms_auth = CmsAuthData::new(tag, id_cert); - let publisher = Arc::new(Publisher::new( + let publisher_request = PublisherRequest::new( name, "token".to_string(), base_uri, Some(cms_auth) - )); + ); + + let init = PublisherInit::from(publisher_request); + + let publisher = Publisher::init(init).unwrap(); + let rrdp_uri = test::http_uri("http://host/rrdp/"); diff --git a/src/remote/rfc8181.rs b/src/remote/rfc8181.rs index 3d12d0bd..d3870cbf 100644 --- a/src/remote/rfc8181.rs +++ b/src/remote/rfc8181.rs @@ -3,7 +3,8 @@ use std::io; use bytes::Bytes; use rpki::uri; -use crate::api::publication; +use crate::api::publication_data; +use crate::api::{Base64, EncodedHash}; use crate::remote::sigmsg::SignedMessage; use crate::util::xml::{ Attributes, @@ -125,7 +126,7 @@ impl Message { /// impl Message { - pub fn list_reply(reply: publication::ListReply) -> Self { + pub fn list_reply(reply: publication_data::ListReply) -> Self { Message::ReplyMessage(ReplyMessage::ListReply(reply)) } @@ -133,7 +134,7 @@ impl Message { Message::ReplyMessage(ReplyMessage::SuccessReply) } - pub fn publish_delta_query(delta: publication::PublishDelta) -> Self { + pub fn publish_delta_query(delta: publication_data::PublishDelta) -> Self { Message::QueryMessage(QueryMessage::PublishDelta(delta)) } @@ -148,7 +149,7 @@ impl Message { /// This type represents query type Publication Messages defined in RFC8181 #[derive(Clone, Debug, Eq, PartialEq)] pub enum QueryMessage { - PublishDelta(publication::PublishDelta), + PublishDelta(publication_data::PublishDelta), ListQuery } @@ -207,10 +208,10 @@ impl QueryMessage { } /// Consumes this and returns this a PublishRequest for our (json) API - pub fn into_publish_request(self) -> publication::PublishRequest { + pub fn into_publish_request(self) -> publication_data::PublishRequest { match self { - QueryMessage::ListQuery => publication::PublishRequest::List, - QueryMessage::PublishDelta(d) => publication::PublishRequest::Delta(d) + QueryMessage::ListQuery => publication_data::PublishRequest::List, + QueryMessage::PublishDelta(d) => publication_data::PublishRequest::Delta(d) } } } @@ -224,9 +225,9 @@ pub struct PublishDeltaXml; #[derive(Clone, Debug, Eq, PartialEq)] pub enum PublishDeltaElement { - Publish(publication::Publish), - Update(publication::Update), - Withdraw(publication::Withdraw) + Publish(publication_data::Publish), + Update(publication_data::Update), + Withdraw(publication_data::Withdraw) } impl PublishDeltaElement { @@ -257,17 +258,19 @@ impl PublishDeltaXml { let uri = uri::Rsync::from_string(a.take_req("uri")?)?; let tag = a.take_req("tag")?; - let object = r.take_bytes_characters()?; + let base64_string = r.take_chars()?; + let base64 = Base64::from(base64_string); - let res = match a.take_opt_hex("hash") { - Some(hash) => { - let update = publication::Update::new( - Some(tag), uri, object, hash + let res = match a.take_opt("hash") { + Some(hash_str) => { + let hash = EncodedHash::from(hash_str); + let update = publication_data::Update::new( + Some(tag), uri, base64, hash ); Ok(PublishDeltaElement::Update(update)) }, None => { - let publish = publication::Publish::new(Some(tag), uri, object); + let publish = publication_data::Publish::new(Some(tag), uri, base64); Ok(PublishDeltaElement::Publish(publish)) } }; @@ -281,13 +284,14 @@ impl PublishDeltaXml { a: &mut Attributes ) -> Result { - let hash = a.take_req_hex("hash")?; + let hash_str = a.take_req("hash")?; + let hash = EncodedHash::from(hash_str); let uri = uri::Rsync::from_string(a.take_req("uri")?)?; let tag = a.take_req("tag")?; a.exhausted()?; - let withdraw = publication::Withdraw::new(Some(tag), uri, hash); + let withdraw = publication_data::Withdraw::new(Some(tag), uri, hash); Ok(PublishDeltaElement::Withdraw(withdraw)) } @@ -318,8 +322,8 @@ impl PublishDeltaXml { /// is processed by PublicationMessage::decode pub fn decode( r: &mut XmlReader - ) -> Result { - let mut bld = publication::PublishDeltaBuilder::new(); + ) -> Result { + let mut bld = publication_data::PublishDeltaBuilder::new(); while let Some(pde) = Self::decode_opt(r)? { match pde { @@ -339,7 +343,7 @@ impl PublishDeltaXml { /// Encodes a PublishDelta to XML in the given writer, for inclusion in an /// RFC8181 CMS. pub fn encode( - delta: &publication::PublishDelta, + delta: &publication_data::PublishDelta, w: &mut XmlWriter ) -> Result<(), io::Error> { for p in delta.publishes() { Self::encode_publish(p, w)?; } @@ -349,12 +353,13 @@ impl PublishDeltaXml { } fn encode_publish( - publish: &publication::Publish, + publish: &publication_data::Publish, w: &mut XmlWriter ) -> Result<(), io::Error> { let uri = publish.uri().to_string(); let tag = publish.tag_for_xml(); + let content = publish.content().to_string(); let a = [ ("tag", tag.as_ref()), @@ -365,23 +370,22 @@ impl PublishDeltaXml { "publish", Some(&a), |w| { - w.put_blob(publish.content()) + w.put_text(content.as_ref()) } ) } fn encode_update( - update: &publication::Update, + update: &publication_data::Update, w: &mut XmlWriter ) -> Result<(), io::Error> { let uri = update.uri().to_string(); - let enc = hex::encode(update.hash()); let tag = update.tag_for_xml(); let a = [ ("tag", tag.as_ref()), - ("hash", enc.as_ref()), + ("hash", update.hash().as_ref()), ("uri", uri.as_ref()) ]; @@ -389,22 +393,21 @@ impl PublishDeltaXml { "publish", Some(&a), |w| { - w.put_blob(update.content()) + w.put_text(update.content().as_ref()) } ) } fn encode_withdraw( - withdraw: &publication::Withdraw, + withdraw: &publication_data::Withdraw, w: &mut XmlWriter ) -> Result<(), io::Error> { let uri = withdraw.uri().to_string(); - let hash = hex::encode(withdraw.hash()); let tag = withdraw.tag_for_xml(); let a = [ - ("hash", hash.as_ref()), + ("hash", withdraw.hash().as_ref()), ("tag", tag.as_ref()), ("uri", uri.as_ref()) ]; @@ -426,7 +429,7 @@ impl PublishDeltaXml { #[derive(Clone, Debug, Eq, PartialEq)] pub enum ReplyMessage { SuccessReply, - ListReply(publication::ListReply), + ListReply(publication_data::ListReply), ErrorReply(ErrorReply) } @@ -468,7 +471,7 @@ impl ReplyMessage { /// Decodes XML to a ListReply. fn decode_list_reply( r: &mut XmlReader - ) -> Result { + ) -> Result { let mut elements = vec![]; @@ -476,11 +479,11 @@ impl ReplyMessage { let e = r.take_opt_element(|t, mut a, _r| { match t.name.as_ref() { "list" => { - let hash = a.take_req_hex("hash")?; + let hash = EncodedHash::from(a.take_req("hash")?); let uri = uri::Rsync::from_string(a.take_req("uri")?)?; a.exhausted()?; - Ok(Some(publication::ListElement::new(uri, hash))) + Ok(Some(publication_data::ListElement::new(uri, hash))) }, _ => { Err(MessageError::UnexpectedStart(t.name.clone())) @@ -493,7 +496,7 @@ impl ReplyMessage { None => break } } - Ok(publication::ListReply::new(elements)) + Ok(publication_data::ListReply::new(elements)) } /// Encodes a ReplyMessage for inclusion in an RFC8181 Protocol CMS. @@ -512,17 +515,16 @@ impl ReplyMessage { /// Encodes a ListReply to XML. fn encode_list_reply( - reply: &publication::ListReply, + reply: &publication_data::ListReply, w: &mut XmlWriter ) -> Result<(), io::Error> { for el in reply.elements() { - let hash = hex::encode(&el.hash()); let uri = el.uri().to_string(); w.put_element( "list", - Some(&[("hash", hash.as_ref()), ("uri", uri.as_ref())]), + Some(&[("hash", el.hash().as_ref()), ("uri", uri.as_ref())]), |w| { w.empty() } )?; } @@ -892,13 +894,13 @@ mod tests { use super::*; use std::str; use rpki::uri::Rsync; - use crate::util::sha256; + use crate::api::EncodedHash; //------------ ListReplyBuilder ---------------------------------------------- /// This type is useful for testing pub struct ListReplyBuilder { - elements: Vec + elements: Vec } impl ListReplyBuilder { @@ -908,8 +910,8 @@ mod tests { } pub fn add(&mut self, object: &Bytes, uri: uri::Rsync) { - let hash = sha256(object); - let el = publication::ListElement::new(uri, hash); + let hash = EncodedHash::from_content(object); + let el = publication_data::ListElement::new(uri, hash); self.elements.push(el); } @@ -917,7 +919,7 @@ mod tests { /// protocol CMS message. pub fn build_message(self) -> Message { Message::list_reply( - publication::ListReply::new(self.elements) + publication_data::ListReply::new(self.elements) ) } } @@ -1013,7 +1015,8 @@ mod tests { #[test] fn should_create_error_reply() { let object = Bytes::from_static(include_bytes!("../../test/remote/cms_ta.cer")); - let publish = publication::Publish::with_hash_tag( + let object = Base64::from_content(&object); + let publish = publication_data::Publish::with_hash_tag( rsync_uri("rsync://host/path/cms-ta.cer"), object ); @@ -1049,26 +1052,28 @@ mod tests { fn should_encode_publish_delta() { let object = Bytes::from_static(include_bytes!("../../test/remote/cms_ta.cer")); let object2 = Bytes::from_static(include_bytes!("../../test/remote/pdu_200.der")); - let object_hash = sha256(&object); + let object_hash = EncodedHash::from_content(&object); + let object = Base64::from_content(&object); + let object2 = Base64::from_content(&object2); - let mut builder = publication::PublishDeltaBuilder::new(); + let mut builder = publication_data::PublishDeltaBuilder::new(); builder.add_withdraw( - publication::Withdraw::with_hash_tag( + publication_data::Withdraw::with_hash_tag( rsync_uri("rsync://host/path/cms-ta.cer"), object_hash.clone() ) ); builder.add_publish( - publication::Publish::with_hash_tag( + publication_data::Publish::with_hash_tag( rsync_uri("rsync://host/path/cms-ta.cer"), object ) ); builder.add_update( - publication::Update::with_hash_tag( + publication_data::Update::with_hash_tag( rsync_uri("rsync://host/path/cms-ta.cer"), object2, object_hash diff --git a/src/remote/rfc8183.rs b/src/remote/rfc8183.rs index d78ecf7f..b58ee2ae 100644 --- a/src/remote/rfc8183.rs +++ b/src/remote/rfc8183.rs @@ -9,7 +9,7 @@ use bcder::decode; use rpki::uri; use rpki::x509; use rpki::x509::Time; -use crate::api::publishers; +use crate::api::publisher_data; use crate::remote::id::IdCert; use crate::util::xml::{AttributesError, XmlReader, XmlReaderErr, XmlWriter}; @@ -130,11 +130,11 @@ impl PublisherRequest { self, token: String, base_uri: uri::Rsync - ) -> publishers::Publisher { - let cms_data = publishers::CmsAuthData::new( + ) -> publisher_data::PublisherRequest { + let cms_data = publisher_data::CmsAuthData::new( self.tag, self.id_cert ); - publishers::Publisher::new( + publisher_data::PublisherRequest::new( self.publisher_handle, token, base_uri, diff --git a/src/storage/events.rs b/src/storage/events.rs deleted file mode 100644 index 027bd1c9..00000000 --- a/src/storage/events.rs +++ /dev/null @@ -1,676 +0,0 @@ -use std::any::Any; -use std::collections::HashMap; -use std::fmt::Display; -use std::fs::File; -use std::io; -use std::io::Write; -use std::ops::Deref; -use std::path::PathBuf; -use std::sync::RwLock; -use serde::Serialize; -use serde::de::DeserializeOwned; -use serde_json; -use crate::util::file; -use std::sync::Arc; - - -//------------ Storable ------------------------------------------------------ - -pub trait Storable: Serialize + DeserializeOwned + Sized {} -impl Storable for T { } - - -//------------ AggregateId --------------------------------------------------- - -#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] -pub struct AggregateId(String); - -impl AggregateId { - pub fn new(s: &str) -> Self { - AggregateId(s.to_string()) - } -} - -//------------ Aggregate ----------------------------------------------------- - -pub trait Aggregate: Storable + Clone { - - type Command: Command; - type Event: Event; - type InitEvent: Event; - type Error: std::error::Error; - - /// Creates a new instance. Expects an event with data needed to - /// initialise the instance. Typically this means that a specific - /// 'create' event is passed, with all the needed data, or just an empty - /// marker if no data is needed. Implementations must return an error in - /// case the instance cannot be created. - fn init(event: Self::InitEvent) -> Result; - - /// Returns the current version of the aggregate. - fn version(&self) -> u64; - - /// Moves this, and applies the event to this. This MUST not result in - /// any errors. Applying the event just updates data and is side-effect - /// free. - /// - /// Note that both self and the event are moved. This is done because we - /// want to enable moving data into the new aggregate without the need for - /// additional allocations. - fn apply(&mut self, event: Self::Event); - - /// Processes a command. I.e. validate the command, and return a list of - /// events that will result in the desired new state, but do not apply - /// these event here. The command processing must be side-effect free. - /// - /// The command is moved, because we want to enable moving its data - /// without reallocating. - fn process_command(&self, command: Self::Command) -> Result, Self::Error>; -} - - -//------------ Event -------------------------------------------------------- - -pub trait Event: Storable { - /// Identifies the aggregate, useful when storing and retrieving the event. - fn id(&self) -> &AggregateId; - - /// The version of the aggregate that this event updates. - /// In other words, an aggregate that is currently at version x, will get - /// version x + 1, when the event for version x is applied. - fn version(&self) -> u64; -} - -#[derive(Deserialize, Serialize)] -pub struct StoredEvent { - id: AggregateId, - version: u64, - #[serde(deserialize_with = "E::deserialize")] - details: E -} - -impl StoredEvent { - pub fn new(id: &AggregateId, version: u64, event: E) -> Self { - StoredEvent { id: id.clone(), version, details: event } - } - - pub fn event(&self) -> &E { & self.details } - - pub fn into_details(self) -> E { self.details } - - pub fn unwrap(self) -> (AggregateId, u64, E) { - (self.id, self.version, self.details) - } -} - -impl Event for StoredEvent { - fn id(&self) -> &AggregateId { - &self.id - } - - fn version(&self) -> u64 { - self.version - } -} - -//------------ Command ------------------------------------------------------- - -/// Commands are used to send an intent to change an aggregate. -/// -/// Think of this as the data container for your update API, plus some -/// meta-data to ensure that the command is sent to the right instance of an -/// Aggregate, and that concurrency issues are handled. -pub trait Command: Storable { - /// Identify the type of event returned by the aggregate that uses this - /// command. This is needed because we may need to check whether a - /// command conflicts with recent events. - type Event: Event; - - /// Identifies the aggregate, useful when storing and retrieving the event. - fn id(&self) -> &AggregateId; - - /// The version of the aggregate that this command updates. If this - /// command should update whatever the latest version happens to be, then - /// use None here. - fn version(&self) -> Option; - - /// In case of concurrent processing of commands, the aggregate may be - /// outdated when a command is applied. In such cases this method expects - /// the list of events that happened since the ['affected_version'] and - /// will return whether there is a conflict. If there is no conflict that - /// the command may be applied again. - /// - /// Note that this defaults to true, which is the safe choice when in - /// doubt. If you choose to implement this, then you will also need to - /// implement the ['set_affected_version'] function. - fn conflicts(&self, _events: &[Self::Event]) -> bool { true } -} - - -//------------ SentCommand --------------------------------------------------- - -/// Convenience wrapper so that implementations can just implement -/// ['CommandDetails'] and leave the id and version boilerplate. -#[derive(Deserialize, Serialize)] -pub struct SentCommand { - id: AggregateId, - version: Option, - #[serde(deserialize_with = "C::deserialize")] - details: C -} - -impl Command for SentCommand { - type Event = C::Event; - - fn id(&self) -> &AggregateId { - &self.id - } - - fn version(&self) -> Option { - self.version - } -} - -impl SentCommand { - pub fn new(id: &AggregateId, version: Option, details: C) -> Self { - SentCommand { id: id.clone(), version, details } - } - - pub fn into_details(self) -> C { self.details } -} - - -//------------ CommandDetails ------------------------------------------------ - -/// Implement this for an enum with CommandDetails, so you you can reuse the -/// id and version boilerplate from ['SentCommand']. -pub trait CommandDetails: Storable { - type Event: Event; -} - - -//------------ AggregateRef -------------------------------------------------- - -/// This type wraps the Aggregate references returned by the AggregateManager, -/// so that we can change the implementation details of the of the latter. -/// This derefs to the Aggregate. -pub struct AggregateRef { - agg: Arc -} - -impl Deref for AggregateRef { - type Target = A; - - fn deref(&self) -> &'_ Self::Target { - &self.agg - } -} - -impl AsRef for AggregateRef { - fn as_ref(&self) -> &A { - &self.agg - } -} - - -//------------ AggregateManager ---------------------------------------------- - -/// This type is responsible for managing Aggregates. I.e. creating new -/// Aggregate instances, returning a reference for reading, dispatching -/// commands to an aggregate, and storing them. -pub struct AggregateManager { - /// Stores a the most recent aggregate. - /// - /// Note. We may wish to change this in future. E.g. just remember the - /// version of the aggregate and get it from storage as needed. Or even - /// make this a choice - keep things we use often in memory, but not - /// other things. - /// - /// We may then also wish to change the return types, and maybe we will - /// have to push the write locking down to the keystore.. I.e. any new - /// save already requires a new key - it is a write once store. So, if we - /// get an error then that is an indication of a concurrency issue. - cache: RwLock>>, - store: S -} - -impl AggregateManager { - - pub fn new(store: S) -> Self { - let values = RwLock::new(HashMap::new()); - AggregateManager { - cache: values, store - } - } - - fn update_cache( - &self, - id: &AggregateId, - mut force: bool - ) -> Result<(), AggMgrErr> { - - let mut cache = self.cache.write().unwrap(); - - let mut has_key = cache.contains_key(id); - - if ! has_key { - let init_key = S::key_for_event(id, 0); - if let Some(init) = self.store.get::(&init_key) - .map_err(AggMgrErr::KeyStoreError)? { - let mut agg = A::init(init).map_err(AggMgrErr::AggregateError)?; - - cache.insert(id.clone(), Arc::new(agg)); - force = true; - has_key = true; - } - } - - if has_key && force { - // We MUST have an entry now - let arc = cache.get_mut(id).unwrap(); - let agg = Arc::make_mut(arc); - - loop { - let ver = agg.version(); - let key = S::key_for_event(id, ver); - if let Some(event) = self.store.get::(&key) - .map_err(AggMgrErr::KeyStoreError)? { - agg.apply(event) - } else { - break - } - } - } - - Ok(()) - } - - - /// Get a reference to the latest version of the aggregate. - #[allow(clippy::type_complexity)] - pub fn get_latest( - &self, - id: &AggregateId - ) -> Result>, AggMgrErr> { - self.update_cache(id, false)?; - Ok(self.cache.read().unwrap().get(id) - .map(|arc| AggregateRef { agg: arc.clone() } )) - } - - #[allow(clippy::type_complexity)] - pub fn create( - &self, - id: &AggregateId, - event: A::InitEvent - ) -> Result<(), AggMgrErr> { - self.update_cache(id, true)?; - - let mut cache = self.cache.write().unwrap(); - if cache.contains_key(id) { - Err(AggMgrErr::AggregateAlreadyExists) - } else { - let key = S::key_for_event(id, 0); - self.store.store(&key, &event).map_err(AggMgrErr::KeyStoreError)?; - - let agg = A::init(event).map_err(AggMgrErr::AggregateError)?; - - cache.insert(id.clone(), Arc::new(agg)); - Ok(()) - } - } - - /// Apply a command to the latest aggregate, save the events and return - /// the updated aggregate. - #[allow(clippy::type_complexity)] - pub fn apply( - &self, - command: A::Command - ) -> Result<(), AggMgrErr> { - let id = command.id().clone(); - self.update_cache(&id, true)?; - - let mut cache = self.cache.write().unwrap(); - - match cache.get_mut(&id) { - None => Err(AggMgrErr::AggregateDoesNotExist), - Some(agg) => { - - let agg = Arc::make_mut(agg); - - if let Some(version) = command.version() { - if version != agg.version() { - // TODO check conflicts - return Err(AggMgrErr::ConcurrentModification) - } - } - - let events = agg.process_command(command) - .map_err(AggMgrErr::AggregateError)?; - - for e in events { - let key = S::key_for_event(&id, e.version()); - self.store.store(&key, &e).map_err(AggMgrErr::KeyStoreError)?; - agg.apply(e); - } - - Ok(()) - } - } - } -} - - -//------------ AggMgrErr ----------------------------------------------------- - -#[derive(Debug, Display)] -pub enum AggMgrErr { - #[display(fmt = "Aggregate does not exist")] - AggregateDoesNotExist, - - #[display(fmt = "Aggregate already exists")] - AggregateAlreadyExists, - - #[display(fmt = "Concurrent modification. Command rejected")] - ConcurrentModification, - - #[display(fmt = "{}", _0)] - AggregateError(A), - - #[display(fmt = "{}", _0)] - KeyStoreError(K), -} - - -//------------ KeyStore ------------------------------------------------------ - -/// Generic KeyStore for AggregateManager -pub trait KeyStore { - type Key; - type Error: std::error::Error; - - fn key_for_snapshot(id: &AggregateId, version: u64) -> Self::Key; - fn key_for_event(id: &AggregateId, version: u64) -> Self::Key; - - /// Returns whether a key already exists. - fn has_key(&self, key: &Self::Key) -> bool; - - /// Throws an error if the key already exists. - fn store(&self, key: &Self::Key, value: &V) -> Result<(), Self::Error>; - - /// Get the value for this key, if any exists. - fn get(&self, key: &Self::Key) -> Result, Self::Error>; -} - - -//------------ DiskKeyStore -------------------------------------------------- - -/// This type can store and retrieve values to/from disk, using json -/// serialization. -pub struct DiskKeyStore { - dir: PathBuf -} - -impl KeyStore for DiskKeyStore { - type Key = PathBuf; - type Error = DiskKeyStoreError; - - fn key_for_snapshot(id: &AggregateId, version: u64) -> Self::Key { - PathBuf::from(format!("snapshot-{}-{}", id.0, version)) - } - - fn key_for_event(id: &AggregateId, version: u64) -> Self::Key { - PathBuf::from(format!("delta-{}-{}", id.0, version)) - } - - fn has_key(&self, key: &Self::Key) -> bool { - self.file_path(key).exists() - } - - fn store(&self, key: &Self::Key, value: &V) -> Result<(), - Self::Error> { - if self.has_key(key) { - Err(DiskKeyStoreError::KeyExists(key.to_string_lossy().to_string())) - } else { - let mut f = file::create_file_with_path(&self.file_path(key))?; - let json = serde_json::to_string(value)?; - f.write_all(json.as_ref())?; - Ok(()) - } - } - - fn get(&self, key: &Self::Key) -> Result, Self::Error> { - if self.has_key(key) { - let f = File::open(self.file_path(&key))?; - let v: V = serde_json::from_reader(f)?; - Ok(Some(v)) - } else { - Ok(None) - } - } -} - -impl DiskKeyStore { - pub fn new(dir: PathBuf) -> Self { - DiskKeyStore { dir } - } - - fn file_path(&self, key: &::Key) -> PathBuf { - let mut file_path = self.dir.clone(); - file_path.push(key); - file_path - } -} - -//------------ DiskKeyStoreError --------------------------------------------- - -/// This type defines possible Errors for KeyStore -#[derive(Debug, Display)] -pub enum DiskKeyStoreError { - #[display(fmt = "{}", _0)] - IoError(io::Error), - - #[display(fmt = "{}", _0)] - JsonError(serde_json::Error), - - #[display(fmt = "Key already exists: {}", _0)] - KeyExists(String) -} - -impl From for DiskKeyStoreError { - fn from(e: io::Error) -> Self { DiskKeyStoreError::IoError(e) } -} - -impl From for DiskKeyStoreError { - fn from(e: serde_json::Error) -> Self { DiskKeyStoreError::JsonError(e) } -} - -impl std::error::Error for DiskKeyStoreError { } - - - -//------------ Tests --------------------------------------------------------- - -#[cfg(test)] -mod tests { - - use super::*; - use crate::util::test; - - #[derive(Clone, Deserialize, Serialize)] - struct Person { - id: AggregateId, - version: u64, - name: String, - age: u8 - } - - impl Person { - pub fn id(&self) -> &AggregateId { &self.id } - pub fn version(&self) -> u64 { self.version } - pub fn name(&self) -> &String { &self.name } - pub fn age(&self) -> u8 { self.age } - } - - #[derive(Deserialize, Serialize)] - pub struct InitPersonDetails { - pub name: String - } - - #[derive(Deserialize, Serialize)] - enum PersonEventDetails { - NameChanged(String), - HadBirthday - } - - #[derive(Deserialize, Serialize)] - enum PersonCommandDetails { - ChangeName(String), - GoAroundTheSun - } - - impl CommandDetails for PersonCommandDetails { - type Event = PersonEvent; - } - - type InitPersonEvent = StoredEvent; - - impl InitPersonEvent { - pub fn init(id: &AggregateId, name: &str) -> Self { - StoredEvent::new(id, 0, InitPersonDetails { name: name.to_string()}) - } - } - - type PersonEvent = StoredEvent; - - impl PersonEvent { - pub fn had_birthday(p: &Person) -> Self { - StoredEvent::new(p.id(), p.version, PersonEventDetails::HadBirthday) - } - - pub fn name_changed(p: &Person, name: String) -> Self { - StoredEvent::new( - p.id(), - p.version, - PersonEventDetails::NameChanged(name)) - } - } - - type PersonCommand = SentCommand; - - impl PersonCommand { - - pub fn go_around_sun(id: &AggregateId, version: Option) -> Self { - Self::new(id, version, PersonCommandDetails::GoAroundTheSun) - } - - pub fn change_name(id: &AggregateId, version: Option, s: &str) -> Self { - let details = PersonCommandDetails::ChangeName(s.to_string()); - Self::new(id, version, details) - } - } - - #[derive(Clone, Debug, Display)] - enum PersonError { - #[display(fmt = "No person can live longer than 255 years")] - TooOld - } - - impl std::error::Error for PersonError {} - - impl Aggregate for Person { - type Command = PersonCommand; - type Event = PersonEvent; - type InitEvent = InitPersonEvent; - type Error = PersonError; - - fn init(event: Self::InitEvent) -> Result { - let (id, _version, init) = event.unwrap(); - Ok(Person { - id, version: 1, name: init.name, age: 0 - }) - } - - fn version(&self) -> u64 { - self.version - } - - fn apply(&mut self, event: Self::Event) { - match event.into_details() { - PersonEventDetails::NameChanged(name) => { self.name = name }, - PersonEventDetails::HadBirthday => { self.age = self.age + 1 } - } - self.version = self.version + 1; - } - - fn process_command(&self, command: Self::Command) -> Result, Self::Error> { - match command.into_details() { - PersonCommandDetails::ChangeName(name) => { - let event = PersonEvent::name_changed(&self, name); - Ok(vec![event]) - }, - PersonCommandDetails::GoAroundTheSun => { - if self.age == 255 { - Err(PersonError::TooOld) - } else { - let event = PersonEvent::had_birthday(&self); - Ok(vec![event]) - } - } - } - } - } - - type PersonManager = AggregateManager; - - - #[test] - fn test() { - test::test_with_tmp_dir(|d| { - - let storage = DiskKeyStore::new(d.clone()); - let manager = PersonManager::new(storage); - - let id_alice = AggregateId::new("alice"); - let alice_init = InitPersonEvent::init(&id_alice, "alice smith"); - - manager.create(&id_alice, alice_init).unwrap(); - - let alice = manager.get_latest(&id_alice).unwrap().unwrap(); - assert_eq!(alice.name(), "alice smith"); - assert_eq!(alice.age(), 0); - - let mut age = 0; - loop { - manager.apply( - PersonCommand::go_around_sun(&id_alice, None) - ).unwrap(); - age = age + 1; - if age == 21 { - break - } - } - - let alice = manager.get_latest(&id_alice).unwrap().unwrap(); - assert_eq!(alice.name(), "alice smith"); - assert_eq!(alice.age(), 21); - - manager.apply( - PersonCommand::change_name(&id_alice, Some(22), "alice smith-doe") - ).unwrap(); - - let alice = manager.get_latest(&id_alice).unwrap().unwrap(); - assert_eq!(alice.name(), "alice smith-doe"); - assert_eq!(alice.age(), 21); - - // Should read state from disk - let storage = DiskKeyStore::new(d); - let manager = PersonManager::new(storage); - - let alice = manager.get_latest(&id_alice).unwrap().unwrap(); - assert_eq!(alice.name(), "alice smith-doe"); - assert_eq!(alice.age(), 21); - }) - } - - -} \ No newline at end of file diff --git a/src/storage/mod.rs b/src/storage/mod.rs index d4042bd4..890db70c 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,4 +1,3 @@ //! Storing values for future use pub mod caching_ks; -pub mod events; pub mod keystore; diff --git a/src/util/ext_serde.rs b/src/util/ext_serde.rs index de13bcc8..b0e258cb 100644 --- a/src/util/ext_serde.rs +++ b/src/util/ext_serde.rs @@ -116,4 +116,3 @@ pub fn de_facility<'de, D>(d: D) -> Result |_| { de::Error::custom( format!("Unsupported syslog_facility: \"{}\"", string))}) } - diff --git a/src/util/file.rs b/src/util/file.rs index 1b9788cd..389f2ea7 100644 --- a/src/util/file.rs +++ b/src/util/file.rs @@ -4,13 +4,11 @@ use std::io::{self, Read, Write}; use std::path::PathBuf; use bytes::Bytes; use rpki::uri; -use crate::api::publication; +use crate::api::{ Base64, EncodedHash }; +use crate::api::publication_data; use crate::util::ext_serde; -use crate::util::sha256; -///-- Some helper functions - /// Creates a sub dir if needed, return full path to it pub fn sub_dir(base: &PathBuf, name: &str) -> Result { let mut full_path = base.clone(); @@ -106,12 +104,31 @@ pub fn delete_in_dir( delete(&full_path) } -fn delete(full_path: &PathBuf) -> Result<(), io::Error> { +pub fn delete(full_path: &PathBuf) -> Result<(), io::Error> { trace!("Removing file: {}", full_path.to_string_lossy()); fs::remove_file(full_path)?; Ok(()) } +pub fn clean_file_and_path(path: &PathBuf) -> Result<(), io::Error> { + if path.exists() { + fs::remove_file(&path)?; + + let mut parent_opt = path.parent(); + + while parent_opt.is_some() { + let parent = parent_opt.unwrap(); + if parent.read_dir()?.count() == 0 { + debug!("Will delete {}", parent.to_string_lossy().to_string()); + fs::remove_dir(parent)?; + } + + parent_opt = parent.parent(); + } + } + Ok(()) +} + fn path_with_rsync(base_path: &PathBuf, uri: &uri::Rsync) -> PathBuf { let mut path = base_path.clone(); path.push(uri.module().authority()); @@ -158,7 +175,7 @@ fn crawl_disk( } else { let uri = derive_uri(base_path, &path, rsync_base)?; let content = read(&path)?; - let current_file = CurrentFile::new(uri, content); + let current_file = CurrentFile::new(uri, &content); res.push(current_file); } @@ -198,72 +215,71 @@ pub struct CurrentFile { serialize_with = "ext_serde::ser_rsync_uri")] uri: uri::Rsync, - #[serde( - deserialize_with = "ext_serde::de_bytes", - serialize_with = "ext_serde::ser_bytes")] /// The actual file content. Note that we may want to store this /// only on disk in future (look up by sha256 hash), to save memory. - content: Bytes, + content: Base64, - #[serde( - deserialize_with = "ext_serde::de_bytes", - serialize_with = "ext_serde::ser_bytes")] /// The sha-256 hash of the file (as is used on the RPKI manifests and /// in the publication protocol for list, update and withdraw). Saving /// this rather than calculating on demand seems a small price for some /// performance gain. - hash: Bytes + hash: EncodedHash } impl CurrentFile { - pub fn new(uri: uri::Rsync, content: Bytes) -> Self { - let hash = sha256(&content); + pub fn new(uri: uri::Rsync, content: &Bytes) -> Self { + let content = Base64::from_content(&content); + let hash = content.to_encoded_hash(); CurrentFile {uri, content, hash} } /// Saves this file under a base directory, based on the (rsync) uri of /// this file. pub fn save(&self, base_path: &PathBuf) -> Result<(), io::Error> { - save_with_rsync_uri(&self.content, &base_path, &self.uri) + save_with_rsync_uri(&self.content.to_bytes(), &base_path, &self.uri) } pub fn uri(&self) -> &uri::Rsync { &self.uri } - pub fn content(&self) -> &Bytes { + pub fn content(&self) -> &Base64 { &self.content } - pub fn hash(&self) -> &Bytes { + pub fn to_bytes(&self) -> Bytes { + self.content.to_bytes() + } + + pub fn hash(&self) -> &EncodedHash { &self.hash } - pub fn as_publish(&self) -> publication::Publish { - let tag = Some(hex::encode(&self.hash)); + pub fn as_publish(&self) -> publication_data::Publish { + let tag = Some(self.hash.to_string()); let uri = self.uri.clone(); let content = self.content.clone(); - publication::Publish::new(tag, uri, content) + publication_data::Publish::new(tag, uri, content) } - pub fn as_update(&self, old_hash: &Bytes) -> publication::Update { + pub fn as_update(&self, old_hash: &EncodedHash) -> publication_data::Update { let tag = None; let uri = self.uri.clone(); let content = self.content.clone(); let hash = old_hash.clone(); - publication::Update::new(tag, uri, content, hash) + publication_data::Update::new(tag, uri, content, hash) } - pub fn as_withdraw(&self) -> publication::Withdraw { + pub fn as_withdraw(&self) -> publication_data::Withdraw { let tag = None; let uri = self.uri.clone(); - let hash = sha256(&self.content); - publication::Withdraw::new(tag, uri, hash) + let hash = self.hash.clone(); + publication_data::Withdraw::new(tag, uri, hash) } - pub fn into_list_element(self) -> publication::ListElement { - publication::ListElement::new(self.uri, self.hash) + pub fn into_list_element(self) -> publication_data::ListElement { + publication_data::ListElement::new(self.uri, self.hash) } } @@ -321,20 +337,19 @@ mod tests { let file_1 = CurrentFile::new( test::rsync_uri("rsync://host:10873/module/alice/file1.txt"), - Bytes::from("content 1") + &Bytes::from("content 1") ); let file_2 = CurrentFile::new( test::rsync_uri("rsync://host:10873/module/alice/file2.txt"), - Bytes::from("content 2") + &Bytes::from("content 2") ); let file_3 = CurrentFile::new( - test::rsync_uri("rsync://host:10873/module/alice/sub/file1\ - .txt"), - Bytes::from("content sub file") + test::rsync_uri("rsync://host:10873/module/alice/sub/file1.txt"), + &Bytes::from("content sub file") ); let file_4 = CurrentFile::new( test::rsync_uri("rsync://host:10873/module/bob/file.txt"), - Bytes::from("content") + &Bytes::from("content") ); file_1.save(&base_dir).unwrap(); diff --git a/src/util/mod.rs b/src/util/mod.rs index c07cef6b..a98e7074 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,6 +1,14 @@ //! General utility modules for use all over the code base +use std::time::Duration; use bytes::Bytes; +use chrono::DateTime; +use chrono::Utc; +use chrono::offset::TimeZone; use rpki::crypto::DigestAlgorithm; +use serde::Serialize; +use serde::Serializer; +use serde::Deserializer; +use serde::Deserialize; pub mod ext_serde; pub mod file; @@ -9,6 +17,45 @@ pub mod softsigner; pub mod test; pub mod xml; -pub fn sha256(object: &Bytes) -> Bytes { - Bytes::from(DigestAlgorithm.digest(object.as_ref()).as_ref()) -} \ No newline at end of file +pub fn sha256(object: &[u8]) -> Bytes { + Bytes::from(DigestAlgorithm.digest(object).as_ref()) +} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct Time(DateTime); + +impl Time { + pub fn now() -> Self { + Time(Utc::now()) + } + + pub fn before_now(dur: Duration) -> Self { + let mut millis = Utc::now().timestamp_millis(); + millis -= dur.as_secs() as i64 * 1000; + + Time(Utc.timestamp_millis(millis)) + } + + pub fn on_or_before(&self, other: &Time) -> bool { + self.0.timestamp_millis() <= other.0.timestamp_millis() + } +} + +impl Serialize for Time { + fn serialize( + &self, serializer: S + ) -> Result where S: Serializer { + serializer.serialize_i64(self.0.timestamp_millis()) + } +} + +impl<'de> Deserialize<'de> for Time { + fn deserialize( + deserializer: D + ) -> Result where D: Deserializer<'de> { + + let timestamp: i64 = i64::deserialize(deserializer)?; + Ok(Time(Utc.timestamp_millis(timestamp))) + } +} + diff --git a/tests/client_api_publish_at_server.rs b/tests/client_api_publish_at_server.rs index 02d20440..bc2daa0a 100644 --- a/tests/client_api_publish_at_server.rs +++ b/tests/client_api_publish_at_server.rs @@ -8,6 +8,7 @@ extern crate tokio; extern crate bytes; use std::{thread, time}; +use std::collections::HashSet; use std::path::PathBuf; use actix::System; use krill::krillc::data::ReportFormat; @@ -25,7 +26,6 @@ use krill::pubc::apiclient; use krill::pubc::apiclient::ApiResponse; use krill::util::file::CurrentFile; use krill::util::file; -use std::collections::HashSet; use krill::util::httpclient; fn list(server_uri: &str, handle: &str, token: &str) -> apiclient::Options { @@ -126,6 +126,7 @@ fn client_publish_at_server() { match res { Err(apiclient::Error::HttpClientError (httpclient::Error::Forbidden)) => {}, + Err(e) => panic!("Expected forbidden, got: {}", e), _ => panic!("Expected forbidden") } } @@ -146,24 +147,25 @@ fn client_publish_at_server() { } } + // Create files on disk to sync let sync_dir = test::create_sub_dir(&d); let file_a = CurrentFile::new( test::rsync_uri("rsync://127.0.0.1/repo/alice/a.txt"), - test::as_bytes("a") + &test::as_bytes("a") ); let file_b = CurrentFile::new( test::rsync_uri("rsync://127.0.0.1/repo/alice/b.txt"), - test::as_bytes("b") + &test::as_bytes("b") ); let file_c = CurrentFile::new( test::rsync_uri("rsync://127.0.0.1/repo/alice/c.txt"), - test::as_bytes("c") + &test::as_bytes("c") ); - file::save_in_dir(file_a.content(), &sync_dir, "a.txt").unwrap(); - file::save_in_dir(file_b.content(), &sync_dir, "b.txt").unwrap(); - file::save_in_dir(file_c.content(), &sync_dir, "c.txt").unwrap(); + file::save_in_dir(&file_a.to_bytes(), &sync_dir, "a.txt").unwrap(); + file::save_in_dir(&file_b.to_bytes(), &sync_dir, "b.txt").unwrap(); + file::save_in_dir(&file_c.to_bytes(), &sync_dir, "c.txt").unwrap(); // Must refuse syncing files outside of publisher base dir @@ -273,10 +275,6 @@ fn client_publish_at_server() { // XXX TODO Must remove files when removing publisher // Expect that c.txt is removed when looking at latest snapshot. file::delete_in_dir(&sync_dir, "c.txt").unwrap(); - - - // Re-add publisher - add_publisher(handle, base_rsync_uri_alice, token); }); } diff --git a/tests/client_cms_publish_at_server.rs b/tests/client_cms_publish_at_server.rs index 9e5b6364..1fc6737d 100644 --- a/tests/client_cms_publish_at_server.rs +++ b/tests/client_cms_publish_at_server.rs @@ -95,20 +95,20 @@ fn client_publish_at_server() { let sync_dir = test::create_sub_dir(&d); let file_a = CurrentFile::new( test::rsync_uri("rsync://127.0.0.1/repo/alice/a.txt"), - test::as_bytes("a") + &test::as_bytes("a") ); let file_b = CurrentFile::new( test::rsync_uri("rsync://127.0.0.1/repo/alice/b.txt"), - test::as_bytes("b") + &test::as_bytes("b") ); let file_c = CurrentFile::new( test::rsync_uri("rsync://127.0.0.1/repo/alice/c.txt"), - test::as_bytes("c") + &test::as_bytes("c") ); - file::save_in_dir(file_a.content(), &sync_dir, "a.txt").unwrap(); - file::save_in_dir(file_b.content(), &sync_dir, "b.txt").unwrap(); - file::save_in_dir(file_c.content(), &sync_dir, "c.txt").unwrap(); + file::save_in_dir(&file_a.to_bytes(), &sync_dir, "a.txt").unwrap(); + file::save_in_dir(&file_b.to_bytes(), &sync_dir, "b.txt").unwrap(); + file::save_in_dir(&file_c.to_bytes(), &sync_dir, "c.txt").unwrap(); client.sync_dir(&sync_dir).unwrap(); diff --git a/tests/manage_publishers.rs b/tests/manage_publishers.rs index 0478368d..d4167cc2 100644 --- a/tests/manage_publishers.rs +++ b/tests/manage_publishers.rs @@ -15,7 +15,6 @@ use krill::krillc::data::{ ApiResponse, ReportFormat }; -use krill::krillc; use krill::krillc::KrillClient; use krill::krillc::options::{ AddPublisherWithCms, @@ -25,9 +24,8 @@ use krill::krillc::options::{ }; use krill::pubc::cmsclient::PubClient; use krill::util::test; -use krill::util::httpclient; use krill::remote::rfc8183::RepositoryResponse; -use reqwest::StatusCode; + /// Tests that we can list publishers through the API #[test] @@ -123,7 +121,7 @@ fn manage_publishers() { match res { ApiResponse::PublisherDetails(details) => { assert_eq!( - details.publisher_handle(), + details.handle(), "alice" ); } @@ -177,17 +175,15 @@ fn manage_publishers() { Command::Publishers(PublishersCommand::Details("alice".to_string())) ); - let res = KrillClient::process(krillc_opts); + let res = KrillClient::process(krillc_opts).unwrap(); match res { - Err(krillc::Error::HttpClientError( - httpclient::Error::BadStatus(code))) => { - assert_eq!(code, StatusCode::NOT_FOUND); - }, - _ => assert!(false) // should have failed! + ApiResponse::PublisherDetails(details) => { + assert_eq!(details.handle(), "alice"); + assert!(details.retired()); + } + _ => panic!("Expected to find alice in retired state") } - - assert!(res.is_err()); } });