diff --git a/src/commons/file.rs b/src/commons/file.rs index 2d5b08ed..751f41df 100644 --- a/src/commons/file.rs +++ b/src/commons/file.rs @@ -1,607 +1,606 @@ -//! Utilities for accessing files. -//! -//! The functions in here primarily provide convenient error conversion. - -use std::{fmt, fs, io}; -use std::borrow::Cow; -use std::fs::File; -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; -use std::str::FromStr; -use bytes::Bytes; -use log::trace; -use serde::{Deserialize, Serialize}; -use serde::de::DeserializeOwned; -use rpki::uri; -use rpki::ca::publication::{Base64, ListElement, Publish, Update, Withdraw}; -use rpki::rrdp::Hash; -use crate::commons::error::KrillIoError; - - -/// Creates a sub dir if needed, return full path to it -pub fn sub_dir(base: &Path, name: &str) -> Result { - let mut full_path = base.to_path_buf(); - full_path.push(name); - create_dir_all(&full_path)?; - Ok(full_path) -} - -/// Creates a dir and any parent dirs which were missing. -pub fn create_dir_all(dir: &Path) -> Result<(), KrillIoError> { - if !dir.is_dir() { - fs::create_dir_all(dir).map_err(|e| { - KrillIoError::new( - format!("could not create dir: {}", dir.to_string_lossy()), - e, - ) - })?; - } - Ok(()) -} - -/// Removes a dir and all its content. -pub fn remove_dir_all(dir: &Path) -> Result<(), KrillIoError> { - if dir.exists() { - fs::remove_dir_all(dir).map_err(|e| { - KrillIoError::new( - format!("could not remove dir: {}", dir.to_string_lossy()), - e, - ) - })?; - } - Ok(()) -} - -/// Creates a new File or opens an exiting one. If the file did not exist, the -/// path will be created if it did not exist yet. -pub fn create_file_with_path(path: &Path) -> Result { - if !path.exists() { - if let Some(parent) = path.parent() { - trace!("Creating path: {}", parent.to_string_lossy()); - fs::create_dir_all(parent).map_err(|e| { - KrillIoError::new( - format!( - "Could not create dir path for: {}", - parent.to_string_lossy() - ), - e, - ) - })?; - } - } - File::create(path).map_err(|e| { - KrillIoError::new( - format!("Could not create file: {}", path.to_string_lossy()), - e, - ) - }) -} - -/// Derive the path for this file. -pub fn file_path(base_path: &Path, file_name: &str) -> PathBuf { - let mut path = base_path.to_path_buf(); - path.push(file_name); - path -} - -/// Saves a file, creating parent dirs as needed -pub fn save(content: &[u8], full_path: &Path) -> Result<(), KrillIoError> { - let mut f = create_file_with_path(full_path)?; - f.write_all(content).map_err(|e| { - KrillIoError::new( - format!("Could not write to: {}", full_path.to_string_lossy()), - e, - ) - })?; - - trace!("Saved file: {}", full_path.to_string_lossy()); - Ok(()) -} - -/// Saves an object to json - unwraps any json errors! -pub fn save_json( - object: &O, - full_path: &Path, -) -> Result<(), KrillIoError> { - let json = serde_json::to_string(object).unwrap(); - save(&Bytes::from(json), full_path) -} - -/// Loads a files and deserializes as json for the expected type. Maps json -/// errors to KrillIoError -pub fn load_json( - full_path: &Path, -) -> Result { - let bytes = read(full_path)?; - serde_json::from_slice(&bytes).map_err(|e| { - KrillIoError::new( - format!( - "Could not load json for file: {}", - full_path.to_string_lossy() - ), - io::Error::other( - format!("could not deserialize json: {e}"), - ), - ) - }) -} - -/// Saves a file, creating parent dirs as needed -pub fn save_in_dir( - content: &Bytes, - base_path: &Path, - name: &str, -) -> Result<(), KrillIoError> { - let mut full_path = base_path.to_path_buf(); - full_path.push(name); - save(content, &full_path) -} - -/// Saves a file under a base directory, using the rsync uri to create -/// sub-directories preserving the rsync authority and module in dir names. -pub fn save_with_rsync_uri( - content: &Bytes, - base_path: &Path, - uri: &uri::Rsync, -) -> Result<(), KrillIoError> { - let path = path_with_rsync(base_path, uri); - save(content, &path) -} - -/// Reads a file to Bytes -pub fn read(path: &Path) -> Result { - let mut f = File::open(path).map_err(|e| { - KrillIoError::new( - format!("Could not open: '{}'", path.to_string_lossy()), - e, - ) - })?; - let mut bytes = Vec::new(); - f.read_to_end(&mut bytes).map_err(|e| { - KrillIoError::new( - format!("Could not read: {}", path.to_string_lossy()), - e, - ) - })?; - Ok(Bytes::from(bytes)) -} - -pub fn read_with_rsync_uri( - base_path: &Path, - uri: &uri::Rsync, -) -> Result { - let path = path_with_rsync(base_path, uri); - read(&path) -} - -pub fn delete_with_rsync_uri( - base_path: &Path, - uri: &uri::Rsync, -) -> Result<(), KrillIoError> { - delete_file(&path_with_rsync(base_path, uri)) -} - -pub fn delete_in_dir( - base_path: &Path, - name: &str, -) -> Result<(), KrillIoError> { - let mut full_path = base_path.to_path_buf(); - full_path.push(name); - delete_file(&full_path) -} - -/// Deletes a file, but does not touch the parent directories. See -/// [`clean_file_and_path`] for an alternative that does. -pub fn delete_file(full_path: &Path) -> Result<(), KrillIoError> { - trace!("Removing file: {}", full_path.to_string_lossy()); - fs::remove_file(full_path).map_err(|e| { - KrillIoError::new( - format!("Could not remove file: {}", full_path.to_string_lossy()), - e, - ) - }) -} - -/// Removes the file and any **empty** directories on the path after removing -/// it. -pub fn clean_file_and_path(path: &Path) -> Result<(), KrillIoError> { - if path.exists() { - delete_file(path)?; - - let mut parent_opt = path.parent(); - - while parent_opt.is_some() { - let parent = parent_opt.unwrap(); - if parent - .read_dir() - .map_err(|e| { - KrillIoError::new( - format!( - "Could not read directory: '{}'", - parent.to_string_lossy() - ), - e, - ) - })? - .count() - == 0 - { - trace!( - "Will delete {}", parent.display(), - ); - fs::remove_dir(parent).map_err(|e| { - KrillIoError::new( - format!( - "Could not remove dir: {}", - parent.display(), - ), - e, - ) - })?; - } - - parent_opt = parent.parent(); - } - } - Ok(()) -} - -fn path_with_rsync(base_path: &Path, uri: &uri::Rsync) -> PathBuf { - let mut path = base_path.to_path_buf(); - #[cfg(unix)] - path.push(uri.authority()); - #[cfg(not(unix))] - path.push(uri.authority().replace(":", "_")); - path.push(uri.module_name()); - path.push(uri.path()); - path -} - -/// Recurses a path on disk and returns all files found as ['CurrentFile'], -/// using the provided rsync_base URI as the rsync prefix. -/// Allows a publication client to publish the contents below some base -/// dir, in their own designated rsync URI name space. -pub fn crawl_incl_rsync_base( - base_path: &Path, - rsync_base: &uri::Rsync, -) -> Result, Error> { - crawl_disk(base_path, base_path, Some(rsync_base)) -} - -/// Recurses a path on disk and returns all files found as ['CurrentFile'], -/// deriving the rsync_base URI from the directory structure. This is -/// useful when reading ['CurrentFile'] instances that were saved in some -/// base directory as is done by the ['FileStore']. -pub fn crawl_derive_rsync_uri( - base_path: &Path, -) -> Result, Error> { - crawl_disk(base_path, base_path, None) -} - -fn crawl_disk( - base_path: &Path, - path: &Path, - rsync_base: Option<&uri::Rsync>, -) -> Result, Error> { - let mut res = Vec::new(); - - for entry in fs::read_dir(path).map_err(|_| Error::cannot_read(path))? { - let entry = entry.map_err(|_| Error::cannot_read(path))?; - let path = entry.path(); - if path.is_dir() { - let mut other = crawl_disk(base_path, &path, rsync_base)?; - res.append(&mut other); - } else { - let uri = derive_uri(base_path, &path, rsync_base)?; - let content = - read(&path).map_err(|_| Error::cannot_read(&path))?; - let current_file = CurrentFile::new(uri, &content); - - res.push(current_file); - } - } - - Ok(res) -} - -fn derive_uri( - base_path: &Path, - path: &Path, - rsync_base: Option<&uri::Rsync>, -) -> Result { - let rel = path - .strip_prefix(base_path) - .map_err(|_| Error::PathOutsideBasePath)?; - - let rel_string = rel.to_string_lossy().to_string(); - - let uri_string = match rsync_base { - Some(rsync_base) => format!("{rsync_base}{rel_string}"), - None => format!("rsync://{rel_string}"), - }; - - #[cfg(not(unix))] - let uri_string = uri_string.replace("\\", "/"); - - let uri = uri::Rsync::from_str(&uri_string) - .map_err(|_| Error::UnsupportedFileName(uri_string))?; - Ok(uri) -} - -/// Recursively copy a base_path (if it's a dir that is), and preserve the -/// permissions timestamps and all that goodness.. -/// -/// This is needed when making a back-up copy when we need to do upgrades on -/// data, which could in theory fail, in which case we want to leave teh old -/// data in place. -pub fn backup_dir(base_path: &Path, target_path: &Path) -> Result<(), Error> { - if base_path.to_string_lossy() == Cow::Borrowed("/") - || target_path.to_string_lossy() == Cow::Borrowed("/") - { - Err(Error::BackupExcessive) - } else if base_path.is_file() { - let mut target = target_path.to_path_buf(); - target.push(base_path.file_name().unwrap()); - - if target.exists() { - Err(Error::backup_target_exists(target_path)) - } else { - fs::copy(base_path, target_path).map_err(|e| { - KrillIoError::new( - format!( - "Could not back up file from '{}' to '{}'", - base_path.to_string_lossy(), - target_path.to_string_lossy() - ), - e, - ) - })?; - Ok(()) - } - } else if base_path.is_dir() { - for entry in fs::read_dir(base_path).map_err(|e| { - KrillIoError::new( - format!( - "Could not read dir '{}' for backup", - base_path.to_string_lossy() - ), - e, - ) - })? { - let path = entry - .map_err(|e| { - KrillIoError::new( - format!( - "Could not read entry for dir '{}' for backup", - base_path.to_string_lossy() - ), - e, - ) - })? - .path(); - let mut target = target_path.to_path_buf(); - target.push(path.file_name().unwrap()); - if path.is_dir() { - backup_dir(&path, &target)?; - } else if path.is_file() { - if let Some(parent) = target.parent() { - fs::create_dir_all(parent).map_err(|e| { - KrillIoError::new( - format!( - "Could not create dir(s) '{}' for backup", - parent.to_string_lossy() - ), - e, - ) - })?; - } - fs::copy(&path, &target).map_err(|e| { - KrillIoError::new( - format!( - "Could not backup '{}' to '{}'", - path.to_string_lossy(), - target.to_string_lossy() - ), - e, - ) - })?; - } else { - return Err(Error::backup_cannot_read(&path)); - } - } - Ok(()) - } else { - Err(Error::backup_cannot_read(base_path)) - } -} - -//------------ CurrentFile --------------------------------------------------- - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct CurrentFile { - /// The full uri for this file. - uri: uri::Rsync, - - /// The actual file content. Note that we may want to store this - /// only on disk in future (look up by sha256 hash), to save memory. - content: Base64, - - /// The sha-256 hash of the file (as is used on the RPKI manifests and - /// in the publication protocol for list, update and withdraw). Saving - /// this rather than calculating on demand seems a small price for some - /// performance gain. - hash: Hash, -} - -impl CurrentFile { - pub fn new(uri: uri::Rsync, content: &Bytes) -> Self { - let content = Base64::from_content(content); - let hash = content.to_hash(); - - CurrentFile { uri, content, hash } - } - - /// Saves this file under a base directory, based on the (rsync) uri of - /// this file. - pub fn save(&self, base_path: &Path) -> Result<(), KrillIoError> { - save_with_rsync_uri(&self.content.to_bytes(), base_path, &self.uri) - } - - pub fn uri(&self) -> &uri::Rsync { - &self.uri - } - - pub fn content(&self) -> &Base64 { - &self.content - } - - pub fn to_bytes(&self) -> Bytes { - self.content.to_bytes() - } - - pub fn hash(&self) -> &Hash { - &self.hash - } - - pub fn as_publish(&self) -> Publish { - let tag = Some(self.hash.to_string()); - let uri = self.uri.clone(); - let content = self.content.clone(); - Publish::new(tag, uri, content) - } - - pub fn as_update(&self, old_hash: &Hash) -> Update { - let tag = None; - let uri = self.uri.clone(); - let content = self.content.clone(); - let hash = *old_hash; - Update::new(tag, uri, content, hash) - } - - pub fn as_withdraw(&self) -> Withdraw { - let tag = None; - let uri = self.uri.clone(); - let hash = self.hash; - Withdraw::new(tag, uri, hash) - } - - pub fn into_list_element(self) -> ListElement { - ListElement::new(self.uri, self.hash) - } -} - -impl PartialEq for CurrentFile { - fn eq(&self, other: &CurrentFile) -> bool { - self.uri == other.uri - && self.hash == other.hash - && self.content == other.content - } -} - -impl Eq for CurrentFile {} - -//------------ Error --------------------------------------------------------- -#[derive(Debug)] -pub enum Error { - CannotRead(String), - UnsupportedFileName(String), - PathOutsideBasePath, - BackupExcessive, - BackupCannotReadSource(String), - BackupTargetExists(String), - Io(KrillIoError), -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Error::CannotRead(s) => write!(f, "Cannot read: {s}"), - Error::UnsupportedFileName(name) => { - write!(f, "Unsupported characters: {name}") - } - Error::PathOutsideBasePath => { - write!(f, "Cannot use path outside of rsync jail") - } - Error::BackupExcessive => write!( - f, - "Do not ever use '/' as the source or target for backups" - ), - Error::BackupCannotReadSource(e) => { - write!(f, "Source for backup cannot be read: {e}") - } - Error::BackupTargetExists(e) => { - write!(f, "Target for backup already exists: {e}") - } - Error::Io(e) => e.fmt(f), - } - } -} - -impl Error { - pub fn cannot_read(path: &Path) -> Error { - let str = path.to_string_lossy().to_string(); - Error::CannotRead(str) - } - - fn backup_cannot_read(path: &Path) -> Error { - let str = path.to_string_lossy().to_string(); - Error::BackupCannotReadSource(str) - } - - fn backup_target_exists(path: &Path) -> Error { - let str = path.to_string_lossy().to_string(); - Error::BackupTargetExists(str) - } -} - -impl std::error::Error for Error {} - -impl From for Error { - fn from(e: KrillIoError) -> Self { - Error::Io(e) - } -} - -//------------ Tests --------------------------------------------------------- - -#[cfg(unix)] -#[cfg(test)] -mod tests { - use crate::commons::test; - - use super::*; - - #[test] - fn should_scan_disk() { - test::test_under_tmp(|base_dir| { - let file_1 = CurrentFile::new( - test::rsync("rsync://host:10873/module/alice/file1.txt"), - &Bytes::from("content 1"), - ); - let file_2 = CurrentFile::new( - test::rsync("rsync://host:10873/module/alice/file2.txt"), - &Bytes::from("content 2"), - ); - let file_3 = CurrentFile::new( - test::rsync("rsync://host:10873/module/alice/sub/file1.txt"), - &Bytes::from("content sub file"), - ); - let file_4 = CurrentFile::new( - test::rsync("rsync://host:10873/module/bob/file.txt"), - &Bytes::from("content"), - ); - - file_1.save(&base_dir).unwrap(); - file_2.save(&base_dir).unwrap(); - file_3.save(&base_dir).unwrap(); - file_4.save(&base_dir).unwrap(); - - let files = crawl_derive_rsync_uri(&base_dir).unwrap(); - - assert!(files.contains(&file_1)); - assert!(files.contains(&file_2)); - assert!(files.contains(&file_3)); - assert!(files.contains(&file_4)); - }); - } -} +//! Utilities for accessing files. +//! +//! The functions in here primarily provide convenient error conversion. + +use std::{fmt, fs, io}; +use std::borrow::Cow; +use std::fs::File; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use bytes::Bytes; +use log::trace; +use serde::{Deserialize, Serialize}; +use serde::de::DeserializeOwned; +use rpki::uri; +use rpki::ca::publication::{Base64, ListElement, Publish, Update, Withdraw}; +use rpki::rrdp::Hash; +use crate::commons::error::KrillIoError; + + +/// Creates a sub dir if needed, return full path to it +pub fn sub_dir(base: &Path, name: &str) -> Result { + let mut full_path = base.to_path_buf(); + full_path.push(name); + create_dir_all(&full_path)?; + Ok(full_path) +} + +/// Creates a dir and any parent dirs which were missing. +pub fn create_dir_all(dir: &Path) -> Result<(), KrillIoError> { + if !dir.is_dir() { + fs::create_dir_all(dir).map_err(|e| { + KrillIoError::new( + format!("could not create dir: {}", dir.to_string_lossy()), + e, + ) + })?; + } + Ok(()) +} + +/// Removes a dir and all its content. +pub fn remove_dir_all(dir: &Path) -> Result<(), KrillIoError> { + if dir.exists() { + fs::remove_dir_all(dir).map_err(|e| { + KrillIoError::new( + format!("could not remove dir: {}", dir.to_string_lossy()), + e, + ) + })?; + } + Ok(()) +} + +/// Creates a new File or opens an exiting one. If the file did not exist, the +/// path will be created if it did not exist yet. +pub fn create_file_with_path(path: &Path) -> Result { + if !path.exists() { + if let Some(parent) = path.parent() { + trace!("Creating path: {}", parent.to_string_lossy()); + fs::create_dir_all(parent).map_err(|e| { + KrillIoError::new( + format!( + "Could not create dir path for: {}", + parent.to_string_lossy() + ), + e, + ) + })?; + } + } + File::create(path).map_err(|e| { + KrillIoError::new( + format!("Could not create file: {}", path.to_string_lossy()), + e, + ) + }) +} + +/// Derive the path for this file. +pub fn file_path(base_path: &Path, file_name: &str) -> PathBuf { + let mut path = base_path.to_path_buf(); + path.push(file_name); + path +} + +/// Saves a file, creating parent dirs as needed +pub fn save(content: &[u8], full_path: &Path) -> Result<(), KrillIoError> { + let mut f = create_file_with_path(full_path)?; + f.write_all(content).map_err(|e| { + KrillIoError::new( + format!("Could not write to: {}", full_path.to_string_lossy()), + e, + ) + })?; + + trace!("Saved file: {}", full_path.to_string_lossy()); + Ok(()) +} + +/// Saves an object to json - unwraps any json errors! +pub fn save_json( + object: &O, + full_path: &Path, +) -> Result<(), KrillIoError> { + let json = serde_json::to_string(object).unwrap(); + save(&Bytes::from(json), full_path) +} + +/// Loads a files and deserializes as json for the expected type. Maps json +/// errors to KrillIoError +pub fn load_json( + full_path: &Path, +) -> Result { + let bytes = read(full_path)?; + serde_json::from_slice(&bytes).map_err(|e| { + KrillIoError::new( + format!( + "Could not load json for file: {}", + full_path.to_string_lossy() + ), + io::Error::other( + format!("could not deserialize json: {e}"), + ), + ) + }) +} + +/// Saves a file, creating parent dirs as needed +pub fn save_in_dir( + content: &Bytes, + base_path: &Path, + name: &str, +) -> Result<(), KrillIoError> { + let mut full_path = base_path.to_path_buf(); + full_path.push(name); + save(content, &full_path) +} + +/// Saves a file under a base directory, using the rsync uri to create +/// sub-directories preserving the rsync authority and module in dir names. +pub fn save_with_rsync_uri( + content: &Bytes, + base_path: &Path, + uri: &uri::Rsync, +) -> Result<(), KrillIoError> { + let path = path_with_rsync(base_path, uri); + save(content, &path) +} + +/// Reads a file to Bytes +pub fn read(path: &Path) -> Result { + let mut f = File::open(path).map_err(|e| { + KrillIoError::new( + format!("Could not open: '{}'", path.to_string_lossy()), + e, + ) + })?; + let mut bytes = Vec::new(); + f.read_to_end(&mut bytes).map_err(|e| { + KrillIoError::new( + format!("Could not read: {}", path.to_string_lossy()), + e, + ) + })?; + Ok(Bytes::from(bytes)) +} + +pub fn read_with_rsync_uri( + base_path: &Path, + uri: &uri::Rsync, +) -> Result { + let path = path_with_rsync(base_path, uri); + read(&path) +} + +pub fn delete_with_rsync_uri( + base_path: &Path, + uri: &uri::Rsync, +) -> Result<(), KrillIoError> { + delete_file(&path_with_rsync(base_path, uri)) +} + +pub fn delete_in_dir( + base_path: &Path, + name: &str, +) -> Result<(), KrillIoError> { + let mut full_path = base_path.to_path_buf(); + full_path.push(name); + delete_file(&full_path) +} + +/// Deletes a file, but does not touch the parent directories. See +/// [`clean_file_and_path`] for an alternative that does. +pub fn delete_file(full_path: &Path) -> Result<(), KrillIoError> { + trace!("Removing file: {}", full_path.to_string_lossy()); + fs::remove_file(full_path).map_err(|e| { + KrillIoError::new( + format!("Could not remove file: {}", full_path.to_string_lossy()), + e, + ) + }) +} + +/// Removes the file and any **empty** directories on the path after removing +/// it. +pub fn clean_file_and_path(path: &Path) -> Result<(), KrillIoError> { + if path.exists() { + delete_file(path)?; + + let mut parent_opt = path.parent(); + + while parent_opt.is_some() { + let parent = parent_opt.unwrap(); + if parent + .read_dir() + .map_err(|e| { + KrillIoError::new( + format!( + "Could not read directory: '{}'", + parent.to_string_lossy() + ), + e, + ) + })? + .count() + == 0 + { + trace!( + "Will delete {}", parent.display(), + ); + fs::remove_dir(parent).map_err(|e| { + KrillIoError::new( + format!( + "Could not remove dir: {}", + parent.display(), + ), + e, + ) + })?; + } + + parent_opt = parent.parent(); + } + } + Ok(()) +} + +fn path_with_rsync(base_path: &Path, uri: &uri::Rsync) -> PathBuf { + let mut path = base_path.to_path_buf(); + #[cfg(unix)] + path.push(uri.authority()); + #[cfg(not(unix))] + path.push(uri.authority().replace(":", "_")); + path.push(uri.module_name()); + path.push(uri.path()); + path +} + +/// Recurses a path on disk and returns all files found as ['CurrentFile'], +/// using the provided rsync_base URI as the rsync prefix. +/// Allows a publication client to publish the contents below some base +/// dir, in their own designated rsync URI name space. +pub fn crawl_incl_rsync_base( + base_path: &Path, + rsync_base: &uri::Rsync, +) -> Result, Error> { + crawl_disk(base_path, base_path, Some(rsync_base)) +} + +/// Recurses a path on disk and returns all files found as ['CurrentFile'], +/// deriving the rsync_base URI from the directory structure. This is +/// useful when reading ['CurrentFile'] instances that were saved in some +/// base directory as is done by the ['FileStore']. +pub fn crawl_derive_rsync_uri( + base_path: &Path, +) -> Result, Error> { + crawl_disk(base_path, base_path, None) +} + +fn crawl_disk( + base_path: &Path, + path: &Path, + rsync_base: Option<&uri::Rsync>, +) -> Result, Error> { + let mut res = Vec::new(); + + for entry in fs::read_dir(path).map_err(|_| Error::cannot_read(path))? { + let entry = entry.map_err(|_| Error::cannot_read(path))?; + let path = entry.path(); + if path.is_dir() { + let mut other = crawl_disk(base_path, &path, rsync_base)?; + res.append(&mut other); + } else { + let uri = derive_uri(base_path, &path, rsync_base)?; + let content = + read(&path).map_err(|_| Error::cannot_read(&path))?; + let current_file = CurrentFile::new(uri, &content); + + res.push(current_file); + } + } + + Ok(res) +} + +fn derive_uri( + base_path: &Path, + path: &Path, + rsync_base: Option<&uri::Rsync>, +) -> Result { + let rel = path + .strip_prefix(base_path) + .map_err(|_| Error::PathOutsideBasePath)?; + + let rel_string = rel.to_string_lossy().to_string(); + + let uri_string = match rsync_base { + Some(rsync_base) => format!("{rsync_base}{rel_string}"), + None => format!("rsync://{rel_string}"), + }; + + #[cfg(not(unix))] + let uri_string = uri_string.replace("\\", "/"); + + let uri = uri::Rsync::from_str(&uri_string) + .map_err(|_| Error::UnsupportedFileName(uri_string))?; + Ok(uri) +} + +/// Recursively copy a base_path (if it's a dir that is), and preserve the +/// permissions timestamps and all that goodness.. +/// +/// This is needed when making a back-up copy when we need to do upgrades on +/// data, which could in theory fail, in which case we want to leave teh old +/// data in place. +pub fn backup_dir(base_path: &Path, target_path: &Path) -> Result<(), Error> { + if base_path.to_string_lossy() == Cow::Borrowed("/") + || target_path.to_string_lossy() == Cow::Borrowed("/") + { + Err(Error::BackupExcessive) + } else if base_path.is_file() { + let mut target = target_path.to_path_buf(); + target.push(base_path.file_name().unwrap()); + + if target.exists() { + Err(Error::backup_target_exists(target_path)) + } else { + fs::copy(base_path, target_path).map_err(|e| { + KrillIoError::new( + format!( + "Could not back up file from '{}' to '{}'", + base_path.to_string_lossy(), + target_path.to_string_lossy() + ), + e, + ) + })?; + Ok(()) + } + } else if base_path.is_dir() { + for entry in fs::read_dir(base_path).map_err(|e| { + KrillIoError::new( + format!( + "Could not read dir '{}' for backup", + base_path.to_string_lossy() + ), + e, + ) + })? { + let path = entry + .map_err(|e| { + KrillIoError::new( + format!( + "Could not read entry for dir '{}' for backup", + base_path.to_string_lossy() + ), + e, + ) + })? + .path(); + let mut target = target_path.to_path_buf(); + target.push(path.file_name().unwrap()); + if path.is_dir() { + backup_dir(&path, &target)?; + } else if path.is_file() { + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|e| { + KrillIoError::new( + format!( + "Could not create dir(s) '{}' for backup", + parent.to_string_lossy() + ), + e, + ) + })?; + } + fs::copy(&path, &target).map_err(|e| { + KrillIoError::new( + format!( + "Could not backup '{}' to '{}'", + path.to_string_lossy(), + target.to_string_lossy() + ), + e, + ) + })?; + } else { + return Err(Error::backup_cannot_read(&path)); + } + } + Ok(()) + } else { + Err(Error::backup_cannot_read(base_path)) + } +} + +//------------ CurrentFile --------------------------------------------------- + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct CurrentFile { + /// The full uri for this file. + uri: uri::Rsync, + + /// The actual file content. Note that we may want to store this + /// only on disk in future (look up by sha256 hash), to save memory. + content: Base64, + + /// The sha-256 hash of the file (as is used on the RPKI manifests and + /// in the publication protocol for list, update and withdraw). Saving + /// this rather than calculating on demand seems a small price for some + /// performance gain. + hash: Hash, +} + +impl CurrentFile { + pub fn new(uri: uri::Rsync, content: &Bytes) -> Self { + let content = Base64::from_content(content); + let hash = content.to_hash(); + + CurrentFile { uri, content, hash } + } + + /// Saves this file under a base directory, based on the (rsync) uri of + /// this file. + pub fn save(&self, base_path: &Path) -> Result<(), KrillIoError> { + save_with_rsync_uri(&self.content.to_bytes(), base_path, &self.uri) + } + + pub fn uri(&self) -> &uri::Rsync { + &self.uri + } + + pub fn content(&self) -> &Base64 { + &self.content + } + + pub fn to_bytes(&self) -> Bytes { + self.content.to_bytes() + } + + pub fn hash(&self) -> &Hash { + &self.hash + } + + pub fn as_publish(&self) -> Publish { + let tag = Some(self.hash.to_string()); + let uri = self.uri.clone(); + let content = self.content.clone(); + Publish::new(tag, uri, content) + } + + pub fn as_update(&self, old_hash: &Hash) -> Update { + let tag = None; + let uri = self.uri.clone(); + let content = self.content.clone(); + let hash = *old_hash; + Update::new(tag, uri, content, hash) + } + + pub fn as_withdraw(&self) -> Withdraw { + let tag = None; + let uri = self.uri.clone(); + let hash = self.hash; + Withdraw::new(tag, uri, hash) + } + + pub fn into_list_element(self) -> ListElement { + ListElement::new(self.uri, self.hash) + } +} + +impl PartialEq for CurrentFile { + fn eq(&self, other: &CurrentFile) -> bool { + self.uri == other.uri + && self.hash == other.hash + && self.content == other.content + } +} + +impl Eq for CurrentFile {} + +//------------ Error --------------------------------------------------------- +#[derive(Debug)] +pub enum Error { + CannotRead(String), + UnsupportedFileName(String), + PathOutsideBasePath, + BackupExcessive, + BackupCannotReadSource(String), + BackupTargetExists(String), + Io(KrillIoError), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Error::CannotRead(s) => write!(f, "Cannot read: {s}"), + Error::UnsupportedFileName(name) => { + write!(f, "Unsupported characters: {name}") + } + Error::PathOutsideBasePath => { + write!(f, "Cannot use path outside of rsync jail") + } + Error::BackupExcessive => write!( + f, + "Do not ever use '/' as the source or target for backups" + ), + Error::BackupCannotReadSource(e) => { + write!(f, "Source for backup cannot be read: {e}") + } + Error::BackupTargetExists(e) => { + write!(f, "Target for backup already exists: {e}") + } + Error::Io(e) => e.fmt(f), + } + } +} + +impl Error { + pub fn cannot_read(path: &Path) -> Error { + let str = path.to_string_lossy().to_string(); + Error::CannotRead(str) + } + + fn backup_cannot_read(path: &Path) -> Error { + let str = path.to_string_lossy().to_string(); + Error::BackupCannotReadSource(str) + } + + fn backup_target_exists(path: &Path) -> Error { + let str = path.to_string_lossy().to_string(); + Error::BackupTargetExists(str) + } +} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(e: KrillIoError) -> Self { + Error::Io(e) + } +} + +//------------ Tests --------------------------------------------------------- + +#[cfg(test)] +mod tests { + use crate::commons::test; + + use super::*; + + #[test] + fn should_scan_disk() { + test::test_under_tmp(|base_dir| { + let file_1 = CurrentFile::new( + test::rsync("rsync://host:10873/module/alice/file1.txt"), + &Bytes::from("content 1"), + ); + let file_2 = CurrentFile::new( + test::rsync("rsync://host:10873/module/alice/file2.txt"), + &Bytes::from("content 2"), + ); + let file_3 = CurrentFile::new( + test::rsync("rsync://host:10873/module/alice/sub/file1.txt"), + &Bytes::from("content sub file"), + ); + let file_4 = CurrentFile::new( + test::rsync("rsync://host:10873/module/bob/file.txt"), + &Bytes::from("content"), + ); + + file_1.save(&base_dir).unwrap(); + file_2.save(&base_dir).unwrap(); + file_3.save(&base_dir).unwrap(); + file_4.save(&base_dir).unwrap(); + + let files = crawl_derive_rsync_uri(&base_dir).unwrap(); + + assert!(files.contains(&file_1)); + assert!(files.contains(&file_2)); + assert!(files.contains(&file_3)); + assert!(files.contains(&file_4)); + }); + } +}