WEBDIENSTE/NLnetLabs-krill
1
0
Fork 0
mirror of https://github.com/NLnetLabs/krill.git synced 2026-09-26 11:24:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

Improve efficiency of Publication Server command migration, and report progress. (#503)

Browse Source
This commit is contained in:
Tim Bruijnzeels
2021-05-12 11:26:23 +02:00
parent 3e78620f19
commit f13ea9b5e6
4 changed files with 74 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cli/client.rs
+4 -3
View File
@@ -22,7 +22,7 @@ use crate::constants::KRILL_CLI_API_ENV;
use crate::daemon::config::Config;
#[cfg(feature = "multi-user")]
use crate::constants::{ PW_HASH_LOG_N, PW_HASH_P, PW_HASH_R };
use crate::constants::{PW_HASH_LOG_N, PW_HASH_P, PW_HASH_R};
fn resolve_uri(server: &uri::Https, path: &str) -> String {
format!("{}{}", server, path)
@@ -431,12 +431,13 @@ impl KrillClient {
}
#[cfg(feature = "multi-user")]
#[allow(clippy::clippy::unnecessary_wraps)]
fn user(&self, details: KrillUserDetails) -> Result<ApiResponse, Error> {
let (password_hash, salt) = {
use scrypt::scrypt;
let password = rpassword::read_password_from_tty(Some("Enter the password to hash: ")).unwrap();
// The scrypt-js NPM documentation (https://www.npmjs.com/package/scrypt-js) says:
// "TL;DR - either only allow ASCII characters in passwords, or use
// String.prototype.normalize('NFKC') on any password"
@@ -448,7 +449,7 @@ impl KrillClient {
let params = scrypt::Params::new(PW_HASH_LOG_N, PW_HASH_R, PW_HASH_P).unwrap();
// hash twice with two different salts
// hash first with a salt the client browser knows how to construct based on the users id and a site
// hash first with a salt the client browser knows how to construct based on the users id and a site
// specific string.
let weak_salt = format!("krill-lagosta-{}", user_id);
src/daemon/auth/providers/config_file/provider.rs
+13 -5
View File
@@ -12,7 +12,7 @@ use crate::daemon::auth::{Auth, AuthProvider, LoggedInUser};
use crate::daemon::config::Config;
use crate::daemon::http::HttpResponse;
use crate::constants::{ PW_HASH_LOG_N, PW_HASH_P, PW_HASH_R };
use crate::constants::{PW_HASH_LOG_N, PW_HASH_P, PW_HASH_R};
// This is NOT an actual relative path to redirect to. Instead it is the path
// string of an entry in the Vue router routes table to "route" to (in the
@@ -43,7 +43,7 @@ fn get_checked_config_user(id: &str, user: &ConfigUserDetails) -> KrillResult<Us
Ok(UserDetails {
password_hash: Token::from(password_hash),
salt: salt,
salt,
attributes: user.attributes.clone(),
})
}
@@ -133,7 +133,7 @@ impl AuthProvider for ConfigFileAuthProvider {
use scrypt::scrypt;
// Do NOT bail out if the user is not known because then the unknown user path would return very quickly
// compared to the known user path and timing differences can aid attackers.
// compared to the known user path and timing differences can aid attackers.
let (user_password_hash, user_salt) = match self.users.get(&id) {
Some(user) => (user.password_hash.to_string(), user.salt.clone()),
None => (self.fake_password_hash.clone(), self.fake_salt.clone()),
@@ -148,14 +148,22 @@ impl AuthProvider for ConfigFileAuthProvider {
let password_hash_bytes = hex::decode(password_hash.as_ref()).unwrap();
let strong_salt = hex::decode(&user_salt).unwrap();
let mut hashed_hash: [u8; 32] = [0; 32];
scrypt(password_hash_bytes.as_slice(), strong_salt.as_slice(), &params, &mut hashed_hash).unwrap();
scrypt(
password_hash_bytes.as_slice(),
strong_salt.as_slice(),
&params,
&mut hashed_hash,
)
.unwrap();
if hex::encode(hashed_hash) == user_password_hash.as_ref() {
// And now finally check the user, so that both known and unknown user code paths do the same work
// and don't result in an obvious timing difference between the two scenarios which could potentially
// be used to discover user names.
if let Some(user) = self.users.get(&id) {
let api_token = self.session_cache.encode(&id, &user.attributes, HashMap::new(), &self.key, None)?;
let api_token =
self.session_cache
.encode(&id, &user.attributes, HashMap::new(), &self.key, None)?;
Ok(LoggedInUser {
token: api_token,
src/upgrades/v0_9_0/ca_objects_migration.rs
+6 -5
View File
@@ -251,6 +251,12 @@ impl UpgradeStore for CasStoreMigration {
let mut total_migrated = 0;
for cmd_key in cmd_keys {
// Do the migration counter first, so that we can just call continue when we need to skip commands
total_migrated += 1;
if total_migrated % 100 == 0 {
info!(" migrated {} commands", total_migrated);
}
debug!(" command: {}", cmd_key);
let mut old_cmd: OldStoredCaCommand = self.get(&cmd_key)?;
@@ -298,11 +304,6 @@ impl UpgradeStore for CasStoreMigration {
let key = KeyStoreKey::scoped(scope.clone(), format!("{}.json", cmd_key));
self.store.store(&key, &migrated_cmd)?;
total_migrated += 1;
if total_migrated % 100 == 0 {
info!(" migrated {} commands", total_migrated);
}
}
info!("Finished migrating commands for CA {}", scope);
src/upgrades/v0_9_0/pubd_objects_migration.rs
+51 -23
View File
@@ -6,7 +6,8 @@ use std::{
sync::Arc,
};
use rpki::{crypto::KeyIdentifier, uri};
use chrono::Duration;
use rpki::{crypto::KeyIdentifier, uri, x509::Time};
use crate::{
commons::{
@@ -110,8 +111,16 @@ impl UpgradeStore for PubdStoreMigration {
let scope = "0";
let handle = Handle::from_str(scope).unwrap();
let info_key = KeyStoreKey::scoped(scope.to_string(), "info.json".to_string());
let mut info: StoredValueInfo = match self.store.get(&info_key) {
// Archive all keys in the scope, then we can write new keys as needed without
// overwriting anything when we renumber.
for key in self.store.keys(Some(scope.to_string()), "")? {
self.archive_to_migration_scope(&key)?;
}
let migration_scope = format!("{}/{}", scope, MIGRATION_SCOPE);
let migration_info_key = KeyStoreKey::scoped(migration_scope.clone(), "info.json".to_string());
let mut info: StoredValueInfo = match self.store.get(&migration_info_key) {
Ok(Some(info)) => info,
_ => StoredValueInfo::default(),
};
@@ -121,10 +130,11 @@ impl UpgradeStore for PubdStoreMigration {
info.last_command = 1;
// migrate init
let init_key = Self::event_key(&scope, 0);
let old_init_key = Self::event_key(&migration_scope, 0);
let init_key = Self::event_key(scope, 0);
let old_init: OldPubdInit = self
.store
.get(&init_key)?
.get(&old_init_key)?
.ok_or_else(|| UpgradeError::custom("Cannot read pubd init event"))?;
let (_, _, old_init) = old_init.unpack();
@@ -133,29 +143,51 @@ impl UpgradeStore for PubdStoreMigration {
self.store.store(&init_key, &init)?;
// migrate commands and events
let cmd_keys = self.command_keys(scope)?;
info!("Will migrate {} commands for publication server", cmd_keys.len());
let old_cmd_keys = self.command_keys(&migration_scope)?;
let mut total_migrated = 0;
let time_started = Time::now();
let total_commands = old_cmd_keys.len();
for cmd_key in cmd_keys {
let mut old_cmd: OldStoredRepositoryCommand = self.get(&cmd_key)?;
self.archive_to_migration_scope(&cmd_key)?;
info!("Will migrate {} commands for publication server", total_commands);
for old_cmd_key in old_cmd_keys {
// Do the migration counter first, so that we can just call continue when we need to skip commands
total_migrated += 1;
if total_migrated % 100 == 0 {
// ETA:
// - (total_migrated / (now - started)) * total
let mut time_passed = (Time::now().timestamp() - time_started.timestamp()) as usize;
if time_passed == 0 {
time_passed = 1; // avoid divide by zero.. we are doing approximate estimates here
}
let migrated_per_second = total_migrated / time_passed;
let expected_seconds = (total_commands / migrated_per_second) as i64;
let eta = time_started + Duration::seconds(expected_seconds);
info!(
" migrated {} commands, expect to finish: {}",
total_migrated,
eta.to_rfc3339()
);
}
if old_cmd_key.name().contains("pubd-publish.json") {
continue; // There is no migration needed for these commands.
}
let mut old_cmd: OldStoredRepositoryCommand = self.get(&old_cmd_key)?;
if let Some(evt_versions) = old_cmd.effect.events() {
debug!(" command: {}", cmd_key);
debug!(" command: {}", old_cmd_key);
let mut events = vec![];
for v in evt_versions {
let event_key = Self::event_key(scope, *v);
debug!(" +- event: {}", event_key);
let old_event_key = Self::event_key(&migration_scope, *v);
debug!(" +- event: {}", old_event_key);
let old_evt: OldPubdEvt = self
.store
.get(&event_key)?
.ok_or_else(|| UpgradeError::Custom(format!("Cannot parse old event: {}", event_key)))?;
self.archive_to_migration_scope(&event_key)?;
.get(&old_event_key)?
.ok_or_else(|| UpgradeError::Custom(format!("Cannot parse old event: {}", old_event_key)))?;
if old_evt.needs_migration() {
info.last_event += 1;
@@ -185,11 +217,6 @@ impl UpgradeStore for PubdStoreMigration {
let key = KeyStoreKey::scoped(scope.to_string(), format!("{}.json", cmd_key));
self.store.store(&key, &migrated_cmd)?;
total_migrated += 1;
if total_migrated % 100 == 0 {
info!(" migrated {} commands", total_migrated);
}
}
info!("Finished migrating publication server commands");
@@ -202,6 +229,7 @@ impl UpgradeStore for PubdStoreMigration {
// update the info file
info.snapshot_version = 0;
info.last_command -= 1;
let info_key = KeyStoreKey::scoped(scope.to_string(), "info.json".to_string());
self.store.store(&info_key, &info)?;
// verify that we can now rebuild the 0.9 publication server based on