Moving all the http client stuff into crate::util::httpclient, and make it a bit smarter especially w.r.t. json handling.

This commit is contained in:
Tim Bruijnzeels
2019-01-24 17:26:25 +01:00
parent 6248f0baf0
commit 252580a3ca
8 changed files with 329 additions and 336 deletions
+51 -171
View File
@@ -2,10 +2,7 @@ pub mod data;
pub mod options;
use std::io;
use std::time::Duration;
use bytes::Bytes;
use reqwest::{Client, Response, StatusCode};
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
use rpki::uri;
use crate::api::data::Publisher;
use crate::krillc::data::{
@@ -21,7 +18,7 @@ use crate::krillc::options::{
};
use crate::remote::rfc8183;
use crate::util::file;
use serde::Serialize;
use crate::util::httpclient;
/// Command line tool for Krill admin tasks
pub struct KrillClient {
@@ -60,7 +57,10 @@ impl KrillClient {
}
fn health(&self) -> Result<ApiResponse, Error> {
self.get("api/v1/health")?;
httpclient::get_ok(
&self.resolve_uri("api/v1/health"),
Some(&self.token)
)?;
Ok(ApiResponse::Health)
}
@@ -70,8 +70,10 @@ impl KrillClient {
) -> Result<ApiResponse, Error> {
match command {
PublishersCommand::List => {
let res = self.get("api/v1/publishers")?;
let list: PublisherList = serde_json::from_str(&res)?;
let list: PublisherList = httpclient::get_json(
&self.resolve_uri("api/v1/publishers"),
Some(&self.token)
)?;
Ok(ApiResponse::PublisherList(list))
},
PublishersCommand::Add(add) => {
@@ -81,48 +83,41 @@ impl KrillClient {
add.base_uri,
None
);
match self.post_json("api/v1/publishers", pbl)? {
Some(body) => {
if body.is_empty() {
Ok(ApiResponse::Empty)
} else {
Ok(ApiResponse::GenericBody(body))
}
},
None => Ok(ApiResponse::Empty)
}
self.add_publisher(pbl)
},
PublishersCommand::AddWithCms(add) => {
let bytes = file::read(&add.xml)?;
let pr = rfc8183::PublisherRequest::decode(bytes.as_ref())?;
let pbl = pr.into_publisher(add.token, add.base_uri);
match self.post_json("api/v1/publishers", pbl)? {
Some(body) => {
if body.is_empty() {
Ok(ApiResponse::Empty)
} else {
Ok(ApiResponse::GenericBody(body))
}
},
None => Ok(ApiResponse::Empty)
}
self.add_publisher(pbl)
},
PublishersCommand::Remove(handle) => {
let uri = format!("api/v1/publishers/{}", handle);
self.delete(&uri)?;
let uri = self.resolve_uri(&uri);
httpclient::delete(&uri, Some(&self.token))?;
Ok(ApiResponse::Empty)
},
PublishersCommand::Details(handle) => {
let uri = format!("api/v1/publishers/{}", handle);
let res = self.get(uri.as_str())?;
let details: PublisherDetails = serde_json::from_str(&res)?;
let uri = self.resolve_uri(&uri);
let details: PublisherDetails = httpclient::get_json(
&uri,
Some(&self.token)
)?;
Ok(ApiResponse::PublisherDetails(details))
},
PublishersCommand::RepositoryResponseXml(handle, file_opt) => {
let uri = format!("api/v1/publishers/{}/response.xml", handle);
let xml = self.get(uri.as_str())?;
let uri = self.resolve_uri(&uri);
let xml = httpclient::get_text(
&uri,
"application/xml",
Some(&self.token)
)?;
match file_opt {
Some(path) => {
file::save(&Bytes::from(xml), &path)?;
@@ -135,8 +130,13 @@ impl KrillClient {
},
PublishersCommand::IdCert(handle, file) => {
let uri = format!("api/v1/publishers/{}", handle);
let res = self.get(uri.as_str())?;
let details: PublisherDetails = serde_json::from_str(&res)?;
let uri = self.resolve_uri(&uri);
let details: PublisherDetails = httpclient::get_json(
&uri,
Some(&self.token)
)?;
match details.identity_cert() {
Some(cert) => {
let bytes = cert.to_bytes();
@@ -145,138 +145,23 @@ impl KrillClient {
},
None => Err(Error::NoIdCert)
}
}
}
}
fn headers(&self) -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(
USER_AGENT,
HeaderValue::from_str("krillc").unwrap()
);
headers.insert(
"Authorization",
HeaderValue::from_str(&format!("Bearer {}", &self.token)).unwrap()
);
headers
fn add_publisher(&self, pbl: Publisher) -> Result<ApiResponse, Error> {
httpclient::post_json(
&self.resolve_uri("api/v1/publishers"),
pbl,
Some(&self.token)
)?;
Ok(ApiResponse::Empty)
}
fn client(&self) -> Result<Client, Error> {
Client::builder()
.gzip(true)
.timeout(Duration::from_secs(30))
.build()
.map_err(|e| Error::RequestError(e))
fn resolve_uri(&self, path: &str) -> String {
format!("{}{}", &self.server, path)
}
fn post_json(
&self,
rel: &str,
data: impl Serialize
) -> Result<Option<String>, Error> {
let headers = self.headers();
let body = serde_json::to_string(&data)?;
let uri = format!("{}{}", &self.server.to_string(), rel);
let mut res = self.client()?.post(&uri)
.headers(headers)
.body(body)
.send()?;
match res.status() {
StatusCode::OK => {
Ok(res.text().ok())
},
status => {
match res.text() {
Ok(body) => {
if body.is_empty() {
Err(Error::BadStatus(status))
} else {
Err(Error::ErrorWithBody(body))
}
},
_ => Err(Error::BadStatus(status))
}
}
}
}
/// Sends a get request to the server, including the token for
/// authorization.
/// Note that the server uri ends with a '/', so leave out the '/'
/// from the start of the rel_path when calling this function.
fn get(
&self,
rel_path: &str
) -> Result<String, Error> {
let mut res = self.get_generic(rel_path)?;
let txt = res.text()?;
Ok(txt)
}
/// Sends a get request to the server, including the token for
/// authorization.
/// Note that the server uri ends with a '/', so leave out the '/'
/// from the start of the rel_path when calling this function.
#[allow(dead_code)]
fn get_binary(
&self,
rel_path: &str
) -> Result<Bytes, Error> {
let mut res = self.get_generic(rel_path)?;
let mut bytes: Vec<u8> = vec![];
res.copy_to(&mut bytes)?;
Ok(Bytes::from(bytes))
}
fn get_generic(
&self,
rel_path: &str
) -> Result<Response, Error> {
let uri = format!("{}{}", &self.server.to_string(), rel_path);
let mut res = self.client()?
.get(&uri)
.headers(self.headers())
.send()?;
match res.status() {
StatusCode::OK => {
Ok(res)
},
status => {
match res.text() {
Ok(body) => {
if body.is_empty() {
Err(Error::BadStatus(status))
} else {
Err(Error::ErrorWithBody(body))
}
},
_ => Err(Error::BadStatus(status))
}
}
}
}
fn delete(
&self,
rel: &str
) -> Result<(), Error> {
let uri = format!("{}{}", &self.server.to_string(), rel);
let res = self.client()?
.delete(&uri)
.headers(self.headers())
.send()?;
match res.status() {
StatusCode::OK => Ok(()),
status => Err(Error::BadStatus(status))
}
}
}
//------------ Error ---------------------------------------------------------
@@ -289,14 +174,8 @@ pub enum Error {
#[display(fmt="Server is not available.")]
ServerDown,
#[display(fmt="Request Error: {}", _0)]
RequestError(reqwest::Error),
#[display(fmt="Received bad status: {}", _0)]
BadStatus(StatusCode),
#[display(fmt="{}", _0)]
ErrorWithBody(String),
HttpClientError(httpclient::Error),
#[display(fmt="Received invalid json response: {}", _0)]
JsonError(serde_json::Error),
@@ -311,13 +190,14 @@ pub enum Error {
NoIdCert,
#[display(fmt="Invalid RFC8183 XML")]
InvalidRfc8183
InvalidRfc8183,
#[display(fmt="Empty response received from server")]
EmptyResponse
}
impl From<reqwest::Error> for Error {
fn from(e: reqwest::Error) -> Self {
Error::RequestError(e)
}
impl From<httpclient::Error> for Error {
fn from(e: httpclient::Error) -> Self { Error::HttpClientError(e) }
}
impl From<serde_json::Error> for Error {
+1 -2
View File
@@ -68,8 +68,7 @@ pub fn publishers(req: &HttpRequest) -> HttpResponse {
Err(e) => server_error(Error::ServerError(e)),
Ok(publishers) => {
render_json(
responses::PublisherList::from(&publishers,
"/api/v1/publishers")
responses::PublisherList::from(&publishers, "/api/v1/publishers")
)
}
}
+1 -1
View File
@@ -10,7 +10,7 @@ use actix_web::{pred, fs, server};
use actix_web::{App, FromRequest, HttpResponse};
use actix_web::dev::MessageBody;
use actix_web::middleware;
use actix_web::http::{Method, StatusCode };
use actix_web::http::{Method, StatusCode};
use bcder::decode;
use futures::Future;
use openssl::ssl::{SslMethod, SslAcceptor, SslAcceptorBuilder, SslFiletype};
+11 -101
View File
@@ -1,12 +1,9 @@
//! Publication Client that uses the JSON/Rest API
use std::path::PathBuf;
use clap::{App, Arg, SubCommand};
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT, CONTENT_TYPE};
use reqwest::{Client, Response, StatusCode};
use rpki::uri;
use serde::Serialize;
use std::time::Duration;
use api::responses;
use crate::api::responses;
use crate::util::httpclient;
//------------ PubClientOptions ----------------------------------------------
@@ -125,15 +122,12 @@ pub fn execute(options: PubClientOptions) -> Result<ApiResponse, Error> {
&options.handle
);
match get_text(&uri, &options.token) {
Err(e) => Err(e),
Ok(None) => Err(Error::NoResponse),
Ok(Some(text)) => {
let list: responses::ListReply =
serde_json::from_str(&text)?;
Ok(ApiResponse::List(list))
}
match httpclient::get_json::<responses::ListReply>(
&uri,
Some(&options.token)
) {
Err(e) => Err(Error::HttpClientError(e)),
Ok(list) => Ok(ApiResponse::List(list))
}
},
Command::Sync(_dir) => {
@@ -143,77 +137,6 @@ pub fn execute(options: PubClientOptions) -> Result<ApiResponse, Error> {
}
fn client() -> Result<Client, Error> {
Client::builder()
.gzip(true)
.timeout(Duration::from_secs(30))
.build()
.map_err(|e| Error::RequestError(e))
}
fn headers(token: &str) -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(
USER_AGENT,
HeaderValue::from_str("krill-pubc").unwrap()
);
headers.insert(
CONTENT_TYPE,
HeaderValue::from_str("application/json").unwrap()
);
headers.insert(
"Authorization",
HeaderValue::from_str(&format!("Bearer {}", token)).unwrap()
);
headers
}
fn process_response(mut res: Response) -> Result<Option<String>, Error> {
match res.status() {
StatusCode::OK => {
Ok(res.text().ok())
},
status => {
match res.text() {
Ok(body) => {
if body.is_empty() {
Err(Error::BadStatus(status))
} else {
Err(Error::ErrorWithBody(body))
}
},
_ => Err(Error::BadStatus(status))
}
}
}
}
fn get_text(
uri: &str,
token: &str
) -> Result<Option<String>, Error> {
let headers = headers(token);
let res = client()?.get(uri).headers(headers).send()?;
process_response(res)
}
#[allow(dead_code)]
fn post_json(
uri: &str,
data: impl Serialize,
token: &str
) -> Result<Option<String>, Error> {
let headers = headers(token);
let body = serde_json::to_string(&data)?;
let client = client()?;
let res = client.post(uri).headers(headers).body(body).send()?;
process_response(res)
}
//------------ Command -------------------------------------------------------
#[derive(Clone, Debug, Eq, PartialEq)]
@@ -252,21 +175,14 @@ pub enum Error {
#[display(fmt = "Specify an action: list, or sync --dir <dir>")]
NoCommand,
#[display(fmt="Request Error: {}", _0)]
RequestError(reqwest::Error),
#[display(fmt="Received bad status: {}", _0)]
BadStatus(StatusCode),
#[display(fmt="{}", _0)]
ErrorWithBody(String),
#[display(fmt = "Expected a response body, but got nothing.")]
NoResponse,
#[display(fmt="{}", _0)]
HttpClientError(httpclient::Error),
#[display(fmt="Received invalid json response: {}", _0)]
JsonError(serde_json::Error),
}
impl From<uri::Error> for Error {
@@ -276,9 +192,3 @@ impl From<uri::Error> for Error {
impl From<serde_json::Error> for Error {
fn from(e: serde_json::Error) -> Self { Error::JsonError(e) }
}
impl From<reqwest::Error> for Error {
fn from(e: reqwest::Error) -> Self {
Error::RequestError(e)
}
}
+16 -60
View File
@@ -1,14 +1,10 @@
use std::io;
use std::io::Read;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use bcder::{Captured, Mode};
use bcder::decode;
use bcder::encode::Values;
use clap::{App, Arg, SubCommand};
use reqwest::{Client, Response, StatusCode};
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT, CONTENT_TYPE};
use rpki::x509::ValidationError;
use rpki::crypto::{PublicKeyFormat, Signer};
use toml;
@@ -20,6 +16,7 @@ use crate::remote::rfc8181;
use crate::remote::sigmsg::SignedMessage;
use crate::storage::caching_ks::CachingDiskKeyStore;
use crate::storage::keystore::{self, Info, Key, KeyStore};
use crate::util::httpclient;
use crate::util::softsigner::{self, OpenSslSigner};
use crate::util::file::{self, CurrentFile, RecursorError};
@@ -277,51 +274,15 @@ impl PubClient {
req: Captured
) -> Result<rfc8181::Message, Error> {
let parent = self.get_my_parent()?;
let post_bytes = req.into_bytes();
let mut headers = HeaderMap::new();
headers.insert(
USER_AGENT,
HeaderValue::from_str("krill-pubc-cms").unwrap()
);
headers.insert(
CONTENT_TYPE,
HeaderValue::from_str("application/rpki-publication").unwrap()
);
let res_bytes = httpclient::post_binary(
&parent.service_uri().to_string(),
&post_bytes,
"application/rpki-publication"
)?;
let client = Client::builder()
.gzip(true)
// 5 minute timeout is enough to sync the entire RIPE NCC repo.
.timeout(Duration::from_secs(300))
.build()?;
let mut res = client.post(&parent.service_uri().to_string())
.headers(headers)
.body(req.to_vec())
.send()?;
match res.status() {
StatusCode::OK => {
self.parse_res(res)
},
_ => {
println!("{}", res.text()
.unwrap_or("unspecified error".to_string()));
Err(Error::PubServerHttpError(res.status()))
}
}
}
fn parse_res(
&mut self,
mut res: Response
) -> Result<rfc8181::Message, Error> {
let parent = self.get_my_parent()?;
let mut bytes: Vec<u8> = vec![];
res.read_to_end(&mut bytes).unwrap();
let bytes = bytes::Bytes::from(bytes);
let signed_msg = SignedMessage::decode(bytes, true)?;
let signed_msg = SignedMessage::decode(res_bytes, true)?;
signed_msg.validate(parent.id_cert())?;
rfc8181::Message::from_signed_message(&signed_msg).map_err(|e| {
Error::MessageError(e)
@@ -526,12 +487,6 @@ pub enum Error {
#[display(fmt="{}", _0)]
KeyStoreError(keystore::Error),
#[display(fmt="Received bad HTTP status code: {}", _0)]
PubServerHttpError(StatusCode),
#[display(fmt="Request Error: {}", _0)]
RequestError(reqwest::Error),
#[display(fmt="{}", _0)]
ValidationError(ValidationError),
@@ -551,7 +506,10 @@ pub enum Error {
RecursorError(RecursorError),
#[display(fmt="{}", _0)]
BuilderError(builder::Error<softsigner::SignerError>)
BuilderError(builder::Error<softsigner::SignerError>),
#[display(fmt="{}", _0)]
HttpClientError(httpclient::Error),
}
impl From<softsigner::SignerError> for Error {
@@ -566,12 +524,6 @@ impl From<keystore::Error> for Error {
}
}
impl From<reqwest::Error> for Error {
fn from(e: reqwest::Error) -> Self {
Error::RequestError(e)
}
}
impl From<ValidationError> for Error {
fn from(e: ValidationError) -> Self {
Error::ValidationError(e)
@@ -602,5 +554,9 @@ impl From<builder::Error<softsigner::SignerError>> for Error {
}
}
impl From<httpclient::Error> for Error {
fn from(e: httpclient::Error) -> Self { Error::HttpClientError(e) }
}
// Tested in integration tests in tests folder
+245
View File
@@ -0,0 +1,245 @@
//! Some helper functions for HTTP calls
use std::io::Read;
use std::time::Duration;
use bytes::Bytes;
use reqwest::{Client, Response, StatusCode};
use reqwest::header::{
HeaderMap,
HeaderValue,
InvalidHeaderValue,
USER_AGENT,
CONTENT_TYPE};
use serde::Serialize;
use serde::de::DeserializeOwned;
const JSON_CONTENT: &'static str = "application/json";
/// Performs a GET request that expects a json response that can be
/// deserialized into the an owned value of the expected type. Returns an error
/// if nothing is returned.
pub fn get_json<T: DeserializeOwned>(
uri: &str,
token: Option<&str>
) -> Result<T, Error> {
let headers = headers(Some(JSON_CONTENT), token)?;
let res = client()?.get(uri).headers(headers).send()?;
process_json_response(res)
}
/// Performs a get request and expects a response that can be turned
/// into a string (in particular, not a binary response).
pub fn get_text(
uri: &str,
content_type: &str,
token: Option<&str>
) -> Result<String, Error> {
let headers = headers(Some(content_type), token)?;
let res = client()?.get(uri).headers(headers).send()?;
match opt_text_response(res)? {
Some(res) => Ok(res),
None => Err(Error::EmptyResponse)
}
}
/// Checks that there is a 200 OK response at the given URI. Discards the
/// response body.
pub fn get_ok(uri: &str, token: Option<&str>) -> Result<(), Error> {
let headers = headers(None, token)?;
let res = client()?.get(uri).headers(headers).send()?;
opt_text_response(res)?; // Will return nice errors with possible body.
Ok(())
}
/// Performs a POST of data that can be serialized into json, and expects
/// a 200 OK response, without a body.
pub fn post_json(
uri: &str,
data: impl Serialize,
token: Option<&str>
) -> Result<(), Error> {
let headers = headers(Some(JSON_CONTENT), token)?;
let body = serde_json::to_string(&data)?;
let res = client()?.post(uri).headers(headers).body(body).send()?;
if let Some(res) = opt_text_response(res)? {
Err(Error::UnexpectedResponse(res))
} else {
Ok(())
}
}
/// Performs a POST of data that can be serialized into json, and expects
/// a json response that can be deserialized into the an owned value of the
/// expected type.
pub fn post_json_with_response<T: DeserializeOwned>(
uri: &str,
data: impl Serialize,
token: Option<&str>
) -> Result<T, Error> {
let headers = headers(Some(JSON_CONTENT), token)?;
let body = serde_json::to_string(&data)?;
let res = client()?.post(uri).headers(headers).body(body).send()?;
process_json_response(res)
}
/// Posts binary data, and expects a binary response.
///
/// Note: Bytes may be empty if the post was successful, but the response was
/// empty.
pub fn post_binary(
uri: &str,
data: &Bytes,
content_type: &str
) -> Result<Bytes, Error> {
let headers = headers(Some(content_type), None)?;
let body = data.to_vec();
let mut res = client()?.post(uri).headers(headers).body(body).send()?;
match res.status() {
StatusCode::OK => {
let mut bytes: Vec<u8> = vec![];
res.read_to_end(&mut bytes).unwrap();
let bytes = bytes::Bytes::from(bytes);
Ok(bytes)
},
status => {
match res.text() {
Ok(body) => {
if body.is_empty() {
Err(Error::BadStatus(status))
} else {
Err(Error::ErrorWithBody(status, body))
}
},
_ => Err(Error::BadStatus(status))
}
}
}
}
/// Sends a delete request to the specified url.
pub fn delete(
uri: &str,
token: Option<&str>
) -> Result<(), Error> {
let headers = headers(None, token)?;
client()?.delete(uri).headers(headers).send()?;
Ok(())
}
fn client() -> Result<Client, Error> {
Client::builder()
.gzip(true)
.timeout(Duration::from_secs(30))
.build()
.map_err(|e| Error::RequestError(e))
}
fn headers(
content_type: Option<&str>,
token: Option<&str>
) -> Result<HeaderMap, Error> {
let mut headers = HeaderMap::new();
headers.insert(
USER_AGENT,
HeaderValue::from_str("krill")?
);
if let Some(content_type) = content_type {
headers.insert(
CONTENT_TYPE,
HeaderValue::from_str(content_type)?
);
}
if let Some(token) = token {
headers.insert(
"Authorization",
HeaderValue::from_str(&format!("Bearer {}", token))?
);
}
Ok(headers)
}
fn process_json_response<T: DeserializeOwned>(
res: Response
) -> Result<T, Error> {
match opt_text_response(res) {
Err(e) => Err(e),
Ok(None) => Err(Error::EmptyResponse),
Ok(Some(s)) => {
let res: T = serde_json::from_str(&s)?;
Ok(res)
}
}
}
fn opt_text_response(mut res: Response) -> Result<Option<String>, Error> {
match res.status() {
StatusCode::OK => {
match res.text().ok() {
None => Ok(None),
Some(s) => {
if s.is_empty() {
Ok(None)
} else {
Ok(Some(s))
}
}
}
},
status => {
match res.text() {
Ok(body) => {
if body.is_empty() {
Err(Error::BadStatus(status))
} else {
Err(Error::ErrorWithBody(status, body))
}
},
_ => Err(Error::BadStatus(status))
}
}
}
}
//------------ Error ---------------------------------------------------------
#[derive(Debug, Display)]
pub enum Error {
#[display(fmt="Request Error: {}", _0)]
RequestError(reqwest::Error),
#[display(fmt="Received bad status: {}", _0)]
BadStatus(StatusCode),
#[display(fmt="Status: {}, Error: {}", _0, _1)]
ErrorWithBody(StatusCode, String),
#[display(fmt="{}", _0)]
JsonError(serde_json::Error),
#[display(fmt="{}", _0)]
InvalidHeader(InvalidHeaderValue),
#[display(fmt="Empty response received from server")]
EmptyResponse,
#[display(fmt="Unexpected response: {}", _0)]
UnexpectedResponse(String)
}
impl From<reqwest::Error> for Error {
fn from(e: reqwest::Error) -> Self { Error::RequestError(e) }
}
impl From<serde_json::Error> for Error {
fn from(e: serde_json::Error) -> Self { Error::JsonError(e) }
}
impl From<InvalidHeaderValue> for Error {
fn from(v: InvalidHeaderValue) -> Self { Error::InvalidHeader(v) }
}
+1
View File
@@ -4,6 +4,7 @@ use rpki::crypto::DigestAlgorithm;
pub mod ext_serde;
pub mod file;
pub mod httpclient;
pub mod softsigner;
pub mod test;
pub mod xml;
+3 -1
View File
@@ -25,6 +25,7 @@ use krill::krillc::options::{
};
use krill::pubc::cmsclient::PubClient;
use krill::util::test;
use krill::util::httpclient;
use krill::remote::rfc8183::RepositoryResponse;
use reqwest::StatusCode;
@@ -179,7 +180,8 @@ fn manage_publishers() {
let res = KrillClient::process(krillc_opts);
match res {
Err(krillc::Error::BadStatus(code)) => {
Err(krillc::Error::HttpClientError(
httpclient::Error::BadStatus(code))) => {
assert_eq!(code, StatusCode::NOT_FOUND);
},
_ => assert!(false) // should have failed!