From 1ac1f7fd24f36af887fa0e06ecb686685e3744a2 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 15 Nov 2018 17:07:10 +0100 Subject: [PATCH 001/344] cleanup module names --- src/api/server.rs | 295 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 src/api/server.rs diff --git a/src/api/server.rs b/src/api/server.rs new file mode 100644 index 00000000..889c3a92 --- /dev/null +++ b/src/api/server.rs @@ -0,0 +1,295 @@ +use crate::api::schema::*; +use crate::api::router::*; +use crate::api::config::*; + +use std::fmt; +use std::path::{PathBuf}; +use std::sync::Arc; + +use failure::*; +use serde_json::{json, Value}; + +use futures::future::{self, Either}; +//use tokio::prelude::*; +//use tokio::timer::Delay; +use tokio::fs::File; +use tokio_codec; +//use bytes::{BytesMut, BufMut}; + +//use hyper::body::Payload; +use hyper::http::request::Parts; +use hyper::{Body, Request, Response, StatusCode}; +use hyper::service::{Service, NewService}; +use hyper::rt::{Future, Stream}; +use hyper::header; + +pub struct RestServer { + pub api_config: Arc, +} + +impl RestServer { + + pub fn new(api_config: ApiConfig) -> Self { + Self { api_config: Arc::new(api_config) } + } +} + +impl NewService for RestServer +{ + type ReqBody = Body; + type ResBody = Body; + type Error = hyper::Error; + type InitError = hyper::Error; + type Service = ApiService; + type Future = Box + Send>; + fn new_service(&self) -> Self::Future { + Box::new(future::ok(ApiService { api_config: self.api_config.clone() })) + } +} + +pub struct ApiService { + pub api_config: Arc, +} + + +impl Service for ApiService { + type ReqBody = Body; + type ResBody = Body; + type Error = hyper::Error; + type Future = Box, Error = Self::Error> + Send>; + + fn call(&mut self, req: Request) -> Self::Future { + + Box::new(handle_request(self.api_config.clone(), req).then(|result| { + match result { + Ok(res) => Ok::<_, hyper::Error>(res), + Err(err) => { + if let Some(apierr) = err.downcast_ref::() { + let mut resp = Response::new(Body::from(apierr.message.clone())); + *resp.status_mut() = apierr.code; + Ok(resp) + } else { + let mut resp = Response::new(Body::from(err.to_string())); + *resp.status_mut() = StatusCode::BAD_REQUEST; + Ok(resp) + } + } + } + })) + } +} + +type BoxFut = Box, Error = failure::Error> + Send>; + +#[derive(Debug, Fail)] +pub struct HttpError { + pub code: StatusCode, + pub message: String, +} + +impl HttpError { + pub fn new(code: StatusCode, message: String) -> Self { + HttpError { code, message } + } +} + +impl fmt::Display for HttpError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Error {}: {}", self.code, self.message) + } +} + +macro_rules! http_err { + ($status:ident, $msg:expr) => {{ + Error::from(HttpError::new(StatusCode::$status, $msg)) + }} +} + +fn get_request_parameters_async( + info: &'static ApiMethod, + parts: Parts, + req_body: Body, +) -> Box + Send> +{ + let resp = req_body + .map_err(|err| http_err!(BAD_REQUEST, format!("Promlems reading request body: {}", err))) + .fold(Vec::new(), |mut acc, chunk| { + if acc.len() + chunk.len() < 64*1024 { //fimxe: max request body size? + acc.extend_from_slice(&*chunk); + Ok(acc) + } + else { Err(http_err!(BAD_REQUEST, format!("Request body too large"))) } + }) + .and_then(move |body| { + + let bytes = String::from_utf8(body.to_vec())?; // why copy?? + + println!("GOT BODY {:?}", bytes); + + let mut test_required = true; + + let mut params = json!({}); + + if bytes.len() > 0 { + params = parse_query_string(&bytes, &info.parameters, true)?; + test_required = false; + } + + if let Some(query_str) = parts.uri.query() { + let query_params = parse_query_string(query_str, &info.parameters, test_required)?; + + for (k, v) in query_params.as_object().unwrap() { + params[k] = v.clone(); // fixme: why clone()?? + } + } + + println!("GOT PARAMS {}", params); + Ok(params) + }); + + Box::new(resp) +} + +fn handle_sync_api_request( + info: &'static ApiMethod, + parts: Parts, + req_body: Body, +) -> BoxFut +{ + let params = get_request_parameters_async(info, parts, req_body); + + let resp = params + .and_then(move |params| { + + println!("GOT PARAMS {}", params); + + /* + let when = Instant::now() + Duration::from_millis(3000); + let task = Delay::new(when).then(|_| { + println!("A LAZY TASK"); + ok(()) + }); + + tokio::spawn(task); + */ + + let res = (info.handler)(params, info)?; + + Ok(res) + + }).then(|result| { + match result { + Ok(ref value) => { + let json_str = value.to_string(); + + Ok(Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(json_str))?) + } + Err(err) => Err(http_err!(BAD_REQUEST, err.to_string())) + } + }); + + Box::new(resp) +} + +fn simple_static_file_download(filename: PathBuf) -> BoxFut { + + Box::new(File::open(filename) + .map_err(|err| http_err!(BAD_REQUEST, format!("File open failed: {}", err))) + .and_then(|file| { + let buf: Vec = Vec::new(); + tokio::io::read_to_end(file, buf) + .map_err(|err| http_err!(BAD_REQUEST, format!("File read failed: {}", err))) + .and_then(|data| Ok(Response::new(data.1.into()))) + })) +} + +fn chuncked_static_file_download(filename: PathBuf) -> BoxFut { + + Box::new(File::open(filename) + .map_err(|err| http_err!(BAD_REQUEST, format!("File open failed: {}", err))) + .and_then(|file| { + let payload = tokio_codec::FramedRead::new(file, tokio_codec::BytesCodec::new()). + map(|bytes| { + //sigh - howto avoid copy here? or the whole map() ?? + hyper::Chunk::from(bytes.to_vec()) + }); + let body = Body::wrap_stream(payload); + // fixme: set content type and other headers + Ok(Response::builder() + .status(StatusCode::OK) + .body(body) + .unwrap()) + })) +} + +fn handle_static_file_download(filename: PathBuf) -> BoxFut { + + let response = tokio::fs::metadata(filename.clone()) + .map_err(|err| http_err!(BAD_REQUEST, format!("File access problems: {}", err))) + .and_then(|metadata| { + if metadata.len() < 1024*32 { + Either::A(simple_static_file_download(filename)) + } else { + Either::B(chuncked_static_file_download(filename)) + } + }); + + return Box::new(response); +} + +pub fn handle_request(api: Arc, req: Request) -> BoxFut { + + let (parts, body) = req.into_parts(); + + let method = parts.method.clone(); + let path = parts.uri.path(); + + // normalize path + // do not allow ".", "..", or hidden files ".XXXX" + // also remove empty path components + + let items = path.split('/'); + let mut path = String::new(); + let mut components = vec![]; + + for name in items { + if name.is_empty() { continue; } + if name.starts_with(".") { + return Box::new(future::err(http_err!(BAD_REQUEST, "Path contains illegal components.".to_string()))); + } + path.push('/'); + path.push_str(name); + components.push(name); + } + + let comp_len = components.len(); + + println!("REQUEST {} {}", method, path); + println!("COMPO {:?}", components); + + if comp_len >= 1 && components[0] == "api3" { + println!("GOT API REQUEST"); + if comp_len >= 2 { + let format = components[1]; + if format != "json" { + return Box::new(future::err(http_err!(BAD_REQUEST, format!("Unsupported output format '{}'.", format)))) + } + + if let Some(api_method) = api.find_method(&components[2..], method) { + // fixme: handle auth + return handle_sync_api_request(api_method, parts, body); + } + } + } else { + // not Auth for accessing files! + + let filename = api.find_alias(&components); + return handle_static_file_download(filename); + } + + Box::new(future::err(http_err!(NOT_FOUND, "Path not found.".to_string()))) +} + From 5106bbc70eca509c617c367858b3cd8b076ad34a Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 15 Nov 2018 17:47:59 +0100 Subject: [PATCH 002/344] allow closure handlers --- src/api/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/server.rs b/src/api/server.rs index 889c3a92..c8162995 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -173,7 +173,7 @@ fn handle_sync_api_request( tokio::spawn(task); */ - let res = (info.handler)(params, info)?; + let res = (*info.handler)(params, info)?; Ok(res) From 17161122850a18c708cc560d967c0a0ec14c691e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 16 Nov 2018 09:15:33 +0100 Subject: [PATCH 003/344] handle uri parameters correctly --- src/api/server.rs | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/api/server.rs b/src/api/server.rs index c8162995..3c61b4ac 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -5,9 +5,11 @@ use crate::api::config::*; use std::fmt; use std::path::{PathBuf}; use std::sync::Arc; +use std::collections::HashMap; use failure::*; -use serde_json::{json, Value}; +use serde_json::{Value}; +use url::form_urlencoded; use futures::future::{self, Either}; //use tokio::prelude::*; @@ -109,6 +111,7 @@ fn get_request_parameters_async( info: &'static ApiMethod, parts: Parts, req_body: Body, + uri_param: HashMap, ) -> Box + Send> { let resp = req_body @@ -126,23 +129,27 @@ fn get_request_parameters_async( println!("GOT BODY {:?}", bytes); - let mut test_required = true; - - let mut params = json!({}); + let mut param_list: Vec<(String, String)> = vec![]; if bytes.len() > 0 { - params = parse_query_string(&bytes, &info.parameters, true)?; - test_required = false; + for (k, v) in form_urlencoded::parse(bytes.as_bytes()).into_owned() { + param_list.push((k, v)); + } + } if let Some(query_str) = parts.uri.query() { - let query_params = parse_query_string(query_str, &info.parameters, test_required)?; - - for (k, v) in query_params.as_object().unwrap() { - params[k] = v.clone(); // fixme: why clone()?? + for (k, v) in form_urlencoded::parse(query_str.as_bytes()).into_owned() { + param_list.push((k, v)); } } + for (k, v) in uri_param { + param_list.push((k.clone(), v.clone())); + } + + let params = parse_parameter_strings(¶m_list, &info.parameters, true)?; + println!("GOT PARAMS {}", params); Ok(params) }); @@ -154,9 +161,10 @@ fn handle_sync_api_request( info: &'static ApiMethod, parts: Parts, req_body: Body, + uri_param: HashMap, ) -> BoxFut { - let params = get_request_parameters_async(info, parts, req_body); + let params = get_request_parameters_async(info, parts, req_body, uri_param); let resp = params .and_then(move |params| { @@ -278,9 +286,11 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { return Box::new(future::err(http_err!(BAD_REQUEST, format!("Unsupported output format '{}'.", format)))) } - if let Some(api_method) = api.find_method(&components[2..], method) { + let mut uri_param = HashMap::new(); + + if let Some(api_method) = api.find_method(&components[2..], method, &mut uri_param) { // fixme: handle auth - return handle_sync_api_request(api_method, parts, body); + return handle_sync_api_request(api_method, parts, body, uri_param); } } } else { @@ -292,4 +302,3 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { Box::new(future::err(http_err!(NOT_FOUND, "Path not found.".to_string()))) } - From dd4b1a797bf1bf7430aa0ac43ac7f8d2dd69a465 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 16 Nov 2018 11:12:00 +0100 Subject: [PATCH 004/344] router: no need to use Fn (fn also works for static closures) --- src/api/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/server.rs b/src/api/server.rs index 3c61b4ac..d1f87135 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -181,7 +181,7 @@ fn handle_sync_api_request( tokio::spawn(task); */ - let res = (*info.handler)(params, info)?; + let res = (info.handler)(params, info)?; Ok(res) From b53007523d1ba9f5209814b69d1b10d39784a818 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 1 Dec 2018 13:37:49 +0100 Subject: [PATCH 005/344] remove www/pbs-index.html.tt, hardcode into rust for now --- src/api/server.rs | 61 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/src/api/server.rs b/src/api/server.rs index d1f87135..07542921 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use std::collections::HashMap; use failure::*; -use serde_json::{Value}; +use serde_json::{json, Value}; use url::form_urlencoded; use futures::future::{self, Either}; @@ -202,6 +202,57 @@ fn handle_sync_api_request( Box::new(resp) } +fn get_index() -> BoxFut { + + let nodename = "unknown"; + let username = ""; + let token = "abc"; + + let setup = json!({ + "Setup": { "auth_cookie_name": "PBSAuthCookie" }, + "NodeName": nodename, + "UserName": username, + "CSRFPreventionToken": token + }); + + let index = format!(r###" + + + + + + + Proxmox Backup Server + + + + + + + + + + + + + + + + +
+ +
+ + +"###, setup.to_string()); + + Box::new(future::ok(Response::new(index.into()))) +} + fn simple_static_file_download(filename: PathBuf) -> BoxFut { Box::new(File::open(filename) @@ -296,8 +347,12 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { } else { // not Auth for accessing files! - let filename = api.find_alias(&components); - return handle_static_file_download(filename); + if comp_len == 0 { + return get_index(); + } else { + let filename = api.find_alias(&components); + return handle_static_file_download(filename); + } } Box::new(future::err(http_err!(NOT_FOUND, "Path not found.".to_string()))) From 4892b328299e7207acdd10593525c12892f9eb87 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 1 Dec 2018 15:21:25 +0100 Subject: [PATCH 006/344] fix file download, listen to 0.0.0.0 --- src/api/server.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/api/server.rs b/src/api/server.rs index 07542921..a8ed1ab7 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -223,23 +223,23 @@ fn get_index() -> BoxFut { Proxmox Backup Server - + - - + + - - + + - - + + - + From 185f4301dce2efda22e4ae92478eee49405eb608 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 2 Dec 2018 11:00:52 +0100 Subject: [PATCH 007/344] set content type for static file download --- src/api/server.rs | 51 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/src/api/server.rs b/src/api/server.rs index a8ed1ab7..485d1e7e 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -3,7 +3,7 @@ use crate::api::router::*; use crate::api::config::*; use std::fmt; -use std::path::{PathBuf}; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::collections::HashMap; @@ -253,32 +253,73 @@ fn get_index() -> BoxFut { Box::new(future::ok(Response::new(index.into()))) } +fn extension_to_content_type(filename: &Path) -> (&'static str, bool) { + + if let Some(ext) = filename.extension().and_then(|osstr| osstr.to_str()) { + return match ext { + "css" => ("text/css", false), + "html" => ("text/html", false), + "js" => ("application/javascript", false), + "json" => ("application/json", false), + "map" => ("application/json", false), + "png" => ("image/png", true), + "ico" => ("image/x-icon", true), + "gif" => ("image/gif", true), + "svg" => ("image/svg+xml", false), + "jar" => ("application/java-archive", true), + "woff" => ("application/font-woff", true), + "woff2" => ("application/font-woff2", true), + "ttf" => ("application/font-snft", true), + "pdf" => ("application/pdf", true), + "epub" => ("application/epub+zip", true), + "mp3" => ("audio/mpeg", true), + "oga" => ("audio/ogg", true), + "tgz" => ("application/x-compressed-tar", true), + _ => ("application/octet-stream", false), + }; + } + + ("application/octet-stream", false) +} + fn simple_static_file_download(filename: PathBuf) -> BoxFut { + let (content_type, _nocomp) = extension_to_content_type(&filename); + Box::new(File::open(filename) .map_err(|err| http_err!(BAD_REQUEST, format!("File open failed: {}", err))) - .and_then(|file| { + .and_then(move |file| { let buf: Vec = Vec::new(); tokio::io::read_to_end(file, buf) .map_err(|err| http_err!(BAD_REQUEST, format!("File read failed: {}", err))) - .and_then(|data| Ok(Response::new(data.1.into()))) + .and_then(move |data| { + let mut response = Response::new(data.1.into()); + response.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static(content_type)); + Ok(response) + }) })) } fn chuncked_static_file_download(filename: PathBuf) -> BoxFut { + let (content_type, _nocomp) = extension_to_content_type(&filename); + Box::new(File::open(filename) .map_err(|err| http_err!(BAD_REQUEST, format!("File open failed: {}", err))) - .and_then(|file| { + .and_then(move |file| { let payload = tokio_codec::FramedRead::new(file, tokio_codec::BytesCodec::new()). map(|bytes| { //sigh - howto avoid copy here? or the whole map() ?? hyper::Chunk::from(bytes.to_vec()) }); let body = Body::wrap_stream(payload); - // fixme: set content type and other headers + + // fixme: set other headers ? Ok(Response::builder() .status(StatusCode::OK) + .header(header::CONTENT_TYPE, content_type) .body(body) .unwrap()) })) From fc45b741cb61723471833e6e48fbecf288cd14a8 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 4 Dec 2018 17:53:10 +0100 Subject: [PATCH 008/344] start the GUI --- src/api/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/server.rs b/src/api/server.rs index 485d1e7e..15104074 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -239,7 +239,7 @@ fn get_index() -> BoxFut { - + From 0f30b2b4c4dc64635f7183e5f81ac635b39e6546 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 5 Dec 2018 10:16:23 +0100 Subject: [PATCH 009/344] move src/api/server.rs -> src/server/rest.rs --- src/{api/server.rs => server/rest.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{api/server.rs => server/rest.rs} (100%) diff --git a/src/api/server.rs b/src/server/rest.rs similarity index 100% rename from src/api/server.rs rename to src/server/rest.rs From 53acf7490b2c5d6bc097221f7f5c0d2b4e291272 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 5 Dec 2018 12:42:25 +0100 Subject: [PATCH 010/344] add output formatter --- src/server/formatter.rs | 51 +++++++++++++++++++++++++++++++++++++++++ src/server/rest.rs | 26 +++++++++++++-------- 2 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 src/server/formatter.rs diff --git a/src/server/formatter.rs b/src/server/formatter.rs new file mode 100644 index 00000000..fa76b4c7 --- /dev/null +++ b/src/server/formatter.rs @@ -0,0 +1,51 @@ +use failure::*; +use serde_json::{json, Value}; + +pub struct OutputFormatter { + + pub format_result: fn(data: &Value) -> (Vec, &'static str), +} + +fn json_format_result(data: &Value) -> (Vec, &'static str) { + + let content_type = "application/json;charset=UTF-8"; + + let result = json!({ + "data": data + }); + + // todo: set result.total result.changes + + let json_str = result.to_string(); + + let raw = json_str.into_bytes(); + + (raw, content_type) +} + +pub static JSON_FORMATTER: OutputFormatter = OutputFormatter { + format_result: json_format_result, +}; + + +fn extjs_format_result(data: &Value) -> (Vec, &'static str) { + + let content_type = "application/json;charset=UTF-8"; + + let result = json!({ + "data": data, + "success": true + }); + + // todo: set result.total result.changes + + let json_str = result.to_string(); + + let raw = json_str.into_bytes(); + + (raw, content_type) +} + +pub static EXTJS_FORMATTER: OutputFormatter = OutputFormatter { + format_result: extjs_format_result, +}; diff --git a/src/server/rest.rs b/src/server/rest.rs index 15104074..b7b8b08c 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -1,6 +1,7 @@ use crate::api::schema::*; use crate::api::router::*; use crate::api::config::*; +use super::formatter::*; use std::fmt; use std::path::{Path, PathBuf}; @@ -66,7 +67,7 @@ impl Service for ApiService { match result { Ok(res) => Ok::<_, hyper::Error>(res), Err(err) => { - if let Some(apierr) = err.downcast_ref::() { + if let Some(apierr) = err.downcast_ref::() { let mut resp = Response::new(Body::from(apierr.message.clone())); *resp.status_mut() = apierr.code; Ok(resp) @@ -140,6 +141,7 @@ fn get_request_parameters_async( if let Some(query_str) = parts.uri.query() { for (k, v) in form_urlencoded::parse(query_str.as_bytes()).into_owned() { + if k == "_dc" { continue; } // skip extjs "disable cache" parameter param_list.push((k, v)); } } @@ -159,6 +161,7 @@ fn get_request_parameters_async( fn handle_sync_api_request( info: &'static ApiMethod, + formatter: &'static OutputFormatter, parts: Parts, req_body: Body, uri_param: HashMap, @@ -185,15 +188,16 @@ fn handle_sync_api_request( Ok(res) - }).then(|result| { + }).then(move |result| { match result { Ok(ref value) => { - let json_str = value.to_string(); + + let (raw, content_type) = (formatter.format_result)(value); Ok(Response::builder() .status(StatusCode::OK) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(json_str))?) + .header(header::CONTENT_TYPE, content_type) + .body(Body::from(raw))?) } Err(err) => Err(http_err!(BAD_REQUEST, err.to_string())) } @@ -374,15 +378,19 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { println!("GOT API REQUEST"); if comp_len >= 2 { let format = components[1]; - if format != "json" { - return Box::new(future::err(http_err!(BAD_REQUEST, format!("Unsupported output format '{}'.", format)))) - } + let formatter = match format { + "json" => &JSON_FORMATTER, + "extjs" => &EXTJS_FORMATTER, + _ => { + return Box::new(future::err(http_err!(BAD_REQUEST, format!("Unsupported output format '{}'.", format)))); + } + }; let mut uri_param = HashMap::new(); if let Some(api_method) = api.find_method(&components[2..], method, &mut uri_param) { // fixme: handle auth - return handle_sync_api_request(api_method, parts, body, uri_param); + return handle_sync_api_request(api_method, formatter, parts, body, uri_param); } } } else { From 2ad4db5d13acd2eb2039e6bf3c34658c896b38e1 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 5 Dec 2018 18:22:56 +0100 Subject: [PATCH 011/344] simplify formatter code --- src/server/formatter.rs | 74 +++++++++++++++++++++++++++++++---------- src/server/rest.rs | 28 +--------------- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/src/server/formatter.rs b/src/server/formatter.rs index fa76b4c7..decd7063 100644 --- a/src/server/formatter.rs +++ b/src/server/formatter.rs @@ -1,49 +1,89 @@ use failure::*; use serde_json::{json, Value}; +use hyper::{Body, Response, StatusCode}; +use hyper::header; + pub struct OutputFormatter { - pub format_result: fn(data: &Value) -> (Vec, &'static str), + pub format_result: fn(data: Result) -> Response, } -fn json_format_result(data: &Value) -> (Vec, &'static str) { +fn json_format_result(data: Result) -> Response { let content_type = "application/json;charset=UTF-8"; - let result = json!({ - "data": data - }); + match data { + Ok(value) => { + let result = json!({ + "data": value + }); - // todo: set result.total result.changes + // todo: set result.total result.changes - let json_str = result.to_string(); + let json_str = result.to_string(); - let raw = json_str.into_bytes(); + let raw = json_str.into_bytes(); - (raw, content_type) + let mut response = Response::new(raw.into()); + response.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static(content_type)); + return response; + } + Err(err) => { + let mut response = Response::new(Body::from(err.to_string())); + response.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static(content_type)); + *response.status_mut() = StatusCode::BAD_REQUEST; + return response; + } + } } pub static JSON_FORMATTER: OutputFormatter = OutputFormatter { format_result: json_format_result, }; - -fn extjs_format_result(data: &Value) -> (Vec, &'static str) { +fn extjs_format_result(data: Result) -> Response { let content_type = "application/json;charset=UTF-8"; - let result = json!({ - "data": data, - "success": true - }); + let result = match data { + Ok(value) => { + let result = json!({ + "data": value, + "success": true + }); - // todo: set result.total result.changes + // todo: set result.total result.changes + + result + } + Err(err) => { + let mut errors = vec![]; + + errors.push(err.to_string()); + + let result = json!({ + "errors": errors, + "success": false + }); + + result + } + }; let json_str = result.to_string(); let raw = json_str.into_bytes(); - (raw, content_type) + let mut response = Response::new(raw.into()); + response.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static(content_type)); + response } pub static EXTJS_FORMATTER: OutputFormatter = OutputFormatter { diff --git a/src/server/rest.rs b/src/server/rest.rs index b7b8b08c..0b2ff930 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -171,36 +171,10 @@ fn handle_sync_api_request( let resp = params .and_then(move |params| { - - println!("GOT PARAMS {}", params); - - /* - let when = Instant::now() + Duration::from_millis(3000); - let task = Delay::new(when).then(|_| { - println!("A LAZY TASK"); - ok(()) - }); - - tokio::spawn(task); - */ - let res = (info.handler)(params, info)?; - Ok(res) - }).then(move |result| { - match result { - Ok(ref value) => { - - let (raw, content_type) = (formatter.format_result)(value); - - Ok(Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, content_type) - .body(Body::from(raw))?) - } - Err(err) => Err(http_err!(BAD_REQUEST, err.to_string())) - } + Ok((formatter.format_result)(result)) }); Box::new(resp) From 3cd4bb8a633fc5d8742bb585477ac37e04f691ee Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Tue, 8 Jan 2019 14:21:54 +0100 Subject: [PATCH 012/344] rest: don't copy the body Signed-off-by: Wolfgang Bumiller --- src/server/rest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 0b2ff930..bd16151c 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -126,7 +126,7 @@ fn get_request_parameters_async( }) .and_then(move |body| { - let bytes = String::from_utf8(body.to_vec())?; // why copy?? + let bytes = std::str::from_utf8(&body)?; println!("GOT BODY {:?}", bytes); From ac1397dedb1efd69b6ceab230ec1e0596b328c6b Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Tue, 8 Jan 2019 14:22:43 +0100 Subject: [PATCH 013/344] rest: rename utf-8-checked 'bytes' to 'utf8' Signed-off-by: Wolfgang Bumiller --- src/server/rest.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index bd16151c..d9475d1a 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -126,14 +126,14 @@ fn get_request_parameters_async( }) .and_then(move |body| { - let bytes = std::str::from_utf8(&body)?; + let utf8 = std::str::from_utf8(&body)?; - println!("GOT BODY {:?}", bytes); + println!("GOT BODY {:?}", utf8); let mut param_list: Vec<(String, String)> = vec![]; - if bytes.len() > 0 { - for (k, v) in form_urlencoded::parse(bytes.as_bytes()).into_owned() { + if utf8.len() > 0 { + for (k, v) in form_urlencoded::parse(utf8.as_bytes()).into_owned() { param_list.push((k, v)); } From c1582dcf39e8073fd8120a9fdd211b11fca83c74 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 14 Jan 2019 12:26:04 +0100 Subject: [PATCH 014/344] api/router.rs: allow different types of api methods Added a prototype for file/backup uploads. --- src/server/rest.rs | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index d9475d1a..5d3f23c3 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -82,8 +82,6 @@ impl Service for ApiService { } } -type BoxFut = Box, Error = failure::Error> + Send>; - #[derive(Debug, Fail)] pub struct HttpError { pub code: StatusCode, @@ -180,6 +178,31 @@ fn handle_sync_api_request( Box::new(resp) } +fn handle_upload_api_request( + info: &'static ApiUploadMethod, + formatter: &'static OutputFormatter, + parts: Parts, + req_body: Body, + uri_param: HashMap, +) -> BoxFut +{ + // fixme: convert parameters to Json + let mut param_list: Vec<(String, String)> = vec![]; + + for (k, v) in uri_param { + param_list.push((k.clone(), v.clone())); + } + + let params = match parse_parameter_strings(¶m_list, &info.parameters, true) { + Ok(v) => v, + Err(err) => { + return Box::new(future::err(err.into())); + } + }; + + (info.handler)(req_body, params, info) +} + fn get_index() -> BoxFut { let nodename = "unknown"; @@ -362,9 +385,15 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { let mut uri_param = HashMap::new(); - if let Some(api_method) = api.find_method(&components[2..], method, &mut uri_param) { - // fixme: handle auth - return handle_sync_api_request(api_method, formatter, parts, body, uri_param); + // fixme: handle auth + match api.find_method(&components[2..], method, &mut uri_param) { + MethodDefinition::None => {} + MethodDefinition::Simple(api_method) => { + return handle_sync_api_request(api_method, formatter, parts, body, uri_param); + } + MethodDefinition::Upload(upload_method) => { + return handle_upload_api_request(upload_method, formatter, parts, body, uri_param); + } } } } else { From c36fa6128790b0bff966ff069e218b3e1314fdc1 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 15 Jan 2019 11:38:26 +0100 Subject: [PATCH 015/344] api3/admin/datastore/upload_catar.rs: implement upload future --- src/server/rest.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 5d3f23c3..d119658d 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -180,8 +180,8 @@ fn handle_sync_api_request( fn handle_upload_api_request( info: &'static ApiUploadMethod, - formatter: &'static OutputFormatter, - parts: Parts, + _formatter: &'static OutputFormatter, + _parts: Parts, req_body: Body, uri_param: HashMap, ) -> BoxFut From 148b327e63ddedc11d888d7f6944e80d5763b190 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 16 Jan 2019 13:58:36 +0100 Subject: [PATCH 016/344] server/rest.rs: correctly pass query/url parameters --- src/server/rest.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index d119658d..4ca98815 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -180,8 +180,8 @@ fn handle_sync_api_request( fn handle_upload_api_request( info: &'static ApiUploadMethod, - _formatter: &'static OutputFormatter, - _parts: Parts, + formatter: &'static OutputFormatter, + parts: Parts, req_body: Body, uri_param: HashMap, ) -> BoxFut @@ -189,6 +189,13 @@ fn handle_upload_api_request( // fixme: convert parameters to Json let mut param_list: Vec<(String, String)> = vec![]; + if let Some(query_str) = parts.uri.query() { + for (k, v) in form_urlencoded::parse(query_str.as_bytes()).into_owned() { + if k == "_dc" { continue; } // skip extjs "disable cache" parameter + param_list.push((k, v)); + } + } + for (k, v) in uri_param { param_list.push((k.clone(), v.clone())); } @@ -196,7 +203,8 @@ fn handle_upload_api_request( let params = match parse_parameter_strings(¶m_list, &info.parameters, true) { Ok(v) => v, Err(err) => { - return Box::new(future::err(err.into())); + let resp = (formatter.format_result)(Err(Error::from(err))); + return Box::new(future::ok(resp)); } }; From 90e1d858e035f9d512a227afe2ff85cca93646ed Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 17 Jan 2019 12:03:38 +0100 Subject: [PATCH 017/344] api/router.rs: return Result in upload handler --- src/server/rest.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 4ca98815..49f3f676 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -208,7 +208,13 @@ fn handle_upload_api_request( } }; - (info.handler)(req_body, params, info) + match (info.handler)(req_body, params, info) { + Ok(future) => future, + Err(err) => { + let resp = (formatter.format_result)(Err(Error::from(err))); + Box::new(future::ok(resp)) + } + } } fn get_index() -> BoxFut { From 6e219aefd318dced6aebbec6bfb463ade4a53184 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 17 Jan 2019 12:43:29 +0100 Subject: [PATCH 018/344] api3/admin/datastore/upload_catar.rs: verify content type ("application/x-proxmox-backup-catar") --- src/server/rest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 49f3f676..1bd724eb 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -208,7 +208,7 @@ fn handle_upload_api_request( } }; - match (info.handler)(req_body, params, info) { + match (info.handler)(parts, req_body, params, info) { Ok(future) => future, Err(err) => { let resp = (formatter.format_result)(Err(Error::from(err))); From 85722a84927e227dabcf5b28b13e8f284abbcdb9 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 19 Jan 2019 16:42:43 +0100 Subject: [PATCH 019/344] api/router.rs: rename ApiUploadMethod to ApiAsyncMethod We can use this for uploads and downloads ... --- src/server/rest.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 1bd724eb..74cf1049 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -178,8 +178,8 @@ fn handle_sync_api_request( Box::new(resp) } -fn handle_upload_api_request( - info: &'static ApiUploadMethod, +fn handle_async_api_request( + info: &'static ApiAsyncMethod, formatter: &'static OutputFormatter, parts: Parts, req_body: Body, @@ -405,8 +405,8 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { MethodDefinition::Simple(api_method) => { return handle_sync_api_request(api_method, formatter, parts, body, uri_param); } - MethodDefinition::Upload(upload_method) => { - return handle_upload_api_request(upload_method, formatter, parts, body, uri_param); + MethodDefinition::Async(async_method) => { + return handle_async_api_request(async_method, formatter, parts, body, uri_param); } } } From e35404deb745a975e95e80f990fc9087cc4b7ee6 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 20 Jan 2019 14:28:06 +0100 Subject: [PATCH 020/344] remove crate tokio-codec (seems to be part of tokio now) --- src/server/rest.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 74cf1049..c4c78880 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -16,7 +16,6 @@ use futures::future::{self, Either}; //use tokio::prelude::*; //use tokio::timer::Delay; use tokio::fs::File; -use tokio_codec; //use bytes::{BytesMut, BufMut}; //use hyper::body::Payload; @@ -324,7 +323,7 @@ fn chuncked_static_file_download(filename: PathBuf) -> BoxFut { Box::new(File::open(filename) .map_err(|err| http_err!(BAD_REQUEST, format!("File open failed: {}", err))) .and_then(move |file| { - let payload = tokio_codec::FramedRead::new(file, tokio_codec::BytesCodec::new()). + let payload = tokio::codec::FramedRead::new(file, tokio::codec::BytesCodec::new()). map(|bytes| { //sigh - howto avoid copy here? or the whole map() ?? hyper::Chunk::from(bytes.to_vec()) From c6430658645a6a398ca47114c046aa6c5af69af0 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 22 Jan 2019 12:10:38 +0100 Subject: [PATCH 021/344] rename api3 back to api2 There is no real need to change the path, so using api2 we can reuse all helpers (like tools from proxmox widget toolkit). --- src/server/rest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index c4c78880..c2b175c2 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -384,7 +384,7 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { println!("REQUEST {} {}", method, path); println!("COMPO {:?}", components); - if comp_len >= 1 && components[0] == "api3" { + if comp_len >= 1 && components[0] == "api2" { println!("GOT API REQUEST"); if comp_len >= 2 { let format = components[1]; From b1be01218acf051c37eea2643b87abda829a0431 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 23 Jan 2019 12:49:10 +0100 Subject: [PATCH 022/344] server/rest.rs: fake login cookie --- src/server/rest.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index c2b175c2..97e80e10 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -1,3 +1,4 @@ +use crate::tools; use crate::api::schema::*; use crate::api::router::*; use crate::api::config::*; @@ -218,15 +219,15 @@ fn handle_async_api_request( fn get_index() -> BoxFut { - let nodename = "unknown"; - let username = ""; + let nodename = tools::nodename(); + let username = "fakelogin"; // todo: implement real auth let token = "abc"; let setup = json!({ "Setup": { "auth_cookie_name": "PBSAuthCookie" }, "NodeName": nodename, "UserName": username, - "CSRFPreventionToken": token + "CSRFPreventionToken": token, }); let index = format!(r###" @@ -264,7 +265,15 @@ fn get_index() -> BoxFut { "###, setup.to_string()); - Box::new(future::ok(Response::new(index.into()))) + let resp = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html") + // emulate succssful login, so that Proxmox:Utils.authOk() returns true + .header(header::SET_COOKIE, "PBSAuthCookie=\"XXX\"") // fixme: remove + .body(index.into()) + .unwrap(); + + Box::new(future::ok(resp)) } fn extension_to_content_type(filename: &Path) -> (&'static str, bool) { From 32f3db27bd2f5dbad02fba8a2b9f6c643769be18 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 26 Jan 2019 14:50:37 +0100 Subject: [PATCH 023/344] api: pass RpcEnvirnment to api handlers --- src/server/formatter.rs | 140 +++++++++++++++++++++------------------- src/server/rest.rs | 35 ++++++++-- 2 files changed, 104 insertions(+), 71 deletions(-) diff --git a/src/server/formatter.rs b/src/server/formatter.rs index decd7063..526c8e07 100644 --- a/src/server/formatter.rs +++ b/src/server/formatter.rs @@ -1,79 +1,22 @@ use failure::*; use serde_json::{json, Value}; +use crate::api::router::RpcEnvironment; + use hyper::{Body, Response, StatusCode}; use hyper::header; pub struct OutputFormatter { - pub format_result: fn(data: Result) -> Response, + pub format_result: fn(data: Value, rpcenv: &RpcEnvironment) -> Response, + + pub format_error: fn(err: Error) -> Response, } -fn json_format_result(data: Result) -> Response { +static json_content_type: &str = "application/json;charset=UTF-8"; - let content_type = "application/json;charset=UTF-8"; - match data { - Ok(value) => { - let result = json!({ - "data": value - }); - - // todo: set result.total result.changes - - let json_str = result.to_string(); - - let raw = json_str.into_bytes(); - - let mut response = Response::new(raw.into()); - response.headers_mut().insert( - header::CONTENT_TYPE, - header::HeaderValue::from_static(content_type)); - return response; - } - Err(err) => { - let mut response = Response::new(Body::from(err.to_string())); - response.headers_mut().insert( - header::CONTENT_TYPE, - header::HeaderValue::from_static(content_type)); - *response.status_mut() = StatusCode::BAD_REQUEST; - return response; - } - } -} - -pub static JSON_FORMATTER: OutputFormatter = OutputFormatter { - format_result: json_format_result, -}; - -fn extjs_format_result(data: Result) -> Response { - - let content_type = "application/json;charset=UTF-8"; - - let result = match data { - Ok(value) => { - let result = json!({ - "data": value, - "success": true - }); - - // todo: set result.total result.changes - - result - } - Err(err) => { - let mut errors = vec![]; - - errors.push(err.to_string()); - - let result = json!({ - "errors": errors, - "success": false - }); - - result - } - }; +fn json_response(result: Value) -> Response { let json_str = result.to_string(); @@ -82,10 +25,77 @@ fn extjs_format_result(data: Result) -> Response { let mut response = Response::new(raw.into()); response.headers_mut().insert( header::CONTENT_TYPE, - header::HeaderValue::from_static(content_type)); + header::HeaderValue::from_static(json_content_type)); + response } +fn json_format_result(data: Value, rpcenv: &RpcEnvironment) -> Response { + + let mut result = json!({ + "data": data + }); + + if let Some(total) = rpcenv.get_result_attrib("total").and_then(|v| v.as_u64()) { + result["total"] = Value::from(total); + } + + if let Some(changes) = rpcenv.get_result_attrib("changes") { + result["changes"] = changes.clone(); + } + + json_response(result) +} + +fn json_format_error(err: Error) -> Response { + + let mut response = Response::new(Body::from(err.to_string())); + response.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static(json_content_type)); + *response.status_mut() = StatusCode::BAD_REQUEST; + + response +} + +pub static JSON_FORMATTER: OutputFormatter = OutputFormatter { + format_result: json_format_result, + format_error: json_format_error, +}; + +fn extjs_format_result(data: Value, rpcenv: &RpcEnvironment) -> Response { + + let mut result = json!({ + "data": data, + "success": true + }); + + if let Some(total) = rpcenv.get_result_attrib("total").and_then(|v| v.as_u64()) { + result["total"] = Value::from(total); + } + + if let Some(changes) = rpcenv.get_result_attrib("changes") { + result["changes"] = changes.clone(); + } + + + json_response(result) +} + +fn extjs_format_error(err: Error) -> Response { + + let mut errors = vec![]; + errors.push(err.to_string()); + + let result = json!({ + "errors": errors, + "success": false + }); + + json_response(result) +} + pub static EXTJS_FORMATTER: OutputFormatter = OutputFormatter { format_result: extjs_format_result, + format_error: extjs_format_error, }; diff --git a/src/server/rest.rs b/src/server/rest.rs index 97e80e10..3812ffb6 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -26,6 +26,27 @@ use hyper::service::{Service, NewService}; use hyper::rt::{Future, Stream}; use hyper::header; +struct RestEnvironment { + result_attributes: HashMap, +} + +impl RestEnvironment { + fn new() -> Self { + Self { result_attributes: HashMap::new() } + } +} + +impl RpcEnvironment for RestEnvironment { + + fn set_result_attrib(&mut self, name: &str, value: Value) { + self.result_attributes.insert(name.into(), value); + } + + fn get_result_attrib(&self, name: &str) -> Option<&Value> { + self.result_attributes.get(name) + } +} + pub struct RestServer { pub api_config: Arc, } @@ -169,10 +190,12 @@ fn handle_sync_api_request( let resp = params .and_then(move |params| { - let res = (info.handler)(params, info)?; - Ok(res) - }).then(move |result| { - Ok((formatter.format_result)(result)) + let mut rpcenv = RestEnvironment::new(); + let resp = match (info.handler)(params, info, &mut rpcenv) { + Ok(data) => (formatter.format_result)(data, &rpcenv), + Err(err) => (formatter.format_error)(err), + }; + Ok(resp) }); Box::new(resp) @@ -203,7 +226,7 @@ fn handle_async_api_request( let params = match parse_parameter_strings(¶m_list, &info.parameters, true) { Ok(v) => v, Err(err) => { - let resp = (formatter.format_result)(Err(Error::from(err))); + let resp = (formatter.format_error)(Error::from(err)); return Box::new(future::ok(resp)); } }; @@ -211,7 +234,7 @@ fn handle_async_api_request( match (info.handler)(parts, req_body, params, info) { Ok(future) => future, Err(err) => { - let resp = (formatter.format_result)(Err(Error::from(err))); + let resp = (formatter.format_error)(Error::from(err)); Box::new(future::ok(resp)) } } From a0a545c72046ac8a70fc42e81804f528ecde4890 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 26 Jan 2019 15:08:02 +0100 Subject: [PATCH 024/344] move rpc environment implementation to separate files --- src/server/environment.rs | 25 +++++++++++++++++++++++++ src/server/formatter.rs | 6 +++--- src/server/rest.rs | 22 +--------------------- 3 files changed, 29 insertions(+), 24 deletions(-) create mode 100644 src/server/environment.rs diff --git a/src/server/environment.rs b/src/server/environment.rs new file mode 100644 index 00000000..18346be4 --- /dev/null +++ b/src/server/environment.rs @@ -0,0 +1,25 @@ +use crate::api::router::*; + +use std::collections::HashMap; +use serde_json::Value; + +pub struct RestEnvironment { + result_attributes: HashMap, +} + +impl RestEnvironment { + pub fn new() -> Self { + Self { result_attributes: HashMap::new() } + } +} + +impl RpcEnvironment for RestEnvironment { + + fn set_result_attrib(&mut self, name: &str, value: Value) { + self.result_attributes.insert(name.into(), value); + } + + fn get_result_attrib(&self, name: &str) -> Option<&Value> { + self.result_attributes.get(name) + } +} diff --git a/src/server/formatter.rs b/src/server/formatter.rs index 526c8e07..8314d7a8 100644 --- a/src/server/formatter.rs +++ b/src/server/formatter.rs @@ -13,7 +13,7 @@ pub struct OutputFormatter { pub format_error: fn(err: Error) -> Response, } -static json_content_type: &str = "application/json;charset=UTF-8"; +static JSON_CONTENT_TYPE: &str = "application/json;charset=UTF-8"; fn json_response(result: Value) -> Response { @@ -25,7 +25,7 @@ fn json_response(result: Value) -> Response { let mut response = Response::new(raw.into()); response.headers_mut().insert( header::CONTENT_TYPE, - header::HeaderValue::from_static(json_content_type)); + header::HeaderValue::from_static(JSON_CONTENT_TYPE)); response } @@ -52,7 +52,7 @@ fn json_format_error(err: Error) -> Response { let mut response = Response::new(Body::from(err.to_string())); response.headers_mut().insert( header::CONTENT_TYPE, - header::HeaderValue::from_static(json_content_type)); + header::HeaderValue::from_static(JSON_CONTENT_TYPE)); *response.status_mut() = StatusCode::BAD_REQUEST; response diff --git a/src/server/rest.rs b/src/server/rest.rs index 3812ffb6..180455d3 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -2,6 +2,7 @@ use crate::tools; use crate::api::schema::*; use crate::api::router::*; use crate::api::config::*; +use super::environment::RestEnvironment; use super::formatter::*; use std::fmt; @@ -26,27 +27,6 @@ use hyper::service::{Service, NewService}; use hyper::rt::{Future, Stream}; use hyper::header; -struct RestEnvironment { - result_attributes: HashMap, -} - -impl RestEnvironment { - fn new() -> Self { - Self { result_attributes: HashMap::new() } - } -} - -impl RpcEnvironment for RestEnvironment { - - fn set_result_attrib(&mut self, name: &str, value: Value) { - self.result_attributes.insert(name.into(), value); - } - - fn get_result_attrib(&self, name: &str) -> Option<&Value> { - self.result_attributes.get(name) - } -} - pub struct RestServer { pub api_config: Arc, } From 084ccdd5906d0a58ea15a73f382226e431a653cb Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 27 Jan 2019 10:18:52 +0100 Subject: [PATCH 025/344] also pass rpcenv to async handlers --- src/server/rest.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 180455d3..8dc80d85 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -159,6 +159,7 @@ fn get_request_parameters_async( } fn handle_sync_api_request( + mut rpcenv: RestEnvironment, info: &'static ApiMethod, formatter: &'static OutputFormatter, parts: Parts, @@ -170,7 +171,6 @@ fn handle_sync_api_request( let resp = params .and_then(move |params| { - let mut rpcenv = RestEnvironment::new(); let resp = match (info.handler)(params, info, &mut rpcenv) { Ok(data) => (formatter.format_result)(data, &rpcenv), Err(err) => (formatter.format_error)(err), @@ -182,6 +182,7 @@ fn handle_sync_api_request( } fn handle_async_api_request( + mut rpcenv: RestEnvironment, info: &'static ApiAsyncMethod, formatter: &'static OutputFormatter, parts: Parts, @@ -211,7 +212,7 @@ fn handle_async_api_request( } }; - match (info.handler)(parts, req_body, params, info) { + match (info.handler)(parts, req_body, params, info, &mut rpcenv) { Ok(future) => future, Err(err) => { let resp = (formatter.format_error)(Error::from(err)); @@ -396,6 +397,8 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { println!("REQUEST {} {}", method, path); println!("COMPO {:?}", components); + let mut rpcenv = RestEnvironment::new(); + if comp_len >= 1 && components[0] == "api2" { println!("GOT API REQUEST"); if comp_len >= 2 { @@ -414,10 +417,10 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { match api.find_method(&components[2..], method, &mut uri_param) { MethodDefinition::None => {} MethodDefinition::Simple(api_method) => { - return handle_sync_api_request(api_method, formatter, parts, body, uri_param); + return handle_sync_api_request(rpcenv, api_method, formatter, parts, body, uri_param); } MethodDefinition::Async(async_method) => { - return handle_async_api_request(async_method, formatter, parts, body, uri_param); + return handle_async_api_request(rpcenv, async_method, formatter, parts, body, uri_param); } } } From 23db39488f45e714d51475673cb4f05e9cc68c96 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 27 Jan 2019 10:33:42 +0100 Subject: [PATCH 026/344] RpcEnvironment: add environment type enum RpcEnvironmentType --- src/server/environment.rs | 12 ++++++++++-- src/server/rest.rs | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/server/environment.rs b/src/server/environment.rs index 18346be4..7345cc8d 100644 --- a/src/server/environment.rs +++ b/src/server/environment.rs @@ -4,12 +4,16 @@ use std::collections::HashMap; use serde_json::Value; pub struct RestEnvironment { + env_type: RpcEnvironmentType, result_attributes: HashMap, } impl RestEnvironment { - pub fn new() -> Self { - Self { result_attributes: HashMap::new() } + pub fn new(env_type: RpcEnvironmentType) -> Self { + Self { + result_attributes: HashMap::new(), + env_type, + } } } @@ -22,4 +26,8 @@ impl RpcEnvironment for RestEnvironment { fn get_result_attrib(&self, name: &str) -> Option<&Value> { self.result_attributes.get(name) } + + fn env_type(&self) -> RpcEnvironmentType { + self.env_type + } } diff --git a/src/server/rest.rs b/src/server/rest.rs index 8dc80d85..eb938c53 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -397,7 +397,7 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { println!("REQUEST {} {}", method, path); println!("COMPO {:?}", components); - let mut rpcenv = RestEnvironment::new(); + let rpcenv = RestEnvironment::new(RpcEnvironmentType::PRIVILEDGED); if comp_len >= 1 && components[0] == "api2" { println!("GOT API REQUEST"); From 42e06fc5ca6f5b33171b33ece3d09402e24c2e0c Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 27 Jan 2019 10:42:45 +0100 Subject: [PATCH 027/344] RpcEnvironment: implement set_user() and get_user() --- src/server/environment.rs | 10 ++++++++++ src/server/rest.rs | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/server/environment.rs b/src/server/environment.rs index 7345cc8d..2d6103f8 100644 --- a/src/server/environment.rs +++ b/src/server/environment.rs @@ -6,12 +6,14 @@ use serde_json::Value; pub struct RestEnvironment { env_type: RpcEnvironmentType, result_attributes: HashMap, + user: Option, } impl RestEnvironment { pub fn new(env_type: RpcEnvironmentType) -> Self { Self { result_attributes: HashMap::new(), + user: None, env_type, } } @@ -30,4 +32,12 @@ impl RpcEnvironment for RestEnvironment { fn env_type(&self) -> RpcEnvironmentType { self.env_type } + + fn set_user(&mut self, user: Option) { + self.user = user; + } + + fn get_user(&self) -> Option { + self.user.clone() + } } diff --git a/src/server/rest.rs b/src/server/rest.rs index eb938c53..ccadae47 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -397,7 +397,7 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { println!("REQUEST {} {}", method, path); println!("COMPO {:?}", components); - let rpcenv = RestEnvironment::new(RpcEnvironmentType::PRIVILEDGED); + let mut rpcenv = RestEnvironment::new(RpcEnvironmentType::PRIVILEDGED); if comp_len >= 1 && components[0] == "api2" { println!("GOT API REQUEST"); @@ -414,6 +414,8 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { let mut uri_param = HashMap::new(); // fixme: handle auth + rpcenv.set_user(Some(String::from("root@pam"))); + match api.find_method(&components[2..], method, &mut uri_param) { MethodDefinition::None => {} MethodDefinition::Simple(api_method) => { From 08e45e35733ddd67d5b6d5064de28d8a50a2090e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 28 Jan 2019 13:17:03 +0100 Subject: [PATCH 028/344] src/bin/proxmox-backup-proxy.rs: implement unpriviledged server We want to run the public server as user www-data. Requests needing root priviledges needs to be proxied to the proxmox-backup.service, which now listens to 127.0.0.1:82. --- src/server/rest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index ccadae47..d585cd03 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -397,7 +397,7 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { println!("REQUEST {} {}", method, path); println!("COMPO {:?}", components); - let mut rpcenv = RestEnvironment::new(RpcEnvironmentType::PRIVILEDGED); + let mut rpcenv = RestEnvironment::new(api.env_type()); if comp_len >= 1 && components[0] == "api2" { println!("GOT API REQUEST"); From 4e5a5728cbe951eabe4fbf6da5d00d9c8a35d85f Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 28 Jan 2019 13:44:48 +0100 Subject: [PATCH 029/344] server/formatter.rs: fix extjs error format --- src/server/formatter.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server/formatter.rs b/src/server/formatter.rs index 8314d7a8..eb3c33d1 100644 --- a/src/server/formatter.rs +++ b/src/server/formatter.rs @@ -85,9 +85,12 @@ fn extjs_format_result(data: Value, rpcenv: &RpcEnvironment) -> Response { fn extjs_format_error(err: Error) -> Response { let mut errors = vec![]; - errors.push(err.to_string()); + + let message = err.to_string(); + errors.push(&message); let result = json!({ + "message": message, "errors": errors, "success": false }); From 1aa3b197a691e5b0d6912e25624132be9bb94f73 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 28 Jan 2019 17:30:39 +0100 Subject: [PATCH 030/344] server/rest.rs: add proxy_sync_api_request() dummy --- src/server/rest.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index d585cd03..97ebd078 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -158,6 +158,19 @@ fn get_request_parameters_async( Box::new(resp) } +fn proxy_sync_api_request( + mut rpcenv: RestEnvironment, + info: &'static ApiMethod, + formatter: &'static OutputFormatter, + parts: Parts, + req_body: Body, + uri_param: HashMap, +) -> BoxFut +{ + + return Box::new(future::err(http_err!(BAD_REQUEST, String::from("implement proxy")))); +} + fn handle_sync_api_request( mut rpcenv: RestEnvironment, info: &'static ApiMethod, @@ -397,7 +410,8 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { println!("REQUEST {} {}", method, path); println!("COMPO {:?}", components); - let mut rpcenv = RestEnvironment::new(api.env_type()); + let env_type = api.env_type(); + let mut rpcenv = RestEnvironment::new(env_type); if comp_len >= 1 && components[0] == "api2" { println!("GOT API REQUEST"); @@ -419,7 +433,11 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { match api.find_method(&components[2..], method, &mut uri_param) { MethodDefinition::None => {} MethodDefinition::Simple(api_method) => { - return handle_sync_api_request(rpcenv, api_method, formatter, parts, body, uri_param); + if api_method.protected && env_type == RpcEnvironmentType::PUBLIC { + return proxy_sync_api_request(rpcenv, api_method, formatter, parts, body, uri_param); + } else { + return handle_sync_api_request(rpcenv, api_method, formatter, parts, body, uri_param); + } } MethodDefinition::Async(async_method) => { return handle_async_api_request(rpcenv, async_method, formatter, parts, body, uri_param); From 8ec1299ab3ab67a424aaad96b3b3da93a66c9d8d Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 28 Jan 2019 18:06:42 +0100 Subject: [PATCH 031/344] server/rest.rs: implement proxy_sync_api_request --- src/server/rest.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 97ebd078..e1d18954 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -159,16 +159,30 @@ fn get_request_parameters_async( } fn proxy_sync_api_request( - mut rpcenv: RestEnvironment, + rpcenv: RestEnvironment, info: &'static ApiMethod, formatter: &'static OutputFormatter, - parts: Parts, + mut parts: Parts, req_body: Body, uri_param: HashMap, ) -> BoxFut { - return Box::new(future::err(http_err!(BAD_REQUEST, String::from("implement proxy")))); + let mut uri_parts = parts.uri.clone().into_parts(); + + uri_parts.scheme = Some(http::uri::Scheme::HTTP); + uri_parts.authority = Some(http::uri::Authority::from_static("127.0.0.1:82")); + let new_uri = http::Uri::from_parts(uri_parts).unwrap(); + + parts.uri = new_uri; + + let request = Request::from_parts(parts, req_body); + + let resp = hyper::client::Client::new() + .request(request) + .map_err(|e| Error::from(e)); + + return Box::new(resp); } fn handle_sync_api_request( From c4f2b212c5fcac90ff2cfb7e98db66ae4bd5b046 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 28 Jan 2019 18:22:16 +0100 Subject: [PATCH 032/344] server/rest.rs: simplify proxy code Only pass neccessary parameters. --- src/server/rest.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index e1d18954..6df857eb 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -158,13 +158,9 @@ fn get_request_parameters_async( Box::new(resp) } -fn proxy_sync_api_request( - rpcenv: RestEnvironment, - info: &'static ApiMethod, - formatter: &'static OutputFormatter, +fn proxy_protected_request( mut parts: Parts, req_body: Body, - uri_param: HashMap, ) -> BoxFut { @@ -448,7 +444,7 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { MethodDefinition::None => {} MethodDefinition::Simple(api_method) => { if api_method.protected && env_type == RpcEnvironmentType::PUBLIC { - return proxy_sync_api_request(rpcenv, api_method, formatter, parts, body, uri_param); + return proxy_protected_request(parts, body); } else { return handle_sync_api_request(rpcenv, api_method, formatter, parts, body, uri_param); } From 1701fd9bd4d1a3bd544137207a925d2e589a59cc Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 30 Jan 2019 15:14:20 +0100 Subject: [PATCH 033/344] api2/access.rs: add ticket api --- src/server/rest.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 6df857eb..1a7831cc 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -247,8 +247,8 @@ fn handle_async_api_request( fn get_index() -> BoxFut { let nodename = tools::nodename(); - let username = "fakelogin"; // todo: implement real auth - let token = "abc"; + let username = ""; // fixme: implement real auth + let token = ""; let setup = json!({ "Setup": { "auth_cookie_name": "PBSAuthCookie" }, From 200b5b87ea7ee66d2cb4307f07ce313c9bfaad29 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 31 Jan 2019 10:08:08 +0100 Subject: [PATCH 034/344] Utils.js: fix cookie handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use unsecure cookie foör testing. --- src/server/rest.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 1a7831cc..d9263778 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -295,8 +295,6 @@ fn get_index() -> BoxFut { let resp = Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "text/html") - // emulate succssful login, so that Proxmox:Utils.authOk() returns true - .header(header::SET_COOKIE, "PBSAuthCookie=\"XXX\"") // fixme: remove .body(index.into()) .unwrap(); From 0ef7c190e1697e782cad270b06b5c9755df03443 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 31 Jan 2019 12:22:00 +0100 Subject: [PATCH 035/344] server/rest.rs: verify auth cookie --- src/server/rest.rs | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index d9263778..787bf20f 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -2,6 +2,7 @@ use crate::tools; use crate::api::schema::*; use crate::api::router::*; use crate::api::config::*; +use crate::auth_helpers::*; use super::environment::RestEnvironment; use super::formatter::*; @@ -421,6 +422,20 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { let env_type = api.env_type(); let mut rpcenv = RestEnvironment::new(env_type); + let delay_unauth_time = std::time::Instant::now() + std::time::Duration::from_millis(3000); + + if let Some(raw_cookie) = parts.headers.get("COOKIE") { + if let Ok(cookie) = raw_cookie.to_str() { + if let Some(ticket) = tools::extract_auth_cookie(cookie, "PBSAuthCookie") { + if let Ok((_, Some(username))) = tools::ticket::verify_rsa_ticket( + public_auth_key(), "PBS", &ticket, None, -300, 3600*2) { + rpcenv.set_user(Some(username)); + } + } + } + } + + if comp_len >= 1 && components[0] == "api2" { println!("GOT API REQUEST"); if comp_len >= 2 { @@ -435,8 +450,21 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { let mut uri_param = HashMap::new(); - // fixme: handle auth - rpcenv.set_user(Some(String::from("root@pam"))); + if comp_len == 4 && components[2] == "access" && components[3] == "ticket" { + // explicitly allow those calls without auth + } else { + if let Some(_username) = rpcenv.get_user() { + // fixme: check permissions + } else { + // always delay unauthorized calls by 3 seconds (from start of request) + let resp = (formatter.format_error)(http_err!(UNAUTHORIZED, "permission check failed.".into())); + let delayed_response = tokio::timer::Delay::new(delay_unauth_time) + .map_err(|err| http_err!(INTERNAL_SERVER_ERROR, format!("tokio timer delay error: {}", err))) + .and_then(|_| Ok(resp)); + + return Box::new(delayed_response); + } + } match api.find_method(&components[2..], method, &mut uri_param) { MethodDefinition::None => {} From 8f75d998be9e470689487d2d58371da6b5fcd0d7 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 31 Jan 2019 13:22:30 +0100 Subject: [PATCH 036/344] move http error class to router.rs --- src/server/rest.rs | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 787bf20f..12010e46 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -6,7 +6,6 @@ use crate::auth_helpers::*; use super::environment::RestEnvironment; use super::formatter::*; -use std::fmt; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::collections::HashMap; @@ -84,30 +83,6 @@ impl Service for ApiService { } } -#[derive(Debug, Fail)] -pub struct HttpError { - pub code: StatusCode, - pub message: String, -} - -impl HttpError { - pub fn new(code: StatusCode, message: String) -> Self { - HttpError { code, message } - } -} - -impl fmt::Display for HttpError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Error {}: {}", self.code, self.message) - } -} - -macro_rules! http_err { - ($status:ident, $msg:expr) => {{ - Error::from(HttpError::new(StatusCode::$status, $msg)) - }} -} - fn get_request_parameters_async( info: &'static ApiMethod, parts: Parts, From 5d6350978769a8924a1a5619397ae3b4d1338ce4 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 31 Jan 2019 14:34:21 +0100 Subject: [PATCH 037/344] delay unauthorized request (rate limit) --- src/server/rest.rs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 12010e46..49fc0280 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -168,13 +168,30 @@ fn handle_sync_api_request( { let params = get_request_parameters_async(info, parts, req_body, uri_param); + let delay_unauth_time = std::time::Instant::now() + std::time::Duration::from_millis(3000); + let resp = params .and_then(move |params| { + let mut delay = false; let resp = match (info.handler)(params, info, &mut rpcenv) { Ok(data) => (formatter.format_result)(data, &rpcenv), - Err(err) => (formatter.format_error)(err), + Err(err) => { + if let Some(httperr) = err.downcast_ref::() { + if httperr.code == StatusCode::UNAUTHORIZED { delay = true; } + } + (formatter.format_error)(err) + } }; - Ok(resp) + + if delay { + let delayed_response = tokio::timer::Delay::new(delay_unauth_time) + .map_err(|err| http_err!(INTERNAL_SERVER_ERROR, format!("tokio timer delay error: {}", err))) + .and_then(|_| Ok(resp)); + + Either::A(delayed_response) + } else { + Either::B(future::ok(resp)) + } }); Box::new(resp) From 9707fdadd74eb8e6b878d99ab08f683de7359b09 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 1 Feb 2019 09:54:56 +0100 Subject: [PATCH 038/344] implement relead_timezone flag --- src/server/rest.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 49fc0280..a22b23d3 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -27,6 +27,8 @@ use hyper::service::{Service, NewService}; use hyper::rt::{Future, Stream}; use hyper::header; +extern "C" { fn tzset(); } + pub struct RestServer { pub api_config: Arc, } @@ -135,6 +137,7 @@ fn get_request_parameters_async( } fn proxy_protected_request( + info: &'static ApiMethod, mut parts: Parts, req_body: Body, ) -> BoxFut @@ -154,6 +157,12 @@ fn proxy_protected_request( .request(request) .map_err(|e| Error::from(e)); + let resp = if info.reload_timezone { + Either::A(resp.then(|resp| {unsafe { tzset() }; resp })) + } else { + Either::B(resp) + }; + return Box::new(resp); } @@ -183,6 +192,10 @@ fn handle_sync_api_request( } }; + if info.reload_timezone { + unsafe { tzset() }; + } + if delay { let delayed_response = tokio::timer::Delay::new(delay_unauth_time) .map_err(|err| http_err!(INTERNAL_SERVER_ERROR, format!("tokio timer delay error: {}", err))) @@ -462,7 +475,7 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { MethodDefinition::None => {} MethodDefinition::Simple(api_method) => { if api_method.protected && env_type == RpcEnvironmentType::PUBLIC { - return proxy_protected_request(parts, body); + return proxy_protected_request(api_method, parts, body); } else { return handle_sync_api_request(rpcenv, api_method, formatter, parts, body, uri_param); } From fe3b25029b6d023969ef35332596e46def300dd4 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Mon, 4 Feb 2019 15:34:38 +0100 Subject: [PATCH 039/344] remove some rather inconvenient debug output Signed-off-by: Wolfgang Bumiller --- src/server/rest.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index a22b23d3..1c63c3e9 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -105,8 +105,6 @@ fn get_request_parameters_async( let utf8 = std::str::from_utf8(&body)?; - println!("GOT BODY {:?}", utf8); - let mut param_list: Vec<(String, String)> = vec![]; if utf8.len() > 0 { @@ -129,7 +127,6 @@ fn get_request_parameters_async( let params = parse_parameter_strings(¶m_list, &info.parameters, true)?; - println!("GOT PARAMS {}", params); Ok(params) }); From 50ff21da59fe04563abc75bd65279c586b6eefea Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 13 Feb 2019 14:31:43 +0100 Subject: [PATCH 040/344] src/client/http_client.rs: try to login use an environment var to store passphrase (PBS_PASSWORD) --- src/server/rest.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 1c63c3e9..49e4a8b4 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -70,7 +70,7 @@ impl Service for ApiService { match result { Ok(res) => Ok::<_, hyper::Error>(res), Err(err) => { - if let Some(apierr) = err.downcast_ref::() { + if let Some(apierr) = err.downcast_ref::() { let mut resp = Response::new(Body::from(apierr.message.clone())); *resp.status_mut() = apierr.code; Ok(resp) @@ -458,6 +458,8 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { if let Some(_username) = rpcenv.get_user() { // fixme: check permissions } else { + println!("Abort UNAUTHORIZED API REQUEST"); + // always delay unauthorized calls by 3 seconds (from start of request) let resp = (formatter.format_error)(http_err!(UNAUTHORIZED, "permission check failed.".into())); let delayed_response = tokio::timer::Delay::new(delay_unauth_time) From e683d9ccb79b575ed29e34e06192bede5f7e911f Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Feb 2019 13:07:34 +0100 Subject: [PATCH 041/344] src/server/rest.rs: log failed requests --- src/server/rest.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 49e4a8b4..e94b7f4a 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -57,6 +57,18 @@ pub struct ApiService { pub api_config: Arc, } +impl ApiService { + fn log_response(path: &str, resp: &Response) { + let status = resp.status(); + if !status.is_success() { + let reason = status.canonical_reason().unwrap_or("unknown reason"); + let client = "unknown"; // fixme: howto get peer_addr ? + let message = "request failed"; + + log::error!("{}: {} {}: [client {}] {}", path, status.as_str(), reason, client, message); + } + } +} impl Service for ApiService { type ReqBody = Body; @@ -65,18 +77,23 @@ impl Service for ApiService { type Future = Box, Error = Self::Error> + Send>; fn call(&mut self, req: Request) -> Self::Future { - - Box::new(handle_request(self.api_config.clone(), req).then(|result| { + let path = req.uri().path().to_owned(); + Box::new(handle_request(self.api_config.clone(), req).then(move |result| { match result { - Ok(res) => Ok::<_, hyper::Error>(res), + Ok(res) => { + Self::log_response(&path, &res); + Ok::<_, hyper::Error>(res) + } Err(err) => { if let Some(apierr) = err.downcast_ref::() { let mut resp = Response::new(Body::from(apierr.message.clone())); *resp.status_mut() = apierr.code; + Self::log_response(&path, &resp); Ok(resp) } else { let mut resp = Response::new(Body::from(err.to_string())); *resp.status_mut() = StatusCode::BAD_REQUEST; + Self::log_response(&path, &resp); Ok(resp) } } @@ -458,8 +475,6 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { if let Some(_username) = rpcenv.get_user() { // fixme: check permissions } else { - println!("Abort UNAUTHORIZED API REQUEST"); - // always delay unauthorized calls by 3 seconds (from start of request) let resp = (formatter.format_error)(http_err!(UNAUTHORIZED, "permission check failed.".into())); let delayed_response = tokio::timer::Delay::new(delay_unauth_time) From 9bbd574fba09f536fca4332f806585bd437c88eb Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Feb 2019 13:28:41 +0100 Subject: [PATCH 042/344] avoid double logging of proxied requests --- src/server/rest.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index e94b7f4a..efd49cd5 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -59,7 +59,11 @@ pub struct ApiService { impl ApiService { fn log_response(path: &str, resp: &Response) { + + if resp.headers().contains_key("PBS_PROXIED") { return; } + let status = resp.status(); + if !status.is_success() { let reason = status.canonical_reason().unwrap_or("unknown reason"); let client = "unknown"; // fixme: howto get peer_addr ? @@ -169,7 +173,11 @@ fn proxy_protected_request( let resp = hyper::client::Client::new() .request(request) - .map_err(|e| Error::from(e)); + .map_err(|e| Error::from(e)) + .map(|mut resp| { + resp.headers_mut().insert("PBS_PROXIED", header::HeaderValue::from_static("1")); + resp + }); let resp = if info.reload_timezone { Either::A(resp.then(|resp| {unsafe { tzset() }; resp })) From 8daf9fd8397369056802077f9e90e6d308743277 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 14 Feb 2019 16:04:24 +0100 Subject: [PATCH 043/344] server/rest.rs: use a protocol extension to avoid double log Instead of modifying the response header itself. --- src/server/rest.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index efd49cd5..8af00b2a 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -60,7 +60,7 @@ pub struct ApiService { impl ApiService { fn log_response(path: &str, resp: &Response) { - if resp.headers().contains_key("PBS_PROXIED") { return; } + if resp.extensions().get::().is_some() { return; }; let status = resp.status(); @@ -154,6 +154,8 @@ fn get_request_parameters_async( Box::new(resp) } +struct NoLogExtension(); + fn proxy_protected_request( info: &'static ApiMethod, mut parts: Parts, @@ -175,7 +177,7 @@ fn proxy_protected_request( .request(request) .map_err(|e| Error::from(e)) .map(|mut resp| { - resp.headers_mut().insert("PBS_PROXIED", header::HeaderValue::from_static("1")); + resp.extensions_mut().insert(NoLogExtension); resp }); From 1314000db7620ee5b28a543d379a426ac66f5c7e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 15 Feb 2019 09:55:12 +0100 Subject: [PATCH 044/344] server/rest.rs: log full error messages --- src/server/formatter.rs | 11 ++++++++++- src/server/rest.rs | 6 +++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/server/formatter.rs b/src/server/formatter.rs index eb3c33d1..c46fc822 100644 --- a/src/server/formatter.rs +++ b/src/server/formatter.rs @@ -6,6 +6,9 @@ use crate::api::router::RpcEnvironment; use hyper::{Body, Response, StatusCode}; use hyper::header; +/// Extension to set error message for server side logging +pub struct ErrorMessageExtension(pub String); + pub struct OutputFormatter { pub format_result: fn(data: Value, rpcenv: &RpcEnvironment) -> Response, @@ -55,6 +58,8 @@ fn json_format_error(err: Error) -> Response { header::HeaderValue::from_static(JSON_CONTENT_TYPE)); *response.status_mut() = StatusCode::BAD_REQUEST; + response.extensions_mut().insert(ErrorMessageExtension(err.to_string())); + response } @@ -95,7 +100,11 @@ fn extjs_format_error(err: Error) -> Response { "success": false }); - json_response(result) + let mut response = json_response(result); + + response.extensions_mut().insert(ErrorMessageExtension(message)); + + response } pub static EXTJS_FORMATTER: OutputFormatter = OutputFormatter { diff --git a/src/server/rest.rs b/src/server/rest.rs index 8af00b2a..954435a1 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -67,7 +67,11 @@ impl ApiService { if !status.is_success() { let reason = status.canonical_reason().unwrap_or("unknown reason"); let client = "unknown"; // fixme: howto get peer_addr ? - let message = "request failed"; + + let mut message = "request failed"; + if let Some(data) = resp.extensions().get::() { + message = &data.0; + } log::error!("{}: {} {}: [client {}] {}", path, status.as_str(), reason, client, message); } From 1aff635a23688473f2508d7665e71c2160619818 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 15 Feb 2019 10:16:12 +0100 Subject: [PATCH 045/344] server/rest.rs: add method to log message --- src/server/rest.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 954435a1..0910f6f4 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -57,24 +57,22 @@ pub struct ApiService { pub api_config: Arc, } -impl ApiService { - fn log_response(path: &str, resp: &Response) { +fn log_response(method: hyper::Method, path: &str, resp: &Response) { - if resp.extensions().get::().is_some() { return; }; + if resp.extensions().get::().is_some() { return; }; - let status = resp.status(); + let status = resp.status(); - if !status.is_success() { - let reason = status.canonical_reason().unwrap_or("unknown reason"); - let client = "unknown"; // fixme: howto get peer_addr ? + if !status.is_success() { + let reason = status.canonical_reason().unwrap_or("unknown reason"); + let client = "unknown"; // fixme: howto get peer_addr ? - let mut message = "request failed"; - if let Some(data) = resp.extensions().get::() { - message = &data.0; - } - - log::error!("{}: {} {}: [client {}] {}", path, status.as_str(), reason, client, message); + let mut message = "request failed"; + if let Some(data) = resp.extensions().get::() { + message = &data.0; } + + log::error!("{} {}: {} {}: [client {}] {}", method.as_str(), path, status.as_str(), reason, client, message); } } @@ -86,22 +84,24 @@ impl Service for ApiService { fn call(&mut self, req: Request) -> Self::Future { let path = req.uri().path().to_owned(); + let method = req.method().clone(); + Box::new(handle_request(self.api_config.clone(), req).then(move |result| { match result { Ok(res) => { - Self::log_response(&path, &res); + log_response(method, &path, &res); Ok::<_, hyper::Error>(res) } Err(err) => { if let Some(apierr) = err.downcast_ref::() { let mut resp = Response::new(Body::from(apierr.message.clone())); *resp.status_mut() = apierr.code; - Self::log_response(&path, &resp); + log_response(method, &path, &resp); Ok(resp) } else { let mut resp = Response::new(Body::from(err.to_string())); *resp.status_mut() = StatusCode::BAD_REQUEST; - Self::log_response(&path, &resp); + log_response(method, &path, &resp); Ok(resp) } } From 124b26b892fbe29f2a3987d6958300f22746b5ca Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 16 Feb 2019 15:52:55 +0100 Subject: [PATCH 046/344] cleanup auth code, verify CSRF prevention token --- src/server/rest.rs | 81 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 21 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 0910f6f4..5d0f8249 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -422,6 +422,48 @@ fn handle_static_file_download(filename: PathBuf) -> BoxFut { return Box::new(response); } +fn extract_auth_data(headers: &http::HeaderMap) -> (Option, Option) { + + let mut ticket = None; + if let Some(raw_cookie) = headers.get("COOKIE") { + if let Ok(cookie) = raw_cookie.to_str() { + ticket = tools::extract_auth_cookie(cookie, "PBSAuthCookie"); + } + } + + let token = match headers.get("CSRFPreventionToken").map(|v| v.to_str()) { + Some(Ok(v)) => Some(v.to_owned()), + _ => None, + }; + + (ticket, token) +} + +fn check_auth(method: &hyper::Method, ticket: Option, token: Option) -> Result { + + let ticket_lifetime = 3600*2; // 2 hours + + let username = match ticket { + Some(ticket) => match tools::ticket::verify_rsa_ticket(public_auth_key(), "PBS", &ticket, None, -300, ticket_lifetime) { + Ok((_age, Some(username))) => username.to_owned(), + Ok((_, None)) => bail!("ticket without username."), + Err(err) => return Err(err), + } + None => bail!("missing ticket"), + }; + + if method != hyper::Method::GET { + if let Some(token) = token { + println!("CSRF prev token: {:?}", token); + verify_csrf_prevention_token(csrf_secret(), &username, &token, -300, ticket_lifetime)?; + } else { + bail!(""); + } + } + + Ok(username) +} + pub fn handle_request(api: Arc, req: Request) -> BoxFut { let (parts, body) = req.into_parts(); @@ -457,20 +499,9 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { let delay_unauth_time = std::time::Instant::now() + std::time::Duration::from_millis(3000); - if let Some(raw_cookie) = parts.headers.get("COOKIE") { - if let Ok(cookie) = raw_cookie.to_str() { - if let Some(ticket) = tools::extract_auth_cookie(cookie, "PBSAuthCookie") { - if let Ok((_, Some(username))) = tools::ticket::verify_rsa_ticket( - public_auth_key(), "PBS", &ticket, None, -300, 3600*2) { - rpcenv.set_user(Some(username)); - } - } - } - } - - if comp_len >= 1 && components[0] == "api2" { println!("GOT API REQUEST"); + if comp_len >= 2 { let format = components[1]; let formatter = match format { @@ -486,16 +517,24 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { if comp_len == 4 && components[2] == "access" && components[3] == "ticket" { // explicitly allow those calls without auth } else { - if let Some(_username) = rpcenv.get_user() { - // fixme: check permissions - } else { - // always delay unauthorized calls by 3 seconds (from start of request) - let resp = (formatter.format_error)(http_err!(UNAUTHORIZED, "permission check failed.".into())); - let delayed_response = tokio::timer::Delay::new(delay_unauth_time) - .map_err(|err| http_err!(INTERNAL_SERVER_ERROR, format!("tokio timer delay error: {}", err))) - .and_then(|_| Ok(resp)); + let (ticket, token) = extract_auth_data(&parts.headers); + match check_auth(&method, ticket, token) { + Ok(username) => { - return Box::new(delayed_response); + // fixme: check permissions + + rpcenv.set_user(Some(username)); + } + Err(err) => { + // always delay unauthorized calls by 3 seconds (from start of request) + let err = http_err!(UNAUTHORIZED, format!("permission check failed - {}", err)); + let resp = (formatter.format_error)(err); + let delayed_response = tokio::timer::Delay::new(delay_unauth_time) + .map_err(|err| http_err!(INTERNAL_SERVER_ERROR, format!("tokio timer delay error: {}", err))) + .and_then(|_| Ok(resp)); + + return Box::new(delayed_response); + } } } From 304bfa59a87b76cf43f595b95848bf67014bfbfd Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 17 Feb 2019 09:59:20 +0100 Subject: [PATCH 047/344] rename src/api to src/api_schema --- src/server/environment.rs | 2 +- src/server/formatter.rs | 2 +- src/server/rest.rs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server/environment.rs b/src/server/environment.rs index 2d6103f8..36bbcd9a 100644 --- a/src/server/environment.rs +++ b/src/server/environment.rs @@ -1,4 +1,4 @@ -use crate::api::router::*; +use crate::api_schema::router::*; use std::collections::HashMap; use serde_json::Value; diff --git a/src/server/formatter.rs b/src/server/formatter.rs index c46fc822..a4b89888 100644 --- a/src/server/formatter.rs +++ b/src/server/formatter.rs @@ -1,7 +1,7 @@ use failure::*; use serde_json::{json, Value}; -use crate::api::router::RpcEnvironment; +use crate::api_schema::router::RpcEnvironment; use hyper::{Body, Response, StatusCode}; use hyper::header; diff --git a/src/server/rest.rs b/src/server/rest.rs index 5d0f8249..ec64824a 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -1,7 +1,7 @@ use crate::tools; -use crate::api::schema::*; -use crate::api::router::*; -use crate::api::config::*; +use crate::api_schema::schema::*; +use crate::api_schema::router::*; +use crate::api_schema::config::*; use crate::auth_helpers::*; use super::environment::RestEnvironment; use super::formatter::*; From b1c1c468ee7d54f1f2e599a33ac55e3fd68ac99a Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 17 Feb 2019 10:16:33 +0100 Subject: [PATCH 048/344] improve api_schema module structure --- src/server/rest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index ec64824a..9c236d82 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -1,5 +1,5 @@ use crate::tools; -use crate::api_schema::schema::*; +use crate::api_schema::*; use crate::api_schema::router::*; use crate::api_schema::config::*; use crate::auth_helpers::*; From fce8be6fe125aaffbcfe7af5a3246a0e1adcd974 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 17 Feb 2019 17:18:44 +0100 Subject: [PATCH 049/344] src/server/rest.rs: improve logs for unauthorized request --- src/server/formatter.rs | 14 +++++++++++--- src/server/rest.rs | 4 ++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/server/formatter.rs b/src/server/formatter.rs index a4b89888..b7ad5a65 100644 --- a/src/server/formatter.rs +++ b/src/server/formatter.rs @@ -1,7 +1,7 @@ use failure::*; use serde_json::{json, Value}; -use crate::api_schema::router::RpcEnvironment; +use crate::api_schema::router::{HttpError, RpcEnvironment}; use hyper::{Body, Response, StatusCode}; use hyper::header; @@ -52,11 +52,19 @@ fn json_format_result(data: Value, rpcenv: &RpcEnvironment) -> Response { fn json_format_error(err: Error) -> Response { - let mut response = Response::new(Body::from(err.to_string())); + let mut response = if let Some(apierr) = err.downcast_ref::() { + let mut resp = Response::new(Body::from(apierr.message.clone())); + *resp.status_mut() = apierr.code; + resp + } else { + let mut resp = Response::new(Body::from(err.to_string())); + *resp.status_mut() = StatusCode::BAD_REQUEST; + resp + }; + response.headers_mut().insert( header::CONTENT_TYPE, header::HeaderValue::from_static(JSON_CONTENT_TYPE)); - *response.status_mut() = StatusCode::BAD_REQUEST; response.extensions_mut().insert(ErrorMessageExtension(err.to_string())); diff --git a/src/server/rest.rs b/src/server/rest.rs index 9c236d82..2a1546ab 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -454,10 +454,10 @@ fn check_auth(method: &hyper::Method, ticket: Option, token: Option Date: Sun, 17 Feb 2019 17:31:53 +0100 Subject: [PATCH 050/344] src/server/rest.rs: factor our normalize_path() --- src/server/rest.rs | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 2a1546ab..75498469 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -464,31 +464,40 @@ fn check_auth(method: &hyper::Method, ticket: Option, token: Option, req: Request) -> BoxFut { - - let (parts, body) = req.into_parts(); - - let method = parts.method.clone(); - let path = parts.uri.path(); - - // normalize path - // do not allow ".", "..", or hidden files ".XXXX" - // also remove empty path components +// normalize path +// do not allow ".", "..", or hidden files ".XXXX" +// also remove empty path components +fn normalize_path(path: &str) -> Result<(String, Vec<&str>), Error> { let items = path.split('/'); + let mut path = String::new(); let mut components = vec![]; for name in items { if name.is_empty() { continue; } if name.starts_with(".") { - return Box::new(future::err(http_err!(BAD_REQUEST, "Path contains illegal components.".to_string()))); + bail!("Path contains illegal components."); } path.push('/'); path.push_str(name); components.push(name); } + Ok((path, components)) +} + +pub fn handle_request(api: Arc, req: Request) -> BoxFut { + + let (parts, body) = req.into_parts(); + + let method = parts.method.clone(); + + let (path, components) = match normalize_path(parts.uri.path()) { + Ok((p,c)) => (p, c), + Err(err) => return Box::new(future::err(http_err!(BAD_REQUEST, err.to_string()))), + }; + let comp_len = components.len(); println!("REQUEST {} {}", method, path); From bc6fa1684e94f6db8aff78f113217dc0b82c63bc Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sun, 17 Feb 2019 18:50:40 +0100 Subject: [PATCH 051/344] src/server/rest.rs: get_index() include username and CSRF token When we have an valid ticket. Also delay get_index() if called with an invalid ticket. --- src/server/rest.rs | 50 +++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 75498469..ab248a94 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -225,11 +225,7 @@ fn handle_sync_api_request( } if delay { - let delayed_response = tokio::timer::Delay::new(delay_unauth_time) - .map_err(|err| http_err!(INTERNAL_SERVER_ERROR, format!("tokio timer delay error: {}", err))) - .and_then(|_| Ok(resp)); - - Either::A(delayed_response) + Either::A(delayed_response(resp, delay_unauth_time)) } else { Either::B(future::ok(resp)) } @@ -278,11 +274,12 @@ fn handle_async_api_request( } } -fn get_index() -> BoxFut { +fn get_index(username: Option, token: Option) -> Response { let nodename = tools::nodename(); - let username = ""; // fixme: implement real auth - let token = ""; + let username = username.unwrap_or(String::from("")); + + let token = token.unwrap_or(String::from("")); let setup = json!({ "Setup": { "auth_cookie_name": "PBSAuthCookie" }, @@ -326,13 +323,11 @@ fn get_index() -> BoxFut { "###, setup.to_string()); - let resp = Response::builder() + Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "text/html") .body(index.into()) - .unwrap(); - - Box::new(future::ok(resp)) + .unwrap() } fn extension_to_content_type(filename: &Path) -> (&'static str, bool) { @@ -439,7 +434,7 @@ fn extract_auth_data(headers: &http::HeaderMap) -> (Option, Option, token: Option) -> Result { +fn check_auth(method: &hyper::Method, ticket: &Option, token: &Option) -> Result { let ticket_lifetime = 3600*2; // 2 hours @@ -487,6 +482,13 @@ fn normalize_path(path: &str) -> Result<(String, Vec<&str>), Error> { Ok((path, components)) } +fn delayed_response(resp: Response, delay_unauth_time: std::time::Instant) -> BoxFut { + + Box::new(tokio::timer::Delay::new(delay_unauth_time) + .map_err(|err| http_err!(INTERNAL_SERVER_ERROR, format!("tokio timer delay error: {}", err))) + .and_then(|_| Ok(resp))) +} + pub fn handle_request(api: Arc, req: Request) -> BoxFut { let (parts, body) = req.into_parts(); @@ -509,7 +511,6 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { let delay_unauth_time = std::time::Instant::now() + std::time::Duration::from_millis(3000); if comp_len >= 1 && components[0] == "api2" { - println!("GOT API REQUEST"); if comp_len >= 2 { let format = components[1]; @@ -527,7 +528,7 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { // explicitly allow those calls without auth } else { let (ticket, token) = extract_auth_data(&parts.headers); - match check_auth(&method, ticket, token) { + match check_auth(&method, &ticket, &token) { Ok(username) => { // fixme: check permissions @@ -537,12 +538,7 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { Err(err) => { // always delay unauthorized calls by 3 seconds (from start of request) let err = http_err!(UNAUTHORIZED, format!("permission check failed - {}", err)); - let resp = (formatter.format_error)(err); - let delayed_response = tokio::timer::Delay::new(delay_unauth_time) - .map_err(|err| http_err!(INTERNAL_SERVER_ERROR, format!("tokio timer delay error: {}", err))) - .and_then(|_| Ok(resp)); - - return Box::new(delayed_response); + return delayed_response((formatter.format_error)(err), delay_unauth_time); } } } @@ -562,10 +558,18 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { } } } else { - // not Auth for accessing files! + // not Auth required for accessing files! if comp_len == 0 { - return get_index(); + let (ticket, token) = extract_auth_data(&parts.headers); + if ticket != None { + match check_auth(&method, &ticket, &token) { + Ok(username) => return Box::new(future::ok(get_index(Some(username), token))), + _ => return delayed_response(get_index(None, None), delay_unauth_time), + } + } else { + return Box::new(future::ok(get_index(None, None))); + } } else { let filename = api.find_alias(&components); return handle_static_file_download(filename); From 435615a34a680219db2aab30a3a85307ec4f4867 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 18 Feb 2019 06:54:12 +0100 Subject: [PATCH 052/344] src/server/rest.rs: correctly insert NoLogExtension() --- src/server/rest.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index ab248a94..824a07c4 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -181,16 +181,17 @@ fn proxy_protected_request( .request(request) .map_err(|e| Error::from(e)) .map(|mut resp| { - resp.extensions_mut().insert(NoLogExtension); + resp.extensions_mut().insert(NoLogExtension()); resp }); + let resp = if info.reload_timezone { Either::A(resp.then(|resp| {unsafe { tzset() }; resp })) } else { Either::B(resp) }; - + return Box::new(resp); } From 3dc99a5049c3440c6c6bc4cc367ee26b370b23c7 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Mon, 18 Feb 2019 13:21:25 +0100 Subject: [PATCH 053/344] cleanup Error::from is already a function taking 1 parameter, there's no need to wrap it with `|e| Error::from(e)`. Signed-off-by: Wolfgang Bumiller --- src/server/rest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 824a07c4..e92406c5 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -179,7 +179,7 @@ fn proxy_protected_request( let resp = hyper::client::Client::new() .request(request) - .map_err(|e| Error::from(e)) + .map_err(Error::from) .map(|mut resp| { resp.extensions_mut().insert(NoLogExtension()); resp From 9b4e1de1c0d561785628bd3887be10c002b4dd49 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 27 Feb 2019 12:12:00 +0100 Subject: [PATCH 054/344] rc/server/rest.rs: allow to pass parameters as application/json --- src/server/rest.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index e92406c5..b5ad9cd0 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -117,6 +117,18 @@ fn get_request_parameters_async( uri_param: HashMap, ) -> Box + Send> { + let mut is_json = false; + + if let Some(value) = parts.headers.get(header::CONTENT_TYPE) { + if value == "application/x-www-form-urlencoded" { + is_json = false; + } else if value == "application/json" { + is_json = true; + } else { + return Box::new(future::err(http_err!(BAD_REQUEST, format!("unsupported content type")))); + } + } + let resp = req_body .map_err(|err| http_err!(BAD_REQUEST, format!("Promlems reading request body: {}", err))) .fold(Vec::new(), |mut acc, chunk| { @@ -130,6 +142,18 @@ fn get_request_parameters_async( let utf8 = std::str::from_utf8(&body)?; + let obj_schema = &info.parameters; + + if is_json { + let mut params: Value = serde_json::from_str(utf8)?; + for (k, v) in uri_param { + if let Some((_optional, prop_schema)) = obj_schema.properties.get::(&k) { + params[&k] = parse_simple_value(&v, prop_schema)?; + } + } + return Ok(params); + } + let mut param_list: Vec<(String, String)> = vec![]; if utf8.len() > 0 { @@ -150,7 +174,7 @@ fn get_request_parameters_async( param_list.push((k.clone(), v.clone())); } - let params = parse_parameter_strings(¶m_list, &info.parameters, true)?; + let params = parse_parameter_strings(¶m_list, obj_schema, true)?; Ok(params) }); From b9febc5f1c37c4fe8a78192729fb6063fcf6e3ea Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Fri, 1 Mar 2019 09:34:29 +0100 Subject: [PATCH 055/344] src/tools/file_logger.rs: class to log into files --- src/tools/file_logger.rs | 73 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/tools/file_logger.rs diff --git a/src/tools/file_logger.rs b/src/tools/file_logger.rs new file mode 100644 index 00000000..743a4fb4 --- /dev/null +++ b/src/tools/file_logger.rs @@ -0,0 +1,73 @@ +use failure::*; +use chrono::Local; +use std::io::Write; + +/// Log messages with timestamps into files +/// +/// Logs messages to file, and optionaly to standart output. +/// +/// +/// #### Example: +/// ``` +/// #[macro_use] extern crate proxmox_backup; +/// # use failure::*; +/// use proxmox_backup::tools::FileLogger; +/// +/// let mut log = FileLogger::new("test.log", true).unwrap(); +/// flog!(log, "A simple log: {}", "Hello!"); +/// ``` + + +pub struct FileLogger { + file: std::fs::File, + to_stdout: bool, +} + +/// Log messages to [FileLogger](tools/struct.FileLogger.html) +#[macro_export] +macro_rules! flog { + ($log:expr, $($arg:tt)*) => ({ + $log.log(format!($($arg)*)); + }) +} + +impl FileLogger { + + pub fn new(file_name: &str, to_stdout: bool) -> Result { + + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(file_name)?; + + Ok(Self { file , to_stdout }) + } + + pub fn log>(&mut self, msg: S) { + + let msg = msg.as_ref(); + + let mut stdout = std::io::stdout(); + if self.to_stdout { + stdout.write(msg.as_bytes()).unwrap(); + stdout.write(b"\n").unwrap(); + } + + + let line = format!("{}: {}\n", Local::now().format("%b %e %T"), msg); + self.file.write(line.as_bytes()).unwrap(); + } +} + +impl std::io::Write for FileLogger { + fn write(&mut self, buf: &[u8]) -> Result { + if self.to_stdout { let _ = std::io::stdout().write(buf); } + self.file.write(buf) + } + + fn flush(&mut self) -> Result<(), std::io::Error> { + if self.to_stdout { let _ = std::io::stdout().flush(); } + self.file.flush() + } +} From 6dd8bfb84bcdf3c78b0cb8107307e5a68965ddc4 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 5 Mar 2019 12:52:39 +0100 Subject: [PATCH 056/344] src/tools/ticket.rs: define const TICKET_LIFETIME --- src/server/rest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index b5ad9cd0..87401f4e 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -461,7 +461,7 @@ fn extract_auth_data(headers: &http::HeaderMap) -> (Option, Option, token: &Option) -> Result { - let ticket_lifetime = 3600*2; // 2 hours + let ticket_lifetime = tools::ticket::TICKET_LIFETIME; let username = match ticket { Some(ticket) => match tools::ticket::verify_rsa_ticket(public_auth_key(), "PBS", &ticket, None, -300, ticket_lifetime) { From 946995d98488594eed4105b04fc179a48941a152 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Tue, 12 Mar 2019 10:40:25 +0100 Subject: [PATCH 057/344] tools: add daemon helpers Signed-off-by: Wolfgang Bumiller --- src/tools/daemon.rs | 115 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 src/tools/daemon.rs diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs new file mode 100644 index 00000000..bdd8450b --- /dev/null +++ b/src/tools/daemon.rs @@ -0,0 +1,115 @@ +//! Helpers for daemons/services. + +use std::ffi::CString; +use std::os::unix::ffi::OsStrExt; +use std::panic::UnwindSafe; + +use failure::*; + +// Unfortunately FnBox is nightly-only and Box is unusable, so just use Box... +pub type BoxedStoreFunc = Box Result + UnwindSafe + Send>; + +/// Helper trait to "store" something in the environment to be re-used after re-executing the +/// service on a reload. +pub trait ReexecContinue: Sized { + fn restore(var: &str) -> Result; + fn get_store_func(&self) -> BoxedStoreFunc; +} + +/// Manages things to be stored and reloaded upon reexec. +/// Anything which should be restorable should be instantiated via this struct's `restore` method, +pub struct ReexecStore { + pre_exec: Vec, +} + +// Currently we only need environment variables for storage, but in theory we could also add +// variants which need temporary files or pipes... +struct PreExecEntry { + name: &'static str, // Feel free to change to String if necessary... + store_fn: BoxedStoreFunc, +} + +impl ReexecStore { + pub fn new() -> Self { + Self { + pre_exec: Vec::new(), + } + } + + /// Restore an object from an environment variable of the given name, or, if none exists, uses + /// the function provided in the `or_create` parameter to instantiate the new "first" instance. + /// + /// Values created via this method will be remembered for later re-execution. + pub fn restore(&mut self, name: &'static str, or_create: F) -> Result + where + T: ReexecContinue, + F: FnOnce() -> Result, + { + let res = match std::env::var(name) { + Ok(varstr) => T::restore(&varstr)?, + Err(std::env::VarError::NotPresent) => or_create()?, + Err(_) => bail!("variable {} has invalid value", name), + }; + + self.pre_exec.push(PreExecEntry { + name, + store_fn: res.get_store_func(), + }); + Ok(res) + } + + fn pre_exec(self) -> Result<(), Error> { + for item in self.pre_exec { + std::env::set_var(item.name, (item.store_fn)()?); + } + Ok(()) + } + + pub fn fork_restart(self) -> Result<(), Error> { + // Get the path to our executable as CString + let exe = CString::new( + std::fs::read_link("/proc/self/exe")? + .into_os_string() + .as_bytes() + )?; + + // Get our parameters as Vec + let args = std::env::args_os(); + let mut new_args = Vec::with_capacity(args.len()); + for arg in args { + new_args.push(CString::new(arg.as_bytes())?); + } + + // Start ourselves in the background: + use nix::unistd::{fork, ForkResult}; + match fork() { + Ok(ForkResult::Child) => { + // At this point we call pre-exec helpers. We must be certain that if they fail for + // whatever reason we can still call `_exit()`, so use catch_unwind. + match std::panic::catch_unwind(move || self.do_exec(exe, new_args)) { + Ok(_) => eprintln!("do_exec returned unexpectedly!"), + Err(_) => eprintln!("panic in re-exec"), + } + // No matter how we managed to get here, this is the time where we bail out quickly: + unsafe { + libc::_exit(-1) + } + } + Ok(ForkResult::Parent { child }) => { + eprintln!("forked off a new server (pid: {})", child); + Ok(()) + } + Err(e) => { + eprintln!("fork() failed, restart delayed: {}", e); + Ok(()) + } + } + } + + fn do_exec(self, exe: CString, args: Vec) -> Result<(), Error> { + self.pre_exec()?; + nix::unistd::setsid()?; + nix::unistd::execvp(&exe, &args)?; + Ok(()) + } +} From 66c138a51a8cce8c09fda8578605794f80578272 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Mon, 18 Mar 2019 11:50:10 +0100 Subject: [PATCH 058/344] tools: daemon: add a default signalfd helper Proxy and daemon for now just want to handle reload via `SIGHUP`, so provide a helper creating the signalfd stream doing that - this is simply a filtered stream which passes the remaining signals through, so it can be used exactly like the signalfd stream could before to add more signals. Signed-off-by: Wolfgang Bumiller --- src/tools/daemon.rs | 55 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index bdd8450b..0df1f2e5 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -5,6 +5,9 @@ use std::os::unix::ffi::OsStrExt; use std::panic::UnwindSafe; use failure::*; +use tokio::prelude::*; + +use crate::tools::signalfd::{SigSet, SignalFd}; // Unfortunately FnBox is nightly-only and Box is unusable, so just use Box... pub type BoxedStoreFunc = Box Result + UnwindSafe + Send>; @@ -113,3 +116,55 @@ impl ReexecStore { Ok(()) } } + +/// Provide a default signal handler for daemons (daemon & proxy). +/// When the first `SIGHUP` is received, the `reexec_store`'s `fork_restart` method will be +/// triggered. Any further `SIGHUP` is "passed through". +pub fn default_signalfd_stream( + reexec_store: ReexecStore, + before_reload: F, +) -> Result, Error> +where + F: FnOnce() -> Result<(), Error>, +{ + use nix::sys::signal::{SigmaskHow, Signal, sigprocmask}; + + // Block SIGHUP for *all* threads and use it for a signalfd handler: + let mut sigs = SigSet::empty(); + sigs.add(Signal::SIGHUP); + sigprocmask(SigmaskHow::SIG_BLOCK, Some(&sigs), None)?; + + let sigfdstream = SignalFd::new(&sigs)?; + let mut reexec_store = Some(reexec_store); + let mut before_reload = Some(before_reload); + + Ok(sigfdstream + .filter_map(move |si| { + // FIXME: logging should be left to the user of this: + eprintln!("received signal: {}", si.ssi_signo); + + if si.ssi_signo == Signal::SIGHUP as u32 { + // The firs time this happens we will try to start a new process which should take + // over. + if let Some(reexec_store) = reexec_store.take() { + if let Err(e) = (before_reload.take().unwrap())() { + return Some(Err(e)); + } + + match reexec_store.fork_restart() { + Ok(_) => return None, + Err(e) => return Some(Err(e)), + } + } + } + + // pass the rest through: + Some(Ok(si)) + }) + // filter_map cannot produce errors, so we create Result<> items instead, iow: + // before: Stream + // after: Stream, Error>. + // use and_then to lift out the wrapped result: + .and_then(|si_res| si_res) + ) +} From 14ed3eb57c0dc84ef57e1c39ea9dbc443cff0b3a Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Mon, 18 Mar 2019 11:52:04 +0100 Subject: [PATCH 059/344] tools: implement ReexecContinue for tokio's TcpListener This is the only thing we currently need to keep alive for reloads. Signed-off-by: Wolfgang Bumiller --- src/tools/daemon.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index 0df1f2e5..b0e60d91 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -1,12 +1,14 @@ //! Helpers for daemons/services. use std::ffi::CString; +use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; use std::os::unix::ffi::OsStrExt; use std::panic::UnwindSafe; use failure::*; use tokio::prelude::*; +use crate::tools::fd_change_cloexec; use crate::tools::signalfd::{SigSet, SignalFd}; // Unfortunately FnBox is nightly-only and Box is unusable, so just use Box... @@ -168,3 +170,28 @@ where .and_then(|si_res| si_res) ) } + +// For now all we need to do is store and reuse a tcp listening socket: +impl ReexecContinue for tokio::net::TcpListener { + // NOTE: The socket must not be closed when the store-function is called: + // FIXME: We could become "independent" of the TcpListener and its reference to the file + // descriptor by `dup()`ing it (and check if the listener still exists via kcmp()?) + fn get_store_func(&self) -> BoxedStoreFunc { + let fd = self.as_raw_fd(); + Box::new(move || { + fd_change_cloexec(fd, false)?; + Ok(fd.to_string()) + }) + } + + fn restore(var: &str) -> Result { + let fd = var.parse::() + .map_err(|e| format_err!("invalid file descriptor: {}", e))? + as RawFd; + fd_change_cloexec(fd, true)?; + Ok(Self::from_std( + unsafe { std::net::TcpListener::from_raw_fd(fd) }, + &tokio::reactor::Handle::default(), + )?) + } +} From 3aa2dbc85750adb254f1e77a42ac57e3addad5b8 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Mon, 18 Mar 2019 13:44:37 +0100 Subject: [PATCH 060/344] tools: daemon: rename some structs Reloadable resources are now 'Reloadable' instead of 'ReexecContinue'. The struct handling the reload is a 'Reloader', not a 'ReexecStore'. Signed-off-by: Wolfgang Bumiller --- src/tools/daemon.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index b0e60d91..f51dab1c 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -6,6 +6,7 @@ use std::os::unix::ffi::OsStrExt; use std::panic::UnwindSafe; use failure::*; +use nix::sys::signalfd::siginfo; use tokio::prelude::*; use crate::tools::fd_change_cloexec; @@ -16,14 +17,14 @@ pub type BoxedStoreFunc = Box Result + UnwindSafe + S /// Helper trait to "store" something in the environment to be re-used after re-executing the /// service on a reload. -pub trait ReexecContinue: Sized { +pub trait Reloadable: Sized { fn restore(var: &str) -> Result; fn get_store_func(&self) -> BoxedStoreFunc; } /// Manages things to be stored and reloaded upon reexec. /// Anything which should be restorable should be instantiated via this struct's `restore` method, -pub struct ReexecStore { +pub struct Reloader { pre_exec: Vec, } @@ -34,7 +35,7 @@ struct PreExecEntry { store_fn: BoxedStoreFunc, } -impl ReexecStore { +impl Reloader { pub fn new() -> Self { Self { pre_exec: Vec::new(), @@ -47,7 +48,7 @@ impl ReexecStore { /// Values created via this method will be remembered for later re-execution. pub fn restore(&mut self, name: &'static str, or_create: F) -> Result where - T: ReexecContinue, + T: Reloadable, F: FnOnce() -> Result, { let res = match std::env::var(name) { @@ -120,12 +121,12 @@ impl ReexecStore { } /// Provide a default signal handler for daemons (daemon & proxy). -/// When the first `SIGHUP` is received, the `reexec_store`'s `fork_restart` method will be +/// When the first `SIGHUP` is received, the `reloader`'s `fork_restart` method will be /// triggered. Any further `SIGHUP` is "passed through". pub fn default_signalfd_stream( - reexec_store: ReexecStore, + reloader: Reloader, before_reload: F, -) -> Result, Error> +) -> Result, Error> where F: FnOnce() -> Result<(), Error>, { @@ -137,7 +138,7 @@ where sigprocmask(SigmaskHow::SIG_BLOCK, Some(&sigs), None)?; let sigfdstream = SignalFd::new(&sigs)?; - let mut reexec_store = Some(reexec_store); + let mut reloader = Some(reloader); let mut before_reload = Some(before_reload); Ok(sigfdstream @@ -148,12 +149,12 @@ where if si.ssi_signo == Signal::SIGHUP as u32 { // The firs time this happens we will try to start a new process which should take // over. - if let Some(reexec_store) = reexec_store.take() { + if let Some(reloader) = reloader.take() { if let Err(e) = (before_reload.take().unwrap())() { return Some(Err(e)); } - match reexec_store.fork_restart() { + match reloader.fork_restart() { Ok(_) => return None, Err(e) => return Some(Err(e)), } @@ -172,7 +173,7 @@ where } // For now all we need to do is store and reuse a tcp listening socket: -impl ReexecContinue for tokio::net::TcpListener { +impl Reloadable for tokio::net::TcpListener { // NOTE: The socket must not be closed when the store-function is called: // FIXME: We could become "independent" of the TcpListener and its reference to the file // descriptor by `dup()`ing it (and check if the listener still exists via kcmp()?) From 50e95f7c392d50832776692337ade01642e682cd Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Mon, 18 Mar 2019 14:13:44 +0100 Subject: [PATCH 061/344] daemon: simplify daemon creation Signed-off-by: Wolfgang Bumiller --- src/tools/daemon.rs | 114 ++++++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 53 deletions(-) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index f51dab1c..db66c316 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -6,7 +6,8 @@ use std::os::unix::ffi::OsStrExt; use std::panic::UnwindSafe; use failure::*; -use nix::sys::signalfd::siginfo; +use futures::future::poll_fn; +use futures::try_ready; use tokio::prelude::*; use crate::tools::fd_change_cloexec; @@ -120,58 +121,6 @@ impl Reloader { } } -/// Provide a default signal handler for daemons (daemon & proxy). -/// When the first `SIGHUP` is received, the `reloader`'s `fork_restart` method will be -/// triggered. Any further `SIGHUP` is "passed through". -pub fn default_signalfd_stream( - reloader: Reloader, - before_reload: F, -) -> Result, Error> -where - F: FnOnce() -> Result<(), Error>, -{ - use nix::sys::signal::{SigmaskHow, Signal, sigprocmask}; - - // Block SIGHUP for *all* threads and use it for a signalfd handler: - let mut sigs = SigSet::empty(); - sigs.add(Signal::SIGHUP); - sigprocmask(SigmaskHow::SIG_BLOCK, Some(&sigs), None)?; - - let sigfdstream = SignalFd::new(&sigs)?; - let mut reloader = Some(reloader); - let mut before_reload = Some(before_reload); - - Ok(sigfdstream - .filter_map(move |si| { - // FIXME: logging should be left to the user of this: - eprintln!("received signal: {}", si.ssi_signo); - - if si.ssi_signo == Signal::SIGHUP as u32 { - // The firs time this happens we will try to start a new process which should take - // over. - if let Some(reloader) = reloader.take() { - if let Err(e) = (before_reload.take().unwrap())() { - return Some(Err(e)); - } - - match reloader.fork_restart() { - Ok(_) => return None, - Err(e) => return Some(Err(e)), - } - } - } - - // pass the rest through: - Some(Ok(si)) - }) - // filter_map cannot produce errors, so we create Result<> items instead, iow: - // before: Stream - // after: Stream, Error>. - // use and_then to lift out the wrapped result: - .and_then(|si_res| si_res) - ) -} - // For now all we need to do is store and reuse a tcp listening socket: impl Reloadable for tokio::net::TcpListener { // NOTE: The socket must not be closed when the store-function is called: @@ -196,3 +145,62 @@ impl Reloadable for tokio::net::TcpListener { )?) } } + +/// This creates a future representing a daemon which reloads itself when receiving a SIGHUP. +/// If this is started regularly, a listening socket is created. In this case, the file descriptor +/// number will be remembered in `PROXMOX_BACKUP_LISTEN_FD`. +/// If the variable already exists, its contents will instead be used to restore the listening +/// socket. The finished listening socket is then passed to the `create_service` function which +/// can be used to setup the TLS and the HTTP daemon. +pub fn create_daemon( + address: std::net::SocketAddr, + create_service: F, +) -> Result, Error> +where + F: FnOnce(tokio::net::TcpListener) -> Result, + S: Future, +{ + let mut reloader = Reloader::new(); + + let listener: tokio::net::TcpListener = reloader.restore( + "PROXMOX_BACKUP_LISTEN_FD", + move || Ok(tokio::net::TcpListener::bind(&address)?), + )?; + + let service = create_service(listener)?; + + // Block SIGHUP for *all* threads and use it for a signalfd handler: + use nix::sys::signal; + let mut sigs = SigSet::empty(); + sigs.add(signal::Signal::SIGHUP); + signal::sigprocmask(signal::SigmaskHow::SIG_BLOCK, Some(&sigs), None)?; + + let mut sigfdstream = SignalFd::new(&sigs)? + .map_err(|e| log::error!("error in signal handler: {}", e)); + + let mut reloader = Some(reloader); + + // Use a Future instead of a Stream for ease-of-use: Poll until we receive a SIGHUP. + let signal_handler = poll_fn(move || { + match try_ready!(sigfdstream.poll()) { + Some(si) => { + log::info!("received signal {}", si.ssi_signo); + if si.ssi_signo == signal::Signal::SIGHUP as u32 { + if let Err(e) = reloader.take().unwrap().fork_restart() { + log::error!("error during reload: {}", e); + } + Ok(Async::Ready(())) + } else { + Ok(Async::NotReady) + } + } + // or the stream ended (which it can't, really) + None => Ok(Async::Ready(())) + } + }); + + Ok(service.select(signal_handler) + .map(|_| log::info!("daemon shutting down...")) + .map_err(|_| ()) + ) +} From 022b626bc05c1d371b22dbd7e7ce7c083d83c5ac Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 19 Mar 2019 12:50:15 +0100 Subject: [PATCH 062/344] src/server/rest.rs: correctly extract content type --- src/server/rest.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 87401f4e..022155db 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -120,12 +120,16 @@ fn get_request_parameters_async( let mut is_json = false; if let Some(value) = parts.headers.get(header::CONTENT_TYPE) { - if value == "application/x-www-form-urlencoded" { - is_json = false; - } else if value == "application/json" { - is_json = true; - } else { - return Box::new(future::err(http_err!(BAD_REQUEST, format!("unsupported content type")))); + match value.to_str().map(|v| v.split(';').next()) { + Ok(Some("application/x-www-form-urlencoded")) => { + is_json = false; + } + Ok(Some("application/json")) => { + is_json = true; + } + _ => { + return Box::new(future::err(http_err!(BAD_REQUEST, format!("unsupported content type")))); + } } } From 24c023fe4736862e3bc0dbe68352dcb45fe54d2a Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 1 Apr 2019 07:52:30 +0200 Subject: [PATCH 063/344] src/server/rest.rs: generate csrf token if we have a valid ticket This is important if the user reloads the browser page. --- src/server/rest.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 022155db..44e9943f 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -589,11 +589,18 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { } else { // not Auth required for accessing files! + if method != hyper::Method::GET { + return Box::new(future::err(http_err!(BAD_REQUEST, format!("Unsupported method")))); + } + if comp_len == 0 { let (ticket, token) = extract_auth_data(&parts.headers); if ticket != None { match check_auth(&method, &ticket, &token) { - Ok(username) => return Box::new(future::ok(get_index(Some(username), token))), + Ok(username) => { + let new_token = assemble_csrf_prevention_token(csrf_secret(), &username); + return Box::new(future::ok(get_index(Some(username), Some(new_token)))); + } _ => return delayed_response(get_index(None, None), delay_unauth_time), } } else { From c76ceea94145fa204f9b46359cd8fab65fe35125 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 1 Apr 2019 08:04:12 +0200 Subject: [PATCH 064/344] src/server/rest.rs: use formatter to encode errors --- src/server/rest.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 44e9943f..1cce06ed 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -573,7 +573,10 @@ pub fn handle_request(api: Arc, req: Request) -> BoxFut { } match api.find_method(&components[2..], method, &mut uri_param) { - MethodDefinition::None => {} + MethodDefinition::None => { + let err = http_err!(NOT_FOUND, "Path not found.".to_string()); + return Box::new(future::ok((formatter.format_error)(err))); + } MethodDefinition::Simple(api_method) => { if api_method.protected && env_type == RpcEnvironmentType::PUBLIC { return proxy_protected_request(api_method, parts, body); From edc588857e14c7d8eebe35f660383fc6233ab17a Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 1 Apr 2019 12:03:47 +0200 Subject: [PATCH 065/344] add global var to indicate server shutdown requests --- src/tools/daemon.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index db66c316..eb1ead05 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -200,7 +200,10 @@ where }); Ok(service.select(signal_handler) - .map(|_| log::info!("daemon shutting down...")) - .map_err(|_| ()) + .map(|_| { + log::info!("daemon shutting down..."); + crate::tools::request_shutdown(); + }) + .map_err(|_| ()) ) } From 1ee4442d87fba352d2afaa0fe4c79930a465136a Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 3 Apr 2019 08:58:43 +0200 Subject: [PATCH 066/344] src/tools/file_logger.rs: change timestamp format to rfc3339 --- src/tools/file_logger.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tools/file_logger.rs b/src/tools/file_logger.rs index 743a4fb4..423e998c 100644 --- a/src/tools/file_logger.rs +++ b/src/tools/file_logger.rs @@ -1,5 +1,5 @@ use failure::*; -use chrono::Local; +use chrono::{TimeZone, Local}; use std::io::Write; /// Log messages with timestamps into files @@ -18,6 +18,7 @@ use std::io::Write; /// ``` +#[derive(Debug)] pub struct FileLogger { file: std::fs::File, to_stdout: bool, @@ -54,8 +55,7 @@ impl FileLogger { stdout.write(b"\n").unwrap(); } - - let line = format!("{}: {}\n", Local::now().format("%b %e %T"), msg); + let line = format!("{}: {}\n", Local.timestamp(Local::now().timestamp(), 0).to_rfc3339(), msg); self.file.write(line.as_bytes()).unwrap(); } } From e3e5ef3929876c33264d2230c65e48bf2d4a72b5 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 3 Apr 2019 14:13:33 +0200 Subject: [PATCH 067/344] src/tools/file_logger.rs: new - accept AsRef --- src/tools/file_logger.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/file_logger.rs b/src/tools/file_logger.rs index 423e998c..0475073b 100644 --- a/src/tools/file_logger.rs +++ b/src/tools/file_logger.rs @@ -34,7 +34,7 @@ macro_rules! flog { impl FileLogger { - pub fn new(file_name: &str, to_stdout: bool) -> Result { + pub fn new>(file_name: P, to_stdout: bool) -> Result { let file = std::fs::OpenOptions::new() .read(true) From ff995ce0e1e10eba78c78a7097612f353c69091e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 6 Apr 2019 09:17:25 +0200 Subject: [PATCH 068/344] src/server.rs: improve crate layout --- src/server/environment.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/environment.rs b/src/server/environment.rs index 36bbcd9a..b002b24f 100644 --- a/src/server/environment.rs +++ b/src/server/environment.rs @@ -3,6 +3,7 @@ use crate::api_schema::router::*; use std::collections::HashMap; use serde_json::Value; +/// Encapsulates information about the runtime environment pub struct RestEnvironment { env_type: RpcEnvironmentType, result_attributes: HashMap, From 71d03f1ef408d8dc3cac36ee601cd3fba65bc183 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Sat, 6 Apr 2019 11:24:37 +0200 Subject: [PATCH 069/344] src/tools/file_logger.rs: fix test --- src/tools/file_logger.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/file_logger.rs b/src/tools/file_logger.rs index 0475073b..a8cab578 100644 --- a/src/tools/file_logger.rs +++ b/src/tools/file_logger.rs @@ -13,6 +13,7 @@ use std::io::Write; /// # use failure::*; /// use proxmox_backup::tools::FileLogger; /// +/// # std::fs::remove_file("test.log"); /// let mut log = FileLogger::new("test.log", true).unwrap(); /// flog!(log, "A simple log: {}", "Hello!"); /// ``` From 9761b81b84e556dd8fdf978a57c5b64ef1078bd3 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 8 Apr 2019 12:21:29 +0200 Subject: [PATCH 070/344] implement server state/signal handling, depend on tokio-signal --- src/server/state.rs | 151 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 src/server/state.rs diff --git a/src/server/state.rs b/src/server/state.rs new file mode 100644 index 00000000..f7ee4300 --- /dev/null +++ b/src/server/state.rs @@ -0,0 +1,151 @@ +use failure::*; +use lazy_static::lazy_static; +use std::sync::Mutex; + +use futures::*; +use futures::stream::Stream; + +use tokio::sync::oneshot; +use tokio_signal::unix::{Signal, SIGHUP, SIGINT}; + +use crate::tools; + +#[derive(PartialEq, Copy, Clone, Debug)] +pub enum ServerMode { + Normal, + Shutdown, +} + +pub struct ServerState { + pub mode: ServerMode, + pub shutdown_listeners: Vec>, + pub last_worker_listeners: Vec>, + pub worker_count: usize, + pub reload_request: bool, +} + + +lazy_static! { + static ref SERVER_STATE: Mutex = Mutex::new(ServerState { + mode: ServerMode::Normal, + shutdown_listeners: vec![], + last_worker_listeners: vec![], + worker_count: 0, + reload_request: false, + }); +} + +pub fn server_state_init() -> Result<(), Error> { + + let stream = Signal::new(SIGINT).flatten_stream(); + + let future = stream.for_each(|_| { + println!("got shutdown request (SIGINT)"); + SERVER_STATE.lock().unwrap().reload_request = false; + tools::request_shutdown(); + Ok(()) + }).map_err(|_| {}); + + let abort_future = last_worker_future().map_err(|_| {}); + let task = future.select(abort_future); + + tokio::spawn(task.map(|_| {}).map_err(|_| {})); + + let stream = Signal::new(SIGHUP).flatten_stream(); + + let future = stream.for_each(|_| { + println!("got reload request (SIGHUP)"); + SERVER_STATE.lock().unwrap().reload_request = true; + tools::request_shutdown(); + Ok(()) + }).map_err(|_| {}); + + let abort_future = last_worker_future().map_err(|_| {}); + let task = future.select(abort_future); + + tokio::spawn(task.map(|_| {}).map_err(|_| {})); + + Ok(()) +} + +pub fn is_reload_request() -> bool { + let data = SERVER_STATE.lock().unwrap(); + + if data.mode == ServerMode::Shutdown && data.reload_request { + true + } else { + false + } +} + +pub fn server_shutdown() { + let mut data = SERVER_STATE.lock().unwrap(); + + println!("SET SHUTDOWN MODE"); + + data.mode = ServerMode::Shutdown; + + notify_listeners(&mut data.shutdown_listeners); + + drop(data); // unlock + + check_last_worker(); +} + +pub fn shutdown_future() -> oneshot::Receiver<()> { + let (tx, rx) = oneshot::channel::<()>(); + + let mut data = SERVER_STATE.lock().unwrap(); + match data.mode { + ServerMode::Normal => { data.shutdown_listeners.push(tx); }, + ServerMode::Shutdown => { let _ = tx.send(()); }, + } + + rx +} + +pub fn last_worker_future() -> oneshot::Receiver<()> { + let (tx, rx) = oneshot::channel::<()>(); + + let mut data = SERVER_STATE.lock().unwrap(); + if data.mode == ServerMode::Shutdown && data.worker_count == 0 { + let _ = tx.send(()); + } else { + data.last_worker_listeners.push(tx); + } + + rx +} + +pub fn set_worker_count(count: usize) { + let mut data = SERVER_STATE.lock().unwrap(); + data.worker_count = count; + + if !(data.mode == ServerMode::Shutdown && data.worker_count == 0) { return; } + + notify_listeners(&mut data.last_worker_listeners); +} + + +pub fn check_last_worker() { + + let mut data = SERVER_STATE.lock().unwrap(); + + if !(data.mode == ServerMode::Shutdown && data.worker_count == 0) { return; } + + notify_listeners(&mut data.last_worker_listeners); +} + +fn notify_listeners(list: &mut Vec>) { + loop { + match list.pop() { + None => { break; }, + Some(ch) => { + println!("SEND ABORT"); + if let Err(_) = ch.send(()) { + eprintln!("SEND ABORT failed"); + } + }, + } + } +} From b9e9f05aaf9946ef928037a1834699ad543f771b Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 8 Apr 2019 14:00:23 +0200 Subject: [PATCH 071/344] src/tools/daemon.rs: use new ServerState handler --- src/tools/daemon.rs | 49 +++++++++++++-------------------------------- 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index eb1ead05..e3becbd5 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -6,12 +6,10 @@ use std::os::unix::ffi::OsStrExt; use std::panic::UnwindSafe; use failure::*; -use futures::future::poll_fn; -use futures::try_ready; use tokio::prelude::*; +use crate::server; use crate::tools::fd_change_cloexec; -use crate::tools::signalfd::{SigSet, SignalFd}; // Unfortunately FnBox is nightly-only and Box is unusable, so just use Box... pub type BoxedStoreFunc = Box Result + UnwindSafe + Send>; @@ -169,40 +167,21 @@ where let service = create_service(listener)?; - // Block SIGHUP for *all* threads and use it for a signalfd handler: - use nix::sys::signal; - let mut sigs = SigSet::empty(); - sigs.add(signal::Signal::SIGHUP); - signal::sigprocmask(signal::SigmaskHow::SIG_BLOCK, Some(&sigs), None)?; - - let mut sigfdstream = SignalFd::new(&sigs)? - .map_err(|e| log::error!("error in signal handler: {}", e)); - let mut reloader = Some(reloader); - // Use a Future instead of a Stream for ease-of-use: Poll until we receive a SIGHUP. - let signal_handler = poll_fn(move || { - match try_ready!(sigfdstream.poll()) { - Some(si) => { - log::info!("received signal {}", si.ssi_signo); - if si.ssi_signo == signal::Signal::SIGHUP as u32 { - if let Err(e) = reloader.take().unwrap().fork_restart() { - log::error!("error during reload: {}", e); - } - Ok(Async::Ready(())) - } else { - Ok(Async::NotReady) - } - } - // or the stream ended (which it can't, really) - None => Ok(Async::Ready(())) - } - }); - - Ok(service.select(signal_handler) - .map(|_| { - log::info!("daemon shutting down..."); - crate::tools::request_shutdown(); + let abort_future = server::shutdown_future().map_err(|_| {}); + Ok(service + .select(abort_future) + .map(move |_| { + crate::tools::request_shutdown(); // make sure we are in shutdown mode + if server::is_reload_request() { + log::info!("daemon reload..."); + if let Err(e) = reloader.take().unwrap().fork_restart() { + log::error!("error during reload: {}", e); + } + } else { + log::info!("daemon shutting down..."); + } }) .map_err(|_| ()) ) From 8c4656ea047ea47c3530de57638e41326ef0b5b0 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Mon, 8 Apr 2019 17:59:39 +0200 Subject: [PATCH 072/344] src/server/command_socket.rs: simple command socket --- src/server/command_socket.rs | 67 ++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/server/command_socket.rs diff --git a/src/server/command_socket.rs b/src/server/command_socket.rs new file mode 100644 index 00000000..7378ed8e --- /dev/null +++ b/src/server/command_socket.rs @@ -0,0 +1,67 @@ +use failure::*; + +use futures::*; +use futures::stream::Stream; + +use tokio::net::unix::UnixListener; +use tokio::io::AsyncRead; + +use std::io::Write; + +use std::path::PathBuf; +use serde_json::Value; +use std::sync::Arc; + +/// Listens on a Unix Socket to handle simple command asynchronously +pub fn create_control_socket(path: P, f: F) -> Result, Error> + where P: Into, + F: Send + Sync +'static + Fn(Value) -> Result, +{ + let path: PathBuf = path.into(); + + let socket = UnixListener::bind(&path)?; + + let f = Arc::new(f); + let path = Arc::new(path); + let path2 = path.clone(); + let path3 = path.clone(); + + let control_future = socket.incoming() + .map_err(move |err| { eprintln!("failed to accept on control socket {:?}: {}", path2, err); }) + .for_each(move |conn| { + let f1 = f.clone(); + + let (rx, mut tx) = conn.split(); + let path = path3.clone(); + let path2 = path3.clone(); + + tokio::io::lines(std::io::BufReader::new(rx)) + .map_err(move |err| { eprintln!("control socket {:?} read error: {}", path, err); }) + .and_then(move |cmd| { + let res = try_block!({ + let param = match cmd.parse::() { + Ok(p) => p, + Err(err) => bail!("ERRER {}", err), + }; + + f1(param) + }); + + let resp = match res { + Ok(v) => format!("OK: {}\n", v), + Err(err) => format!("ERROR: {}\n", err), + }; + Ok(resp) + }) + .for_each(move |resp| { + tx.write_all(resp.as_bytes()) + .map_err(|err| { eprintln!("control socket {:?} write response error: {}", path2, err); }) + }) + + }); + + let abort_future = super::last_worker_future().map_err(|_| {}); + let task = control_future.select(abort_future).map(|_| {}).map_err(|_| {}); + + Ok(task) +} From 143256104499cd346d64f3532523fd9778a01452 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 9 Apr 2019 11:47:23 +0200 Subject: [PATCH 073/344] src/server/command_socket.rs: implement auto_remove flag Remove the socket file on close. --- src/server/command_socket.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/server/command_socket.rs b/src/server/command_socket.rs index 7378ed8e..07ee5fe1 100644 --- a/src/server/command_socket.rs +++ b/src/server/command_socket.rs @@ -13,18 +13,20 @@ use serde_json::Value; use std::sync::Arc; /// Listens on a Unix Socket to handle simple command asynchronously -pub fn create_control_socket(path: P, f: F) -> Result, Error> +pub fn create_control_socket(path: P, auto_remove: bool, f: F) -> Result, Error> where P: Into, F: Send + Sync +'static + Fn(Value) -> Result, { let path: PathBuf = path.into(); + let path1: PathBuf = path.clone(); + + if auto_remove { let _ = std::fs::remove_file(&path); } let socket = UnixListener::bind(&path)?; let f = Arc::new(f); - let path = Arc::new(path); - let path2 = path.clone(); - let path3 = path.clone(); + let path2 = Arc::new(path); + let path3 = path2.clone(); let control_future = socket.incoming() .map_err(move |err| { eprintln!("failed to accept on control socket {:?}: {}", path2, err); }) @@ -61,7 +63,12 @@ pub fn create_control_socket(path: P, f: F) -> Result Date: Tue, 9 Apr 2019 12:47:42 +0200 Subject: [PATCH 074/344] src/server/command_socket.rs: code cleanup - fix error message --- src/server/command_socket.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/server/command_socket.rs b/src/server/command_socket.rs index 07ee5fe1..b69866dc 100644 --- a/src/server/command_socket.rs +++ b/src/server/command_socket.rs @@ -43,7 +43,7 @@ pub fn create_control_socket(path: P, auto_remove: bool, f: F) -> Result() { Ok(p) => p, - Err(err) => bail!("ERRER {}", err), + Err(err) => bail!("unable to parse json value - {}", err), }; f1(param) @@ -63,7 +63,6 @@ pub fn create_control_socket(path: P, auto_remove: bool, f: F) -> Result Date: Wed, 10 Apr 2019 08:24:32 +0200 Subject: [PATCH 075/344] start hyper server using with_graceful_shutdown() Without, hyper keeps some futures running, and the server does not correctly shutdown. --- src/tools/daemon.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index e3becbd5..de679520 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -169,9 +169,7 @@ where let mut reloader = Some(reloader); - let abort_future = server::shutdown_future().map_err(|_| {}); Ok(service - .select(abort_future) .map(move |_| { crate::tools::request_shutdown(); // make sure we are in shutdown mode if server::is_reload_request() { From 116990f26472647ee9792106e79ad28ba715207b Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 10 Apr 2019 09:03:17 +0200 Subject: [PATCH 076/344] src/server/worker_task.rs: use abstract socket --- src/server/command_socket.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/server/command_socket.rs b/src/server/command_socket.rs index b69866dc..814d160b 100644 --- a/src/server/command_socket.rs +++ b/src/server/command_socket.rs @@ -13,15 +13,13 @@ use serde_json::Value; use std::sync::Arc; /// Listens on a Unix Socket to handle simple command asynchronously -pub fn create_control_socket(path: P, auto_remove: bool, f: F) -> Result, Error> +pub fn create_control_socket(path: P, f: F) -> Result, Error> where P: Into, F: Send + Sync +'static + Fn(Value) -> Result, { let path: PathBuf = path.into(); let path1: PathBuf = path.clone(); - if auto_remove { let _ = std::fs::remove_file(&path); } - let socket = UnixListener::bind(&path)?; let f = Arc::new(f); @@ -64,10 +62,7 @@ pub fn create_control_socket(path: P, auto_remove: bool, f: F) -> Result Date: Wed, 10 Apr 2019 11:05:00 +0200 Subject: [PATCH 077/344] src/server/command_socket.rs: correctly handle/spawn handle parallel connections --- src/server/command_socket.rs | 50 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/src/server/command_socket.rs b/src/server/command_socket.rs index 814d160b..92a1322c 100644 --- a/src/server/command_socket.rs +++ b/src/server/command_socket.rs @@ -18,7 +18,6 @@ pub fn create_control_socket(path: P, f: F) -> Result Result, { let path: PathBuf = path.into(); - let path1: PathBuf = path.clone(); let socket = UnixListener::bind(&path)?; @@ -35,29 +34,34 @@ pub fn create_control_socket(path: P, f: F) -> Result() { - Ok(p) => p, - Err(err) => bail!("unable to parse json value - {}", err), + let abort_future = super::last_worker_future().map_err(|_| {}); + + tokio::spawn( + tokio::io::lines(std::io::BufReader::new(rx)) + .map_err(move |err| { eprintln!("control socket {:?} read error: {}", path, err); }) + .and_then(move |cmd| { + let res = try_block!({ + let param = match cmd.parse::() { + Ok(p) => p, + Err(err) => bail!("unable to parse json value - {}", err), + }; + + f1(param) + }); + + let resp = match res { + Ok(v) => format!("OK: {}\n", v), + Err(err) => format!("ERROR: {}\n", err), }; - - f1(param) - }); - - let resp = match res { - Ok(v) => format!("OK: {}\n", v), - Err(err) => format!("ERROR: {}\n", err), - }; - Ok(resp) - }) - .for_each(move |resp| { - tx.write_all(resp.as_bytes()) - .map_err(|err| { eprintln!("control socket {:?} write response error: {}", path2, err); }) - }) - + Ok(resp) + }) + .for_each(move |resp| { + tx.write_all(resp.as_bytes()) + .map_err(|err| { eprintln!("control socket {:?} write response error: {}", path2, err); }) + }) + .select(abort_future) + .then(move |_| { Ok(()) }) + ) }); let abort_future = super::last_worker_future().map_err(|_| {}); From dfb73ee28689689847fde30a4594d6628594786e Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Wed, 10 Apr 2019 12:42:24 +0200 Subject: [PATCH 078/344] src/server/worker_task.rs: implement abort_worker (via command_socket) --- src/server/command_socket.rs | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/server/command_socket.rs b/src/server/command_socket.rs index 92a1322c..57a54a3a 100644 --- a/src/server/command_socket.rs +++ b/src/server/command_socket.rs @@ -70,3 +70,52 @@ pub fn create_control_socket(path: P, f: F) -> Result( + path: P, + params: Value +) -> impl Future + where P: Into, + +{ + let path: PathBuf = path.into(); + + tokio::net::UnixStream::connect(path) + .map_err(move |err| format_err!("control socket connect failed - {}", err)) + .and_then(move |conn| { + + let (rx, tx) = conn.split(); + + let mut command_string = params.to_string(); + command_string.push('\n'); + + tokio::io::write_all(tx, command_string) + .and_then(|(tx,_)| tokio::io::shutdown(tx)) + .map_err(|err| format_err!("control socket write error - {}", err)) + .and_then(move |_| { + tokio::io::lines(std::io::BufReader::new(rx)) + .into_future() + .then(|test| { + match test { + Ok((Some(data), _)) => { + if data.starts_with("OK: ") { + match data[4..].parse::() { + Ok(v) => Ok(v), + Err(err) => bail!("unable to parse json response - {}", err), + } + } else if data.starts_with("ERROR: ") { + bail!("{}", &data[7..]); + } else { + bail!("unable to parse response: {}", data); + } + } + Ok((None, _)) => { + bail!("no response"); + } + Err((err, _)) => Err(Error::from(err)), + } + }) + }) + }) +} From 04f7276b1a438e9258ec918b50908d4280996c9d Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Wed, 10 Apr 2019 15:17:11 +0200 Subject: [PATCH 079/344] tools/daemon: dup the TcpListener file descriptor Now that we let hyper shutdown gracefully we need an owned version of the listening socket to prevent it from closing before running the reload preparations. Signed-off-by: Wolfgang Bumiller --- src/tools/daemon.rs | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index de679520..432756d8 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -1,7 +1,7 @@ //! Helpers for daemons/services. use std::ffi::CString; -use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; +use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; use std::os::unix::ffi::OsStrExt; use std::panic::UnwindSafe; @@ -9,16 +9,16 @@ use failure::*; use tokio::prelude::*; use crate::server; -use crate::tools::fd_change_cloexec; +use crate::tools::{fd_change_cloexec, self}; // Unfortunately FnBox is nightly-only and Box is unusable, so just use Box... -pub type BoxedStoreFunc = Box Result + UnwindSafe + Send>; +pub type BoxedStoreFunc = Box Result + UnwindSafe + Send>; /// Helper trait to "store" something in the environment to be re-used after re-executing the /// service on a reload. pub trait Reloadable: Sized { fn restore(var: &str) -> Result; - fn get_store_func(&self) -> BoxedStoreFunc; + fn get_store_func(&self) -> Result; } /// Manages things to be stored and reloaded upon reexec. @@ -58,13 +58,13 @@ impl Reloader { self.pre_exec.push(PreExecEntry { name, - store_fn: res.get_store_func(), + store_fn: res.get_store_func()?, }); Ok(res) } fn pre_exec(self) -> Result<(), Error> { - for item in self.pre_exec { + for mut item in self.pre_exec { std::env::set_var(item.name, (item.store_fn)()?); } Ok(()) @@ -124,12 +124,15 @@ impl Reloadable for tokio::net::TcpListener { // NOTE: The socket must not be closed when the store-function is called: // FIXME: We could become "independent" of the TcpListener and its reference to the file // descriptor by `dup()`ing it (and check if the listener still exists via kcmp()?) - fn get_store_func(&self) -> BoxedStoreFunc { - let fd = self.as_raw_fd(); - Box::new(move || { - fd_change_cloexec(fd, false)?; - Ok(fd.to_string()) - }) + fn get_store_func(&self) -> Result { + let mut fd_opt = Some(tools::Fd( + nix::fcntl::fcntl(self.as_raw_fd(), nix::fcntl::FcntlArg::F_DUPFD_CLOEXEC(0))? + )); + Ok(Box::new(move || { + let fd = fd_opt.take().unwrap(); + fd_change_cloexec(fd.as_raw_fd(), false)?; + Ok(fd.into_raw_fd().to_string()) + })) } fn restore(var: &str) -> Result { From 3e4290e9569a96a0b5944dd8ceb886db6659bf91 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Thu, 11 Apr 2019 10:51:59 +0200 Subject: [PATCH 080/344] src/server/command_socket.rs: check control socket permissions --- src/server/command_socket.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/server/command_socket.rs b/src/server/command_socket.rs index 57a54a3a..2f1af86d 100644 --- a/src/server/command_socket.rs +++ b/src/server/command_socket.rs @@ -11,6 +11,8 @@ use std::io::Write; use std::path::PathBuf; use serde_json::Value; use std::sync::Arc; +use std::os::unix::io::AsRawFd; +use nix::sys::socket; /// Listens on a Unix Socket to handle simple command asynchronously pub fn create_control_socket(path: P, f: F) -> Result, Error> @@ -26,6 +28,21 @@ pub fn create_control_socket(path: P, f: F) -> Result { + let mygid = unsafe { libc::getgid() }; + if !(cred.uid() == 0 || cred.gid() == mygid) { + bail!("no permissions for {:?}", cred); + } + } + Err(err) => bail!("no permissions - unable to read peer credential - {}", err), + } + Ok(conn) + }) .map_err(move |err| { eprintln!("failed to accept on control socket {:?}: {}", path2, err); }) .for_each(move |conn| { let f1 = f.clone(); From e6bdfe06743ce22bc337c31201756b3659f353e4 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Tue, 16 Apr 2019 10:36:04 +0200 Subject: [PATCH 081/344] api_schema: allow generic api handler functions Signed-off-by: Wolfgang Bumiller --- src/server/rest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index 1cce06ed..eda55f36 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -239,7 +239,7 @@ fn handle_sync_api_request( let resp = params .and_then(move |params| { let mut delay = false; - let resp = match (info.handler)(params, info, &mut rpcenv) { + let resp = match (info.handler.as_ref().unwrap())(params, info, &mut rpcenv) { Ok(data) => (formatter.format_result)(data, &rpcenv), Err(err) => { if let Some(httperr) = err.downcast_ref::() { From efd898a71c08a83e100cc06b4fa3822f6bde776e Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Thu, 25 Apr 2019 08:35:01 +0000 Subject: [PATCH 082/344] tools/daemon: add sd_notify wrapper Signed-off-by: Wolfgang Bumiller --- src/tools/daemon.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index 432756d8..870e5610 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -1,6 +1,7 @@ //! Helpers for daemons/services. use std::ffi::CString; +use std::os::raw::{c_char, c_int}; use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd}; use std::os::unix::ffi::OsStrExt; use std::panic::UnwindSafe; @@ -187,3 +188,34 @@ where .map_err(|_| ()) ) } + +#[link(name = "systemd")] +extern "C" { + fn sd_notify(unset_environment: c_int, state: *const c_char) -> c_int; +} + +pub enum SystemdNotify { + Ready, + Reloading, + Stopping, + Status(String), + MainPid(nix::unistd::Pid), +} + +pub fn systemd_notify(state: SystemdNotify) -> Result<(), Error> { + let message = match state { + SystemdNotify::Ready => CString::new("READY=1"), + SystemdNotify::Reloading => CString::new("RELOADING=1"), + SystemdNotify::Stopping => CString::new("STOPPING=1"), + SystemdNotify::Status(msg) => CString::new(format!("STATUS={}", msg)), + SystemdNotify::MainPid(pid) => CString::new(format!("MAINPID={}", pid)), + }?; + let rc = unsafe { sd_notify(0, message.as_ptr()) }; + if rc < 0 { + bail!( + "systemd_notify failed: {}", + std::io::Error::from_raw_os_error(-rc), + ); + } + Ok(()) +} From 30150eef0f210728895d7a8f901dc727f9b74322 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Thu, 25 Apr 2019 08:38:26 +0000 Subject: [PATCH 083/344] use service Type=notify Signed-off-by: Wolfgang Bumiller --- src/tools/daemon.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index 870e5610..9dc3dc8e 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -103,6 +103,9 @@ impl Reloader { } Ok(ForkResult::Parent { child }) => { eprintln!("forked off a new server (pid: {})", child); + if let Err(e) = systemd_notify(SystemdNotify::MainPid(child)) { + log::error!("failed to notify systemd about the new main pid: {}", e); + } Ok(()) } Err(e) => { @@ -178,8 +181,12 @@ where crate::tools::request_shutdown(); // make sure we are in shutdown mode if server::is_reload_request() { log::info!("daemon reload..."); + if let Err(e) = systemd_notify(SystemdNotify::Reloading) { + log::error!("failed to notify systemd about the state change: {}", e); + } if let Err(e) = reloader.take().unwrap().fork_restart() { log::error!("error during reload: {}", e); + let _ = systemd_notify(SystemdNotify::Status(format!("error during reload"))); } } else { log::info!("daemon shutting down..."); From 88aaa1841aea48e783514601ffd9635265dc8e59 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Thu, 25 Apr 2019 11:00:02 +0000 Subject: [PATCH 084/344] use double-fork for reload To ensure the new process' parent is pid 1, so systemd won't complain about supervising a process it does not own. Fixes the following log spam on reloads: Apr 25 10:50:54 deb-dev systemd[1]: proxmox-backup.service: Supervising process 1625 which is not our child. We'll most likely not notice when it exits. Signed-off-by: Wolfgang Bumiller --- src/tools/daemon.rs | 57 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/src/tools/daemon.rs b/src/tools/daemon.rs index 9dc3dc8e..af71828b 100644 --- a/src/tools/daemon.rs +++ b/src/tools/daemon.rs @@ -11,6 +11,8 @@ use tokio::prelude::*; use crate::server; use crate::tools::{fd_change_cloexec, self}; +use crate::tools::read::*; +use crate::tools::write::*; // Unfortunately FnBox is nightly-only and Box is unusable, so just use Box... pub type BoxedStoreFunc = Box Result + UnwindSafe + Send>; @@ -86,15 +88,43 @@ impl Reloader { new_args.push(CString::new(arg.as_bytes())?); } + // Synchronisation pipe: + let (pin, pout) = super::pipe()?; + // Start ourselves in the background: use nix::unistd::{fork, ForkResult}; match fork() { Ok(ForkResult::Child) => { - // At this point we call pre-exec helpers. We must be certain that if they fail for - // whatever reason we can still call `_exit()`, so use catch_unwind. - match std::panic::catch_unwind(move || self.do_exec(exe, new_args)) { - Ok(_) => eprintln!("do_exec returned unexpectedly!"), - Err(_) => eprintln!("panic in re-exec"), + // Double fork so systemd can supervise us without nagging... + match fork() { + Ok(ForkResult::Child) => { + std::mem::drop(pin); + // At this point we call pre-exec helpers. We must be certain that if they fail for + // whatever reason we can still call `_exit()`, so use catch_unwind. + match std::panic::catch_unwind(move || { + let mut pout = unsafe { + std::fs::File::from_raw_fd(pout.into_raw_fd()) + }; + let pid = nix::unistd::Pid::this(); + if let Err(e) = pout.write_value(&pid.as_raw()) { + log::error!("failed to send new server PID to parent: {}", e); + unsafe { + libc::_exit(-1); + } + } + std::mem::drop(pout); + self.do_exec(exe, new_args) + }) + { + Ok(_) => eprintln!("do_exec returned unexpectedly!"), + Err(_) => eprintln!("panic in re-exec"), + } + } + Ok(ForkResult::Parent { child }) => { + std::mem::drop((pin, pout)); + log::debug!("forked off a new server (second pid: {})", child); + } + Err(e) => log::error!("fork() failed, restart delayed: {}", e), } // No matter how we managed to get here, this is the time where we bail out quickly: unsafe { @@ -102,14 +132,27 @@ impl Reloader { } } Ok(ForkResult::Parent { child }) => { - eprintln!("forked off a new server (pid: {})", child); + log::debug!("forked off a new server (first pid: {}), waiting for 2nd pid", child); + std::mem::drop(pout); + let mut pin = unsafe { + std::fs::File::from_raw_fd(pin.into_raw_fd()) + }; + let child = nix::unistd::Pid::from_raw(match pin.read_value() { + Ok(v) => v, + Err(e) => { + log::error!("failed to receive pid of double-forked child process: {}", e); + // systemd will complain but won't kill the service... + return Ok(()); + } + }); + if let Err(e) = systemd_notify(SystemdNotify::MainPid(child)) { log::error!("failed to notify systemd about the new main pid: {}", e); } Ok(()) } Err(e) => { - eprintln!("fork() failed, restart delayed: {}", e); + log::error!("fork() failed, restart delayed: {}", e); Ok(()) } } From 83f663b7a38b17ad1d408e1193dcc4f0f8ce4e96 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 30 Apr 2019 10:21:48 +0200 Subject: [PATCH 085/344] src/server/state.rs: use new BroadcastData helper --- src/server/state.rs | 53 ++++++++++----------------------------------- 1 file changed, 12 insertions(+), 41 deletions(-) diff --git a/src/server/state.rs b/src/server/state.rs index f7ee4300..3f8e47e6 100644 --- a/src/server/state.rs +++ b/src/server/state.rs @@ -5,10 +5,9 @@ use std::sync::Mutex; use futures::*; use futures::stream::Stream; -use tokio::sync::oneshot; use tokio_signal::unix::{Signal, SIGHUP, SIGINT}; -use crate::tools; +use crate::tools::{self, BroadcastData}; #[derive(PartialEq, Copy, Clone, Debug)] pub enum ServerMode { @@ -18,8 +17,8 @@ pub enum ServerMode { pub struct ServerState { pub mode: ServerMode, - pub shutdown_listeners: Vec>, - pub last_worker_listeners: Vec>, + pub shutdown_listeners: BroadcastData<()>, + pub last_worker_listeners: BroadcastData<()>, pub worker_count: usize, pub reload_request: bool, } @@ -28,8 +27,8 @@ pub struct ServerState { lazy_static! { static ref SERVER_STATE: Mutex = Mutex::new(ServerState { mode: ServerMode::Normal, - shutdown_listeners: vec![], - last_worker_listeners: vec![], + shutdown_listeners: BroadcastData::new(), + last_worker_listeners: BroadcastData::new(), worker_count: 0, reload_request: false, }); @@ -85,36 +84,22 @@ pub fn server_shutdown() { data.mode = ServerMode::Shutdown; - notify_listeners(&mut data.shutdown_listeners); + data.shutdown_listeners.notify_listeners(Ok(())); drop(data); // unlock check_last_worker(); } -pub fn shutdown_future() -> oneshot::Receiver<()> { - let (tx, rx) = oneshot::channel::<()>(); - +pub fn shutdown_future() -> impl Future { let mut data = SERVER_STATE.lock().unwrap(); - match data.mode { - ServerMode::Normal => { data.shutdown_listeners.push(tx); }, - ServerMode::Shutdown => { let _ = tx.send(()); }, - } - - rx + data.shutdown_listeners.listen() } -pub fn last_worker_future() -> oneshot::Receiver<()> { - let (tx, rx) = oneshot::channel::<()>(); +pub fn last_worker_future() -> impl Future { let mut data = SERVER_STATE.lock().unwrap(); - if data.mode == ServerMode::Shutdown && data.worker_count == 0 { - let _ = tx.send(()); - } else { - data.last_worker_listeners.push(tx); - } - - rx + data.last_worker_listeners.listen() } pub fn set_worker_count(count: usize) { @@ -123,7 +108,7 @@ pub fn set_worker_count(count: usize) { if !(data.mode == ServerMode::Shutdown && data.worker_count == 0) { return; } - notify_listeners(&mut data.last_worker_listeners); + data.last_worker_listeners.notify_listeners(Ok(())); } @@ -133,19 +118,5 @@ pub fn check_last_worker() { if !(data.mode == ServerMode::Shutdown && data.worker_count == 0) { return; } - notify_listeners(&mut data.last_worker_listeners); -} - -fn notify_listeners(list: &mut Vec>) { - loop { - match list.pop() { - None => { break; }, - Some(ch) => { - println!("SEND ABORT"); - if let Err(_) = ch.send(()) { - eprintln!("SEND ABORT failed"); - } - }, - } - } + data.last_worker_listeners.notify_listeners(Ok(())); } From e53d4dadaad11d66032df5626403c1bb7c7d0df9 Mon Sep 17 00:00:00 2001 From: Dietmar Maurer Date: Tue, 7 May 2019 09:44:34 +0200 Subject: [PATCH 086/344] move normalize_path to tools::normalize_uri_path --- src/server/rest.rs | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/src/server/rest.rs b/src/server/rest.rs index eda55f36..cdd8f6fb 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -488,29 +488,6 @@ fn check_auth(method: &hyper::Method, ticket: &Option, token: &Option