From b5c05fc85cde6faccea04427c4b3ff43c1754c88 Mon Sep 17 00:00:00 2001 From: Wolfgang Bumiller Date: Thu, 6 Jun 2019 15:21:58 +0200 Subject: [PATCH] import proxmox-api-macro crate Signed-off-by: Wolfgang Bumiller --- Cargo.toml | 1 + proxmox-api-macro/Cargo.toml | 24 ++ proxmox-api-macro/src/api_def.rs | 91 ++++ proxmox-api-macro/src/api_macro.rs | 577 ++++++++++++++++++++++++++ proxmox-api-macro/src/lib.rs | 140 +++++++ proxmox-api-macro/src/parsing.rs | 261 ++++++++++++ proxmox-api-macro/src/router_macro.rs | 371 +++++++++++++++++ proxmox-api-macro/src/util.rs | 19 + proxmox-api-macro/tests/basic.rs | 136 ++++++ proxmox/Cargo.toml | 1 + proxmox/src/lib.rs | 8 +- 11 files changed, 1628 insertions(+), 1 deletion(-) create mode 100644 proxmox-api-macro/Cargo.toml create mode 100644 proxmox-api-macro/src/api_def.rs create mode 100644 proxmox-api-macro/src/api_macro.rs create mode 100644 proxmox-api-macro/src/lib.rs create mode 100644 proxmox-api-macro/src/parsing.rs create mode 100644 proxmox-api-macro/src/router_macro.rs create mode 100644 proxmox-api-macro/src/util.rs create mode 100644 proxmox-api-macro/tests/basic.rs diff --git a/Cargo.toml b/Cargo.toml index efd01e25..5b182f9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,5 +2,6 @@ members = [ "proxmox-tools", "proxmox-api", + "proxmox-api-macro", "proxmox", ] diff --git a/proxmox-api-macro/Cargo.toml b/proxmox-api-macro/Cargo.toml new file mode 100644 index 00000000..6f767fa8 --- /dev/null +++ b/proxmox-api-macro/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "proxmox-api-macro" +edition = "2018" +version = "0.1.0" +authors = [ "Wolfgang Bumiller " ] + +[lib] +proc-macro = true + +[dependencies] +derive_builder = "0.7" +failure = "0.1" +proc-macro2 = "0.4" +quote = "0.6" +syn = { version = "0.15", features = [ "full" ] } + +[dev-dependencies] +bytes = "0.4" +futures-preview = { version = "0.3.0-alpha.16", features = [ "compat" ] } +http = "0.1" +proxmox-api = { path = "../proxmox-api" } +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" diff --git a/proxmox-api-macro/src/api_def.rs b/proxmox-api-macro/src/api_def.rs new file mode 100644 index 00000000..99b6788f --- /dev/null +++ b/proxmox-api-macro/src/api_def.rs @@ -0,0 +1,91 @@ +use std::collections::HashMap; + +use proc_macro2::{Ident, TokenStream}; + +use derive_builder::Builder; +use failure::{bail, Error}; +use quote::{quote, ToTokens}; + +use super::parsing::Value; + +#[derive(Builder)] +pub struct ParameterDefinition { + pub description: syn::LitStr, + #[builder(default)] + pub validate: Option, + #[builder(default)] + pub minimum: Option, + #[builder(default)] + pub maximum: Option, +} + +impl ParameterDefinition { + pub fn builder() -> ParameterDefinitionBuilder { + ParameterDefinitionBuilder::default() + } + + pub fn from_object(obj: HashMap) -> Result { + let mut def = ParameterDefinition::builder(); + + for (key, value) in obj { + match key.as_str() { + "description" => { + def.description(value.expect_lit_str()?); + } + "validate" => { + def.validate(Some(value.expect_ident()?)); + } + "minimum" => { + def.minimum(Some(value.expect_lit()?)); + } + "maximum" => { + def.maximum(Some(value.expect_lit()?)); + } + other => bail!("invalid key in type definition: {}", other), + } + } + + match def.build() { + Ok(r) => Ok(r), + Err(err) => bail!("{}", err), + } + } + + pub fn add_verifiers( + &self, + name_str: &str, + this: TokenStream, + verifiers: &mut Vec, + ) { + verifiers.push(match self.validate { + Some(ref ident) => quote! { #ident(&#this)?; }, + None => quote! { proxmox_api::ApiType::verify(&#this)?; }, + }); + + if let Some(ref lit) = self.minimum { + let errstr = format!( + "parameter '{}' out of range: (must be >= {})", + name_str, + lit.clone().into_token_stream().to_string(), + ); + verifiers.push(quote! { + if #this < #lit { + bail!("{}", #errstr); + } + }); + } + + if let Some(ref lit) = self.maximum { + let errstr = format!( + "parameter '{}' out of range: (must be <= {})", + name_str, + lit.clone().into_token_stream().to_string(), + ); + verifiers.push(quote! { + if #this > #lit { + bail!("{}", #errstr); + } + }); + } + } +} diff --git a/proxmox-api-macro/src/api_macro.rs b/proxmox-api-macro/src/api_macro.rs new file mode 100644 index 00000000..9d86fe21 --- /dev/null +++ b/proxmox-api-macro/src/api_macro.rs @@ -0,0 +1,577 @@ +use std::collections::HashMap; + +use proc_macro2::{Delimiter, Ident, Span, TokenStream, TokenTree}; + +use failure::{bail, format_err, Error}; +use quote::{quote, ToTokens}; +use syn::Token; + +use super::api_def::ParameterDefinition; +use super::parsing::*; + +pub fn api_macro(attr: TokenStream, item: TokenStream) -> Result { + let definition = attr + .into_iter() + .next() + .expect("expected api definition in braces"); + + let definition = match definition { + TokenTree::Group(ref group) if group.delimiter() == Delimiter::Brace => group.stream(), + _ => bail!("expected api definition in braces"), + }; + + let definition = parse_object(definition)?; + + // Now parse the item, based on which we decide whether this is an API method which needs a + // wrapper, or an API type which needs an ApiType implementation! + let item: syn::Item = syn::parse2(item).unwrap(); + + match item { + syn::Item::Struct(ref itemstruct) => { + let extra = handle_struct(definition, itemstruct)?; + let mut output = item.into_token_stream(); + output.extend(extra); + Ok(output) + } + syn::Item::Fn(func) => handle_function(definition, func), + _ => bail!("api macro currently only applies to structs and functions"), + } +} + +fn handle_function( + mut definition: HashMap, + mut item: syn::ItemFn, +) -> Result { + if item.decl.generics.lt_token.is_some() { + bail!("cannot use generic functions for api macros currently"); + // Not until we stabilize our generated representation! + } + + // We cannot use #{foo.bar} in quote!, we can only use #foo, so these must all be local + // variables. (I'd prefer a struct and using `#{func.description}`, `#{func.protected}` etc. + // but that's not supported. + + let fn_api_description = definition.remove("description") + .ok_or_else(|| format_err!("missing 'description' in method definition"))? + .expect_lit_str()?; + + let fn_api_protected = definition.remove("protected") + .map(|v| v.expect_lit_bool()) + .transpose()? + .unwrap_or_else(|| syn::LitBool { + span: Span::call_site(), + value: false, + }); + + let fn_api_reload_timezone = definition.remove("reload_timezone") + .map(|v| v.expect_lit_bool()) + .transpose()? + .unwrap_or_else(|| syn::LitBool { + span: Span::call_site(), + value: false, + }); + + let vis = std::mem::replace(&mut item.vis, syn::Visibility::Inherited); + let span = item.ident.span(); + let name_str = item.ident.to_string(); + let impl_str = format!("{}_impl", name_str); + let impl_ident = Ident::new(&impl_str, span); + let name = std::mem::replace(&mut item.ident, impl_ident.clone()); + let mut return_type = match item.decl.output { + syn::ReturnType::Default => syn::Type::Tuple(syn::TypeTuple { + paren_token: syn::token::Paren { span: Span::call_site() }, + elems: syn::punctuated::Punctuated::new(), + }), + syn::ReturnType::Type(_, ref ty) => ty.as_ref().clone(), + }; + + let mut extracted_args = syn::punctuated::Punctuated::::new(); + let mut passed_args = syn::punctuated::Punctuated::::new(); + let mut arg_extraction = Vec::new(); + + let inputs = item.decl.inputs.clone(); + for arg in item.decl.inputs.iter() { + let arg = match arg { + syn::FnArg::Captured(ref arg) => arg, + other => bail!("unhandled type of method parameter ({:?})", other), + }; + + let name = match &arg.pat { + syn::Pat::Ident(name) => &name.ident, + other => bail!("invalid kind of parameter pattern: {:?}", other), + }; + passed_args.push(name.clone()); + let name_str = name.to_string(); + + let arg_name = Ident::new(&format!("arg_{}", name_str), name.span()); + extracted_args.push(arg_name.clone()); + + arg_extraction.push(quote! { + let #arg_name = ::serde_json::from_value( + args + .remove(#name_str) + .unwrap_or(::serde_json::Value::Null) + )?; + }); + } + + use std::iter::FromIterator; + let arg_extraction = TokenStream::from_iter(arg_extraction.into_iter()); + + // The router expects an ApiMethod, or more accurately, an object implementing ApiMethodInfo. + // This is because we need access to a bunch of additional attributes of the functions both at + // runtime and when doing command line parsing/completion/help output. + // + // When manually implementing methods, we usually just write them out as an `ApiMethod` which + // is a type requiring all the info made available by the ApiMethodInfo trait as members. + // + // While we could just generate a `const ApiMethod` for our functions, we would like them to + // also be usable as functions simply because the syntax we use to create them makes them + // *look* like functions, so it would be nice if they also *behaved* like real functions. + // + // Therefore all the fields of an ApiMethod are accessed via methods from the ApiMethodInfo + // trait and we perform the same trick lazy_static does: Create a new type implementing + // ApiMethodInfo, and make its instance Deref to an actual function. + // This way the function can still be used normally. Validators for parameters will be + // executed, serialization happens only when coming from the method's `handler`. + + let name_str = name.to_string(); + let struct_name = Ident::new(&super::util::to_camel_case(&name_str), name.span()); + let mut body = Vec::new(); + body.push(quote! { + // This is our helper struct which Derefs to a wrapper of our original function, which + // applies the added validators. + #vis struct #struct_name(); + + #[allow(non_upper_case_globals)] + const #name: &#struct_name = &#struct_name(); + + // Namespace some of our code into the helper type: + impl #struct_name { + // This is the original function, renamed to `#impl_ident` + #item + + // This is the handler used by our router, which extracts the parameters out of a + // serde_json::Value, running the actual method, then serializing the output into an + // API response. + // + // FIXME: For now this always returns status 200, we're going to have to figure out how + // to use different success status values. + // This could be a simple optional parameter to just replace the number, or + // alternatively we could just recognize functions returning a http::Response and not + // perform the serialization/http::Response-building automatically. + // (Alternatively we could do exactly that with a trait so we don't have to parse the + // return type?) + fn wrapped_api_handler(args: ::serde_json::Value) -> ::proxmox_api::ApiFuture { + async fn handler(mut args: ::serde_json::Value) -> ::proxmox_api::ApiOutput { + let mut empty_args = ::serde_json::map::Map::new(); + let args = args.as_object_mut() + .unwrap_or(&mut empty_args); + + #arg_extraction + + if !args.is_empty() { + let mut extra = String::new(); + for arg in args.keys() { + if !extra.is_empty() { + extra.push_str(", "); + } + extra.push_str(arg); + } + bail!("unexpected extra parameters: {}", extra); + } + + let output = #struct_name::#impl_ident(#extracted_args).await?; + ::proxmox_api::IntoApiOutput::into_api_output(output) + } + Box::pin(handler(args)) + } + } + }); + + if item.asyncness.is_some() { + // An async function is expected to return its value, so we wrap it a bit: + + body.push(quote! { + // Our helper type derefs to a wrapper performing input validation and returning a + // Pin>. + // Unfortunately we cannot return the actual function since that won't work for + // `async fn`, since an `async fn` cannot appear as a return type :( + impl ::std::ops::Deref for #struct_name { + type Target = fn(#inputs) -> ::std::pin::Pin + >>; + + fn deref(&self) -> &Self::Target { + const FUNC: fn(#inputs) -> ::std::pin::Pin>> = |#inputs| { + Box::pin(#struct_name::#impl_ident(#passed_args)) + }; + &FUNC + } + } + }); + } else { + // Non async fn must return an ApiFuture already! + return_type = syn::Type::Verbatim(syn::TypeVerbatim { + tts: definition.remove("returns") + .ok_or_else(|| format_err!( + "non async-fn must return a Response \ + and specify its return type via the `returns` property", + ))? + .expect_ident()? + .into_token_stream(), + }); + + body.push(quote! { + // Our helper type derefs to a wrapper performing input validation and returning a + // Pin>. + // Unfortunately we cannot return the actual function since that won't work for + // `async fn`, since an `async fn` cannot appear as a return type :( + impl ::std::ops::Deref for #struct_name { + type Target = fn(#inputs) -> ::proxmox_api::ApiFuture; + + fn deref(&self) -> &Self::Target { + const FUNC: fn(#inputs) -> ::proxmox_api::ApiFuture = |#inputs| { + #struct_name::#impl_ident(#passed_args) + }; + &FUNC + } + } + }); + } + + body.push(quote! { + // We now need to provide all the info required for routing, command line completion, API + // documentation, etc. + // + // Note that technically we don't need the `description` member in this trait, as this is + // mostly used at compile time for documentation! + impl ::proxmox_api::ApiMethodInfo for #struct_name { + fn description(&self) -> &'static str { + #fn_api_description + } + + fn parameters(&self) -> &'static [::proxmox_api::Parameter] { + // FIXME! + &[] + } + + fn return_type(&self) -> &'static ::proxmox_api::TypeInfo { + <#return_type as ::proxmox_api::ApiType>::type_info() + } + + fn protected(&self) -> bool { + #fn_api_protected + } + + fn reload_timezone(&self) -> bool { + #fn_api_reload_timezone + } + + fn handler(&self) -> fn(::serde_json::Value) -> ::proxmox_api::ApiFuture { + #struct_name::wrapped_api_handler + } + } + }); + + let body = TokenStream::from_iter(body); + //dbg!("{}", &body); + Ok(body) +} + +fn handle_struct( + definition: HashMap, + item: &syn::ItemStruct, +) -> Result { + if item.generics.lt_token.is_some() { + bail!("generic types are currently not supported"); + } + + let name = &item.ident; + + match item.fields { + syn::Fields::Unit => bail!("unit types are not allowed"), + syn::Fields::Unnamed(ref fields) => handle_struct_unnamed(definition, name, fields), + syn::Fields::Named(ref fields) => handle_struct_named(definition, name, fields), + } +} + +fn handle_struct_unnamed( + definition: HashMap, + name: &Ident, + item: &syn::FieldsUnnamed, +) -> Result { + let fields = &item.unnamed; + if fields.len() != 1 { + bail!("only 1 unnamed field is currently allowed for api types"); + } + + //let field = fields.first().unwrap().value(); + + let apidef = ParameterDefinition::from_object(definition)?; + + let validator = match apidef.validate { + Some(ident) => quote! { #ident(&self.0) }, + None => quote! { proxmox_api::ApiType::verify(&self.0) }, + }; + + Ok(quote! { + impl ::proxmox_api::ApiType for #name { + fn type_info() -> &'static ::proxmox_api::TypeInfo { + const INFO: ::proxmox_api::TypeInfo = ::proxmox_api::TypeInfo { + name: stringify!(#name), + description: "FIXME", + complete_fn: None, // FIXME! + }; + &INFO + } + + fn verify(&self) -> Result<(), Error> { + #validator + } + } + }) +} + +fn handle_struct_named( + definition: HashMap, + name: &Ident, + item: &syn::FieldsNamed, +) -> Result { + let mut verify_entries = None; + let mut description = None; + for (key, value) in definition { + match key.as_str() { + "fields" => { + verify_entries = Some(handle_named_struct_fields(item, value.expect_object()?)?); + } + "description" => { + description = Some(value.expect_lit_str()?); + } + other => bail!("unknown api definition field: {}", other), + } + } + + let description = description + .ok_or_else(|| format_err!("missing 'description' for type {}", name.to_string()))?; + + use std::iter::FromIterator; + let verifiers = TokenStream::from_iter( + verify_entries.ok_or_else(|| format_err!("missing 'fields' definition for struct"))?, + ); + + Ok(quote! { + impl ::proxmox_api::ApiType for #name { + fn type_info() -> &'static ::proxmox_api::TypeInfo { + const INFO: ::proxmox_api::TypeInfo = ::proxmox_api::TypeInfo { + name: stringify!(#name), + description: #description, + complete_fn: None, // FIXME! + }; + &INFO + } + + fn verify(&self) -> Result<(), Error> { + #verifiers + Ok(()) + } + } + }) +} + +fn handle_named_struct_fields( + item: &syn::FieldsNamed, + mut field_def: HashMap, +) -> Result, Error> { + let mut verify_entries = Vec::new(); + + for field in item.named.iter() { + let name = &field.ident; + let name_str = name + .as_ref() + .expect("field name in struct of named fields") + .to_string(); + + let this = quote! { self.#name }; + + let def = field_def + .remove(&name_str) + .ok_or_else(|| format_err!("missing field in definition: '{}'", name_str))? + .expect_object()?; + + let def = ParameterDefinition::from_object(def)?; + def.add_verifiers(&name_str, this, &mut verify_entries); + } + + if !field_def.is_empty() { + // once SliceConcatExt is stable we can join(",") on the fields... + let mut missing = String::new(); + for key in field_def.keys() { + if !missing.is_empty() { + missing.push_str(", "); + } + missing.push_str(&key); + } + bail!( + "the following struct fields are not handled in the api definition: {}", + missing + ); + } + + Ok(verify_entries) +} + +//fn parse_api_definition(def: &mut ApiDefinitionBuilder, tokens: TokenStream) -> Result<(), Error> { +// let obj = parse_object(tokens)?; +// for (key, value) in obj { +// match (key.as_str(), value) { +// ("parameters", Value::Object(members)) => { +// def.parameters(handle_parameter_list(members)?); +// } +// ("parameters", other) => bail!("not a parameter list: {:?}", other), +// ("unauthenticated", value) => { +// def.unauthenticated(value.to_bool()?); +// } +// (key, _) => bail!("unexpected api definition parameter: {}", key), +// } +// } +// Ok(()) +//} +// +//fn handle_parameter_list(obj: HashMap) -> Result, Error> { +// let mut out = HashMap::new(); +// +// for (key, value) in obj { +// let parameter = match value { +// Value::Description(ident, description) => { +// make_default_parameter(&ident.to_string(), description)? +// } +// Value::Optional(ident, description) => { +// let mut parameter = make_default_parameter(&ident.to_string(), description)?; +// parameter.optional = true; +// parameter +// } +// Value::Object(obj) => handle_parameter(&key, obj)?, +// other => bail!("expected parameter type for {}, at {:?}", key, other), +// }; +// +// if out.insert(key.clone(), parameter).is_some() { +// bail!("duplicate parameter entry: {}", key); +// } +// } +// +// Ok(out) +//} +// +//fn make_default_parameter(ident: &str, description: String) -> Result { +// let mut parameter = Parameter::default(); +// parameter.description = description; +// parameter.parameter_type = match ident { +// "bool" => ParameterType::Bool, +// "string" => ParameterType::String(StringParameter::default()), +// "float" => ParameterType::Float(FloatParameter::default()), +// "object" => { +// let mut obj = ObjectParameter::default(); +// obj.allow_unknown_keys = true; +// ParameterType::Object(obj) +// } +// other => bail!("invalid parameter type name: {}", other), +// }; +// Ok(parameter) +//} +// +//fn handle_parameter(key: &str, mut obj: HashMap) -> Result { +// let mut builder = ParameterBuilder::default(); +// +// builder.name(key.to_string()); +// +// if let Some(optional) = obj.remove("optional") { +// builder.optional(optional.to_bool()?); +// } else { +// builder.optional(false); +// } +// +// builder.description( +// obj.remove("description") +// .ok_or_else(|| { +// format_err!("`description` field is not optional in parameter definition") +// })? +// .to_string()?, +// ); +// +// let type_name = obj +// .remove("type") +// .ok_or_else(|| format_err!("missing type name in parameter {}", key))?; +// +// let type_name = match type_name { +// Value::Ident(ident) => ident.to_string(), +// other => bail!("bad type name for parameter {}: {:?}", key, other), +// }; +// +// builder.parameter_type(match type_name.as_str() { +// "integer" => handle_integer_type(&mut obj)?, +// "float" => handle_float_type(&mut obj)?, +// "string" => handle_string_type(&mut obj)?, +// _ => bail!("unknown type name: {}", type_name), +// }); +// +// if !obj.is_empty() { +// bail!( +// "unknown keys for type {}: {}", +// type_name, +// obj.keys().fold(String::new(), |acc, key| { +// if acc.is_empty() { +// key.to_string() +// } else { +// format!("{}, {}", acc, key) +// } +// }) +// ) +// } +// +// builder.build().map_err(|e| format_err!("{}", e)) +//} +// +//fn handle_string_type(obj: &mut HashMap) -> Result { +// let mut param = StringParameter::default(); +// +// if let Some(value) = obj.remove("minimum_length") { +// param.minimum_length = Some(value.to_unsigned()?); +// } +// +// if let Some(value) = obj.remove("maximum_length") { +// param.maximum_length = Some(value.to_unsigned()?); +// } +// +// Ok(ParameterType::String(param)) +//} +// +//fn handle_integer_type(obj: &mut HashMap) -> Result { +// let mut param = IntegerParameter::default(); +// +// if let Some(value) = obj.remove("minimum") { +// param.minimum = Some(value.to_integer()?); +// } +// +// if let Some(value) = obj.remove("maximum") { +// param.maximum = Some(value.to_integer()?); +// } +// +// Ok(ParameterType::Integer(param)) +//} +// +//fn handle_float_type(obj: &mut HashMap) -> Result { +// let mut param = FloatParameter::default(); +// +// if let Some(value) = obj.remove("minimum") { +// param.minimum = Some(value.to_float()?); +// } +// +// if let Some(value) = obj.remove("maximum") { +// param.maximum = Some(value.to_float()?); +// } +// +// Ok(ParameterType::Float(param)) +//} diff --git a/proxmox-api-macro/src/lib.rs b/proxmox-api-macro/src/lib.rs new file mode 100644 index 00000000..e820ede2 --- /dev/null +++ b/proxmox-api-macro/src/lib.rs @@ -0,0 +1,140 @@ +#![recursion_limit = "256"] + +extern crate proc_macro; +extern crate proc_macro2; + +use proc_macro::TokenStream; + +mod api_def; +mod parsing; +mod util; + +mod api_macro; +mod router_macro; + +/// This is the `#[api(api definition)]` attribute for functions. An Api definition defines the +/// parameters and return type of an API call. The function will automatically be wrapped in a +/// function taking and returning a json `Value`, while performing validity checks on both input +/// and output. +/// +/// Example: +/// ```ignore +/// #[api({ +/// parameters: { +/// // Short form: [`optional`] TYPE ("description") +/// name: string ("A person's name"), +/// gender: optional string ("A person's gender"), +/// // Long form uses json-ish syntax: +/// coolness: { +/// type: integer, // we don't enclose type names in quotes though... +/// description: "the coolness of a person, using the coolness scale", +/// minimum: 0, +/// maximum: 10, +/// }, +/// // Hyphenated parameters are allowed, but need quotes (due to how proc_macro +/// // TokenStreams work) +/// "is-weird": optional float ("hyphenated names must be enclosed in quotes") +/// }, +/// // TODO: returns: {} +/// })] +/// fn test() { +/// } +/// ``` +#[proc_macro_attribute] +pub fn api(attr: TokenStream, item: TokenStream) -> TokenStream { + match api_macro::api_macro(attr.into(), item.into()) { + Ok(output) => output.into(), + Err(err) => panic!("error in api definition: {}", err), + } +} + +/// The router macro helps to avoid having to type out strangely nested `Router` expressions. +/// +/// Note that without `proc_macro_hack` we currently cannot use macros in expression position, so +/// this cannot be used inline within an expression. +/// +/// Example: +/// ```ignore +/// router!{ +/// let my_router = { +/// /people/{person}: { +/// POST: create_person, +/// GET: get_person, +/// PUT: update_person, +/// DELETE: delete_person, +/// }, +/// /people/{person}/kick: { POST: kick_person }, +/// /groups/{group}: { +/// /: { +/// POST: create_group, +/// PUT: update_group_info, +/// GET: get_group_info, +/// DELETE: delete_group, +/// }, +/// /people/{person}: { +/// POST: add_person_to_group, +/// DELETE: delete_person_from_group, +/// PUT: update_person_details_for_group, +/// GET: get_person_details_from_group, +/// }, +/// }, +/// /other: (an_external_router) +/// }; +/// } +/// ``` +/// +/// The above should produce the following output: +/// ```ignore +/// let my_router = Router::new() +/// .subdir( +/// "people", +/// Router::new() +/// .parameter_subdir( +/// "person", +/// Router::new() +/// .post(create_person) +/// .get(get_person) +/// .put(update_person) +/// .delete(delete_person) +/// .subdir( +/// "kick", +/// Router::new() +/// .post(kick_person) +/// ) +/// ) +/// ) +/// .subdir( +/// "groups", +/// Router::new() +/// .parameter_subdir( +/// "group", +/// Router::new() +/// .post(create_group) +/// .put(update_group_info) +/// .get(get_group_info) +/// .delete(delete_group_info) +/// .subdir( +/// "people", +/// Router::new() +/// .parameter_subdir( +/// "person", +/// Router::new() +/// .post(add_person_to_group) +/// .delete(delete_person_from_group) +/// .put(update_person_details_for_group) +/// .get(get_person_details_from_group) +/// ) +/// ) +/// ) +/// ) +/// .subdir("other", an_external_router) +/// ; +/// ``` +#[proc_macro] +pub fn router(input: TokenStream) -> TokenStream { + // TODO... + match router_macro::router_macro(input.into()) { + Ok(output) => output.into(), + Err(err) => panic!("error in router macro: {}", err), + } +} diff --git a/proxmox-api-macro/src/parsing.rs b/proxmox-api-macro/src/parsing.rs new file mode 100644 index 00000000..eb9cf236 --- /dev/null +++ b/proxmox-api-macro/src/parsing.rs @@ -0,0 +1,261 @@ +use std::collections::HashMap; + +use proc_macro2::{Delimiter, Group, Ident, Span, TokenStream, TokenTree}; + +use failure::{bail, Error}; +use syn::Lit; + +pub type RawTokenIter = proc_macro2::token_stream::IntoIter; +pub type TokenIter = std::iter::Peekable; + +pub fn match_keyword(tokens: &mut TokenIter, keyword: &'static str) -> Result<(), Error> { + if let Some(tt) = tokens.next() { + if let TokenTree::Ident(ident) = tt { + if ident.to_string() == keyword { + return Ok(()); + } + } + } + bail!("expected `{}` keyword", keyword); +} + +pub fn need_ident(tokens: &mut TokenIter) -> Result { + match tokens.next() { + Some(TokenTree::Ident(ident)) => Ok(ident), + other => bail!("expected ident: {:?}", other), + } +} + +pub fn match_punct(tokens: &mut TokenIter, punct: char) -> Result<(), Error> { + if let Some(tt) = tokens.next() { + if let TokenTree::Punct(p) = tt { + if p.as_char() == punct { + return Ok(()); + } + } + } + bail!("expected `{}`", punct); +} + +pub fn need_group(tokens: &mut TokenIter, delimiter: Delimiter) -> Result { + if let Some(TokenTree::Group(group)) = tokens.next() { + if group.delimiter() == delimiter { + return Ok(group); + } + } + bail!("expected group surrounded by {:?}", delimiter); +} + +pub fn match_colon(tokens: &mut TokenIter) -> Result<(), Error> { + match tokens.next() { + Some(TokenTree::Punct(ref punct)) if punct.as_char() == ':' => Ok(()), + Some(other) => bail!("expected colon at {:?}", other.span()), + None => bail!("colon expected"), + } +} + +pub fn maybe_comma(tokens: &mut TokenIter) -> Result { + match tokens.next() { + Some(TokenTree::Punct(ref punct)) if punct.as_char() == ',' => Ok(true), + Some(other) => bail!("expected comma at {:?}", other.span()), + None => Ok(false), + } +} + +pub fn need_comma(tokens: &mut TokenIter) -> Result<(), Error> { + if !maybe_comma(tokens)? { + bail!("comma expected"); + } + Ok(()) +} + +// returns whther there was a comma +pub fn comma_or_end(tokens: &mut TokenIter) -> Result<(), Error> { + if tokens.peek().is_some() { + need_comma(tokens)?; + } + Ok(()) +} + +/// A more relaxed version of Ident which allows hyphens. +pub struct Name(String, Span); + +impl Name { + pub fn new(name: String, span: Span) -> Result { + let beg = name.as_bytes()[0]; + if !(beg.is_ascii_alphanumeric() || beg == b'_') + || !name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') + { + bail!("`{}` is not a valid name", name); + } + Ok(Self(name, span)) + } + + pub fn to_string(&self) -> String { + self.0.clone() + } + + pub fn into_string(self) -> String { + self.0 + } +} + +impl From for Name { + fn from(ident: Ident) -> Name { + Name(ident.to_string(), ident.span()) + } +} + +pub fn need_hyphenated_name(tokens: &mut TokenIter) -> Result { + let start = need_ident(&mut *tokens)?; + finish_hyphenated_name(&mut *tokens, start) +} + +pub fn finish_hyphenated_name(tokens: &mut TokenIter, name: Ident) -> Result { + let span = name.span(); + let mut name = name.to_string(); + + loop { + if let Some(TokenTree::Punct(punct)) = tokens.peek() { + if punct.as_char() == '-' { + name.push('-'); + let _ = tokens.next(); + } else { + break; + } + } else { + break; + } + + // after a hyphen we *need* another text: + match tokens.next() { + Some(TokenTree::Ident(ident)) => name.push_str(&ident.to_string()), + Some(other) => bail!("expected name (possibly with hyphens): {:?}", other), + None => bail!("unexpected end in name"), + } + } + + Ok(Name(name, span)) +} + +// parse an object notation: +// object := '{' [ member * ] '}' +// member := ':' +// member_value := [ "optional" ] ( | | ) +#[derive(Debug)] +pub enum Value { + //Ident(Ident), // eg. `string` or `integer` + //Description(syn::LitStr), // eg. `"some text"` + Ident(Ident), // eg. `foo`, for referencing stuff, may become `expression`? + Literal(syn::Lit), // eg. `123` + Negative(syn::Lit), // eg. `-123` + Object(HashMap), // eg. `{ key: value }` +} + +impl Value { + pub fn expect_lit(self) -> Result { + match self { + Value::Literal(lit) => Ok(lit), + other => bail!("expected string literal, got: {:?}", other), + } + } + + pub fn expect_lit_str(self) -> Result { + match self { + Value::Literal(syn::Lit::Str(lit)) => Ok(lit), + Value::Literal(other) => bail!("expected string literal, got: {:?}", other), + other => bail!("expected string literal, got: {:?}", other), + } + } + + pub fn expect_ident(self) -> Result { + match self { + Value::Ident(ident) => Ok(ident), + other => bail!("expected ident, got: {:?}", other), + } + } + + pub fn expect_object(self) -> Result, Error> { + match self { + Value::Object(obj) => Ok(obj), + other => bail!("expected ident, got: {:?}", other), + } + } + + pub fn expect_lit_bool(self) -> Result { + match self { + Value::Literal(syn::Lit::Bool(lit)) => Ok(lit), + Value::Literal(other) => bail!("expected booleanliteral, got: {:?}", other), + other => bail!("expected boolean literal, got: {:?}", other), + } + } +} + +pub fn parse_object(tokens: TokenStream) -> Result, Error> { + let mut tokens = tokens.into_iter().peekable(); + let mut out = HashMap::new(); + + loop { + if tokens.peek().is_none() { + break; + } + + let key = need_ident_or_string(&mut tokens)?; + match_colon(&mut tokens)?; + + let key_name = key.to_string(); + + let member = match tokens.next() { + Some(TokenTree::Group(group)) => { + if group.delimiter() == Delimiter::Brace { + Value::Object(parse_object(group.stream())?) + } else { + bail!("invalid group delimiter: {:?}", group.delimiter()); + } + } + Some(TokenTree::Punct(ref punct)) if punct.as_char() == '-' => { + if let Some(TokenTree::Literal(literal)) = tokens.next() { + let lit = Lit::new(literal); + match lit { + Lit::Int(_) | Lit::Float(_) => Value::Negative(lit), + _ => bail!("expected literal after unary minus"), + } + } else { + bail!("expected literal value"); + } + } + Some(TokenTree::Literal(literal)) => Value::Literal(Lit::new(literal)), + Some(TokenTree::Ident(ident)) => Value::Ident(ident), + Some(other) => bail!("expected member value at {}", other), + None => bail!("missing member value after {}", key_name), + }; + + if out.insert(key_name.clone(), member).is_some() { + bail!("duplicate entry: {}", key_name); + } + + comma_or_end(&mut tokens)?; + } + + Ok(out) +} + +fn need_ident_or_string(tokens: &mut TokenIter) -> Result { + match tokens.next() { + Some(TokenTree::Ident(ident)) => Ok(ident.into()), + Some(TokenTree::Literal(literal)) => { + let span = literal.span(); + match Lit::new(literal) { + Lit::Str(value) => Ok(Name::new(value.value(), span)?), + _ => bail!("expected ident or string as key: {:?}", span), + } + } + Some(other) => bail!( + "expected colon after key in api definition at {:?}", + other.span() + ), + None => bail!("ident expected"), + } +} diff --git a/proxmox-api-macro/src/router_macro.rs b/proxmox-api-macro/src/router_macro.rs new file mode 100644 index 00000000..6ba7a66e --- /dev/null +++ b/proxmox-api-macro/src/router_macro.rs @@ -0,0 +1,371 @@ +use std::collections::HashMap; + +use proc_macro2::{Delimiter, Ident, TokenStream, TokenTree}; + +use failure::{bail, Error}; +use quote::quote; + +use super::parsing::*; + +pub fn router_macro(input: TokenStream) -> Result { + let mut input = input.into_iter().peekable(); + + let mut out = TokenStream::new(); + + loop { + if input.peek().is_none() { + break; + } + + match_keyword(&mut input, "static")?; + let router_name = need_ident(&mut input)?; + match_punct(&mut input, '=')?; + let content = need_group(&mut input, Delimiter::Brace)?; + + let router = parse_router(content.stream().into_iter().peekable())?; + let router = router.into_token_stream(Some(router_name)); + + out.extend(router); + + match_punct(&mut input, ';')?; + } + + Ok(out) +} + +/// A sub-route entry. This represents subdirectories in a route entry. +/// +/// This can either be a fixed set of directories, or a parameter name, in which case it matches +/// all directory names into the parameter of the specified name. +pub enum SubRoute { + Directories(HashMap), + Parameter(String, Box), +} + +impl SubRoute { + /// Create an ampty directories entry. + fn directories() -> Self { + SubRoute::Directories(HashMap::new()) + } + + /// Create a parameter entry with an empty default router. + fn parameter(name: String) -> Self { + SubRoute::Parameter(name, Box::new(Router::default())) + } +} + +/// A set of operations for a specific directory entry, and an optional sub router. +#[derive(Default)] +pub struct Router { + pub get: Option, + pub put: Option, + pub post: Option, + pub delete: Option, + pub subroute: Option, +} + +/// An entry for a router. +/// +/// While parsing a router we either get a `path: router` key/value entry, or a +/// `method: function_name` entry. +enum Entry { + /// This entry represents a path containing a sub router. + Path(Path), + /// This entry represents a method name. + Method(Ident), +} + +/// The components making up a path. +enum Component { + /// This component is a fixed sub directory name. Eg. `foo` or `baz` in `/foo/{bar}/baz`. + Name(String), + /// This component matches everything into a parameter. Eg. `bar` in `/foo/{bar}/baz`. + Match(String), +} + +/// A path is just a list of components. +type Path = Vec; + +impl Router { + /// Insert a new router at a specific path. + /// + /// Note that this does not allow replacing an already existing router node. + fn insert(&mut self, path: Path, router: Router) -> Result<(), Error> { + let mut at = self; + let mut created = false; + for component in path { + created = false; + match component { + Component::Name(name) => { + let subroute = at.subroute.get_or_insert_with(SubRoute::directories); + match subroute { + SubRoute::Directories(hash) => { + at = hash.entry(name).or_insert_with(|| { + created = true; + Router::default() + }); + } + SubRoute::Parameter(_param, _router) => { + bail!("subdirectory '{}' clashes with parameter matcher", name); + } + } + } + Component::Match(name) => { + let subroute = at.subroute.get_or_insert_with(|| { + created = true; + SubRoute::parameter(name.clone()) + }); + match subroute { + SubRoute::Directories(_) => { + bail!( + "parameter matcher '{}' clashes with existing directory", + name + ); + } + SubRoute::Parameter(existing_name, router) => { + if name != *existing_name { + bail!( + "paramter matcher '{}' clashes with existing name '{}'", + name, + existing_name, + ); + } + at = router.as_mut(); + } + } + } + } + } + + if !created { + bail!("tried to replace existing path in router"); + } + std::mem::replace(at, router); + Ok(()) + } + + fn into_token_stream(self, name: Option) -> TokenStream { + use std::iter::FromIterator; + + use proc_macro2::{Group, Literal, Punct, Spacing, Span}; + + let mut out = vec![ + TokenTree::Ident(Ident::new("Router", Span::call_site())), + TokenTree::Punct(Punct::new(':', Spacing::Joint)), + TokenTree::Punct(Punct::new(':', Spacing::Alone)), + TokenTree::Ident(Ident::new("new", Span::call_site())), + TokenTree::Group(Group::new(Delimiter::Parenthesis, TokenStream::new())), + ]; + + fn add_method(out: &mut Vec, name: &str, func_name: Ident) { + out.push(TokenTree::Punct(Punct::new('.', Spacing::Alone))); + out.push(TokenTree::Ident(Ident::new(name, Span::call_site()))); + out.push(TokenTree::Group(Group::new( + Delimiter::Parenthesis, + TokenStream::from_iter(vec![TokenTree::Ident(func_name)]), + ))); + } + + if let Some(method) = self.get { + add_method(&mut out, "get", method); + } + if let Some(method) = self.put { + add_method(&mut out, "put", method); + } + if let Some(method) = self.post { + add_method(&mut out, "post", method); + } + if let Some(method) = self.delete { + add_method(&mut out, "delete", method); + } + + match self.subroute { + None => (), + Some(SubRoute::Parameter(name, router)) => { + out.push(TokenTree::Punct(Punct::new('.', Spacing::Alone))); + out.push(TokenTree::Ident(Ident::new( + "parameter_subdir", + Span::call_site(), + ))); + let mut sub_route = TokenStream::from_iter(vec![ + TokenTree::Literal(Literal::string(&name)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + ]); + sub_route.extend(router.into_token_stream(None)); + out.push(TokenTree::Group(Group::new( + Delimiter::Parenthesis, + sub_route, + ))); + } + Some(SubRoute::Directories(hash)) => { + for (name, router) in hash { + out.push(TokenTree::Punct(Punct::new('.', Spacing::Alone))); + out.push(TokenTree::Ident(Ident::new("subdir", Span::call_site()))); + let mut sub_route = TokenStream::from_iter(vec![ + TokenTree::Literal(Literal::string(&name)), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + ]); + sub_route.extend(router.into_token_stream(None)); + out.push(TokenTree::Group(Group::new( + Delimiter::Parenthesis, + sub_route, + ))); + } + } + } + + if let Some(name) = name { + let type_name = Ident::new(&format!("{}_TYPE", name.to_string()), name.span()); + let var_name = name; + let router_expression = TokenStream::from_iter(out); + + quote! { + #[allow(non_camel_case_types)] + struct #type_name(std::cell::Cell>, std::sync::Once); + unsafe impl Sync for #type_name {} + impl std::ops::Deref for #type_name { + type Target = Router; + fn deref(&self) -> &Self::Target { + self.1.call_once(|| unsafe { + self.0.set(Some(#router_expression)); + }); + unsafe { + (*self.0.as_ptr()).as_ref().unwrap() + } + } + } + static #var_name : #type_name = #type_name( + std::cell::Cell::new(None), + std::sync::Once::new(), + ); + } + } else { + TokenStream::from_iter(out) + } + } +} + +fn parse_router(mut input: TokenIter) -> Result { + let mut router = Router::default(); + loop { + match parse_entry_key(&mut input)? { + Some(Entry::Method(name)) => { + let function = need_ident(&mut input)?; + + let method_ptr = match name.to_string().as_str() { + "GET" => &mut router.get, + "PUT" => &mut router.put, + "POST" => &mut router.post, + "DELETE" => &mut router.delete, + other => bail!("not a valid method name: {}", other.to_string()), + }; + + if method_ptr.is_some() { + bail!("duplicate method entry: {}", name.to_string()); + } + + *method_ptr = Some(function); + } + Some(Entry::Path(path)) => { + let sub_content = need_group(&mut input, Delimiter::Brace)?; + let sub_router = parse_router(sub_content.stream().into_iter().peekable())?; + router.insert(path, sub_router)?; + } + None => break, + } + comma_or_end(&mut input)?; + } + Ok(router) +} + +fn parse_entry_key(tokens: &mut TokenIter) -> Result, Error> { + match tokens.next() { + None => Ok(None), + Some(TokenTree::Punct(ref punct)) if punct.as_char() == '/' => { + Ok(Some(Entry::Path(parse_path_name(tokens)?))) + } + Some(TokenTree::Ident(ident)) => { + match_colon(tokens)?; + Ok(Some(Entry::Method(ident))) + } + Some(other) => bail!("invalid router entry: {:?}", other), + } +} + +fn parse_path_name(tokens: &mut TokenIter) -> Result { + let mut path = Path::new(); + let mut component = String::new(); + loop { + match tokens.next() { + None => bail!("expected path component"), + Some(TokenTree::Group(group)) => { + if group.delimiter() != Delimiter::Brace { + bail!("invalid path component: {:?}", group); + } + let name = need_hyphenated_name(&mut group.stream().into_iter().peekable())?; + if !component.is_empty() { + path.push(Component::Name(component)); + component = String::new(); + } + path.push(Component::Match(name.into_string())); + + // Now: + // `component` is empty + // Next tokens: + // `:` (and we're done) + // `/` (and we start the next component) + } + Some(TokenTree::Punct(ref punct)) if punct.as_char() == ':' => { + if !component.is_empty() { + // this only happens when we hit the '-' case + bail!("name must not end with a hyphen"); + } + break; + } + Some(TokenTree::Ident(ident)) => { + component.push_str(&ident.to_string()); + + // Now: + // `component` is partially or fully filled + // Next tokens: + // `:` (and we're done) + // `/` (and we start the next component) + // `-` (the component name is not finished yet) + } + Some(other) => bail!("invalid path component: {:?}", other), + } + + // there may be hyphens here, but we don't allow space separated paths or other symbols + match tokens.next() { + None => break, + Some(TokenTree::Punct(punct)) => match punct.as_char() { + ':' => break, // okay in both cases + '-' => { + if component.is_empty() { + bail!("unexpected hyphen after parameter matcher"); + } + component.push('-'); + // `component` is partially filled, we need more + } + '/' => { + if !component.is_empty() { + path.push(Component::Name(component)); + component = String::new(); + } + // `component` is cleared, we start the next one + } + other => bail!("invalid punctuation in path: {:?}", other), + }, + Some(other) => bail!( + "invalid path component, expected hyphen or slash: {:?}", + other + ), + } + } + + if !component.is_empty() { + path.push(Component::Name(component)); + } + + Ok(path) +} diff --git a/proxmox-api-macro/src/util.rs b/proxmox-api-macro/src/util.rs new file mode 100644 index 00000000..1bc7a825 --- /dev/null +++ b/proxmox-api-macro/src/util.rs @@ -0,0 +1,19 @@ +pub fn to_camel_case(text: &str) -> String { + let mut out = String::new(); + + let mut capitalize = true; + for c in text.chars() { + if c == '_' { + capitalize = true; + } else { + if capitalize { + out.extend(c.to_uppercase()); + capitalize = false; + } else { + out.push(c); + } + } + } + + out +} diff --git a/proxmox-api-macro/tests/basic.rs b/proxmox-api-macro/tests/basic.rs new file mode 100644 index 00000000..b21a3692 --- /dev/null +++ b/proxmox-api-macro/tests/basic.rs @@ -0,0 +1,136 @@ +#![feature(async_await)] + +use bytes::Bytes; +use failure::{bail, format_err, Error}; +use http::Response; +use serde_derive::{Deserialize, Serialize}; +use serde_json::Value; + +use proxmox_api::Router; +use proxmox_api_macro::api; + +#[api({ + description: "A hostname or IP address", + validate: validate_hostname, +})] +#[derive(Deserialize, Serialize)] +#[repr(transparent)] +pub struct HostOrIp(String); + +// Simplified for example purposes +fn validate_hostname(name: &str) -> Result<(), Error> { + if name == "" { + bail!("found bad hostname"); + } + Ok(()) +} + +#[api({ + description: "A person definition containing name and ID", + fields: { + name: { + description: "The person's full name", + }, + id: { + description: "The person's ID number", + minimum: 1000, + maximum: 10000, + }, + }, +})] +#[derive(Deserialize, Serialize)] +pub struct Person { + name: String, + id: usize, +} + +#[api({ + description: "A test function returning a fixed text", + parameters: {}, +})] +async fn test_body() -> Result<&'static str, Error> { + Ok("test body") +} + +#[api({ + description: "Loopback the `input` parameter", + parameters: { + input: "the input", + }, +})] +async fn get_loopback(param: String) -> Result { + Ok(param) +} + +#[api({ + description: "Loopback the `input` parameter", + parameters: { + input: "the input", + }, + returns: String +})] +fn non_async_test(param: String) -> proxmox_api::ApiFuture { + Box::pin((async move || proxmox_api::IntoApiOutput::into_api_output(param))()) +} + +proxmox_api_macro::router! { + static TEST_ROUTER = { + GET: test_body, + + /subdir: { GET: test_body }, + /subdir/repeated: { GET: test_body }, + + /other: { GET: test_body }, + /other/subdir: { GET: test_body }, + + /more/{param}: { GET: get_loopback }, + /more/{param}/info: { GET: get_loopback }, + + /another/{param}: { + GET: get_loopback, + + /dir: { GET: non_async_test }, + }, + }; +} + +fn check_body(router: &Router, path: &str, expect: &'static str) { + let (router, parameters) = router + .lookup(path) + .expect("expected method to exist on test router"); + let method = router + .get + .as_ref() + .expect("expected GET method on router at path"); + let fut = method.handler()(parameters.unwrap_or(Value::Null)); + let resp = futures::executor::block_on(fut) + .expect("expected `GET` on test_body to return successfully"); + assert!(resp.status() == 200, "test response should have status 200"); + let body = resp.into_body(); + let body = std::str::from_utf8(&body).expect("expected test body to be valid utf8"); + assert!( + body == expect, + "expected test body output to be {:?}, found: {:?}", + expect, + body + ); +} + +#[test] +fn router() { + check_body(&TEST_ROUTER, "/subdir", r#"{"data":"test body"}"#); + check_body(&TEST_ROUTER, "/subdir/repeated", r#"{"data":"test body"}"#); + check_body(&TEST_ROUTER, "/more/argvalue", r#"{"data":"argvalue"}"#); + check_body( + &TEST_ROUTER, + "/more/argvalue/info", + r#"{"data":"argvalue"}"#, + ); + check_body(&TEST_ROUTER, "/another/foo", r#"{"data":"foo"}"#); + check_body(&TEST_ROUTER, "/another/foo/dir", r#"{"data":"foo"}"#); + + // And can I... + let res = futures::executor::block_on(get_loopback("FOO".to_string())) + .expect("expected result from get_loopback"); + assert!(res == "FOO", "expected FOO from direct get_loopback('FOO') call"); +} diff --git a/proxmox/Cargo.toml b/proxmox/Cargo.toml index f2be2475..441f7e34 100644 --- a/proxmox/Cargo.toml +++ b/proxmox/Cargo.toml @@ -9,4 +9,5 @@ authors = [ [dependencies] proxmox-api = { path = "../proxmox-api" } +proxmox-api-macro = { path = "../proxmox-api-macro" } proxmox-tools = { path = "../proxmox-tools" } diff --git a/proxmox/src/lib.rs b/proxmox/src/lib.rs index 29604835..c2124297 100644 --- a/proxmox/src/lib.rs +++ b/proxmox/src/lib.rs @@ -1,2 +1,8 @@ -pub use proxmox_api as api; pub use proxmox_tools as tools; + +// Both `proxmox_api` and the 2 macros from `proxmox_api_macro` should be +// exposed via `proxmox::api`. +pub mod api { + pub use proxmox_api::*; + pub use proxmox_api_macro::{api, router}; +}