Improvements to wasi-headers tool (#160)

* wasi-headers: update WASI submodule, handle changes to witx ast

* wasi-headers: restructure lib and exe to be more flexible

just factor out some of the hard-coded stuff

tools/wasi-headers/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "wasi-headers"
-version = "0.0.0"
-authors = ["Dan Gohman <sunfish@mozilla.com>"]
+version = "0.0.1"
+authors = ["Dan Gohman <sunfish@mozilla.com>", "Pat Hickey <phickey@fastly.com>"]
 license = "Apache-2.0"
 edition = "2018"
 publish = false
@@ -10,6 +10,7 @@ publish = false
 heck = "0.3.1"
 witx = { path = "WASI/tools/witx" }
 anyhow = "1.0.22"
+clap = "2.23"
 
 [dev-dependencies]
 diff = "0.1.11"

tools/wasi-headers/WASI

@@ -1 +1 @@
-Subproject commit 8ff4e845d7baf48bf7cf7b6d9cc2a32a973da04e
+Subproject commit 04d4eba571dc1d6fe9ab129ea9343911bcc256dc

tools/wasi-headers/src/c_header.rs

@@ -78,9 +78,10 @@ fn print_datatype(ret: &mut String, nt: &NamedType) {
         ret.push_str(" */\n");
     }
 
-    match &nt.dt {
+    match &nt.tref {
         TypeRef::Value(v) => match &**v {
             Type::Enum(e) => print_enum(ret, &nt.name, e),
+            Type::Int(i) => print_int(ret, &nt.name, i),
             Type::Flags(f) => print_flags(ret, &nt.name, f),
             Type::Struct(s) => print_struct(ret, &nt.name, s),
             Type::Union(u) => print_union(ret, &nt.name, u),
@@ -88,9 +89,9 @@ fn print_datatype(ret: &mut String, nt: &NamedType) {
             Type::Builtin { .. }
             | Type::Array { .. }
             | Type::Pointer { .. }
-            | Type::ConstPointer { .. } => print_alias(ret, &nt.name, &nt.dt),
+            | Type::ConstPointer { .. } => print_alias(ret, &nt.name, &nt.tref),
         },
-        TypeRef::Name(_) => print_alias(ret, &nt.name, &nt.dt),
+        TypeRef::Name(_) => print_alias(ret, &nt.name, &nt.tref),
     }
 }
 
@@ -147,6 +148,46 @@ fn print_enum(ret: &mut String, name: &Id, e: &EnumDatatype) {
     }
 }
 
+fn print_int(ret: &mut String, name: &Id, i: &IntDatatype) {
+    ret.push_str(&format!(
+        "typedef {} __wasi_{}_t;\n",
+        intrepr_name(i.repr),
+        ident_name(name)
+    ));
+    ret.push_str("\n");
+
+    for (index, const_) in i.consts.iter().enumerate() {
+        if !const_.docs.is_empty() {
+            ret.push_str("/**\n");
+            for line in const_.docs.lines() {
+                ret.push_str(&format!(" * {}\n", line));
+            }
+            ret.push_str(" */\n");
+        }
+        ret.push_str(&format!(
+            "#define __WASI_{}_{} ({}({}))\n",
+            ident_name(&name).to_shouty_snake_case(),
+            ident_name(&const_.name).to_shouty_snake_case(),
+            intrepr_const(i.repr),
+            index
+        ));
+        ret.push_str("\n");
+    }
+
+    ret.push_str(&format!(
+        "_Static_assert(sizeof(__wasi_{}_t) == {}, \"witx calculated size\");\n",
+        ident_name(name),
+        i.repr.mem_size()
+    ));
+    ret.push_str(&format!(
+        "_Static_assert(_Alignof(__wasi_{}_t) == {}, \"witx calculated align\");\n",
+        ident_name(name),
+        i.repr.mem_align()
+    ));
+
+    ret.push_str("\n");
+}
+
 fn print_flags(ret: &mut String, name: &Id, f: &FlagsDatatype) {
     ret.push_str(&format!(
         "typedef {} __wasi_{}_t;\n",
@@ -377,7 +418,10 @@ fn ident_name(i: &Id) -> String {
 
 fn builtin_type_name(b: BuiltinType) -> &'static str {
     match b {
-        BuiltinType::String => "string",
+        BuiltinType::String | BuiltinType::Char8 => {
+            panic!("no type name for string or char8 builtins")
+        }
+        BuiltinType::USize => "size_t",
         BuiltinType::U8 => "uint8_t",
         BuiltinType::U16 => "uint16_t",
         BuiltinType::U32 => "uint32_t",
@@ -392,13 +436,26 @@ fn builtin_type_name(b: BuiltinType) -> &'static str {
 }
 
 fn typeref_name(tref: &TypeRef) -> String {
+    match &*tref.type_() {
+        Type::Builtin(BuiltinType::String) | Type::Builtin(BuiltinType::Char8) | Type::Array(_) => {
+            panic!("unsupported grammar: cannot construct name of string or array",)
+        }
+        _ => {}
+    }
+
     match tref {
-        TypeRef::Name(named_type) => format!("__wasi_{}_t", named_type.name.as_str()),
-        TypeRef::Value(anon_type) => match &**anon_type {
-            Type::Builtin(b) => builtin_type_name(*b).to_string(),
-            Type::Array(_) => unreachable!("arrays should be special-cased"),
+        TypeRef::Name(named_type) => match &*named_type.type_() {
             Type::Pointer(p) => format!("{} *", typeref_name(&*p)),
             Type::ConstPointer(p) => format!("const {} *", typeref_name(&*p)),
+            Type::Array(_) => unreachable!("arrays excluded above"),
+            _ => format!("__wasi_{}_t", named_type.name.as_str()),
+        },
+        TypeRef::Value(anon_type) => match &**anon_type {
+            Type::Array(_) => unreachable!("arrays excluded above"),
+            Type::Builtin(b) => builtin_type_name(*b).to_string(),
+            Type::Pointer(p) => format!("{} *", typeref_name(&*p)),
+            Type::ConstPointer(p) => format!("const {} *", typeref_name(&*p)),
+            Type::Int(i) => format!("{}", intrepr_name(i.repr)),
             Type::Struct { .. }
             | Type::Union { .. }
             | Type::Enum { .. }

tools/wasi-headers/src/lib.rs

@@ -2,17 +2,12 @@ mod c_header;
 
 use anyhow::{anyhow, Result};
 use c_header::to_c_header;
-use std::fs::read_dir;
+use std::fs;
 use std::io;
+use std::path::PathBuf;
 use witx::load;
 
-pub fn generate() -> Result<String> {
-    let mut inputs = read_dir("WASI/phases/snapshot/witx")?
-        .map(|res| res.map(|e| e.path()))
-        .collect::<Result<Vec<_>, io::Error>>()?;
-
-    inputs.sort();
-
+pub fn generate(inputs: &[PathBuf]) -> Result<String> {
     // TODO: drop the anyhow! part once witx switches to anyhow.
     let doc = load(&inputs).map_err(|e| anyhow!(e.to_string()))?;
 
@@ -24,3 +19,18 @@ pub fn generate() -> Result<String> {
 
     Ok(to_c_header(&doc, &inputs_str))
 }
+
+pub fn snapshot_witx_files() -> Result<Vec<PathBuf>> {
+    let snapshot_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("WASI/phases/snapshot/witx");
+    let mut inputs = fs::read_dir(snapshot_dir)?
+        .map(|res| res.map(|e| e.path()))
+        .collect::<Result<Vec<_>, io::Error>>()?;
+
+    inputs.sort();
+    Ok(inputs)
+}
+
+pub fn libc_wasi_api_header() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("../../libc-bottom-half/headers/public/wasi/api.h")
+}

tools/wasi-headers/src/main.rs

@@ -1,11 +1,60 @@
+#[macro_use]
+extern crate clap;
+
 use anyhow::Result;
+use clap::{Arg, SubCommand};
 use std::fs::File;
 use std::io::Write;
-use wasi_headers::generate;
+use std::path::PathBuf;
+use wasi_headers::{generate, libc_wasi_api_header, snapshot_witx_files};
 
-pub fn main() -> Result<()> {
-    let c_header = generate()?;
-    let mut file = File::create("../../libc-bottom-half/headers/public/wasi/api.h")?;
-    file.write_all(c_header.as_bytes())?;
+struct GenerateCommand {
+    /// Input witx file
+    inputs: Vec<PathBuf>,
+    /// Output header file
+    output: PathBuf,
+}
+
+impl GenerateCommand {
+    pub fn execute(&self) -> Result<()> {
+        let c_header = generate(&self.inputs)?;
+        let mut file = File::create(&self.output)?;
+        file.write_all(c_header.as_bytes())?;
+        Ok(())
+    }
+}
+
+fn main() -> Result<()> {
+    let matches = app_from_crate!()
+        .arg(Arg::with_name("inputs").required(true).multiple(true))
+        .arg(
+            Arg::with_name("output")
+                .short("o")
+                .long("output")
+                .takes_value(true)
+                .required(true),
+        )
+        .subcommand(
+            SubCommand::with_name("generate-libc")
+                .about("generate libc api.h from current snapshot"),
+        )
+        .get_matches();
+
+    let cmd = if matches.subcommand_matches("generate-libc").is_some() {
+        let inputs = snapshot_witx_files()?;
+        let output = libc_wasi_api_header();
+        GenerateCommand { inputs, output }
+    } else {
+        GenerateCommand {
+            inputs: matches
+                .values_of("inputs")
+                .expect("inputs required")
+                .map(PathBuf::from)
+                .collect(),
+            output: PathBuf::from(matches.value_of("output").expect("output required")),
+        }
+    };
+
+    cmd.execute()?;
     Ok(())
 }

tools/wasi-headers/tests/verify.rs

@@ -1,7 +1,11 @@
+use std::fs;
+
 #[test]
 fn assert_same_as_src() {
-    let actual = include_str!("../../../libc-bottom-half/headers/public/wasi/api.h");
-    let expected = wasi_headers::generate().expect("header generation should succeed");
+    let actual =
+        fs::read_to_string(wasi_headers::libc_wasi_api_header()).expect("read libc wasi/api.h");
+    let witx_files = wasi_headers::snapshot_witx_files().expect("parse snapshot witx files");
+    let expected = wasi_headers::generate(&witx_files).expect("header generation");
     if actual == expected {
         return;
     }
@@ -63,7 +67,9 @@ fn assert_same_as_src() {
     }
 
     eprintln!();
-    eprintln!("To regenerate the files, run `cd tools/wasi-headers && cargo run`.");
+    eprintln!(
+        "To regenerate the files, run `cd tools/wasi-headers && cargo run -- generate-libc`."
+    );
     eprintln!();
     panic!();
 }