test: improve test infrastructure (#554)

This change represents a rather large re-design in how `wasi-libc` builds and runs its tests. Initially, #346 retrieved the `libc-test` repository and built a subset of those tests to give us some amount of test coverage. Later, because there was no way to add custom C tests, #522 added a `smoke` directory which allowed this. But (a) each of these test suites was built and run separately and (b) it was unclear how to add more tests flexibly--some tests should only run on `*p2` targets or `*-threads` targets, e.g. This change reworks all of this so that all tests are built the same way, in the same place. For downloaded tests like those from `libc-test`, I chose to add "stub tests" that `#include` the original version. This not only keeps all enabled tests in one place, it also allows us to add "directives," C comments that the `Makefile` uses to filter out tests for certain targets or add special compile, link or run flags. These rudimentary scripts, along with other Bash logic I moved out of the Makefile now live in the `scripts` directory. Finally, all of this is explained more clearly in an updated `README.md`. The hope with documenting this a bit better is that it would be easier for drive-by contributors to be able to either dump in new C tests for regressions they may find or enable more libc-tests. As of my current count, we only enable 40/75 of libc-test's functional tests, 0/228 math tests, 0/69 regression tests, and 0/79 API tests. Though many of these may not apply to WASI programs, it would be nice to explore how many more of these tests can be enabled to increase wasi-libc's test coverage. This change should explain how to do that and, with directives, make it possible to condition how the tests compile and run.
2025-10-04 09:00:56 +00:00 · 2024-12-09 13:45:31 -08:00 · 2024-12-09 13:45:31 -08:00 · 2b853ff079
commit 2b853ff079
parent 29c22a4b68
49 changed files with 530 additions and 165 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -125,17 +125,13 @@ jobs:
        export WASIP1_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasip1/)
        export WASIP2_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasip2/)
        mkdir -p $WASI_DIR $WASIP1_DIR $WASIP2_DIR
-        cp download/libclang_rt.builtins-wasm32.a $WASI_DIR
-        cp download/libclang_rt.builtins-wasm32.a $WASIP1_DIR
-        cp download/libclang_rt.builtins-wasm32.a $WASIP2_DIR
+        cp build/download/libclang_rt.builtins-wasm32.a $WASI_DIR
+        cp build/download/libclang_rt.builtins-wasm32.a $WASIP1_DIR
+        cp build/download/libclang_rt.builtins-wasm32.a $WASIP2_DIR
        TARGET_TRIPLE=wasm32-wasi make test
-        rm -r build
        TARGET_TRIPLE=wasm32-wasip1 make test
-        rm -r build
        TARGET_TRIPLE=wasm32-wasip2 make test
-        rm -r build
        TARGET_TRIPLE=wasm32-wasi-threads make test
-        rm -r build
        TARGET_TRIPLE=wasm32-wasip1-threads make test
      # The older version of Clang does not provide the expected symbol for the
      # test entrypoints: `undefined symbol: __main_argc_argv`.
--- a/test/.gitignore
+++ b/test/.gitignore
@ -1,5 +1,2 @@
 build
-download
 run
-
-smoke/*.dir
--- a/test/Makefile
+++ b/test/Makefile
@ -9,15 +9,28 @@
 # - `run`: execute the benchmarks with a Wasm `$(ENGINE)` of choice (e.g.,
 #   Wasmtime)

-test: run run_smoke
-
-# Unlike the `libc-test` directory, we will output all artifacts to the `build`
-# directory (keeping with the `wasi-libc` conventions).
-OBJDIR ?= $(CURDIR)/build
-DOWNDIR ?= $(CURDIR)/download
+test: run

+# Decide which target to build for and which libc to use.
 TARGET_TRIPLE ?= wasm32-wasi

+# Setup various paths used by the tests.
+OBJDIR ?= build/$(TARGET_TRIPLE)
+DOWNDIR ?= build/download
+SRCDIR ?= src
+RUNDIR ?= run/$(TARGET_TRIPLE)
+
+# We also need to know the location the wasi-libc sysroot we're building
+# against.
+SYSROOT_DIR ?= ../sysroot
+SYSROOT := $(SYSROOT_DIR)/lib/$(TARGET_TRIPLE)
+$(SYSROOT):
+	@echo "No sysroot for $(TARGET_TRIPLE) available at $(SYSROOT_DIR); to build it, e.g.:"
+	@echo "  cd $(dir $(SYSROOT_DIR))"
+	@echo "  make TARGET_TRIPLE=$(TARGET_TRIPLE)"
+	@exit 1
+
+
 ##### DOWNLOAD #################################################################

 LIBC_TEST_URL ?= https://github.com/bytecodealliance/libc-test
@ -25,21 +38,14 @@ LIBC_TEST = $(DOWNDIR)/libc-test
 LIBRT_URL ?= https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/libclang_rt.builtins-wasm32-wasi-24.0.tar.gz
 LIBRT = $(DOWNDIR)/libclang_rt.builtins-wasm32.a
 WASMTIME_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasmtime-v26.0.1-x86_64-linux.tar.xz
-WASMTIME = $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime
+WASMTIME = $(abspath $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime)
 WASM_TOOLS_URL ?= https://github.com/bytecodealliance/wasm-tools/releases/download/v1.220.0/wasm-tools-1.220.0-x86_64-linux.tar.gz
 WASM_TOOLS = $(DOWNDIR)/$(shell basename $(WASM_TOOLS_URL) .tar.gz)/wasm-tools
 ADAPTER_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v26.0.1/wasi_snapshot_preview1.command.wasm
 ADAPTER = $(DOWNDIR)/wasi_snapshot_preview1.command.wasm

-TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME)
-ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
-TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS)
-endif
-
-download: $(TO_DOWNLOAD)
-
 $(DOWNDIR):
-	mkdir -p download
+	@mkdir -p $@

 $(LIBC_TEST): | $(DOWNDIR)
 	git clone --depth 1 $(LIBC_TEST_URL) $@
@ -61,151 +67,149 @@ $(WASM_TOOLS): | $(DOWNDIR)
 $(ADAPTER): | $(DOWNDIR)
 	wget --no-clobber --directory-prefix=$(DOWNDIR) $(ADAPTER_URL)

+# Target to download all necessary dependencies.
+TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME)
+ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
+TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS)
+endif
+DOWNLOADED := $(DOWNDIR)/downloaded.stamp
+$(DOWNLOADED): $(TO_DOWNLOAD)
+	touch $@
+download: $(DOWNLOADED)
+
 clean::
 	rm -rf $(DOWNDIR)

-##### BUILD ####################################################################
+##### INFRA ####################################################################

-# For now, we list out the tests that we can currently build and run. This is
-# heavily focused on the functional tests; in the future it would be good to
-# fill out the missing tests and also include some `src/api` and `src/math`
-# tests (TODO).
-TESTS := \
-	$(LIBC_TEST)/src/functional/argv.c \
-	$(LIBC_TEST)/src/functional/basename.c \
-	$(LIBC_TEST)/src/functional/clocale_mbfuncs.c \
-	$(LIBC_TEST)/src/functional/clock_gettime.c \
-	$(LIBC_TEST)/src/functional/crypt.c \
-	$(LIBC_TEST)/src/functional/dirname.c \
-	$(LIBC_TEST)/src/functional/env.c \
-	$(LIBC_TEST)/src/functional/fnmatch.c \
-	$(LIBC_TEST)/src/functional/iconv_open.c \
-	$(LIBC_TEST)/src/functional/mbc.c \
-	$(LIBC_TEST)/src/functional/memstream.c \
-	$(LIBC_TEST)/src/functional/qsort.c \
-	$(LIBC_TEST)/src/functional/random.c \
-	$(LIBC_TEST)/src/functional/search_hsearch.c \
-	$(LIBC_TEST)/src/functional/search_insque.c \
-	$(LIBC_TEST)/src/functional/search_lsearch.c \
-	$(LIBC_TEST)/src/functional/search_tsearch.c \
-	$(LIBC_TEST)/src/functional/snprintf.c \
-	$(LIBC_TEST)/src/functional/sscanf.c \
-	$(LIBC_TEST)/src/functional/strftime.c \
-	$(LIBC_TEST)/src/functional/string.c \
-	$(LIBC_TEST)/src/functional/string_memcpy.c \
-	$(LIBC_TEST)/src/functional/string_memmem.c \
-	$(LIBC_TEST)/src/functional/string_memset.c \
-	$(LIBC_TEST)/src/functional/string_strchr.c \
-	$(LIBC_TEST)/src/functional/string_strcspn.c \
-	$(LIBC_TEST)/src/functional/string_strstr.c \
-	$(LIBC_TEST)/src/functional/strtod.c \
-	$(LIBC_TEST)/src/functional/strtod_long.c \
-	$(LIBC_TEST)/src/functional/strtod_simple.c \
-	$(LIBC_TEST)/src/functional/strtof.c \
-	$(LIBC_TEST)/src/functional/strtol.c \
-	$(LIBC_TEST)/src/functional/swprintf.c \
-	$(LIBC_TEST)/src/functional/tgmath.c \
-	$(LIBC_TEST)/src/functional/udiv.c \
-	$(LIBC_TEST)/src/functional/wcsstr.c \
-	$(LIBC_TEST)/src/functional/wcstol.c
+INFRA_OBJDIR := $(OBJDIR)/common
+$(INFRA_OBJDIR):
+	@mkdir -p $@

-# Part of the problem including more tests is that the `libc-test`
-# infrastructure code is not all Wasm-compilable. As we include more tests
-# above, this list will also likely need to grow.
-COMMON_TEST_INFRA = \
-	$(LIBC_TEST)/src/common/path.c \
+# Build the common test infrastructure. Part of the problem including more tests
+# is that the `libc-test` infrastructure code is not all Wasm-compilable. As we
+# include more tests above, this list will also likely need to grow.
+INFRA_FILES = $(LIBC_TEST)/src/common/path.c \
 	$(LIBC_TEST)/src/common/print.c \
 	$(LIBC_TEST)/src/common/rand.c \
 	$(LIBC_TEST)/src/common/utf8.c
+$(INFRA_FILES): $(DOWNLOADED)
+INFRA_WASM_OBJS := $(patsubst $(LIBC_TEST)/src/common/%.c,$(OBJDIR)/common/%.wasm.o,$(INFRA_FILES))
+$(OBJDIR)/common/%.wasm.o: $(LIBC_TEST)/src/common/%.c | $(INFRA_OBJDIR)
+	$(CC) $(CFLAGS) -c $< -o $@
+
+# Also, include the `libc-test` infrastructure headers.
+INFRA_HEADERS_DIR := $(LIBC_TEST)/src/common
+INFRA_HEADERS := $(shell find $(INFRA_HEADERS_DIR) -name '*.h')
+$(INFRA_HEADERS): $(DOWNLOADED)
+
+##### BUILD ####################################################################

 # Create various lists containing the various artifacts to be built: mainly,
 # $(WASM_OBJS) are compiled in the $(OBJDIRS) and then linked together to form
 # the $(WASMS) tests.
-NAMES := $(TESTS:$(LIBC_TEST)/src/%.c=%)
-WASMS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.core.wasm)
-WASM_OBJS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
-INFRA_WASM_OBJS := $(COMMON_TEST_INFRA:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
+ALL_TESTS := $(shell find $(SRCDIR) -name '*.c')
+TESTS := $(shell TARGET_TRIPLE=$(TARGET_TRIPLE) scripts/filter.py $(ALL_TESTS))
+WASM_OBJS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.wasm.o)
 WASM_OBJS += $(INFRA_WASM_OBJS)
-DIRS := $(patsubst $(OBJDIR)/%/,%,$(sort $(dir $(WASM_OBJS))))
-OBJDIRS := $(DIRS:%=$(OBJDIR)/%)
+ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
+WASMS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.component.wasm)
+else
+WASMS := $(TESTS:$(SRCDIR)/%.c=$(OBJDIR)/%.core.wasm)
+endif

-# Allow $(CC) to be set from the command line; ?= doesn't work for CC because
-# make has a default value for it.
+
+# Setup the compiler. We allow $(CC) to be set from the command line; ?= doesn't
+# work for CC because make has a default value for it.
 ifeq ($(origin CC), default)
 CC := clang
 endif
 LDFLAGS ?=
-CFLAGS ?= --target=$(TARGET_TRIPLE) --sysroot=../sysroot
+CFLAGS ?= --target=$(TARGET_TRIPLE) --sysroot=$(SYSROOT_DIR)
 # Always include the `libc-test` infrastructure headers.
-CFLAGS += -I$(LIBC_TEST)/src/common
+CFLAGS += -I$(INFRA_HEADERS_DIR)
 ifneq ($(findstring -threads,$(TARGET_TRIPLE)),)
 CFLAGS += -pthread
 endif

+# Build up all the `*.wasm.o` object files; these are the same regardless of
+# whether we're building core modules or components.
+$(WASM_OBJS): $(INFRA_HEADERS)
+$(OBJDIR)/%.wasm.o: $(SRCDIR)/%.c $(DOWNLOADED) $(SYSROOT)
+	@mkdir -p $(@D)
+	$(CC) $(CFLAGS) $(shell scripts/add-flags.py CFLAGS $<) -c $< -o $@
+
+# Build up all the `*.wasm` files.
+obj_to_c = $(patsubst $(OBJDIR)/%.wasm.o,$(SRCDIR)/%.c,$1)
+$(OBJDIR)/%.core.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS)
+	@mkdir -p $(@D)
+	$(CC) $(CFLAGS) $(LDFLAGS) $(shell scripts/add-flags.py LDFLAGS $(call obj_to_c,$<)) $^ -o $@
+
+# For wasip2, we include an additional step to wrap up the core module into
+# a component.
+$(OBJDIR)/%.component.wasm: $(OBJDIR)/%.core.wasm
+	$(WASM_TOOLS) component new --adapt $(ADAPTER) $< -o $@
+
 # Compile each selected test using Clang. Note that failures here are likely
 # due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory.
 # This location is system-dependent, but could be fixed by something like:
 #  $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi
 #  $ sudo cp download/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/
-build: download $(WASMS)
-
-$(WASMS): | $(OBJDIRS)
-$(OBJDIR)/%.core.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS)
-	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
-
-ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
-$(OBJDIR)/%.wasm: $(OBJDIR)/%.core.wasm
-	$(WASM_TOOLS) component new --adapt $(ADAPTER) $< -o $@
-endif
-
-$(WASM_OBJS): $(LIBC_TEST)/src/common/test.h | $(OBJDIRS)
-$(OBJDIR)/%.wasm.o: $(LIBC_TEST)/src/%.c
-	$(CC) $(CFLAGS) -c -o $@ $<
-
-$(OBJDIRS):
-	mkdir -p $@
+build: $(DOWNLOADED) $(WASMS)

 clean::
 	rm -rf $(OBJDIR)

+##### GENERATE #################################################################
+
+# Not all of the downloaded `libc-test` tests can be built and run with
+# `wasi-libc`. Thus, we only include the subset that can be in `src/libc-test`
+# as stub files that `#include` the original test files. When we want to add
+# more tests, though, the `generate-stubs` target will generate stubs for the
+# missing tests which we can delete or alter as needed.
+
+STUBDIR := $(SRCDIR)/libc-test
+generate-stubs:
+	FROM_DIR=$(LIBC_TEST) TO_DIR=$(STUBDIR) scripts/generate-stubs.sh
+
 ##### RUN ######################################################################

 ENGINE ?= $(WASMTIME) run
-ERRS:=$(WASMS:%.core.wasm=%.wasm.err)
+ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
+ENGINE += --wasm component-model
+OBJPAT := $(OBJDIR)/%.component.wasm
+else
+OBJPAT := $(OBJDIR)/%.core.wasm
+endif
+
+# Each Wasm test is run every time, generating a folder containing a `cmd.sh`
+# script and an `output.log` file (see `scripts/run-test.sh` for details). The
+# `success` file is never generated, which means the test will rerun every time.
+# To ignore a test temporarily, `touch .../success:`.
+RUNTESTS:=$(WASMS:$(OBJPAT)=$(RUNDIR)/%/success)
+wasm_to_c = $(patsubst $(OBJPAT),$(SRCDIR)/%.c,$1)
+$(RUNDIR)/%/success: $(OBJPAT)
+	@mkdir -p $(@D)
+	@DIR="$(abspath $(@D))" \
+	  WASM="$(abspath $<)" \
+	  ENGINE="$(ENGINE) $(shell scripts/add-flags.py RUN $(call wasm_to_c,$<))" \
+	    scripts/run-test.sh

 # Use the provided Wasm engine to execute each test, emitting its output into
 # a `.err` file.
-run: build $(ERRS)
-	@echo "Tests passed"
-
-$(ERRS): | $(OBJDIRS)
-
-ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
-%.wasm.err: %.wasm
-	$(ENGINE) --wasm component-model $< >$@
-else
-%.wasm.err: %.core.wasm
-	$(ENGINE) $< >$@
-endif
+run: build $(RUNTESTS)
+	@if scripts/failed-tests.sh $(RUNDIR); then \
+		echo "Tests passed"; \
+	else \
+		echo "Tests failed:"; \
+		VERBOSE=1 scripts/failed-tests.sh $(RUNDIR); \
+	fi

 clean::
-	rm -rf $(OBJDIR)/*/*.err
-
-##### SMOKE TEST SUITE #########################################################
-
-include smoke/smoke.mk
+	rm -rf $(RUNDIR)

 ##### MISC #####################################################################

 # Note: the `clean` target has been built up by all of the previous sections.

-debug:
-	@echo NAMES $(NAMES)
-	@echo TESTS $(TESTS)
-	@echo WASMS $(WASMS)
-	@echo WASM_OBJS $(WASM_OBJS)
-	@echo ERRS $(ERRS)
-	@echo DIRS $(DIRS)
-	@echo OBJDIRS $(OBJDIRS)
-
-.PHONY: test download build run clean
+.PHONY: test download build run generate-stubs clean
--- a/test/README.md
+++ b/test/README.md
@ -1,6 +1,85 @@
 # Test

-This directory runs a subset of libc-test using the sysroot produced by
-`wasi-libc`.
+This directory runs C tests compiled to WebAssembly against `wasi-libc` to check
+its functionality. It enables a subset of [libc-test] as well as adding custom C
+tests; all enabled tests are contained in the [`src`] directory.
+
+### Pre-requisites
+
+- Clang
+- [`libc-test`]
+- `libclang_rt.builtins-wasm32.a`
+- a WebAssembly engine
+- other WebAssembly tools, especially for `wasm32-wasip2` support (see the
+  [`Makefile] for a complete list)
+
+All but Clang are downloaded automatically by the `make download` target.
+
+### Build and run
+
+To build and run all tests:
+
+```sh
+$ make TARGET_TRIPLE=...
+Tests passed
+```
+
+Note that `wasm-ld` must be available, so an additional
+`CC=<wasi-sdk>/bin/clang` may be necessary. Each test runs in a directory that
+looks like (see [`run-test.sh`]):
+
+```sh
+$ ls run/$TARGET_TRIPLE/misc/some-test
+cmd.sh      # the exact command used to run the test
+fs          # a directory containing any test-created files
+output.log  # the captured printed output--only for errors
+```
+
+### Adding tests
+
+To add a test, create a new C file in [`src/misc`]:
+
+```c
+//! filter.py(TARGET_TRIPLE): !wasm32-wasip2
+//! add-flags.py(CFLAGS): ...
+//! add-flags.py(LDFLAGS): ...
+//! add-flags.py(RUN): ...
+void main() { ... }
+```
+
+- to pass, the `main` function must exit successfully and avoid printing output
+- the `filter.py` directive controls when the test builds and runs (e.g., not
+  for `wasip2`)
+- the `add-flags.py` directive adds extra information for building or running
+  the test (see the [`Makefile`] for precise use).
+
+### Enabling more [libc-test] tests
+
+[libc-test] has more tests available that are not yet enabled (e.g., to count
+the enabled subset, `find src -name *.c | wc -l`). Each enabled test contains a
+stub file in [`src/libc-test`] that `#include`s its downloaded version and adds
+various `filter.py` and `add-flags.py` directives.
+
+To quickly create stub files for not-yet-enabled tests:
+
+```sh
+$ make generate-stubs
+$ git status
+...
+src/libc-test/functional/tls_align.c
+src/libc-test/functional/tls_align_dlopen.c
+src/libc-test/functional/tls_align_dso.c
+src/libc-test/functional/tls_init.c
+```
+
+Then modify the directives for these new stub files to get the new tests to
+compile and successfully run.
+
+

 [libc-test]: https://wiki.musl-libc.org/libc-test.html
+[`Makefile`]: Makefile
+[`run-test.sh`]: scripts/run-test.sh
+[`src`]: src
+[`src/libc-test`]: src/libc-test
+[`src/misc`]: src/misc
--- a/test/scripts/add-flags.py
+++ b/test/scripts/add-flags.py
@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+# Find additional compilation flags specified in test files.
+#
+# This script accepts a single file as an argument and looks for a comment like
+# the following: `// add-flags.py(<NAME>): <FLAGS>`. If found, the `<FLAGS>` are
+# printed to stdout.
+#
+# Example:
+# ```
+# $ head one.c
+# //! add-flags.py(CFLAGS): -DFOO
+# $ ./add-flags.py CFLAGS one.c
+# -DFOO
+
+import sys
+import os
+import re
+import logging
+
+"""
+Match a C comment like the following: `//! add-flags.py: <FLAGS>`.
+"""
+PATTERN = re.compile('\\s*//\\!\\s*add-flags\\.py\\(([^)]+)\\):\\s*(.*)')
+
+
+def find_flags(name, file):
+    with open(file, 'r') as f:
+        for lineno, line in enumerate(f, start=1):
+            match = PATTERN.match(line)
+            if match and match[1] == name:
+                pos = f'[{file}:{lineno}]'
+                logging.debug(f'{pos} found flags')
+                return match[2].strip()
+
+
+def main(name, file):
+    flags = find_flags(name, file)
+    if flags:
+        print(flags)
+
+
+if __name__ == "__main__":
+    logging.getLogger().name = os.path.basename(__file__)
+    if os.environ.get('VERBOSE'):
+        logging.basicConfig(level=logging.DEBUG)
+    if len(sys.argv) != 3:
+        print(f'usage: {sys.argv[0]} <name> <file>')
+        sys.exit(1)
+    main(sys.argv[1], sys.argv[2])
--- a/test/scripts/failed-tests.sh
+++ b/test/scripts/failed-tests.sh
@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# This script checks for failed tests in the given directory. Failed tests will
+# include text in a *.log file.
+#
+# Usage: failed-tests.sh [TESTDIR]
+#
+# Optionally set the VERBOSE environment variable to see the list of failed
+# tests.
+
+set -e
+
+TESTDIR=${1:-.}
+FAILED=$(find $TESTDIR -type f -and -name *.log -and -not -size 0)
+
+if [[ -n "$VERBOSE" ]]; then
+    for failed in "$FAILED"; do
+        echo "$failed";
+    done
+fi
+
+if [[ -n "$FAILED" ]]; then
+    exit 1
+fi
--- a/test/scripts/filter.py
+++ b/test/scripts/filter.py
@ -0,0 +1,93 @@
+#!/usr/bin/env python
+
+# Filter out test files that do not match a configuration variable.
+#
+# This script accepts a list of files as arguments and looks for a comment like
+# the following in each: `//! filter.py(<VARIABLE>): <EXPRESSION>`. The
+# `VARIABLE` value is looked up in the environment and compared with the
+# expression. An `EXPRESSION` is a comma-separated list of terms: either
+# `<string>` or `!<string>`. If the `VARIABLE` value matches the expression, the
+# file is printed; otherwise, it is filtered out.
+#
+# Example:
+# ```
+# $ head one.c
+# //! filter.py(FOO): bar
+# $ head two.c
+# //! filter.py(FOO):!bar, baz
+# $ FOO=bar ./filter.py one.c two.c
+# one.c
+# $ FOO=baz ./filter.py one.c two.c
+# two.c
+# $ ./filter.py one.c two.c
+# one.c two.c
+# ```
+
+import sys
+import os
+import re
+import logging
+
+
+def parse_expression(expr):
+    """
+    An expression is a comma-separated list of terms: either <string> or
+    !<string>.
+    """
+    return [s.strip() for s in expr.split(',')]
+
+
+def matches_expression(value, expr, pos):
+    """
+    A value matches an expression if it is in the list of strings or if it is
+    not explicitly rejected; order matters, because the first term that matches,
+    wins.
+    """
+    for e in expr:
+        if e.startswith('!') and value == e[1:]:
+            logging.debug(f'{pos} {value} != {expr}, REJECTED')
+            return False
+        elif value == e:
+            logging.debug(f'{pos} {value} == {expr}, ALLOWED')
+            return True
+    logging.debug(f'{pos} {value} not in {expr}, REJECTED')
+    return False
+
+
+"""
+Match a C comment like the following: `//! filter.py(<VARIABLE>): <EXPRESSION>`.
+"""
+PATTERN = re.compile('\\s*//\\!\\s*filter\\.py\\(([^)]+)\\):\\s*(.*)')
+
+
+def line_matches(line, env, pos):
+    match = PATTERN.match(line)
+    if match:
+        value = env.get(match[1])
+        if value is None:
+            logging.debug(f'{pos} no value for {match[1]}, ALLOWED')
+            return True
+        else:
+            expr = parse_expression(match[2])
+            return matches_expression(value, expr, pos)
+    return True
+
+
+def file_matches(file, env):
+    with open(file, 'r') as f:
+        print = True
+        for lineno, line in enumerate(f, start=1):
+            print &= line_matches(line, env, f'[{file}:{lineno}]')
+        return print
+
+
+def main(env, files):
+    filtered = [file for file in files if file_matches(file, env)]
+    print(' '.join(filtered))
+
+
+if __name__ == "__main__":
+    logging.getLogger().name = os.path.basename(__file__)
+    if os.environ.get('VERBOSE'):
+        logging.basicConfig(level=logging.DEBUG)
+    main(os.environ, sys.argv[1:])
--- a/test/scripts/generate-stubs.sh
+++ b/test/scripts/generate-stubs.sh
@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Create stub tests in `$TO_DIR` that `#include` a test in the `$FROM_DIR`
+# directory. If the stub already exists, do not overwrite it.
+#
+# Usage: FROM_DIR=... TO_DIR=... ./generate-stubs.sh
+
+FROM_DIR="${FROM_DIR:-build/download/libc-test}"
+TO_DIR="${TO_DIR:-src/libc-test}"
+# For now, only retrieve the functional tests.
+FUNCTIONAL_TESTS=$(find $FROM_DIR/src/math -name '*.c')
+
+for from in $FUNCTIONAL_TESTS; do
+    to="${from/"$FROM_DIR/src"/"$TO_DIR"}"
+	if [ ! -f $to ]; then \
+		mkdir -p $(dirname $to)
+		echo '// Auto-generated by generate-stubs.sh' > $to
+		echo '//! add-flags.py(CFLAGS): -I.' >> $to
+		echo "#include \"$from\"" >> $to
+	fi
+done
--- a/test/scripts/run-test.sh
+++ b/test/scripts/run-test.sh
@ -0,0 +1,20 @@
+#/usr/bin/env bash
+
+# Run a previously built test, generating in `$DIR`:
+# - a `cmd.sh` script containing the actual command used to run the test in an
+#   engine
+# - a `fs` directory which the test may have used for file IO
+# - an `output.log` file containing the output of the test
+#
+# Usage: DIR=... WASM=... ENGINE=... ./run-test.sh
+
+ENGINE="${ENGINE:-wasmtime}"
+[ -n "$WASM" ] || (echo "missing WASM variable" && exit 1)
+[ -n "$DIR" ] || (echo "missing DIR variable" && exit 1)
+
+cd $DIR
+mkdir -p fs
+echo "$ENGINE $WASM" > cmd.sh
+chmod +x cmd.sh
+./cmd.sh &> output.log
+[ $? -eq 0 ] || echo "Test failed" >> output.log
--- a/test/smoke/smoke.mk
+++ b/test/smoke/smoke.mk
@ -1,35 +0,0 @@
-# Smoke test suite specific to wasi-libc
-#
-# This Makefile is included by the parent Makefile
-
-SMOKE_TESTS_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
-SMOKE_TESTS := $(wildcard $(SMOKE_TESTS_DIR)/*.c)
-SMOKE_OBJDIR := $(OBJDIR)/smoke
-SMOKE_WASMS := $(SMOKE_TESTS:$(SMOKE_TESTS_DIR)/%.c=$(SMOKE_OBJDIR)/%.core.wasm)
-SMOKE_ERRS := $(SMOKE_WASMS:%.core.wasm=%.wasm.err)
-
-$(SMOKE_OBJDIR):
-	mkdir -p $@
-
-$(SMOKE_OBJDIR)/%.core.wasm: $(SMOKE_TESTS_DIR)/%.c | $(SMOKE_OBJDIR)
-	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
-
-ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
-$(SMOKE_OBJDIR)/%.wasm: $(SMOKE_OBJDIR)/%.core.wasm
-	$(WASM_TOOLS) component new --adapt $(ADAPTER) $< -o $@
-endif
-
-ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
-$(SMOKE_OBJDIR)/%.wasm.err: $(SMOKE_OBJDIR)/%.wasm
-	rm -rf $(SMOKE_TESTS_DIR)/$*.dir
-	mkdir -p $(SMOKE_TESTS_DIR)/$*.dir
-	$(ENGINE) --dir $(SMOKE_TESTS_DIR)/$*.dir::/ --wasm component-model $< >$@
-else
-$(SMOKE_OBJDIR)/%.wasm.err: $(SMOKE_OBJDIR)/%.core.wasm
-	rm -rf $(SMOKE_TESTS_DIR)/$*.dir
-	mkdir -p $(SMOKE_TESTS_DIR)/$*.dir
-	$(ENGINE) --dir $(SMOKE_TESTS_DIR)/$*.dir::/ $< >$@
-endif
-
-run_smoke: $(SMOKE_ERRS)
-	@echo "Smoke tests passed"
--- a/test/src/libc-test/functional/argv.c
+++ b/test/src/libc-test/functional/argv.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/argv.c"
--- a/test/src/libc-test/functional/basename.c
+++ b/test/src/libc-test/functional/basename.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/basename.c"
--- a/test/src/libc-test/functional/clocale_mbfuncs.c
+++ b/test/src/libc-test/functional/clocale_mbfuncs.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/clocale_mbfuncs.c"
--- a/test/src/libc-test/functional/clock_gettime.c
+++ b/test/src/libc-test/functional/clock_gettime.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/clock_gettime.c"
--- a/test/src/libc-test/functional/crypt.c
+++ b/test/src/libc-test/functional/crypt.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/crypt.c"
--- a/test/src/libc-test/functional/dirname.c
+++ b/test/src/libc-test/functional/dirname.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/dirname.c"
--- a/test/src/libc-test/functional/env.c
+++ b/test/src/libc-test/functional/env.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/env.c"
--- a/test/src/libc-test/functional/fnmatch.c
+++ b/test/src/libc-test/functional/fnmatch.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/fnmatch.c"
--- a/test/src/libc-test/functional/iconv_open.c
+++ b/test/src/libc-test/functional/iconv_open.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/iconv_open.c"
--- a/test/src/libc-test/functional/mbc.c
+++ b/test/src/libc-test/functional/mbc.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/mbc.c"
--- a/test/src/libc-test/functional/memstream.c
+++ b/test/src/libc-test/functional/memstream.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/memstream.c"
--- a/test/src/libc-test/functional/qsort.c
+++ b/test/src/libc-test/functional/qsort.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/qsort.c"
--- a/test/src/libc-test/functional/random.c
+++ b/test/src/libc-test/functional/random.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/random.c"
--- a/test/src/libc-test/functional/search_hsearch.c
+++ b/test/src/libc-test/functional/search_hsearch.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/search_hsearch.c"
--- a/test/src/libc-test/functional/search_insque.c
+++ b/test/src/libc-test/functional/search_insque.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/search_insque.c"
--- a/test/src/libc-test/functional/search_lsearch.c
+++ b/test/src/libc-test/functional/search_lsearch.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/search_lsearch.c"
--- a/test/src/libc-test/functional/search_tsearch.c
+++ b/test/src/libc-test/functional/search_tsearch.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/search_tsearch.c"
--- a/test/src/libc-test/functional/snprintf.c
+++ b/test/src/libc-test/functional/snprintf.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/snprintf.c"
--- a/test/src/libc-test/functional/sscanf.c
+++ b/test/src/libc-test/functional/sscanf.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/sscanf.c"
--- a/test/src/libc-test/functional/strftime.c
+++ b/test/src/libc-test/functional/strftime.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/strftime.c"
--- a/test/src/libc-test/functional/string.c
+++ b/test/src/libc-test/functional/string.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/string.c"
--- a/test/src/libc-test/functional/string_memcpy.c
+++ b/test/src/libc-test/functional/string_memcpy.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/string_memcpy.c"
--- a/test/src/libc-test/functional/string_memmem.c
+++ b/test/src/libc-test/functional/string_memmem.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/string_memmem.c"
--- a/test/src/libc-test/functional/string_memset.c
+++ b/test/src/libc-test/functional/string_memset.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/string_memset.c"
--- a/test/src/libc-test/functional/string_strchr.c
+++ b/test/src/libc-test/functional/string_strchr.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/string_strchr.c"
--- a/test/src/libc-test/functional/string_strcspn.c
+++ b/test/src/libc-test/functional/string_strcspn.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/string_strcspn.c"
--- a/test/src/libc-test/functional/string_strstr.c
+++ b/test/src/libc-test/functional/string_strstr.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/string_strstr.c"
--- a/test/src/libc-test/functional/strtod.c
+++ b/test/src/libc-test/functional/strtod.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/strtod.c"
--- a/test/src/libc-test/functional/strtod_long.c
+++ b/test/src/libc-test/functional/strtod_long.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/strtod_long.c"
--- a/test/src/libc-test/functional/strtod_simple.c
+++ b/test/src/libc-test/functional/strtod_simple.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/strtod_simple.c"
--- a/test/src/libc-test/functional/strtof.c
+++ b/test/src/libc-test/functional/strtof.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/strtof.c"
--- a/test/src/libc-test/functional/strtol.c
+++ b/test/src/libc-test/functional/strtol.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/strtol.c"
--- a/test/src/libc-test/functional/strtold.c
+++ b/test/src/libc-test/functional/strtold.c
@ -0,0 +1,3 @@
+//! add-flags.py(CFLAGS): -I.
+//! add-flags.py(LDFLAGS): -lc-printscan-long-double
+#include "build/download/libc-test/src/functional/strtold.c"
--- a/test/src/libc-test/functional/swprintf.c
+++ b/test/src/libc-test/functional/swprintf.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/swprintf.c"
--- a/test/src/libc-test/functional/tgmath.c
+++ b/test/src/libc-test/functional/tgmath.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/tgmath.c"
--- a/test/src/libc-test/functional/udiv.c
+++ b/test/src/libc-test/functional/udiv.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/udiv.c"
--- a/test/src/libc-test/functional/wcsstr.c
+++ b/test/src/libc-test/functional/wcsstr.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/wcsstr.c"
--- a/test/src/libc-test/functional/wcstol.c
+++ b/test/src/libc-test/functional/wcstol.c
@ -0,0 +1,3 @@
+// Auto-generated by generate-stubs.sh
+//! add-flags.py(CFLAGS): -I.
+#include "build/download/libc-test/src/functional/wcstol.c"
--- a/test/smoke/test_fts.c
+++ b/test/smoke/test_fts.c
@ -1,3 +1,5 @@
+//! add-flags.py(RUN): --dir fs::/
+
 /*
 * We modified musl-fts not to use fchdir() and we made FTS_NOCHDIR
 * the default behavior. This test is to make sure that the modified