Fix make THREAD_MODEL=posix (#311)

* Fixes for the THREAD_MODEL=posix build

* Fix expected symbols from previous commit

* Enable `lock` in `random.c` when threads are enabled

This uses the `_REENTRANT` definition to indicate when the `lock` should
be available.

* Disable `aio.h` when compiling for threads

In talking to @sunfishcode about `aio.h`, this functionality is not yet
a primary concern (it was already disabled in the default,
single-threaded mode). Additionally, this change adds expectation lines
for the new symbols/includes added and removed by `pthread.h`.

This change was reached by running:
```console
$ git diff --no-index expected/wasm32-wasi sysroot/share/wasm32-wasi > patch.diff
# replace "sysroot/share" with "expected" in `patch.diff`
$ git apply patch.diff --reject
# manually fix any rejections
```

* Specify the TLS model until LLVM 15 is released

The `-ftls-model` configuration can be removed once https://reviews.llvm.org/D130053 makes its way into an upstream release.

* Rename `__wasi_libc_pthread_self` to `__wasilibc_pthread_self`

The symbol is still undefined, though.

* Add different sets of expected output based on THREAD_MODEL

* Re-add trailing whitespace to `predefined-macros.txt`

@sbc100 wanted to retain the whitespace trailing after certain
predefined macro lines. This change restores that whitespace from
upstream and re-generates the POSIX version using the following command:

```console
$ git diff --no-index expected/wasm32-wasi/posix/predefined-macros.txt sysroot/share/wasm32-wasi/predefined-macros.txt | sed 's/sysroot\/share\/wasm32-wasi/expected\/wasm32-wasi\/posix/' | git apply
```

* Protect `preopens.c` against concurrent access

* Only build thread-capable wasi-libc on latest version of Clang

* Use `thrd_sleep` from MUSL instead of aliasing `nanosleep`

* Define `pthread_setcancelstate` in `THREAD_MODEL=posix` builds

There are other options here (e.g., always define the `pthread_*`
symbols with stubs) but until we discuss that this is an intermediate
working step.

* Define a Wasm global to store `pthread_self`

* Remove `g_needs_dynamic_alloc` global

* Document the state of pthread support

* review: de-duplicate symbols based on #314

* review: only define `__wasilibc_cwd_{un}lock` when needed

* review: add #ifdefs to `__pthread_setcancelstate`

* review: add additional #ifdefs to `pthread_self.c`

* review: put lock definition behind #ifdef _REENTRANT

* review: remove pthread_setcancelstate.c

* review: re-fix indentation

* review: alias __clock_nanosleep in bottom half

* review: remove extra line

Co-authored-by: Sam Clegg <sbc@chromium.org>

.github/workflows/main.yml

@@ -66,6 +66,14 @@ jobs:
         name: ${{ format( 'sysroot-{0}.tgz', matrix.os) }}
         path: sysroot
 
+    - name: Build libc + threads
+      # Only build the thread-capable wasi-libc in the latest supported Clang
+      # version; the earliest version does not have all necessary builtins
+      # (e.g., `__builtin_wasm_memory_atomic_notify`).
+      if: matrix.clang_version != '10.0.0'
+      shell: bash
+      run: make -j4 THREAD_MODEL=posix
+
   # Disable the headerstest job for now, while WASI transitions from the
   # witx snapshots to wit proposals, and we have a few manual edits to the
   # generated header to make life easier for folks.

Makefile

@@ -13,7 +13,7 @@ EXTRA_CFLAGS ?= -O2 -DNDEBUG
 SYSROOT ?= $(CURDIR)/sysroot
 # A directory to install to for "make install".
 INSTALL_DIR ?= /usr/local
-# single or posix
+# single or posix; note that pthread support is still a work-in-progress.
 THREAD_MODEL ?= single
 # dlmalloc or none
 MALLOC_IMPL ?= dlmalloc
@@ -147,6 +147,7 @@ LIBC_TOP_HALF_MUSL_SOURCES = \
         unistd/posix_close.c \
         stat/futimesat.c \
         legacy/getpagesize.c \
+        thread/thrd_sleep.c \
     ) \
     $(filter-out %/procfdname.c %/syscall.c %/syscall_ret.c %/vdso.c %/version.c, \
                  $(wildcard $(LIBC_TOP_HALF_MUSL_SRC_DIR)/internal/*.c)) \
@@ -185,6 +186,7 @@ LIBC_TOP_HALF_MUSL_SOURCES = \
                  %/cimagf.c %/cimag.c %cimagl.c, \
                  $(wildcard $(LIBC_TOP_HALF_MUSL_SRC_DIR)/complex/*.c)) \
     $(wildcard $(LIBC_TOP_HALF_MUSL_SRC_DIR)/crypt/*.c)
+
 MUSL_PRINTSCAN_SOURCES = \
     $(LIBC_TOP_HALF_MUSL_SRC_DIR)/internal/floatscan.c \
     $(LIBC_TOP_HALF_MUSL_SRC_DIR)/stdio/vfprintf.c \
@@ -227,7 +229,9 @@ ifeq ($(THREAD_MODEL), single)
 CFLAGS += -mthread-model single
 endif
 ifeq ($(THREAD_MODEL), posix)
-CFLAGS += -mthread-model posix -pthread
+# Specify the tls-model until LLVM 15 is released (which should contain
+# https://reviews.llvm.org/D130053).
+CFLAGS += -mthread-model posix -pthread -ftls-model=local-exec
 endif
 
 # Expose the public headers to the implementation. We use `-isystem` for
@@ -356,11 +360,12 @@ MUSL_OMIT_HEADERS += \
     "netinet/ether.h" \
     "sys/timerfd.h" \
     "libintl.h" \
-    "sys/sysmacros.h"
+    "sys/sysmacros.h" \
+    "aio.h"
 
 ifeq ($(THREAD_MODEL), single)
 # Remove headers not supported in single-threaded mode.
-MUSL_OMIT_HEADERS += "aio.h" "pthread.h"
+MUSL_OMIT_HEADERS += "pthread.h"
 endif
 
 default: finish
@@ -595,7 +600,7 @@ check-symbols: startup_files libc
 
 	# Check that the computed metadata matches the expected metadata.
 	# This ignores whitespace because on Windows the output has CRLF line endings.
-	diff -wur "$(CURDIR)/expected/$(MULTIARCH_TRIPLE)" "$(SYSROOT_SHARE)"
+	diff -wur "$(CURDIR)/expected/$(MULTIARCH_TRIPLE)/$(THREAD_MODEL)" "$(SYSROOT_SHARE)"
 
 install: finish
 	mkdir -p "$(INSTALL_DIR)"

README.md

@@ -7,7 +7,8 @@ environment variables, program startup, and many other APIs.
 
 WASI Libc is sufficiently stable and usable for many purposes, as most of the
 POSIX-compatible APIs are stable, though it is continuing to evolve to better
-align with wasm and WASI.
+align with wasm and WASI. For example, pthread support is still a work in
+progress.
 
 ## Usage
 

expected/wasm32-wasi/posix/include-all.c

@@ -0,0 +1,171 @@
+#include <__errno.h>
+#include <__errno_values.h>
+#include <__fd_set.h>
+#include <__function___isatty.h>
+#include <__functions_malloc.h>
+#include <__functions_memcpy.h>
+#include <__header_dirent.h>
+#include <__header_fcntl.h>
+#include <__header_inttypes.h>
+#include <__header_netinet_in.h>
+#include <__header_poll.h>
+#include <__header_stdlib.h>
+#include <__header_string.h>
+#include <__header_sys_ioctl.h>
+#include <__header_sys_resource.h>
+#include <__header_sys_socket.h>
+#include <__header_sys_stat.h>
+#include <__header_time.h>
+#include <__header_unistd.h>
+#include <__macro_FD_SETSIZE.h>
+#include <__macro_PAGESIZE.h>
+#include <__mode_t.h>
+#include <__seek.h>
+#include <__struct_dirent.h>
+#include <__struct_in6_addr.h>
+#include <__struct_in_addr.h>
+#include <__struct_iovec.h>
+#include <__struct_msghdr.h>
+#include <__struct_pollfd.h>
+#include <__struct_rusage.h>
+#include <__struct_sockaddr.h>
+#include <__struct_sockaddr_in.h>
+#include <__struct_sockaddr_in6.h>
+#include <__struct_sockaddr_storage.h>
+#include <__struct_sockaddr_un.h>
+#include <__struct_stat.h>
+#include <__struct_timespec.h>
+#include <__struct_timeval.h>
+#include <__struct_tm.h>
+#include <__struct_tms.h>
+#include <__typedef_DIR.h>
+#include <__typedef_blkcnt_t.h>
+#include <__typedef_blksize_t.h>
+#include <__typedef_clock_t.h>
+#include <__typedef_clockid_t.h>
+#include <__typedef_dev_t.h>
+#include <__typedef_fd_set.h>
+#include <__typedef_gid_t.h>
+#include <__typedef_in_addr_t.h>
+#include <__typedef_in_port_t.h>
+#include <__typedef_ino_t.h>
+#include <__typedef_mode_t.h>
+#include <__typedef_nfds_t.h>
+#include <__typedef_nlink_t.h>
+#include <__typedef_off_t.h>
+#include <__typedef_sa_family_t.h>
+#include <__typedef_sigset_t.h>
+#include <__typedef_socklen_t.h>
+#include <__typedef_ssize_t.h>
+#include <__typedef_suseconds_t.h>
+#include <__typedef_time_t.h>
+#include <__typedef_uid_t.h>
+#include <alloca.h>
+#include <ar.h>
+#include <arpa/ftp.h>
+#include <arpa/inet.h>
+#include <arpa/nameser.h>
+#include <arpa/nameser_compat.h>
+#include <arpa/telnet.h>
+#include <arpa/tftp.h>
+#include <assert.h>
+#include <byteswap.h>
+#include <complex.h>
+#include <cpio.h>
+#include <crypt.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <endian.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <features.h>
+#include <fenv.h>
+#include <float.h>
+#include <fmtmsg.h>
+#include <fnmatch.h>
+#include <ftw.h>
+#include <getopt.h>
+#include <glob.h>
+#include <iconv.h>
+#include <ifaddrs.h>
+#include <inttypes.h>
+#include <iso646.h>
+#include <langinfo.h>
+#include <libgen.h>
+#include <limits.h>
+#include <locale.h>
+#include <malloc.h>
+#include <math.h>
+#include <memory.h>
+#include <monetary.h>
+#include <mqueue.h>
+#include <netinet/icmp6.h>
+#include <netinet/igmp.h>
+#include <netinet/in.h>
+#include <netinet/in_systm.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/ip_icmp.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <netpacket/packet.h>
+#include <nl_types.h>
+#include <poll.h>
+#include <pthread.h>
+#include <regex.h>
+#include <sched.h>
+#include <search.h>
+#include <semaphore.h>
+#include <stdalign.h>
+#include <stdbool.h>
+#include <stdc-predef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdio_ext.h>
+#include <stdlib.h>
+#include <stdnoreturn.h>
+#include <string.h>
+#include <strings.h>
+#include <stropts.h>
+#include <sys/dir.h>
+#include <sys/errno.h>
+#include <sys/eventfd.h>
+#include <sys/fcntl.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/param.h>
+#include <sys/poll.h>
+#include <sys/random.h>
+#include <sys/reg.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/stropts.h>
+#include <sys/syscall.h>
+#include <sys/sysinfo.h>
+#include <sys/time.h>
+#include <sys/timeb.h>
+#include <sys/timex.h>
+#include <sys/ttydefaults.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <sys/utsname.h>
+#include <syscall.h>
+#include <sysexits.h>
+#include <tar.h>
+#include <tgmath.h>
+#include <threads.h>
+#include <time.h>
+#include <uchar.h>
+#include <unistd.h>
+#include <utime.h>
+#include <values.h>
+#include <wasi/api.h>
+#include <wasi/libc-environ.h>
+#include <wasi/libc-find-relpath.h>
+#include <wasi/libc-nocwd.h>
+#include <wasi/libc.h>
+#include <wchar.h>
+#include <wctype.h>

expected/wasm32-wasi/posix/undefined-symbols.txt

@@ -0,0 +1,75 @@
+__addtf3
+__divtf3
+__eqtf2
+__extenddftf2
+__extendsftf2
+__fixtfdi
+__fixtfsi
+__fixunstfsi
+__floatsitf
+__floatunsitf
+__getf2
+__gttf2
+__heap_base
+__imported_wasi_snapshot_preview1_args_get
+__imported_wasi_snapshot_preview1_args_sizes_get
+__imported_wasi_snapshot_preview1_clock_res_get
+__imported_wasi_snapshot_preview1_clock_time_get
+__imported_wasi_snapshot_preview1_environ_get
+__imported_wasi_snapshot_preview1_environ_sizes_get
+__imported_wasi_snapshot_preview1_fd_advise
+__imported_wasi_snapshot_preview1_fd_allocate
+__imported_wasi_snapshot_preview1_fd_close
+__imported_wasi_snapshot_preview1_fd_datasync
+__imported_wasi_snapshot_preview1_fd_fdstat_get
+__imported_wasi_snapshot_preview1_fd_fdstat_set_flags
+__imported_wasi_snapshot_preview1_fd_fdstat_set_rights
+__imported_wasi_snapshot_preview1_fd_filestat_get
+__imported_wasi_snapshot_preview1_fd_filestat_set_size
+__imported_wasi_snapshot_preview1_fd_filestat_set_times
+__imported_wasi_snapshot_preview1_fd_pread
+__imported_wasi_snapshot_preview1_fd_prestat_dir_name
+__imported_wasi_snapshot_preview1_fd_prestat_get
+__imported_wasi_snapshot_preview1_fd_pwrite
+__imported_wasi_snapshot_preview1_fd_read
+__imported_wasi_snapshot_preview1_fd_readdir
+__imported_wasi_snapshot_preview1_fd_renumber
+__imported_wasi_snapshot_preview1_fd_seek
+__imported_wasi_snapshot_preview1_fd_sync
+__imported_wasi_snapshot_preview1_fd_tell
+__imported_wasi_snapshot_preview1_fd_write
+__imported_wasi_snapshot_preview1_path_create_directory
+__imported_wasi_snapshot_preview1_path_filestat_get
+__imported_wasi_snapshot_preview1_path_filestat_set_times
+__imported_wasi_snapshot_preview1_path_link
+__imported_wasi_snapshot_preview1_path_open
+__imported_wasi_snapshot_preview1_path_readlink
+__imported_wasi_snapshot_preview1_path_remove_directory
+__imported_wasi_snapshot_preview1_path_rename
+__imported_wasi_snapshot_preview1_path_symlink
+__imported_wasi_snapshot_preview1_path_unlink_file
+__imported_wasi_snapshot_preview1_poll_oneoff
+__imported_wasi_snapshot_preview1_proc_exit
+__imported_wasi_snapshot_preview1_random_get
+__imported_wasi_snapshot_preview1_sched_yield
+__imported_wasi_snapshot_preview1_sock_accept
+__imported_wasi_snapshot_preview1_sock_recv
+__imported_wasi_snapshot_preview1_sock_send
+__imported_wasi_snapshot_preview1_sock_shutdown
+__letf2
+__lock
+__lockfile
+__lttf2
+__main_argc_argv
+__netf2
+__stack_pointer
+__subtf3
+__tls_base
+__trunctfdf2
+__trunctfsf2
+__unlock
+__unlockfile
+__unordtf2
+__wasilibc_pthread_self
+__wasm_call_ctors
+pthread_setcancelstate

expected/wasm32-wasi/single/defined-symbols.txt

@@ -18,6 +18,7 @@ __c_dot_utf8_locale
 __c_locale
 __clock
 __clock_gettime
+__clock_nanosleep
 __cos
 __cosdf
 __cosl
@@ -1082,6 +1083,7 @@ tfind
 tgamma
 tgammaf
 tgammal
+thrd_sleep
 time
 timegm
 times

libc-bottom-half/cloudlibc/src/libc/time/clock_nanosleep.c

@@ -33,3 +33,5 @@ int clock_nanosleep(clockid_t clock_id, int flags, const struct timespec *rqtp,
   __wasi_errno_t error = __wasi_poll_oneoff(&sub, &ev, 1, &nevents);
   return error == 0 && ev.error == 0 ? 0 : ENOTSUP;
 }
+
+weak_alias(clock_nanosleep, __clock_nanosleep);

libc-bottom-half/cloudlibc/src/libc/time/nanosleep.c

@@ -3,9 +3,7 @@
 // SPDX-License-Identifier: BSD-2-Clause
 
 #include <errno.h>
-#include <threads.h>
 #include <time.h>
-#include <_/cdefs.h>
 
 int nanosleep(const struct timespec *rqtp, struct timespec *rem) {
   int error = clock_nanosleep(CLOCK_REALTIME, 0, rqtp, rem);
@@ -15,7 +13,3 @@ int nanosleep(const struct timespec *rqtp, struct timespec *rem) {
   }
   return 0;
 }
-
-#if defined(_REENTRANT)
-__strong_reference(nanosleep, thrd_sleep);
-#endif

libc-bottom-half/sources/chdir.c

@@ -9,9 +9,12 @@
 #include <wasi/libc.h>
 
 #ifdef _REENTRANT
-#error "chdir doesn't yet support multiple threads"
+void __wasilibc_cwd_lock(void);
+void __wasilibc_cwd_unlock(void);
+#else
+#define __wasilibc_cwd_lock() (void)0
+#define __wasilibc_cwd_unlock() (void)0
 #endif
-
 extern char *__wasilibc_cwd;
 static int __wasilibc_cwd_mallocd = 0;
 
@@ -60,8 +63,10 @@ int chdir(const char *path)
 
     // And set our new malloc'd buffer into the global cwd, freeing the
     // previous one if necessary.
+    __wasilibc_cwd_lock();
     char *prev_cwd = __wasilibc_cwd;
     __wasilibc_cwd = new_cwd;
+    __wasilibc_cwd_unlock();
     if (__wasilibc_cwd_mallocd)
         free(prev_cwd);
     __wasilibc_cwd_mallocd = 1;
@@ -77,11 +82,13 @@ static const char *make_absolute(const char *path) {
         return path;
     }
 
+#ifndef _REENTRANT
     // If the path is empty, or points to the current directory, then return
     // the current directory.
     if (path[0] == 0 || !strcmp(path, ".") || !strcmp(path, "./")) {
         return __wasilibc_cwd;
     }
+#endif
 
     // If the path starts with `./` then we won't be appending that to the cwd.
     if (path[0] == '.' && path[1] == '/')
@@ -90,18 +97,30 @@ static const char *make_absolute(const char *path) {
     // Otherwise we'll take the current directory, add a `/`, and then add the
     // input `path`. Note that this doesn't do any normalization (like removing
     // `/./`).
+    __wasilibc_cwd_lock();
     size_t cwd_len = strlen(__wasilibc_cwd);
-    size_t path_len = strlen(path);
+    size_t path_len = path ? strlen(path) : 0;
+    __wasilibc_cwd_unlock();
     int need_slash = __wasilibc_cwd[cwd_len - 1] == '/' ? 0 : 1;
     size_t alloc_len = cwd_len + path_len + 1 + need_slash;
     if (alloc_len > make_absolute_len) {
         char *tmp = realloc(make_absolute_buf, alloc_len);
-        if (tmp == NULL)
+        if (tmp == NULL) {
+            __wasilibc_cwd_unlock();
             return NULL;
+        }
         make_absolute_buf = tmp;
         make_absolute_len = alloc_len;
     }
     strcpy(make_absolute_buf, __wasilibc_cwd);
+    __wasilibc_cwd_unlock();
+
+#ifdef _REENTRANT
+    if (path[0] == 0 || !strcmp(path, ".") || !strcmp(path, "./")) {
+        return make_absolute_buf;
+    }
+#endif
+
     if (need_slash)
         strcpy(make_absolute_buf + cwd_len, "/");
     strcpy(make_absolute_buf + cwd_len + need_slash, path);

libc-bottom-half/sources/getcwd.c

@@ -1,30 +1,39 @@
 #include <unistd.h>
 #include <errno.h>
 #include <string.h>
-
-// For threads this needs to synchronize with chdir
-#ifdef _REENTRANT
-#error "getcwd doesn't yet support multiple threads"
-#endif
+#include "lock.h"
 
 char *__wasilibc_cwd = "/";
 
+#ifdef _REENTRANT
+static volatile int lock[1];
+void __wasilibc_cwd_lock(void) { LOCK(lock); }
+void __wasilibc_cwd_unlock(void) { UNLOCK(lock); }
+#else
+#define __wasilibc_cwd_lock() (void)0
+#define __wasilibc_cwd_unlock() (void)0
+#endif
+
 char *getcwd(char *buf, size_t size)
 {
+    __wasilibc_cwd_lock();
     if (!buf) {
         buf = strdup(__wasilibc_cwd);
         if (!buf) {
             errno = ENOMEM;
+            __wasilibc_cwd_unlock();
             return NULL;
         }
     } else {
         size_t len = strlen(__wasilibc_cwd);
         if (size < len + 1) {
             errno = ERANGE;
+            __wasilibc_cwd_unlock();
             return NULL;
         }
         strcpy(buf, __wasilibc_cwd);
     }
+    __wasilibc_cwd_unlock();
     return buf;
 }
 

libc-bottom-half/sources/getentropy.c

@@ -1,10 +1,6 @@
-#include <wasi/api.h>
 #include <errno.h>
 #include <unistd.h>
-
-#ifdef _REENTRANT
-#error With threads support, getentropy is not intended to be a cancellation point.
-#endif
+#include <wasi/api.h>
 
 int __getentropy(void *buffer, size_t len) {
     if (len > 256) {

libc-bottom-half/sources/preopens.c

@@ -2,14 +2,11 @@
 //! environment, with associated path prefixes, which can be used to map
 //! absolute paths to capabilities with relative paths.
 
-#ifdef _REENTRANT
-#error "__wasilibc_register_preopened_fd doesn't yet support multiple threads"
-#endif
-
 #include <assert.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <limits.h>
+#include <lock.h>
 #include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
@@ -32,6 +29,12 @@ static preopen *preopens;
 static size_t num_preopens;
 static size_t preopen_capacity;
 
+/// Access to the the above preopen must be protected in the presence of
+/// threads.
+#ifdef _REENTRANT
+static volatile int lock[1];
+#endif
+
 #ifdef NDEBUG
 #define assert_invariants() // assertions disabled
 #else
@@ -55,6 +58,7 @@ static void assert_invariants(void) {
 
 /// Allocate space for more preopens. Returns 0 on success and -1 on failure.
 static int resize(void) {
+    LOCK(lock);
     size_t start_capacity = 4;
     size_t old_capacity = preopen_capacity;
     size_t new_capacity = old_capacity == 0 ? start_capacity : old_capacity * 2;
@@ -62,6 +66,7 @@ static int resize(void) {
     preopen *old_preopens = preopens;
     preopen *new_preopens = calloc(sizeof(preopen), new_capacity);
     if (new_preopens == NULL)
+        UNLOCK(lock);
         return -1;
 
     memcpy(new_preopens, old_preopens, num_preopens * sizeof(preopen));
@@ -70,6 +75,7 @@ static int resize(void) {
     free(old_preopens);
 
     assert_invariants();
+    UNLOCK(lock);
     return 0;
 }
 
@@ -97,6 +103,8 @@ static const char *strip_prefixes(const char *path) {
 ///
 /// This function takes ownership of `prefix`.
 static int internal_register_preopened_fd(__wasi_fd_t fd, const char *relprefix) {
+    LOCK(lock);
+
     // Check preconditions.
     assert_invariants();
     assert(fd != AT_FDCWD);
@@ -104,14 +112,17 @@ static int internal_register_preopened_fd(__wasi_fd_t fd, const char *relprefix)
     assert(relprefix != NULL);
 
     if (num_preopens == preopen_capacity && resize() != 0)
+        UNLOCK(lock);
         return -1;
 
     char *prefix = strdup(strip_prefixes(relprefix));
     if (prefix == NULL)
+        UNLOCK(lock);
         return -1;
     preopens[num_preopens++] = (preopen) { prefix, fd, };
 
     assert_invariants();
+    UNLOCK(lock);
     return 0;
 }
 
@@ -166,6 +177,7 @@ int __wasilibc_find_abspath(const char *path,
     // recently added preopens take precedence over less recently addded ones.
     size_t match_len = 0;
     int fd = -1;
+    LOCK(lock);
     for (size_t i = num_preopens; i > 0; --i) {
         const preopen *pre = &preopens[i - 1];
         const char *prefix = pre->prefix;
@@ -182,6 +194,7 @@ int __wasilibc_find_abspath(const char *path,
             *abs_prefix = prefix;
         }
     }
+    UNLOCK(lock);
 
     if (fd == -1) {
         errno = ENOENT;

libc-top-half/musl/arch/wasm32/atomic_arch.h

@@ -1,7 +1,3 @@
-#ifdef _REENTRANT
-#error "multiple threads not supported in musl yet"
-#endif
-
 #define a_barrier() (__sync_synchronize())
 #define a_cas(p, t, s) (__sync_val_compare_and_swap((p), (t), (s)))
 #define a_crash() (__builtin_trap())

libc-top-half/musl/arch/wasm32/pthread_arch.h

@@ -1,12 +1,11 @@
-#ifdef _REENTRANT
-#error "multiple threads not supported in musl yet"
+uintptr_t __get_tp(void) {
+#if _REENTRANT
+  int val;
+  __asm__("global.get __wasilibc_pthread_self\n"
+          "local.set %0"
+          : "=r"(val));
+  return val;
+#else
+  return 0;
 #endif
-
-static inline struct pthread *__pthread_self(void)
-{
-  return (struct pthread *)-1;
 }
-
-#define TP_ADJ(p) (p)
-
-#define tls_mod_off_t unsigned long long

libc-top-half/musl/include/pthread.h

@@ -103,8 +103,10 @@ int pthread_setcanceltype(int, int *);
 void pthread_testcancel(void);
 int pthread_cancel(pthread_t);
 
+#ifdef __wasilibc_unmodified_upstream /* WASI has no CPU scheduling support. */
 int pthread_getschedparam(pthread_t, int *__restrict, struct sched_param *__restrict);
 int pthread_setschedparam(pthread_t, int, const struct sched_param *);
+#endif
 int pthread_setschedprio(pthread_t, int);
 
 int pthread_once(pthread_once_t *, void (*)(void));
@@ -167,8 +169,10 @@ int pthread_attr_getscope(const pthread_attr_t *__restrict, int *__restrict);
 int pthread_attr_setscope(pthread_attr_t *, int);
 int pthread_attr_getschedpolicy(const pthread_attr_t *__restrict, int *__restrict);
 int pthread_attr_setschedpolicy(pthread_attr_t *, int);
+#ifdef __wasilibc_unmodified_upstream /* WASI has no CPU scheduling support. */
 int pthread_attr_getschedparam(const pthread_attr_t *__restrict, struct sched_param *__restrict);
 int pthread_attr_setschedparam(pthread_attr_t *__restrict, const struct sched_param *__restrict);
+#endif
 int pthread_attr_getinheritsched(const pthread_attr_t *__restrict, int *__restrict);
 int pthread_attr_setinheritsched(pthread_attr_t *, int);
 

libc-top-half/musl/src/conf/sysconf.c

@@ -251,7 +251,7 @@ long sysconf(int name)
 		return DELAYTIMER_MAX;
 	case JT_NPROCESSORS_CONF & 255:
 	case JT_NPROCESSORS_ONLN & 255: ;
-#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
+#ifdef __wasilibc_unmodified_upstream
 		unsigned char set[128] = {1};
 		int i, cnt;
 		__syscall(SYS_sched_getaffinity, 0, sizeof set, set);
@@ -259,7 +259,7 @@ long sysconf(int name)
 			for (; set[i]; set[i]&=set[i]-1, cnt++);
 		return cnt;
 #else
-		// With no thread support, just say there's 1 processor.
+		// WASI has no way to query the processor count
 		return 1;
 #endif
 #ifdef __wasilibc_unmodified_upstream // WASI has no sysinfo

libc-top-half/musl/src/internal/pthread_impl.h

@@ -2,12 +2,18 @@
 #define _PTHREAD_IMPL_H
 
 #include <pthread.h>
+#ifdef __wasilibc_unmodified_upstream
 #include <signal.h>
+#endif
 #include <errno.h>
 #include <limits.h>
+#ifdef __wasilibc_unmodified_upstream
 #include <sys/mman.h>
+#endif
 #include "libc.h"
+#ifdef __wasilibc_unmodified_upstream
 #include "syscall.h"
+#endif
 #include "atomic.h"
 #include "futex.h"
 
@@ -159,7 +165,9 @@ extern hidden volatile int __eintr_valid_flag;
 
 hidden int __clone(int (*)(void *), void *, int, void *, ...);
 hidden int __set_thread_area(void *);
+#ifdef __wasilibc_unmodified_upstream /* WASI has no sigaction */
 hidden int __libc_sigaction(int, const struct sigaction *, struct sigaction *);
+#endif
 hidden void __unmapself(void *, size_t);
 
 hidden int __timedwait(volatile int *, int, clockid_t, const struct timespec *, int);
@@ -169,14 +177,22 @@ static inline void __wake(volatile void *addr, int cnt, int priv)
 {
 	if (priv) priv = FUTEX_PRIVATE;
 	if (cnt<0) cnt = INT_MAX;
+#ifdef __wasilibc_unmodified_upstream
 	__syscall(SYS_futex, addr, FUTEX_WAKE|priv, cnt) != -ENOSYS ||
 	__syscall(SYS_futex, addr, FUTEX_WAKE, cnt);
+#else
+	__builtin_wasm_memory_atomic_notify((int*)addr, cnt);
+#endif
 }
 static inline void __futexwait(volatile void *addr, int val, int priv)
 {
+#ifdef __wasilibc_unmodified_upstream
 	if (priv) priv = FUTEX_PRIVATE;
 	__syscall(SYS_futex, addr, FUTEX_WAIT|priv, val, 0) != -ENOSYS ||
 	__syscall(SYS_futex, addr, FUTEX_WAIT, val, 0);
+#else
+	__wait(addr, NULL, val, priv);
+#endif
 }
 
 hidden void __acquire_ptc(void);

libc-top-half/musl/src/prng/random.c

@@ -23,7 +23,7 @@ static int n = 31;
 static int i = 3;
 static int j = 0;
 static uint32_t *x = init+1;
-#ifdef __wasilibc_unmodified_upstream
+#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
 static volatile int lock[1];
 volatile int *const __random_lockptr = lock;
 #endif

libc-top-half/musl/src/thread/pthread_self.c

@@ -1,6 +1,13 @@
 #include "pthread_impl.h"
 #include <threads.h>
 
+#if !defined(__wasilibc_unmodified_upstream) && defined(__wasm__) &&           \
+    defined(_REENTRANT)
+// We need some place to store the thread ID. This WebAssembly global fits the
+// bill and is used by `__get_tp` elsewhere.
+__asm__(".globaltype __wasilibc_pthread_self, i32\n");
+#endif
+
 static pthread_t __pthread_self_internal()
 {
 	return __pthread_self();

libc-top-half/musl/src/thread/pthread_setcancelstate.c

@@ -2,10 +2,12 @@
 
 int __pthread_setcancelstate(int new, int *old)
 {
+#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
 	if (new > 2U) return EINVAL;
 	struct pthread *self = __pthread_self();
 	if (old) *old = self->canceldisable;
 	self->canceldisable = new;
+#endif
 	return 0;
 }
 

libc-top-half/musl/src/thread/thrd_sleep.c

@@ -1,7 +1,9 @@
 #include <threads.h>
 #include <time.h>
 #include <errno.h>
+#ifdef __wasilibc_unmodified_upstream
 #include "syscall.h"
+#endif
 
 int thrd_sleep(const struct timespec *req, struct timespec *rem)
 {