diff --git a/debian/changelog b/debian/changelog
index 1fb0c981..2adffe35 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -2,6 +2,8 @@ llvm-toolchain-17 (1:17.0.2-1~exp2) UNRELEASED; urgency=medium
 
   [ Matthias Klose ]
   * Consolidate the build flags for the wasm builds.
+  * Disable the clc and wasm builds when not building binary indep packages.
+  * Limit the parallel links to 2 on riscv64.
 
  -- Sylvestre Ledru <sylvestre@debian.org>  Wed, 04 Oct 2023 09:46:49 +0200
 
diff --git a/debian/rules b/debian/rules
index c536c257..f4b06101 100755
--- a/debian/rules
+++ b/debian/rules
@@ -195,6 +195,7 @@ endif
 # debug symbols OOM in Ubuntu Ubuntu and slows down build process.
 ifneq (,$(filter $(DEB_HOST_ARCH),riscv64))
   opt_flags = -O2 -DNDEBUG -g0
+  STAGE_ALL_CMAKE_EXTRA += -DLLVM_PARALLEL_LINK_JOBS=2
 else
   opt_flags = -O2 -DNDEBUG -g1
 endif
@@ -960,8 +961,8 @@ sccache-stats:
 override_dh_auto_build: \
   debian-full-build \
   debian-libfuzzer-build \
-  debian-libclc-build \
-  debian-wasm-build \
+  $(if $(filter libclc-%, $(shell dh_listpackages)),debian-libclc-build) \
+  $(if $(filter %-wasm32 %-wasm64, $(shell dh_listpackages)),debian-wasm-build) \
   sccache-stats
 
 override_dh_prep: build_doc
@@ -1112,10 +1113,16 @@ endif
 	# Remove an osx file
 	rm -f $(DEB_INST)/usr/lib/llvm-$(LLVM_VERSION)/lib/libbolt_rt_instr_osx.a
 
-	DESTDIR=$(DEB_INST) ninja $(VERBOSE) -C libclc/build install
+	if test -d libclc/build; then \
+		DESTDIR=$(DEB_INST) ninja $(VERBOSE) -C libclc/build install; \
+	fi
 
-	DESTDIR=$(DEB_INST) ninja $(VERBOSE) -C build-wasm/compiler-rt-wasm32 install
-	DESTDIR=$(DEB_INST) ninja $(VERBOSE) -C build-wasm/compiler-rt-wasm64 install
+	if test -d build-wasm/compiler-rt-wasm32; then \
+		DESTDIR=$(DEB_INST) ninja $(VERBOSE) -C build-wasm/compiler-rt-wasm32 install; \
+	fi
+	if test -d build-wasm/compiler-rt-wasm64; then \
+		DESTDIR=$(DEB_INST) ninja $(VERBOSE) -C build-wasm/compiler-rt-wasm64 install; \
+	fi
 	if test -d build-wasm/libcxx-wasm32-wasi; then \
 		DESTDIR=$(DEB_INST) ninja $(VERBOSE) -C build-wasm/libcxx-wasm32-wasi install; \
 	fi