diff --git a/configure.ac b/configure.ac
index c9af89c8..d304d248 100644
--- a/configure.ac
+++ b/configure.ac
@@ -294,11 +294,10 @@ if test "x$enable_hardening" != "xno"; then
 		HARDENING_CFLAGS="-fstack-protector-strong "
 	fi
 
-	dnl Must not have -O0 but must have a -O for -D_FORTIFY_SOURCE=2
-	TMP1="$(echo $CFLAGS | sed -n 's/.*\(-O0\).*/\1/p')"
-	TMP2="$(echo $CFLAGS | sed -n 's/.*\(-O\).*/\1/p')"
-	if test -z "$TMP1" && test -n "$TPM2"; then
-		HARDENING_CFLAGS="$HARDENING_CFLAGS -D_FORTIFY_SOURCE=2 "
+	dnl Only support -D_FORTIFY_SOURCE=2 and have higher levels passed in by user
+	dnl since they may create more overhead
+	if $CC $CFLAGS -Werror -D_FORTIFY_SOURCE=2 $srcdir/include/libtpms/tpm_library.h 2>/dev/null; then
+		HARDENING_CFLAGS="$HARDENING_CFLAGS -D_FORTIFY_SOURCE=2"
 	fi
 	dnl Check linker for 'relro' and 'now'
 	save_CFLAGS="$CFLAGS"