github: Enable AddressSanitizer if leak-detective is disabled

At least for the tests where it is available and works.  It conflicts
with the instrumentation used by the coverage and fuzzing (and possibly
sonarcloud) tests, the toolchain for the Windows builds doesn't seem to
support it, and on FreeBSD the test executables hang due to a
compatibility issue with FreeBSD's qsort(), which has been fixed [1],
but that has not made it into the clang version in the base system.

For the custom OpenSSL build, debug symbols are enabled so we can
suppress some leaks properly.

[1] https://github.com/llvm/llvm-project/issues/46176

scripts/test.sh

@@ -96,7 +96,7 @@ build_openssl()
 	SSL_DIR=$DEPS_BUILD_DIR/$SSL_PKG
 	SSL_SRC=https://www.openssl.org/source/$SSL_PKG.tar.gz
 	SSL_INS=$DEPS_PREFIX/ssl
-	SSL_OPT="shared no-tls no-dtls no-ssl3 no-zlib no-comp no-idea no-psk no-srp
+	SSL_OPT="-d shared no-tls no-dtls no-ssl3 no-zlib no-comp no-idea no-psk no-srp
 			 no-stdio no-tests enable-rfc3779 enable-ec_nistp_64_gcc_128"
 
 	if test -d "$SSL_DIR"; then
@@ -471,6 +471,21 @@ CONFIG="$CONFIG
 	--enable-monolithic=${MONOLITHIC-no}
 	--enable-leak-detective=${LEAK_DETECTIVE-no}"
 
+case "$TEST" in
+	coverage|freebsd|fuzzing|sonarcloud|win*)
+		# don't use AddressSanitizer if it's not available or causes conflicts
+		CONFIG="$CONFIG --disable-asan"
+		;;
+	*)
+		if [ "$ID" = "ubuntu" -a "$VERSION_ID" = "18.04" ]; then
+			# the libstdc++ workaround for libbotan doesn't work on Ubuntu 18.04
+			CONFIG="$CONFIG --disable-asan"
+		elif [ "$LEAK_DETECTIVE" != "yes" ]; then
+			CONFIG="$CONFIG --enable-asan"
+		fi
+		;;
+esac
+
 echo "$ ./autogen.sh"
 ./autogen.sh || exit $?
 echo "$ CC=$CC CFLAGS=\"$CFLAGS\" ./configure $CONFIG"