Support AddressSanitizer on GCC

.github/workflows/main.yml

@@ -6,6 +6,9 @@ env:
   DEFAULT_RUBY_VERSION: 3.1
   BUNDLE_CLEAN: true
   USE_ASAN: true
+  # We probably don't have true memory leaks, but AddressSanitizer complains
+  # about it anyway. Disable leak detection for now.
+  ASAN_OPTIONS: detect_leaks=0
 
 on:
   push: {}
@@ -59,6 +62,7 @@ jobs:
           sudo env
           PATH="$PATH"
           GEM_PATH="$GEM_PATH"
+          ASAN_OPTIONS="$ASAN_OPTIONS"
           ../buildout/test/cxx/main
         working-directory: test
 
@@ -127,6 +131,7 @@ jobs:
           sudo env
           PATH="$PATH"
           GEM_PATH="$GEM_PATH"
+          ASAN_OPTIONS="$ASAN_OPTIONS"
           PASSENGER_DUMP_WITH_CRASH_WATCH=false
           PASSENGER_FORCE_TERMINATE_ON_ABORT=true
           gtimeout --signal KILL --verbose 600

bin/passenger-install-nginx-module

@@ -553,7 +553,7 @@ private
 
     extra_ldflags = [
       PlatformInfo.openssl_extra_ldflags,
-      boolean_option('USE_ASAN') ? PlatformInfo.address_sanitizer_flags : nil,
+      boolean_option('USE_ASAN') ? PlatformInfo.address_sanitizer_c_ldflags : nil,
     ].compact.join(" ").strip
 
     command = "sh ./configure --prefix='#{prefix}' "

build/apache2.rb

@@ -126,7 +126,8 @@
       PlatformInfo.apache2_module_cxx_ldflags,
       PlatformInfo.portability_cxx_ldflags,
       OPTIMIZE ? '-O' : nil,
-      USE_ASAN ? PlatformInfo.address_sanitizer_flags : nil,
+      USE_ASAN ? PlatformInfo.address_sanitizer_cxx_shlib_ldflags : nil,
+      USE_UBSAN ? PlatformInfo.undefined_behavior_sanitizer_cxx_ldflags : nil,
     ].compact
   )
 end

build/basics.rb

@@ -114,20 +114,16 @@
 let(:extra_cflags) do
   result = PlatformInfo.default_extra_cflags.dup
   result << " " << compiler_flag_option('EXTRA_CFLAGS') if !compiler_flag_option('EXTRA_CFLAGS').empty?
-  result << " #{PlatformInfo.address_sanitizer_flags}" if USE_ASAN && PlatformInfo.address_sanitizer_flags
-  result << " #{PlatformInfo.undefined_behavior_sanitizer_flags}" if USE_UBSAN && PlatformInfo.undefined_behavior_sanitizer_flags
-  result << " -fno-omit-frame-pointer" if USE_ASAN
-  result << " -fno-optimize-sibling-calls" if USE_ASAN && PlatformInfo.cc_supports_fno_optimize_sibling_calls_flag?
+  result << " #{PlatformInfo.address_sanitizer_cflags}" if USE_ASAN && PlatformInfo.address_sanitizer_cflags
+  result << " #{PlatformInfo.undefined_behavior_sanitizer_cflags}" if USE_UBSAN && PlatformInfo.undefined_behavior_sanitizer_cflags
   result << " -DPASSENGER_DISABLE_THREAD_LOCAL_STORAGE" if !boolean_option('PASSENGER_THREAD_LOCAL_STORAGE', true)
   result
 end
 let(:extra_cxxflags) do
   result = PlatformInfo.default_extra_cxxflags.dup
   result << " " << compiler_flag_option('EXTRA_CXXFLAGS') if !compiler_flag_option('EXTRA_CXXFLAGS').empty?
-  result << " #{PlatformInfo.address_sanitizer_flags}" if USE_ASAN && PlatformInfo.address_sanitizer_flags
-  result << " #{PlatformInfo.undefined_behavior_sanitizer_flags}" if USE_UBSAN && PlatformInfo.undefined_behavior_sanitizer_flags
-  result << " -fno-omit-frame-pointer" if USE_ASAN
-  result << " -fno-optimize-sibling-calls" if USE_ASAN && PlatformInfo.cxx_supports_fno_optimize_sibling_calls_flag?
+  result << " #{PlatformInfo.address_sanitizer_cxxflags}" if USE_ASAN && PlatformInfo.address_sanitizer_cxxflags
+  result << " #{PlatformInfo.undefined_behavior_sanitizer_cxxflags}" if USE_UBSAN && PlatformInfo.undefined_behavior_sanitizer_cxxflags
   result << " -DPASSENGER_DISABLE_THREAD_LOCAL_STORAGE" if !boolean_option('PASSENGER_THREAD_LOCAL_STORAGE', true)
   result
 end
@@ -141,17 +137,17 @@
 # These should be included last in the command string, even after portability_*_ldflags.
 let(:extra_c_ldflags) do
   result = []
-  result << PlatformInfo.address_sanitizer_flags if USE_ASAN
-  result << PlatformInfo.undefined_behavior_sanitizer_flags if USE_UBSAN
+  result << PlatformInfo.address_sanitizer_c_ldflags if USE_ASAN
+  result << PlatformInfo.undefined_behavior_sanitizer_c_ldflags if USE_UBSAN
   result << compiler_flag_option('EXTRA_LDFLAGS')
   result << compiler_flag_option('EXTRA_C_LDFLAGS')
   result.compact!
   result.join(' ')
 end
 let(:extra_cxx_ldflags) do
   result = []
-  result << PlatformInfo.address_sanitizer_flags if USE_ASAN
-  result << PlatformInfo.undefined_behavior_sanitizer_flags if USE_UBSAN
+  result << PlatformInfo.address_sanitizer_cxx_ldflags if USE_ASAN
+  result << PlatformInfo.undefined_behavior_sanitizer_cxx_ldflags if USE_UBSAN
   result << compiler_flag_option('EXTRA_LDFLAGS')
   result << compiler_flag_option('EXTRA_CXX_LDFLAGS')
   result.compact!

src/ruby_supportlib/phusion_passenger/config/nginx_engine_compiler.rb

@@ -55,7 +55,7 @@ def self.configure_script_options(address_sanitizer: false)
         extra_ldflags = [
           PlatformInfo.openssl_extra_ldflags,
           PlatformInfo.pcre_extra_ldflags,
-          address_sanitizer ? PlatformInfo.address_sanitizer_flags : nil,
+          address_sanitizer ? PlatformInfo.address_sanitizer_c_ldflags : nil,
         ].compact.join(" ").strip
         if !extra_ldflags.empty?
           result << "--with-ld-opt=#{Shellwords.escape extra_ldflags} "

src/ruby_supportlib/phusion_passenger/platform_info/compiler.rb

@@ -505,25 +505,116 @@ def self.cxx_visibility_flag_generates_warnings?
     end
     memoize :cxx_visibility_flag_generates_warnings?, true
 
-    def self.address_sanitizer_flags
+    def self.cc_supports_fsanitize_address?
       if cc_is_clang?
-        if `#{cc} --help` =~ /-fsanitize=/
-          "-fsanitize=address -fsanitize-address-use-after-return=always"
-        else
-          "-faddress-sanitizer"
-        end
+        `#{cc} --help 2>&1` =~ /-fsanitize=/
+      elsif cc_is_gcc?
+        `#{cc} --help=common 2>&1` =~ /-fsanitize=/
+      else
+        false
+      end
+    end
+    memoize :cc_supports_fsanitize_address?, true
+
+    def self.cxx_supports_fsanitize_address?
+      if cxx_is_clang?
+        `#{cxx} --help 2>&1` =~ /-fsanitize=/
+      elsif cxx_is_gcc?
+        `#{cxx} --help=common 2>&1` =~ /-fsanitize=/
       else
-        nil
+        false
       end
     end
+    memoize :cxx_supports_fsanitize_address?, true
 
-    def self.undefined_behavior_sanitizer_flags
+    def self.address_sanitizer_cflags
       if cc_is_clang?
+        if cc_supports_fsanitize_address?
+          result = "-fsanitize=address -fsanitize-address-use-after-return=always"
+        else
+          result = "-faddress-sanitizer"
+        end
+      elsif cc_supports_fsanitize_address?
+        result = "-fsanitize=address"
+      end
+
+      if result
+        result << " -fno-omit-frame-pointer"
+        result << " -fno-optimize-sibling-calls" if cc_supports_fno_optimize_sibling_calls_flag?
+        result
+      end
+    end
+    memoize :address_sanitizer_cflags
+
+    def self.address_sanitizer_cxxflags
+      if cxx_is_clang?
+        if cxx_supports_fsanitize_address?
+          result = "-fsanitize=address -fsanitize-address-use-after-return=always"
+        else
+          result = "-faddress-sanitizer"
+        end
+      elsif cxx_supports_fsanitize_address?
+        result = "-fsanitize=address"
+      end
+
+      if result
+        result << " -fno-omit-frame-pointer"
+        result << " -fno-optimize-sibling-calls" if cxx_supports_fno_optimize_sibling_calls_flag?
+        result
+      end
+    end
+    memoize :address_sanitizer_cxxflags
+
+    def self.address_sanitizer_c_ldflags
+      if cc_supports_fsanitize_address?
+        "-fsanitize=address"
+      end
+    end
+    memoize :address_sanitizer_c_ldflags
+
+    def self.address_sanitizer_cxx_ldflags
+      if cxx_supports_fsanitize_address?
+        "-fsanitize=address"
+      end
+    end
+    memoize :address_sanitizer_cxx_ldflags
+
+    def self.address_sanitizer_cxx_shlib_ldflags
+      if cxx_supports_fsanitize_address?
+        result = "-fsanitize=address"
+        result << " -shared-libasan" if cxx_is_clang?
+        result
+      end
+    end
+    memoize :address_sanitizer_cxx_shlib_ldflags
+
+    def self.undefined_behavior_sanitizer_cflags
+      if (cc_is_clang? || cc_is_gcc?) && cc_supports_fsanitize_address?
+        "-fsanitize=undefined"
+      end
+    end
+    memoize :undefined_behavior_sanitizer_cflags
+
+    def self.undefined_behavior_sanitizer_cxxflags
+      if (cxx_is_clang? || cxx_is_gcc?) && cxx_supports_fsanitize_address?
+        "-fsanitize=undefined"
+      end
+    end
+    memoize :undefined_behavior_sanitizer_cxxflags
+
+    def self.undefined_behavior_sanitizer_c_ldflags
+      if (cc_is_clang? || cc_is_gcc?) && cc_supports_fsanitize_address?
+        "-fsanitize=undefined"
+      end
+    end
+    memoize :undefined_behavior_sanitizer_c_ldflags
+
+    def self.undefined_behavior_sanitizer_cxx_ldflags
+      if (cxx_is_clang? || cxx_is_gcc?) && cxx_supports_fsanitize_address?
         "-fsanitize=undefined"
-      else
-        nil
       end
     end
+    memoize :undefined_behavior_sanitizer_cxx_ldflags
 
     def self.cxx_11_flag
       # C++11 support on FreeBSD 10.0 + Clang seems to be bugged.