[sanitizer] Fail __sanitizer_symbolize_demangle instead of returning input

LLVMSymbolizer::DemangleName returns the same input if it can't demangle.
We can't tell if this is already demangled or format is unsupported.

Internally DemangleName uses nonMicrosoftDemangle which can report a failure.

Author: vitalybuka

compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp

@@ -17,6 +17,7 @@
 
 #include "llvm/DebugInfo/Symbolize/DIPrinter.h"
 #include "llvm/DebugInfo/Symbolize/Symbolize.h"
+#include "llvm/Demangle/Demangle.h"
 
 static llvm::symbolize::LLVMSymbolizer *Symbolizer = nullptr;
 static bool Demangle = true;
@@ -117,8 +118,9 @@ void __sanitizer_symbolize_flush() {
 
 bool __sanitizer_symbolize_demangle(const char *Name, char *Buffer,
                                    int MaxLength) {
-  std::string Result =
-      llvm::symbolize::LLVMSymbolizer::DemangleName(Name, nullptr);
+  std::string Result;
+  if (!llvm::nonMicrosoftDemangle(Name, Result))
+    return false;
   return __sanitizer::internal_snprintf(Buffer, MaxLength, "%s",
                                         Result.c_str()) < MaxLength;
 }

compiler-rt/test/sanitizer_common/TestCases/demangle_internal.cpp

@@ -0,0 +1,25 @@
+// RUN: %clangxx -O0 %s -o %t && %run %t
+
+// REQUIRES: internal_symbolizer
+
+// FIXME: link internal_symbolizer.
+// XFAIL: asan, hwasan, ubsan
+
+#include <algorithm>
+#include <assert.h>
+#include <string.h>
+
+extern "C" bool __sanitizer_symbolize_demangle(const char *Name, char *Buffer,
+                                               int MaxLength);
+
+int main() {
+  char out[128];
+  assert(!__sanitizer_symbolize_demangle("1A", out, sizeof(out)));
+
+  const char name[] = "_Z3fooi";
+  for (int i = 1; i < sizeof(out); ++i) {
+    memset(out, 1, sizeof(out));
+    assert(__sanitizer_symbolize_demangle(name, out, i) == (i > 8));
+    assert(i < 9 || 0 == strncmp(out, "foo(int)", i - 1));
+  }
+}

compiler-rt/test/ubsan/TestCases/TypeCheck/vptr-virtual-base.cpp

@@ -1,9 +1,6 @@
 // RUN: %clangxx -frtti -fsanitize=null,vptr -fno-sanitize-recover=vptr -g %s -O3 -o %t
 // RUN: not %run %t 2>&1 | FileCheck %s
 
-// FIXME: Investigate.
-// XFAIL: internal_symbolizer && (ubsan-tsan || ubsan-msan)
-
 // REQUIRES: shared_cxxabi
 // REQUIRES: cxxabi
 // UNSUPPORTED: target={{.*windows-msvc.*}}

compiler-rt/test/ubsan/TestCases/TypeCheck/vptr.cpp

@@ -36,9 +36,6 @@
 // RUN: echo "vptr_check:S" > %t.loc-supp
 // RUN: %env_ubsan_opts=halt_on_error=1:suppressions='"%t.loc-supp"' not %run %t x- 2>&1 | FileCheck %s --check-prefix=CHECK-LOC-SUPPRESS
 
-// FIXME: Investigate.
-// XFAIL: internal_symbolizer && (ubsan-tsan || ubsan-msan)
-
 // REQUIRES: stable-runtime, cxxabi
 // UNSUPPORTED: target={{.*windows-msvc.*}}
 // Suppressions file not pushed to the device.