1 files changed, 132 insertions, 53 deletions

diff --git a/libsanitizer/asan/asan_rtl.cc b/libsanitizer/asan/asan_rtl.cc
index 175d37788c2..e551b6fbc26 100644
--- a/libsanitizer/asan/asan_rtl.cc
+++ b/libsanitizer/asan/asan_rtl.cc
@@ -25,6 +25,8 @@
 
 namespace __asan {
 
+uptr AsanMappingProfile[kAsanMappingProfileSize];
+
 static void AsanDie() {
   static atomic_uint32_t num_calls;
   if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) {
@@ -35,13 +37,19 @@ static void AsanDie() {
     Report("Sleeping for %d second(s)\n", flags()->sleep_before_dying);
     SleepForSeconds(flags()->sleep_before_dying);
   }
-  if (flags()->unmap_shadow_on_exit)
-    UnmapOrDie((void*)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
+  if (flags()->unmap_shadow_on_exit) {
+    if (kMidMemBeg) {
+      UnmapOrDie((void*)kLowShadowBeg, kMidMemBeg - kLowShadowBeg);
+      UnmapOrDie((void*)kMidMemEnd, kHighShadowEnd - kMidMemEnd);
+    } else {
+      UnmapOrDie((void*)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg);
+    }
+  }
   if (death_callback)
     death_callback();
   if (flags()->abort_on_error)
     Abort();
-  Exit(flags()->exitcode);
+  internal__exit(flags()->exitcode);
 }
 
 static void AsanCheckFailed(const char *file, int line, const char *cond,
@@ -66,6 +74,17 @@ static const char *MaybeCallAsanDefaultOptions() {
   return (&__asan_default_options) ? __asan_default_options() : "";
 }
 
+static const char *MaybeUseAsanDefaultOptionsCompileDefiniton() {
+#ifdef ASAN_DEFAULT_OPTIONS
+// Stringize the macro value.
+# define ASAN_STRINGIZE(x) #x
+# define ASAN_STRINGIZE_OPTIONS(options) ASAN_STRINGIZE(options)
+  return ASAN_STRINGIZE_OPTIONS(ASAN_DEFAULT_OPTIONS);
+#else
+  return "";
+#endif
+}
+
 static void ParseFlagsFromString(Flags *f, const char *str) {
   ParseFlag(str, &f->quarantine_size, "quarantine_size");
   ParseFlag(str, &f->symbolize, "symbolize");
@@ -146,6 +165,9 @@ void InitializeFlags(Flags *f, const char *env) {
   f->alloc_dealloc_mismatch = true;
   f->use_stack_depot = true;  // Only affects allocator2.
 
+  // Override from compile definition.
+  ParseFlagsFromString(f, MaybeUseAsanDefaultOptionsCompileDefiniton());
+
   // Override from user-specified string.
   ParseFlagsFromString(f, MaybeCallAsanDefaultOptions());
   if (flags()->verbosity) {
@@ -161,7 +183,10 @@ void InitializeFlags(Flags *f, const char *env) {
 int asan_inited;
 bool asan_init_is_running;
 void (*death_callback)(void);
-uptr kHighMemEnd;
+
+#if !ASAN_FIXED_MAPPING
+uptr kHighMemEnd, kMidMemBeg, kMidMemEnd;
+#endif
 
 // -------------------------- Misc ---------------- {{{1
 void ShowStatsAndAbort() {
@@ -209,6 +234,17 @@ ASAN_REPORT_ERROR(store, true, 4)
 ASAN_REPORT_ERROR(store, true, 8)
 ASAN_REPORT_ERROR(store, true, 16)
 
+#define ASAN_REPORT_ERROR_N(type, is_write)                    \
+extern "C" NOINLINE INTERFACE_ATTRIBUTE                        \
+void __asan_report_ ## type ## _n(uptr addr, uptr size);       \
+void __asan_report_ ## type ## _n(uptr addr, uptr size) {      \
+  GET_CALLER_PC_BP_SP;                                         \
+  __asan_report_error(pc, bp, sp, addr, is_write, size);       \
+}
+
+ASAN_REPORT_ERROR_N(load, false)
+ASAN_REPORT_ERROR_N(store, true)
+
 // Force the linker to keep the symbols for various ASan interface functions.
 // We want to keep those in the executable in order to let the instrumented
 // dynamic libraries access the symbol even if it is not used by the executable
@@ -259,9 +295,15 @@ static NOINLINE void force_interface_symbols() {
 static void asan_atexit() {
   Printf("AddressSanitizer exit stats:\n");
   __asan_print_accumulated_stats();
+  // Print AsanMappingProfile.
+  for (uptr i = 0; i < kAsanMappingProfileSize; i++) {
+    if (AsanMappingProfile[i] == 0) continue;
+    Printf("asan_mapping.h:%zd -- %zd\n", i, AsanMappingProfile[i]);
+  }
 }
 
 static void InitializeHighMemEnd() {
+#if !ASAN_FIXED_MAPPING
 #if SANITIZER_WORDSIZE == 64
 # if defined(__powerpc64__)
   // FIXME:
@@ -277,6 +319,58 @@ static void InitializeHighMemEnd() {
 #else  // SANITIZER_WORDSIZE == 32
   kHighMemEnd = (1ULL << 32) - 1;  // 0xffffffff;
 #endif  // SANITIZER_WORDSIZE
+#endif  // !ASAN_FIXED_MAPPING
+}
+
+static void ProtectGap(uptr a, uptr size) {
+  CHECK_EQ(a, (uptr)Mprotect(a, size));
+}
+
+static void PrintAddressSpaceLayout() {
+  Printf("|| `[%p, %p]` || HighMem    ||\n",
+         (void*)kHighMemBeg, (void*)kHighMemEnd);
+  Printf("|| `[%p, %p]` || HighShadow ||\n",
+         (void*)kHighShadowBeg, (void*)kHighShadowEnd);
+  if (kMidMemBeg) {
+    Printf("|| `[%p, %p]` || ShadowGap3 ||\n",
+           (void*)kShadowGap3Beg, (void*)kShadowGap3End);
+    Printf("|| `[%p, %p]` || MidMem     ||\n",
+           (void*)kMidMemBeg, (void*)kMidMemEnd);
+    Printf("|| `[%p, %p]` || ShadowGap2 ||\n",
+           (void*)kShadowGap2Beg, (void*)kShadowGap2End);
+    Printf("|| `[%p, %p]` || MidShadow  ||\n",
+           (void*)kMidShadowBeg, (void*)kMidShadowEnd);
+  }
+  Printf("|| `[%p, %p]` || ShadowGap  ||\n",
+         (void*)kShadowGapBeg, (void*)kShadowGapEnd);
+  if (kLowShadowBeg) {
+    Printf("|| `[%p, %p]` || LowShadow  ||\n",
+           (void*)kLowShadowBeg, (void*)kLowShadowEnd);
+    Printf("|| `[%p, %p]` || LowMem     ||\n",
+           (void*)kLowMemBeg, (void*)kLowMemEnd);
+  }
+  Printf("MemToShadow(shadow): %p %p %p %p",
+         (void*)MEM_TO_SHADOW(kLowShadowBeg),
+         (void*)MEM_TO_SHADOW(kLowShadowEnd),
+         (void*)MEM_TO_SHADOW(kHighShadowBeg),
+         (void*)MEM_TO_SHADOW(kHighShadowEnd));
+  if (kMidMemBeg) {
+    Printf(" %p %p",
+           (void*)MEM_TO_SHADOW(kMidShadowBeg),
+           (void*)MEM_TO_SHADOW(kMidShadowEnd));
+  }
+  Printf("\n");
+  Printf("red_zone=%zu\n", (uptr)flags()->redzone);
+  Printf("malloc_context_size=%zu\n", (uptr)flags()->malloc_context_size);
+
+  Printf("SHADOW_SCALE: %zx\n", (uptr)SHADOW_SCALE);
+  Printf("SHADOW_GRANULARITY: %zx\n", (uptr)SHADOW_GRANULARITY);
+  Printf("SHADOW_OFFSET: %zx\n", (uptr)SHADOW_OFFSET);
+  CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
+  if (kMidMemBeg)
+    CHECK(kMidShadowBeg > kLowShadowEnd &&
+          kMidMemBeg > kMidShadowEnd &&
+          kHighShadowBeg > kMidMemEnd);
 }
 
 }  // namespace __asan
@@ -352,49 +446,48 @@ void __asan_init() {
   ReplaceSystemMalloc();
   ReplaceOperatorsNewAndDelete();
 
-  if (flags()->verbosity) {
-    Printf("|| `[%p, %p]` || HighMem    ||\n",
-           (void*)kHighMemBeg, (void*)kHighMemEnd);
-    Printf("|| `[%p, %p]` || HighShadow ||\n",
-           (void*)kHighShadowBeg, (void*)kHighShadowEnd);
-    Printf("|| `[%p, %p]` || ShadowGap  ||\n",
-           (void*)kShadowGapBeg, (void*)kShadowGapEnd);
-    Printf("|| `[%p, %p]` || LowShadow  ||\n",
-           (void*)kLowShadowBeg, (void*)kLowShadowEnd);
-    Printf("|| `[%p, %p]` || LowMem     ||\n",
-           (void*)kLowMemBeg, (void*)kLowMemEnd);
-    Printf("MemToShadow(shadow): %p %p %p %p\n",
-           (void*)MEM_TO_SHADOW(kLowShadowBeg),
-           (void*)MEM_TO_SHADOW(kLowShadowEnd),
-           (void*)MEM_TO_SHADOW(kHighShadowBeg),
-           (void*)MEM_TO_SHADOW(kHighShadowEnd));
-    Printf("red_zone=%zu\n", (uptr)flags()->redzone);
-    Printf("malloc_context_size=%zu\n", (uptr)flags()->malloc_context_size);
-
-    Printf("SHADOW_SCALE: %zx\n", (uptr)SHADOW_SCALE);
-    Printf("SHADOW_GRANULARITY: %zx\n", (uptr)SHADOW_GRANULARITY);
-    Printf("SHADOW_OFFSET: %zx\n", (uptr)SHADOW_OFFSET);
-    CHECK(SHADOW_SCALE >= 3 && SHADOW_SCALE <= 7);
+  uptr shadow_start = kLowShadowBeg;
+  if (kLowShadowBeg) shadow_start -= GetMmapGranularity();
+  uptr shadow_end = kHighShadowEnd;
+  bool full_shadow_is_available =
+      MemoryRangeIsAvailable(shadow_start, shadow_end);
+
+#if ASAN_LINUX && defined(__x86_64__) && !ASAN_FIXED_MAPPING
+  if (!full_shadow_is_available) {
+    kMidMemBeg = kLowMemEnd < 0x3000000000ULL ? 0x3000000000ULL : 0;
+    kMidMemEnd = kLowMemEnd < 0x3000000000ULL ? 0x3fffffffffULL : 0;
   }
+#endif
+
+  if (flags()->verbosity)
+    PrintAddressSpaceLayout();
 
   if (flags()->disable_core) {
     DisableCoreDumper();
   }
 
-  uptr shadow_start = kLowShadowBeg;
-  if (kLowShadowBeg > 0) shadow_start -= GetMmapGranularity();
-  uptr shadow_end = kHighShadowEnd;
-  if (MemoryRangeIsAvailable(shadow_start, shadow_end)) {
-    if (kLowShadowBeg != kLowShadowEnd) {
-      // mmap the low shadow plus at least one page.
-      ReserveShadowMemoryRange(kLowShadowBeg - GetMmapGranularity(),
-                               kLowShadowEnd);
-    }
+  if (full_shadow_is_available) {
+    // mmap the low shadow plus at least one page at the left.
+    if (kLowShadowBeg)
+      ReserveShadowMemoryRange(shadow_start, kLowShadowEnd);
+    // mmap the high shadow.
+    ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd);
+    // protect the gap.
+    ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
+  } else if (kMidMemBeg &&
+      MemoryRangeIsAvailable(shadow_start, kMidMemBeg - 1) &&
+      MemoryRangeIsAvailable(kMidMemEnd + 1, shadow_end)) {
+    CHECK(kLowShadowBeg != kLowShadowEnd);
+    // mmap the low shadow plus at least one page at the left.
+    ReserveShadowMemoryRange(shadow_start, kLowShadowEnd);
+    // mmap the mid shadow.
+    ReserveShadowMemoryRange(kMidShadowBeg, kMidShadowEnd);
     // mmap the high shadow.
     ReserveShadowMemoryRange(kHighShadowBeg, kHighShadowEnd);
-    // protect the gap
-    void *prot = Mprotect(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
-    CHECK(prot == (void*)kShadowGapBeg);
+    // protect the gaps.
+    ProtectGap(kShadowGapBeg, kShadowGapEnd - kShadowGapBeg + 1);
+    ProtectGap(kShadowGap2Beg, kShadowGap2End - kShadowGap2Beg + 1);
+    ProtectGap(kShadowGap3Beg, kShadowGap3End - kShadowGap3Beg + 1);
   } else {
     Report("Shadow memory range interleaves with an existing memory mapping. "
            "ASan cannot proceed correctly. ABORTING.\n");
@@ -426,17 +519,3 @@ void __asan_init() {
     Report("AddressSanitizer Init done\n");
   }
 }
-
-#if defined(ASAN_USE_PREINIT_ARRAY)
-  // On Linux, we force __asan_init to be called before anyone else
-  // by placing it into .preinit_array section.
-  // FIXME: do we have anything like this on Mac?
-  __attribute__((section(".preinit_array")))
-    typeof(__asan_init) *__asan_preinit =__asan_init;
-#elif defined(_WIN32) && defined(_DLL)
-  // On Windows, when using dynamic CRT (/MD), we can put a pointer
-  // to __asan_init into the global list of C initializers.
-  // See crt0dat.c in the CRT sources for the details.
-  #pragma section(".CRT$XIB", long, read)  // NOLINT
-  __declspec(allocate(".CRT$XIB")) void (*__asan_preinit)() = __asan_init;
-#endif