Report fatal error in the case of out of memory

Analysis of fails in the case of out of memory errors can be tricky on
Windows. Such error emerges at the point where memory allocation function
fails, but manifests itself when null pointer is used. These two points
may be distant from each other. Besides, next runs may not exhibit
allocation error.

Usual programming practice does not require checking result of 'operator
new' because it throws 'std::bad_alloc' in the case of allocation error.
However, LLVM is usually built with exceptions turned off, so 'new' can
return null pointer. This change installs custom new handler, which causes
fatal error in the case of out of memory. The handler is installed
automatically prior to call to 'main' during construction of a static
object defined in 'lib/Support/ErrorHandling.cpp'. If the application does
not use this file, the handler may be installed manually by a call to
'llvm::install_out_of_memory_new_handler', declared in
'include/llvm/Support/ErrorHandling.h".

There are calls to C allocation functions, malloc, calloc and realloc.
They are used for interoperability with C code, when allocated object has
variable size and when it is necessary to avoid call of constructors. In
many calls the result is not checked against null pointer. To simplify
checks, new functions are defined in the namespace 'llvm' with the
same names as these C function. These functions produce fatal error if
allocation fails. User should use 'llvm::malloc' instead of 'std::malloc'
in order to use the safe variant. This change replaces 'std::malloc'
in the cases when the result of allocation function is not checked against
null pointer.

Finally, there are plain C code, that uses malloc and similar functions. If
the result is not checked, assert statements are added.

Differential Revision: https://reviews.llvm.org/D43010

llvm-svn: 325224

llvm/include/llvm/ADT/BitVector.h

@@ -828,7 +828,8 @@ private:
   }
 
   MutableArrayRef<BitWord> allocate(size_t NumWords) {
-    BitWord *RawBits = (BitWord *)std::malloc(NumWords * sizeof(BitWord));
+    BitWord *RawBits = static_cast<BitWord *>(
+        llvm::malloc(NumWords * sizeof(BitWord)));
     return MutableArrayRef<BitWord>(RawBits, NumWords);
   }
 
@@ -867,8 +868,8 @@ private:
   void grow(unsigned NewSize) {
     size_t NewCapacity = std::max<size_t>(NumBitWords(NewSize), Bits.size() * 2);
     assert(NewCapacity > 0 && "realloc-ing zero space");
-    BitWord *NewBits =
-        (BitWord *)std::realloc(Bits.data(), NewCapacity * sizeof(BitWord));
+    BitWord *NewBits = static_cast<BitWord *>(
+        llvm::realloc(Bits.data(), NewCapacity * sizeof(BitWord)));
     Bits = MutableArrayRef<BitWord>(NewBits, NewCapacity);
     clear_unused_bits();
   }

llvm/include/llvm/ADT/SparseMultiSet.h

@@ -211,7 +211,7 @@ public:
     // The Sparse array doesn't actually need to be initialized, so malloc
     // would be enough here, but that will cause tools like valgrind to
     // complain about branching on uninitialized data.
-    Sparse = reinterpret_cast<SparseT*>(calloc(U, sizeof(SparseT)));
+    Sparse = static_cast<SparseT*>(llvm::calloc(U, sizeof(SparseT)));
     Universe = U;
   }
 

llvm/include/llvm/ADT/SparseSet.h

@@ -22,6 +22,7 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Allocator.h"
 #include <cassert>
 #include <cstdint>
 #include <cstdlib>
@@ -163,7 +164,7 @@ public:
     // The Sparse array doesn't actually need to be initialized, so malloc
     // would be enough here, but that will cause tools like valgrind to
     // complain about branching on uninitialized data.
-    Sparse = reinterpret_cast<SparseT*>(calloc(U, sizeof(SparseT)));
+    Sparse = static_cast<SparseT*>(llvm::calloc(U, sizeof(SparseT)));
     Universe = U;
   }
 

llvm/include/llvm/Support/Allocator.h

@@ -439,6 +439,33 @@ public:
   T *Allocate(size_t num = 1) { return Allocator.Allocate<T>(num); }
 };
 
+/// \{
+/// Counterparts of allocation functions defined in namespace 'std', which crash
+/// on allocation failure instead of returning null pointer.
+
+LLVM_ATTRIBUTE_RETURNS_NONNULL inline void *malloc(size_t Sz) {
+  void *Result = std::malloc(Sz);
+  if (Result == nullptr)
+    report_bad_alloc_error("Allocation failed.");
+  return Result;
+}
+
+LLVM_ATTRIBUTE_RETURNS_NONNULL inline void *calloc(size_t Count, size_t Sz) {
+  void *Result = std::calloc(Count, Sz);
+  if (Result == nullptr)
+    report_bad_alloc_error("Allocation failed.");
+  return Result;
+}
+
+LLVM_ATTRIBUTE_RETURNS_NONNULL inline void *realloc(void *Ptr, size_t Sz) {
+  void *Result = std::realloc(Ptr, Sz);
+  if (Result == nullptr)
+    report_bad_alloc_error("Allocation failed.");
+  return Result;
+}
+
+/// \}
+
 } // end namespace llvm
 
 template <typename AllocatorT, size_t SlabSize, size_t SizeThreshold>

llvm/include/llvm/Support/ErrorHandling.h

@@ -100,6 +100,8 @@ void install_bad_alloc_error_handler(fatal_error_handler_t handler,
 /// Restores default bad alloc error handling behavior.
 void remove_bad_alloc_error_handler();
 
+void install_out_of_memory_new_handler();
+
 /// Reports a bad alloc error, calling any user defined bad alloc
 /// error handler. In contrast to the generic 'report_fatal_error'
 /// functions, this function is expected to return, e.g. the user

llvm/include/llvm/Support/OnDiskHashTable.h

@@ -95,7 +95,8 @@ private:
 
   /// \brief Resize the hash table, moving the old entries into the new buckets.
   void resize(size_t NewSize) {
-    Bucket *NewBuckets = (Bucket *)std::calloc(NewSize, sizeof(Bucket));
+    Bucket *NewBuckets = static_cast<Bucket *>(
+        llvm::calloc(NewSize, sizeof(Bucket)));
     // Populate NewBuckets with the old entries.
     for (size_t I = 0; I < NumBuckets; ++I)
       for (Item *E = Buckets[I].Head; E;) {
@@ -226,7 +227,7 @@ public:
     NumBuckets = 64;
     // Note that we do not need to run the constructors of the individual
     // Bucket objects since 'calloc' returns bytes that are all 0.
-    Buckets = (Bucket *)std::calloc(NumBuckets, sizeof(Bucket));
+    Buckets = static_cast<Bucket *>(llvm::calloc(NumBuckets, sizeof(Bucket)));
   }
 
   ~OnDiskChainedHashTableGenerator() { std::free(Buckets); }

llvm/lib/CodeGen/InterferenceCache.cpp

@@ -48,8 +48,8 @@ void InterferenceCache::reinitPhysRegEntries() {
   if (PhysRegEntriesCount == TRI->getNumRegs()) return;
   free(PhysRegEntries);
   PhysRegEntriesCount = TRI->getNumRegs();
-  PhysRegEntries = (unsigned char*)
-    calloc(PhysRegEntriesCount, sizeof(unsigned char));
+  PhysRegEntries = static_cast<unsigned char*>(
+      llvm::calloc(PhysRegEntriesCount, sizeof(unsigned char)));
 }
 
 void InterferenceCache::init(MachineFunction *mf,

llvm/lib/CodeGen/LiveIntervalUnion.cpp

@@ -187,7 +187,7 @@ void LiveIntervalUnion::Array::init(LiveIntervalUnion::Allocator &Alloc,
   clear();
   Size = NSize;
   LIUs = static_cast<LiveIntervalUnion*>(
-    malloc(sizeof(LiveIntervalUnion)*NSize));
+      llvm::malloc(sizeof(LiveIntervalUnion)*NSize));
   for (unsigned i = 0; i != Size; ++i)
     new(LIUs + i) LiveIntervalUnion(Alloc);
 }

llvm/lib/CodeGen/RegisterPressure.cpp

@@ -635,7 +635,8 @@ void PressureDiffs::init(unsigned N) {
   }
   Max = Size;
   free(PDiffArray);
-  PDiffArray = reinterpret_cast<PressureDiff*>(calloc(N, sizeof(PressureDiff)));
+  PDiffArray = static_cast<PressureDiff*>(
+      llvm::calloc(N, sizeof(PressureDiff)));
 }
 
 void PressureDiffs::addInstruction(unsigned Idx,

llvm/lib/ExecutionEngine/Interpreter/Execution.cpp

@@ -974,7 +974,7 @@ void Interpreter::visitAllocaInst(AllocaInst &I) {
   unsigned MemToAlloc = std::max(1U, NumElements * TypeSize);
 
   // Allocate enough memory to hold the type...
-  void *Memory = malloc(MemToAlloc);
+  void *Memory = llvm::malloc(MemToAlloc);
 
   DEBUG(dbgs() << "Allocated Type: " << *Ty << " (" << TypeSize << " bytes) x " 
                << NumElements << " (Total: " << MemToAlloc << ") at "

llvm/lib/Object/Object.cpp

@@ -228,7 +228,7 @@ uint64_t LLVMGetRelocationType(LLVMRelocationIteratorRef RI) {
 const char *LLVMGetRelocationTypeName(LLVMRelocationIteratorRef RI) {
   SmallVector<char, 0> ret;
   (*unwrap(RI))->getTypeName(ret);
-  char *str = static_cast<char*>(malloc(ret.size()));
+  char *str = static_cast<char*>(llvm::malloc(ret.size()));
   std::copy(ret.begin(), ret.end(), str);
   return str;
 }

llvm/lib/Support/ErrorHandling.cpp

@@ -175,6 +175,39 @@ void llvm::report_bad_alloc_error(const char *Reason, bool GenCrashDiag) {
 #endif
 }
 
+#ifdef LLVM_ENABLE_EXCEPTIONS
+// Do not set custom new handler if exceptions are enabled. In this case OOM
+// errors are handled by throwing 'std::bad_alloc'.
+void llvm::install_out_of_memory_new_handler() {
+}
+#else
+// Causes crash on allocation failure. It is called prior to the handler set by
+// 'install_bad_alloc_error_handler'.
+static void out_of_memory_new_handler() {
+  llvm::report_bad_alloc_error("Allocation failed");
+}
+
+// Installs new handler that causes crash on allocation failure. It does not
+// need to be called explicitly, if this file is linked to application, because
+// in this case it is called during construction of 'new_handler_installer'.
+void llvm::install_out_of_memory_new_handler() {
+  static bool out_of_memory_new_handler_installed = false;
+  if (!out_of_memory_new_handler_installed) {
+    std::set_new_handler(out_of_memory_new_handler);
+    out_of_memory_new_handler_installed = true;
+  }
+}
+
+// Static object that causes installation of 'out_of_memory_new_handler' before
+// execution of 'main'.
+static class NewHandlerInstaller {
+public:
+  NewHandlerInstaller() {
+    install_out_of_memory_new_handler();
+  }
+} new_handler_installer;
+#endif
+
 void llvm::llvm_unreachable_internal(const char *msg, const char *file,
                                      unsigned line) {
   // This code intentionally doesn't call the ErrorHandler callback, because

llvm/lib/Support/FoldingSet.cpp

@@ -214,7 +214,8 @@ static void **GetBucketFor(unsigned Hash, void **Buckets, unsigned NumBuckets) {
 
 /// AllocateBuckets - Allocated initialized bucket memory.
 static void **AllocateBuckets(unsigned NumBuckets) {
-  void **Buckets = static_cast<void**>(calloc(NumBuckets+1, sizeof(void*)));
+  void **Buckets = static_cast<void**>(
+      llvm::calloc(NumBuckets+1, sizeof(void*)));
 
   if (Buckets == nullptr)
     report_bad_alloc_error("Allocation of Buckets failed.");

llvm/lib/Support/RWMutex.cpp

@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Support/Allocator.h"
 #include "llvm/Support/RWMutex.h"
 #include "llvm/Config/config.h"
 
@@ -49,7 +50,7 @@ RWMutexImpl::RWMutexImpl()
 {
   // Declare the pthread_rwlock data structures
   pthread_rwlock_t* rwlock =
-    static_cast<pthread_rwlock_t*>(malloc(sizeof(pthread_rwlock_t)));
+    static_cast<pthread_rwlock_t*>(llvm::malloc(sizeof(pthread_rwlock_t)));
 
 #ifdef __APPLE__
   // Workaround a bug/mis-feature in Darwin's pthread_rwlock_init.

llvm/lib/Support/StringMap.cpp

@@ -57,10 +57,9 @@ void StringMapImpl::init(unsigned InitSize) {
   NumItems = 0;
   NumTombstones = 0;
   
-  TheTable = (StringMapEntryBase **)calloc(NewNumBuckets+1,
-                                           sizeof(StringMapEntryBase **) +
-                                           sizeof(unsigned));
-
+  TheTable = static_cast<StringMapEntryBase **>(
+      std::calloc(NewNumBuckets+1,
+                  sizeof(StringMapEntryBase **) + sizeof(unsigned)));
   if (TheTable == nullptr)
     report_bad_alloc_error("Allocation of StringMap table failed.");
 
@@ -219,10 +218,8 @@ unsigned StringMapImpl::RehashTable(unsigned BucketNo) {
   unsigned NewBucketNo = BucketNo;
   // Allocate one extra bucket which will always be non-empty.  This allows the
   // iterators to stop at end.
-  StringMapEntryBase **NewTableArray =
-    (StringMapEntryBase **)calloc(NewSize+1, sizeof(StringMapEntryBase *) +
-                                             sizeof(unsigned));
-
+  StringMapEntryBase **NewTableArray = static_cast<StringMapEntryBase **>(
+      std::calloc(NewSize+1, sizeof(StringMapEntryBase *) + sizeof(unsigned)));
   if (NewTableArray == nullptr)
     report_bad_alloc_error("Allocation of StringMap hash table failed.");
 

llvm/lib/Support/Unix/Signals.inc

@@ -138,7 +138,7 @@ static void CreateSigAltStack() {
     return;
 
   stack_t AltStack = {};
-  AltStack.ss_sp = reinterpret_cast<char *>(malloc(AltStackSize));
+  AltStack.ss_sp = static_cast<char *>(llvm::malloc(AltStackSize));
   NewAltStackPointer = AltStack.ss_sp; // Save to avoid reporting a leak.
   AltStack.ss_size = AltStackSize;
   if (sigaltstack(&AltStack, &OldAltStack) != 0)

llvm/lib/Support/Windows/RWMutex.inc

@@ -74,10 +74,10 @@ static bool loadSRW() {
 
 sys::RWMutexImpl::RWMutexImpl() {
   if (loadSRW()) {
-    data_ = calloc(1, sizeof(SRWLOCK));
+    data_ = llvm::calloc(1, sizeof(SRWLOCK));
     fpInitializeSRWLock(static_cast<PSRWLOCK>(data_));
   } else {
-    data_ = calloc(1, sizeof(CRITICAL_SECTION));
+    data_ = llvm::calloc(1, sizeof(CRITICAL_SECTION));
     InitializeCriticalSection(static_cast<LPCRITICAL_SECTION>(data_));
   }
 }

llvm/tools/llvm-c-test/attributes.c

@@ -14,6 +14,7 @@
 
 #include "llvm-c-test.h"
 
+#include <assert.h>
 #include <stdlib.h>
 
 int llvm_test_function_attributes(void) {
@@ -30,6 +31,7 @@ int llvm_test_function_attributes(void) {
       int AttrCount = LLVMGetAttributeCountAtIndex(F, Idx);
       LLVMAttributeRef *Attrs =
           (LLVMAttributeRef *)malloc(AttrCount * sizeof(LLVMAttributeRef));
+      assert(Attrs);
       LLVMGetAttributesAtIndex(F, Idx, Attrs);
       free(Attrs);
     }
@@ -61,6 +63,7 @@ int llvm_test_callsite_attributes(void) {
             int AttrCount = LLVMGetCallSiteAttributeCount(I, Idx);
             LLVMAttributeRef *Attrs = (LLVMAttributeRef *)malloc(
                 AttrCount * sizeof(LLVMAttributeRef));
+            assert(Attrs);
             LLVMGetCallSiteAttributes(I, Idx, Attrs);
             free(Attrs);
           }

llvm/tools/llvm-c-test/echo.cpp

@@ -90,7 +90,8 @@ struct TypeCloner {
         unsigned ParamCount = LLVMCountParamTypes(Src);
         LLVMTypeRef* Params = nullptr;
         if (ParamCount > 0) {
-          Params = (LLVMTypeRef*) malloc(ParamCount * sizeof(LLVMTypeRef));
+          Params = static_cast<LLVMTypeRef*>(
+              llvm::malloc(ParamCount * sizeof(LLVMTypeRef)));
           LLVMGetParamTypes(Src, Params);
           for (unsigned i = 0; i < ParamCount; i++)
             Params[i] = Clone(Params[i]);

llvm/unittests/Support/AllocatorTest.cpp

@@ -147,7 +147,7 @@ public:
     // Allocate space for the alignment, the slab, and a void* that goes right
     // before the slab.
     size_t Alignment = 4096;
-    void *MemBase = malloc(Size + Alignment - 1 + sizeof(void*));
+    void *MemBase = llvm::malloc(Size + Alignment - 1 + sizeof(void*));
 
     // Find the slab start.
     void *Slab = (void *)alignAddr((char*)MemBase + sizeof(void *), Alignment);

llvm/unittests/Support/ManagedStatic.cpp

@@ -6,6 +6,8 @@
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
+
+#include "llvm/Support/Allocator.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Config/config.h"
 #ifdef HAVE_PTHREAD_H
@@ -30,7 +32,7 @@ namespace test1 {
   // Valgrind's leak checker complains glibc's stack allocation.
   // To appease valgrind, we provide our own stack for each thread.
   void *allocate_stack(pthread_attr_t &a, size_t n = 65536) {
-    void *stack = malloc(n);
+    void *stack = llvm::malloc(n);
     pthread_attr_init(&a);
 #if defined(__linux__)
     pthread_attr_setstack(&a, stack, n);
@@ -83,7 +85,7 @@ TEST(ManagedStaticTest, NestedStatics) {
 namespace CustomCreatorDeletor {
 struct CustomCreate {
   static void *call() {
-    void *Mem = std::malloc(sizeof(int));
+    void *Mem = llvm::malloc(sizeof(int));
     *((int *)Mem) = 42;
     return Mem;
   }