+ /// \brief Deallocate all but the current slab and reset the current pointer
+ /// to the beginning of it, freeing all memory allocated so far.
+ void Reset() {
+ if (Slabs.empty())
+ return;
+
+ // Reset the state.
+ BytesAllocated = 0;
+ CurPtr = (char *)Slabs.front();
+ End = CurPtr + SlabSize;
+
+ // Deallocate all but the first slab, and all custome sized slabs.
+ DeallocateSlabs(std::next(Slabs.begin()), Slabs.end());
+ Slabs.erase(std::next(Slabs.begin()), Slabs.end());
+ DeallocateCustomSizedSlabs();
+ CustomSizedSlabs.clear();
+ }
+
+ /// \brief Allocate space at the specified alignment.
+ LLVM_ATTRIBUTE_RETURNS_NONNULL void *Allocate(size_t Size, size_t Alignment) {
+ assert(Alignment > 0 && "0-byte alignnment is not allowed. Use 1 instead.");
+
+ // Keep track of how many bytes we've allocated.
+ BytesAllocated += Size;
+
+ size_t Adjustment = alignmentAdjustment(CurPtr, Alignment);
+ assert(Adjustment + Size >= Size && "Adjustment + Size must not overflow");
+
+ // Check if we have enough space.
+ if (Adjustment + Size <= size_t(End - CurPtr)) {
+ char *AlignedPtr = CurPtr + Adjustment;
+ CurPtr = AlignedPtr + Size;
+ // Update the allocation point of this memory block in MemorySanitizer.
+ // Without this, MemorySanitizer messages for values originated from here
+ // will point to the allocation of the entire slab.
+ __msan_allocated_memory(AlignedPtr, Size);
+ return AlignedPtr;
+ }
+
+ // If Size is really big, allocate a separate slab for it.
+ size_t PaddedSize = Size + Alignment - 1;
+ if (PaddedSize > SizeThreshold) {
+ void *NewSlab = Allocator.Allocate(PaddedSize, 0);
+ CustomSizedSlabs.push_back(std::make_pair(NewSlab, PaddedSize));
+
+ uintptr_t AlignedAddr = alignAddr(NewSlab, Alignment);
+ assert(AlignedAddr + Size <= (uintptr_t)NewSlab + PaddedSize);
+ char *AlignedPtr = (char*)AlignedAddr;
+ __msan_allocated_memory(AlignedPtr, Size);
+ return AlignedPtr;
+ }
+
+ // Otherwise, start a new slab and try again.
+ StartNewSlab();
+ uintptr_t AlignedAddr = alignAddr(CurPtr, Alignment);
+ assert(AlignedAddr + Size <= (uintptr_t)End &&
+ "Unable to allocate memory!");
+ char *AlignedPtr = (char*)AlignedAddr;
+ CurPtr = AlignedPtr + Size;
+ __msan_allocated_memory(AlignedPtr, Size);
+ return AlignedPtr;
+ }
+
+ // Pull in base class overloads.
+ using AllocatorBase<BumpPtrAllocatorImpl>::Allocate;
+
+ void Deallocate(const void * /*Ptr*/, size_t /*Size*/) {}
+
+ // Pull in base class overloads.
+ using AllocatorBase<BumpPtrAllocatorImpl>::Deallocate;
+
+ size_t GetNumSlabs() const { return Slabs.size() + CustomSizedSlabs.size(); }
+
+ size_t getTotalMemory() const {
+ size_t TotalMemory = 0;
+ for (auto I = Slabs.begin(), E = Slabs.end(); I != E; ++I)
+ TotalMemory += computeSlabSize(std::distance(Slabs.begin(), I));
+ for (auto &PtrAndSize : CustomSizedSlabs)
+ TotalMemory += PtrAndSize.second;
+ return TotalMemory;
+ }
+
+ void PrintStats() const {
+ detail::printBumpPtrAllocatorStats(Slabs.size(), BytesAllocated,
+ getTotalMemory());
+ }
+
+private:
+ /// \brief The current pointer into the current slab.
+ ///
+ /// This points to the next free byte in the slab.
+ char *CurPtr;
+
+ /// \brief The end of the current slab.
+ char *End;
+
+ /// \brief The slabs allocated so far.
+ SmallVector<void *, 4> Slabs;
+
+ /// \brief Custom-sized slabs allocated for too-large allocation requests.
+ SmallVector<std::pair<void *, size_t>, 0> CustomSizedSlabs;
+
+ /// \brief How many bytes we've allocated.
+ ///
+ /// Used so that we can compute how much space was wasted.
+ size_t BytesAllocated;
+
+ /// \brief The allocator instance we use to get slabs of memory.
+ AllocatorT Allocator;
+
+ static size_t computeSlabSize(unsigned SlabIdx) {
+ // Scale the actual allocated slab size based on the number of slabs
+ // allocated. Every 128 slabs allocated, we double the allocated size to
+ // reduce allocation frequency, but saturate at multiplying the slab size by
+ // 2^30.
+ return SlabSize * ((size_t)1 << std::min<size_t>(30, SlabIdx / 128));
+ }
+
+ /// \brief Allocate a new slab and move the bump pointers over into the new
+ /// slab, modifying CurPtr and End.
+ void StartNewSlab() {
+ size_t AllocatedSlabSize = computeSlabSize(Slabs.size());
+
+ void *NewSlab = Allocator.Allocate(AllocatedSlabSize, 0);
+ Slabs.push_back(NewSlab);
+ CurPtr = (char *)(NewSlab);
+ End = ((char *)NewSlab) + AllocatedSlabSize;
+ }
+
+ /// \brief Deallocate a sequence of slabs.
+ void DeallocateSlabs(SmallVectorImpl<void *>::iterator I,
+ SmallVectorImpl<void *>::iterator E) {
+ for (; I != E; ++I) {
+ size_t AllocatedSlabSize =
+ computeSlabSize(std::distance(Slabs.begin(), I));
+#ifndef NDEBUG
+ // Poison the memory so stale pointers crash sooner. Note we must
+ // preserve the Size and NextPtr fields at the beginning.
+ if (AllocatedSlabSize != 0) {
+ sys::Memory::setRangeWritable(*I, AllocatedSlabSize);
+ memset(*I, 0xCD, AllocatedSlabSize);
+ }
+#endif
+ Allocator.Deallocate(*I, AllocatedSlabSize);
+ }
+ }
+
+ /// \brief Deallocate all memory for custom sized slabs.
+ void DeallocateCustomSizedSlabs() {
+ for (auto &PtrAndSize : CustomSizedSlabs) {
+ void *Ptr = PtrAndSize.first;
+ size_t Size = PtrAndSize.second;
+#ifndef NDEBUG
+ // Poison the memory so stale pointers crash sooner. Note we must
+ // preserve the Size and NextPtr fields at the beginning.
+ sys::Memory::setRangeWritable(Ptr, Size);
+ memset(Ptr, 0xCD, Size);
+#endif
+ Allocator.Deallocate(Ptr, Size);
+ }
+ }
+
+ template <typename T> friend class SpecificBumpPtrAllocator;
+};
+
+/// \brief The standard BumpPtrAllocator which just uses the default template
+/// paramaters.
+typedef BumpPtrAllocatorImpl<> BumpPtrAllocator;
+
+/// \brief A BumpPtrAllocator that allows only elements of a specific type to be
+/// allocated.
+///
+/// This allows calling the destructor in DestroyAll() and when the allocator is
+/// destroyed.
+template <typename T> class SpecificBumpPtrAllocator {
+ BumpPtrAllocator Allocator;
+
+public:
+ SpecificBumpPtrAllocator() : Allocator() {}
+ SpecificBumpPtrAllocator(SpecificBumpPtrAllocator &&Old)
+ : Allocator(std::move(Old.Allocator)) {}
+ ~SpecificBumpPtrAllocator() { DestroyAll(); }
+
+ SpecificBumpPtrAllocator &operator=(SpecificBumpPtrAllocator &&RHS) {
+ Allocator = std::move(RHS.Allocator);
+ return *this;
+ }
+
+ /// Call the destructor of each allocated object and deallocate all but the
+ /// current slab and reset the current pointer to the beginning of it, freeing
+ /// all memory allocated so far.
+ void DestroyAll() {
+ auto DestroyElements = [](char *Begin, char *End) {
+ assert(Begin == (char*)alignAddr(Begin, alignOf<T>()));
+ for (char *Ptr = Begin; Ptr + sizeof(T) <= End; Ptr += sizeof(T))
+ reinterpret_cast<T *>(Ptr)->~T();
+ };
+
+ for (auto I = Allocator.Slabs.begin(), E = Allocator.Slabs.end(); I != E;
+ ++I) {
+ size_t AllocatedSlabSize = BumpPtrAllocator::computeSlabSize(
+ std::distance(Allocator.Slabs.begin(), I));
+ char *Begin = (char*)alignAddr(*I, alignOf<T>());
+ char *End = *I == Allocator.Slabs.back() ? Allocator.CurPtr
+ : (char *)*I + AllocatedSlabSize;
+
+ DestroyElements(Begin, End);
+ }
+
+ for (auto &PtrAndSize : Allocator.CustomSizedSlabs) {
+ void *Ptr = PtrAndSize.first;
+ size_t Size = PtrAndSize.second;
+ DestroyElements((char*)alignAddr(Ptr, alignOf<T>()), (char *)Ptr + Size);
+ }
+
+ Allocator.Reset();
+ }
+
+ /// \brief Allocate space for an array of objects without constructing them.
+ T *Allocate(size_t num = 1) { return Allocator.Allocate<T>(num); }