Move OperandList to be allocated prior to User for hung off subclasses.

For hung off uses, we need a Use* to tell use where the operands are.
This was User::OperandList but we want to remove that to save space
of all subclasses which aren't making use of 'hung off uses'.

Hung off uses now allocate their own 'OperandList' Use* in the
User::new which they call.

getOperandList() now uses the hung off uses bit to work out where the
Use* for the OperandList lives.  If a User has hung off uses, then this
bit tells them to go back a single Use* from the User* and use that
value as the OperandList.

If a User has no hung off uses, then we get the first operand by
subtracting (NumOperands * sizeof(Use)) from the User this pointer.

This saves a pointer from User and all subclasses.  Given the average
size of a subclass of User is 112 or 128 bytes, this saves around 7% of space
With malloc tending to align to 16-bytes the real saving is typically more like 3.5%.

On 'opt -O2 verify-uselistorder.lto.bc', peak memory usage prior to this change
is 149MB and after is 143MB so the savings are around 2.5% of peak.

Looking at some passes which allocate many Instructions and Values, parseIR drops
from 54.25MB to 52.21MB while the Inliner calls to Instruction::clone() drops
from 28.20MB to 27.05MB.

Reviewed by Duncan Exon Smith.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@239623 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/llvm/IR/User.h b/include/llvm/IR/User.h
index 743eefb707790914e7f0bea80903be6b6e810636..f1aa89f37152946535b08fa534eaec692d64eedb 100644 (file)
--- a/include/llvm/IR/User.h
+++ b/include/llvm/IR/User.h
@@ -37,14 +37,6 @@ class User : public Value {
   template <unsigned>
   friend struct HungoffOperandTraits;
   virtual void anchor();
-protected:
-  /// \brief This is a pointer to the array of Uses for this User.
-  ///
-  /// For nodes of fixed arity (e.g. a binary operator) this array will live
-  /// prefixed to some derived class instance.  For nodes of resizable variable
-  /// arity (e.g. PHINodes, SwitchInst etc.), this memory will be dynamically
-  /// allocated and should be destroyed by the classes' virtual dtor.
-  Use *LegacyOperandList;
 
 protected:
   /// Allocate a User with an operand pointer co-allocated.
@@ -60,9 +52,12 @@ protected:
 
   User(Type *ty, unsigned vty, Use *OpList, unsigned NumOps)
       : Value(ty, vty) {
-    setOperandList(OpList);
     assert(NumOps < (1u << NumUserOperandsBits) && "Too many operands");
     NumUserOperands = NumOps;
+    // If we have hung off uses, then the operand list should initially be
+    // null.
+    assert((!HasHungOffUses || !getOperandList()) &&
+           "Error in initializing hung off uses for User");
   }
 
   /// \brief Allocate the array of Uses, followed by a pointer
@@ -77,14 +72,6 @@ protected:
 
 public:
   ~User() override {
-    // drop the hung off uses.
-    Use::zap(getOperandList(), getOperandList() + NumUserOperands,
-             HasHungOffUses);
-    if (HasHungOffUses) {
-      setOperandList(nullptr);
-      // Reset NumOperands so User::operator delete() does the right thing.
-      NumUserOperands = 0;
-    }
   }
   /// \brief Free memory allocated for User and Use objects.
   void operator delete(void *Usr);
@@ -109,12 +96,23 @@ protected:
     return OpFrom<Idx>(this);
   }
 private:
+  Use *&getHungOffOperands() { return *(reinterpret_cast<Use **>(this) - 1); }
+
+  Use *getIntrusiveOperands() {
+    return reinterpret_cast<Use *>(this) - NumUserOperands;
+  }
+
   void setOperandList(Use *NewList) {
-    LegacyOperandList = NewList;
+    assert(HasHungOffUses &&
+           "Setting operand list only required for hung off uses");
+    getHungOffOperands() = NewList;
   }
 public:
-  Use *getOperandList() const {
-    return LegacyOperandList;
+  Use *getOperandList() {
+    return HasHungOffUses ? getHungOffOperands() : getIntrusiveOperands();
+  }
+  const Use *getOperandList() const {
+    return const_cast<User *>(this)->getOperandList();
   }
   Value *getOperand(unsigned i) const {
     assert(i < NumUserOperands && "getOperand() out of range!");

diff --git a/lib/IR/User.cpp b/lib/IR/User.cpp
index 5ccf00e1552793d04e52ed5b52e80b885b0f9757..c6e4e89bb34cf3d047d7cd7ff0cf626c687c3ee7 100644 (file)
--- a/lib/IR/User.cpp
+++ b/lib/IR/User.cpp
@@ -90,19 +90,20 @@ void *User::operator new(size_t Size, unsigned Us) {
   Use *Start = static_cast<Use*>(Storage);
   Use *End = Start + Us;
   User *Obj = reinterpret_cast<User*>(End);
-  Obj->setOperandList(Start);
-  Obj->HasHungOffUses = false;
   Obj->NumUserOperands = Us;
+  Obj->HasHungOffUses = false;
   Use::initTags(Start, End);
   return Obj;
 }
 
 void *User::operator new(size_t Size) {
-  void *Storage = ::operator new(Size);
-  User *Obj = reinterpret_cast<User*>(Storage);
-  Obj->setOperandList(nullptr);
-  Obj->HasHungOffUses = true;
+  // Allocate space for a single Use*
+  void *Storage = ::operator new(Size + sizeof(Use *));
+  Use **HungOffOperandList = static_cast<Use **>(Storage);
+  User *Obj = reinterpret_cast<User *>(HungOffOperandList + 1);
   Obj->NumUserOperands = 0;
+  Obj->HasHungOffUses = true;
+  *HungOffOperandList = nullptr;
   return Obj;
 }
 
@@ -111,11 +112,21 @@ void *User::operator new(size_t Size) {
 //===----------------------------------------------------------------------===//
 
 void User::operator delete(void *Usr) {
-  User *Start = static_cast<User*>(Usr);
-  Use *Storage = static_cast<Use*>(Usr) - Start->NumUserOperands;
-  // If there were hung-off uses, they will have been freed already and
-  // NumOperands reset to 0, so here we just free the User itself.
-  ::operator delete(Storage);
+  // Hung off uses use a single Use* before the User, while other subclasses
+  // use a Use[] allocated prior to the user.
+  User *Obj = static_cast<User *>(Usr);
+  if (Obj->HasHungOffUses) {
+    Use **HungOffOperandList = static_cast<Use **>(Usr) - 1;
+    // drop the hung off uses.
+    Use::zap(*HungOffOperandList, *HungOffOperandList + Obj->NumUserOperands,
+             /* Delete */ true);
+    ::operator delete(HungOffOperandList);
+  } else {
+    Use *Storage = static_cast<Use *>(Usr) - Obj->NumUserOperands;
+    Use::zap(Storage, Storage + Obj->NumUserOperands,
+             /* Delete */ false);
+    ::operator delete(Storage);
+  }
 }
 
 //===----------------------------------------------------------------------===//