From 101b25c028706c61c7dd9fb92d0b3c1541cb12b6 Mon Sep 17 00:00:00 2001 From: Nate Begeman Date: Tue, 15 Sep 2009 19:05:41 +0000 Subject: [PATCH] Better solution for tracking both the original alignment of the access, and the current alignment based on the source value offset. This avoids increasing the size of mem nodes. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@81897 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/CodeGen/SelectionDAG.h | 7 ++-- include/llvm/CodeGen/SelectionDAGNodes.h | 42 +++++++++---------- .../SelectionDAG/LegalizeVectorTypes.cpp | 31 ++++++-------- lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 41 +++++++++--------- .../SelectionDAG/SelectionDAGBuild.cpp | 5 +-- 5 files changed, 55 insertions(+), 71 deletions(-) diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 064fc874551..d6dc7c274a1 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -544,7 +544,7 @@ public: /// SDValue getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, bool isVolatile=false, - unsigned Alignment=0, unsigned OrigAlignment=0); + unsigned Alignment=0); SDValue getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, EVT EVT, bool isVolatile=false, @@ -554,14 +554,13 @@ public: SDValue getLoad(ISD::MemIndexedMode AM, DebugLoc dl, ISD::LoadExtType ExtType, EVT VT, SDValue Chain, SDValue Ptr, SDValue Offset, const Value *SV, int SVOffset, EVT EVT, - bool isVolatile=false, unsigned Alignment=0, - unsigned OrigAlignment=0); + bool isVolatile=false, unsigned Alignment=0); /// getStore - Helper function to build ISD::STORE nodes. /// SDValue getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, bool isVolatile=false, - unsigned Alignment=0, unsigned OrigAlignment=0); + unsigned Alignment=0); SDValue getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, EVT TVT, bool isVolatile=false, unsigned Alignment=0); diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index a3ce5d432d3..04e1b853fe7 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -28,6 +28,7 @@ #include "llvm/CodeGen/ValueTypes.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/Support/Allocator.h" +#include "llvm/Support/MathExtras.h" #include "llvm/Support/RecyclingAllocator.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/DebugLoc.h" @@ -1518,23 +1519,22 @@ private: //! SVOffset - Memory location offset. Note that base is defined in MemSDNode int SVOffset; - - //! OrigAlign - The original alignment of this MemSDNode in the case where - // this node was created by legalize from a MemSDNode with known alignment. - unsigned OrigAlign; - public: MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT MemoryVT, - const Value *srcValue, int SVOff, - unsigned alignment, bool isvolatile, unsigned oalign); + const Value *srcValue, int SVOff, unsigned alignment, + bool isvolatile); MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops, unsigned NumOps, EVT MemoryVT, const Value *srcValue, int SVOff, - unsigned alignment, bool isvolatile, unsigned oalign); + unsigned alignment, bool isvolatile); /// Returns alignment and volatility of the memory access - unsigned getAlignment() const { return (1u << (SubclassData >> 6)) >> 1; } - unsigned getOriginalAlignment() const { return OrigAlign; } + unsigned getOriginalAlignment() const { + return (1u << (SubclassData >> 6)) >> 1; + } + unsigned getAlignment() const { + return MinAlign(getOriginalAlignment(), SVOffset); + } bool isVolatile() const { return (SubclassData >> 5) & 1; } /// getRawSubclassData - Return the SubclassData value, which contains an @@ -1605,14 +1605,14 @@ public: SDValue Cmp, SDValue Swp, const Value* SrcVal, unsigned Align=0) : MemSDNode(Opc, dl, VTL, MemVT, SrcVal, /*SVOffset=*/0, - Align, /*isVolatile=*/true, /* OrigAlign=*/0) { + Align, /*isVolatile=*/true) { InitOperands(Ops, Chain, Ptr, Cmp, Swp); } AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value* SrcVal, unsigned Align=0) : MemSDNode(Opc, dl, VTL, MemVT, SrcVal, /*SVOffset=*/0, - Align, /*isVolatile=*/true, /* OrigAlign=*/0) { + Align, /*isVolatile=*/true) { InitOperands(Ops, Chain, Ptr, Val); } @@ -1653,8 +1653,7 @@ public: const SDValue *Ops, unsigned NumOps, EVT MemoryVT, const Value *srcValue, int SVO, unsigned Align, bool Vol, bool ReadMem, bool WriteMem) - : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, srcValue, SVO, Align, Vol, - /* OrigAlign=*/0), + : MemSDNode(Opc, dl, VTs, Ops, NumOps, MemoryVT, srcValue, SVO, Align, Vol), ReadMem(ReadMem), WriteMem(WriteMem) { } @@ -2269,9 +2268,8 @@ class LSBaseSDNode : public MemSDNode { public: LSBaseSDNode(ISD::NodeType NodeTy, DebugLoc dl, SDValue *Operands, unsigned numOperands, SDVTList VTs, ISD::MemIndexedMode AM, - EVT VT, const Value *SV, int SVO, unsigned Align, bool Vol, - unsigned OAlign) - : MemSDNode(NodeTy, dl, VTs, VT, SV, SVO, Align, Vol, OAlign) { + EVT VT, const Value *SV, int SVO, unsigned Align, bool Vol) + : MemSDNode(NodeTy, dl, VTs, VT, SV, SVO, Align, Vol) { assert(Align != 0 && "Loads and stores should have non-zero aligment"); SubclassData |= AM << 2; assert(getAddressingMode() == AM && "MemIndexedMode encoding error!"); @@ -2309,10 +2307,9 @@ class LoadSDNode : public LSBaseSDNode { friend class SelectionDAG; LoadSDNode(SDValue *ChainPtrOff, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT LVT, - const Value *SV, int O=0, unsigned Align=0, bool Vol=false, - unsigned OAlign=0) + const Value *SV, int O=0, unsigned Align=0, bool Vol=false) : LSBaseSDNode(ISD::LOAD, dl, ChainPtrOff, 3, - VTs, AM, LVT, SV, O, Align, Vol, OAlign) { + VTs, AM, LVT, SV, O, Align, Vol) { SubclassData |= (unsigned short)ETy; assert(getExtensionType() == ETy && "LoadExtType encoding error!"); } @@ -2339,10 +2336,9 @@ class StoreSDNode : public LSBaseSDNode { friend class SelectionDAG; StoreSDNode(SDValue *ChainValuePtrOff, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, bool isTrunc, EVT SVT, - const Value *SV, int O=0, unsigned Align=0, bool Vol=false, - unsigned OAlign=0) + const Value *SV, int O=0, unsigned Align=0, bool Vol=false) : LSBaseSDNode(ISD::STORE, dl, ChainValuePtrOff, 4, - VTs, AM, SVT, SV, O, Align, Vol, OAlign) { + VTs, AM, SVT, SV, O, Align, Vol) { SubclassData |= (unsigned short)isTrunc; assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!"); } diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 33adf08f674..a03f825a9f0 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -170,8 +170,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) { DAG.getUNDEF(N->getBasePtr().getValueType()), N->getSrcValue(), N->getSrcValueOffset(), N->getMemoryVT().getVectorElementType(), - N->isVolatile(), N->getAlignment(), - N->getOriginalAlignment()); + N->isVolatile(), N->getOriginalAlignment()); // Legalized the chain result - switch anything that used the old chain to // use the new one. @@ -361,8 +360,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){ return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)), N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(), - N->isVolatile(), N->getAlignment(), - N->getOriginalAlignment()); + N->isVolatile(), N->getOriginalAlignment()); } @@ -716,23 +714,21 @@ void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, const Value *SV = LD->getSrcValue(); int SVOffset = LD->getSrcValueOffset(); EVT MemoryVT = LD->getMemoryVT(); - unsigned Alignment = LD->getAlignment(); - unsigned OrigAlignment = LD->getOriginalAlignment(); + unsigned Alignment = LD->getOriginalAlignment(); bool isVolatile = LD->isVolatile(); EVT LoMemVT, HiMemVT; GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT); Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset, - SV, SVOffset, LoMemVT, isVolatile, Alignment, OrigAlignment); + SV, SVOffset, LoMemVT, isVolatile, Alignment); unsigned IncrementSize = LoMemVT.getSizeInBits()/8; Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, DAG.getIntPtrConstant(IncrementSize)); SVOffset += IncrementSize; - Alignment = MinAlign(Alignment, IncrementSize); Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset, - SV, SVOffset, HiMemVT, isVolatile, Alignment, OrigAlignment); + SV, SVOffset, HiMemVT, isVolatile, Alignment); // Build a factor node to remember that this load is independent of the // other one. @@ -1081,8 +1077,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { SDValue Ptr = N->getBasePtr(); int SVOffset = N->getSrcValueOffset(); EVT MemoryVT = N->getMemoryVT(); - unsigned Alignment = N->getAlignment(); - unsigned OrigAlignment = N->getOriginalAlignment(); + unsigned Alignment = N->getOriginalAlignment(); bool isVol = N->isVolatile(); SDValue Lo, Hi; GetSplitVector(N->getOperand(1), Lo, Hi); @@ -1097,21 +1092,19 @@ SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) { LoMemVT, isVol, Alignment); else Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset, - isVol, Alignment, OrigAlignment); + isVol, Alignment); // Increment the pointer to the other half. Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, DAG.getIntPtrConstant(IncrementSize)); + SVOffset += IncrementSize; if (isTruncating) - Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, - N->getSrcValue(), SVOffset+IncrementSize, - HiMemVT, - isVol, MinAlign(Alignment, IncrementSize)); + Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, + HiMemVT, isVol, Alignment); else - Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize, - isVol, MinAlign(Alignment, IncrementSize), - OrigAlignment); + Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset, + isVol, Alignment); return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index fe45319339e..d0d45b5c77a 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -432,12 +432,14 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { const LoadSDNode *LD = cast(N); ID.AddInteger(LD->getMemoryVT().getRawBits()); ID.AddInteger(LD->getRawSubclassData()); + ID.AddInteger(LD->getSrcValueOffset()); break; } case ISD::STORE: { const StoreSDNode *ST = cast(N); ID.AddInteger(ST->getMemoryVT().getRawBits()); ID.AddInteger(ST->getRawSubclassData()); + ID.AddInteger(ST->getSrcValueOffset()); break; } case ISD::ATOMIC_CMP_SWAP: @@ -3646,12 +3648,9 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, ISD::LoadExtType ExtType, EVT VT, SDValue Chain, SDValue Ptr, SDValue Offset, const Value *SV, int SVOffset, EVT EVT, - bool isVolatile, unsigned Alignment, - unsigned OrigAlignment) { + bool isVolatile, unsigned Alignment) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(VT); - if (OrigAlignment == 0) - OrigAlignment = Alignment; if (VT == EVT) { ExtType = ISD::NON_EXTLOAD; @@ -3682,12 +3681,13 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); ID.AddInteger(EVT.getRawBits()); ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, isVolatile, Alignment)); + ID.AddInteger(SVOffset); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); SDNode *N = NodeAllocator.Allocate(); new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, EVT, SV, SVOffset, - Alignment, isVolatile, OrigAlignment); + Alignment, isVolatile); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3696,11 +3696,10 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, - bool isVolatile, unsigned Alignment, - unsigned OrigAlignment) { + bool isVolatile, unsigned Alignment) { SDValue Undef = getUNDEF(Ptr.getValueType()); return getLoad(ISD::UNINDEXED, dl, ISD::NON_EXTLOAD, VT, Chain, Ptr, Undef, - SV, SVOffset, VT, isVolatile, Alignment, OrigAlignment); + SV, SVOffset, VT, isVolatile, Alignment); } SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, @@ -3727,14 +3726,11 @@ SelectionDAG::getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base, SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, - bool isVolatile, unsigned Alignment, - unsigned OrigAlignment) { + bool isVolatile, unsigned Alignment) { EVT VT = Val.getValueType(); if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(VT); - if (OrigAlignment == 0) - OrigAlignment = Alignment; SDVTList VTs = getVTList(MVT::Other); SDValue Undef = getUNDEF(Ptr.getValueType()); @@ -3744,12 +3740,13 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, ID.AddInteger(VT.getRawBits()); ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, isVolatile, Alignment)); + ID.AddInteger(SVOffset); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); SDNode *N = NodeAllocator.Allocate(); new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false, - VT, SV, SVOffset, Alignment, isVolatile, OrigAlignment); + VT, SV, SVOffset, Alignment, isVolatile); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3779,6 +3776,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, ID.AddInteger(SVT.getRawBits()); ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, isVolatile, Alignment)); + ID.AddInteger(SVOffset); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); @@ -4976,25 +4974,24 @@ GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, const GlobalValue *GA, MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, const Value *srcValue, int SVO, unsigned alignment, - bool vol, unsigned origAlign) - : SDNode(Opc, dl, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO), - OrigAlign(origAlign) { + bool vol) + : SDNode(Opc, dl, VTs), MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) { SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment); assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!"); - assert(getAlignment() == alignment && "Alignment representation error!"); - assert(isVolatile() == vol && "Volatile representation error!"); + assert(getOriginalAlignment() == alignment && "Alignment encoding error!"); + assert(isVolatile() == vol && "Volatile encoding error!"); } MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, const SDValue *Ops, unsigned NumOps, EVT memvt, const Value *srcValue, int SVO, unsigned alignment, - bool vol, unsigned origAlign) + bool vol) : SDNode(Opc, dl, VTs, Ops, NumOps), - MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO), OrigAlign(origAlign) { + MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO) { SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, vol, alignment); assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!"); - assert(getAlignment() == alignment && "Alignment representation error!"); - assert(isVolatile() == vol && "Volatile representation error!"); + assert(getOriginalAlignment() == alignment && "Alignment encoding error!"); + assert(isVolatile() == vol && "Volatile encoding error!"); } /// getMemOperand - Return a MachineMemOperand object describing the memory diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp index 8b9a5a5459d..c946bdad5d3 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuild.cpp @@ -2815,7 +2815,7 @@ void SelectionDAGLowering::visitLoad(LoadInst &I) { DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, Ptr, DAG.getConstant(Offsets[i], PtrVT)), - SV, Offsets[i], isVolatile, Alignment, Alignment); + SV, Offsets[i], isVolatile, Alignment); Values[i] = L; Chains[i] = L.getValue(1); } @@ -2864,8 +2864,7 @@ void SelectionDAGLowering::visitStore(StoreInst &I) { DAG.getNode(ISD::ADD, getCurDebugLoc(), PtrVT, Ptr, DAG.getConstant(Offsets[i], PtrVT)), - PtrV, Offsets[i], - isVolatile, Alignment, Alignment); + PtrV, Offsets[i], isVolatile, Alignment); DAG.setRoot(DAG.getNode(ISD::TokenFactor, getCurDebugLoc(), MVT::Other, &Chains[0], NumValues)); -- 2.34.1