X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAG.cpp;h=3643ea7c5315d00846867b1c38dc853bbb76cab2;hb=a4f2bb08de92ccec21cbb3d0a9518384b0173660;hp=f6fed2145d994b06bca6467c63d1530293f8c912;hpb=c76909abfec876c6b751d693ebd3df07df686aa0;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index f6fed2145d9..3643ea7c531 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -10,7 +10,10 @@ // This implements the SelectionDAG class. // //===----------------------------------------------------------------------===// + #include "llvm/CodeGen/SelectionDAG.h" +#include "SDNodeOrdering.h" +#include "SDNodeDbgValue.h" #include "llvm/Constants.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Function.h" @@ -27,11 +30,14 @@ #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetData.h" +#include "llvm/Target/TargetFrameInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MathExtras.h" @@ -199,19 +205,6 @@ bool ISD::isScalarToVector(const SDNode *N) { return true; } - -/// isDebugLabel - Return true if the specified node represents a debug -/// label (i.e. ISD::DBG_LABEL or TargetInstrInfo::DBG_LABEL node). -bool ISD::isDebugLabel(const SDNode *N) { - SDValue Zero; - if (N->getOpcode() == ISD::DBG_LABEL) - return true; - if (N->isMachineOpcode() && - N->getMachineOpcode() == TargetInstrInfo::DBG_LABEL) - return true; - return false; -} - /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) /// when given the operation for (X op Y). ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { @@ -392,13 +385,7 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { case ISD::Register: ID.AddInteger(cast(N)->getReg()); break; - case ISD::DBG_STOPPOINT: { - const DbgStopPointSDNode *DSP = cast(N); - ID.AddInteger(DSP->getLine()); - ID.AddInteger(DSP->getColumn()); - ID.AddPointer(DSP->getCompileUnit()); - break; - } + case ISD::SRCVALUE: ID.AddPointer(cast(N)->getValue()); break; @@ -459,6 +446,12 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { ID.AddInteger(SVN->getMaskElt(i)); break; } + case ISD::TargetBlockAddress: + case ISD::BlockAddress: { + ID.AddPointer(cast(N)->getBlockAddress()); + ID.AddInteger(cast(N)->getTargetFlags()); + break; + } } // end switch (N->getOpcode()) } @@ -476,18 +469,20 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { } /// encodeMemSDNodeFlags - Generic routine for computing a value for use in -/// the CSE map that carries volatility, indexing mode, and +/// the CSE map that carries volatility, temporalness, indexing mode, and /// extension/truncation information. /// static inline unsigned -encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile) { +encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, + bool isNonTemporal) { assert((ConvType & 3) == ConvType && "ConvType may not require more than 2 bits!"); assert((AM & 7) == AM && "AM may not require more than 3 bits!"); return ConvType | (AM << 2) | - (isVolatile << 5); + (isVolatile << 5) | + (isNonTemporal << 6); } //===----------------------------------------------------------------------===// @@ -502,8 +497,6 @@ static bool doNotCSE(SDNode *N) { switch (N->getOpcode()) { default: break; case ISD::HANDLENODE: - case ISD::DBG_LABEL: - case ISD::DBG_STOPPOINT: case ISD::EH_LABEL: return true; // Never CSE these nodes. } @@ -601,6 +594,14 @@ void SelectionDAG::DeallocateNode(SDNode *N) { N->NodeType = ISD::DELETED_NODE; NodeAllocator.Deallocate(AllNodes.remove(N)); + + // Remove the ordering of this node. + Ordering->remove(N); + + // If any of the SDDbgValue nodes refer to this SDNode, invalidate them. + SmallVector &DbgVals = DbgInfo->getSDDbgValues(N); + for (unsigned i = 0, e = DbgVals.size(); i != e; ++i) + DbgVals[i]->setIsInvalidated(); } /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that @@ -652,7 +653,7 @@ bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Flag && !N->isMachineOpcode() && !doNotCSE(N)) { N->dump(this); - errs() << "\n"; + dbgs() << "\n"; llvm_unreachable("Node is not in map!"); } #endif @@ -704,7 +705,8 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, FoldingSetNodeID ID; AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); AddNodeIDCustom(ID, N); - return CSEMap.FindNodeOrInsertPos(ID, InsertPos); + SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); + return Node; } /// FindModifiedNodeSlot - Find a slot for the specified node if its operands @@ -721,7 +723,8 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, FoldingSetNodeID ID; AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); AddNodeIDCustom(ID, N); - return CSEMap.FindNodeOrInsertPos(ID, InsertPos); + SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); + return Node; } @@ -738,7 +741,8 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, FoldingSetNodeID ID; AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); AddNodeIDCustom(ID, N); - return CSEMap.FindNodeOrInsertPos(ID, InsertPos); + SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); + return Node; } /// VerifyNode - Sanity check the given node. Aborts if it is invalid. @@ -790,9 +794,11 @@ unsigned SelectionDAG::getEVTAlignment(EVT VT) const { // EntryNode could meaningfully have debug info if we can find it... SelectionDAG::SelectionDAG(TargetLowering &tli, FunctionLoweringInfo &fli) : TLI(tli), FLI(fli), DW(0), - EntryNode(ISD::EntryToken, DebugLoc::getUnknownLoc(), - getVTList(MVT::Other)), Root(getEntryNode()) { + EntryNode(ISD::EntryToken, DebugLoc(), getVTList(MVT::Other)), + Root(getEntryNode()), Ordering(0) { AllNodes.push_back(&EntryNode); + Ordering = new SDNodeOrdering(); + DbgInfo = new SDDbgInfo(); } void SelectionDAG::init(MachineFunction &mf, MachineModuleInfo *mmi, @@ -805,6 +811,9 @@ void SelectionDAG::init(MachineFunction &mf, MachineModuleInfo *mmi, SelectionDAG::~SelectionDAG() { allnodes_clear(); + delete Ordering; + DbgInfo->clear(); + delete DbgInfo; } void SelectionDAG::allnodes_clear() { @@ -830,11 +839,32 @@ void SelectionDAG::clear() { EntryNode.UseList = 0; AllNodes.push_back(&EntryNode); Root = getEntryNode(); + delete Ordering; + Ordering = new SDNodeOrdering(); + DbgInfo->clear(); + delete DbgInfo; + DbgInfo = new SDDbgInfo(); +} + +SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { + return VT.bitsGT(Op.getValueType()) ? + getNode(ISD::SIGN_EXTEND, DL, VT, Op) : + getNode(ISD::TRUNCATE, DL, VT, Op); +} + +SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { + return VT.bitsGT(Op.getValueType()) ? + getNode(ISD::ZERO_EXTEND, DL, VT, Op) : + getNode(ISD::TRUNCATE, DL, VT, Op); } SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, DebugLoc DL, EVT VT) { + assert(!VT.isVector() && + "getZeroExtendInReg should use the vector element type instead of " + "the vector type!"); if (Op.getValueType() == VT) return Op; - APInt Imm = APInt::getLowBitsSet(Op.getValueSizeInBits(), + unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); + APInt Imm = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); return getNode(ISD::AND, DL, Op.getValueType(), Op, getConstant(Imm, Op.getValueType())); @@ -843,14 +873,14 @@ SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, DebugLoc DL, EVT VT) { /// getNOT - Create a bitwise NOT operation as (XOR Val, -1). /// SDValue SelectionDAG::getNOT(DebugLoc DL, SDValue Val, EVT VT) { - EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT; + EVT EltVT = VT.getScalarType(); SDValue NegOne = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); return getNode(ISD::XOR, DL, VT, Val, NegOne); } SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { - EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT; + EVT EltVT = VT.getScalarType(); assert((EltVT.getSizeInBits() >= 64 || (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && "getConstant with a uint64_t value that doesn't fit in the type!"); @@ -864,7 +894,7 @@ SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { assert(VT.isInteger() && "Cannot create FP integer constant!"); - EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT; + EVT EltVT = VT.getScalarType(); assert(Val.getBitWidth() == EltVT.getSizeInBits() && "APInt size does not match type size!"); @@ -877,9 +907,9 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) if (!VT.isVector()) return SDValue(N, 0); + if (!N) { - N = NodeAllocator.Allocate(); - new (N) ConstantSDNode(isT, &Val, EltVT); + N = new (NodeAllocator) ConstantSDNode(isT, &Val, EltVT); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); } @@ -888,8 +918,7 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { if (VT.isVector()) { SmallVector Ops; Ops.assign(VT.getVectorNumElements(), Result); - Result = getNode(ISD::BUILD_VECTOR, DebugLoc::getUnknownLoc(), - VT, &Ops[0], Ops.size()); + Result = getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, &Ops[0], Ops.size()); } return Result; } @@ -906,8 +935,7 @@ SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); - EVT EltVT = - VT.isVector() ? VT.getVectorElementType() : VT; + EVT EltVT = VT.getScalarType(); // Do the map lookup using the actual bit pattern for the floating point // value, so that we don't have problems with 0.0 comparing equal to -0.0, and @@ -921,9 +949,9 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) if (!VT.isVector()) return SDValue(N, 0); + if (!N) { - N = NodeAllocator.Allocate(); - new (N) ConstantFPSDNode(isTarget, &V, EltVT); + N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); } @@ -933,15 +961,13 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ SmallVector Ops; Ops.assign(VT.getVectorNumElements(), Result); // FIXME DebugLoc info might be appropriate here - Result = getNode(ISD::BUILD_VECTOR, DebugLoc::getUnknownLoc(), - VT, &Ops[0], Ops.size()); + Result = getNode(ISD::BUILD_VECTOR, DebugLoc(), VT, &Ops[0], Ops.size()); } return Result; } SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { - EVT EltVT = - VT.isVector() ? VT.getVectorElementType() : VT; + EVT EltVT = VT.getScalarType(); if (EltVT==MVT::f32) return getConstantFP(APFloat((float)Val), VT, isTarget); else @@ -982,8 +1008,9 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) GlobalAddressSDNode(Opc, GV, VT, Offset, TargetFlags); + + SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, GV, VT, + Offset, TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -997,8 +1024,8 @@ SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) FrameIndexSDNode(FI, VT, isTarget); + + SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -1016,8 +1043,9 @@ SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) JumpTableSDNode(JTI, VT, isTarget, TargetFlags); + + SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, + TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -1041,8 +1069,9 @@ SDValue SelectionDAG::getConstantPool(Constant *C, EVT VT, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) ConstantPoolSDNode(isTarget, C, VT, Offset, Alignment, TargetFlags); + + SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, + Alignment, TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -1067,8 +1096,9 @@ SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) ConstantPoolSDNode(isTarget, C, VT, Offset, Alignment, TargetFlags); + + SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, + Alignment, TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -1081,8 +1111,8 @@ SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) BasicBlockSDNode(MBB); + + SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -1097,8 +1127,7 @@ SDValue SelectionDAG::getValueType(EVT VT) { ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; if (N) return SDValue(N, 0); - N = NodeAllocator.Allocate(); - new (N) VTSDNode(VT); + N = new (NodeAllocator) VTSDNode(VT); AllNodes.push_back(N); return SDValue(N, 0); } @@ -1106,8 +1135,7 @@ SDValue SelectionDAG::getValueType(EVT VT) { SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { SDNode *&N = ExternalSymbols[Sym]; if (N) return SDValue(N, 0); - N = NodeAllocator.Allocate(); - new (N) ExternalSymbolSDNode(false, Sym, 0, VT); + N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); AllNodes.push_back(N); return SDValue(N, 0); } @@ -1118,8 +1146,7 @@ SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, TargetExternalSymbols[std::pair(Sym, TargetFlags)]; if (N) return SDValue(N, 0); - N = NodeAllocator.Allocate(); - new (N) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); + N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); AllNodes.push_back(N); return SDValue(N, 0); } @@ -1129,11 +1156,11 @@ SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { CondCodeNodes.resize(Cond+1); if (CondCodeNodes[Cond] == 0) { - CondCodeSDNode *N = NodeAllocator.Allocate(); - new (N) CondCodeSDNode(Cond); + CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); CondCodeNodes[Cond] = N; AllNodes.push_back(N); } + return SDValue(CondCodeNodes[Cond], 0); } @@ -1234,8 +1261,8 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, DebugLoc dl, SDValue N1, int *MaskAlloc = OperandAllocator.Allocate(NElts); memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); - ShuffleVectorSDNode *N = NodeAllocator.Allocate(); - new (N) ShuffleVectorSDNode(VT, dl, N1, N2, MaskAlloc); + ShuffleVectorSDNode *N = + new (NodeAllocator) ShuffleVectorSDNode(VT, dl, N1, N2, MaskAlloc); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -1252,12 +1279,14 @@ SDValue SelectionDAG::getConvertRndSat(EVT VT, DebugLoc dl, return Val; FoldingSetNodeID ID; + SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; + AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); void* IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - CvtRndSatSDNode *N = NodeAllocator.Allocate(); - SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; - new (N) CvtRndSatSDNode(VT, dl, Ops, 5, Code); + + CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl, Ops, 5, + Code); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -1270,42 +1299,50 @@ SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) RegisterSDNode(RegNo, VT); + + SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); } -SDValue SelectionDAG::getDbgStopPoint(DebugLoc DL, SDValue Root, - unsigned Line, unsigned Col, - MDNode *CU) { - SDNode *N = NodeAllocator.Allocate(); - new (N) DbgStopPointSDNode(Root, Line, Col, CU); - N->setDebugLoc(DL); +SDValue SelectionDAG::getEHLabel(DebugLoc dl, SDValue Root, MCSymbol *Label) { + FoldingSetNodeID ID; + SDValue Ops[] = { Root }; + AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); + ID.AddPointer(Label); + void *IP = 0; + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) + return SDValue(E, 0); + + SDNode *N = new (NodeAllocator) EHLabelSDNode(dl, Root, Label); + CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); } -SDValue SelectionDAG::getLabel(unsigned Opcode, DebugLoc dl, - SDValue Root, - unsigned LabelID) { + +SDValue SelectionDAG::getBlockAddress(BlockAddress *BA, EVT VT, + bool isTarget, + unsigned char TargetFlags) { + unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; + FoldingSetNodeID ID; - SDValue Ops[] = { Root }; - AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), &Ops[0], 1); - ID.AddInteger(LabelID); + AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); + ID.AddPointer(BA); + ID.AddInteger(TargetFlags); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) LabelSDNode(Opcode, dl, Root, LabelID); + + SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); } SDValue SelectionDAG::getSrcValue(const Value *V) { - assert((!V || isa(V->getType())) && + assert((!V || V->getType()->isPointerTy()) && "SrcValue is not a pointer?"); FoldingSetNodeID ID; @@ -1316,8 +1353,7 @@ SDValue SelectionDAG::getSrcValue(const Value *V) { if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) SrcValueSDNode(V); + SDNode *N = new (NodeAllocator) SrcValueSDNode(V); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -1343,7 +1379,7 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { unsigned StackAlign = std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), minAlign); - int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign); + int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); return getFrameIndex(FrameIdx, TLI.getPointerTy()); } @@ -1359,7 +1395,7 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { TD->getPrefTypeAlignment(Ty2)); MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); - int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align); + int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); return getFrameIndex(FrameIdx, TLI.getPointerTy()); } @@ -1472,7 +1508,7 @@ bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { if (Op.getValueType().isVector()) return false; - unsigned BitWidth = Op.getValueSizeInBits(); + unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); } @@ -1495,7 +1531,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, APInt &KnownZero, APInt &KnownOne, unsigned Depth) const { unsigned BitWidth = Mask.getBitWidth(); - assert(BitWidth == Op.getValueType().getSizeInBits() && + assert(BitWidth == Op.getValueType().getScalarType().getSizeInBits() && "Mask size mismatches value type size!"); KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. @@ -1700,7 +1736,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, return; case ISD::SIGN_EXTEND_INREG: { EVT EVT = cast(Op.getOperand(1))->getVT(); - unsigned EBits = EVT.getSizeInBits(); + unsigned EBits = EVT.getScalarType().getSizeInBits(); // Sign extension. Compute the demanded bits in the result that are not // present in the input. @@ -1745,14 +1781,14 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, if (ISD::isZEXTLoad(Op.getNode())) { LoadSDNode *LD = cast(Op); EVT VT = LD->getMemoryVT(); - unsigned MemBits = VT.getSizeInBits(); + unsigned MemBits = VT.getScalarType().getSizeInBits(); KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits) & Mask; } return; } case ISD::ZERO_EXTEND: { EVT InVT = Op.getOperand(0).getValueType(); - unsigned InBits = InVT.getSizeInBits(); + unsigned InBits = InVT.getScalarType().getSizeInBits(); APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; APInt InMask = Mask; InMask.trunc(InBits); @@ -1766,7 +1802,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, } case ISD::SIGN_EXTEND: { EVT InVT = Op.getOperand(0).getValueType(); - unsigned InBits = InVT.getSizeInBits(); + unsigned InBits = InVT.getScalarType().getSizeInBits(); APInt InSignBit = APInt::getSignBit(InBits); APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; APInt InMask = Mask; @@ -1807,7 +1843,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, } case ISD::ANY_EXTEND: { EVT InVT = Op.getOperand(0).getValueType(); - unsigned InBits = InVT.getSizeInBits(); + unsigned InBits = InVT.getScalarType().getSizeInBits(); APInt InMask = Mask; InMask.trunc(InBits); KnownZero.trunc(InBits); @@ -1819,7 +1855,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, } case ISD::TRUNCATE: { EVT InVT = Op.getOperand(0).getValueType(); - unsigned InBits = InVT.getSizeInBits(); + unsigned InBits = InVT.getScalarType().getSizeInBits(); APInt InMask = Mask; InMask.zext(InBits); KnownZero.zext(InBits); @@ -1886,19 +1922,28 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, } case ISD::SREM: if (ConstantSDNode *Rem = dyn_cast(Op.getOperand(1))) { - const APInt &RA = Rem->getAPIntValue(); - if (RA.isPowerOf2() || (-RA).isPowerOf2()) { - APInt LowBits = RA.isStrictlyPositive() ? (RA - 1) : ~RA; + const APInt &RA = Rem->getAPIntValue().abs(); + if (RA.isPowerOf2()) { + APInt LowBits = RA - 1; APInt Mask2 = LowBits | APInt::getSignBit(BitWidth); ComputeMaskedBits(Op.getOperand(0), Mask2,KnownZero2,KnownOne2,Depth+1); - // If the sign bit of the first operand is zero, the sign bit of - // the result is zero. If the first operand has no one bits below - // the second operand's single 1 bit, its sign will be zero. + // The low bits of the first operand are unchanged by the srem. + KnownZero = KnownZero2 & LowBits; + KnownOne = KnownOne2 & LowBits; + + // If the first operand is non-negative or has all low bits zero, then + // the upper bits are all zero. if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) - KnownZero2 |= ~LowBits; + KnownZero |= ~LowBits; + + // If the first operand is negative and not all low bits are zero, then + // the upper bits are all one. + if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) + KnownOne |= ~LowBits; - KnownZero |= KnownZero2 & Mask; + KnownZero &= Mask; + KnownOne &= Mask; assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); } @@ -1952,7 +1997,7 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ EVT VT = Op.getValueType(); assert(VT.isInteger() && "Invalid VT!"); - unsigned VTBits = VT.getSizeInBits(); + unsigned VTBits = VT.getScalarType().getSizeInBits(); unsigned Tmp, Tmp2; unsigned FirstAnswer = 1; @@ -1979,12 +2024,13 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ } case ISD::SIGN_EXTEND: - Tmp = VTBits-Op.getOperand(0).getValueType().getSizeInBits(); + Tmp = VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; case ISD::SIGN_EXTEND_INREG: // Max of the input and what this extends. - Tmp = cast(Op.getOperand(1))->getVT().getSizeInBits(); + Tmp = + cast(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); Tmp = VTBits-Tmp+1; Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); @@ -2128,10 +2174,10 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ switch (ExtType) { default: break; case ISD::SEXTLOAD: // '17' bits known - Tmp = LD->getMemoryVT().getSizeInBits(); + Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); return VTBits-Tmp+1; case ISD::ZEXTLOAD: // '16' bits known - Tmp = LD->getMemoryVT().getSizeInBits(); + Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); return VTBits-Tmp; } } @@ -2183,6 +2229,29 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { return false; } +bool SelectionDAG::isKnownNeverZero(SDValue Op) const { + // If the value is a constant, we can obviously see if it is a zero or not. + if (const ConstantFPSDNode *C = dyn_cast(Op)) + return !C->isZero(); + + // TODO: Recognize more cases here. + + return false; +} + +bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { + // Check the obvious case. + if (A == B) return true; + + // For for negative and positive zero. + if (const ConstantFPSDNode *CA = dyn_cast(A)) + if (const ConstantFPSDNode *CB = dyn_cast(B)) + if (CA->isZero() && CB->isZero()) return true; + + // Otherwise they may not be equal. + return false; +} + bool SelectionDAG::isVerifiedDebugInfoDesc(SDValue Op) const { GlobalAddressSDNode *GA = dyn_cast(Op); if (!GA) return false; @@ -2232,8 +2301,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT) { void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) SDNode(Opcode, DL, getVTList(VT)); + + SDNode *N = new (NodeAllocator) SDNode(Opcode, DL, getVTList(VT)); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); @@ -2248,22 +2317,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, // Constant fold unary operations with an integer constant operand. if (ConstantSDNode *C = dyn_cast(Operand.getNode())) { const APInt &Val = C->getAPIntValue(); - unsigned BitWidth = VT.getSizeInBits(); switch (Opcode) { default: break; case ISD::SIGN_EXTEND: - return getConstant(APInt(Val).sextOrTrunc(BitWidth), VT); + return getConstant(APInt(Val).sextOrTrunc(VT.getSizeInBits()), VT); case ISD::ANY_EXTEND: case ISD::ZERO_EXTEND: case ISD::TRUNCATE: - return getConstant(APInt(Val).zextOrTrunc(BitWidth), VT); + return getConstant(APInt(Val).zextOrTrunc(VT.getSizeInBits()), VT); case ISD::UINT_TO_FP: case ISD::SINT_TO_FP: { const uint64_t zero[] = {0, 0}; - // No compile time operations on this type. - if (VT==MVT::ppcf128) - break; - APFloat apf = APFloat(APInt(BitWidth, 2, zero)); + // No compile time operations on ppcf128. + if (VT == MVT::ppcf128) break; + APFloat apf = APFloat(APInt(VT.getSizeInBits(), 2, zero)); (void)apf.convertFromAPInt(Val, Opcode==ISD::SINT_TO_FP, APFloat::rmNearestTiesToEven); @@ -2341,6 +2408,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, assert(VT.isFloatingPoint() && Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); if (Operand.getValueType() == VT) return Operand; // noop conversion. + assert((!VT.isVector() || + VT.getVectorNumElements() == + Operand.getValueType().getVectorNumElements()) && + "Vector element count mismatch!"); if (Operand.getOpcode() == ISD::UNDEF) return getUNDEF(VT); break; @@ -2348,8 +2419,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, assert(VT.isInteger() && Operand.getValueType().isInteger() && "Invalid SIGN_EXTEND!"); if (Operand.getValueType() == VT) return Operand; // noop extension - assert(Operand.getValueType().bitsLT(VT) - && "Invalid sext node, dst < src!"); + assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && + "Invalid sext node, dst < src!"); + assert((!VT.isVector() || + VT.getVectorNumElements() == + Operand.getValueType().getVectorNumElements()) && + "Vector element count mismatch!"); if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); break; @@ -2357,8 +2432,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, assert(VT.isInteger() && Operand.getValueType().isInteger() && "Invalid ZERO_EXTEND!"); if (Operand.getValueType() == VT) return Operand; // noop extension - assert(Operand.getValueType().bitsLT(VT) - && "Invalid zext node, dst < src!"); + assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && + "Invalid zext node, dst < src!"); + assert((!VT.isVector() || + VT.getVectorNumElements() == + Operand.getValueType().getVectorNumElements()) && + "Vector element count mismatch!"); if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getNode()->getOperand(0)); @@ -2367,8 +2446,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, assert(VT.isInteger() && Operand.getValueType().isInteger() && "Invalid ANY_EXTEND!"); if (Operand.getValueType() == VT) return Operand; // noop extension - assert(Operand.getValueType().bitsLT(VT) - && "Invalid anyext node, dst < src!"); + assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && + "Invalid anyext node, dst < src!"); + assert((!VT.isVector() || + VT.getVectorNumElements() == + Operand.getValueType().getVectorNumElements()) && + "Vector element count mismatch!"); if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND) // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); @@ -2377,14 +2460,19 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, assert(VT.isInteger() && Operand.getValueType().isInteger() && "Invalid TRUNCATE!"); if (Operand.getValueType() == VT) return Operand; // noop truncate - assert(Operand.getValueType().bitsGT(VT) - && "Invalid truncate node, src < dst!"); + assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && + "Invalid truncate node, src < dst!"); + assert((!VT.isVector() || + VT.getVectorNumElements() == + Operand.getValueType().getVectorNumElements()) && + "Vector element count mismatch!"); if (OpOpcode == ISD::TRUNCATE) return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); else if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ANY_EXTEND) { // If the source is smaller than the dest, we still need an extend. - if (Operand.getNode()->getOperand(0).getValueType().bitsLT(VT)) + if (Operand.getNode()->getOperand(0).getValueType().getScalarType() + .bitsLT(VT.getScalarType())) return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); else if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); @@ -2441,12 +2529,11 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - N = NodeAllocator.Allocate(); - new (N) UnarySDNode(Opcode, DL, VTs, Operand); + + N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTs, Operand); CSEMap.InsertNode(N, IP); } else { - N = NodeAllocator.Allocate(); - new (N) UnarySDNode(Opcode, DL, VTs, Operand); + N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTs, Operand); } AllNodes.push_back(N); @@ -2592,12 +2679,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, // size of the value, the shift/rotate count is guaranteed to be zero. if (VT == MVT::i1) return N1; + if (N2C && N2C->isNullValue()) + return N1; break; case ISD::FP_ROUND_INREG: { EVT EVT = cast(N2)->getVT(); assert(VT == N1.getValueType() && "Not an inreg round!"); assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && "Cannot FP_ROUND_INREG integer types"); + assert(EVT.isVector() == VT.isVector() && + "FP_ROUND_INREG type should be vector iff the operand " + "type is vector!"); + assert((!EVT.isVector() || + EVT.getVectorNumElements() == VT.getVectorNumElements()) && + "Vector element counts must match in FP_ROUND_INREG"); assert(EVT.bitsLE(VT) && "Not rounding down!"); if (cast(N2)->getVT() == VT) return N1; // Not actually rounding. break; @@ -2615,6 +2710,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, assert(VT == N1.getValueType() && "Not an inreg extend!"); assert(VT.isInteger() && EVT.isInteger() && "Cannot *_EXTEND_INREG FP types"); + assert(!EVT.isVector() && + "AssertSExt/AssertZExt type should be the vector element type " + "rather than the vector type!"); assert(EVT.bitsLE(VT) && "Not extending!"); if (VT == EVT) return N1; // noop assertion. break; @@ -2624,12 +2722,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, assert(VT == N1.getValueType() && "Not an inreg extend!"); assert(VT.isInteger() && EVT.isInteger() && "Cannot *_EXTEND_INREG FP types"); + assert(EVT.isVector() == VT.isVector() && + "SIGN_EXTEND_INREG type should be vector iff the operand " + "type is vector!"); + assert((!EVT.isVector() || + EVT.getVectorNumElements() == VT.getVectorNumElements()) && + "Vector element counts must match in SIGN_EXTEND_INREG"); assert(EVT.bitsLE(VT) && "Not extending!"); if (EVT == VT) return N1; // Not actually extending if (N1C) { APInt Val = N1C->getAPIntValue(); - unsigned FromBits = cast(N2)->getVT().getSizeInBits(); + unsigned FromBits = EVT.getScalarType().getSizeInBits(); Val <<= Val.getBitWidth()-FromBits; Val = Val.ashr(Val.getBitWidth()-FromBits); return getConstant(Val, VT); @@ -2675,13 +2779,16 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector // operations are lowered to scalars. if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { - // If the indices are the same, return the inserted element. - if (N1.getOperand(2) == N2) - return N1.getOperand(1); - // If the indices are known different, extract the element from + // If the indices are the same, return the inserted element else + // if the indices are known different, extract the element from // the original vector. - else if (isa(N1.getOperand(2)) && - isa(N2)) + if (N1.getOperand(2) == N2) { + if (VT == N1.getOperand(1).getValueType()) + return N1.getOperand(1); + else + return getSExtOrTrunc(N1.getOperand(1), DL, VT); + } else if (isa(N1.getOperand(2)) && + isa(N2)) return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); } break; @@ -2853,12 +2960,11 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - N = NodeAllocator.Allocate(); - new (N) BinarySDNode(Opcode, DL, VTs, N1, N2); + + N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTs, N1, N2); CSEMap.InsertNode(N, IP); } else { - N = NodeAllocator.Allocate(); - new (N) BinarySDNode(Opcode, DL, VTs, N1, N2); + N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTs, N1, N2); } AllNodes.push_back(N); @@ -2930,13 +3036,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - N = NodeAllocator.Allocate(); - new (N) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); + + N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); CSEMap.InsertNode(N, IP); } else { - N = NodeAllocator.Allocate(); - new (N) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); + N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTs, N1, N2, N3); } + AllNodes.push_back(N); #ifndef NDEBUG VerifyNode(N); @@ -2985,8 +3091,9 @@ SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { /// operand. static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, DebugLoc dl) { - unsigned NumBits = VT.isVector() ? - VT.getVectorElementType().getSizeInBits() : VT.getSizeInBits(); + assert(Value.getOpcode() != ISD::UNDEF); + + unsigned NumBits = VT.getScalarType().getSizeInBits(); if (ConstantSDNode *C = dyn_cast(Value)) { APInt Val = APInt(NumBits, C->getZExtValue() & 255); unsigned Shift = 8; @@ -3024,11 +3131,17 @@ static SDValue getMemsetStringVal(EVT VT, DebugLoc dl, SelectionDAG &DAG, if (Str.empty()) { if (VT.isInteger()) return DAG.getConstant(0, VT); - unsigned NumElts = VT.getVectorNumElements(); - MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; - return DAG.getNode(ISD::BIT_CONVERT, dl, VT, - DAG.getConstant(0, - EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts))); + else if (VT.getSimpleVT().SimpleTy == MVT::f32 || + VT.getSimpleVT().SimpleTy == MVT::f64) + return DAG.getConstantFP(0.0, VT); + else if (VT.isVector()) { + unsigned NumElts = VT.getVectorNumElements(); + MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; + return DAG.getNode(ISD::BIT_CONVERT, dl, VT, + DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), + EltVT, NumElts))); + } else + llvm_unreachable("Expected type!"); } assert(!VT.isVector() && "Can't handle vector type here!"); @@ -3076,51 +3189,34 @@ static bool isMemSrcFromString(SDValue Src, std::string &Str) { return false; } -/// MeetsMaxMemopRequirement - Determines if the number of memory ops required -/// to replace the memset / memcpy is below the threshold. It also returns the -/// types of the sequence of memory ops to perform memset / memcpy. -static -bool MeetsMaxMemopRequirement(std::vector &MemOps, - SDValue Dst, SDValue Src, - unsigned Limit, uint64_t Size, unsigned &Align, - std::string &Str, bool &isSrcStr, - SelectionDAG &DAG, - const TargetLowering &TLI) { - isSrcStr = isMemSrcFromString(Src, Str); - bool isSrcConst = isa(Src); - EVT VT = TLI.getOptimalMemOpType(Size, Align, isSrcConst, isSrcStr, DAG); - bool AllowUnalign = TLI.allowsUnalignedMemoryAccesses(VT); - if (VT != MVT::iAny) { +/// FindOptimalMemOpLowering - Determines the optimial series memory ops +/// to replace the memset / memcpy. Return true if the number of memory ops +/// is below the threshold. It returns the types of the sequence of +/// memory ops to perform memset / memcpy by reference. +static bool FindOptimalMemOpLowering(std::vector &MemOps, + unsigned Limit, uint64_t Size, + unsigned DstAlign, unsigned SrcAlign, + bool NonScalarIntSafe, + SelectionDAG &DAG, + const TargetLowering &TLI) { + assert((SrcAlign == 0 || SrcAlign >= DstAlign) && + "Expecting memcpy / memset source to meet alignment requirement!"); + // If 'SrcAlign' is zero, that means the memory operation does not need load + // the value, i.e. memset or memcpy from constant string. Otherwise, it's + // the inferred alignment of the source. 'DstAlign', on the other hand, is the + // specified alignment of the memory operation. If it is zero, that means + // it's possible to change the alignment of the destination. + EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, + NonScalarIntSafe, DAG); + + if (VT == MVT::Other) { + VT = TLI.getPointerTy(); const Type *Ty = VT.getTypeForEVT(*DAG.getContext()); - unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); - // If source is a string constant, this will require an unaligned load. - if (NewAlign > Align && (isSrcConst || AllowUnalign)) { - if (Dst.getOpcode() != ISD::FrameIndex) { - // Can't change destination alignment. It requires a unaligned store. - if (AllowUnalign) - VT = MVT::iAny; - } else { - int FI = cast(Dst)->getIndex(); - MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); - if (MFI->isFixedObjectIndex(FI)) { - // Can't change destination alignment. It requires a unaligned store. - if (AllowUnalign) - VT = MVT::iAny; - } else { - // Give the stack frame object a larger alignment if needed. - if (MFI->getObjectAlignment(FI) < NewAlign) - MFI->setObjectAlignment(FI, NewAlign); - Align = NewAlign; - } - } - } - } - - if (VT == MVT::iAny) { - if (TLI.allowsUnalignedMemoryAccesses(MVT::i64)) { + if (DstAlign >= TLI.getTargetData()->getABITypeAlignment(Ty) || + TLI.allowsUnalignedMemoryAccesses(VT)) { VT = MVT::i64; } else { - switch (Align & 7) { + switch (DstAlign & 7) { case 0: VT = MVT::i64; break; case 4: VT = MVT::i32; break; case 2: VT = MVT::i16; break; @@ -3142,7 +3238,7 @@ bool MeetsMaxMemopRequirement(std::vector &MemOps, unsigned VTSize = VT.getSizeInBits() / 8; while (VTSize > Size) { // For now, only use non-vector load / store's for the left-over pieces. - if (VT.isVector()) { + if (VT.isVector() || VT.isFloatingPoint()) { VT = MVT::i64; while (!TLI.isTypeLegal(VT)) VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); @@ -3165,28 +3261,49 @@ bool MeetsMaxMemopRequirement(std::vector &MemOps, } static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, - SDValue Chain, SDValue Dst, - SDValue Src, uint64_t Size, - unsigned Align, bool AlwaysInline, - const Value *DstSV, uint64_t DstSVOff, - const Value *SrcSV, uint64_t SrcSVOff){ - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDValue Chain, SDValue Dst, + SDValue Src, uint64_t Size, + unsigned Align, bool AlwaysInline, + const Value *DstSV, uint64_t DstSVOff, + const Value *SrcSV, uint64_t SrcSVOff) { + // Turn a memcpy of undef to nop. + if (Src.getOpcode() == ISD::UNDEF) + return Chain; // Expand memcpy to a series of load and store ops if the size operand falls // below a certain threshold. + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); std::vector MemOps; uint64_t Limit = -1ULL; if (!AlwaysInline) Limit = TLI.getMaxStoresPerMemcpy(); - unsigned DstAlign = Align; // Destination alignment can change. + bool DstAlignCanChange = false; + MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); + FrameIndexSDNode *FI = dyn_cast(Dst); + if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) + DstAlignCanChange = true; + unsigned SrcAlign = DAG.InferPtrAlignment(Src); + if (Align > SrcAlign) + SrcAlign = Align; std::string Str; - bool CopyFromStr; - if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, Limit, Size, DstAlign, - Str, CopyFromStr, DAG, TLI)) + bool CopyFromStr = isMemSrcFromString(Src, Str); + bool isZeroStr = CopyFromStr && Str.empty(); + if (!FindOptimalMemOpLowering(MemOps, Limit, Size, + (DstAlignCanChange ? 0 : Align), + (isZeroStr ? 0 : SrcAlign), true, DAG, TLI)) return SDValue(); + if (DstAlignCanChange) { + const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); + unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); + if (NewAlign > Align) { + // Give the stack frame object a larger alignment if needed. + if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) + MFI->setObjectAlignment(FI->getIndex(), NewAlign); + Align = NewAlign; + } + } - bool isZeroStr = CopyFromStr && Str.empty(); SmallVector OutChains; unsigned NumMemOps = MemOps.size(); uint64_t SrcOff = 0, DstOff = 0; @@ -3195,16 +3312,17 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, unsigned VTSize = VT.getSizeInBits() / 8; SDValue Value, Store; - if (CopyFromStr && (isZeroStr || !VT.isVector())) { + if (CopyFromStr && + (isZeroStr || (VT.isInteger() && !VT.isVector()))) { // It's unlikely a store of a vector immediate can be done in a single // instruction. It would require a load from a constantpool first. - // We also handle store a vector with all zero's. + // We only handle zero vectors here. // FIXME: Handle other cases where store of vector immediate is done in // a single instruction. Value = getMemsetStringVal(VT, dl, DAG, TLI, Str, SrcOff); Store = DAG.getStore(Chain, dl, Value, getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstSVOff + DstOff, false, DstAlign); + DstSV, DstSVOff + DstOff, false, false, Align); } else { // The type might not be legal for the target. This should only happen // if the type is smaller than a legal type, as on PPC, so the right @@ -3215,10 +3333,12 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, assert(NVT.bitsGE(VT)); Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, getMemBasePlusOffset(Src, SrcOff, DAG), - SrcSV, SrcSVOff + SrcOff, VT, false, Align); + SrcSV, SrcSVOff + SrcOff, VT, false, false, + MinAlign(SrcAlign, SrcOff)); Store = DAG.getTruncStore(Chain, dl, Value, - getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstSVOff + DstOff, VT, false, DstAlign); + getMemBasePlusOffset(Dst, DstOff, DAG), + DstSV, DstSVOff + DstOff, VT, false, false, + Align); } OutChains.push_back(Store); SrcOff += VTSize; @@ -3230,28 +3350,48 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, } static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, - SDValue Chain, SDValue Dst, - SDValue Src, uint64_t Size, - unsigned Align, bool AlwaysInline, - const Value *DstSV, uint64_t DstSVOff, - const Value *SrcSV, uint64_t SrcSVOff){ - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDValue Chain, SDValue Dst, + SDValue Src, uint64_t Size, + unsigned Align,bool AlwaysInline, + const Value *DstSV, uint64_t DstSVOff, + const Value *SrcSV, uint64_t SrcSVOff) { + // Turn a memmove of undef to nop. + if (Src.getOpcode() == ISD::UNDEF) + return Chain; // Expand memmove to a series of load and store ops if the size operand falls // below a certain threshold. + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); std::vector MemOps; uint64_t Limit = -1ULL; if (!AlwaysInline) Limit = TLI.getMaxStoresPerMemmove(); - unsigned DstAlign = Align; // Destination alignment can change. - std::string Str; - bool CopyFromStr; - if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, Limit, Size, DstAlign, - Str, CopyFromStr, DAG, TLI)) + bool DstAlignCanChange = false; + MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); + FrameIndexSDNode *FI = dyn_cast(Dst); + if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) + DstAlignCanChange = true; + unsigned SrcAlign = DAG.InferPtrAlignment(Src); + if (Align > SrcAlign) + SrcAlign = Align; + + if (!FindOptimalMemOpLowering(MemOps, Limit, Size, + (DstAlignCanChange ? 0 : Align), + SrcAlign, true, DAG, TLI)) return SDValue(); - uint64_t SrcOff = 0, DstOff = 0; + if (DstAlignCanChange) { + const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); + unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); + if (NewAlign > Align) { + // Give the stack frame object a larger alignment if needed. + if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) + MFI->setObjectAlignment(FI->getIndex(), NewAlign); + Align = NewAlign; + } + } + uint64_t SrcOff = 0, DstOff = 0; SmallVector LoadValues; SmallVector LoadChains; SmallVector OutChains; @@ -3263,7 +3403,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, Value = DAG.getLoad(VT, dl, Chain, getMemBasePlusOffset(Src, SrcOff, DAG), - SrcSV, SrcSVOff + SrcOff, false, Align); + SrcSV, SrcSVOff + SrcOff, false, false, SrcAlign); LoadValues.push_back(Value); LoadChains.push_back(Value.getValue(1)); SrcOff += VTSize; @@ -3278,7 +3418,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, Store = DAG.getStore(Chain, dl, LoadValues[i], getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstSVOff + DstOff, false, DstAlign); + DstSV, DstSVOff + DstOff, false, false, Align); OutChains.push_back(Store); DstOff += VTSize; } @@ -3288,24 +3428,43 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, } static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, - SDValue Chain, SDValue Dst, - SDValue Src, uint64_t Size, - unsigned Align, - const Value *DstSV, uint64_t DstSVOff) { - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDValue Chain, SDValue Dst, + SDValue Src, uint64_t Size, + unsigned Align, + const Value *DstSV, uint64_t DstSVOff) { + // Turn a memset of undef to nop. + if (Src.getOpcode() == ISD::UNDEF) + return Chain; // Expand memset to a series of load/store ops if the size operand // falls below a certain threshold. + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); std::vector MemOps; - std::string Str; - bool CopyFromStr; - if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, TLI.getMaxStoresPerMemset(), - Size, Align, Str, CopyFromStr, DAG, TLI)) + bool DstAlignCanChange = false; + MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); + FrameIndexSDNode *FI = dyn_cast(Dst); + if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) + DstAlignCanChange = true; + bool NonScalarIntSafe = + isa(Src) && cast(Src)->isNullValue(); + if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(), + Size, (DstAlignCanChange ? 0 : Align), 0, + NonScalarIntSafe, DAG, TLI)) return SDValue(); + if (DstAlignCanChange) { + const Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); + unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty); + if (NewAlign > Align) { + // Give the stack frame object a larger alignment if needed. + if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) + MFI->setObjectAlignment(FI->getIndex(), NewAlign); + Align = NewAlign; + } + } + SmallVector OutChains; uint64_t DstOff = 0; - unsigned NumMemOps = MemOps.size(); for (unsigned i = 0; i < NumMemOps; i++) { EVT VT = MemOps[i]; @@ -3313,7 +3472,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, SDValue Value = getMemsetValue(Src, VT, DAG, dl); SDValue Store = DAG.getStore(Chain, dl, Value, getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstSVOff + DstOff); + DstSV, DstSVOff + DstOff, false, false, 0); OutChains.push_back(Store); DstOff += VTSize; } @@ -3336,10 +3495,9 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, if (ConstantSize->isNullValue()) return Chain; - SDValue Result = - getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, - ConstantSize->getZExtValue(), - Align, false, DstSV, DstSVOff, SrcSV, SrcSVOff); + SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, + ConstantSize->getZExtValue(),Align, + false, DstSV, DstSVOff, SrcSV, SrcSVOff); if (Result.getNode()) return Result; } @@ -3443,9 +3601,9 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, if (ConstantSize->isNullValue()) return Chain; - SDValue Result = - getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), - Align, DstSV, DstSVOff); + SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src, + ConstantSize->getZExtValue(), + Align, DstSV, DstSVOff); if (Result.getNode()) return Result; } @@ -3535,8 +3693,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, cast(E)->refineAlignment(MMO); return SDValue(E, 0); } - SDNode* N = NodeAllocator.Allocate(); - new (N) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Cmp, Swp, MMO); + SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, + Ptr, Cmp, Swp, MMO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3598,8 +3756,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, cast(E)->refineAlignment(MMO); return SDValue(E, 0); } - SDNode* N = NodeAllocator.Allocate(); - new (N) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, Ptr, Val, MMO); + SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain, + Ptr, Val, MMO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3677,12 +3835,12 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, return SDValue(E, 0); } - N = NodeAllocator.Allocate(); - new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); + N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, + MemVT, MMO); CSEMap.InsertNode(N, IP); } else { - N = NodeAllocator.Allocate(); - new (N) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); + N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl, VTList, Ops, NumOps, + MemVT, MMO); } AllNodes.push_back(N); return SDValue(N, 0); @@ -3693,7 +3851,8 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, ISD::LoadExtType ExtType, EVT VT, SDValue Chain, SDValue Ptr, SDValue Offset, const Value *SV, int SVOffset, EVT MemVT, - bool isVolatile, unsigned Alignment) { + bool isVolatile, bool isNonTemporal, + unsigned Alignment) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(VT); @@ -3707,6 +3866,8 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, unsigned Flags = MachineMemOperand::MOLoad; if (isVolatile) Flags |= MachineMemOperand::MOVolatile; + if (isNonTemporal) + Flags |= MachineMemOperand::MONonTemporal; MachineMemOperand *MMO = MF.getMachineMemOperand(SV, Flags, SVOffset, MemVT.getStoreSize(), Alignment); @@ -3724,16 +3885,15 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, assert(VT == MemVT && "Non-extending load from different memory type!"); } else { // Extending load. - if (VT.isVector()) - assert(MemVT.getVectorNumElements() == VT.getVectorNumElements() && - "Invalid vector extload!"); - else - assert(MemVT.bitsLT(VT) && - "Should only be an extending load, not truncating!"); - assert((ExtType == ISD::EXTLOAD || VT.isInteger()) && - "Cannot sign/zero extend a FP/Vector load!"); + assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && + "Should only be an extending load, not truncating!"); assert(VT.isInteger() == MemVT.isInteger() && "Cannot convert from FP to Int or Int -> FP!"); + assert(VT.isVector() == MemVT.isVector() && + "Cannot use trunc store to convert to or from a vector!"); + assert((!VT.isVector() || + VT.getVectorNumElements() == MemVT.getVectorNumElements()) && + "Cannot use trunc store to change the number of vector elements!"); } bool Indexed = AM != ISD::UNINDEXED; @@ -3746,14 +3906,15 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); ID.AddInteger(MemVT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile())); + ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), + MMO->isNonTemporal())); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { cast(E)->refineAlignment(MMO); return SDValue(E, 0); } - SDNode *N = NodeAllocator.Allocate(); - new (N) LoadSDNode(Ops, dl, VTs, AM, ExtType, MemVT, MMO); + SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl, VTs, AM, ExtType, + MemVT, MMO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3762,20 +3923,22 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, - bool isVolatile, unsigned Alignment) { + bool isVolatile, bool isNonTemporal, + unsigned Alignment) { SDValue Undef = getUNDEF(Ptr.getValueType()); return getLoad(ISD::UNINDEXED, dl, ISD::NON_EXTLOAD, VT, Chain, Ptr, Undef, - SV, SVOffset, VT, isVolatile, Alignment); + SV, SVOffset, VT, isVolatile, isNonTemporal, Alignment); } SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, SDValue Chain, SDValue Ptr, const Value *SV, int SVOffset, EVT MemVT, - bool isVolatile, unsigned Alignment) { + bool isVolatile, bool isNonTemporal, + unsigned Alignment) { SDValue Undef = getUNDEF(Ptr.getValueType()); return getLoad(ISD::UNINDEXED, dl, ExtType, VT, Chain, Ptr, Undef, - SV, SVOffset, MemVT, isVolatile, Alignment); + SV, SVOffset, MemVT, isVolatile, isNonTemporal, Alignment); } SDValue @@ -3787,12 +3950,13 @@ SelectionDAG::getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base, return getLoad(AM, dl, LD->getExtensionType(), OrigLoad.getValueType(), LD->getChain(), Base, Offset, LD->getSrcValue(), LD->getSrcValueOffset(), LD->getMemoryVT(), - LD->isVolatile(), LD->getAlignment()); + LD->isVolatile(), LD->isNonTemporal(), LD->getAlignment()); } SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, - bool isVolatile, unsigned Alignment) { + bool isVolatile, bool isNonTemporal, + unsigned Alignment) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(Val.getValueType()); @@ -3806,6 +3970,8 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, unsigned Flags = MachineMemOperand::MOStore; if (isVolatile) Flags |= MachineMemOperand::MOVolatile; + if (isNonTemporal) + Flags |= MachineMemOperand::MONonTemporal; MachineMemOperand *MMO = MF.getMachineMemOperand(SV, Flags, SVOffset, Val.getValueType().getStoreSize(), Alignment); @@ -3822,14 +3988,15 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); ID.AddInteger(VT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile())); + ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), + MMO->isNonTemporal())); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { cast(E)->refineAlignment(MMO); return SDValue(E, 0); } - SDNode *N = NodeAllocator.Allocate(); - new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, false, VT, MMO); + SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, + false, VT, MMO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3838,7 +4005,8 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, SDValue Ptr, const Value *SV, int SVOffset, EVT SVT, - bool isVolatile, unsigned Alignment) { + bool isVolatile, bool isNonTemporal, + unsigned Alignment) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(SVT); @@ -3852,6 +4020,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, unsigned Flags = MachineMemOperand::MOStore; if (isVolatile) Flags |= MachineMemOperand::MOVolatile; + if (isNonTemporal) + Flags |= MachineMemOperand::MONonTemporal; MachineMemOperand *MMO = MF.getMachineMemOperand(SV, Flags, SVOffset, SVT.getStoreSize(), Alignment); @@ -3866,10 +4036,15 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, if (VT == SVT) return getStore(Chain, dl, Val, Ptr, MMO); - assert(VT.bitsGT(SVT) && "Not a truncation?"); + assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && + "Should only be a truncating store, not extending!"); assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!"); - + assert(VT.isVector() == SVT.isVector() && + "Cannot use trunc store to convert to or from a vector!"); + assert((!VT.isVector() || + VT.getVectorNumElements() == SVT.getVectorNumElements()) && + "Cannot use trunc store to change the number of vector elements!"); SDVTList VTs = getVTList(MVT::Other); SDValue Undef = getUNDEF(Ptr.getValueType()); @@ -3877,14 +4052,15 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); ID.AddInteger(SVT.getRawBits()); - ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile())); + ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), + MMO->isNonTemporal())); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { cast(E)->refineAlignment(MMO); return SDValue(E, 0); } - SDNode *N = NodeAllocator.Allocate(); - new (N) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, true, SVT, MMO); + SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, ISD::UNINDEXED, + true, SVT, MMO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3905,10 +4081,11 @@ SelectionDAG::getIndexedStore(SDValue OrigStore, DebugLoc dl, SDValue Base, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = NodeAllocator.Allocate(); - new (N) StoreSDNode(Ops, dl, VTs, AM, - ST->isTruncatingStore(), ST->getMemoryVT(), - ST->getMemOperand()); + + SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl, VTs, AM, + ST->isTruncatingStore(), + ST->getMemoryVT(), + ST->getMemOperand()); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDValue(N, 0); @@ -3979,12 +4156,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - N = NodeAllocator.Allocate(); - new (N) SDNode(Opcode, DL, VTs, Ops, NumOps); + N = new (NodeAllocator) SDNode(Opcode, DL, VTs, Ops, NumOps); CSEMap.InsertNode(N, IP); } else { - N = NodeAllocator.Allocate(); - new (N) SDNode(Opcode, DL, VTs, Ops, NumOps); + N = new (NodeAllocator) SDNode(Opcode, DL, VTs, Ops, NumOps); } AllNodes.push_back(N); @@ -4029,7 +4204,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, if (ConstantSDNode *AndRHS = dyn_cast(N3.getOperand(1))) { // If the and is only masking out bits that cannot effect the shift, // eliminate the and. - unsigned NumBits = VT.getSizeInBits()*2; + unsigned NumBits = VT.getScalarType().getSizeInBits()*2; if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); } @@ -4045,33 +4220,28 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, SDVTList VTList, void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); + if (NumOps == 1) { - N = NodeAllocator.Allocate(); - new (N) UnarySDNode(Opcode, DL, VTList, Ops[0]); + N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTList, Ops[0]); } else if (NumOps == 2) { - N = NodeAllocator.Allocate(); - new (N) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); + N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); } else if (NumOps == 3) { - N = NodeAllocator.Allocate(); - new (N) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], Ops[2]); + N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], + Ops[2]); } else { - N = NodeAllocator.Allocate(); - new (N) SDNode(Opcode, DL, VTList, Ops, NumOps); + N = new (NodeAllocator) SDNode(Opcode, DL, VTList, Ops, NumOps); } CSEMap.InsertNode(N, IP); } else { if (NumOps == 1) { - N = NodeAllocator.Allocate(); - new (N) UnarySDNode(Opcode, DL, VTList, Ops[0]); + N = new (NodeAllocator) UnarySDNode(Opcode, DL, VTList, Ops[0]); } else if (NumOps == 2) { - N = NodeAllocator.Allocate(); - new (N) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); + N = new (NodeAllocator) BinarySDNode(Opcode, DL, VTList, Ops[0], Ops[1]); } else if (NumOps == 3) { - N = NodeAllocator.Allocate(); - new (N) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], Ops[2]); + N = new (NodeAllocator) TernarySDNode(Opcode, DL, VTList, Ops[0], Ops[1], + Ops[2]); } else { - N = NodeAllocator.Allocate(); - new (N) SDNode(Opcode, DL, VTList, Ops, NumOps); + N = new (NodeAllocator) SDNode(Opcode, DL, VTList, Ops, NumOps); } } AllNodes.push_back(N); @@ -4158,7 +4328,7 @@ SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { I->VTs[2] == VT3 && I->VTs[3] == VT4) return *I; - EVT *Array = Allocator.Allocate(3); + EVT *Array = Allocator.Allocate(4); Array[0] = VT1; Array[1] = VT2; Array[2] = VT3; @@ -4174,6 +4344,7 @@ SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { case 1: return getVTList(VTs[0]); case 2: return getVTList(VTs[0], VTs[1]); case 3: return getVTList(VTs[0], VTs[1], VTs[2]); + case 4: return getVTList(VTs[0], VTs[1], VTs[2], VTs[3]); default: break; } @@ -4433,91 +4604,13 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs, const SDValue *Ops, unsigned NumOps) { - return MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT) { - SDVTList VTs = getVTList(VT); - return MorphNodeTo(N, Opc, VTs, 0, 0); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT, SDValue Op1) { - SDVTList VTs = getVTList(VT); - SDValue Ops[] = { Op1 }; - return MorphNodeTo(N, Opc, VTs, Ops, 1); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT, SDValue Op1, - SDValue Op2) { - SDVTList VTs = getVTList(VT); - SDValue Ops[] = { Op1, Op2 }; - return MorphNodeTo(N, Opc, VTs, Ops, 2); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT, SDValue Op1, - SDValue Op2, SDValue Op3) { - SDVTList VTs = getVTList(VT); - SDValue Ops[] = { Op1, Op2, Op3 }; - return MorphNodeTo(N, Opc, VTs, Ops, 3); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT, const SDValue *Ops, - unsigned NumOps) { - SDVTList VTs = getVTList(VT); - return MorphNodeTo(N, Opc, VTs, Ops, NumOps); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT1, EVT VT2, const SDValue *Ops, - unsigned NumOps) { - SDVTList VTs = getVTList(VT1, VT2); - return MorphNodeTo(N, Opc, VTs, Ops, NumOps); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT1, EVT VT2) { - SDVTList VTs = getVTList(VT1, VT2); - return MorphNodeTo(N, Opc, VTs, (SDValue *)0, 0); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT1, EVT VT2, EVT VT3, - const SDValue *Ops, unsigned NumOps) { - SDVTList VTs = getVTList(VT1, VT2, VT3); - return MorphNodeTo(N, Opc, VTs, Ops, NumOps); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT1, EVT VT2, - SDValue Op1) { - SDVTList VTs = getVTList(VT1, VT2); - SDValue Ops[] = { Op1 }; - return MorphNodeTo(N, Opc, VTs, Ops, 1); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT1, EVT VT2, - SDValue Op1, SDValue Op2) { - SDVTList VTs = getVTList(VT1, VT2); - SDValue Ops[] = { Op1, Op2 }; - return MorphNodeTo(N, Opc, VTs, Ops, 2); -} - -SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, - EVT VT1, EVT VT2, - SDValue Op1, SDValue Op2, - SDValue Op3) { - SDVTList VTs = getVTList(VT1, VT2); - SDValue Ops[] = { Op1, Op2, Op3 }; - return MorphNodeTo(N, Opc, VTs, Ops, 3); + N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); + // Reset the NodeID to -1. + N->setNodeId(-1); + return N; } -/// MorphNodeTo - These *mutate* the specified node to have the specified +/// MorphNodeTo - This *mutates* the specified node to have the specified /// return type, opcode, and operands. /// /// Note that MorphNodeTo returns the resultant node. If there is already a @@ -4573,7 +4666,7 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, // remainder of the current SelectionDAG iteration, so we can allocate // the operands directly out of a pool with no recycling metadata. MN->InitOperands(OperandAllocator.Allocate(NumOps), - Ops, NumOps); + Ops, NumOps); else MN->InitOperands(MN->LocalOperands, Ops, NumOps); MN->OperandsNeedDelete = false; @@ -4588,17 +4681,19 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, N->InitOperands(new SDUse[NumOps], Ops, NumOps); N->OperandsNeedDelete = true; } else - MN->InitOperands(MN->OperandList, Ops, NumOps); + N->InitOperands(N->OperandList, Ops, NumOps); } // Delete any nodes that are still dead after adding the uses for the // new operands. - SmallVector DeadNodes; - for (SmallPtrSet::iterator I = DeadNodeSet.begin(), - E = DeadNodeSet.end(); I != E; ++I) - if ((*I)->use_empty()) - DeadNodes.push_back(*I); - RemoveDeadNodes(DeadNodes); + if (!DeadNodeSet.empty()) { + SmallVector DeadNodes; + for (SmallPtrSet::iterator I = DeadNodeSet.begin(), + E = DeadNodeSet.end(); I != E; ++I) + if ((*I)->use_empty()) + DeadNodes.push_back(*I); + RemoveDeadNodes(DeadNodes); + } if (IP) CSEMap.InsertNode(N, IP); // Memoize the new node. @@ -4612,115 +4707,126 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, /// Note that getMachineNode returns the resultant node. If there is already a /// node of the specified opcode and operands, it returns that node instead of /// the current one. -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT) { SDVTList VTs = getVTList(VT); return getMachineNode(Opcode, dl, VTs, 0, 0); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, - SDValue Op1) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, SDValue Op1) { SDVTList VTs = getVTList(VT); SDValue Ops[] = { Op1 }; return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, - SDValue Op1, SDValue Op2) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, + SDValue Op1, SDValue Op2) { SDVTList VTs = getVTList(VT); SDValue Ops[] = { Op1, Op2 }; return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, - SDValue Op1, SDValue Op2, - SDValue Op3) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, + SDValue Op1, SDValue Op2, SDValue Op3) { SDVTList VTs = getVTList(VT); SDValue Ops[] = { Op1, Op2, Op3 }; return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, - const SDValue *Ops, unsigned NumOps) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT, + const SDValue *Ops, unsigned NumOps) { SDVTList VTs = getVTList(VT); return getMachineNode(Opcode, dl, VTs, Ops, NumOps); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, - EVT VT1, EVT VT2) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, EVT VT2) { SDVTList VTs = getVTList(VT1, VT2); return getMachineNode(Opcode, dl, VTs, 0, 0); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, - EVT VT2, SDValue Op1) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, + EVT VT1, EVT VT2, SDValue Op1) { SDVTList VTs = getVTList(VT1, VT2); SDValue Ops[] = { Op1 }; return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, - EVT VT2, SDValue Op1, - SDValue Op2) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, + EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { SDVTList VTs = getVTList(VT1, VT2); SDValue Ops[] = { Op1, Op2 }; return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, - EVT VT2, SDValue Op1, - SDValue Op2, SDValue Op3) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, + EVT VT1, EVT VT2, SDValue Op1, + SDValue Op2, SDValue Op3) { SDVTList VTs = getVTList(VT1, VT2); SDValue Ops[] = { Op1, Op2, Op3 }; return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, - EVT VT1, EVT VT2, - const SDValue *Ops, unsigned NumOps) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, + EVT VT1, EVT VT2, + const SDValue *Ops, unsigned NumOps) { SDVTList VTs = getVTList(VT1, VT2); return getMachineNode(Opcode, dl, VTs, Ops, NumOps); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, - EVT VT1, EVT VT2, EVT VT3, - SDValue Op1, SDValue Op2) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, + EVT VT1, EVT VT2, EVT VT3, + SDValue Op1, SDValue Op2) { SDVTList VTs = getVTList(VT1, VT2, VT3); SDValue Ops[] = { Op1, Op2 }; return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, - EVT VT1, EVT VT2, EVT VT3, - SDValue Op1, SDValue Op2, - SDValue Op3) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, + EVT VT1, EVT VT2, EVT VT3, + SDValue Op1, SDValue Op2, SDValue Op3) { SDVTList VTs = getVTList(VT1, VT2, VT3); SDValue Ops[] = { Op1, Op2, Op3 }; return getMachineNode(Opcode, dl, VTs, Ops, array_lengthof(Ops)); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, - EVT VT1, EVT VT2, EVT VT3, - const SDValue *Ops, unsigned NumOps) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, + EVT VT1, EVT VT2, EVT VT3, + const SDValue *Ops, unsigned NumOps) { SDVTList VTs = getVTList(VT1, VT2, VT3); return getMachineNode(Opcode, dl, VTs, Ops, NumOps); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, - EVT VT2, EVT VT3, EVT VT4, - const SDValue *Ops, unsigned NumOps) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, EVT VT1, + EVT VT2, EVT VT3, EVT VT4, + const SDValue *Ops, unsigned NumOps) { SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); return getMachineNode(Opcode, dl, VTs, Ops, NumOps); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, - const std::vector &ResultTys, - const SDValue *Ops, unsigned NumOps) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc dl, + const std::vector &ResultTys, + const SDValue *Ops, unsigned NumOps) { SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); return getMachineNode(Opcode, dl, VTs, Ops, NumOps); } -SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, - const SDValue *Ops, unsigned NumOps) { +MachineSDNode * +SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, + const SDValue *Ops, unsigned NumOps) { bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Flag; MachineSDNode *N; void *IP; @@ -4730,12 +4836,11 @@ SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) - return E; + return cast(E); } // Allocate a new MachineSDNode. - N = NodeAllocator.Allocate(); - new (N) MachineSDNode(~Opcode, DL, VTs); + N = new (NodeAllocator) MachineSDNode(~Opcode, DL, VTs); // Initialize the operands list. if (NumOps > array_lengthof(N->LocalOperands)) @@ -4759,16 +4864,27 @@ SDNode *SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, } /// getTargetExtractSubreg - A convenience function for creating -/// TargetInstrInfo::EXTRACT_SUBREG nodes. +/// TargetOpcode::EXTRACT_SUBREG nodes. SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, DebugLoc DL, EVT VT, SDValue Operand) { SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); - SDNode *Subreg = getMachineNode(TargetInstrInfo::EXTRACT_SUBREG, DL, + SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, VT, Operand, SRIdxVal); return SDValue(Subreg, 0); } +/// getTargetInsertSubreg - A convenience function for creating +/// TargetOpcode::INSERT_SUBREG nodes. +SDValue +SelectionDAG::getTargetInsertSubreg(int SRIdx, DebugLoc DL, EVT VT, + SDValue Operand, SDValue Subreg) { + SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); + SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, + VT, Operand, Subreg, SRIdxVal); + return SDValue(Result, 0); +} + /// getNodeIfExists - Get the specified node if it's already available, or /// else return NULL. SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, @@ -4783,6 +4899,63 @@ SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, return NULL; } +/// getDbgValue - Creates a SDDbgValue node. +/// +SDDbgValue * +SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, + DebugLoc DL, unsigned O) { + return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); +} + +SDDbgValue * +SelectionDAG::getDbgValue(MDNode *MDPtr, Value *C, uint64_t Off, + DebugLoc DL, unsigned O) { + return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); +} + +SDDbgValue * +SelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, + DebugLoc DL, unsigned O) { + return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); +} + +namespace { + +/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node +/// pointed to by a use iterator is deleted, increment the use iterator +/// so that it doesn't dangle. +/// +/// This class also manages a "downlink" DAGUpdateListener, to forward +/// messages to ReplaceAllUsesWith's callers. +/// +class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { + SelectionDAG::DAGUpdateListener *DownLink; + SDNode::use_iterator &UI; + SDNode::use_iterator &UE; + + virtual void NodeDeleted(SDNode *N, SDNode *E) { + // Increment the iterator as needed. + while (UI != UE && N == *UI) + ++UI; + + // Then forward the message. + if (DownLink) DownLink->NodeDeleted(N, E); + } + + virtual void NodeUpdated(SDNode *N) { + // Just forward the message. + if (DownLink) DownLink->NodeUpdated(N); + } + +public: + RAUWUpdateListener(SelectionDAG::DAGUpdateListener *dl, + SDNode::use_iterator &ui, + SDNode::use_iterator &ue) + : DownLink(dl), UI(ui), UE(ue) {} +}; + +} + /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. /// This can cause recursive merging of nodes in the DAG. /// @@ -4803,6 +4976,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To, // is replaced by To, we don't want to replace of all its users with To // too. See PR3018 for more info. SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); + RAUWUpdateListener Listener(UpdateListener, UI, UE); while (UI != UE) { SDNode *User = *UI; @@ -4821,7 +4995,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To, // Now that we have modified User, add it back to the CSE maps. If it // already exists there, recursively merge the results together. - AddModifiedNodeToCSEMaps(User, UpdateListener); + AddModifiedNodeToCSEMaps(User, &Listener); } } @@ -4847,6 +5021,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To, // Iterate over just the existing users of From. See the comments in // the ReplaceAllUsesWith above. SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); + RAUWUpdateListener Listener(UpdateListener, UI, UE); while (UI != UE) { SDNode *User = *UI; @@ -4865,7 +5040,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To, // Now that we have modified User, add it back to the CSE maps. If it // already exists there, recursively merge the results together. - AddModifiedNodeToCSEMaps(User, UpdateListener); + AddModifiedNodeToCSEMaps(User, &Listener); } } @@ -4883,6 +5058,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, // Iterate over just the existing users of From. See the comments in // the ReplaceAllUsesWith above. SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); + RAUWUpdateListener Listener(UpdateListener, UI, UE); while (UI != UE) { SDNode *User = *UI; @@ -4902,7 +5078,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, // Now that we have modified User, add it back to the CSE maps. If it // already exists there, recursively merge the results together. - AddModifiedNodeToCSEMaps(User, UpdateListener); + AddModifiedNodeToCSEMaps(User, &Listener); } } @@ -4924,6 +5100,7 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To, // the ReplaceAllUsesWith above. SDNode::use_iterator UI = From.getNode()->use_begin(), UE = From.getNode()->use_end(); + RAUWUpdateListener Listener(UpdateListener, UI, UE); while (UI != UE) { SDNode *User = *UI; bool UserRemovedFromCSEMaps = false; @@ -4959,7 +5136,7 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To, // Now that we have modified User, add it back to the CSE maps. If it // already exists there, recursively merge the results together. - AddModifiedNodeToCSEMaps(User, UpdateListener); + AddModifiedNodeToCSEMaps(User, &Listener); } } @@ -5058,6 +5235,7 @@ unsigned SelectionDAG::AssignTopologicalOrder() { // count of outstanding operands. for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { SDNode *N = I++; + checkForCycles(N); unsigned Degree = N->getNumOperands(); if (Degree == 0) { // A node with no uses, add it to the result array immediately. @@ -5065,6 +5243,7 @@ unsigned SelectionDAG::AssignTopologicalOrder() { allnodes_iterator Q = N; if (Q != SortedPos) SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); + assert(SortedPos != AllNodes.end() && "Overran node list"); ++SortedPos; } else { // Temporarily use the Node Id as scratch space for the degree count. @@ -5076,22 +5255,35 @@ unsigned SelectionDAG::AssignTopologicalOrder() { // such that by the time the end is reached all nodes will be sorted. for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { SDNode *N = I; + checkForCycles(N); + // N is in sorted position, so all its uses have one less operand + // that needs to be sorted. for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); UI != UE; ++UI) { SDNode *P = *UI; unsigned Degree = P->getNodeId(); + assert(Degree != 0 && "Invalid node degree"); --Degree; if (Degree == 0) { // All of P's operands are sorted, so P may sorted now. P->setNodeId(DAGSize++); if (P != SortedPos) SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); + assert(SortedPos != AllNodes.end() && "Overran node list"); ++SortedPos; } else { // Update P's outstanding operand count. P->setNodeId(Degree); } } + if (I == SortedPos) { +#ifndef NDEBUG + SDNode *S = ++I; + dbgs() << "Overran sorted position:\n"; + S->dumprFull(); +#endif + llvm_unreachable(0); + } } assert(SortedPos == AllNodes.end() && @@ -5110,7 +5302,25 @@ unsigned SelectionDAG::AssignTopologicalOrder() { return DAGSize; } +/// AssignOrdering - Assign an order to the SDNode. +void SelectionDAG::AssignOrdering(const SDNode *SD, unsigned Order) { + assert(SD && "Trying to assign an order to a null node!"); + Ordering->add(SD, Order); +} +/// GetOrdering - Get the order for the SDNode. +unsigned SelectionDAG::GetOrdering(const SDNode *SD) const { + assert(SD && "Trying to get the order of a null node!"); + return Ordering->getOrder(SD); +} + +/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the +/// value is produced by SD. +void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD) { + DbgInfo->add(DB, SD); + if (SD) + SD->setHasDebugValue(true); +} //===----------------------------------------------------------------------===// // SDNode Class @@ -5122,16 +5332,18 @@ HandleSDNode::~HandleSDNode() { GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, const GlobalValue *GA, EVT VT, int64_t o, unsigned char TF) - : SDNode(Opc, DebugLoc::getUnknownLoc(), getSDVTList(VT)), - Offset(o), TargetFlags(TF) { + : SDNode(Opc, DebugLoc(), getSDVTList(VT)), Offset(o), TargetFlags(TF) { TheGlobal = const_cast(GA); } MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, MachineMemOperand *mmo) : SDNode(Opc, dl, VTs), MemoryVT(memvt), MMO(mmo) { - SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile()); + SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), + MMO->isNonTemporal()); assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); + assert(isNonTemporal() == MMO->isNonTemporal() && + "Non-temporal encoding error!"); assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); } @@ -5140,7 +5352,8 @@ MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, MachineMemOperand *mmo) : SDNode(Opc, dl, VTs, Ops, NumOps), MemoryVT(memvt), MMO(mmo) { - SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile()); + SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), + MMO->isNonTemporal()); assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); } @@ -5272,31 +5485,26 @@ bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, return false; } - -static void findPredecessor(SDNode *N, const SDNode *P, bool &found, - SmallPtrSet &Visited) { - if (found || !Visited.insert(N)) - return; - - for (unsigned i = 0, e = N->getNumOperands(); !found && i != e; ++i) { - SDNode *Op = N->getOperand(i).getNode(); - if (Op == P) { - found = true; - return; - } - findPredecessor(Op, P, found, Visited); - } -} - /// isPredecessorOf - Return true if this node is a predecessor of N. This node -/// is either an operand of N or it can be reached by recursively traversing -/// up the operands. +/// is either an operand of N or it can be reached by traversing up the operands. /// NOTE: this is an expensive method. Use it carefully. bool SDNode::isPredecessorOf(SDNode *N) const { SmallPtrSet Visited; - bool found = false; - findPredecessor(N, this, found, Visited); - return found; + SmallVector Worklist; + Worklist.push_back(N); + + do { + N = Worklist.pop_back_val(); + for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { + SDNode *Op = N->getOperand(i).getNode(); + if (Op == this) + return true; + if (Visited.insert(Op)) + Worklist.push_back(Op); + } + } while (!Worklist.empty()); + + return false; } uint64_t SDNode::getConstantOperandVal(unsigned Num) const { @@ -5314,15 +5522,15 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { if (const TargetInstrInfo *TII = G->getTarget().getInstrInfo()) if (getMachineOpcode() < TII->getNumOpcodes()) return TII->get(getMachineOpcode()).getName(); - return "<>"; + return "<>"; } if (G) { const TargetLowering &TLI = G->getTargetLoweringInfo(); const char *Name = TLI.getTargetNodeName(getOpcode()); if (Name) return Name; - return "<>"; + return "<>"; } - return "<>"; + return "<>"; #ifndef NDEBUG case ISD::DELETED_NODE: @@ -5370,14 +5578,17 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::EH_RETURN: return "EH_RETURN"; case ISD::ConstantPool: return "ConstantPool"; case ISD::ExternalSymbol: return "ExternalSymbol"; - case ISD::INTRINSIC_WO_CHAIN: { - unsigned IID = cast(getOperand(0))->getZExtValue(); - return Intrinsic::getName((Intrinsic::ID)IID); - } + case ISD::BlockAddress: return "BlockAddress"; + case ISD::INTRINSIC_WO_CHAIN: case ISD::INTRINSIC_VOID: case ISD::INTRINSIC_W_CHAIN: { - unsigned IID = cast(getOperand(1))->getZExtValue(); - return Intrinsic::getName((Intrinsic::ID)IID); + unsigned OpNo = getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1; + unsigned IID = cast(getOperand(OpNo))->getZExtValue(); + if (IID < Intrinsic::num_intrinsics) + return Intrinsic::getName((Intrinsic::ID)IID); + else if (const TargetIntrinsicInfo *TII = G->getTarget().getIntrinsicInfo()) + return TII->getName(IID); + llvm_unreachable("Invalid intrinsic ID"); } case ISD::BUILD_VECTOR: return "BUILD_VECTOR"; @@ -5389,13 +5600,13 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::TargetJumpTable: return "TargetJumpTable"; case ISD::TargetConstantPool: return "TargetConstantPool"; case ISD::TargetExternalSymbol: return "TargetExternalSymbol"; + case ISD::TargetBlockAddress: return "TargetBlockAddress"; case ISD::CopyToReg: return "CopyToReg"; case ISD::CopyFromReg: return "CopyFromReg"; case ISD::UNDEF: return "undef"; case ISD::MERGE_VALUES: return "merge_values"; case ISD::INLINEASM: return "inlineasm"; - case ISD::DBG_LABEL: return "dbg_label"; case ISD::EH_LABEL: return "eh_label"; case ISD::HANDLENODE: return "handlenode"; @@ -5484,6 +5695,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::FP_TO_SINT: return "fp_to_sint"; case ISD::FP_TO_UINT: return "fp_to_uint"; case ISD::BIT_CONVERT: return "bit_convert"; + case ISD::FP16_TO_FP32: return "fp16_to_fp32"; + case ISD::FP32_TO_FP16: return "fp32_to_fp16"; case ISD::CONVERT_RNDSAT: { switch (cast(this)->getCvtCode()) { @@ -5529,10 +5742,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::CTTZ: return "cttz"; case ISD::CTLZ: return "ctlz"; - // Debug info - case ISD::DBG_STOPPOINT: return "dbg_stoppoint"; - case ISD::DEBUG_LOC: return "debug_loc"; - // Trampolines case ISD::TRAMPOLINE: return "trampoline"; @@ -5606,7 +5815,7 @@ std::string ISD::ArgFlagsTy::getArgFlagsString() { void SDNode::dump() const { dump(0); } void SDNode::dump(const SelectionDAG *G) const { - print(errs(), G); + print(dbgs(), G); } void SDNode::print_types(raw_ostream &OS, const SelectionDAG *G) const { @@ -5698,9 +5907,9 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { } else if (const RegisterSDNode *R = dyn_cast(this)) { if (G && R->getReg() && TargetRegisterInfo::isPhysicalRegister(R->getReg())) { - OS << " " << G->getTarget().getRegisterInfo()->getName(R->getReg()); + OS << " %" << G->getTarget().getRegisterInfo()->getName(R->getReg()); } else { - OS << " #" << R->getReg(); + OS << " %reg" << R->getReg(); } } else if (const ExternalSymbolSDNode *ES = dyn_cast(this)) { @@ -5716,7 +5925,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { OS << ":" << N->getVT().getEVTString(); } else if (const LoadSDNode *LD = dyn_cast(this)) { - OS << " <" << *LD->getMemOperand(); + OS << "<" << *LD->getMemOperand(); bool doExt = true; switch (LD->getExtensionType()) { @@ -5734,7 +5943,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { OS << ">"; } else if (const StoreSDNode *ST = dyn_cast(this)) { - OS << " <" << *ST->getMemOperand(); + OS << "<" << *ST->getMemOperand(); if (ST->isTruncatingStore()) OS << ", trunc to " << ST->getMemoryVT().getEVTString(); @@ -5745,15 +5954,30 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { OS << ">"; } else if (const MemSDNode* M = dyn_cast(this)) { - OS << " <" << *M->getMemOperand() << ">"; + OS << "<" << *M->getMemOperand() << ">"; + } else if (const BlockAddressSDNode *BA = + dyn_cast(this)) { + OS << "<"; + WriteAsOperand(OS, BA->getBlockAddress()->getFunction(), false); + OS << ", "; + WriteAsOperand(OS, BA->getBlockAddress()->getBasicBlock(), false); + OS << ">"; + if (unsigned int TF = BA->getTargetFlags()) + OS << " [TF=" << TF << ']'; } + + if (G) + if (unsigned Order = G->GetOrdering(this)) + OS << " [ORD=" << Order << ']'; + + if (getNodeId() != -1) + OS << " [ID=" << getNodeId() << ']'; } void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { print_types(OS, G); - OS << " "; for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { - if (i) OS << ", "; + if (i) OS << ", "; else OS << " "; OS << (void*)getOperand(i).getNode(); if (unsigned RN = getOperand(i).getResNo()) OS << ":" << RN; @@ -5761,22 +5985,233 @@ void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { print_details(OS, G); } +static void printrWithDepthHelper(raw_ostream &OS, const SDNode *N, + const SelectionDAG *G, unsigned depth, + unsigned indent) +{ + if (depth == 0) + return; + + OS.indent(indent); + + N->print(OS, G); + + if (depth < 1) + return; + + for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { + OS << '\n'; + printrWithDepthHelper(OS, N->getOperand(i).getNode(), G, depth-1, indent+2); + } +} + +void SDNode::printrWithDepth(raw_ostream &OS, const SelectionDAG *G, + unsigned depth) const { + printrWithDepthHelper(OS, this, G, depth, 0); +} + +void SDNode::printrFull(raw_ostream &OS, const SelectionDAG *G) const { + // Don't print impossibly deep things. + printrWithDepth(OS, G, 100); +} + +void SDNode::dumprWithDepth(const SelectionDAG *G, unsigned depth) const { + printrWithDepth(dbgs(), G, depth); +} + +void SDNode::dumprFull(const SelectionDAG *G) const { + // Don't print impossibly deep things. + dumprWithDepth(G, 100); +} + static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) { for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) if (N->getOperand(i).getNode()->hasOneUse()) DumpNodes(N->getOperand(i).getNode(), indent+2, G); else - errs() << "\n" << std::string(indent+2, ' ') - << (void*)N->getOperand(i).getNode() << ": "; + dbgs() << "\n" << std::string(indent+2, ' ') + << (void*)N->getOperand(i).getNode() << ": "; - errs() << "\n"; - errs().indent(indent); + dbgs() << "\n"; + dbgs().indent(indent); N->dump(G); } +SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { + assert(N->getNumValues() == 1 && + "Can't unroll a vector with multiple results!"); + + EVT VT = N->getValueType(0); + unsigned NE = VT.getVectorNumElements(); + EVT EltVT = VT.getVectorElementType(); + DebugLoc dl = N->getDebugLoc(); + + SmallVector Scalars; + SmallVector Operands(N->getNumOperands()); + + // If ResNE is 0, fully unroll the vector op. + if (ResNE == 0) + ResNE = NE; + else if (NE > ResNE) + NE = ResNE; + + unsigned i; + for (i= 0; i != NE; ++i) { + for (unsigned j = 0; j != N->getNumOperands(); ++j) { + SDValue Operand = N->getOperand(j); + EVT OperandVT = Operand.getValueType(); + if (OperandVT.isVector()) { + // A vector operand; extract a single element. + EVT OperandEltVT = OperandVT.getVectorElementType(); + Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, + OperandEltVT, + Operand, + getConstant(i, MVT::i32)); + } else { + // A scalar operand; just use it as is. + Operands[j] = Operand; + } + } + + switch (N->getOpcode()) { + default: + Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, + &Operands[0], Operands.size())); + break; + case ISD::SHL: + case ISD::SRA: + case ISD::SRL: + case ISD::ROTL: + case ISD::ROTR: + Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], + getShiftAmountOperand(Operands[1]))); + break; + case ISD::SIGN_EXTEND_INREG: + case ISD::FP_ROUND_INREG: { + EVT ExtVT = cast(Operands[1])->getVT().getVectorElementType(); + Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, + Operands[0], + getValueType(ExtVT))); + } + } + } + + for (; i < ResNE; ++i) + Scalars.push_back(getUNDEF(EltVT)); + + return getNode(ISD::BUILD_VECTOR, dl, + EVT::getVectorVT(*getContext(), EltVT, ResNE), + &Scalars[0], Scalars.size()); +} + + +/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a +/// location that is 'Dist' units away from the location that the 'Base' load +/// is loading from. +bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, + unsigned Bytes, int Dist) const { + if (LD->getChain() != Base->getChain()) + return false; + EVT VT = LD->getValueType(0); + if (VT.getSizeInBits() / 8 != Bytes) + return false; + + SDValue Loc = LD->getOperand(1); + SDValue BaseLoc = Base->getOperand(1); + if (Loc.getOpcode() == ISD::FrameIndex) { + if (BaseLoc.getOpcode() != ISD::FrameIndex) + return false; + const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); + int FI = cast(Loc)->getIndex(); + int BFI = cast(BaseLoc)->getIndex(); + int FS = MFI->getObjectSize(FI); + int BFS = MFI->getObjectSize(BFI); + if (FS != BFS || FS != (int)Bytes) return false; + return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); + } + if (Loc.getOpcode() == ISD::ADD && Loc.getOperand(0) == BaseLoc) { + ConstantSDNode *V = dyn_cast(Loc.getOperand(1)); + if (V && (V->getSExtValue() == Dist*Bytes)) + return true; + } + + GlobalValue *GV1 = NULL; + GlobalValue *GV2 = NULL; + int64_t Offset1 = 0; + int64_t Offset2 = 0; + bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); + bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); + if (isGA1 && isGA2 && GV1 == GV2) + return Offset1 == (Offset2 + Dist*Bytes); + return false; +} + + +/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if +/// it cannot be inferred. +unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { + // If this is a GlobalAddress + cst, return the alignment. + GlobalValue *GV; + int64_t GVOffset = 0; + if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { + // If GV has specified alignment, then use it. Otherwise, use the preferred + // alignment. + unsigned Align = GV->getAlignment(); + if (!Align) { + if (GlobalVariable *GVar = dyn_cast(GV)) { + if (GVar->hasInitializer()) { + const TargetData *TD = TLI.getTargetData(); + Align = TD->getPreferredAlignment(GVar); + } + } + } + return MinAlign(Align, GVOffset); + } + + // If this is a direct reference to a stack slot, use information about the + // stack slot's alignment. + int FrameIdx = 1 << 31; + int64_t FrameOffset = 0; + if (FrameIndexSDNode *FI = dyn_cast(Ptr)) { + FrameIdx = FI->getIndex(); + } else if (Ptr.getOpcode() == ISD::ADD && + isa(Ptr.getOperand(1)) && + isa(Ptr.getOperand(0))) { + FrameIdx = cast(Ptr.getOperand(0))->getIndex(); + FrameOffset = Ptr.getConstantOperandVal(1); + } + + if (FrameIdx != (1 << 31)) { + // FIXME: Handle FI+CST. + const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); + unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), + FrameOffset); + if (MFI.isFixedObjectIndex(FrameIdx)) { + int64_t ObjectOffset = MFI.getObjectOffset(FrameIdx) + FrameOffset; + + // The alignment of the frame index can be determined from its offset from + // the incoming frame position. If the frame object is at offset 32 and + // the stack is guaranteed to be 16-byte aligned, then we know that the + // object is 16-byte aligned. + unsigned StackAlign = getTarget().getFrameInfo()->getStackAlignment(); + unsigned Align = MinAlign(ObjectOffset, StackAlign); + + // Finally, the frame object itself may have a known alignment. Factor + // the alignment + offset into a new alignment. For example, if we know + // the FI is 8 byte aligned, but the pointer is 4 off, we really have a + // 4-byte alignment of the resultant pointer. Likewise align 4 + 4-byte + // offset = 4-byte alignment, align 4 + 1-byte offset = align 1, etc. + return std::max(Align, FIInfoAlign); + } + return FIInfoAlign; + } + + return 0; +} + void SelectionDAG::dump() const { - errs() << "SelectionDAG has " << AllNodes.size() << " nodes:"; + dbgs() << "SelectionDAG has " << AllNodes.size() << " nodes:"; for (allnodes_const_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) { @@ -5787,7 +6222,7 @@ void SelectionDAG::dump() const { if (getRoot().getNode()) DumpNodes(getRoot().getNode(), 2, this); - errs() << "\n\n"; + dbgs() << "\n\n"; } void SDNode::printr(raw_ostream &OS, const SelectionDAG *G) const { @@ -5800,25 +6235,31 @@ static void DumpNodesr(raw_ostream &OS, const SDNode *N, unsigned indent, const SelectionDAG *G, VisitedSDNodeSet &once) { if (!once.insert(N)) // If we've been here before, return now. return; + // Dump the current SDNode, but don't end the line yet. OS << std::string(indent, ' '); N->printr(OS, G); + // Having printed this SDNode, walk the children: for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { const SDNode *child = N->getOperand(i).getNode(); + if (i) OS << ","; OS << " "; + if (child->getNumOperands() == 0) { // This child has no grandchildren; print it inline right here. child->printr(OS, G); once.insert(child); - } else { // Just the address. FIXME: also print the child's opcode + } else { // Just the address. FIXME: also print the child's opcode. OS << (void*)child; if (unsigned RN = N->getOperand(i).getResNo()) OS << ":" << RN; } } + OS << "\n"; + // Dump children that have grandchildren on their own line(s). for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { const SDNode *child = N->getOperand(i).getNode(); @@ -5828,12 +6269,12 @@ static void DumpNodesr(raw_ostream &OS, const SDNode *N, unsigned indent, void SDNode::dumpr() const { VisitedSDNodeSet once; - DumpNodesr(errs(), this, 0, 0, once); + DumpNodesr(dbgs(), this, 0, 0, once); } void SDNode::dumpr(const SelectionDAG *G) const { VisitedSDNodeSet once; - DumpNodesr(errs(), this, 0, G, once); + DumpNodesr(dbgs(), this, 0, G, once); } @@ -5853,7 +6294,8 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, - unsigned MinSplatBits) { + unsigned MinSplatBits, + bool isBigEndian) { EVT VT = getValueType(0); assert(VT.isVector() && "Expected a vector type"); unsigned sz = VT.getSizeInBits(); @@ -5870,12 +6312,14 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, unsigned int nOps = getNumOperands(); assert(nOps > 0 && "isConstantSplat has 0-size build vector"); unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); - for (unsigned i = 0; i < nOps; ++i) { + + for (unsigned j = 0; j < nOps; ++j) { + unsigned i = isBigEndian ? nOps-1-j : j; SDValue OpVal = getOperand(i); - unsigned BitPos = i * EltBitSize; + unsigned BitPos = j * EltBitSize; if (OpVal.getOpcode() == ISD::UNDEF) - SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos +EltBitSize); + SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); else if (ConstantSDNode *CN = dyn_cast(OpVal)) SplatValue |= (APInt(CN->getAPIntValue()).zextOrTrunc(EltBitSize). zextOrTrunc(sz) << BitPos); @@ -5927,3 +6371,41 @@ bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { return false; return true; } + +#ifdef XDEBUG +static void checkForCyclesHelper(const SDNode *N, + SmallPtrSet &Visited, + SmallPtrSet &Checked) { + // If this node has already been checked, don't check it again. + if (Checked.count(N)) + return; + + // If a node has already been visited on this depth-first walk, reject it as + // a cycle. + if (!Visited.insert(N)) { + dbgs() << "Offending node:\n"; + N->dumprFull(); + errs() << "Detected cycle in SelectionDAG\n"; + abort(); + } + + for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) + checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); + + Checked.insert(N); + Visited.erase(N); +} +#endif + +void llvm::checkForCycles(const llvm::SDNode *N) { +#ifdef XDEBUG + assert(N && "Checking nonexistant SDNode"); + SmallPtrSet visited; + SmallPtrSet checked; + checkForCyclesHelper(N, visited, checked); +#endif +} + +void llvm::checkForCycles(const llvm::SelectionDAG *DAG) { + checkForCycles(DAG->getRoot().getNode()); +}