X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAG.cpp;h=ac6a64ec6b89f26dd06ac5d5b0c8531fd4ea1898;hb=2ad40d34947ced36d96f351f6c2a4d57a044e2a4;hp=8c0554d186d928dbac5a483bafbd646d2cebcf7e;hpb=18141eed26d90b225ed4451a6b6521fb01f8eb31;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 8c0554d186d..ac6a64ec6b8 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -15,6 +15,7 @@ #include "SDNodeOrdering.h" #include "SDNodeDbgValue.h" #include "llvm/Constants.h" +#include "llvm/Analysis/DebugInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Function.h" #include "llvm/GlobalAlias.h" @@ -32,6 +33,7 @@ #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetFrameInfo.h" #include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetSelectionDAGInfo.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetIntrinsicInfo.h" @@ -304,10 +306,6 @@ ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, return Result; } -const TargetMachine &SelectionDAG::getTarget() const { - return MF->getTarget(); -} - //===----------------------------------------------------------------------===// // SDNode Profile Support //===----------------------------------------------------------------------===// @@ -792,8 +790,8 @@ unsigned SelectionDAG::getEVTAlignment(EVT VT) const { } // EntryNode could meaningfully have debug info if we can find it... -SelectionDAG::SelectionDAG(TargetLowering &tli, FunctionLoweringInfo &fli) - : TLI(tli), FLI(fli), +SelectionDAG::SelectionDAG(const TargetMachine &tm) + : TM(tm), TLI(*tm.getTargetLowering()), TSI(*tm.getSelectionDAGInfo()), EntryNode(ISD::EntryToken, DebugLoc(), getVTList(MVT::Other)), Root(getEntryNode()), Ordering(0) { AllNodes.push_back(&EntryNode); @@ -809,7 +807,6 @@ void SelectionDAG::init(MachineFunction &mf) { SelectionDAG::~SelectionDAG() { allnodes_clear(); delete Ordering; - DbgInfo->clear(); delete DbgInfo; } @@ -836,11 +833,8 @@ void SelectionDAG::clear() { EntryNode.UseList = 0; AllNodes.push_back(&EntryNode); Root = getEntryNode(); - delete Ordering; - Ordering = new SDNodeOrdering(); + Ordering->clear(); DbgInfo->clear(); - delete DbgInfo; - DbgInfo = new SDDbgInfo(); } SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, DebugLoc DL, EVT VT) { @@ -967,11 +961,21 @@ SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { EVT EltVT = VT.getScalarType(); if (EltVT==MVT::f32) return getConstantFP(APFloat((float)Val), VT, isTarget); - else + else if (EltVT==MVT::f64) return getConstantFP(APFloat(Val), VT, isTarget); + else if (EltVT==MVT::f80 || EltVT==MVT::f128) { + bool ignored; + APFloat apf = APFloat(Val); + apf.convert(*EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, + &ignored); + return getConstantFP(apf, VT, isTarget); + } else { + assert(0 && "Unsupported type in getConstantFP"); + return SDValue(); + } } -SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, +SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, DebugLoc DL, EVT VT, int64_t Offset, bool isTargetGA, unsigned char TargetFlags) { @@ -1006,7 +1010,7 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, GV, VT, + SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL, GV, VT, Offset, TargetFlags); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); @@ -1048,7 +1052,7 @@ SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, return SDValue(N, 0); } -SDValue SelectionDAG::getConstantPool(Constant *C, EVT VT, +SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, unsigned Alignment, int Offset, bool isTarget, unsigned char TargetFlags) { @@ -1319,7 +1323,7 @@ SDValue SelectionDAG::getEHLabel(DebugLoc dl, SDValue Root, MCSymbol *Label) { } -SDValue SelectionDAG::getBlockAddress(BlockAddress *BA, EVT VT, +SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, bool isTarget, unsigned char TargetFlags) { unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; @@ -1356,6 +1360,23 @@ SDValue SelectionDAG::getSrcValue(const Value *V) { return SDValue(N, 0); } +/// getMDNode - Return an MDNodeSDNode which holds an MDNode. +SDValue SelectionDAG::getMDNode(const MDNode *MD) { + FoldingSetNodeID ID; + AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); + ID.AddPointer(MD); + + void *IP = 0; + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) + return SDValue(E, 0); + + SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); + CSEMap.InsertNode(N, IP); + AllNodes.push_back(N); + return SDValue(N, 0); +} + + /// getShiftAmountOperand - Return the specified value casted to /// the target's desired shift amount type. SDValue SelectionDAG::getShiftAmountOperand(SDValue Op) { @@ -1904,7 +1925,8 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, // Output known-0 bits are known if clear or set in both the low clear bits // common to both LHS & RHS. For example, 8+(X<<3) is known to have the // low 3 bits clear. - APInt Mask2 = APInt::getLowBitsSet(BitWidth, Mask.countTrailingOnes()); + APInt Mask2 = APInt::getLowBitsSet(BitWidth, + BitWidth - Mask.countLeadingZeros()); ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero2, KnownOne2, Depth+1); assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); @@ -2214,7 +2236,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { // If we're told that NaNs won't happen, assume they won't. - if (FiniteOnlyFPMath()) + if (NoNaNsFPMath) return true; // If the value is a constant, we can obviously see if it is a NaN or not. @@ -2253,42 +2275,12 @@ bool SelectionDAG::isVerifiedDebugInfoDesc(SDValue Op) const { GlobalAddressSDNode *GA = dyn_cast(Op); if (!GA) return false; if (GA->getOffset() != 0) return false; - GlobalVariable *GV = dyn_cast(GA->getGlobal()); + const GlobalVariable *GV = dyn_cast(GA->getGlobal()); if (!GV) return false; return MF->getMMI().hasDebugInfo(); } -/// getShuffleScalarElt - Returns the scalar element that will make up the ith -/// element of the result of the vector shuffle. -SDValue SelectionDAG::getShuffleScalarElt(const ShuffleVectorSDNode *N, - unsigned i) { - EVT VT = N->getValueType(0); - DebugLoc dl = N->getDebugLoc(); - if (N->getMaskElt(i) < 0) - return getUNDEF(VT.getVectorElementType()); - unsigned Index = N->getMaskElt(i); - unsigned NumElems = VT.getVectorNumElements(); - SDValue V = (Index < NumElems) ? N->getOperand(0) : N->getOperand(1); - Index %= NumElems; - - if (V.getOpcode() == ISD::BIT_CONVERT) { - V = V.getOperand(0); - EVT VVT = V.getValueType(); - if (!VVT.isVector() || VVT.getVectorNumElements() != (unsigned)NumElems) - return SDValue(); - } - if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) - return (Index == 0) ? V.getOperand(0) - : getUNDEF(VT.getVectorElementType()); - if (V.getOpcode() == ISD::BUILD_VECTOR) - return V.getOperand(Index); - if (const ShuffleVectorSDNode *SVN = dyn_cast(V)) - return getShuffleScalarElt(SVN, Index); - return SDValue(); -} - - /// getNode - Gets or creates the specified node. /// SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT) { @@ -2448,9 +2440,18 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, VT.getVectorNumElements() == Operand.getValueType().getVectorNumElements()) && "Vector element count mismatch!"); - if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND) + + if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || + OpOpcode == ISD::ANY_EXTEND) // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); + + // (ext (trunx x)) -> x + if (OpOpcode == ISD::TRUNCATE) { + SDValue OpOp = Operand.getNode()->getOperand(0); + if (OpOp.getValueType() == VT) + return OpOp; + } break; case ISD::TRUNCATE: assert(VT.isInteger() && Operand.getValueType().isInteger() && @@ -2594,13 +2595,15 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, // one big BUILD_VECTOR. if (N1.getOpcode() == ISD::BUILD_VECTOR && N2.getOpcode() == ISD::BUILD_VECTOR) { - SmallVector Elts(N1.getNode()->op_begin(), N1.getNode()->op_end()); - Elts.insert(Elts.end(), N2.getNode()->op_begin(), N2.getNode()->op_end()); + SmallVector Elts(N1.getNode()->op_begin(), + N1.getNode()->op_end()); + Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); } break; case ISD::AND: - assert(VT.isInteger() && N1.getValueType() == N2.getValueType() && + assert(VT.isInteger() && "This operator does not apply to FP types!"); + assert(N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && "Binary operator types must match!"); // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's // worth handling here. @@ -2613,7 +2616,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, case ISD::XOR: case ISD::ADD: case ISD::SUB: - assert(VT.isInteger() && N1.getValueType() == N2.getValueType() && + assert(VT.isInteger() && "This operator does not apply to FP types!"); + assert(N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && "Binary operator types must match!"); // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so // it's worth handling here. @@ -2628,7 +2632,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, case ISD::SDIV: case ISD::SREM: assert(VT.isInteger() && "This operator does not apply to FP types!"); - // fall through + assert(N1.getValueType() == N2.getValueType() && + N1.getValueType() == VT && "Binary operator types must match!"); + break; case ISD::FADD: case ISD::FSUB: case ISD::FMUL: @@ -2651,6 +2657,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, return N1; } } + assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); assert(N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && "Binary operator types must match!"); break; @@ -2778,14 +2785,19 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, // If the indices are the same, return the inserted element else // if the indices are known different, extract the element from // the original vector. - if (N1.getOperand(2) == N2) { - if (VT == N1.getOperand(1).getValueType()) - return N1.getOperand(1); - else - return getSExtOrTrunc(N1.getOperand(1), DL, VT); - } else if (isa(N1.getOperand(2)) && - isa(N2)) + SDValue N1Op2 = N1.getOperand(2); + ConstantSDNode *N1Op2C = dyn_cast(N1Op2.getNode()); + + if (N1Op2C && N2C) { + if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { + if (VT == N1.getOperand(1).getValueType()) + return N1.getOperand(1); + else + return getSExtOrTrunc(N1.getOperand(1), DL, VT); + } + return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); + } } break; case ISD::EXTRACT_ELEMENT: @@ -2974,7 +2986,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, SDValue N1, SDValue N2, SDValue N3) { // Perform various simplifications. ConstantSDNode *N1C = dyn_cast(N1.getNode()); - ConstantSDNode *N2C = dyn_cast(N2.getNode()); switch (Opcode) { case ISD::CONCAT_VECTORS: // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to @@ -2982,9 +2993,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, if (N1.getOpcode() == ISD::BUILD_VECTOR && N2.getOpcode() == ISD::BUILD_VECTOR && N3.getOpcode() == ISD::BUILD_VECTOR) { - SmallVector Elts(N1.getNode()->op_begin(), N1.getNode()->op_end()); - Elts.insert(Elts.end(), N2.getNode()->op_begin(), N2.getNode()->op_end()); - Elts.insert(Elts.end(), N3.getNode()->op_begin(), N3.getNode()->op_end()); + SmallVector Elts(N1.getNode()->op_begin(), + N1.getNode()->op_end()); + Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); + Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); } break; @@ -3004,14 +3016,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, if (N2 == N3) return N2; // select C, X, X -> X break; - case ISD::BRCOND: - if (N2C) { - if (N2C->getZExtValue()) // Unconditional branch - return getNode(ISD::BR, DL, MVT::Other, N1, N3); - else - return N1; // Never-taken branch - } - break; case ISD::VECTOR_SHUFFLE: llvm_unreachable("should use getVectorShuffle constructor!"); break; @@ -3178,7 +3182,7 @@ static bool isMemSrcFromString(SDValue Src, std::string &Str) { if (!G) return false; - GlobalVariable *GV = dyn_cast(G->getGlobal()); + const GlobalVariable *GV = dyn_cast(G->getGlobal()); if (GV && GetConstantStringInfo(GV, Str, SrcDelta, false)) return true; @@ -3193,6 +3197,7 @@ static bool FindOptimalMemOpLowering(std::vector &MemOps, unsigned Limit, uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool NonScalarIntSafe, + bool MemcpyStrSrc, SelectionDAG &DAG, const TargetLowering &TLI) { assert((SrcAlign == 0 || SrcAlign >= DstAlign) && @@ -3201,9 +3206,12 @@ static bool FindOptimalMemOpLowering(std::vector &MemOps, // the value, i.e. memset or memcpy from constant string. Otherwise, it's // the inferred alignment of the source. 'DstAlign', on the other hand, is the // specified alignment of the memory operation. If it is zero, that means - // it's possible to change the alignment of the destination. + // it's possible to change the alignment of the destination. 'MemcpyStrSrc' + // indicates whether the memcpy source is constant so it does not need to be + // loaded. EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, - NonScalarIntSafe, DAG); + NonScalarIntSafe, MemcpyStrSrc, + DAG.getMachineFunction()); if (VT == MVT::Other) { if (DstAlign >= TLI.getTargetData()->getPointerPrefAlignment() || @@ -3226,6 +3234,15 @@ static bool FindOptimalMemOpLowering(std::vector &MemOps, if (VT.bitsGT(LVT)) VT = LVT; } + + // If we're optimizing for size, and there is a limit, bump the maximum number + // of operations inserted down to 4. This is a wild guess that approximates + // the size of a call to memcpy or memset (3 arguments + call). + if (Limit != ~0U) { + const Function *F = DAG.getMachineFunction().getFunction(); + if (F->hasFnAttr(Attribute::OptimizeForSize)) + Limit = 4; + } unsigned NumMemOps = 0; while (Size != 0) { @@ -3259,8 +3276,8 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, SDValue Src, uint64_t Size, unsigned Align, bool isVol, bool AlwaysInline, - const Value *DstSV, uint64_t DstSVOff, - const Value *SrcSV, uint64_t SrcSVOff) { + MachinePointerInfo DstPtrInfo, + MachinePointerInfo SrcPtrInfo) { // Turn a memcpy of undef to nop. if (Src.getOpcode() == ISD::UNDEF) return Chain; @@ -3269,9 +3286,6 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, // below a certain threshold. const TargetLowering &TLI = DAG.getTargetLoweringInfo(); std::vector MemOps; - uint64_t Limit = -1ULL; - if (!AlwaysInline) - Limit = TLI.getMaxStoresPerMemcpy(); bool DstAlignCanChange = false; MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); FrameIndexSDNode *FI = dyn_cast(Dst); @@ -3283,9 +3297,12 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, std::string Str; bool CopyFromStr = isMemSrcFromString(Src, Str); bool isZeroStr = CopyFromStr && Str.empty(); + unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(); + if (!FindOptimalMemOpLowering(MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align), - (isZeroStr ? 0 : SrcAlign), true, DAG, TLI)) + (isZeroStr ? 0 : SrcAlign), + true, CopyFromStr, DAG, TLI)) return SDValue(); if (DstAlignCanChange) { @@ -3317,7 +3334,8 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, Value = getMemsetStringVal(VT, dl, DAG, TLI, Str, SrcOff); Store = DAG.getStore(Chain, dl, Value, getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstSVOff + DstOff, isVol, false, Align); + DstPtrInfo.getWithOffset(DstOff), isVol, + false, Align); } else { // The type might not be legal for the target. This should only happen // if the type is smaller than a legal type, as on PPC, so the right @@ -3326,14 +3344,14 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, // FIXME does the case above also need this? EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); assert(NVT.bitsGE(VT)); - Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, + Value = DAG.getExtLoad(ISD::EXTLOAD, NVT, dl, Chain, getMemBasePlusOffset(Src, SrcOff, DAG), - SrcSV, SrcSVOff + SrcOff, VT, isVol, false, + SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, MinAlign(SrcAlign, SrcOff)); Store = DAG.getTruncStore(Chain, dl, Value, getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstSVOff + DstOff, VT, isVol, false, - Align); + DstPtrInfo.getWithOffset(DstOff), VT, isVol, + false, Align); } OutChains.push_back(Store); SrcOff += VTSize; @@ -3349,8 +3367,8 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, SDValue Src, uint64_t Size, unsigned Align, bool isVol, bool AlwaysInline, - const Value *DstSV, uint64_t DstSVOff, - const Value *SrcSV, uint64_t SrcSVOff) { + MachinePointerInfo DstPtrInfo, + MachinePointerInfo SrcPtrInfo) { // Turn a memmove of undef to nop. if (Src.getOpcode() == ISD::UNDEF) return Chain; @@ -3359,9 +3377,6 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, // below a certain threshold. const TargetLowering &TLI = DAG.getTargetLoweringInfo(); std::vector MemOps; - uint64_t Limit = -1ULL; - if (!AlwaysInline) - Limit = TLI.getMaxStoresPerMemmove(); bool DstAlignCanChange = false; MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); FrameIndexSDNode *FI = dyn_cast(Dst); @@ -3370,10 +3385,11 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, unsigned SrcAlign = DAG.InferPtrAlignment(Src); if (Align > SrcAlign) SrcAlign = Align; + unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(); if (!FindOptimalMemOpLowering(MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align), - SrcAlign, true, DAG, TLI)) + SrcAlign, true, false, DAG, TLI)) return SDValue(); if (DstAlignCanChange) { @@ -3399,7 +3415,8 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, Value = DAG.getLoad(VT, dl, Chain, getMemBasePlusOffset(Src, SrcOff, DAG), - SrcSV, SrcSVOff + SrcOff, isVol, false, SrcAlign); + SrcPtrInfo.getWithOffset(SrcOff), isVol, + false, SrcAlign); LoadValues.push_back(Value); LoadChains.push_back(Value.getValue(1)); SrcOff += VTSize; @@ -3414,7 +3431,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl, Store = DAG.getStore(Chain, dl, LoadValues[i], getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstSVOff + DstOff, isVol, false, Align); + DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); OutChains.push_back(Store); DstOff += VTSize; } @@ -3427,7 +3444,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, unsigned Align, bool isVol, - const Value *DstSV, uint64_t DstSVOff) { + MachinePointerInfo DstPtrInfo) { // Turn a memset of undef to nop. if (Src.getOpcode() == ISD::UNDEF) return Chain; @@ -3445,7 +3462,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, isa(Src) && cast(Src)->isNullValue(); if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(), Size, (DstAlignCanChange ? 0 : Align), 0, - NonScalarIntSafe, DAG, TLI)) + NonScalarIntSafe, false, DAG, TLI)) return SDValue(); if (DstAlignCanChange) { @@ -3468,7 +3485,8 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, SDValue Value = getMemsetValue(Src, VT, DAG, dl); SDValue Store = DAG.getStore(Chain, dl, Value, getMemBasePlusOffset(Dst, DstOff, DAG), - DstSV, DstSVOff + DstOff, isVol, false, 0); + DstPtrInfo.getWithOffset(DstOff), + isVol, false, 0); OutChains.push_back(Store); DstOff += VTSize; } @@ -3480,8 +3498,8 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl, SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool AlwaysInline, - const Value *DstSV, uint64_t DstSVOff, - const Value *SrcSV, uint64_t SrcSVOff) { + MachinePointerInfo DstPtrInfo, + MachinePointerInfo SrcPtrInfo) { // Check to see if we should lower the memcpy to loads and stores first. // For cases within the target-specified limits, this is the best choice. @@ -3493,7 +3511,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),Align, - isVol, false, DstSV, DstSVOff, SrcSV, SrcSVOff); + isVol, false, DstPtrInfo, SrcPtrInfo); if (Result.getNode()) return Result; } @@ -3501,9 +3519,9 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, // Then check to see if we should lower the memcpy with target-specific // code. If the target chooses to do this, this is the next best. SDValue Result = - TLI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, + TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline, - DstSV, DstSVOff, SrcSV, SrcSVOff); + DstPtrInfo, SrcPtrInfo); if (Result.getNode()) return Result; @@ -3513,7 +3531,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, assert(ConstantSize && "AlwaysInline requires a constant size!"); return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Align, isVol, - true, DstSV, DstSVOff, SrcSV, SrcSVOff); + true, DstPtrInfo, SrcPtrInfo); } // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc @@ -3544,8 +3562,8 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, - const Value *DstSV, uint64_t DstSVOff, - const Value *SrcSV, uint64_t SrcSVOff) { + MachinePointerInfo DstPtrInfo, + MachinePointerInfo SrcPtrInfo) { // Check to see if we should lower the memmove to loads and stores first. // For cases within the target-specified limits, this is the best choice. @@ -3558,7 +3576,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue Result = getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Align, isVol, - false, DstSV, DstSVOff, SrcSV, SrcSVOff); + false, DstPtrInfo, SrcPtrInfo); if (Result.getNode()) return Result; } @@ -3566,13 +3584,15 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, // Then check to see if we should lower the memmove with target-specific // code. If the target chooses to do this, this is the next best. SDValue Result = - TLI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, - DstSV, DstSVOff, SrcSV, SrcSVOff); + TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, + DstPtrInfo, SrcPtrInfo); if (Result.getNode()) return Result; + // FIXME: If the memmove is volatile, lowering it to plain libc memmove may + // not be safe. See memcpy above for more details. + // Emit a library call. - assert(!isVol && "library memmove does not support volatile"); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext()); @@ -3594,7 +3614,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, - const Value *DstSV, uint64_t DstSVOff) { + MachinePointerInfo DstPtrInfo) { // Check to see if we should lower the memset to stores first. // For cases within the target-specified limits, this is the best choice. @@ -3606,7 +3626,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), - Align, isVol, DstSV, DstSVOff); + Align, isVol, DstPtrInfo); if (Result.getNode()) return Result; @@ -3615,13 +3635,12 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, // Then check to see if we should lower the memset with target-specific // code. If the target chooses to do this, this is the next best. SDValue Result = - TLI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, - DstSV, DstSVOff); + TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, + DstPtrInfo); if (Result.getNode()) return Result; - // Emit a library call. - assert(!isVol && "library memset does not support volatile"); + // Emit a library call. const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext()); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; @@ -3653,19 +3672,12 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst, } SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, - SDValue Chain, - SDValue Ptr, SDValue Cmp, - SDValue Swp, const Value* PtrVal, + SDValue Chain, SDValue Ptr, SDValue Cmp, + SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); - // Check if the memory reference references a frame index - if (!PtrVal) - if (const FrameIndexSDNode *FI = - dyn_cast(Ptr.getNode())) - PtrVal = PseudoSourceValue::getFixedStack(FI->getIndex()); - MachineFunction &MF = getMachineFunction(); unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; @@ -3673,8 +3685,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, Flags |= MachineMemOperand::MOVolatile; MachineMemOperand *MMO = - MF.getMachineMemOperand(PtrVal, Flags, 0, - MemVT.getStoreSize(), Alignment); + MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO); } @@ -3713,12 +3724,6 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(MemVT); - // Check if the memory reference references a frame index - if (!PtrVal) - if (const FrameIndexSDNode *FI = - dyn_cast(Ptr.getNode())) - PtrVal = PseudoSourceValue::getFixedStack(FI->getIndex()); - MachineFunction &MF = getMachineFunction(); unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; @@ -3726,7 +3731,7 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, Flags |= MachineMemOperand::MOVolatile; MachineMemOperand *MMO = - MF.getMachineMemOperand(PtrVal, Flags, 0, + MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, MemVT.getStoreSize(), Alignment); return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO); @@ -3787,18 +3792,18 @@ SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, const EVT *VTs, unsigned NumVTs, const SDValue *Ops, unsigned NumOps, - EVT MemVT, const Value *srcValue, int SVOff, + EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol, bool ReadMem, bool WriteMem) { return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, - MemVT, srcValue, SVOff, Align, Vol, + MemVT, PtrInfo, Align, Vol, ReadMem, WriteMem); } SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, const SDValue *Ops, unsigned NumOps, - EVT MemVT, const Value *srcValue, int SVOff, + EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol, bool ReadMem, bool WriteMem) { if (Align == 0) // Ensure that codegen never sees alignment 0 @@ -3813,8 +3818,7 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, if (Vol) Flags |= MachineMemOperand::MOVolatile; MachineMemOperand *MMO = - MF.getMachineMemOperand(srcValue, Flags, SVOff, - MemVT.getStoreSize(), Align); + MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); } @@ -3851,37 +3855,71 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList, return SDValue(N, 0); } +/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a +/// MachinePointerInfo record from it. This is particularly useful because the +/// code generator has many cases where it doesn't bother passing in a +/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". +static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { + // If this is FI+Offset, we can model it. + if (const FrameIndexSDNode *FI = dyn_cast(Ptr)) + return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); + + // If this is (FI+Offset1)+Offset2, we can model it. + if (Ptr.getOpcode() != ISD::ADD || + !isa(Ptr.getOperand(1)) || + !isa(Ptr.getOperand(0))) + return MachinePointerInfo(); + + int FI = cast(Ptr.getOperand(0))->getIndex(); + return MachinePointerInfo::getFixedStack(FI, Offset+ + cast(Ptr.getOperand(1))->getSExtValue()); +} + +/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a +/// MachinePointerInfo record from it. This is particularly useful because the +/// code generator has many cases where it doesn't bother passing in a +/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". +static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { + // If the 'Offset' value isn't a constant, we can't handle this. + if (ConstantSDNode *OffsetNode = dyn_cast(OffsetOp)) + return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); + if (OffsetOp.getOpcode() == ISD::UNDEF) + return InferPointerInfo(Ptr); + return MachinePointerInfo(); +} + + SDValue -SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, - ISD::LoadExtType ExtType, EVT VT, SDValue Chain, +SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, + EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, SDValue Offset, - const Value *SV, int SVOffset, EVT MemVT, + MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, bool isNonTemporal, - unsigned Alignment) { + unsigned Alignment, const MDNode *TBAAInfo) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(VT); - // Check if the memory reference references a frame index - if (!SV) - if (const FrameIndexSDNode *FI = - dyn_cast(Ptr.getNode())) - SV = PseudoSourceValue::getFixedStack(FI->getIndex()); - - MachineFunction &MF = getMachineFunction(); unsigned Flags = MachineMemOperand::MOLoad; if (isVolatile) Flags |= MachineMemOperand::MOVolatile; if (isNonTemporal) Flags |= MachineMemOperand::MONonTemporal; + + // If we don't have a PtrInfo, infer the trivial frame index case to simplify + // clients. + if (PtrInfo.V == 0) + PtrInfo = InferPointerInfo(Ptr, Offset); + + MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = - MF.getMachineMemOperand(SV, Flags, SVOffset, - MemVT.getStoreSize(), Alignment); - return getLoad(AM, dl, ExtType, VT, Chain, Ptr, Offset, MemVT, MMO); + MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, + TBAAInfo); + return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); } SDValue -SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, - ISD::LoadExtType ExtType, EVT VT, SDValue Chain, +SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, + EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, SDValue Offset, EVT MemVT, MachineMemOperand *MMO) { if (VT == MemVT) { @@ -3927,59 +3965,59 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, DebugLoc dl, SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, - const Value *SV, int SVOffset, + MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, - unsigned Alignment) { + unsigned Alignment, const MDNode *TBAAInfo) { SDValue Undef = getUNDEF(Ptr.getValueType()); - return getLoad(ISD::UNINDEXED, dl, ISD::NON_EXTLOAD, VT, Chain, Ptr, Undef, - SV, SVOffset, VT, isVolatile, isNonTemporal, Alignment); + return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, + PtrInfo, VT, isVolatile, isNonTemporal, Alignment, TBAAInfo); } -SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, DebugLoc dl, EVT VT, +SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, - const Value *SV, - int SVOffset, EVT MemVT, + MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, bool isNonTemporal, - unsigned Alignment) { + unsigned Alignment, const MDNode *TBAAInfo) { SDValue Undef = getUNDEF(Ptr.getValueType()); - return getLoad(ISD::UNINDEXED, dl, ExtType, VT, Chain, Ptr, Undef, - SV, SVOffset, MemVT, isVolatile, isNonTemporal, Alignment); + return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, + PtrInfo, MemVT, isVolatile, isNonTemporal, Alignment, + TBAAInfo); } + SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM) { LoadSDNode *LD = cast(OrigLoad); assert(LD->getOffset().getOpcode() == ISD::UNDEF && "Load is already a indexed load!"); - return getLoad(AM, dl, LD->getExtensionType(), OrigLoad.getValueType(), - LD->getChain(), Base, Offset, LD->getSrcValue(), - LD->getSrcValueOffset(), LD->getMemoryVT(), + return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, + LD->getChain(), Base, Offset, LD->getPointerInfo(), + LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), LD->getAlignment()); } SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, - SDValue Ptr, const Value *SV, int SVOffset, + SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, - unsigned Alignment) { + unsigned Alignment, const MDNode *TBAAInfo) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(Val.getValueType()); - // Check if the memory reference references a frame index - if (!SV) - if (const FrameIndexSDNode *FI = - dyn_cast(Ptr.getNode())) - SV = PseudoSourceValue::getFixedStack(FI->getIndex()); - - MachineFunction &MF = getMachineFunction(); unsigned Flags = MachineMemOperand::MOStore; if (isVolatile) Flags |= MachineMemOperand::MOVolatile; if (isNonTemporal) Flags |= MachineMemOperand::MONonTemporal; + + if (PtrInfo.V == 0) + PtrInfo = InferPointerInfo(Ptr); + + MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = - MF.getMachineMemOperand(SV, Flags, SVOffset, - Val.getValueType().getStoreSize(), Alignment); + MF.getMachineMemOperand(PtrInfo, Flags, + Val.getValueType().getStoreSize(), Alignment, + TBAAInfo); return getStore(Chain, dl, Val, Ptr, MMO); } @@ -4008,27 +4046,26 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val, } SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val, - SDValue Ptr, const Value *SV, - int SVOffset, EVT SVT, - bool isVolatile, bool isNonTemporal, - unsigned Alignment) { + SDValue Ptr, MachinePointerInfo PtrInfo, + EVT SVT,bool isVolatile, bool isNonTemporal, + unsigned Alignment, + const MDNode *TBAAInfo) { if (Alignment == 0) // Ensure that codegen never sees alignment 0 Alignment = getEVTAlignment(SVT); - // Check if the memory reference references a frame index - if (!SV) - if (const FrameIndexSDNode *FI = - dyn_cast(Ptr.getNode())) - SV = PseudoSourceValue::getFixedStack(FI->getIndex()); - - MachineFunction &MF = getMachineFunction(); unsigned Flags = MachineMemOperand::MOStore; if (isVolatile) Flags |= MachineMemOperand::MOVolatile; if (isNonTemporal) Flags |= MachineMemOperand::MONonTemporal; + + if (PtrInfo.V == 0) + PtrInfo = InferPointerInfo(Ptr); + + MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = - MF.getMachineMemOperand(SV, Flags, SVOffset, SVT.getStoreSize(), Alignment); + MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, + TBAAInfo); return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); } @@ -4098,9 +4135,10 @@ SelectionDAG::getIndexedStore(SDValue OrigStore, DebugLoc dl, SDValue Base, SDValue SelectionDAG::getVAArg(EVT VT, DebugLoc dl, SDValue Chain, SDValue Ptr, - SDValue SV) { - SDValue Ops[] = { Chain, Ptr, SV }; - return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 3); + SDValue SV, + unsigned Align) { + SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; + return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); } SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, @@ -4382,17 +4420,16 @@ SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { /// already exists. If the resultant node does not exist in the DAG, the /// input node is returned. As a degenerate case, if you specify the same /// input operands as the node already has, the input node is returned. -SDValue SelectionDAG::UpdateNodeOperands(SDValue InN, SDValue Op) { - SDNode *N = InN.getNode(); +SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); // Check to see if there is no change. - if (Op == N->getOperand(0)) return InN; + if (Op == N->getOperand(0)) return N; // See if the modified node already exists. void *InsertPos = 0; if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) - return SDValue(Existing, InN.getResNo()); + return Existing; // Nope it doesn't. Remove the node from its current place in the maps. if (InsertPos) @@ -4404,22 +4441,20 @@ SDValue SelectionDAG::UpdateNodeOperands(SDValue InN, SDValue Op) { // If this gets put into a CSE map, add it. if (InsertPos) CSEMap.InsertNode(N, InsertPos); - return InN; + return N; } -SDValue SelectionDAG:: -UpdateNodeOperands(SDValue InN, SDValue Op1, SDValue Op2) { - SDNode *N = InN.getNode(); +SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); // Check to see if there is no change. if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) - return InN; // No operands changed, just return the input node. + return N; // No operands changed, just return the input node. // See if the modified node already exists. void *InsertPos = 0; if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) - return SDValue(Existing, InN.getResNo()); + return Existing; // Nope it doesn't. Remove the node from its current place in the maps. if (InsertPos) @@ -4434,32 +4469,31 @@ UpdateNodeOperands(SDValue InN, SDValue Op1, SDValue Op2) { // If this gets put into a CSE map, add it. if (InsertPos) CSEMap.InsertNode(N, InsertPos); - return InN; + return N; } -SDValue SelectionDAG:: -UpdateNodeOperands(SDValue N, SDValue Op1, SDValue Op2, SDValue Op3) { +SDNode *SelectionDAG:: +UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { SDValue Ops[] = { Op1, Op2, Op3 }; return UpdateNodeOperands(N, Ops, 3); } -SDValue SelectionDAG:: -UpdateNodeOperands(SDValue N, SDValue Op1, SDValue Op2, +SDNode *SelectionDAG:: +UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3, SDValue Op4) { SDValue Ops[] = { Op1, Op2, Op3, Op4 }; return UpdateNodeOperands(N, Ops, 4); } -SDValue SelectionDAG:: -UpdateNodeOperands(SDValue N, SDValue Op1, SDValue Op2, +SDNode *SelectionDAG:: +UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3, SDValue Op4, SDValue Op5) { SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; return UpdateNodeOperands(N, Ops, 5); } -SDValue SelectionDAG:: -UpdateNodeOperands(SDValue InN, const SDValue *Ops, unsigned NumOps) { - SDNode *N = InN.getNode(); +SDNode *SelectionDAG:: +UpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { assert(N->getNumOperands() == NumOps && "Update with wrong number of operands"); @@ -4473,12 +4507,12 @@ UpdateNodeOperands(SDValue InN, const SDValue *Ops, unsigned NumOps) { } // No operands changed, just return the input node. - if (!AnyChange) return InN; + if (!AnyChange) return N; // See if the modified node already exists. void *InsertPos = 0; if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) - return SDValue(Existing, InN.getResNo()); + return Existing; // Nope it doesn't. Remove the node from its current place in the maps. if (InsertPos) @@ -4492,7 +4526,7 @@ UpdateNodeOperands(SDValue InN, const SDValue *Ops, unsigned NumOps) { // If this gets put into a CSE map, add it. if (InsertPos) CSEMap.InsertNode(N, InsertPos); - return InN; + return N; } /// DropOperands - Release the operands and set this node to have @@ -4913,7 +4947,7 @@ SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, } SDDbgValue * -SelectionDAG::getDbgValue(MDNode *MDPtr, Value *C, uint64_t Off, +SelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, DebugLoc DL, unsigned O) { return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); } @@ -5321,8 +5355,8 @@ unsigned SelectionDAG::GetOrdering(const SDNode *SD) const { /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the /// value is produced by SD. -void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD) { - DbgInfo->add(DB, SD); +void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { + DbgInfo->add(DB, SD, isParameter); if (SD) SD->setHasDebugValue(true); } @@ -5335,10 +5369,11 @@ HandleSDNode::~HandleSDNode() { DropOperands(); } -GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, const GlobalValue *GA, +GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, DebugLoc DL, + const GlobalValue *GA, EVT VT, int64_t o, unsigned char TF) - : SDNode(Opc, DebugLoc(), getSDVTList(VT)), Offset(o), TargetFlags(TF) { - TheGlobal = const_cast(GA); + : SDNode(Opc, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { + TheGlobal = GA; } MemSDNode::MemSDNode(unsigned Opc, DebugLoc dl, SDVTList VTs, EVT memvt, @@ -5392,6 +5427,8 @@ const EVT *SDNode::getValueTypeList(EVT VT) { sys::SmartScopedLock Lock(*VTMutex); return &(*EVTs->insert(VT).first); } else { + assert(VT.getSimpleVT().SimpleTy < MVT::LAST_VALUETYPE && + "Value type out of range!"); return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; } } @@ -5462,9 +5499,9 @@ bool SDNode::isOperandOf(SDNode *N) const { /// reachesChainWithoutSideEffects - Return true if this operand (which must /// be a chain) reaches the specified operand without crossing any -/// side-effecting instructions. In practice, this looks through token -/// factors and non-volatile loads. In order to remain efficient, this only -/// looks a couple of nodes in, it does not do an exhaustive search. +/// side-effecting instructions on any chain path. In practice, this looks +/// through token factors and non-volatile loads. In order to remain efficient, +/// this only looks a couple of nodes in, it does not do an exhaustive search. bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth) const { if (*this == Dest) return true; @@ -5474,12 +5511,12 @@ bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, if (Depth == 0) return false; // If this is a token factor, all inputs to the TF happen in parallel. If any - // of the operands of the TF reach dest, then we can do the xform. + // of the operands of the TF does not reach dest, then we cannot do the xform. if (getOpcode() == ISD::TokenFactor) { for (unsigned i = 0, e = getNumOperands(); i != e; ++i) - if (getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) - return true; - return false; + if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) + return false; + return true; } // Loads don't have side effects, look through them. @@ -5558,6 +5595,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::PCMARKER: return "PCMarker"; case ISD::READCYCLECOUNTER: return "ReadCycleCounter"; case ISD::SRCVALUE: return "SrcValue"; + case ISD::MDNODE_SDNODE: return "MDNode"; case ISD::EntryToken: return "EntryToken"; case ISD::TokenFactor: return "TokenFactor"; case ISD::AssertSext: return "AssertSext"; @@ -5581,6 +5619,9 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::LSDAADDR: return "LSDAADDR"; case ISD::EHSELECTION: return "EHSELECTION"; case ISD::EH_RETURN: return "EH_RETURN"; + case ISD::EH_SJLJ_SETJMP: return "EH_SJLJ_SETJMP"; + case ISD::EH_SJLJ_LONGJMP: return "EH_SJLJ_LONGJMP"; + case ISD::EH_SJLJ_DISPATCHSETUP: return "EH_SJLJ_DISPATCHSETUP"; case ISD::ConstantPool: return "ConstantPool"; case ISD::ExternalSymbol: return "ExternalSymbol"; case ISD::BlockAddress: return "BlockAddress"; @@ -5621,13 +5662,16 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::FSQRT: return "fsqrt"; case ISD::FSIN: return "fsin"; case ISD::FCOS: return "fcos"; - case ISD::FPOWI: return "fpowi"; - case ISD::FPOW: return "fpow"; case ISD::FTRUNC: return "ftrunc"; case ISD::FFLOOR: return "ffloor"; case ISD::FCEIL: return "fceil"; case ISD::FRINT: return "frint"; case ISD::FNEARBYINT: return "fnearbyint"; + case ISD::FEXP: return "fexp"; + case ISD::FEXP2: return "fexp2"; + case ISD::FLOG: return "flog"; + case ISD::FLOG2: return "flog2"; + case ISD::FLOG10: return "flog10"; // Binary operators case ISD::ADD: return "add"; @@ -5658,7 +5702,9 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::FREM: return "frem"; case ISD::FCOPYSIGN: return "fcopysign"; case ISD::FGETSIGN: return "fgetsign"; + case ISD::FPOW: return "fpow"; + case ISD::FPOWI: return "fpowi"; case ISD::SETCC: return "setcc"; case ISD::VSETCC: return "vsetcc"; case ISD::SELECT: return "select"; @@ -5821,6 +5867,7 @@ std::string ISD::ArgFlagsTy::getArgFlagsString() { void SDNode::dump() const { dump(0); } void SDNode::dump(const SelectionDAG *G) const { print(dbgs(), G); + dbgs() << '\n'; } void SDNode::print_types(raw_ostream &OS, const SelectionDAG *G) const { @@ -5844,7 +5891,7 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { for (MachineSDNode::mmo_iterator i = MN->memoperands_begin(), e = MN->memoperands_end(); i != e; ++i) { OS << **i; - if (next(i) != e) + if (llvm::next(i) != e) OS << " "; } OS << ">"; @@ -5926,6 +5973,11 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { OS << "<" << M->getValue() << ">"; else OS << ""; + } else if (const MDNodeSDNode *MD = dyn_cast(this)) { + if (MD->getMD()) + OS << "<" << MD->getMD() << ">"; + else + OS << ""; } else if (const VTSDNode *N = dyn_cast(this)) { OS << ":" << N->getVT().getEVTString(); } @@ -5977,6 +6029,21 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { if (getNodeId() != -1) OS << " [ID=" << getNodeId() << ']'; + + DebugLoc dl = getDebugLoc(); + if (G && !dl.isUnknown()) { + DIScope + Scope(dl.getScope(G->getMachineFunction().getFunction()->getContext())); + OS << " dbg:"; + // Omit the directory, since it's usually long and uninteresting. + if (Scope.Verify()) + OS << Scope.getFilename(); + else + OS << ""; + OS << ':' << dl.getLine(); + if (dl.getCol() != 0) + OS << ':' << dl.getCol(); + } } void SDNode::print(raw_ostream &OS, const SelectionDAG *G) const { @@ -6063,7 +6130,7 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { unsigned i; for (i= 0; i != NE; ++i) { - for (unsigned j = 0; j != N->getNumOperands(); ++j) { + for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { SDValue Operand = N->getOperand(j); EVT OperandVT = Operand.getValueType(); if (OperandVT.isVector()) { @@ -6141,8 +6208,8 @@ bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, return true; } - GlobalValue *GV1 = NULL; - GlobalValue *GV2 = NULL; + const GlobalValue *GV1 = NULL; + const GlobalValue *GV2 = NULL; int64_t Offset1 = 0; int64_t Offset2 = 0; bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); @@ -6157,14 +6224,14 @@ bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, /// it cannot be inferred. unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { // If this is a GlobalAddress + cst, return the alignment. - GlobalValue *GV; + const GlobalValue *GV; int64_t GVOffset = 0; if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { // If GV has specified alignment, then use it. Otherwise, use the preferred // alignment. unsigned Align = GV->getAlignment(); if (!Align) { - if (GlobalVariable *GVar = dyn_cast(GV)) { + if (const GlobalVariable *GVar = dyn_cast(GV)) { if (GVar->hasInitializer()) { const TargetData *TD = TLI.getTargetData(); Align = TD->getPreferredAlignment(GVar); @@ -6192,23 +6259,6 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), FrameOffset); - if (MFI.isFixedObjectIndex(FrameIdx)) { - int64_t ObjectOffset = MFI.getObjectOffset(FrameIdx) + FrameOffset; - - // The alignment of the frame index can be determined from its offset from - // the incoming frame position. If the frame object is at offset 32 and - // the stack is guaranteed to be 16-byte aligned, then we know that the - // object is 16-byte aligned. - unsigned StackAlign = getTarget().getFrameInfo()->getStackAlignment(); - unsigned Align = MinAlign(ObjectOffset, StackAlign); - - // Finally, the frame object itself may have a known alignment. Factor - // the alignment + offset into a new alignment. For example, if we know - // the FI is 8 byte aligned, but the pointer is 4 off, we really have a - // 4-byte alignment of the resultant pointer. Likewise align 4 + 4-byte - // offset = 4-byte alignment, align 4 + 1-byte offset = align 1, etc. - return std::max(Align, FIInfoAlign); - } return FIInfoAlign; } @@ -6326,8 +6376,8 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, if (OpVal.getOpcode() == ISD::UNDEF) SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); else if (ConstantSDNode *CN = dyn_cast(OpVal)) - SplatValue |= (APInt(CN->getAPIntValue()).zextOrTrunc(EltBitSize). - zextOrTrunc(sz) << BitPos); + SplatValue |= APInt(CN->getAPIntValue()).zextOrTrunc(EltBitSize). + zextOrTrunc(sz) << BitPos; else if (ConstantFPSDNode *CN = dyn_cast(OpVal)) SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <