X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAG.cpp;h=1a1243285976c00a7f832fc681b53f048d5e60cf;hb=e10efce22502d1a1855d25baf1458660f4ba6f33;hp=d295a607777ebe0c58defb2666ede2d5736ca05f;hpb=af47b11b959713d70c45bee1922e468adfaeaff0;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index d295a607777..1a124328597 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -2,17 +2,17 @@ // // The LLVM Compiler Infrastructure // -// This file was developed by the LLVM research group and is distributed under -// the University of Illinois Open Source License. See LICENSE.TXT for details. +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This implements the SelectionDAG class. // //===----------------------------------------------------------------------===// - #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/Constants.h" +#include "llvm/GlobalAlias.h" #include "llvm/GlobalVariable.h" #include "llvm/Intrinsics.h" #include "llvm/DerivedTypes.h" @@ -20,8 +20,10 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/Support/MathExtras.h" -#include "llvm/Target/MRegisterInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetInstrInfo.h" @@ -42,6 +44,19 @@ static SDVTList makeVTList(const MVT::ValueType *VTs, unsigned NumVTs) { return Res; } +static const fltSemantics *MVTToAPFloatSemantics(MVT::ValueType VT) { + switch (VT) { + default: assert(0 && "Unknown FP format"); + case MVT::f32: return &APFloat::IEEEsingle; + case MVT::f64: return &APFloat::IEEEdouble; + case MVT::f80: return &APFloat::x87DoubleExtended; + case MVT::f128: return &APFloat::IEEEquad; + case MVT::ppcf128: return &APFloat::PPCDoubleDouble; + } +} + +SelectionDAG::DAGUpdateListener::~DAGUpdateListener() {} + //===----------------------------------------------------------------------===// // ConstantFPSDNode Class //===----------------------------------------------------------------------===// @@ -56,28 +71,20 @@ bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { bool ConstantFPSDNode::isValueValidForType(MVT::ValueType VT, const APFloat& Val) { + assert(MVT::isFloatingPoint(VT) && "Can only convert between FP types"); + + // Anything can be extended to ppc long double. + if (VT == MVT::ppcf128) + return true; + + // PPC long double cannot be shrunk to anything though. + if (&Val.getSemantics() == &APFloat::PPCDoubleDouble) + return false; + // convert modifies in place, so make a copy. APFloat Val2 = APFloat(Val); - switch (VT) { - default: - return false; // These can't be represented as floating point! - - // FIXME rounding mode needs to be more flexible - case MVT::f32: - return &Val2.getSemantics() == &APFloat::IEEEsingle || - Val2.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven) == - APFloat::opOK; - case MVT::f64: - return &Val2.getSemantics() == &APFloat::IEEEsingle || - &Val2.getSemantics() == &APFloat::IEEEdouble || - Val2.convert(APFloat::IEEEdouble, APFloat::rmNearestTiesToEven) == - APFloat::opOK; - // TODO: Figure out how to test if we can use a shorter type instead! - case MVT::f80: - case MVT::f128: - case MVT::ppcf128: - return true; - } + return Val2.convert(*MVTToAPFloatSemantics(VT), + APFloat::rmNearestTiesToEven) == APFloat::opOK; } //===----------------------------------------------------------------------===// @@ -109,17 +116,9 @@ bool ISD::isBuildVectorAllOnes(const SDNode *N) { if (!cast(NotZero)->isAllOnesValue()) return false; } else if (isa(NotZero)) { - MVT::ValueType VT = NotZero.getValueType(); - if (VT== MVT::f64) { - if (((cast(NotZero)->getValueAPF(). - convertToAPInt().getZExtValue())) != (uint64_t)-1) - return false; - } else { - if ((uint32_t)cast(NotZero)-> - getValueAPF().convertToAPInt().getZExtValue() != - (uint32_t)-1) - return false; - } + if (!cast(NotZero)->getValueAPF(). + convertToAPInt().isAllOnesValue()) + return false; } else return false; @@ -172,6 +171,43 @@ bool ISD::isBuildVectorAllZeros(const SDNode *N) { return true; } +/// isScalarToVector - Return true if the specified node is a +/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low +/// element is not an undef. +bool ISD::isScalarToVector(const SDNode *N) { + if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) + return true; + + if (N->getOpcode() != ISD::BUILD_VECTOR) + return false; + if (N->getOperand(0).getOpcode() == ISD::UNDEF) + return false; + unsigned NumElems = N->getNumOperands(); + for (unsigned i = 1; i < NumElems; ++i) { + SDOperand V = N->getOperand(i); + if (V.getOpcode() != ISD::UNDEF) + return false; + } + return true; +} + + +/// isDebugLabel - Return true if the specified node represents a debug +/// label (i.e. ISD::LABEL or TargetInstrInfo::LABEL node and third operand +/// is 0). +bool ISD::isDebugLabel(const SDNode *N) { + SDOperand Zero; + if (N->getOpcode() == ISD::LABEL) + Zero = N->getOperand(2); + else if (N->isTargetOpcode() && + N->getTargetOpcode() == TargetInstrInfo::LABEL) + // Chain moved to last operand. + Zero = N->getOperand(1); + else + return false; + return isa(Zero) && cast(Zero)->isNullValue(); +} + /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) /// when given the operation for (X op Y). ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { @@ -318,13 +354,16 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) { // Handle SDNode leafs with special info. switch (N->getOpcode()) { default: break; // Normal nodes don't need extra info. + case ISD::ARG_FLAGS: + ID.AddInteger(cast(N)->getArgFlags().getRawBits()); + break; case ISD::TargetConstant: case ISD::Constant: - ID.AddInteger(cast(N)->getValue()); + ID.Add(cast(N)->getAPIntValue()); break; case ISD::TargetConstantFP: case ISD::ConstantFP: { - ID.AddAPFloat(cast(N)->getValueAPF()); + ID.Add(cast(N)->getValueAPF()); break; } case ISD::TargetGlobalAddress: @@ -342,10 +381,16 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) { case ISD::Register: ID.AddInteger(cast(N)->getReg()); break; - case ISD::SRCVALUE: { - SrcValueSDNode *SV = cast(N); - ID.AddPointer(SV->getValue()); - ID.AddInteger(SV->getOffset()); + case ISD::SRCVALUE: + ID.AddPointer(cast(N)->getValue()); + break; + case ISD::MEMOPERAND: { + const MemOperand &MO = cast(N)->MO; + ID.AddPointer(MO.getValue()); + ID.AddInteger(MO.getFlags()); + ID.AddInteger(MO.getOffset()); + ID.AddInteger(MO.getSize()); + ID.AddInteger(MO.getAlignment()); break; } case ISD::FrameIndex: @@ -371,9 +416,7 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) { LoadSDNode *LD = cast(N); ID.AddInteger(LD->getAddressingMode()); ID.AddInteger(LD->getExtensionType()); - ID.AddInteger((unsigned int)(LD->getLoadedVT())); - ID.AddPointer(LD->getSrcValue()); - ID.AddInteger(LD->getSrcValueOffset()); + ID.AddInteger((unsigned int)(LD->getMemoryVT())); ID.AddInteger(LD->getAlignment()); ID.AddInteger(LD->isVolatile()); break; @@ -382,9 +425,7 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) { StoreSDNode *ST = cast(N); ID.AddInteger(ST->getAddressingMode()); ID.AddInteger(ST->isTruncatingStore()); - ID.AddInteger((unsigned int)(ST->getStoredVT())); - ID.AddPointer(ST->getSrcValue()); - ID.AddInteger(ST->getSrcValueOffset()); + ID.AddInteger((unsigned int)(ST->getMemoryVT())); ID.AddInteger(ST->getAlignment()); ID.AddInteger(ST->isVolatile()); break; @@ -423,14 +464,15 @@ void SelectionDAG::RemoveDeadNodes() { // no cycles in the graph. for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { SDNode *Operand = I->Val; - Operand->removeUser(N); + Operand->removeUser(std::distance(N->op_begin(), I), N); // Now that we removed this operand, see if there are no uses of it left. if (Operand->use_empty()) DeadNodes.push_back(Operand); } - if (N->OperandsNeedDelete) + if (N->OperandsNeedDelete) { delete[] N->OperandList; + } N->OperandList = 0; N->NumOperands = 0; @@ -442,7 +484,7 @@ void SelectionDAG::RemoveDeadNodes() { setRoot(Dummy.getValue()); } -void SelectionDAG::RemoveDeadNode(SDNode *N, std::vector &Deleted) { +void SelectionDAG::RemoveDeadNode(SDNode *N, DAGUpdateListener *UpdateListener){ SmallVector DeadNodes; DeadNodes.push_back(N); @@ -452,6 +494,9 @@ void SelectionDAG::RemoveDeadNode(SDNode *N, std::vector &Deleted) { SDNode *N = DeadNodes.back(); DeadNodes.pop_back(); + if (UpdateListener) + UpdateListener->NodeDeleted(N); + // Take the node out of the appropriate CSE map. RemoveNodeFromCSEMaps(N); @@ -459,19 +504,19 @@ void SelectionDAG::RemoveDeadNode(SDNode *N, std::vector &Deleted) { // no cycles in the graph. for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { SDNode *Operand = I->Val; - Operand->removeUser(N); + Operand->removeUser(std::distance(N->op_begin(), I), N); // Now that we removed this operand, see if there are no uses of it left. if (Operand->use_empty()) DeadNodes.push_back(Operand); } - if (N->OperandsNeedDelete) + if (N->OperandsNeedDelete) { delete[] N->OperandList; + } N->OperandList = 0; N->NumOperands = 0; // Finally, remove N itself. - Deleted.push_back(N); AllNodes.erase(N); } } @@ -494,9 +539,10 @@ void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { // Drop all of the operands and decrement used nodes use counts. for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) - I->Val->removeUser(N); - if (N->OperandsNeedDelete) + I->Val->removeUser(std::distance(N->op_begin(), I), N); + if (N->OperandsNeedDelete) { delete[] N->OperandList; + } N->OperandList = 0; N->NumOperands = 0; @@ -527,10 +573,16 @@ void SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { Erased = TargetExternalSymbols.erase(cast(N)->getSymbol()); break; - case ISD::VALUETYPE: - Erased = ValueTypeNodes[cast(N)->getVT()] != 0; - ValueTypeNodes[cast(N)->getVT()] = 0; + case ISD::VALUETYPE: { + MVT::ValueType VT = cast(N)->getVT(); + if (MVT::isExtendedVT(VT)) { + Erased = ExtendedValueTypeNodes.erase(VT); + } else { + Erased = ValueTypeNodes[VT] != 0; + ValueTypeNodes[VT] = 0; + } break; + } default: // Remove it from the CSE Map. Erased = CSEMap.RemoveNode(N); @@ -632,17 +684,13 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, if (const LoadSDNode *LD = dyn_cast(N)) { ID.AddInteger(LD->getAddressingMode()); ID.AddInteger(LD->getExtensionType()); - ID.AddInteger((unsigned int)(LD->getLoadedVT())); - ID.AddPointer(LD->getSrcValue()); - ID.AddInteger(LD->getSrcValueOffset()); + ID.AddInteger((unsigned int)(LD->getMemoryVT())); ID.AddInteger(LD->getAlignment()); ID.AddInteger(LD->isVolatile()); } else if (const StoreSDNode *ST = dyn_cast(N)) { ID.AddInteger(ST->getAddressingMode()); ID.AddInteger(ST->isTruncatingStore()); - ID.AddInteger((unsigned int)(ST->getStoredVT())); - ID.AddPointer(ST->getSrcValue()); - ID.AddInteger(ST->getSrcValueOffset()); + ID.AddInteger((unsigned int)(ST->getMemoryVT())); ID.AddInteger(ST->getAlignment()); ID.AddInteger(ST->isVolatile()); } @@ -655,8 +703,9 @@ SelectionDAG::~SelectionDAG() { while (!AllNodes.empty()) { SDNode *N = AllNodes.begin(); N->SetNextInBucket(0); - if (N->OperandsNeedDelete) + if (N->OperandsNeedDelete) { delete [] N->OperandList; + } N->OperandList = 0; N->NumOperands = 0; AllNodes.pop_front(); @@ -665,7 +714,8 @@ SelectionDAG::~SelectionDAG() { SDOperand SelectionDAG::getZeroExtendInReg(SDOperand Op, MVT::ValueType VT) { if (Op.getValueType() == VT) return Op; - int64_t Imm = ~0ULL >> (64-MVT::getSizeInBits(VT)); + APInt Imm = APInt::getLowBitsSet(Op.getValueSizeInBits(), + MVT::getSizeInBits(VT)); return getNode(ISD::AND, Op.getValueType(), Op, getConstant(Imm, Op.getValueType())); } @@ -680,25 +730,50 @@ SDOperand SelectionDAG::getString(const std::string &Val) { } SDOperand SelectionDAG::getConstant(uint64_t Val, MVT::ValueType VT, bool isT) { + MVT::ValueType EltVT = + MVT::isVector(VT) ? MVT::getVectorElementType(VT) : VT; + + return getConstant(APInt(MVT::getSizeInBits(EltVT), Val), VT, isT); +} + +SDOperand SelectionDAG::getConstant(const APInt &Val, MVT::ValueType VT, bool isT) { assert(MVT::isInteger(VT) && "Cannot create FP integer constant!"); - assert(!MVT::isVector(VT) && "Cannot create Vector ConstantSDNodes!"); + + MVT::ValueType EltVT = + MVT::isVector(VT) ? MVT::getVectorElementType(VT) : VT; - // Mask out any bits that are not valid for this constant. - Val &= MVT::getIntVTBitMask(VT); + assert(Val.getBitWidth() == MVT::getSizeInBits(EltVT) && + "APInt size does not match type size!"); unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; FoldingSetNodeID ID; - AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); - ID.AddInteger(Val); + AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); + ID.Add(Val); void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) - return SDOperand(E, 0); - SDNode *N = new ConstantSDNode(isT, Val, VT); - CSEMap.InsertNode(N, IP); - AllNodes.push_back(N); - return SDOperand(N, 0); + SDNode *N = NULL; + if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) + if (!MVT::isVector(VT)) + return SDOperand(N, 0); + if (!N) { + N = new ConstantSDNode(isT, Val, EltVT); + CSEMap.InsertNode(N, IP); + AllNodes.push_back(N); + } + + SDOperand Result(N, 0); + if (MVT::isVector(VT)) { + SmallVector Ops; + Ops.assign(MVT::getVectorNumElements(VT), Result); + Result = getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); + } + return Result; } +SDOperand SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { + return getConstant(Val, TLI.getPointerTy(), isTarget); +} + + SDOperand SelectionDAG::getConstantFP(const APFloat& V, MVT::ValueType VT, bool isTarget) { assert(MVT::isFloatingPoint(VT) && "Cannot create integer FP constant!"); @@ -712,7 +787,7 @@ SDOperand SelectionDAG::getConstantFP(const APFloat& V, MVT::ValueType VT, unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; FoldingSetNodeID ID; AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); - ID.AddAPFloat(V); + ID.Add(V); void *IP = 0; SDNode *N = NULL; if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) @@ -746,12 +821,20 @@ SDOperand SelectionDAG::getConstantFP(double Val, MVT::ValueType VT, SDOperand SelectionDAG::getGlobalAddress(const GlobalValue *GV, MVT::ValueType VT, int Offset, bool isTargetGA) { - const GlobalVariable *GVar = dyn_cast(GV); unsigned Opc; + + const GlobalVariable *GVar = dyn_cast(GV); + if (!GVar) { + // If GV is an alias then use the aliasee for determining thread-localness. + if (const GlobalAlias *GA = dyn_cast(GV)) + GVar = dyn_cast_or_null(GA->resolveAliasedGlobal()); + } + if (GVar && GVar->isThreadLocal()) Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; else Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; + FoldingSetNodeID ID; AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); ID.AddPointer(GV); @@ -846,16 +929,30 @@ SDOperand SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { return SDOperand(N, 0); } +SDOperand SelectionDAG::getArgFlags(ISD::ArgFlagsTy Flags) { + FoldingSetNodeID ID; + AddNodeIDNode(ID, ISD::ARG_FLAGS, getVTList(MVT::Other), 0, 0); + ID.AddInteger(Flags.getRawBits()); + void *IP = 0; + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) + return SDOperand(E, 0); + SDNode *N = new ARG_FLAGSSDNode(Flags); + CSEMap.InsertNode(N, IP); + AllNodes.push_back(N); + return SDOperand(N, 0); +} + SDOperand SelectionDAG::getValueType(MVT::ValueType VT) { - assert(!MVT::isExtendedVT(VT) && "Expecting a simple value type!"); - if ((unsigned)VT >= ValueTypeNodes.size()) + if (!MVT::isExtendedVT(VT) && (unsigned)VT >= ValueTypeNodes.size()) ValueTypeNodes.resize(VT+1); - if (ValueTypeNodes[VT] == 0) { - ValueTypeNodes[VT] = new VTSDNode(VT); - AllNodes.push_back(ValueTypeNodes[VT]); - } - return SDOperand(ValueTypeNodes[VT], 0); + SDNode *&N = MVT::isExtendedVT(VT) ? + ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT]; + + if (N) return SDOperand(N, 0); + N = new VTSDNode(VT); + AllNodes.push_back(N); + return SDOperand(N, 0); } SDOperand SelectionDAG::getExternalSymbol(const char *Sym, MVT::ValueType VT) { @@ -899,18 +996,42 @@ SDOperand SelectionDAG::getRegister(unsigned RegNo, MVT::ValueType VT) { return SDOperand(N, 0); } -SDOperand SelectionDAG::getSrcValue(const Value *V, int Offset) { +SDOperand SelectionDAG::getSrcValue(const Value *V) { assert((!V || isa(V->getType())) && "SrcValue is not a pointer?"); FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); ID.AddPointer(V); - ID.AddInteger(Offset); + + void *IP = 0; + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) + return SDOperand(E, 0); + + SDNode *N = new SrcValueSDNode(V); + CSEMap.InsertNode(N, IP); + AllNodes.push_back(N); + return SDOperand(N, 0); +} + +SDOperand SelectionDAG::getMemOperand(const MemOperand &MO) { + const Value *v = MO.getValue(); + assert((!v || isa(v->getType())) && + "SrcValue is not a pointer?"); + + FoldingSetNodeID ID; + AddNodeIDNode(ID, ISD::MEMOPERAND, getVTList(MVT::Other), 0, 0); + ID.AddPointer(v); + ID.AddInteger(MO.getFlags()); + ID.AddInteger(MO.getOffset()); + ID.AddInteger(MO.getSize()); + ID.AddInteger(MO.getAlignment()); + void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDOperand(E, 0); - SDNode *N = new SrcValueSDNode(V, Offset); + + SDNode *N = new MemOperandSDNode(MO); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDOperand(N, 0); @@ -953,32 +1074,26 @@ SDOperand SelectionDAG::FoldSetCC(MVT::ValueType VT, SDOperand N1, } if (ConstantSDNode *N2C = dyn_cast(N2.Val)) { - uint64_t C2 = N2C->getValue(); + const APInt &C2 = N2C->getAPIntValue(); if (ConstantSDNode *N1C = dyn_cast(N1.Val)) { - uint64_t C1 = N1C->getValue(); - - // Sign extend the operands if required - if (ISD::isSignedIntSetCC(Cond)) { - C1 = N1C->getSignExtended(); - C2 = N2C->getSignExtended(); - } + const APInt &C1 = N1C->getAPIntValue(); switch (Cond) { default: assert(0 && "Unknown integer setcc!"); case ISD::SETEQ: return getConstant(C1 == C2, VT); case ISD::SETNE: return getConstant(C1 != C2, VT); - case ISD::SETULT: return getConstant(C1 < C2, VT); - case ISD::SETUGT: return getConstant(C1 > C2, VT); - case ISD::SETULE: return getConstant(C1 <= C2, VT); - case ISD::SETUGE: return getConstant(C1 >= C2, VT); - case ISD::SETLT: return getConstant((int64_t)C1 < (int64_t)C2, VT); - case ISD::SETGT: return getConstant((int64_t)C1 > (int64_t)C2, VT); - case ISD::SETLE: return getConstant((int64_t)C1 <= (int64_t)C2, VT); - case ISD::SETGE: return getConstant((int64_t)C1 >= (int64_t)C2, VT); + case ISD::SETULT: return getConstant(C1.ult(C2), VT); + case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); + case ISD::SETULE: return getConstant(C1.ule(C2), VT); + case ISD::SETUGE: return getConstant(C1.uge(C2), VT); + case ISD::SETLT: return getConstant(C1.slt(C2), VT); + case ISD::SETGT: return getConstant(C1.sgt(C2), VT); + case ISD::SETLE: return getConstant(C1.sle(C2), VT); + case ISD::SETGE: return getConstant(C1.sge(C2), VT); } } } - if (ConstantFPSDNode *N1C = dyn_cast(N1.Val)) + if (ConstantFPSDNode *N1C = dyn_cast(N1.Val)) { if (ConstantFPSDNode *N2C = dyn_cast(N2.Val)) { // No compile time operations on this type yet. if (N1C->getValueType(0) == MVT::ppcf128) @@ -1030,21 +1145,25 @@ SDOperand SelectionDAG::FoldSetCC(MVT::ValueType VT, SDOperand N1, // Ensure that the constant occurs on the RHS. return getSetCC(VT, N2, N1, ISD::getSetCCSwappedOperands(Cond)); } - + } + // Could not fold it. return SDOperand(); } +/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We +/// use this predicate to simplify operations downstream. +bool SelectionDAG::SignBitIsZero(SDOperand Op, unsigned Depth) const { + unsigned BitWidth = Op.getValueSizeInBits(); + return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); +} + /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use /// this predicate to simplify operations downstream. Mask is known to be zero /// for bits that V cannot have. -bool SelectionDAG::MaskedValueIsZero(SDOperand Op, uint64_t Mask, +bool SelectionDAG::MaskedValueIsZero(SDOperand Op, const APInt &Mask, unsigned Depth) const { - // The masks are not wide enough to represent this type! Should use APInt. - if (Op.getValueType() == MVT::i128) - return false; - - uint64_t KnownZero, KnownOne; + APInt KnownZero, KnownOne; ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth); assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); return (KnownZero & Mask) == Mask; @@ -1054,30 +1173,30 @@ bool SelectionDAG::MaskedValueIsZero(SDOperand Op, uint64_t Mask, /// known to be either zero or one and return them in the KnownZero/KnownOne /// bitsets. This code only analyzes bits in Mask, in order to short-circuit /// processing. -void SelectionDAG::ComputeMaskedBits(SDOperand Op, uint64_t Mask, - uint64_t &KnownZero, uint64_t &KnownOne, +void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask, + APInt &KnownZero, APInt &KnownOne, unsigned Depth) const { - KnownZero = KnownOne = 0; // Don't know anything. + unsigned BitWidth = Mask.getBitWidth(); + assert(BitWidth == MVT::getSizeInBits(Op.getValueType()) && + "Mask size mismatches value type size!"); + + KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. if (Depth == 6 || Mask == 0) return; // Limit search depth. - // The masks are not wide enough to represent this type! Should use APInt. - if (Op.getValueType() == MVT::i128) - return; - - uint64_t KnownZero2, KnownOne2; + APInt KnownZero2, KnownOne2; switch (Op.getOpcode()) { case ISD::Constant: // We know all of the bits for a constant! - KnownOne = cast(Op)->getValue() & Mask; + KnownOne = cast(Op)->getAPIntValue() & Mask; KnownZero = ~KnownOne & Mask; return; case ISD::AND: // If either the LHS or the RHS are Zero, the result is zero. ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); - Mask &= ~KnownZero; - ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero2, KnownOne2, Depth+1); + ComputeMaskedBits(Op.getOperand(0), Mask & ~KnownZero, + KnownZero2, KnownOne2, Depth+1); assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); @@ -1088,8 +1207,8 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, uint64_t Mask, return; case ISD::OR: ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); - Mask &= ~KnownOne; - ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero2, KnownOne2, Depth+1); + ComputeMaskedBits(Op.getOperand(0), Mask & ~KnownOne, + KnownZero2, KnownOne2, Depth+1); assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); @@ -1105,7 +1224,7 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, uint64_t Mask, assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); // Output known-0 bits are known if clear or set in both the LHS & RHS. - uint64_t KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); + APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); // Output known-1 are known to be set if set in only one of the LHS, RHS. KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); KnownZero = KnownZeroOut; @@ -1133,88 +1252,94 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, uint64_t Mask, return; case ISD::SETCC: // If we know the result of a setcc has the top bits zero, use this info. - if (TLI.getSetCCResultContents() == TargetLowering::ZeroOrOneSetCCResult) - KnownZero |= (MVT::getIntVTBitMask(Op.getValueType()) ^ 1ULL); + if (TLI.getSetCCResultContents() == TargetLowering::ZeroOrOneSetCCResult && + BitWidth > 1) + KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); return; case ISD::SHL: // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 if (ConstantSDNode *SA = dyn_cast(Op.getOperand(1))) { - ComputeMaskedBits(Op.getOperand(0), Mask >> SA->getValue(), + unsigned ShAmt = SA->getValue(); + + // If the shift count is an invalid immediate, don't do anything. + if (ShAmt >= BitWidth) + return; + + ComputeMaskedBits(Op.getOperand(0), Mask.lshr(ShAmt), KnownZero, KnownOne, Depth+1); assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); - KnownZero <<= SA->getValue(); - KnownOne <<= SA->getValue(); - KnownZero |= (1ULL << SA->getValue())-1; // low bits known zero. + KnownZero <<= ShAmt; + KnownOne <<= ShAmt; + // low bits known zero. + KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); } return; case ISD::SRL: // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 if (ConstantSDNode *SA = dyn_cast(Op.getOperand(1))) { - MVT::ValueType VT = Op.getValueType(); unsigned ShAmt = SA->getValue(); - uint64_t TypeMask = MVT::getIntVTBitMask(VT); - ComputeMaskedBits(Op.getOperand(0), (Mask << ShAmt) & TypeMask, + // If the shift count is an invalid immediate, don't do anything. + if (ShAmt >= BitWidth) + return; + + ComputeMaskedBits(Op.getOperand(0), (Mask << ShAmt), KnownZero, KnownOne, Depth+1); assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); - KnownZero &= TypeMask; - KnownOne &= TypeMask; - KnownZero >>= ShAmt; - KnownOne >>= ShAmt; + KnownZero = KnownZero.lshr(ShAmt); + KnownOne = KnownOne.lshr(ShAmt); - uint64_t HighBits = (1ULL << ShAmt)-1; - HighBits <<= MVT::getSizeInBits(VT)-ShAmt; + APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt) & Mask; KnownZero |= HighBits; // High bits known zero. } return; case ISD::SRA: if (ConstantSDNode *SA = dyn_cast(Op.getOperand(1))) { - MVT::ValueType VT = Op.getValueType(); unsigned ShAmt = SA->getValue(); - // Compute the new bits that are at the top now. - uint64_t TypeMask = MVT::getIntVTBitMask(VT); + // If the shift count is an invalid immediate, don't do anything. + if (ShAmt >= BitWidth) + return; - uint64_t InDemandedMask = (Mask << ShAmt) & TypeMask; + APInt InDemandedMask = (Mask << ShAmt); // If any of the demanded bits are produced by the sign extension, we also // demand the input sign bit. - uint64_t HighBits = (1ULL << ShAmt)-1; - HighBits <<= MVT::getSizeInBits(VT) - ShAmt; - if (HighBits & Mask) - InDemandedMask |= MVT::getIntVTSignBit(VT); + APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt) & Mask; + if (HighBits.getBoolValue()) + InDemandedMask |= APInt::getSignBit(BitWidth); ComputeMaskedBits(Op.getOperand(0), InDemandedMask, KnownZero, KnownOne, Depth+1); assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); - KnownZero &= TypeMask; - KnownOne &= TypeMask; - KnownZero >>= ShAmt; - KnownOne >>= ShAmt; + KnownZero = KnownZero.lshr(ShAmt); + KnownOne = KnownOne.lshr(ShAmt); // Handle the sign bits. - uint64_t SignBit = MVT::getIntVTSignBit(VT); - SignBit >>= ShAmt; // Adjust to where it is now in the mask. + APInt SignBit = APInt::getSignBit(BitWidth); + SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. - if (KnownZero & SignBit) { + if (KnownZero.intersects(SignBit)) { KnownZero |= HighBits; // New bits are known zero. - } else if (KnownOne & SignBit) { + } else if (KnownOne.intersects(SignBit)) { KnownOne |= HighBits; // New bits are known one. } } return; case ISD::SIGN_EXTEND_INREG: { MVT::ValueType EVT = cast(Op.getOperand(1))->getVT(); + unsigned EBits = MVT::getSizeInBits(EVT); // Sign extension. Compute the demanded bits in the result that are not // present in the input. - uint64_t NewBits = ~MVT::getIntVTBitMask(EVT) & Mask; + APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits) & Mask; - uint64_t InSignBit = MVT::getIntVTSignBit(EVT); - int64_t InputDemandedBits = Mask & MVT::getIntVTBitMask(EVT); + APInt InSignBit = APInt::getSignBit(EBits); + APInt InputDemandedBits = Mask & APInt::getLowBitsSet(BitWidth, EBits); // If the sign extended bits are demanded, we know that the sign // bit is demanded. - if (NewBits) + InSignBit.zext(BitWidth); + if (NewBits.getBoolValue()) InputDemandedBits |= InSignBit; ComputeMaskedBits(Op.getOperand(0), InputDemandedBits, @@ -1223,10 +1348,10 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, uint64_t Mask, // If the sign bit of the input is known set or clear, then we know the // top bits of the result. - if (KnownZero & InSignBit) { // Input sign bit known clear + if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear KnownZero |= NewBits; KnownOne &= ~NewBits; - } else if (KnownOne & InSignBit) { // Input sign bit known set + } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set KnownOne |= NewBits; KnownZero &= ~NewBits; } else { // Input sign bit unknown @@ -1238,79 +1363,113 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, uint64_t Mask, case ISD::CTTZ: case ISD::CTLZ: case ISD::CTPOP: { - MVT::ValueType VT = Op.getValueType(); - unsigned LowBits = Log2_32(MVT::getSizeInBits(VT))+1; - KnownZero = ~((1ULL << LowBits)-1) & MVT::getIntVTBitMask(VT); - KnownOne = 0; + unsigned LowBits = Log2_32(BitWidth)+1; + KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); + KnownOne = APInt(BitWidth, 0); return; } case ISD::LOAD: { if (ISD::isZEXTLoad(Op.Val)) { LoadSDNode *LD = cast(Op); - MVT::ValueType VT = LD->getLoadedVT(); - KnownZero |= ~MVT::getIntVTBitMask(VT) & Mask; + MVT::ValueType VT = LD->getMemoryVT(); + unsigned MemBits = MVT::getSizeInBits(VT); + KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits) & Mask; } return; } case ISD::ZERO_EXTEND: { - uint64_t InMask = MVT::getIntVTBitMask(Op.getOperand(0).getValueType()); - uint64_t NewBits = (~InMask) & Mask; - ComputeMaskedBits(Op.getOperand(0), Mask & InMask, KnownZero, - KnownOne, Depth+1); - KnownZero |= NewBits & Mask; - KnownOne &= ~NewBits; + MVT::ValueType InVT = Op.getOperand(0).getValueType(); + unsigned InBits = MVT::getSizeInBits(InVT); + APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; + APInt InMask = Mask; + InMask.trunc(InBits); + KnownZero.trunc(InBits); + KnownOne.trunc(InBits); + ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); + KnownZero.zext(BitWidth); + KnownOne.zext(BitWidth); + KnownZero |= NewBits; return; } case ISD::SIGN_EXTEND: { MVT::ValueType InVT = Op.getOperand(0).getValueType(); - unsigned InBits = MVT::getSizeInBits(InVT); - uint64_t InMask = MVT::getIntVTBitMask(InVT); - uint64_t InSignBit = 1ULL << (InBits-1); - uint64_t NewBits = (~InMask) & Mask; - uint64_t InDemandedBits = Mask & InMask; + unsigned InBits = MVT::getSizeInBits(InVT); + APInt InSignBit = APInt::getSignBit(InBits); + APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask; + APInt InMask = Mask; + InMask.trunc(InBits); // If any of the sign extended bits are demanded, we know that the sign - // bit is demanded. - if (NewBits & Mask) - InDemandedBits |= InSignBit; - - ComputeMaskedBits(Op.getOperand(0), InDemandedBits, KnownZero, - KnownOne, Depth+1); - // If the sign bit is known zero or one, the top bits match. - if (KnownZero & InSignBit) { + // bit is demanded. Temporarily set this bit in the mask for our callee. + if (NewBits.getBoolValue()) + InMask |= InSignBit; + + KnownZero.trunc(InBits); + KnownOne.trunc(InBits); + ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); + + // Note if the sign bit is known to be zero or one. + bool SignBitKnownZero = KnownZero.isNegative(); + bool SignBitKnownOne = KnownOne.isNegative(); + assert(!(SignBitKnownZero && SignBitKnownOne) && + "Sign bit can't be known to be both zero and one!"); + + // If the sign bit wasn't actually demanded by our caller, we don't + // want it set in the KnownZero and KnownOne result values. Reset the + // mask and reapply it to the result values. + InMask = Mask; + InMask.trunc(InBits); + KnownZero &= InMask; + KnownOne &= InMask; + + KnownZero.zext(BitWidth); + KnownOne.zext(BitWidth); + + // If the sign bit is known zero or one, the top bits match. + if (SignBitKnownZero) KnownZero |= NewBits; - KnownOne &= ~NewBits; - } else if (KnownOne & InSignBit) { + else if (SignBitKnownOne) KnownOne |= NewBits; - KnownZero &= ~NewBits; - } else { // Otherwise, top bits aren't known. - KnownOne &= ~NewBits; - KnownZero &= ~NewBits; - } return; } case ISD::ANY_EXTEND: { - MVT::ValueType VT = Op.getOperand(0).getValueType(); - ComputeMaskedBits(Op.getOperand(0), Mask & MVT::getIntVTBitMask(VT), - KnownZero, KnownOne, Depth+1); + MVT::ValueType InVT = Op.getOperand(0).getValueType(); + unsigned InBits = MVT::getSizeInBits(InVT); + APInt InMask = Mask; + InMask.trunc(InBits); + KnownZero.trunc(InBits); + KnownOne.trunc(InBits); + ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); + KnownZero.zext(BitWidth); + KnownOne.zext(BitWidth); return; } case ISD::TRUNCATE: { - ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero, KnownOne, Depth+1); + MVT::ValueType InVT = Op.getOperand(0).getValueType(); + unsigned InBits = MVT::getSizeInBits(InVT); + APInt InMask = Mask; + InMask.zext(InBits); + KnownZero.zext(InBits); + KnownOne.zext(InBits); + ComputeMaskedBits(Op.getOperand(0), InMask, KnownZero, KnownOne, Depth+1); assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); - uint64_t OutMask = MVT::getIntVTBitMask(Op.getValueType()); - KnownZero &= OutMask; - KnownOne &= OutMask; + KnownZero.trunc(BitWidth); + KnownOne.trunc(BitWidth); break; } case ISD::AssertZext: { MVT::ValueType VT = cast(Op.getOperand(1))->getVT(); - uint64_t InMask = MVT::getIntVTBitMask(VT); + APInt InMask = APInt::getLowBitsSet(BitWidth, MVT::getSizeInBits(VT)); ComputeMaskedBits(Op.getOperand(0), Mask & InMask, KnownZero, KnownOne, Depth+1); KnownZero |= (~InMask) & Mask; return; } + case ISD::FGETSIGN: + // All bits are zero except the low bit. + KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); + return; + case ISD::ADD: { // If either the LHS or the RHS are Zero, the result is zero. ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); @@ -1321,11 +1480,11 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, uint64_t Mask, // Output known-0 bits are known if clear or set in both the low clear bits // common to both LHS & RHS. For example, 8+(X<<3) is known to have the // low 3 bits clear. - uint64_t KnownZeroOut = std::min(CountTrailingZeros_64(~KnownZero), - CountTrailingZeros_64(~KnownZero2)); + unsigned KnownZeroOut = std::min(KnownZero.countTrailingOnes(), + KnownZero2.countTrailingOnes()); - KnownZero = (1ULL << KnownZeroOut) - 1; - KnownOne = 0; + KnownZero = APInt::getLowBitsSet(BitWidth, KnownZeroOut); + KnownOne = APInt(BitWidth, 0); return; } case ISD::SUB: { @@ -1335,21 +1494,21 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, uint64_t Mask, // We know that the top bits of C-X are clear if X contains less bits // than C (i.e. no wrap-around can happen). For example, 20-X is // positive if we can prove that X is >= 0 and < 16. - MVT::ValueType VT = CLHS->getValueType(0); - if ((CLHS->getValue() & MVT::getIntVTSignBit(VT)) == 0) { // sign bit clear - unsigned NLZ = CountLeadingZeros_64(CLHS->getValue()+1); - uint64_t MaskV = (1ULL << (63-NLZ))-1; // NLZ can't be 64 with no sign bit - MaskV = ~MaskV & MVT::getIntVTBitMask(VT); + if (CLHS->getAPIntValue().isNonNegative()) { + unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); + // NLZ can't be BitWidth with no sign bit + APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); ComputeMaskedBits(Op.getOperand(1), MaskV, KnownZero, KnownOne, Depth+1); // If all of the MaskV bits are known to be zero, then we know the output // top bits are zero, because we now know that the output is from [0-C]. if ((KnownZero & MaskV) == MaskV) { - unsigned NLZ2 = CountLeadingZeros_64(CLHS->getValue()); - KnownZero = ~((1ULL << (64-NLZ2))-1) & Mask; // Top bits known zero. - KnownOne = 0; // No one bits known. + unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); + // Top bits known zero. + KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2) & Mask; + KnownOne = APInt(BitWidth, 0); // No one bits known. } else { - KnownZero = KnownOne = 0; // Otherwise, nothing known. + KnownZero = KnownOne = APInt(BitWidth, 0); // Otherwise, nothing known. } } return; @@ -1390,17 +1549,13 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{ return VTBits-Tmp; case ISD::Constant: { - uint64_t Val = cast(Op)->getValue(); - // If negative, invert the bits, then look at it. - if (Val & MVT::getIntVTSignBit(VT)) - Val = ~Val; + const APInt &Val = cast(Op)->getAPIntValue(); + // If negative, return # leading ones. + if (Val.isNegative()) + return Val.countLeadingOnes(); - // Shift the bits so they are the leading bits in the int64_t. - Val <<= 64-VTBits; - - // Return # leading zeros. We use 'min' here in case Val was zero before - // shifting. We don't want to return '64' as for an i32 "0". - return std::min(VTBits, CountLeadingZeros_64(Val)); + // Return # leading zeros. + return Val.countLeadingZeros(); } case ISD::SIGN_EXTEND: @@ -1477,18 +1632,18 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{ // Special case decrementing a value (ADD X, -1): if (ConstantSDNode *CRHS = dyn_cast(Op.getOperand(0))) if (CRHS->isAllOnesValue()) { - uint64_t KnownZero, KnownOne; - uint64_t Mask = MVT::getIntVTBitMask(VT); + APInt KnownZero, KnownOne; + APInt Mask = APInt::getAllOnesValue(VTBits); ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero, KnownOne, Depth+1); // If the input is known to be 0 or 1, the output is 0/-1, which is all // sign bits set. - if ((KnownZero|1) == Mask) + if ((KnownZero | APInt(VTBits, 1)) == Mask) return VTBits; // If we are subtracting one from a positive number, there is no carry // out of the result. - if (KnownZero & MVT::getIntVTSignBit(VT)) + if (KnownZero.isNegative()) return Tmp; } @@ -1503,18 +1658,18 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{ // Handle NEG. if (ConstantSDNode *CLHS = dyn_cast(Op.getOperand(0))) - if (CLHS->getValue() == 0) { - uint64_t KnownZero, KnownOne; - uint64_t Mask = MVT::getIntVTBitMask(VT); + if (CLHS->isNullValue()) { + APInt KnownZero, KnownOne; + APInt Mask = APInt::getAllOnesValue(VTBits); ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1); // If the input is known to be 0 or 1, the output is 0/-1, which is all // sign bits set. - if ((KnownZero|1) == Mask) + if ((KnownZero | APInt(VTBits, 1)) == Mask) return VTBits; // If the input is known to be positive (the sign bit is known clear), // the output of the NEG has the same number of sign bits as the input. - if (KnownZero & MVT::getIntVTSignBit(VT)) + if (KnownZero.isNegative()) return Tmp2; // Otherwise, we treat this like a SUB. @@ -1539,10 +1694,10 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{ switch (ExtType) { default: break; case ISD::SEXTLOAD: // '17' bits known - Tmp = MVT::getSizeInBits(LD->getLoadedVT()); + Tmp = MVT::getSizeInBits(LD->getMemoryVT()); return VTBits-Tmp+1; case ISD::ZEXTLOAD: // '16' bits known - Tmp = MVT::getSizeInBits(LD->getLoadedVT()); + Tmp = MVT::getSizeInBits(LD->getMemoryVT()); return VTBits-Tmp; } } @@ -1558,14 +1713,13 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{ // Finally, if we can prove that the top bits of the result are 0's or 1's, // use this information. - uint64_t KnownZero, KnownOne; - uint64_t Mask = MVT::getIntVTBitMask(VT); + APInt KnownZero, KnownOne; + APInt Mask = APInt::getAllOnesValue(VTBits); ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth); - uint64_t SignBit = MVT::getIntVTSignBit(VT); - if (KnownZero & SignBit) { // SignBit is 0 + if (KnownZero.isNegative()) { // sign bit is 0 Mask = KnownZero; - } else if (KnownOne & SignBit) { // SignBit is 1; + } else if (KnownOne.isNegative()) { // sign bit is 1; Mask = KnownOne; } else { // Nothing known. @@ -1574,11 +1728,21 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{ // Okay, we know that the sign bit in Mask is set. Use CLZ to determine // the number of identical bits in the top of the input value. - Mask ^= ~0ULL; - Mask <<= 64-VTBits; + Mask = ~Mask; + Mask <<= Mask.getBitWidth()-VTBits; // Return # leading zeros. We use 'min' here in case Val was zero before // shifting. We don't want to return '64' as for an i32 "0". - return std::min(VTBits, CountLeadingZeros_64(Mask)); + return std::min(VTBits, Mask.countLeadingZeros()); +} + + +bool SelectionDAG::isVerifiedDebugInfoDesc(SDOperand Op) const { + GlobalAddressSDNode *GA = dyn_cast(Op); + if (!GA) return false; + GlobalVariable *GV = dyn_cast(GA->getGlobal()); + if (!GV) return false; + MachineModuleInfo *MMI = getMachineModuleInfo(); + return MMI && MMI->hasDebugInfo() && MMI->isVerified(GV); } @@ -1599,127 +1763,84 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT) { SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, SDOperand Operand) { - unsigned Tmp1; // Constant fold unary operations with an integer constant operand. if (ConstantSDNode *C = dyn_cast(Operand.Val)) { - uint64_t Val = C->getValue(); + const APInt &Val = C->getAPIntValue(); + unsigned BitWidth = MVT::getSizeInBits(VT); switch (Opcode) { default: break; - case ISD::SIGN_EXTEND: return getConstant(C->getSignExtended(), VT); + case ISD::SIGN_EXTEND: + return getConstant(APInt(Val).sextOrTrunc(BitWidth), VT); case ISD::ANY_EXTEND: - case ISD::ZERO_EXTEND: return getConstant(Val, VT); - case ISD::TRUNCATE: return getConstant(Val, VT); + case ISD::ZERO_EXTEND: + case ISD::TRUNCATE: + return getConstant(APInt(Val).zextOrTrunc(BitWidth), VT); case ISD::UINT_TO_FP: case ISD::SINT_TO_FP: { const uint64_t zero[] = {0, 0}; - APFloat apf = APFloat(APInt(MVT::getSizeInBits(VT), 2, zero)); - (void)apf.convertFromZeroExtendedInteger(&Val, - MVT::getSizeInBits(Operand.getValueType()), - Opcode==ISD::SINT_TO_FP, - APFloat::rmNearestTiesToEven); + // No compile time operations on this type. + if (VT==MVT::ppcf128) + break; + APFloat apf = APFloat(APInt(BitWidth, 2, zero)); + (void)apf.convertFromAPInt(Val, + Opcode==ISD::SINT_TO_FP, + APFloat::rmNearestTiesToEven); return getConstantFP(apf, VT); } case ISD::BIT_CONVERT: if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) - return getConstantFP(BitsToFloat(Val), VT); + return getConstantFP(Val.bitsToFloat(), VT); else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) - return getConstantFP(BitsToDouble(Val), VT); + return getConstantFP(Val.bitsToDouble(), VT); break; case ISD::BSWAP: - switch(VT) { - default: assert(0 && "Invalid bswap!"); break; - case MVT::i16: return getConstant(ByteSwap_16((unsigned short)Val), VT); - case MVT::i32: return getConstant(ByteSwap_32((unsigned)Val), VT); - case MVT::i64: return getConstant(ByteSwap_64(Val), VT); - } - break; + return getConstant(Val.byteSwap(), VT); case ISD::CTPOP: - switch(VT) { - default: assert(0 && "Invalid ctpop!"); break; - case MVT::i1: return getConstant(Val != 0, VT); - case MVT::i8: - Tmp1 = (unsigned)Val & 0xFF; - return getConstant(CountPopulation_32(Tmp1), VT); - case MVT::i16: - Tmp1 = (unsigned)Val & 0xFFFF; - return getConstant(CountPopulation_32(Tmp1), VT); - case MVT::i32: - return getConstant(CountPopulation_32((unsigned)Val), VT); - case MVT::i64: - return getConstant(CountPopulation_64(Val), VT); - } + return getConstant(Val.countPopulation(), VT); case ISD::CTLZ: - switch(VT) { - default: assert(0 && "Invalid ctlz!"); break; - case MVT::i1: return getConstant(Val == 0, VT); - case MVT::i8: - Tmp1 = (unsigned)Val & 0xFF; - return getConstant(CountLeadingZeros_32(Tmp1)-24, VT); - case MVT::i16: - Tmp1 = (unsigned)Val & 0xFFFF; - return getConstant(CountLeadingZeros_32(Tmp1)-16, VT); - case MVT::i32: - return getConstant(CountLeadingZeros_32((unsigned)Val), VT); - case MVT::i64: - return getConstant(CountLeadingZeros_64(Val), VT); - } + return getConstant(Val.countLeadingZeros(), VT); case ISD::CTTZ: - switch(VT) { - default: assert(0 && "Invalid cttz!"); break; - case MVT::i1: return getConstant(Val == 0, VT); - case MVT::i8: - Tmp1 = (unsigned)Val | 0x100; - return getConstant(CountTrailingZeros_32(Tmp1), VT); - case MVT::i16: - Tmp1 = (unsigned)Val | 0x10000; - return getConstant(CountTrailingZeros_32(Tmp1), VT); - case MVT::i32: - return getConstant(CountTrailingZeros_32((unsigned)Val), VT); - case MVT::i64: - return getConstant(CountTrailingZeros_64(Val), VT); - } + return getConstant(Val.countTrailingZeros(), VT); } } // Constant fold unary operations with a floating point constant operand. if (ConstantFPSDNode *C = dyn_cast(Operand.Val)) { APFloat V = C->getValueAPF(); // make copy - switch (Opcode) { - case ISD::FNEG: - V.changeSign(); - return getConstantFP(V, VT); - case ISD::FABS: - V.clearSign(); - return getConstantFP(V, VT); - case ISD::FP_ROUND: - case ISD::FP_EXTEND: - // This can return overflow, underflow, or inexact; we don't care. - // FIXME need to be more flexible about rounding mode. - (void) V.convert(VT==MVT::f32 ? APFloat::IEEEsingle : - VT==MVT::f64 ? APFloat::IEEEdouble : - VT==MVT::f80 ? APFloat::x87DoubleExtended : - VT==MVT::f128 ? APFloat::IEEEquad : - APFloat::Bogus, - APFloat::rmNearestTiesToEven); - return getConstantFP(V, VT); - case ISD::FP_TO_SINT: - case ISD::FP_TO_UINT: { - integerPart x; - assert(integerPartWidth >= 64); - // FIXME need to be more flexible about rounding mode. - APFloat::opStatus s = V.convertToInteger(&x, 64U, - Opcode==ISD::FP_TO_SINT, - APFloat::rmTowardZero); - if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual + if (VT != MVT::ppcf128 && Operand.getValueType() != MVT::ppcf128) { + switch (Opcode) { + case ISD::FNEG: + V.changeSign(); + return getConstantFP(V, VT); + case ISD::FABS: + V.clearSign(); + return getConstantFP(V, VT); + case ISD::FP_ROUND: + case ISD::FP_EXTEND: + // This can return overflow, underflow, or inexact; we don't care. + // FIXME need to be more flexible about rounding mode. + (void)V.convert(*MVTToAPFloatSemantics(VT), + APFloat::rmNearestTiesToEven); + return getConstantFP(V, VT); + case ISD::FP_TO_SINT: + case ISD::FP_TO_UINT: { + integerPart x; + assert(integerPartWidth >= 64); + // FIXME need to be more flexible about rounding mode. + APFloat::opStatus s = V.convertToInteger(&x, 64U, + Opcode==ISD::FP_TO_SINT, + APFloat::rmTowardZero); + if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual + break; + return getConstant(x, VT); + } + case ISD::BIT_CONVERT: + if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) + return getConstant((uint32_t)V.convertToAPInt().getZExtValue(), VT); + else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) + return getConstant(V.convertToAPInt().getZExtValue(), VT); break; - return getConstant(x, VT); - } - case ISD::BIT_CONVERT: - if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) - return getConstant((uint32_t)V.convertToAPInt().getZExtValue(), VT); - else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) - return getConstant(V.convertToAPInt().getZExtValue(), VT); - break; + } } } @@ -1727,10 +1848,13 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, switch (Opcode) { case ISD::TokenFactor: return Operand; // Factor of one node? No factor. - case ISD::FP_ROUND: + case ISD::FP_ROUND: assert(0 && "Invalid method to make FP_ROUND node"); case ISD::FP_EXTEND: assert(MVT::isFloatingPoint(VT) && MVT::isFloatingPoint(Operand.getValueType()) && "Invalid FP cast!"); + if (Operand.getValueType() == VT) return Operand; // noop conversion. + if (Operand.getOpcode() == ISD::UNDEF) + return getNode(ISD::UNDEF, VT); break; case ISD::SIGN_EXTEND: assert(MVT::isInteger(VT) && MVT::isInteger(Operand.getValueType()) && @@ -1795,6 +1919,14 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, assert(MVT::isVector(VT) && !MVT::isVector(Operand.getValueType()) && MVT::getVectorElementType(VT) == Operand.getValueType() && "Illegal SCALAR_TO_VECTOR node!"); + if (OpOpcode == ISD::UNDEF) + return getNode(ISD::UNDEF, VT); + // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. + if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && + isa(Operand.getOperand(1)) && + Operand.getConstantOperandVal(1) == 0 && + Operand.getOperand(0).getValueType() == VT) + return Operand.getOperand(0); break; case ISD::FNEG: if (OpOpcode == ISD::FSUB) // -(X-Y) -> (Y-X) @@ -1831,15 +1963,36 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, SDOperand N1, SDOperand N2) { -#ifndef NDEBUG + ConstantSDNode *N1C = dyn_cast(N1.Val); + ConstantSDNode *N2C = dyn_cast(N2.Val); switch (Opcode) { + default: break; case ISD::TokenFactor: assert(VT == MVT::Other && N1.getValueType() == MVT::Other && N2.getValueType() == MVT::Other && "Invalid token factor!"); + // Fold trivial token factors. + if (N1.getOpcode() == ISD::EntryToken) return N2; + if (N2.getOpcode() == ISD::EntryToken) return N1; break; case ISD::AND: + assert(MVT::isInteger(VT) && N1.getValueType() == N2.getValueType() && + N1.getValueType() == VT && "Binary operator types must match!"); + // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's + // worth handling here. + if (N2C && N2C->isNullValue()) + return N2; + if (N2C && N2C->isAllOnesValue()) // X & -1 -> X + return N1; + break; case ISD::OR: case ISD::XOR: + assert(MVT::isInteger(VT) && N1.getValueType() == N2.getValueType() && + N1.getValueType() == VT && "Binary operator types must match!"); + // (X ^| 0) -> X. This commonly occurs when legalizing i64 values, so it's + // worth handling here. + if (N2C && N2C->isNullValue()) + return N1; + break; case ISD::UDIV: case ISD::UREM: case ISD::MULHU: @@ -1851,8 +2004,6 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, case ISD::MUL: case ISD::SDIV: case ISD::SREM: - assert(MVT::isInteger(N1.getValueType()) && "Should use F* for FP ops"); - // fall through. case ISD::FADD: case ISD::FSUB: case ISD::FMUL: @@ -1884,10 +2035,27 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, "Cannot FP_ROUND_INREG integer types"); assert(MVT::getSizeInBits(EVT) <= MVT::getSizeInBits(VT) && "Not rounding down!"); + if (cast(N2)->getVT() == VT) return N1; // Not actually rounding. break; } + case ISD::FP_ROUND: + assert(MVT::isFloatingPoint(VT) && + MVT::isFloatingPoint(N1.getValueType()) && + MVT::getSizeInBits(VT) <= MVT::getSizeInBits(N1.getValueType()) && + isa(N2) && "Invalid FP_ROUND!"); + if (N1.getValueType() == VT) return N1; // noop conversion. + break; case ISD::AssertSext: - case ISD::AssertZext: + case ISD::AssertZext: { + MVT::ValueType EVT = cast(N2)->getVT(); + assert(VT == N1.getValueType() && "Not an inreg extend!"); + assert(MVT::isInteger(VT) && MVT::isInteger(EVT) && + "Cannot *_EXTEND_INREG FP types"); + assert(MVT::getSizeInBits(EVT) <= MVT::getSizeInBits(VT) && + "Not extending!"); + if (VT == EVT) return N1; // noop assertion. + break; + } case ISD::SIGN_EXTEND_INREG: { MVT::ValueType EVT = cast(N2)->getVT(); assert(VT == N1.getValueType() && "Not an inreg extend!"); @@ -1895,55 +2063,104 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, "Cannot *_EXTEND_INREG FP types"); assert(MVT::getSizeInBits(EVT) <= MVT::getSizeInBits(VT) && "Not extending!"); + if (EVT == VT) return N1; // Not actually extending + + if (N1C) { + APInt Val = N1C->getAPIntValue(); + unsigned FromBits = MVT::getSizeInBits(cast(N2)->getVT()); + Val <<= Val.getBitWidth()-FromBits; + Val = Val.ashr(Val.getBitWidth()-FromBits); + return getConstant(Val, VT); + } + break; } + case ISD::EXTRACT_VECTOR_ELT: + assert(N2C && "Bad EXTRACT_VECTOR_ELT!"); - default: break; + // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. + if (N1.getOpcode() == ISD::UNDEF) + return getNode(ISD::UNDEF, VT); + + // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is + // expanding copies of large vectors from registers. + if (N1.getOpcode() == ISD::CONCAT_VECTORS && + N1.getNumOperands() > 0) { + unsigned Factor = + MVT::getVectorNumElements(N1.getOperand(0).getValueType()); + return getNode(ISD::EXTRACT_VECTOR_ELT, VT, + N1.getOperand(N2C->getValue() / Factor), + getConstant(N2C->getValue() % Factor, N2.getValueType())); + } + + // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is + // expanding large vector constants. + if (N1.getOpcode() == ISD::BUILD_VECTOR) + return N1.getOperand(N2C->getValue()); + + // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector + // operations are lowered to scalars. + if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) + if (ConstantSDNode *IEC = dyn_cast(N1.getOperand(2))) { + if (IEC == N2C) + return N1.getOperand(1); + else + return getNode(ISD::EXTRACT_VECTOR_ELT, VT, N1.getOperand(0), N2); + } + break; + case ISD::EXTRACT_ELEMENT: + assert(N2C && (unsigned)N2C->getValue() < 2 && "Bad EXTRACT_ELEMENT!"); + assert(!MVT::isVector(N1.getValueType()) && + MVT::isInteger(N1.getValueType()) && + !MVT::isVector(VT) && MVT::isInteger(VT) && + "EXTRACT_ELEMENT only applies to integers!"); + + // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding + // 64-bit integers into 32-bit parts. Instead of building the extract of + // the BUILD_PAIR, only to have legalize rip it apart, just do it now. + if (N1.getOpcode() == ISD::BUILD_PAIR) + return N1.getOperand(N2C->getValue()); + + // EXTRACT_ELEMENT of a constant int is also very common. + if (ConstantSDNode *C = dyn_cast(N1)) { + unsigned ElementSize = MVT::getSizeInBits(VT); + unsigned Shift = ElementSize * N2C->getValue(); + APInt ShiftedVal = C->getAPIntValue().lshr(Shift); + return getConstant(ShiftedVal.trunc(ElementSize), VT); + } + break; + case ISD::EXTRACT_SUBVECTOR: + if (N1.getValueType() == VT) // Trivial extraction. + return N1; + break; } -#endif - ConstantSDNode *N1C = dyn_cast(N1.Val); - ConstantSDNode *N2C = dyn_cast(N2.Val); if (N1C) { - if (Opcode == ISD::SIGN_EXTEND_INREG) { - int64_t Val = N1C->getValue(); - unsigned FromBits = MVT::getSizeInBits(cast(N2)->getVT()); - Val <<= 64-FromBits; - Val >>= 64-FromBits; - return getConstant(Val, VT); - } - if (N2C) { - uint64_t C1 = N1C->getValue(), C2 = N2C->getValue(); + APInt C1 = N1C->getAPIntValue(), C2 = N2C->getAPIntValue(); switch (Opcode) { case ISD::ADD: return getConstant(C1 + C2, VT); case ISD::SUB: return getConstant(C1 - C2, VT); case ISD::MUL: return getConstant(C1 * C2, VT); case ISD::UDIV: - if (C2) return getConstant(C1 / C2, VT); + if (C2.getBoolValue()) return getConstant(C1.udiv(C2), VT); break; case ISD::UREM : - if (C2) return getConstant(C1 % C2, VT); + if (C2.getBoolValue()) return getConstant(C1.urem(C2), VT); break; case ISD::SDIV : - if (C2) return getConstant(N1C->getSignExtended() / - N2C->getSignExtended(), VT); + if (C2.getBoolValue()) return getConstant(C1.sdiv(C2), VT); break; case ISD::SREM : - if (C2) return getConstant(N1C->getSignExtended() % - N2C->getSignExtended(), VT); + if (C2.getBoolValue()) return getConstant(C1.srem(C2), VT); break; case ISD::AND : return getConstant(C1 & C2, VT); case ISD::OR : return getConstant(C1 | C2, VT); case ISD::XOR : return getConstant(C1 ^ C2, VT); case ISD::SHL : return getConstant(C1 << C2, VT); - case ISD::SRL : return getConstant(C1 >> C2, VT); - case ISD::SRA : return getConstant(N1C->getSignExtended() >>(int)C2, VT); - case ISD::ROTL : - return getConstant((C1 << C2) | (C1 >> (MVT::getSizeInBits(VT) - C2)), - VT); - case ISD::ROTR : - return getConstant((C1 >> C2) | (C1 << (MVT::getSizeInBits(VT) - C2)), - VT); + case ISD::SRL : return getConstant(C1.lshr(C2), VT); + case ISD::SRA : return getConstant(C1.ashr(C2), VT); + case ISD::ROTL : return getConstant(C1.rotl(C2), VT); + case ISD::ROTR : return getConstant(C1.rotr(C2), VT); default: break; } } else { // Cannonicalize constant to RHS if commutative @@ -1954,16 +2171,21 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, } } + // Constant fold FP operations. ConstantFPSDNode *N1CFP = dyn_cast(N1.Val); ConstantFPSDNode *N2CFP = dyn_cast(N2.Val); if (N1CFP) { - if (N2CFP) { + if (!N2CFP && isCommutativeBinOp(Opcode)) { + // Cannonicalize constant to RHS if commutative + std::swap(N1CFP, N2CFP); + std::swap(N1, N2); + } else if (N2CFP && VT != MVT::ppcf128) { APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); APFloat::opStatus s; switch (Opcode) { case ISD::FADD: s = V1.add(V2, APFloat::rmNearestTiesToEven); - if (s!=APFloat::opInvalidOp) + if (s != APFloat::opInvalidOp) return getConstantFP(V1, VT); break; case ISD::FSUB: @@ -1991,11 +2213,6 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, return getConstantFP(V1, VT); default: break; } - } else { // Cannonicalize constant to RHS if commutative - if (isCommutativeBinOp(Opcode)) { - std::swap(N1CFP, N2CFP); - std::swap(N1, N2); - } } } @@ -2031,6 +2248,12 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, // Fold a bunch of operators when the RHS is undef. if (N2.getOpcode() == ISD::UNDEF) { switch (Opcode) { + case ISD::XOR: + if (N1.getOpcode() == ISD::UNDEF) + // Handle undef ^ undef -> 0 special case. This is a common + // idiom (misuse). + return getConstant(0, VT); + // fallthrough case ISD::ADD: case ISD::ADDC: case ISD::ADDE: @@ -2044,7 +2267,6 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, case ISD::SDIV: case ISD::UREM: case ISD::SREM: - case ISD::XOR: return N2; // fold op(arg1, undef) -> undef case ISD::MUL: case ISD::AND: @@ -2066,102 +2288,6 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, } } - // Fold operations. - switch (Opcode) { - case ISD::TokenFactor: - // Fold trivial token factors. - if (N1.getOpcode() == ISD::EntryToken) return N2; - if (N2.getOpcode() == ISD::EntryToken) return N1; - break; - - case ISD::AND: - // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's - // worth handling here. - if (N2C && N2C->getValue() == 0) - return N2; - break; - case ISD::OR: - case ISD::XOR: - // (X ^| 0) -> X. This commonly occurs when legalizing i64 values, so it's - // worth handling here. - if (N2C && N2C->getValue() == 0) - return N1; - break; - case ISD::FP_ROUND_INREG: - if (cast(N2)->getVT() == VT) return N1; // Not actually rounding. - break; - case ISD::SIGN_EXTEND_INREG: { - MVT::ValueType EVT = cast(N2)->getVT(); - if (EVT == VT) return N1; // Not actually extending - break; - } - case ISD::EXTRACT_VECTOR_ELT: - assert(N2C && "Bad EXTRACT_VECTOR_ELT!"); - - // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is - // expanding copies of large vectors from registers. - if (N1.getOpcode() == ISD::CONCAT_VECTORS && - N1.getNumOperands() > 0) { - unsigned Factor = - MVT::getVectorNumElements(N1.getOperand(0).getValueType()); - return getNode(ISD::EXTRACT_VECTOR_ELT, VT, - N1.getOperand(N2C->getValue() / Factor), - getConstant(N2C->getValue() % Factor, N2.getValueType())); - } - - // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is - // expanding large vector constants. - if (N1.getOpcode() == ISD::BUILD_VECTOR) - return N1.getOperand(N2C->getValue()); - - // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector - // operations are lowered to scalars. - if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) - if (ConstantSDNode *IEC = dyn_cast(N1.getOperand(2))) { - if (IEC == N2C) - return N1.getOperand(1); - else - return getNode(ISD::EXTRACT_VECTOR_ELT, VT, N1.getOperand(0), N2); - } - break; - case ISD::EXTRACT_ELEMENT: - assert(N2C && (unsigned)N2C->getValue() < 2 && "Bad EXTRACT_ELEMENT!"); - - // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding - // 64-bit integers into 32-bit parts. Instead of building the extract of - // the BUILD_PAIR, only to have legalize rip it apart, just do it now. - if (N1.getOpcode() == ISD::BUILD_PAIR) - return N1.getOperand(N2C->getValue()); - - // EXTRACT_ELEMENT of a constant int is also very common. - if (ConstantSDNode *C = dyn_cast(N1)) { - unsigned Shift = MVT::getSizeInBits(VT) * N2C->getValue(); - return getConstant(C->getValue() >> Shift, VT); - } - break; - - // FIXME: figure out how to safely handle things like - // int foo(int x) { return 1 << (x & 255); } - // int bar() { return foo(256); } -#if 0 - case ISD::SHL: - case ISD::SRL: - case ISD::SRA: - if (N2.getOpcode() == ISD::SIGN_EXTEND_INREG && - cast(N2.getOperand(1))->getVT() != MVT::i1) - return getNode(Opcode, VT, N1, N2.getOperand(0)); - else if (N2.getOpcode() == ISD::AND) - if (ConstantSDNode *AndRHS = dyn_cast(N2.getOperand(1))) { - // If the and is only masking out bits that cannot effect the shift, - // eliminate the and. - unsigned NumBits = MVT::getSizeInBits(VT); - if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) - return getNode(Opcode, VT, N1, N2.getOperand(0)); - } - break; -#endif - } - // Memoize this node if possible. SDNode *N; SDVTList VTs = getVTList(VT); @@ -2195,20 +2321,22 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, break; } case ISD::SELECT: - if (N1C) - if (N1C->getValue()) + if (N1C) { + if (N1C->getValue()) return N2; // select true, X, Y -> X else return N3; // select false, X, Y -> Y + } if (N2 == N3) return N2; // select C, X, X -> X break; case ISD::BRCOND: - if (N2C) + if (N2C) { if (N2C->getValue()) // Unconditional branch return getNode(ISD::BR, MVT::Other, N1, N3); else return N1; // Never-taken branch + } break; case ISD::VECTOR_SHUFFLE: assert(VT == N1.getValueType() && VT == N2.getValueType() && @@ -2257,65 +2385,74 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, return getNode(Opcode, VT, Ops, 5); } -SDOperand SelectionDAG::getLoad(MVT::ValueType VT, - SDOperand Chain, SDOperand Ptr, - const Value *SV, int SVOffset, - bool isVolatile, unsigned Alignment) { - if (Alignment == 0) { // Ensure that codegen never sees alignment 0 - const Type *Ty = 0; - if (VT != MVT::iPTR) { - Ty = MVT::getTypeForValueType(VT); - } else if (SV) { - const PointerType *PT = dyn_cast(SV->getType()); - assert(PT && "Value for load must be a pointer"); - Ty = PT->getElementType(); - } - assert(Ty && "Could not get type information for load"); - Alignment = TLI.getTargetData()->getABITypeAlignment(Ty); - } - SDVTList VTs = getVTList(VT, MVT::Other); - SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType()); - SDOperand Ops[] = { Chain, Ptr, Undef }; +SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dest, + SDOperand Src, SDOperand Size, + SDOperand Align, + SDOperand AlwaysInline) { + SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline }; + return getNode(ISD::MEMCPY, MVT::Other, Ops, 6); +} + +SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dest, + SDOperand Src, SDOperand Size, + SDOperand Align, + SDOperand AlwaysInline) { + SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline }; + return getNode(ISD::MEMMOVE, MVT::Other, Ops, 6); +} + +SDOperand SelectionDAG::getMemset(SDOperand Chain, SDOperand Dest, + SDOperand Src, SDOperand Size, + SDOperand Align, + SDOperand AlwaysInline) { + SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline }; + return getNode(ISD::MEMSET, MVT::Other, Ops, 6); +} + +SDOperand SelectionDAG::getAtomic(unsigned Opcode, SDOperand Chain, + SDOperand Ptr, SDOperand Cmp, + SDOperand Swp, MVT::ValueType VT) { + assert(Opcode == ISD::ATOMIC_LCS && "Invalid Atomic Op"); + assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); + SDVTList VTs = getVTList(Cmp.getValueType(), MVT::Other); FoldingSetNodeID ID; - AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); - ID.AddInteger(ISD::UNINDEXED); - ID.AddInteger(ISD::NON_EXTLOAD); + SDOperand Ops[] = {Chain, Ptr, Cmp, Swp}; + AddNodeIDNode(ID, Opcode, VTs, Ops, 4); ID.AddInteger((unsigned int)VT); - ID.AddPointer(SV); - ID.AddInteger(SVOffset); - ID.AddInteger(Alignment); - ID.AddInteger(isVolatile); - void *IP = 0; + void* IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDOperand(E, 0); - SDNode *N = new LoadSDNode(Ops, VTs, ISD::UNINDEXED, - ISD::NON_EXTLOAD, VT, SV, SVOffset, Alignment, - isVolatile); + SDNode* N = new AtomicSDNode(Opcode, VTs, Chain, Ptr, Cmp, Swp, VT); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDOperand(N, 0); } -SDOperand SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, MVT::ValueType VT, - SDOperand Chain, SDOperand Ptr, - const Value *SV, - int SVOffset, MVT::ValueType EVT, - bool isVolatile, unsigned Alignment) { - // If they are asking for an extending load from/to the same thing, return a - // normal load. - if (VT == EVT) - ExtType = ISD::NON_EXTLOAD; - - if (MVT::isVector(VT)) - assert(EVT == MVT::getVectorElementType(VT) && "Invalid vector extload!"); - else - assert(MVT::getSizeInBits(EVT) < MVT::getSizeInBits(VT) && - "Should only be an extending load, not truncating!"); - assert((ExtType == ISD::EXTLOAD || MVT::isInteger(VT)) && - "Cannot sign/zero extend a FP/Vector load!"); - assert(MVT::isInteger(VT) == MVT::isInteger(EVT) && - "Cannot convert from FP to Int or Int -> FP!"); +SDOperand SelectionDAG::getAtomic(unsigned Opcode, SDOperand Chain, + SDOperand Ptr, SDOperand Val, + MVT::ValueType VT) { + assert((Opcode == ISD::ATOMIC_LAS || Opcode == ISD::ATOMIC_SWAP) + && "Invalid Atomic Op"); + SDVTList VTs = getVTList(Val.getValueType(), MVT::Other); + FoldingSetNodeID ID; + SDOperand Ops[] = {Chain, Ptr, Val}; + AddNodeIDNode(ID, Opcode, VTs, Ops, 3); + ID.AddInteger((unsigned int)VT); + void* IP = 0; + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) + return SDOperand(E, 0); + SDNode* N = new AtomicSDNode(Opcode, VTs, Chain, Ptr, Val, VT); + CSEMap.InsertNode(N, IP); + AllNodes.push_back(N); + return SDOperand(N, 0); +} +SDOperand +SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, + MVT::ValueType VT, SDOperand Chain, + SDOperand Ptr, SDOperand Offset, + const Value *SV, int SVOffset, MVT::ValueType EVT, + bool isVolatile, unsigned Alignment) { if (Alignment == 0) { // Ensure that codegen never sees alignment 0 const Type *Ty = 0; if (VT != MVT::iPTR) { @@ -2324,60 +2461,81 @@ SDOperand SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, MVT::ValueType VT, const PointerType *PT = dyn_cast(SV->getType()); assert(PT && "Value for load must be a pointer"); Ty = PT->getElementType(); - } + } assert(Ty && "Could not get type information for load"); Alignment = TLI.getTargetData()->getABITypeAlignment(Ty); } - SDVTList VTs = getVTList(VT, MVT::Other); - SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType()); - SDOperand Ops[] = { Chain, Ptr, Undef }; + + if (VT == EVT) { + ExtType = ISD::NON_EXTLOAD; + } else if (ExtType == ISD::NON_EXTLOAD) { + assert(VT == EVT && "Non-extending load from different memory type!"); + } else { + // Extending load. + if (MVT::isVector(VT)) + assert(EVT == MVT::getVectorElementType(VT) && "Invalid vector extload!"); + else + assert(MVT::getSizeInBits(EVT) < MVT::getSizeInBits(VT) && + "Should only be an extending load, not truncating!"); + assert((ExtType == ISD::EXTLOAD || MVT::isInteger(VT)) && + "Cannot sign/zero extend a FP/Vector load!"); + assert(MVT::isInteger(VT) == MVT::isInteger(EVT) && + "Cannot convert from FP to Int or Int -> FP!"); + } + + bool Indexed = AM != ISD::UNINDEXED; + assert(Indexed || Offset.getOpcode() == ISD::UNDEF && + "Unindexed load with an offset!"); + + SDVTList VTs = Indexed ? + getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); + SDOperand Ops[] = { Chain, Ptr, Offset }; FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); - ID.AddInteger(ISD::UNINDEXED); + ID.AddInteger(AM); ID.AddInteger(ExtType); ID.AddInteger((unsigned int)EVT); - ID.AddPointer(SV); - ID.AddInteger(SVOffset); ID.AddInteger(Alignment); ID.AddInteger(isVolatile); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDOperand(E, 0); - SDNode *N = new LoadSDNode(Ops, VTs, ISD::UNINDEXED, ExtType, EVT, - SV, SVOffset, Alignment, isVolatile); + SDNode *N = new LoadSDNode(Ops, VTs, AM, ExtType, EVT, SV, SVOffset, + Alignment, isVolatile); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); return SDOperand(N, 0); } +SDOperand SelectionDAG::getLoad(MVT::ValueType VT, + SDOperand Chain, SDOperand Ptr, + const Value *SV, int SVOffset, + bool isVolatile, unsigned Alignment) { + SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType()); + return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, Chain, Ptr, Undef, + SV, SVOffset, VT, isVolatile, Alignment); +} + +SDOperand SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, MVT::ValueType VT, + SDOperand Chain, SDOperand Ptr, + const Value *SV, + int SVOffset, MVT::ValueType EVT, + bool isVolatile, unsigned Alignment) { + SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType()); + return getLoad(ISD::UNINDEXED, ExtType, VT, Chain, Ptr, Undef, + SV, SVOffset, EVT, isVolatile, Alignment); +} + SDOperand SelectionDAG::getIndexedLoad(SDOperand OrigLoad, SDOperand Base, SDOperand Offset, ISD::MemIndexedMode AM) { LoadSDNode *LD = cast(OrigLoad); assert(LD->getOffset().getOpcode() == ISD::UNDEF && "Load is already a indexed load!"); - MVT::ValueType VT = OrigLoad.getValueType(); - SDVTList VTs = getVTList(VT, Base.getValueType(), MVT::Other); - SDOperand Ops[] = { LD->getChain(), Base, Offset }; - FoldingSetNodeID ID; - AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); - ID.AddInteger(AM); - ID.AddInteger(LD->getExtensionType()); - ID.AddInteger((unsigned int)(LD->getLoadedVT())); - ID.AddPointer(LD->getSrcValue()); - ID.AddInteger(LD->getSrcValueOffset()); - ID.AddInteger(LD->getAlignment()); - ID.AddInteger(LD->isVolatile()); - void *IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) - return SDOperand(E, 0); - SDNode *N = new LoadSDNode(Ops, VTs, AM, - LD->getExtensionType(), LD->getLoadedVT(), - LD->getSrcValue(), LD->getSrcValueOffset(), - LD->getAlignment(), LD->isVolatile()); - CSEMap.InsertNode(N, IP); - AllNodes.push_back(N); - return SDOperand(N, 0); + return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), + LD->getChain(), Base, Offset, LD->getSrcValue(), + LD->getSrcValueOffset(), LD->getMemoryVT(), + LD->isVolatile(), LD->getAlignment()); } SDOperand SelectionDAG::getStore(SDOperand Chain, SDOperand Val, @@ -2405,8 +2563,6 @@ SDOperand SelectionDAG::getStore(SDOperand Chain, SDOperand Val, ID.AddInteger(ISD::UNINDEXED); ID.AddInteger(false); ID.AddInteger((unsigned int)VT); - ID.AddPointer(SV); - ID.AddInteger(SVOffset); ID.AddInteger(Alignment); ID.AddInteger(isVolatile); void *IP = 0; @@ -2424,7 +2580,9 @@ SDOperand SelectionDAG::getTruncStore(SDOperand Chain, SDOperand Val, int SVOffset, MVT::ValueType SVT, bool isVolatile, unsigned Alignment) { MVT::ValueType VT = Val.getValueType(); - bool isTrunc = VT != SVT; + + if (VT == SVT) + return getStore(Chain, Val, Ptr, SV, SVOffset, isVolatile, Alignment); assert(MVT::getSizeInBits(VT) > MVT::getSizeInBits(SVT) && "Not a truncation?"); @@ -2449,16 +2607,14 @@ SDOperand SelectionDAG::getTruncStore(SDOperand Chain, SDOperand Val, FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); ID.AddInteger(ISD::UNINDEXED); - ID.AddInteger(isTrunc); + ID.AddInteger(1); ID.AddInteger((unsigned int)SVT); - ID.AddPointer(SV); - ID.AddInteger(SVOffset); ID.AddInteger(Alignment); ID.AddInteger(isVolatile); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDOperand(E, 0); - SDNode *N = new StoreSDNode(Ops, VTs, ISD::UNINDEXED, isTrunc, + SDNode *N = new StoreSDNode(Ops, VTs, ISD::UNINDEXED, true, SVT, SV, SVOffset, Alignment, isVolatile); CSEMap.InsertNode(N, IP); AllNodes.push_back(N); @@ -2477,16 +2633,14 @@ SelectionDAG::getIndexedStore(SDOperand OrigStore, SDOperand Base, AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); ID.AddInteger(AM); ID.AddInteger(ST->isTruncatingStore()); - ID.AddInteger((unsigned int)(ST->getStoredVT())); - ID.AddPointer(ST->getSrcValue()); - ID.AddInteger(ST->getSrcValueOffset()); + ID.AddInteger((unsigned int)(ST->getMemoryVT())); ID.AddInteger(ST->getAlignment()); ID.AddInteger(ST->isVolatile()); void *IP = 0; if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) return SDOperand(E, 0); SDNode *N = new StoreSDNode(Ops, VTs, AM, - ST->isTruncatingStore(), ST->getStoredVT(), + ST->isTruncatingStore(), ST->getMemoryVT(), ST->getSrcValue(), ST->getSrcValueOffset(), ST->getAlignment(), ST->isVolatile()); CSEMap.InsertNode(N, IP); @@ -2743,9 +2897,10 @@ UpdateNodeOperands(SDOperand InN, SDOperand Op) { RemoveNodeFromCSEMaps(N); // Now we update the operands. - N->OperandList[0].Val->removeUser(N); - Op.Val->addUser(N); + N->OperandList[0].Val->removeUser(0, N); N->OperandList[0] = Op; + N->OperandList[0].setUser(N); + Op.Val->addUser(0, N); // If this gets put into a CSE map, add it. if (InsertPos) CSEMap.InsertNode(N, InsertPos); @@ -2772,14 +2927,16 @@ UpdateNodeOperands(SDOperand InN, SDOperand Op1, SDOperand Op2) { // Now we update the operands. if (N->OperandList[0] != Op1) { - N->OperandList[0].Val->removeUser(N); - Op1.Val->addUser(N); + N->OperandList[0].Val->removeUser(0, N); N->OperandList[0] = Op1; + N->OperandList[0].setUser(N); + Op1.Val->addUser(0, N); } if (N->OperandList[1] != Op2) { - N->OperandList[1].Val->removeUser(N); - Op2.Val->addUser(N); + N->OperandList[1].Val->removeUser(1, N); N->OperandList[1] = Op2; + N->OperandList[1].setUser(N); + Op2.Val->addUser(1, N); } // If this gets put into a CSE map, add it. @@ -2807,7 +2964,6 @@ UpdateNodeOperands(SDOperand N, SDOperand Op1, SDOperand Op2, return UpdateNodeOperands(N, Ops, 5); } - SDOperand SelectionDAG:: UpdateNodeOperands(SDOperand InN, SDOperand *Ops, unsigned NumOps) { SDNode *N = InN.Val; @@ -2838,9 +2994,10 @@ UpdateNodeOperands(SDOperand InN, SDOperand *Ops, unsigned NumOps) { // Now we update the operands. for (unsigned i = 0; i != NumOps; ++i) { if (N->OperandList[i] != Ops[i]) { - N->OperandList[i].Val->removeUser(N); - Ops[i].Val->addUser(N); + N->OperandList[i].Val->removeUser(i, N); N->OperandList[i] = Ops[i]; + N->OperandList[i].setUser(N); + Ops[i].Val->addUser(i, N); } } @@ -2849,7 +3006,6 @@ UpdateNodeOperands(SDOperand InN, SDOperand *Ops, unsigned NumOps) { return InN; } - /// MorphNodeTo - This frees the operands of the current node, resets the /// opcode, types, and operands to the specified value. This should only be /// used by the SelectionDAG class. @@ -2862,13 +3018,14 @@ void SDNode::MorphNodeTo(unsigned Opc, SDVTList L, // Clear the operands list, updating used nodes to remove this from their // use list. for (op_iterator I = op_begin(), E = op_end(); I != E; ++I) - I->Val->removeUser(this); + I->Val->removeUser(std::distance(op_begin(), I), this); // If NumOps is larger than the # of operands we currently have, reallocate // the operand list. if (NumOps > NumOperands) { - if (OperandsNeedDelete) + if (OperandsNeedDelete) { delete [] OperandList; + } OperandList = new SDOperand[NumOps]; OperandsNeedDelete = true; } @@ -2878,8 +3035,10 @@ void SDNode::MorphNodeTo(unsigned Opc, SDVTList L, for (unsigned i = 0, e = NumOps; i != e; ++i) { OperandList[i] = Ops[i]; + OperandList[i].setUser(this); SDNode *N = OperandList[i].Val; - N->Uses.push_back(this); + N->addUser(i, this); + ++N->UsesSize; } } @@ -3121,40 +3280,67 @@ SDNode *SelectionDAG::getTargetNode(unsigned Opcode, Ops, NumOps).Val; } +/// getNodeIfExists - Get the specified node if it's already available, or +/// else return NULL. +SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, + const SDOperand *Ops, unsigned NumOps) { + if (VTList.VTs[VTList.NumVTs-1] != MVT::Flag) { + FoldingSetNodeID ID; + AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); + void *IP = 0; + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) + return E; + } + return NULL; +} + + /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. /// This can cause recursive merging of nodes in the DAG. /// -/// This version assumes From/To have a single result value. +/// This version assumes From has a single result value. /// -void SelectionDAG::ReplaceAllUsesWith(SDOperand FromN, SDOperand ToN, - std::vector *Deleted) { - SDNode *From = FromN.Val, *To = ToN.Val; - assert(From->getNumValues() == 1 && To->getNumValues() == 1 && +void SelectionDAG::ReplaceAllUsesWith(SDOperand FromN, SDOperand To, + DAGUpdateListener *UpdateListener) { + SDNode *From = FromN.Val; + assert(From->getNumValues() == 1 && FromN.ResNo == 0 && "Cannot replace with this method!"); - assert(From != To && "Cannot replace uses of with self"); - + assert(From != To.Val && "Cannot replace uses of with self"); + + SmallSetVector Users; while (!From->use_empty()) { - // Process users until they are all gone. - SDNode *U = *From->use_begin(); - + SDNode::use_iterator UI = From->use_begin(); + SDNode *U = UI->getUser(); + + // Remember that this node is about to morph. + if (Users.count(U)) + continue; + Users.insert(U); // This node is about to morph, remove its old self from the CSE maps. RemoveNodeFromCSEMaps(U); - - for (SDOperand *I = U->OperandList, *E = U->OperandList+U->NumOperands; - I != E; ++I) + int operandNum = 0; + for (SDNode::op_iterator I = U->op_begin(), E = U->op_end(); + I != E; ++I, ++operandNum) if (I->Val == From) { - From->removeUser(U); - I->Val = To; - To->addUser(U); - } + From->removeUser(operandNum, U); + *I = To; + I->setUser(U); + To.Val->addUser(operandNum, U); + } // Now that we have modified U, add it back to the CSE maps. If it already // exists there, recursively merge the results together. if (SDNode *Existing = AddNonLeafNodeToCSEMaps(U)) { - ReplaceAllUsesWith(U, Existing, Deleted); - // U is now dead. - if (Deleted) Deleted->push_back(U); + ReplaceAllUsesWith(U, Existing, UpdateListener); + // U is now dead. Inform the listener if it exists and delete it. + if (UpdateListener) + UpdateListener->NodeDeleted(U); DeleteNodeNotInCSEMaps(U); + } else { + // If the node doesn't already exist, we updated it. Inform a listener if + // it exists. + if (UpdateListener) + UpdateListener->NodeUpdated(U); } } } @@ -3166,37 +3352,47 @@ void SelectionDAG::ReplaceAllUsesWith(SDOperand FromN, SDOperand ToN, /// values. /// void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To, - std::vector *Deleted) { + DAGUpdateListener *UpdateListener) { assert(From != To && "Cannot replace uses of with self"); assert(From->getNumValues() == To->getNumValues() && "Cannot use this version of ReplaceAllUsesWith!"); - if (From->getNumValues() == 1) { // If possible, use the faster version. - ReplaceAllUsesWith(SDOperand(From, 0), SDOperand(To, 0), Deleted); - return; - } + if (From->getNumValues() == 1) // If possible, use the faster version. + return ReplaceAllUsesWith(SDOperand(From, 0), SDOperand(To, 0), + UpdateListener); + SmallSetVector Users; while (!From->use_empty()) { - // Process users until they are all gone. - SDNode *U = *From->use_begin(); - + SDNode::use_iterator UI = From->use_begin(); + SDNode *U = UI->getUser(); + + // Remember that this node is about to morph. + if (Users.count(U)) + continue; + Users.insert(U); // This node is about to morph, remove its old self from the CSE maps. RemoveNodeFromCSEMaps(U); - - for (SDOperand *I = U->OperandList, *E = U->OperandList+U->NumOperands; - I != E; ++I) + int operandNum = 0; + for (SDNode::op_iterator I = U->op_begin(), E = U->op_end(); + I != E; ++I, ++operandNum) if (I->Val == From) { - From->removeUser(U); + From->removeUser(operandNum, U); I->Val = To; - To->addUser(U); + To->addUser(operandNum, U); } - + // Now that we have modified U, add it back to the CSE maps. If it already // exists there, recursively merge the results together. if (SDNode *Existing = AddNonLeafNodeToCSEMaps(U)) { - ReplaceAllUsesWith(U, Existing, Deleted); - // U is now dead. - if (Deleted) Deleted->push_back(U); + ReplaceAllUsesWith(U, Existing, UpdateListener); + // U is now dead. Inform the listener if it exists and delete it. + if (UpdateListener) + UpdateListener->NodeDeleted(U); DeleteNodeNotInCSEMaps(U); + } else { + // If the node doesn't already exist, we updated it. Inform a listener if + // it exists. + if (UpdateListener) + UpdateListener->NodeUpdated(U); } } } @@ -3208,61 +3404,102 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To, /// number and types of values returned by From. void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDOperand *To, - std::vector *Deleted) { - if (From->getNumValues() == 1 && To[0].Val->getNumValues() == 1) { - // Degenerate case handled above. - ReplaceAllUsesWith(SDOperand(From, 0), To[0], Deleted); - return; - } + DAGUpdateListener *UpdateListener) { + if (From->getNumValues() == 1) // Handle the simple case efficiently. + return ReplaceAllUsesWith(SDOperand(From, 0), To[0], UpdateListener); + SmallSetVector Users; while (!From->use_empty()) { - // Process users until they are all gone. - SDNode *U = *From->use_begin(); - + SDNode::use_iterator UI = From->use_begin(); + SDNode *U = UI->getUser(); + + // Remember that this node is about to morph. + if (Users.count(U)) + continue; + Users.insert(U); // This node is about to morph, remove its old self from the CSE maps. RemoveNodeFromCSEMaps(U); - - for (SDOperand *I = U->OperandList, *E = U->OperandList+U->NumOperands; - I != E; ++I) + int operandNum = 0; + for (SDNode::op_iterator I = U->op_begin(), E = U->op_end(); + I != E; ++I, ++operandNum) if (I->Val == From) { const SDOperand &ToOp = To[I->ResNo]; - From->removeUser(U); + From->removeUser(operandNum, U); *I = ToOp; - ToOp.Val->addUser(U); + I->setUser(U); + ToOp.Val->addUser(operandNum, U); } - + // Now that we have modified U, add it back to the CSE maps. If it already // exists there, recursively merge the results together. if (SDNode *Existing = AddNonLeafNodeToCSEMaps(U)) { - ReplaceAllUsesWith(U, Existing, Deleted); - // U is now dead. - if (Deleted) Deleted->push_back(U); + ReplaceAllUsesWith(U, Existing, UpdateListener); + // U is now dead. Inform the listener if it exists and delete it. + if (UpdateListener) + UpdateListener->NodeDeleted(U); DeleteNodeNotInCSEMaps(U); + } else { + // If the node doesn't already exist, we updated it. Inform a listener if + // it exists. + if (UpdateListener) + UpdateListener->NodeUpdated(U); } } } +namespace { + /// ChainedSetUpdaterListener - This class is a DAGUpdateListener that removes + /// any deleted nodes from the set passed into its constructor and recursively + /// notifies another update listener if specified. + class ChainedSetUpdaterListener : + public SelectionDAG::DAGUpdateListener { + SmallSetVector &Set; + SelectionDAG::DAGUpdateListener *Chain; + public: + ChainedSetUpdaterListener(SmallSetVector &set, + SelectionDAG::DAGUpdateListener *chain) + : Set(set), Chain(chain) {} + + virtual void NodeDeleted(SDNode *N) { + Set.remove(N); + if (Chain) Chain->NodeDeleted(N); + } + virtual void NodeUpdated(SDNode *N) { + if (Chain) Chain->NodeUpdated(N); + } + }; +} + /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving /// uses of other values produced by From.Val alone. The Deleted vector is -/// handled the same was as for ReplaceAllUsesWith. +/// handled the same way as for ReplaceAllUsesWith. void SelectionDAG::ReplaceAllUsesOfValueWith(SDOperand From, SDOperand To, - std::vector *Deleted) { + DAGUpdateListener *UpdateListener){ assert(From != To && "Cannot replace a value with itself"); + // Handle the simple, trivial, case efficiently. - if (From.Val->getNumValues() == 1 && To.Val->getNumValues() == 1) { - ReplaceAllUsesWith(From, To, Deleted); + if (From.Val->getNumValues() == 1) { + ReplaceAllUsesWith(From, To, UpdateListener); return; } - + + if (From.use_empty()) return; + // Get all of the users of From.Val. We want these in a nice, // deterministically ordered and uniqued set, so we use a SmallSetVector. - SmallSetVector Users(From.Val->use_begin(), From.Val->use_end()); + SmallSetVector Users; + for (SDNode::use_iterator UI = From.Val->use_begin(), + E = From.Val->use_end(); UI != E; ++UI) { + SDNode *User = UI->getUser(); + if (!Users.count(User)) + Users.insert(User); + } - std::vector LocalDeletionVector; + // When one of the recursive merges deletes nodes from the graph, we need to + // make sure that UpdateListener is notified *and* that the node is removed + // from Users if present. CSUL does this. + ChainedSetUpdaterListener CSUL(Users, UpdateListener); - // Pick a deletion vector to use. If the user specified one, use theirs, - // otherwise use a local one. - std::vector *DeleteVector = Deleted ? Deleted : &LocalDeletionVector; while (!Users.empty()) { // We know that this user uses some value of From. If it is the right // value, update it. @@ -3270,7 +3507,7 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDOperand From, SDOperand To, Users.pop_back(); // Scan for an operand that matches From. - SDOperand *Op = User->OperandList, *E = User->OperandList+User->NumOperands; + SDNode::op_iterator Op = User->op_begin(), E = User->op_end(); for (; Op != E; ++Op) if (*Op == From) break; @@ -3281,43 +3518,34 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDOperand From, SDOperand To, // from the CSE maps. RemoveNodeFromCSEMaps(User); - // Update all operands that match "From". + // Update all operands that match "From" in case there are multiple uses. for (; Op != E; ++Op) { if (*Op == From) { - From.Val->removeUser(User); - *Op = To; - To.Val->addUser(User); + From.Val->removeUser(Op-User->op_begin(), User); + *Op = To; + Op->setUser(User); + To.Val->addUser(Op-User->op_begin(), User); } } // Now that we have modified User, add it back to the CSE maps. If it // already exists there, recursively merge the results together. SDNode *Existing = AddNonLeafNodeToCSEMaps(User); - if (!Existing) continue; // Continue on to next user. + if (!Existing) { + if (UpdateListener) UpdateListener->NodeUpdated(User); + continue; // Continue on to next user. + } // If there was already an existing matching node, use ReplaceAllUsesWith - // to replace the dead one with the existing one. However, this can cause + // to replace the dead one with the existing one. This can cause // recursive merging of other unrelated nodes down the line. The merging - // can cause deletion of nodes that used the old value. In this case, - // we have to be certain to remove them from the Users set. - unsigned NumDeleted = DeleteVector->size(); - ReplaceAllUsesWith(User, Existing, DeleteVector); + // can cause deletion of nodes that used the old value. To handle this, we + // use CSUL to remove them from the Users set. + ReplaceAllUsesWith(User, Existing, &CSUL); - // User is now dead. - DeleteVector->push_back(User); + // User is now dead. Notify a listener if present. + if (UpdateListener) UpdateListener->NodeDeleted(User); DeleteNodeNotInCSEMaps(User); - - // We have to be careful here, because ReplaceAllUsesWith could have - // deleted a user of From, which means there may be dangling pointers - // in the "Users" setvector. Scan over the deleted node pointers and - // remove them from the setvector. - for (unsigned i = NumDeleted, e = DeleteVector->size(); i != e; ++i) - Users.remove((*DeleteVector)[i]); - - // If the user doesn't need the set of deleted elements, don't retain them - // to the next loop iteration. - if (Deleted == 0) - LocalDeletionVector.clear(); } } @@ -3395,12 +3623,15 @@ void JumpTableSDNode::ANCHOR() {} void ConstantPoolSDNode::ANCHOR() {} void BasicBlockSDNode::ANCHOR() {} void SrcValueSDNode::ANCHOR() {} +void MemOperandSDNode::ANCHOR() {} void RegisterSDNode::ANCHOR() {} void ExternalSymbolSDNode::ANCHOR() {} void CondCodeSDNode::ANCHOR() {} +void ARG_FLAGSSDNode::ANCHOR() {} void VTSDNode::ANCHOR() {} void LoadSDNode::ANCHOR() {} void StoreSDNode::ANCHOR() {} +void AtomicSDNode::ANCHOR() {} HandleSDNode::~HandleSDNode() { SDVTList VTs = { 0, 0 }; @@ -3419,6 +3650,26 @@ GlobalAddressSDNode::GlobalAddressSDNode(bool isTarget, const GlobalValue *GA, TheGlobal = const_cast(GA); } +/// getMemOperand - Return a MemOperand object describing the memory +/// reference performed by this load or store. +MemOperand LSBaseSDNode::getMemOperand() const { + int Size = (MVT::getSizeInBits(getMemoryVT()) + 7) >> 3; + int Flags = + getOpcode() == ISD::LOAD ? MemOperand::MOLoad : MemOperand::MOStore; + if (IsVolatile) Flags |= MemOperand::MOVolatile; + + // Check if the load references a frame index, and does not have + // an SV attached. + const FrameIndexSDNode *FI = + dyn_cast(getBasePtr().Val); + if (!getSrcValue() && FI) + return MemOperand(PseudoSourceValue::getFixedStack(), Flags, + FI->getIndex(), Size, Alignment); + else + return MemOperand(getSrcValue(), Flags, + getSrcValueOffset(), Size, Alignment); +} + /// Profile - Gather unique data for the node. /// void SDNode::Profile(FoldingSetNodeID &ID) { @@ -3427,10 +3678,10 @@ void SDNode::Profile(FoldingSetNodeID &ID) { /// getValueTypeList - Return a pointer to the specified value type. /// -MVT::ValueType *SDNode::getValueTypeList(MVT::ValueType VT) { +const MVT::ValueType *SDNode::getValueTypeList(MVT::ValueType VT) { if (MVT::isExtendedVT(VT)) { static std::set EVTs; - return (MVT::ValueType *)&(*EVTs.insert(VT).first); + return &(*EVTs.insert(VT).first); } else { static MVT::ValueType VTs[MVT::LAST_VALUETYPE]; VTs[VT] = VT; @@ -3453,16 +3704,13 @@ bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { SmallPtrSet UsersHandled; - for (SDNode::use_iterator UI = Uses.begin(), E = Uses.end(); UI != E; ++UI) { - SDNode *User = *UI; - if (User->getNumOperands() == 1 || - UsersHandled.insert(User)) // First time we've seen this? - for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) - if (User->getOperand(i) == TheValue) { - if (NUses == 0) - return false; // too many uses - --NUses; - } + // TODO: Only iterate over uses of a given value of the node + for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { + if (*UI == TheValue) { + if (NUses == 0) + return false; + --NUses; + } } // Found exactly the right number of uses? @@ -3475,14 +3723,14 @@ bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { bool SDNode::hasAnyUseOfValue(unsigned Value) const { assert(Value < getNumValues() && "Bad value!"); - if (use_size() == 0) return false; + if (use_empty()) return false; SDOperand TheValue(const_cast(this), Value); SmallPtrSet UsersHandled; - for (SDNode::use_iterator UI = Uses.begin(), E = Uses.end(); UI != E; ++UI) { - SDNode *User = *UI; + for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { + SDNode *User = UI->getUser(); if (User->getNumOperands() == 1 || UsersHandled.insert(User)) // First time we've seen this? for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) @@ -3495,12 +3743,12 @@ bool SDNode::hasAnyUseOfValue(unsigned Value) const { } -/// isOnlyUse - Return true if this node is the only use of N. +/// isOnlyUseOf - Return true if this node is the only use of N. /// -bool SDNode::isOnlyUse(SDNode *N) const { +bool SDNode::isOnlyUseOf(SDNode *N) const { bool Seen = false; for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { - SDNode *User = *I; + SDNode *User = I->getUser(); if (User == this) Seen = true; else @@ -3512,20 +3760,51 @@ bool SDNode::isOnlyUse(SDNode *N) const { /// isOperand - Return true if this node is an operand of N. /// -bool SDOperand::isOperand(SDNode *N) const { +bool SDOperandImpl::isOperandOf(SDNode *N) const { for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) if (*this == N->getOperand(i)) return true; return false; } -bool SDNode::isOperand(SDNode *N) const { +bool SDNode::isOperandOf(SDNode *N) const { for (unsigned i = 0, e = N->NumOperands; i != e; ++i) if (this == N->OperandList[i].Val) return true; return false; } +/// reachesChainWithoutSideEffects - Return true if this operand (which must +/// be a chain) reaches the specified operand without crossing any +/// side-effecting instructions. In practice, this looks through token +/// factors and non-volatile loads. In order to remain efficient, this only +/// looks a couple of nodes in, it does not do an exhaustive search. +bool SDOperandImpl::reachesChainWithoutSideEffects(SDOperandImpl Dest, + unsigned Depth) const { + if (*this == Dest) return true; + + // Don't search too deeply, we just want to be able to see through + // TokenFactor's etc. + if (Depth == 0) return false; + + // If this is a token factor, all inputs to the TF happen in parallel. If any + // of the operands of the TF reach dest, then we can do the xform. + if (getOpcode() == ISD::TokenFactor) { + for (unsigned i = 0, e = getNumOperands(); i != e; ++i) + if (getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) + return true; + return false; + } + + // Loads don't have side effects, look through them. + if (LoadSDNode *Ld = dyn_cast(*this)) { + if (!Ld->isVolatile()) + return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); + } + return false; +} + + static void findPredecessor(SDNode *N, const SDNode *P, bool &found, SmallPtrSet &Visited) { if (found || !Visited.insert(N)) @@ -3541,11 +3820,11 @@ static void findPredecessor(SDNode *N, const SDNode *P, bool &found, } } -/// isPredecessor - Return true if this node is a predecessor of N. This node +/// isPredecessorOf - Return true if this node is a predecessor of N. This node /// is either an operand of N or it can be reached by recursively traversing /// up the operands. /// NOTE: this is an expensive method. Use it carefully. -bool SDNode::isPredecessor(SDNode *N) const { +bool SDNode::isPredecessorOf(SDNode *N) const { SmallPtrSet Visited; bool found = false; findPredecessor(N, this, found, Visited); @@ -3566,7 +3845,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { if (G) { if (const TargetInstrInfo *TII = G->getTarget().getInstrInfo()) if (getOpcode()-ISD::BUILTIN_OP_END < TII->getNumOpcodes()) - return TII->getName(getOpcode()-ISD::BUILTIN_OP_END); + return TII->get(getOpcode()-ISD::BUILTIN_OP_END).getName(); TargetLowering &TLI = G->getTargetLoweringInfo(); const char *Name = @@ -3577,9 +3856,15 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { return "<>"; } + case ISD::PREFETCH: return "Prefetch"; + case ISD::MEMBARRIER: return "MemBarrier"; + case ISD::ATOMIC_LCS: return "AtomicLCS"; + case ISD::ATOMIC_LAS: return "AtomicLAS"; + case ISD::ATOMIC_SWAP: return "AtomicSWAP"; case ISD::PCMARKER: return "PCMarker"; case ISD::READCYCLECOUNTER: return "ReadCycleCounter"; case ISD::SRCVALUE: return "SrcValue"; + case ISD::MEMOPERAND: return "MemOperand"; case ISD::EntryToken: return "EntryToken"; case ISD::TokenFactor: return "TokenFactor"; case ISD::AssertSext: return "AssertSext"; @@ -3587,6 +3872,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::STRING: return "String"; case ISD::BasicBlock: return "BasicBlock"; + case ISD::ARG_FLAGS: return "ArgFlags"; case ISD::VALUETYPE: return "ValueType"; case ISD::Register: return "Register"; @@ -3631,6 +3917,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::MERGE_VALUES: return "merge_values"; case ISD::INLINEASM: return "inlineasm"; case ISD::LABEL: return "label"; + case ISD::DECLARE: return "declare"; case ISD::HANDLENODE: return "handlenode"; case ISD::FORMAL_ARGUMENTS: return "formal_arguments"; case ISD::CALL: return "call"; @@ -3672,6 +3959,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::FDIV: return "fdiv"; case ISD::FREM: return "frem"; case ISD::FCOPYSIGN: return "fcopysign"; + case ISD::FGETSIGN: return "fgetsign"; case ISD::SETCC: return "setcc"; case ISD::SELECT: return "select"; @@ -3701,6 +3989,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::SIGN_EXTEND_INREG: return "sign_extend_inreg"; case ISD::TRUNCATE: return "truncate"; case ISD::FP_ROUND: return "fp_round"; + case ISD::FLT_ROUNDS_: return "flt_rounds"; case ISD::FP_ROUND_INREG: return "fp_round_inreg"; case ISD::FP_EXTEND: return "fp_extend"; @@ -3732,7 +4021,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::BUILD_PAIR: return "build_pair"; case ISD::STACKSAVE: return "stacksave"; case ISD::STACKRESTORE: return "stackrestore"; - + case ISD::TRAP: return "trap"; + // Block memory operations. case ISD::MEMSET: return "memset"; case ISD::MEMCPY: return "memcpy"; @@ -3795,6 +4085,30 @@ const char *SDNode::getIndexedModeName(ISD::MemIndexedMode AM) { } } +std::string ISD::ArgFlagsTy::getArgFlagsString() { + std::string S = "< "; + + if (isZExt()) + S += "zext "; + if (isSExt()) + S += "sext "; + if (isInReg()) + S += "inreg "; + if (isSRet()) + S += "sret "; + if (isByVal()) + S += "byval "; + if (isNest()) + S += "nest "; + if (getByValAlign()) + S += "byval-align:" + utostr(getByValAlign()) + " "; + if (getOrigAlign()) + S += "orig-align:" + utostr(getOrigAlign()) + " "; + if (getByValSize()) + S += "byval-size:" + utostr(getByValSize()) + " "; + return S + ">"; +} + void SDNode::dump() const { dump(0); } void SDNode::dump(const SelectionDAG *G) const { cerr << (void*)this << ": "; @@ -3816,6 +4130,19 @@ void SDNode::dump(const SelectionDAG *G) const { cerr << ":" << RN; } + if (!isTargetOpcode() && getOpcode() == ISD::VECTOR_SHUFFLE) { + SDNode *Mask = getOperand(2).Val; + cerr << "<"; + for (unsigned i = 0, e = Mask->getNumOperands(); i != e; ++i) { + if (i) cerr << ","; + if (Mask->getOperand(i).getOpcode() == ISD::UNDEF) + cerr << "u"; + else + cerr << cast(Mask->getOperand(i))->getValue(); + } + cerr << ">"; + } + if (const ConstantSDNode *CSDN = dyn_cast(this)) { cerr << "<" << CSDN->getValue() << ">"; } else if (const ConstantFPSDNode *CSDN = dyn_cast(this)) { @@ -3858,8 +4185,9 @@ void SDNode::dump(const SelectionDAG *G) const { cerr << LBB->getName() << " "; cerr << (const void*)BBDN->getBasicBlock() << ">"; } else if (const RegisterSDNode *R = dyn_cast(this)) { - if (G && R->getReg() && MRegisterInfo::isPhysicalRegister(R->getReg())) { - cerr << " " <getTarget().getRegisterInfo()->getName(R->getReg()); + if (G && R->getReg() && + TargetRegisterInfo::isPhysicalRegister(R->getReg())) { + cerr << " " << G->getTarget().getRegisterInfo()->getName(R->getReg()); } else { cerr << " #" << R->getReg(); } @@ -3868,12 +4196,28 @@ void SDNode::dump(const SelectionDAG *G) const { cerr << "'" << ES->getSymbol() << "'"; } else if (const SrcValueSDNode *M = dyn_cast(this)) { if (M->getValue()) - cerr << "<" << M->getValue() << ":" << M->getOffset() << ">"; + cerr << "<" << M->getValue() << ">"; + else + cerr << ""; + } else if (const MemOperandSDNode *M = dyn_cast(this)) { + if (M->MO.getValue()) + cerr << "<" << M->MO.getValue() << ":" << M->MO.getOffset() << ">"; else - cerr << "getOffset() << ">"; + cerr << "MO.getOffset() << ">"; + } else if (const ARG_FLAGSSDNode *N = dyn_cast(this)) { + cerr << N->getArgFlags().getArgFlagsString(); } else if (const VTSDNode *N = dyn_cast(this)) { cerr << ":" << MVT::getValueTypeString(N->getVT()); } else if (const LoadSDNode *LD = dyn_cast(this)) { + const Value *SrcValue = LD->getSrcValue(); + int SrcOffset = LD->getSrcValueOffset(); + cerr << " <"; + if (SrcValue) + cerr << SrcValue; + else + cerr << "null"; + cerr << ":" << SrcOffset << ">"; + bool doExt = true; switch (LD->getExtensionType()) { default: doExt = false; break; @@ -3888,19 +4232,34 @@ void SDNode::dump(const SelectionDAG *G) const { break; } if (doExt) - cerr << MVT::getValueTypeString(LD->getLoadedVT()) << ">"; + cerr << MVT::getValueTypeString(LD->getMemoryVT()) << ">"; const char *AM = getIndexedModeName(LD->getAddressingMode()); if (*AM) cerr << " " << AM; + if (LD->isVolatile()) + cerr << " "; + cerr << " alignment=" << LD->getAlignment(); } else if (const StoreSDNode *ST = dyn_cast(this)) { + const Value *SrcValue = ST->getSrcValue(); + int SrcOffset = ST->getSrcValueOffset(); + cerr << " <"; + if (SrcValue) + cerr << SrcValue; + else + cerr << "null"; + cerr << ":" << SrcOffset << ">"; + if (ST->isTruncatingStore()) cerr << " getStoredVT()) << ">"; + << MVT::getValueTypeString(ST->getMemoryVT()) << ">"; const char *AM = getIndexedModeName(ST->getAddressingMode()); if (*AM) cerr << " " << AM; + if (ST->isVolatile()) + cerr << " "; + cerr << " alignment=" << ST->getAlignment(); } }