X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAGISel.cpp;h=bf68040c97aebf62e3831692f2d4016728e40d84;hb=bf304c20651b80309af4c0fb3a14c0d73eaa984f;hp=e767c280d42fbfe343f5754aa76f8c0a6a1e8d3b;hpb=86ca3cacc5a56e862c8258ac9c9ee1e37598ebd8;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index e767c280d42..bf68040c97a 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -42,12 +42,19 @@ #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" -#include "llvm/Support/MathExtras.h" -#include "llvm/Support/Debug.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/Timer.h" #include using namespace llvm; +static cl::opt +EnableValueProp("enable-value-prop", cl::Hidden); +static cl::opt +EnableLegalizeTypes("enable-legalize-types", cl::Hidden); + + #ifndef NDEBUG static cl::opt ViewISelDAGs("view-isel-dags", cl::Hidden, @@ -74,53 +81,166 @@ MachinePassRegistry RegisterScheduler::Registry; /// ISHeuristic command line option for instruction schedulers. /// //===---------------------------------------------------------------------===// -namespace { - cl::opt > - ISHeuristic("pre-RA-sched", - cl::init(&createDefaultScheduler), - cl::desc("Instruction schedulers available (before register" - " allocation):")); - - static RegisterScheduler - defaultListDAGScheduler("default", " Best scheduler for the target", - createDefaultScheduler); -} // namespace +static cl::opt > +ISHeuristic("pre-RA-sched", + cl::init(&createDefaultScheduler), + cl::desc("Instruction schedulers available (before register" + " allocation):")); + +static RegisterScheduler +defaultListDAGScheduler("default", " Best scheduler for the target", + createDefaultScheduler); + +namespace { struct SDISelAsmOperandInfo; } + +/// ComputeLinearIndex - Given an LLVM IR aggregate type and a sequence +/// insertvalue or extractvalue indices that identify a member, return +/// the linearized index of the start of the member. +/// +static unsigned ComputeLinearIndex(const TargetLowering &TLI, const Type *Ty, + const unsigned *Indices, + const unsigned *IndicesEnd, + unsigned CurIndex = 0) { + // Base case: We're done. + if (Indices && Indices == IndicesEnd) + return CurIndex; + + // Given a struct type, recursively traverse the elements. + if (const StructType *STy = dyn_cast(Ty)) { + for (StructType::element_iterator EB = STy->element_begin(), + EI = EB, + EE = STy->element_end(); + EI != EE; ++EI) { + if (Indices && *Indices == unsigned(EI - EB)) + return ComputeLinearIndex(TLI, *EI, Indices+1, IndicesEnd, CurIndex); + CurIndex = ComputeLinearIndex(TLI, *EI, 0, 0, CurIndex); + } + } + // Given an array type, recursively traverse the elements. + else if (const ArrayType *ATy = dyn_cast(Ty)) { + const Type *EltTy = ATy->getElementType(); + for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) { + if (Indices && *Indices == i) + return ComputeLinearIndex(TLI, EltTy, Indices+1, IndicesEnd, CurIndex); + CurIndex = ComputeLinearIndex(TLI, EltTy, 0, 0, CurIndex); + } + } + // We haven't found the type we're looking for, so keep searching. + return CurIndex + 1; +} -namespace { struct AsmOperandInfo; } +/// ComputeValueVTs - Given an LLVM IR type, compute a sequence of +/// MVTs that represent all the individual underlying +/// non-aggregate types that comprise it. +/// +/// If Offsets is non-null, it points to a vector to be filled in +/// with the in-memory offsets of each of the individual values. +/// +static void ComputeValueVTs(const TargetLowering &TLI, const Type *Ty, + SmallVectorImpl &ValueVTs, + SmallVectorImpl *Offsets = 0, + uint64_t StartingOffset = 0) { + // Given a struct type, recursively traverse the elements. + if (const StructType *STy = dyn_cast(Ty)) { + const StructLayout *SL = TLI.getTargetData()->getStructLayout(STy); + for (StructType::element_iterator EB = STy->element_begin(), + EI = EB, + EE = STy->element_end(); + EI != EE; ++EI) + ComputeValueVTs(TLI, *EI, ValueVTs, Offsets, + StartingOffset + SL->getElementOffset(EI - EB)); + return; + } + // Given an array type, recursively traverse the elements. + if (const ArrayType *ATy = dyn_cast(Ty)) { + const Type *EltTy = ATy->getElementType(); + uint64_t EltSize = TLI.getTargetData()->getABITypeSize(EltTy); + for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) + ComputeValueVTs(TLI, EltTy, ValueVTs, Offsets, + StartingOffset + i * EltSize); + return; + } + // Base case: we can get an MVT for this LLVM IR type. + ValueVTs.push_back(TLI.getValueType(Ty)); + if (Offsets) + Offsets->push_back(StartingOffset); +} namespace { - /// RegsForValue - This struct represents the physical registers that a - /// particular value is assigned and the type information about the value. - /// This is needed because values can be promoted into larger registers and - /// expanded into multiple smaller registers than the value. + /// RegsForValue - This struct represents the registers (physical or virtual) + /// that a particular set of values is assigned, and the type information about + /// the value. The most common situation is to represent one value at a time, + /// but struct or array values are handled element-wise as multiple values. + /// The splitting of aggregates is performed recursively, so that we never + /// have aggregate-typed registers. The values at this point do not necessarily + /// have legal types, so each value may require one or more registers of some + /// legal type. + /// struct VISIBILITY_HIDDEN RegsForValue { - /// Regs - This list holds the register (for legal and promoted values) - /// or register set (for expanded values) that the value should be assigned - /// to. - std::vector Regs; + /// TLI - The TargetLowering object. + /// + const TargetLowering *TLI; + + /// ValueVTs - The value types of the values, which may not be legal, and + /// may need be promoted or synthesized from one or more registers. + /// + SmallVector ValueVTs; - /// RegVT - The value type of each register. + /// RegVTs - The value types of the registers. This is the same size as + /// ValueVTs and it records, for each value, what the type of the assigned + /// register or registers are. (Individual values are never synthesized + /// from more than one type of register.) /// - MVT::ValueType RegVT; + /// With virtual registers, the contents of RegVTs is redundant with TLI's + /// getRegisterType member function, however when with physical registers + /// it is necessary to have a separate record of the types. + /// + SmallVector RegVTs; - /// ValueVT - The value type of the LLVM value, which may be promoted from - /// RegVT or made from merging the two expanded parts. - MVT::ValueType ValueVT; + /// Regs - This list holds the registers assigned to the values. + /// Each legal or promoted value requires one register, and each + /// expanded value requires multiple registers. + /// + SmallVector Regs; - RegsForValue() : RegVT(MVT::Other), ValueVT(MVT::Other) {} + RegsForValue() : TLI(0) {} - RegsForValue(unsigned Reg, MVT::ValueType regvt, MVT::ValueType valuevt) - : RegVT(regvt), ValueVT(valuevt) { - Regs.push_back(Reg); + RegsForValue(const TargetLowering &tli, + const SmallVector ®s, + MVT regvt, MVT valuevt) + : TLI(&tli), ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs) {} + RegsForValue(const TargetLowering &tli, + const SmallVector ®s, + const SmallVector ®vts, + const SmallVector &valuevts) + : TLI(&tli), ValueVTs(valuevts), RegVTs(regvts), Regs(regs) {} + RegsForValue(const TargetLowering &tli, + unsigned Reg, const Type *Ty) : TLI(&tli) { + ComputeValueVTs(tli, Ty, ValueVTs); + + for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { + MVT ValueVT = ValueVTs[Value]; + unsigned NumRegs = TLI->getNumRegisters(ValueVT); + MVT RegisterVT = TLI->getRegisterType(ValueVT); + for (unsigned i = 0; i != NumRegs; ++i) + Regs.push_back(Reg + i); + RegVTs.push_back(RegisterVT); + Reg += NumRegs; + } } - RegsForValue(const std::vector ®s, - MVT::ValueType regvt, MVT::ValueType valuevt) - : Regs(regs), RegVT(regvt), ValueVT(valuevt) { + + /// append - Add the specified values to this one. + void append(const RegsForValue &RHS) { + TLI = RHS.TLI; + ValueVTs.append(RHS.ValueVTs.begin(), RHS.ValueVTs.end()); + RegVTs.append(RHS.RegVTs.begin(), RHS.RegVTs.end()); + Regs.append(RHS.Regs.begin(), RHS.Regs.end()); } + /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from - /// this value and returns the result as a ValueVT value. This uses + /// this value and returns the result as a ValueVTs value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. SDOperand getCopyFromRegs(SelectionDAG &DAG, @@ -147,15 +267,16 @@ namespace llvm { /// for the target. ScheduleDAG* createDefaultScheduler(SelectionDAGISel *IS, SelectionDAG *DAG, - MachineBasicBlock *BB) { + MachineBasicBlock *BB, + bool Fast) { TargetLowering &TLI = IS->getTargetLowering(); if (TLI.getSchedulingPreference() == TargetLowering::SchedulingForLatency) { - return createTDListDAGScheduler(IS, DAG, BB); + return createTDListDAGScheduler(IS, DAG, BB, Fast); } else { assert(TLI.getSchedulingPreference() == TargetLowering::SchedulingForRegPressure && "Unknown sched type!"); - return createBURRListDAGScheduler(IS, DAG, BB); + return createBURRListDAGScheduler(IS, DAG, BB, Fast); } } @@ -190,7 +311,7 @@ namespace llvm { SmallSet CatchInfoFound; #endif - unsigned MakeReg(MVT::ValueType VT) { + unsigned MakeReg(MVT VT) { return RegInfo.createVirtualRegister(TLI.getRegClassFor(VT)); } @@ -207,6 +328,16 @@ namespace llvm { assert(R == 0 && "Already initialized this value register!"); return R = CreateRegForValue(V); } + + struct LiveOutInfo { + unsigned NumSignBits; + APInt KnownOne, KnownZero; + LiveOutInfo() : NumSignBits(0) {} + }; + + /// LiveOutRegInfo - Information about live out vregs, indexed by their + /// register number offset by 'FirstVirtualRegister'. + std::vector LiveOutRegInfo; }; } @@ -221,7 +352,7 @@ static bool isSelector(Instruction *I) { /// isUsedOutsideOfDefiningBlock - Return true if this instruction is used by /// PHI nodes or outside of the basic block that defines it, or used by a -/// switch instruction, which may expand to multiple basic blocks. +/// switch or atomic instruction, which may expand to multiple basic blocks. static bool isUsedOutsideOfDefiningBlock(Instruction *I) { if (isa(I)) return true; BasicBlock *BB = I->getParent(); @@ -285,9 +416,9 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, // also creates the initial PHI MachineInstrs, though none of the input // operands are populated. for (BB = Fn.begin(), EB = Fn.end(); BB != EB; ++BB) { - MachineBasicBlock *MBB = new MachineBasicBlock(BB); + MachineBasicBlock *MBB = mf.CreateMachineBasicBlock(BB); MBBMap[BB] = MBB; - MF.getBasicBlockList().push_back(MBB); + MF.push_back(MBB); // Create Machine PHI nodes for LLVM PHI nodes, lowering them as // appropriate. @@ -295,7 +426,7 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, for (BasicBlock::iterator I = BB->begin();(PN = dyn_cast(I)); ++I){ if (PN->use_empty()) continue; - MVT::ValueType VT = TLI.getValueType(PN->getType()); + MVT VT = TLI.getValueType(PN->getType()); unsigned NumRegisters = TLI.getNumRegisters(VT); unsigned PHIReg = ValueMap[PN]; assert(PHIReg && "PHI node does not have an assigned virtual register!"); @@ -309,17 +440,26 @@ FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, /// CreateRegForValue - Allocate the appropriate number of virtual registers of /// the correctly promoted or expanded types. Assign these registers /// consecutive vreg numbers and return the first assigned number. +/// +/// In the case that the given value has struct or array type, this function +/// will assign registers for each member or element. +/// unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) { - MVT::ValueType VT = TLI.getValueType(V->getType()); - - unsigned NumRegisters = TLI.getNumRegisters(VT); - MVT::ValueType RegisterVT = TLI.getRegisterType(VT); + SmallVector ValueVTs; + ComputeValueVTs(TLI, V->getType(), ValueVTs); - unsigned R = MakeReg(RegisterVT); - for (unsigned i = 1; i != NumRegisters; ++i) - MakeReg(RegisterVT); + unsigned FirstReg = 0; + for (unsigned Value = 0, e = ValueVTs.size(); Value != e; ++Value) { + MVT ValueVT = ValueVTs[Value]; + MVT RegisterVT = TLI.getRegisterType(ValueVT); - return R; + unsigned NumRegs = TLI.getNumRegisters(ValueVT); + for (unsigned i = 0; i != NumRegs; ++i) { + unsigned R = MakeReg(RegisterVT); + if (!FirstReg) FirstReg = R; + } + } + return FirstReg; } //===----------------------------------------------------------------------===// @@ -337,7 +477,14 @@ class SelectionDAGLowering { /// them up and then emit token factor nodes when possible. This allows us to /// get simple disambiguation between loads without worrying about alias /// analysis. - std::vector PendingLoads; + SmallVector PendingLoads; + + /// PendingExports - CopyToReg nodes that copy values to virtual registers + /// for export to other blocks need to be emitted before any terminator + /// instruction, but they have no other ordering requirements. We bunch them + /// up and the emit a single tokenfactor for them just before terminator + /// instructions. + std::vector PendingExports; /// Case - A struct to record the Value for a switch case, and the /// case's target basic block. @@ -440,7 +587,10 @@ public: FuncInfo(funcinfo), GCI(gci) { } - /// getRoot - Return the current virtual root of the Selection DAG. + /// getRoot - Return the current virtual root of the Selection DAG, + /// flushing any PendingLoad items. This must be done before emitting + /// a store or any other node that may need to be ordered after any + /// prior load instructions. /// SDOperand getRoot() { if (PendingLoads.empty()) @@ -461,7 +611,38 @@ public: return Root; } - SDOperand CopyValueToVirtualRegister(Value *V, unsigned Reg); + /// getControlRoot - Similar to getRoot, but instead of flushing all the + /// PendingLoad items, flush all the PendingExports items. It is necessary + /// to do this before emitting a terminator instruction. + /// + SDOperand getControlRoot() { + SDOperand Root = DAG.getRoot(); + + if (PendingExports.empty()) + return Root; + + // Turn all of the CopyToReg chains into one factored node. + if (Root.getOpcode() != ISD::EntryToken) { + unsigned i = 0, e = PendingExports.size(); + for (; i != e; ++i) { + assert(PendingExports[i].Val->getNumOperands() > 1); + if (PendingExports[i].Val->getOperand(0) == Root) + break; // Don't add the root if we already indirectly depend on it. + } + + if (i == e) + PendingExports.push_back(Root); + } + + Root = DAG.getNode(ISD::TokenFactor, MVT::Other, + &PendingExports[0], + PendingExports.size()); + PendingExports.clear(); + DAG.setRoot(Root); + return Root; + } + + void CopyValueToVirtualRegister(Value *V, unsigned Reg); void visit(Instruction &I) { visit(I.getOpcode(), I); } @@ -480,10 +661,6 @@ public: void setCurrentBasicBlock(MachineBasicBlock *MBB) { CurMBB = MBB; } - SDOperand getLoadFrom(const Type *Ty, SDOperand Ptr, - const Value *SV, SDOperand Root, - bool isVolatile, unsigned Alignment); - SDOperand getValue(const Value *V); void setValue(const Value *V, SDOperand NewN) { @@ -492,7 +669,7 @@ public: N = NewN; } - void GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber, + void GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, std::set &OutputRegs, std::set &InputRegs); @@ -569,6 +746,8 @@ public: void visitAShr(User &I) { visitShift(I, ISD::SRA); } void visitICmp(User &I); void visitFCmp(User &I); + void visitVICmp(User &I); + void visitVFCmp(User &I); // Visit the conversion instructions void visitTrunc(User &I); void visitZExt(User &I); @@ -587,6 +766,9 @@ public: void visitInsertElement(User &I); void visitShuffleVector(User &I); + void visitExtractValue(ExtractValueInst &I); + void visitInsertValue(InsertValueInst &I); + void visitGetElementPtr(User &I); void visitSelect(User &I); @@ -606,7 +788,7 @@ public: void visitVAEnd(CallInst &I); void visitVACopy(CallInst &I); - void visitMemIntrinsic(CallInst &I, unsigned Op); + void visitGetResult(GetResultInst &I); void visitUserOp1(Instruction &I) { assert(0 && "UserOp1 should not exist at instruction selection time!"); @@ -616,6 +798,10 @@ public: assert(0 && "UserOp2 should not exist at instruction selection time!"); abort(); } + +private: + inline const char *implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op); + }; } // end namespace llvm @@ -624,35 +810,33 @@ public: /// combined into the value they represent. If the parts combine to a type /// larger then ValueVT then AssertOp can be used to specify whether the extra /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT -/// (ISD::AssertSext). Likewise TruncExact is used for floating point types to -/// indicate that the extra bits can be discarded without losing precision. +/// (ISD::AssertSext). static SDOperand getCopyFromParts(SelectionDAG &DAG, const SDOperand *Parts, unsigned NumParts, - MVT::ValueType PartVT, - MVT::ValueType ValueVT, - ISD::NodeType AssertOp = ISD::DELETED_NODE, - bool TruncExact = false) { + MVT PartVT, + MVT ValueVT, + ISD::NodeType AssertOp = ISD::DELETED_NODE) { assert(NumParts > 0 && "No parts to assemble!"); TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDOperand Val = Parts[0]; if (NumParts > 1) { // Assemble the value from multiple parts. - if (!MVT::isVector(ValueVT)) { - unsigned PartBits = MVT::getSizeInBits(PartVT); - unsigned ValueBits = MVT::getSizeInBits(ValueVT); + if (!ValueVT.isVector()) { + unsigned PartBits = PartVT.getSizeInBits(); + unsigned ValueBits = ValueVT.getSizeInBits(); // Assemble the power of 2 part. unsigned RoundParts = NumParts & (NumParts - 1) ? 1 << Log2_32(NumParts) : NumParts; unsigned RoundBits = PartBits * RoundParts; - MVT::ValueType RoundVT = RoundBits == ValueBits ? - ValueVT : MVT::getIntegerType(RoundBits); + MVT RoundVT = RoundBits == ValueBits ? + ValueVT : MVT::getIntegerVT(RoundBits); SDOperand Lo, Hi; if (RoundParts > 2) { - MVT::ValueType HalfVT = MVT::getIntegerType(RoundBits/2); + MVT HalfVT = MVT::getIntegerVT(RoundBits/2); Lo = getCopyFromParts(DAG, Parts, RoundParts/2, PartVT, HalfVT); Hi = getCopyFromParts(DAG, Parts+RoundParts/2, RoundParts/2, PartVT, HalfVT); @@ -667,30 +851,30 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, if (RoundParts < NumParts) { // Assemble the trailing non-power-of-2 part. unsigned OddParts = NumParts - RoundParts; - MVT::ValueType OddVT = MVT::getIntegerType(OddParts * PartBits); + MVT OddVT = MVT::getIntegerVT(OddParts * PartBits); Hi = getCopyFromParts(DAG, Parts+RoundParts, OddParts, PartVT, OddVT); // Combine the round and odd parts. Lo = Val; if (TLI.isBigEndian()) std::swap(Lo, Hi); - MVT::ValueType TotalVT = MVT::getIntegerType(NumParts * PartBits); + MVT TotalVT = MVT::getIntegerVT(NumParts * PartBits); Hi = DAG.getNode(ISD::ANY_EXTEND, TotalVT, Hi); Hi = DAG.getNode(ISD::SHL, TotalVT, Hi, - DAG.getConstant(MVT::getSizeInBits(Lo.getValueType()), + DAG.getConstant(Lo.getValueType().getSizeInBits(), TLI.getShiftAmountTy())); Lo = DAG.getNode(ISD::ZERO_EXTEND, TotalVT, Lo); Val = DAG.getNode(ISD::OR, TotalVT, Lo, Hi); } } else { // Handle a multi-element vector. - MVT::ValueType IntermediateVT, RegisterVT; + MVT IntermediateVT, RegisterVT; unsigned NumIntermediates; unsigned NumRegs = TLI.getVectorTypeBreakdown(ValueVT, IntermediateVT, NumIntermediates, RegisterVT); - assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); + NumParts = NumRegs; // Silence a compiler warning. assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); assert(RegisterVT == Parts[0].getValueType() && "Part type doesn't match part!"); @@ -716,7 +900,7 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the intermediate // operands. - Val = DAG.getNode(MVT::isVector(IntermediateVT) ? + Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR, ValueVT, &Ops[0], NumIntermediates); } @@ -728,21 +912,21 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, if (PartVT == ValueVT) return Val; - if (MVT::isVector(PartVT)) { - assert(MVT::isVector(ValueVT) && "Unknown vector conversion!"); + if (PartVT.isVector()) { + assert(ValueVT.isVector() && "Unknown vector conversion!"); return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val); } - if (MVT::isVector(ValueVT)) { - assert(MVT::getVectorElementType(ValueVT) == PartVT && - MVT::getVectorNumElements(ValueVT) == 1 && + if (ValueVT.isVector()) { + assert(ValueVT.getVectorElementType() == PartVT && + ValueVT.getVectorNumElements() == 1 && "Only trivial scalar-to-vector conversions should get here!"); return DAG.getNode(ISD::BUILD_VECTOR, ValueVT, Val); } - if (MVT::isInteger(PartVT) && - MVT::isInteger(ValueVT)) { - if (MVT::getSizeInBits(ValueVT) < MVT::getSizeInBits(PartVT)) { + if (PartVT.isInteger() && + ValueVT.isInteger()) { + if (ValueVT.bitsLT(PartVT)) { // For a truncate, see if we have any information to // indicate whether the truncated bits will always be // zero or sign-extension. @@ -755,14 +939,19 @@ static SDOperand getCopyFromParts(SelectionDAG &DAG, } } - if (MVT::isFloatingPoint(PartVT) && MVT::isFloatingPoint(ValueVT)) - return DAG.getNode(ISD::FP_ROUND, ValueVT, Val, - DAG.getIntPtrConstant(TruncExact)); + if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { + if (ValueVT.bitsLT(Val.getValueType())) + // FP_ROUND's are always exact here. + return DAG.getNode(ISD::FP_ROUND, ValueVT, Val, + DAG.getIntPtrConstant(1)); + return DAG.getNode(ISD::FP_EXTEND, ValueVT, Val); + } - if (MVT::getSizeInBits(PartVT) == MVT::getSizeInBits(ValueVT)) + if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) return DAG.getNode(ISD::BIT_CONVERT, ValueVT, Val); assert(0 && "Unknown mismatch!"); + return SDOperand(); } /// getCopyToParts - Create a series of nodes that contain the specified value @@ -772,43 +961,43 @@ static void getCopyToParts(SelectionDAG &DAG, SDOperand Val, SDOperand *Parts, unsigned NumParts, - MVT::ValueType PartVT, + MVT PartVT, ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { TargetLowering &TLI = DAG.getTargetLoweringInfo(); - MVT::ValueType PtrVT = TLI.getPointerTy(); - MVT::ValueType ValueVT = Val.getValueType(); - unsigned PartBits = MVT::getSizeInBits(PartVT); + MVT PtrVT = TLI.getPointerTy(); + MVT ValueVT = Val.getValueType(); + unsigned PartBits = PartVT.getSizeInBits(); assert(TLI.isTypeLegal(PartVT) && "Copying to an illegal type!"); if (!NumParts) return; - if (!MVT::isVector(ValueVT)) { + if (!ValueVT.isVector()) { if (PartVT == ValueVT) { assert(NumParts == 1 && "No-op copy with multiple parts!"); Parts[0] = Val; return; } - if (NumParts * PartBits > MVT::getSizeInBits(ValueVT)) { + if (NumParts * PartBits > ValueVT.getSizeInBits()) { // If the parts cover more bits than the value has, promote the value. - if (MVT::isFloatingPoint(PartVT) && MVT::isFloatingPoint(ValueVT)) { + if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { assert(NumParts == 1 && "Do not know what to promote to!"); Val = DAG.getNode(ISD::FP_EXTEND, PartVT, Val); - } else if (MVT::isInteger(PartVT) && MVT::isInteger(ValueVT)) { - ValueVT = MVT::getIntegerType(NumParts * PartBits); + } else if (PartVT.isInteger() && ValueVT.isInteger()) { + ValueVT = MVT::getIntegerVT(NumParts * PartBits); Val = DAG.getNode(ExtendKind, ValueVT, Val); } else { assert(0 && "Unknown mismatch!"); } - } else if (PartBits == MVT::getSizeInBits(ValueVT)) { + } else if (PartBits == ValueVT.getSizeInBits()) { // Different types of the same size. assert(NumParts == 1 && PartVT != ValueVT); Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val); - } else if (NumParts * PartBits < MVT::getSizeInBits(ValueVT)) { + } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { // If the parts cover less bits than value has, truncate the value. - if (MVT::isInteger(PartVT) && MVT::isInteger(ValueVT)) { - ValueVT = MVT::getIntegerType(NumParts * PartBits); + if (PartVT.isInteger() && ValueVT.isInteger()) { + ValueVT = MVT::getIntegerVT(NumParts * PartBits); Val = DAG.getNode(ISD::TRUNCATE, ValueVT, Val); } else { assert(0 && "Unknown mismatch!"); @@ -817,7 +1006,7 @@ static void getCopyToParts(SelectionDAG &DAG, // The value may have changed - recompute ValueVT. ValueVT = Val.getValueType(); - assert(NumParts * PartBits == MVT::getSizeInBits(ValueVT) && + assert(NumParts * PartBits == ValueVT.getSizeInBits() && "Failed to tile the value with PartVT!"); if (NumParts == 1) { @@ -829,7 +1018,7 @@ static void getCopyToParts(SelectionDAG &DAG, // Expand the value into multiple parts. if (NumParts & (NumParts - 1)) { // The number of parts is not a power of 2. Split off and copy the tail. - assert(MVT::isInteger(PartVT) && MVT::isInteger(ValueVT) && + assert(PartVT.isInteger() && ValueVT.isInteger() && "Do not know what to expand to!"); unsigned RoundParts = 1 << Log2_32(NumParts); unsigned RoundBits = RoundParts * PartBits; @@ -842,25 +1031,31 @@ static void getCopyToParts(SelectionDAG &DAG, // The odd parts were reversed by getCopyToParts - unreverse them. std::reverse(Parts + RoundParts, Parts + NumParts); NumParts = RoundParts; - ValueVT = MVT::getIntegerType(NumParts * PartBits); + ValueVT = MVT::getIntegerVT(NumParts * PartBits); Val = DAG.getNode(ISD::TRUNCATE, ValueVT, Val); } // The number of parts is a power of 2. Repeatedly bisect the value using // EXTRACT_ELEMENT. - Parts[0] = Val; + Parts[0] = DAG.getNode(ISD::BIT_CONVERT, + MVT::getIntegerVT(ValueVT.getSizeInBits()), + Val); for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { for (unsigned i = 0; i < NumParts; i += StepSize) { unsigned ThisBits = StepSize * PartBits / 2; - MVT::ValueType ThisVT = - ThisBits == PartBits ? PartVT : MVT::getIntegerType (ThisBits); - - Parts[i+StepSize/2] = - DAG.getNode(ISD::EXTRACT_ELEMENT, ThisVT, Parts[i], - DAG.getConstant(1, PtrVT)); - Parts[i] = - DAG.getNode(ISD::EXTRACT_ELEMENT, ThisVT, Parts[i], - DAG.getConstant(0, PtrVT)); + MVT ThisVT = MVT::getIntegerVT (ThisBits); + SDOperand &Part0 = Parts[i]; + SDOperand &Part1 = Parts[i+StepSize/2]; + + Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, ThisVT, Part0, + DAG.getConstant(1, PtrVT)); + Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, ThisVT, Part0, + DAG.getConstant(0, PtrVT)); + + if (ThisBits == PartBits && ThisVT != PartVT) { + Part0 = DAG.getNode(ISD::BIT_CONVERT, PartVT, Part0); + Part1 = DAG.getNode(ISD::BIT_CONVERT, PartVT, Part1); + } } } @@ -873,11 +1068,11 @@ static void getCopyToParts(SelectionDAG &DAG, // Vector ValueVT. if (NumParts == 1) { if (PartVT != ValueVT) { - if (MVT::isVector(PartVT)) { + if (PartVT.isVector()) { Val = DAG.getNode(ISD::BIT_CONVERT, PartVT, Val); } else { - assert(MVT::getVectorElementType(ValueVT) == PartVT && - MVT::getVectorNumElements(ValueVT) == 1 && + assert(ValueVT.getVectorElementType() == PartVT && + ValueVT.getVectorNumElements() == 1 && "Only trivial vector-to-scalar conversions should get here!"); Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, PartVT, Val, DAG.getConstant(0, PtrVT)); @@ -889,21 +1084,22 @@ static void getCopyToParts(SelectionDAG &DAG, } // Handle a multi-element vector. - MVT::ValueType IntermediateVT, RegisterVT; + MVT IntermediateVT, RegisterVT; unsigned NumIntermediates; unsigned NumRegs = DAG.getTargetLoweringInfo() .getVectorTypeBreakdown(ValueVT, IntermediateVT, NumIntermediates, RegisterVT); - unsigned NumElements = MVT::getVectorNumElements(ValueVT); + unsigned NumElements = ValueVT.getVectorNumElements(); assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); + NumParts = NumRegs; // Silence a compiler warning. assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); // Split the vector into intermediate operands. SmallVector Ops(NumIntermediates); for (unsigned i = 0; i != NumIntermediates; ++i) - if (MVT::isVector(IntermediateVT)) + if (IntermediateVT.isVector()) Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, IntermediateVT, Val, DAG.getConstant(i * (NumElements / NumIntermediates), @@ -935,70 +1131,114 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { SDOperand &N = NodeMap[V]; if (N.Val) return N; - const Type *VTy = V->getType(); - MVT::ValueType VT = TLI.getValueType(VTy); if (Constant *C = const_cast(dyn_cast(V))) { + MVT VT = TLI.getValueType(V->getType(), true); + + if (ConstantInt *CI = dyn_cast(C)) + return N = DAG.getConstant(CI->getValue(), VT); + + if (GlobalValue *GV = dyn_cast(C)) + return N = DAG.getGlobalAddress(GV, VT); + + if (isa(C)) + return N = DAG.getConstant(0, TLI.getPointerTy()); + + if (ConstantFP *CFP = dyn_cast(C)) + return N = DAG.getConstantFP(CFP->getValueAPF(), VT); + + if (isa(C) && !isa(V->getType()) && + !V->getType()->isAggregateType()) + return N = DAG.getNode(ISD::UNDEF, VT); + if (ConstantExpr *CE = dyn_cast(C)) { visit(CE->getOpcode(), *CE); SDOperand N1 = NodeMap[V]; assert(N1.Val && "visit didn't populate the ValueMap!"); return N1; - } else if (GlobalValue *GV = dyn_cast(C)) { - return N = DAG.getGlobalAddress(GV, VT); - } else if (isa(C)) { - return N = DAG.getConstant(0, TLI.getPointerTy()); - } else if (isa(C)) { - if (!isa(VTy)) - return N = DAG.getNode(ISD::UNDEF, VT); + } + + if (isa(C) || isa(C)) { + SmallVector Constants; + for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end(); + OI != OE; ++OI) { + SDNode *Val = getValue(*OI).Val; + for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) + Constants.push_back(SDOperand(Val, i)); + } + return DAG.getMergeValues(&Constants[0], Constants.size()); + } - // Create a BUILD_VECTOR of undef nodes. - const VectorType *PTy = cast(VTy); - unsigned NumElements = PTy->getNumElements(); - MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); + if (const ArrayType *ATy = dyn_cast(C->getType())) { + assert((isa(C) || isa(C)) && + "Unknown array constant!"); + unsigned NumElts = ATy->getNumElements(); + if (NumElts == 0) + return SDOperand(); // empty array + MVT EltVT = TLI.getValueType(ATy->getElementType()); + SmallVector Constants(NumElts); + for (unsigned i = 0, e = NumElts; i != e; ++i) { + if (isa(C)) + Constants[i] = DAG.getNode(ISD::UNDEF, EltVT); + else if (EltVT.isFloatingPoint()) + Constants[i] = DAG.getConstantFP(0, EltVT); + else + Constants[i] = DAG.getConstant(0, EltVT); + } + return DAG.getMergeValues(&Constants[0], Constants.size()); + } - SmallVector Ops; - Ops.assign(NumElements, DAG.getNode(ISD::UNDEF, PVT)); - - // Create a VConstant node with generic Vector type. - MVT::ValueType VT = MVT::getVectorType(PVT, NumElements); - return N = DAG.getNode(ISD::BUILD_VECTOR, VT, - &Ops[0], Ops.size()); - } else if (ConstantFP *CFP = dyn_cast(C)) { - return N = DAG.getConstantFP(CFP->getValueAPF(), VT); - } else if (const VectorType *PTy = dyn_cast(VTy)) { - unsigned NumElements = PTy->getNumElements(); - MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); - - // Now that we know the number and type of the elements, push a - // Constant or ConstantFP node onto the ops list for each element of - // the vector constant. - SmallVector Ops; - if (ConstantVector *CP = dyn_cast(C)) { - for (unsigned i = 0; i != NumElements; ++i) - Ops.push_back(getValue(CP->getOperand(i))); - } else { - assert(isa(C) && "Unknown vector constant!"); - SDOperand Op; - if (MVT::isFloatingPoint(PVT)) - Op = DAG.getConstantFP(0, PVT); + if (const StructType *STy = dyn_cast(C->getType())) { + assert((isa(C) || isa(C)) && + "Unknown struct constant!"); + unsigned NumElts = STy->getNumElements(); + if (NumElts == 0) + return SDOperand(); // empty struct + SmallVector Constants(NumElts); + for (unsigned i = 0, e = NumElts; i != e; ++i) { + MVT EltVT = TLI.getValueType(STy->getElementType(i)); + if (isa(C)) + Constants[i] = DAG.getNode(ISD::UNDEF, EltVT); + else if (EltVT.isFloatingPoint()) + Constants[i] = DAG.getConstantFP(0, EltVT); else - Op = DAG.getConstant(0, PVT); - Ops.assign(NumElements, Op); + Constants[i] = DAG.getConstant(0, EltVT); } - - // Create a BUILD_VECTOR node. - MVT::ValueType VT = MVT::getVectorType(PVT, NumElements); - return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], - Ops.size()); + return DAG.getMergeValues(&Constants[0], Constants.size()); + } + + const VectorType *VecTy = cast(V->getType()); + unsigned NumElements = VecTy->getNumElements(); + + // Now that we know the number and type of the elements, get that number of + // elements into the Ops array based on what kind of constant it is. + SmallVector Ops; + if (ConstantVector *CP = dyn_cast(C)) { + for (unsigned i = 0; i != NumElements; ++i) + Ops.push_back(getValue(CP->getOperand(i))); } else { - // Canonicalize all constant ints to be unsigned. - return N = DAG.getConstant(cast(C)->getZExtValue(),VT); + assert((isa(C) || isa(C)) && + "Unknown vector constant!"); + MVT EltVT = TLI.getValueType(VecTy->getElementType()); + + SDOperand Op; + if (isa(C)) + Op = DAG.getNode(ISD::UNDEF, EltVT); + else if (EltVT.isFloatingPoint()) + Op = DAG.getConstantFP(0, EltVT); + else + Op = DAG.getConstant(0, EltVT); + Ops.assign(NumElements, Op); } + + // Create a BUILD_VECTOR node. + return NodeMap[V] = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } + // If this is a static alloca, generate it as the frameindex instead of + // computation. if (const AllocaInst *AI = dyn_cast(V)) { std::map::iterator SI = - FuncInfo.StaticAllocaMap.find(AI); + FuncInfo.StaticAllocaMap.find(AI); if (SI != FuncInfo.StaticAllocaMap.end()) return DAG.getFrameIndex(SI->second, TLI.getPointerTy()); } @@ -1006,55 +1246,54 @@ SDOperand SelectionDAGLowering::getValue(const Value *V) { unsigned InReg = FuncInfo.ValueMap[V]; assert(InReg && "Value not in map!"); - MVT::ValueType RegisterVT = TLI.getRegisterType(VT); - unsigned NumRegs = TLI.getNumRegisters(VT); - - std::vector Regs(NumRegs); - for (unsigned i = 0; i != NumRegs; ++i) - Regs[i] = InReg + i; - - RegsForValue RFV(Regs, RegisterVT, VT); + RegsForValue RFV(TLI, InReg, V->getType()); SDOperand Chain = DAG.getEntryNode(); - return RFV.getCopyFromRegs(DAG, Chain, NULL); } void SelectionDAGLowering::visitRet(ReturnInst &I) { if (I.getNumOperands() == 0) { - DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, getRoot())); + DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, getControlRoot())); return; } + SmallVector NewValues; - NewValues.push_back(getRoot()); - for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { + NewValues.push_back(getControlRoot()); + for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { SDOperand RetOp = getValue(I.getOperand(i)); - MVT::ValueType VT = RetOp.getValueType(); - - // FIXME: C calling convention requires the return type to be promoted to - // at least 32-bit. But this is not necessary for non-C calling conventions. - if (MVT::isInteger(VT)) { - MVT::ValueType MinVT = TLI.getRegisterType(MVT::i32); - if (MVT::getSizeInBits(VT) < MVT::getSizeInBits(MinVT)) - VT = MinVT; - } - - unsigned NumParts = TLI.getNumRegisters(VT); - MVT::ValueType PartVT = TLI.getRegisterType(VT); - SmallVector Parts(NumParts); - ISD::NodeType ExtendKind = ISD::ANY_EXTEND; - - const Function *F = I.getParent()->getParent(); - if (F->paramHasAttr(0, ParamAttr::SExt)) - ExtendKind = ISD::SIGN_EXTEND; - else if (F->paramHasAttr(0, ParamAttr::ZExt)) - ExtendKind = ISD::ZERO_EXTEND; - getCopyToParts(DAG, RetOp, &Parts[0], NumParts, PartVT, ExtendKind); + SmallVector ValueVTs; + ComputeValueVTs(TLI, I.getOperand(i)->getType(), ValueVTs); + for (unsigned j = 0, f = ValueVTs.size(); j != f; ++j) { + MVT VT = ValueVTs[j]; + + // FIXME: C calling convention requires the return type to be promoted to + // at least 32-bit. But this is not necessary for non-C calling conventions. + if (VT.isInteger()) { + MVT MinVT = TLI.getRegisterType(MVT::i32); + if (VT.bitsLT(MinVT)) + VT = MinVT; + } - for (unsigned i = 0; i < NumParts; ++i) { - NewValues.push_back(Parts[i]); - NewValues.push_back(DAG.getConstant(false, MVT::i32)); + unsigned NumParts = TLI.getNumRegisters(VT); + MVT PartVT = TLI.getRegisterType(VT); + SmallVector Parts(NumParts); + ISD::NodeType ExtendKind = ISD::ANY_EXTEND; + + const Function *F = I.getParent()->getParent(); + if (F->paramHasAttr(0, ParamAttr::SExt)) + ExtendKind = ISD::SIGN_EXTEND; + else if (F->paramHasAttr(0, ParamAttr::ZExt)) + ExtendKind = ISD::ZERO_EXTEND; + + getCopyToParts(DAG, SDOperand(RetOp.Val, RetOp.ResNo + j), + &Parts[0], NumParts, PartVT, ExtendKind); + + for (unsigned i = 0; i < NumParts; ++i) { + NewValues.push_back(Parts[i]); + NewValues.push_back(DAG.getArgFlags(ISD::ArgFlagsTy())); + } } } DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, @@ -1072,7 +1311,7 @@ void SelectionDAGLowering::ExportFromCurrentBlock(Value *V) { if (FuncInfo.isExportedInst(V)) return; unsigned Reg = FuncInfo.InitializeRegForValue(V); - PendingLoads.push_back(CopyValueToVirtualRegister(V, Reg)); + CopyValueToVirtualRegister(V, Reg); } bool SelectionDAGLowering::isExportableFromCurrentBlock(Value *V, @@ -1160,8 +1399,8 @@ void SelectionDAGLowering::FindMergedConditions(Value *Cond, case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; - case FCmpInst::FCMP_ORD: FOC = ISD::SETEQ; FPC = ISD::SETO; break; - case FCmpInst::FCMP_UNO: FOC = ISD::SETNE; FPC = ISD::SETUO; break; + case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; + case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; @@ -1195,8 +1434,9 @@ void SelectionDAGLowering::FindMergedConditions(Value *Cond, // Create TmpBB after CurBB. MachineFunction::iterator BBI = CurBB; - MachineBasicBlock *TmpBB = new MachineBasicBlock(CurBB->getBasicBlock()); - CurBB->getParent()->getBasicBlockList().insert(++BBI, TmpBB); + MachineFunction &MF = DAG.getMachineFunction(); + MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); + CurBB->getParent()->insert(++BBI, TmpBB); if (Opc == Instruction::Or) { // Codegen X | Y as: @@ -1261,13 +1501,13 @@ void SelectionDAGLowering::visitBr(BranchInst &I) { NextBlock = BBI; if (I.isUnconditional()) { + // Update machine-CFG edges. + CurMBB->addSuccessor(Succ0MBB); + // If this is not a fall-through branch, emit the branch. if (Succ0MBB != NextBlock) - DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getRoot(), + DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getControlRoot(), DAG.getBasicBlock(Succ0MBB))); - - // Update machine-CFG edges. - CurMBB->addSuccessor(Succ0MBB); return; } @@ -1317,7 +1557,7 @@ void SelectionDAGLowering::visitBr(BranchInst &I) { // Okay, we decided not to do this, remove any inserted MBB's and clear // SwitchCases. for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) - CurMBB->getParent()->getBasicBlockList().erase(SwitchCases[i].ThisBB); + CurMBB->getParent()->erase(SwitchCases[i].ThisBB); SwitchCases.clear(); } @@ -1355,7 +1595,7 @@ void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { uint64_t High = cast(CB.CmpRHS)->getSExtValue(); SDOperand CmpOp = getValue(CB.CmpMHS); - MVT::ValueType VT = CmpOp.getValueType(); + MVT VT = CmpOp.getValueType(); if (cast(CB.CmpLHS)->isMinValue(true)) { Cond = DAG.getSetCC(MVT::i1, CmpOp, DAG.getConstant(High, VT), ISD::SETLE); @@ -1364,9 +1604,12 @@ void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { Cond = DAG.getSetCC(MVT::i1, SUB, DAG.getConstant(High-Low, VT), ISD::SETULE); } - } + // Update successor info + CurMBB->addSuccessor(CB.TrueBB); + CurMBB->addSuccessor(CB.FalseBB); + // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; @@ -1381,24 +1624,21 @@ void SelectionDAGLowering::visitSwitchCase(SelectionDAGISel::CaseBlock &CB) { SDOperand True = DAG.getConstant(1, Cond.getValueType()); Cond = DAG.getNode(ISD::XOR, Cond.getValueType(), Cond, True); } - SDOperand BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, getRoot(), Cond, + SDOperand BrCond = DAG.getNode(ISD::BRCOND, MVT::Other, getControlRoot(), Cond, DAG.getBasicBlock(CB.TrueBB)); if (CB.FalseBB == NextBlock) DAG.setRoot(BrCond); else DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrCond, DAG.getBasicBlock(CB.FalseBB))); - // Update successor info - CurMBB->addSuccessor(CB.TrueBB); - CurMBB->addSuccessor(CB.FalseBB); } /// visitJumpTable - Emit JumpTable node in the current MBB void SelectionDAGLowering::visitJumpTable(SelectionDAGISel::JumpTable &JT) { // Emit the code for the jump table assert(JT.Reg != -1U && "Should lower JT Header first!"); - MVT::ValueType PTy = TLI.getPointerTy(); - SDOperand Index = DAG.getCopyFromReg(getRoot(), JT.Reg, PTy); + MVT PTy = TLI.getPointerTy(); + SDOperand Index = DAG.getCopyFromReg(getControlRoot(), JT.Reg, PTy); SDOperand Table = DAG.getJumpTable(JT.JTI, PTy); DAG.setRoot(DAG.getNode(ISD::BR_JT, MVT::Other, Index.getValue(1), Table, Index)); @@ -1413,7 +1653,7 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, // and conditional branch to default mbb if the result is greater than the // difference between smallest and largest cases. SDOperand SwitchOp = getValue(JTH.SValue); - MVT::ValueType VT = SwitchOp.getValueType(); + MVT VT = SwitchOp.getValueType(); SDOperand SUB = DAG.getNode(ISD::SUB, VT, SwitchOp, DAG.getConstant(JTH.First, VT)); @@ -1422,19 +1662,19 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, // register so it can be used as an index into the jump table in a // subsequent basic block. This value may be smaller or larger than the // target's pointer type, and therefore require extension or truncating. - if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy())) + if (VT.bitsGT(TLI.getPointerTy())) SwitchOp = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), SUB); else SwitchOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), SUB); unsigned JumpTableReg = FuncInfo.MakeReg(TLI.getPointerTy()); - SDOperand CopyTo = DAG.getCopyToReg(getRoot(), JumpTableReg, SwitchOp); + SDOperand CopyTo = DAG.getCopyToReg(getControlRoot(), JumpTableReg, SwitchOp); JT.Reg = JumpTableReg; // Emit the range check for the jump table, and branch to the default // block for the switch statement if the value being switched on exceeds // the largest case in the switch. - SDOperand CMP = DAG.getSetCC(TLI.getSetCCResultTy(), SUB, + SDOperand CMP = DAG.getSetCC(TLI.getSetCCResultType(SUB), SUB, DAG.getConstant(JTH.Last-JTH.First,VT), ISD::SETUGT); @@ -1462,17 +1702,17 @@ void SelectionDAGLowering::visitJumpTableHeader(SelectionDAGISel::JumpTable &JT, void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B) { // Subtract the minimum value SDOperand SwitchOp = getValue(B.SValue); - MVT::ValueType VT = SwitchOp.getValueType(); + MVT VT = SwitchOp.getValueType(); SDOperand SUB = DAG.getNode(ISD::SUB, VT, SwitchOp, DAG.getConstant(B.First, VT)); // Check range - SDOperand RangeCmp = DAG.getSetCC(TLI.getSetCCResultTy(), SUB, + SDOperand RangeCmp = DAG.getSetCC(TLI.getSetCCResultType(SUB), SUB, DAG.getConstant(B.Range, VT), ISD::SETUGT); SDOperand ShiftOp; - if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getShiftAmountTy())) + if (VT.bitsGT(TLI.getShiftAmountTy())) ShiftOp = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), SUB); else ShiftOp = DAG.getNode(ISD::ZERO_EXTEND, TLI.getShiftAmountTy(), SUB); @@ -1483,12 +1723,9 @@ void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B) ShiftOp); unsigned SwitchReg = FuncInfo.MakeReg(TLI.getPointerTy()); - SDOperand CopyTo = DAG.getCopyToReg(getRoot(), SwitchReg, SwitchVal); + SDOperand CopyTo = DAG.getCopyToReg(getControlRoot(), SwitchReg, SwitchVal); B.Reg = SwitchReg; - SDOperand BrRange = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, RangeCmp, - DAG.getBasicBlock(B.Default)); - // Set NextBlock to be the MBB immediately after the current one, if any. // This is used to avoid emitting unnecessary branches to the next block. MachineBasicBlock *NextBlock = 0; @@ -1497,15 +1734,19 @@ void SelectionDAGLowering::visitBitTestHeader(SelectionDAGISel::BitTestBlock &B) NextBlock = BBI; MachineBasicBlock* MBB = B.Cases[0].ThisBB; + + CurMBB->addSuccessor(B.Default); + CurMBB->addSuccessor(MBB); + + SDOperand BrRange = DAG.getNode(ISD::BRCOND, MVT::Other, CopyTo, RangeCmp, + DAG.getBasicBlock(B.Default)); + if (MBB == NextBlock) DAG.setRoot(BrRange); else DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, CopyTo, DAG.getBasicBlock(MBB))); - CurMBB->addSuccessor(B.Default); - CurMBB->addSuccessor(MBB); - return; } @@ -1514,16 +1755,19 @@ void SelectionDAGLowering::visitBitTestCase(MachineBasicBlock* NextMBB, unsigned Reg, SelectionDAGISel::BitTestCase &B) { // Emit bit tests and jumps - SDOperand SwitchVal = DAG.getCopyFromReg(getRoot(), Reg, TLI.getPointerTy()); + SDOperand SwitchVal = DAG.getCopyFromReg(getControlRoot(), Reg, + TLI.getPointerTy()); - SDOperand AndOp = DAG.getNode(ISD::AND, TLI.getPointerTy(), - SwitchVal, - DAG.getConstant(B.Mask, - TLI.getPointerTy())); - SDOperand AndCmp = DAG.getSetCC(TLI.getSetCCResultTy(), AndOp, + SDOperand AndOp = DAG.getNode(ISD::AND, TLI.getPointerTy(), SwitchVal, + DAG.getConstant(B.Mask, TLI.getPointerTy())); + SDOperand AndCmp = DAG.getSetCC(TLI.getSetCCResultType(AndOp), AndOp, DAG.getConstant(0, TLI.getPointerTy()), ISD::SETNE); - SDOperand BrAnd = DAG.getNode(ISD::BRCOND, MVT::Other, getRoot(), + + CurMBB->addSuccessor(B.TargetBB); + CurMBB->addSuccessor(NextMBB); + + SDOperand BrAnd = DAG.getNode(ISD::BRCOND, MVT::Other, getControlRoot(), AndCmp, DAG.getBasicBlock(B.TargetBB)); // Set NextBlock to be the MBB immediately after the current one, if any. @@ -1539,9 +1783,6 @@ void SelectionDAGLowering::visitBitTestCase(MachineBasicBlock* NextMBB, DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, BrAnd, DAG.getBasicBlock(NextMBB))); - CurMBB->addSuccessor(B.TargetBB); - CurMBB->addSuccessor(NextMBB); - return; } @@ -1560,16 +1801,16 @@ void SelectionDAGLowering::visitInvoke(InvokeInst &I) { if (!I.use_empty()) { DenseMap::iterator VMI = FuncInfo.ValueMap.find(&I); if (VMI != FuncInfo.ValueMap.end()) - DAG.setRoot(CopyValueToVirtualRegister(&I, VMI->second)); + CopyValueToVirtualRegister(&I, VMI->second); } - // Drop into normal successor. - DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getRoot(), - DAG.getBasicBlock(Return))); - // Update successor info CurMBB->addSuccessor(Return); CurMBB->addSuccessor(LandingPad); + + // Drop into normal successor. + DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getControlRoot(), + DAG.getBasicBlock(Return))); } void SelectionDAGLowering::visitUnwind(UnwindInst &I) { @@ -1623,8 +1864,8 @@ bool SelectionDAGLowering::handleSmallSwitchRange(CaseRec& CR, for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) { MachineBasicBlock *FallThrough; if (I != E-1) { - FallThrough = new MachineBasicBlock(CurBlock->getBasicBlock()); - CurMF->getBasicBlockList().insert(BBI, FallThrough); + FallThrough = CurMF->CreateMachineBasicBlock(CurBlock->getBasicBlock()); + CurMF->insert(BBI, FallThrough); } else { // If the last case doesn't match, go to the default block. FallThrough = Default; @@ -1707,8 +1948,8 @@ bool SelectionDAGLowering::handleJTSwitchCase(CaseRec& CR, // of the jump table, and jumping to it. Update successor information; // we will either branch to the default case for the switch, or the jump // table. - MachineBasicBlock *JumpTableBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, JumpTableBB); + MachineBasicBlock *JumpTableBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, JumpTableBB); CR.CaseBB->addSuccessor(Default); CR.CaseBB->addSuccessor(JumpTableBB); @@ -1845,8 +2086,8 @@ bool SelectionDAGLowering::handleBTSplitSwitchCase(CaseRec& CR, (cast(CR.GE)->getSExtValue() + 1LL)) { TrueBB = LHSR.first->BB; } else { - TrueBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, TrueBB); + TrueBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, TrueBB); WorkList.push_back(CaseRec(TrueBB, C, CR.GE, LHSR)); } @@ -1859,8 +2100,8 @@ bool SelectionDAGLowering::handleBTSplitSwitchCase(CaseRec& CR, (cast(CR.LT)->getSExtValue() - 1LL)) { FalseBB = RHSR.first->BB; } else { - FalseBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, FalseBB); + FalseBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, FalseBB); WorkList.push_back(CaseRec(FalseBB,CR.LT,C,RHSR)); } @@ -1885,7 +2126,7 @@ bool SelectionDAGLowering::handleBitTestsSwitchCase(CaseRec& CR, CaseRecVector& WorkList, Value* SV, MachineBasicBlock* Default){ - unsigned IntPtrBits = MVT::getSizeInBits(TLI.getPointerTy()); + unsigned IntPtrBits = TLI.getPointerTy().getSizeInBits(); Case& FrontCase = *CR.Range.first; Case& BackCase = *(CR.Range.second-1); @@ -1982,8 +2223,8 @@ bool SelectionDAGLowering::handleBitTestsSwitchCase(CaseRec& CR, DOUT << "Mask: " << CasesBits[i].Mask << ", Bits: " << CasesBits[i].Bits << ", BB: " << CasesBits[i].BB << "\n"; - MachineBasicBlock *CaseBB = new MachineBasicBlock(LLVMBB); - CurMF->getBasicBlockList().insert(BBI, CaseBB); + MachineBasicBlock *CaseBB = CurMF->CreateMachineBasicBlock(LLVMBB); + CurMF->insert(BBI, CaseBB); BTC.push_back(SelectionDAGISel::BitTestCase(CasesBits[i].Mask, CaseBB, CasesBits[i].BB)); @@ -2002,7 +2243,7 @@ bool SelectionDAGLowering::handleBitTestsSwitchCase(CaseRec& CR, } -// Clusterify - Transform simple list of Cases into list of CaseRange's +/// Clusterify - Transform simple list of Cases into list of CaseRange's unsigned SelectionDAGLowering::Clusterify(CaseVector& Cases, const SwitchInst& SI) { unsigned numCmps = 0; @@ -2058,11 +2299,11 @@ void SelectionDAGLowering::visitSwitch(SwitchInst &SI) { // Update machine-CFG edges. // If this is not a fall-through branch, emit the branch. + CurMBB->addSuccessor(Default); if (Default != NextBlock) - DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getRoot(), + DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getControlRoot(), DAG.getBasicBlock(Default))); - - CurMBB->addSuccessor(Default); + return; } @@ -2151,10 +2392,9 @@ void SelectionDAGLowering::visitShift(User &I, unsigned Opcode) { SDOperand Op1 = getValue(I.getOperand(0)); SDOperand Op2 = getValue(I.getOperand(1)); - if (MVT::getSizeInBits(TLI.getShiftAmountTy()) < - MVT::getSizeInBits(Op2.getValueType())) + if (TLI.getShiftAmountTy().bitsLT(Op2.getValueType())) Op2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Op2); - else if (TLI.getShiftAmountTy() > Op2.getValueType()) + else if (TLI.getShiftAmountTy().bitsGT(Op2.getValueType())) Op2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Op2); setValue(&I, DAG.getNode(Opcode, Op1.getValueType(), Op1, Op2)); @@ -2205,8 +2445,8 @@ void SelectionDAGLowering::visitFCmp(User &I) { case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; - case FCmpInst::FCMP_ORD: FOC = ISD::SETEQ; FPC = ISD::SETO; break; - case FCmpInst::FCMP_UNO: FOC = ISD::SETNE; FPC = ISD::SETUO; break; + case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; + case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; @@ -2226,6 +2466,75 @@ void SelectionDAGLowering::visitFCmp(User &I) { setValue(&I, DAG.getSetCC(MVT::i1, Op1, Op2, Condition)); } +void SelectionDAGLowering::visitVICmp(User &I) { + ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; + if (VICmpInst *IC = dyn_cast(&I)) + predicate = IC->getPredicate(); + else if (ConstantExpr *IC = dyn_cast(&I)) + predicate = ICmpInst::Predicate(IC->getPredicate()); + SDOperand Op1 = getValue(I.getOperand(0)); + SDOperand Op2 = getValue(I.getOperand(1)); + ISD::CondCode Opcode; + switch (predicate) { + case ICmpInst::ICMP_EQ : Opcode = ISD::SETEQ; break; + case ICmpInst::ICMP_NE : Opcode = ISD::SETNE; break; + case ICmpInst::ICMP_UGT : Opcode = ISD::SETUGT; break; + case ICmpInst::ICMP_UGE : Opcode = ISD::SETUGE; break; + case ICmpInst::ICMP_ULT : Opcode = ISD::SETULT; break; + case ICmpInst::ICMP_ULE : Opcode = ISD::SETULE; break; + case ICmpInst::ICMP_SGT : Opcode = ISD::SETGT; break; + case ICmpInst::ICMP_SGE : Opcode = ISD::SETGE; break; + case ICmpInst::ICMP_SLT : Opcode = ISD::SETLT; break; + case ICmpInst::ICMP_SLE : Opcode = ISD::SETLE; break; + default: + assert(!"Invalid ICmp predicate value"); + Opcode = ISD::SETEQ; + break; + } + setValue(&I, DAG.getVSetCC(Op1.getValueType(), Op1, Op2, Opcode)); +} + +void SelectionDAGLowering::visitVFCmp(User &I) { + FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; + if (VFCmpInst *FC = dyn_cast(&I)) + predicate = FC->getPredicate(); + else if (ConstantExpr *FC = dyn_cast(&I)) + predicate = FCmpInst::Predicate(FC->getPredicate()); + SDOperand Op1 = getValue(I.getOperand(0)); + SDOperand Op2 = getValue(I.getOperand(1)); + ISD::CondCode Condition, FOC, FPC; + switch (predicate) { + case FCmpInst::FCMP_FALSE: FOC = FPC = ISD::SETFALSE; break; + case FCmpInst::FCMP_OEQ: FOC = ISD::SETEQ; FPC = ISD::SETOEQ; break; + case FCmpInst::FCMP_OGT: FOC = ISD::SETGT; FPC = ISD::SETOGT; break; + case FCmpInst::FCMP_OGE: FOC = ISD::SETGE; FPC = ISD::SETOGE; break; + case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; + case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; + case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; + case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; + case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; + case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; + case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; + case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; + case FCmpInst::FCMP_ULT: FOC = ISD::SETLT; FPC = ISD::SETULT; break; + case FCmpInst::FCMP_ULE: FOC = ISD::SETLE; FPC = ISD::SETULE; break; + case FCmpInst::FCMP_UNE: FOC = ISD::SETNE; FPC = ISD::SETUNE; break; + case FCmpInst::FCMP_TRUE: FOC = FPC = ISD::SETTRUE; break; + default: + assert(!"Invalid VFCmp predicate value"); + FOC = FPC = ISD::SETFALSE; + break; + } + if (FiniteOnlyFPMath()) + Condition = FOC; + else + Condition = FPC; + + MVT DestVT = TLI.getValueType(I.getType()); + + setValue(&I, DAG.getVSetCC(DestVT, Op1, Op2, Condition)); +} + void SelectionDAGLowering::visitSelect(User &I) { SDOperand Cond = getValue(I.getOperand(0)); SDOperand TrueVal = getValue(I.getOperand(1)); @@ -2238,7 +2547,7 @@ void SelectionDAGLowering::visitSelect(User &I) { void SelectionDAGLowering::visitTrunc(User &I) { // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::TRUNCATE, DestVT, N)); } @@ -2246,7 +2555,7 @@ void SelectionDAGLowering::visitZExt(User &I) { // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). // ZExt also can't be a cast to bool for same reason. So, nothing much to do SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, DestVT, N)); } @@ -2254,49 +2563,49 @@ void SelectionDAGLowering::visitSExt(User &I) { // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). // SExt also can't be a cast to bool for same reason. So, nothing much to do SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, DestVT, N)); } void SelectionDAGLowering::visitFPTrunc(User &I) { // FPTrunc is never a no-op cast, no need to check SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_ROUND, DestVT, N, DAG.getIntPtrConstant(0))); } void SelectionDAGLowering::visitFPExt(User &I){ // FPTrunc is never a no-op cast, no need to check SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_EXTEND, DestVT, N)); } void SelectionDAGLowering::visitFPToUI(User &I) { // FPToUI is never a no-op cast, no need to check SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_TO_UINT, DestVT, N)); } void SelectionDAGLowering::visitFPToSI(User &I) { // FPToSI is never a no-op cast, no need to check SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::FP_TO_SINT, DestVT, N)); } void SelectionDAGLowering::visitUIToFP(User &I) { // UIToFP is never a no-op cast, no need to check SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::UINT_TO_FP, DestVT, N)); } void SelectionDAGLowering::visitSIToFP(User &I){ // UIToFP is never a no-op cast, no need to check SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getNode(ISD::SINT_TO_FP, DestVT, N)); } @@ -2304,10 +2613,10 @@ void SelectionDAGLowering::visitPtrToInt(User &I) { // What to do depends on the size of the integer and the size of the pointer. // We can either truncate, zero extend, or no-op, accordingly. SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType SrcVT = N.getValueType(); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT SrcVT = N.getValueType(); + MVT DestVT = TLI.getValueType(I.getType()); SDOperand Result; - if (MVT::getSizeInBits(DestVT) < MVT::getSizeInBits(SrcVT)) + if (DestVT.bitsLT(SrcVT)) Result = DAG.getNode(ISD::TRUNCATE, DestVT, N); else // Note: ZERO_EXTEND can handle cases where the sizes are equal too @@ -2319,9 +2628,9 @@ void SelectionDAGLowering::visitIntToPtr(User &I) { // What to do depends on the size of the integer and the size of the pointer. // We can either truncate, zero extend, or no-op, accordingly. SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType SrcVT = N.getValueType(); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); - if (MVT::getSizeInBits(DestVT) < MVT::getSizeInBits(SrcVT)) + MVT SrcVT = N.getValueType(); + MVT DestVT = TLI.getValueType(I.getType()); + if (DestVT.bitsLT(SrcVT)) setValue(&I, DAG.getNode(ISD::TRUNCATE, DestVT, N)); else // Note: ZERO_EXTEND can handle cases where the sizes are equal too @@ -2330,7 +2639,7 @@ void SelectionDAGLowering::visitIntToPtr(User &I) { void SelectionDAGLowering::visitBitCast(User &I) { SDOperand N = getValue(I.getOperand(0)); - MVT::ValueType DestVT = TLI.getValueType(I.getType()); + MVT DestVT = TLI.getValueType(I.getType()); // BitCast assures us that source and destination are the same size so this // is either a BIT_CONVERT or a no-op. @@ -2369,6 +2678,72 @@ void SelectionDAGLowering::visitShuffleVector(User &I) { V1, V2, Mask)); } +void SelectionDAGLowering::visitInsertValue(InsertValueInst &I) { + const Value *Op0 = I.getOperand(0); + const Value *Op1 = I.getOperand(1); + const Type *AggTy = I.getType(); + const Type *ValTy = Op1->getType(); + bool IntoUndef = isa(Op0); + bool FromUndef = isa(Op1); + + unsigned LinearIndex = ComputeLinearIndex(TLI, AggTy, + I.idx_begin(), I.idx_end()); + + SmallVector AggValueVTs; + ComputeValueVTs(TLI, AggTy, AggValueVTs); + SmallVector ValValueVTs; + ComputeValueVTs(TLI, ValTy, ValValueVTs); + + unsigned NumAggValues = AggValueVTs.size(); + unsigned NumValValues = ValValueVTs.size(); + SmallVector Values(NumAggValues); + + SDOperand Agg = getValue(Op0); + SDOperand Val = getValue(Op1); + unsigned i = 0; + // Copy the beginning value(s) from the original aggregate. + for (; i != LinearIndex; ++i) + Values[i] = IntoUndef ? DAG.getNode(ISD::UNDEF, AggValueVTs[i]) : + SDOperand(Agg.Val, Agg.ResNo + i); + // Copy values from the inserted value(s). + for (; i != LinearIndex + NumValValues; ++i) + Values[i] = FromUndef ? DAG.getNode(ISD::UNDEF, AggValueVTs[i]) : + SDOperand(Val.Val, Val.ResNo + i - LinearIndex); + // Copy remaining value(s) from the original aggregate. + for (; i != NumAggValues; ++i) + Values[i] = IntoUndef ? DAG.getNode(ISD::UNDEF, AggValueVTs[i]) : + SDOperand(Agg.Val, Agg.ResNo + i); + + setValue(&I, DAG.getMergeValues(DAG.getVTList(&AggValueVTs[0], NumAggValues), + &Values[0], NumAggValues)); +} + +void SelectionDAGLowering::visitExtractValue(ExtractValueInst &I) { + const Value *Op0 = I.getOperand(0); + const Type *AggTy = Op0->getType(); + const Type *ValTy = I.getType(); + bool OutOfUndef = isa(Op0); + + unsigned LinearIndex = ComputeLinearIndex(TLI, AggTy, + I.idx_begin(), I.idx_end()); + + SmallVector ValValueVTs; + ComputeValueVTs(TLI, ValTy, ValValueVTs); + + unsigned NumValValues = ValValueVTs.size(); + SmallVector Values(NumValValues); + + SDOperand Agg = getValue(Op0); + // Copy out the selected value(s). + for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) + Values[i - LinearIndex] = + OutOfUndef ? DAG.getNode(ISD::UNDEF, Agg.Val->getValueType(Agg.ResNo + i)) : + SDOperand(Agg.Val, Agg.ResNo + i); + + setValue(&I, DAG.getMergeValues(DAG.getVTList(&ValValueVTs[0], NumValValues), + &Values[0], NumValValues)); +} + void SelectionDAGLowering::visitGetElementPtr(User &I) { SDOperand N = getValue(I.getOperand(0)); @@ -2405,9 +2780,9 @@ void SelectionDAGLowering::visitGetElementPtr(User &I) { // If the index is smaller or larger than intptr_t, truncate or extend // it. - if (IdxN.getValueType() < N.getValueType()) { + if (IdxN.getValueType().bitsLT(N.getValueType())) { IdxN = DAG.getNode(ISD::SIGN_EXTEND, N.getValueType(), IdxN); - } else if (IdxN.getValueType() > N.getValueType()) + } else if (IdxN.getValueType().bitsGT(N.getValueType())) IdxN = DAG.getNode(ISD::TRUNCATE, N.getValueType(), IdxN); // If this is a multiply by a power of two, turn it into a shl @@ -2441,10 +2816,10 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { I.getAlignment()); SDOperand AllocSize = getValue(I.getArraySize()); - MVT::ValueType IntPtr = TLI.getPointerTy(); - if (IntPtr < AllocSize.getValueType()) + MVT IntPtr = TLI.getPointerTy(); + if (IntPtr.bitsLT(AllocSize.getValueType())) AllocSize = DAG.getNode(ISD::TRUNCATE, IntPtr, AllocSize); - else if (IntPtr > AllocSize.getValueType()) + else if (IntPtr.bitsGT(AllocSize.getValueType())) AllocSize = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, AllocSize); AllocSize = DAG.getNode(ISD::MUL, IntPtr, AllocSize, @@ -2467,7 +2842,7 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { DAG.getIntPtrConstant(~(uint64_t)(StackAlign-1))); SDOperand Ops[] = { getRoot(), AllocSize, DAG.getIntPtrConstant(Align) }; - const MVT::ValueType *VTs = DAG.getNodeValueTypes(AllocSize.getValueType(), + const MVT *VTs = DAG.getNodeValueTypes(AllocSize.getValueType(), MVT::Other); SDOperand DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, VTs, 2, Ops, 3); setValue(&I, DSA); @@ -2479,7 +2854,19 @@ void SelectionDAGLowering::visitAlloca(AllocaInst &I) { } void SelectionDAGLowering::visitLoad(LoadInst &I) { - SDOperand Ptr = getValue(I.getOperand(0)); + const Value *SV = I.getOperand(0); + SDOperand Ptr = getValue(SV); + + const Type *Ty = I.getType(); + bool isVolatile = I.isVolatile(); + unsigned Alignment = I.getAlignment(); + + SmallVector ValueVTs; + SmallVector Offsets; + ComputeValueVTs(TLI, Ty, ValueVTs, &Offsets); + unsigned NumValues = ValueVTs.size(); + if (NumValues == 0) + return; SDOperand Root; if (I.isVolatile()) @@ -2489,33 +2876,57 @@ void SelectionDAGLowering::visitLoad(LoadInst &I) { Root = DAG.getRoot(); } - setValue(&I, getLoadFrom(I.getType(), Ptr, I.getOperand(0), - Root, I.isVolatile(), I.getAlignment())); -} - -SDOperand SelectionDAGLowering::getLoadFrom(const Type *Ty, SDOperand Ptr, - const Value *SV, SDOperand Root, - bool isVolatile, - unsigned Alignment) { - SDOperand L = - DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SV, 0, - isVolatile, Alignment); - + SmallVector Values(NumValues); + SmallVector Chains(NumValues); + MVT PtrVT = Ptr.getValueType(); + for (unsigned i = 0; i != NumValues; ++i) { + SDOperand L = DAG.getLoad(ValueVTs[i], Root, + DAG.getNode(ISD::ADD, PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)), + SV, Offsets[i], + isVolatile, Alignment); + Values[i] = L; + Chains[i] = L.getValue(1); + } + + SDOperand Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, + &Chains[0], NumValues); if (isVolatile) - DAG.setRoot(L.getValue(1)); + DAG.setRoot(Chain); else - PendingLoads.push_back(L.getValue(1)); - - return L; + PendingLoads.push_back(Chain); + + setValue(&I, DAG.getMergeValues(DAG.getVTList(&ValueVTs[0], NumValues), + &Values[0], NumValues)); } void SelectionDAGLowering::visitStore(StoreInst &I) { Value *SrcV = I.getOperand(0); SDOperand Src = getValue(SrcV); - SDOperand Ptr = getValue(I.getOperand(1)); - DAG.setRoot(DAG.getStore(getRoot(), Src, Ptr, I.getOperand(1), 0, - I.isVolatile(), I.getAlignment())); + Value *PtrV = I.getOperand(1); + SDOperand Ptr = getValue(PtrV); + + SmallVector ValueVTs; + SmallVector Offsets; + ComputeValueVTs(TLI, SrcV->getType(), ValueVTs, &Offsets); + unsigned NumValues = ValueVTs.size(); + if (NumValues == 0) + return; + + SDOperand Root = getRoot(); + SmallVector Chains(NumValues); + MVT PtrVT = Ptr.getValueType(); + bool isVolatile = I.isVolatile(); + unsigned Alignment = I.getAlignment(); + for (unsigned i = 0; i != NumValues; ++i) + Chains[i] = DAG.getStore(Root, SDOperand(Src.Val, Src.ResNo + i), + DAG.getNode(ISD::ADD, PtrVT, Ptr, + DAG.getConstant(Offsets[i], PtrVT)), + PtrV, Offsets[i], + isVolatile, Alignment); + + DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumValues)); } /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC @@ -2547,14 +2958,14 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, Ops.push_back(Op); } - std::vector VTs; + std::vector VTs; if (I.getType() != Type::VoidTy) { - MVT::ValueType VT = TLI.getValueType(I.getType()); - if (MVT::isVector(VT)) { + MVT VT = TLI.getValueType(I.getType()); + if (VT.isVector()) { const VectorType *DestTy = cast(I.getType()); - MVT::ValueType EltVT = TLI.getValueType(DestTy->getElementType()); + MVT EltVT = TLI.getValueType(DestTy->getElementType()); - VT = MVT::getVectorType(EltVT, DestTy->getNumElements()); + VT = MVT::getVectorVT(EltVT, DestTy->getNumElements()); assert(VT != MVT::Other && "Intrinsic uses a non-legal type?"); } @@ -2564,7 +2975,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, if (HasChain) VTs.push_back(MVT::Other); - const MVT::ValueType *VTList = DAG.getNodeValueTypes(VTs); + const MVT *VTList = DAG.getNodeValueTypes(VTs); // Create the node. SDOperand Result; @@ -2587,7 +2998,7 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, } if (I.getType() != Type::VoidTy) { if (const VectorType *PTy = dyn_cast(I.getType())) { - MVT::ValueType VT = TLI.getValueType(PTy); + MVT VT = TLI.getValueType(PTy); Result = DAG.getNode(ISD::BIT_CONVERT, VT, Result); } setValue(&I, Result); @@ -2596,9 +3007,9 @@ void SelectionDAGLowering::visitTargetIntrinsic(CallInst &I, /// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V. static GlobalVariable *ExtractTypeInfo (Value *V) { - V = IntrinsicInst::StripPointerCasts(V); + V = V->stripPointerCasts(); GlobalVariable *GV = dyn_cast(V); - assert (GV || isa(V) && + assert ((GV || isa(V)) && "TypeInfo must be a global variable or NULL"); return GV; } @@ -2657,6 +3068,22 @@ static void addCatchInfo(CallInst &I, MachineModuleInfo *MMI, } } + +/// Inlined utility function to implement binary input atomic intrinsics for +// visitIntrinsicCall: I is a call instruction +// Op is the associated NodeType for I +const char * +SelectionDAGLowering::implVisitBinaryAtomic(CallInst& I, ISD::NodeType Op) { + SDOperand Root = getRoot(); + SDOperand L = DAG.getAtomic(Op, Root, + getValue(I.getOperand(1)), + getValue(I.getOperand(2)), + I.getOperand(1)); + setValue(&I, L); + DAG.setRoot(L.getValue(1)); + return 0; +} + /// visitIntrinsicCall - Lower the call to the specified intrinsic function. If /// we want to emit this as a call to a named external function, return the name /// otherwise lower it and return null. @@ -2685,36 +3112,58 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return "_longjmp"+!TLI.usesUnderscoreLongJmp(); break; case Intrinsic::memcpy_i32: - case Intrinsic::memcpy_i64: - visitMemIntrinsic(I, ISD::MEMCPY); + case Intrinsic::memcpy_i64: { + SDOperand Op1 = getValue(I.getOperand(1)); + SDOperand Op2 = getValue(I.getOperand(2)); + SDOperand Op3 = getValue(I.getOperand(3)); + unsigned Align = cast(I.getOperand(4))->getZExtValue(); + DAG.setRoot(DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Align, false, + I.getOperand(1), 0, I.getOperand(2), 0)); return 0; + } case Intrinsic::memset_i32: - case Intrinsic::memset_i64: - visitMemIntrinsic(I, ISD::MEMSET); + case Intrinsic::memset_i64: { + SDOperand Op1 = getValue(I.getOperand(1)); + SDOperand Op2 = getValue(I.getOperand(2)); + SDOperand Op3 = getValue(I.getOperand(3)); + unsigned Align = cast(I.getOperand(4))->getZExtValue(); + DAG.setRoot(DAG.getMemset(getRoot(), Op1, Op2, Op3, Align, + I.getOperand(1), 0)); return 0; + } case Intrinsic::memmove_i32: - case Intrinsic::memmove_i64: - visitMemIntrinsic(I, ISD::MEMMOVE); + case Intrinsic::memmove_i64: { + SDOperand Op1 = getValue(I.getOperand(1)); + SDOperand Op2 = getValue(I.getOperand(2)); + SDOperand Op3 = getValue(I.getOperand(3)); + unsigned Align = cast(I.getOperand(4))->getZExtValue(); + + // If the source and destination are known to not be aliases, we can + // lower memmove as memcpy. + uint64_t Size = -1ULL; + if (ConstantSDNode *C = dyn_cast(Op3)) + Size = C->getValue(); + if (AA.alias(I.getOperand(1), Size, I.getOperand(2), Size) == + AliasAnalysis::NoAlias) { + DAG.setRoot(DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Align, false, + I.getOperand(1), 0, I.getOperand(2), 0)); + return 0; + } + + DAG.setRoot(DAG.getMemmove(getRoot(), Op1, Op2, Op3, Align, + I.getOperand(1), 0, I.getOperand(2), 0)); return 0; - + } case Intrinsic::dbg_stoppoint: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); DbgStopPointInst &SPI = cast(I); if (MMI && SPI.getContext() && MMI->Verify(SPI.getContext())) { - SDOperand Ops[5]; - - Ops[0] = getRoot(); - Ops[1] = getValue(SPI.getLineValue()); - Ops[2] = getValue(SPI.getColumnValue()); - DebugInfoDesc *DD = MMI->getDescFor(SPI.getContext()); assert(DD && "Not a debug information descriptor"); - CompileUnitDesc *CompileUnit = cast(DD); - - Ops[3] = DAG.getString(CompileUnit->getFileName()); - Ops[4] = DAG.getString(CompileUnit->getDirectory()); - - DAG.setRoot(DAG.getNode(ISD::LOCATION, MVT::Other, Ops, 5)); + DAG.setRoot(DAG.getDbgStopPoint(getRoot(), + SPI.getLine(), + SPI.getColumn(), + cast(DD))); } return 0; @@ -2724,9 +3173,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DbgRegionStartInst &RSI = cast(I); if (MMI && RSI.getContext() && MMI->Verify(RSI.getContext())) { unsigned LabelID = MMI->RecordRegionStart(RSI.getContext()); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(LabelID, MVT::i32), - DAG.getConstant(0, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getRoot(), LabelID)); } return 0; @@ -2736,9 +3183,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DbgRegionEndInst &REI = cast(I); if (MMI && REI.getContext() && MMI->Verify(REI.getContext())) { unsigned LabelID = MMI->RecordRegionEnd(REI.getContext()); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(LabelID, MVT::i32), - DAG.getConstant(0, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::DBG_LABEL, getRoot(), LabelID)); } return 0; @@ -2755,8 +3200,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { assert(DD && "Not a debug information descriptor"); SubprogramDesc *Subprogram = cast(DD); const CompileUnitDesc *CompileUnit = Subprogram->getFile(); - unsigned SrcFile = MMI->RecordSource(CompileUnit->getDirectory(), - CompileUnit->getFileName()); + unsigned SrcFile = MMI->RecordSource(CompileUnit); // Record the source line but does create a label. It will be emitted // at asm emission time. MMI->RecordSourceLine(Subprogram->getLine(), 0, SrcFile); @@ -2775,32 +3219,28 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::eh_exception: { - if (ExceptionHandling) { - if (!CurMBB->isLandingPad()) { - // FIXME: Mark exception register as live in. Hack for PR1508. - unsigned Reg = TLI.getExceptionAddressRegister(); - if (Reg) CurMBB->addLiveIn(Reg); - } - // Insert the EXCEPTIONADDR instruction. - SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other); - SDOperand Ops[1]; - Ops[0] = DAG.getRoot(); - SDOperand Op = DAG.getNode(ISD::EXCEPTIONADDR, VTs, Ops, 1); - setValue(&I, Op); - DAG.setRoot(Op.getValue(1)); - } else { - setValue(&I, DAG.getConstant(0, TLI.getPointerTy())); + if (!CurMBB->isLandingPad()) { + // FIXME: Mark exception register as live in. Hack for PR1508. + unsigned Reg = TLI.getExceptionAddressRegister(); + if (Reg) CurMBB->addLiveIn(Reg); } + // Insert the EXCEPTIONADDR instruction. + SDVTList VTs = DAG.getVTList(TLI.getPointerTy(), MVT::Other); + SDOperand Ops[1]; + Ops[0] = DAG.getRoot(); + SDOperand Op = DAG.getNode(ISD::EXCEPTIONADDR, VTs, Ops, 1); + setValue(&I, Op); + DAG.setRoot(Op.getValue(1)); return 0; } case Intrinsic::eh_selector_i32: case Intrinsic::eh_selector_i64: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - MVT::ValueType VT = (Intrinsic == Intrinsic::eh_selector_i32 ? + MVT VT = (Intrinsic == Intrinsic::eh_selector_i32 ? MVT::i32 : MVT::i64); - if (ExceptionHandling && MMI) { + if (MMI) { if (CurMBB->isLandingPad()) addCatchInfo(I, MMI, CurMBB); else { @@ -2830,7 +3270,7 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::eh_typeid_for_i32: case Intrinsic::eh_typeid_for_i64: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - MVT::ValueType VT = (Intrinsic == Intrinsic::eh_typeid_for_i32 ? + MVT VT = (Intrinsic == Intrinsic::eh_typeid_for_i32 ? MVT::i32 : MVT::i64); if (MMI) { @@ -2850,11 +3290,11 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { case Intrinsic::eh_return: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - if (MMI && ExceptionHandling) { + if (MMI) { MMI->setCallsEHReturn(true); DAG.setRoot(DAG.getNode(ISD::EH_RETURN, MVT::Other, - getRoot(), + getControlRoot(), getValue(I.getOperand(1)), getValue(I.getOperand(2)))); } else { @@ -2873,32 +3313,27 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { } case Intrinsic::eh_dwarf_cfa: { - if (ExceptionHandling) { - MVT::ValueType VT = getValue(I.getOperand(1)).getValueType(); - SDOperand CfaArg; - if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(TLI.getPointerTy())) - CfaArg = DAG.getNode(ISD::TRUNCATE, - TLI.getPointerTy(), getValue(I.getOperand(1))); - else - CfaArg = DAG.getNode(ISD::SIGN_EXTEND, - TLI.getPointerTy(), getValue(I.getOperand(1))); - - SDOperand Offset = DAG.getNode(ISD::ADD, - TLI.getPointerTy(), - DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, - TLI.getPointerTy()), - CfaArg); - setValue(&I, DAG.getNode(ISD::ADD, - TLI.getPointerTy(), - DAG.getNode(ISD::FRAMEADDR, - TLI.getPointerTy(), - DAG.getConstant(0, - TLI.getPointerTy())), - Offset)); - } else { - setValue(&I, DAG.getConstant(0, TLI.getPointerTy())); - } - + MVT VT = getValue(I.getOperand(1)).getValueType(); + SDOperand CfaArg; + if (VT.bitsGT(TLI.getPointerTy())) + CfaArg = DAG.getNode(ISD::TRUNCATE, + TLI.getPointerTy(), getValue(I.getOperand(1))); + else + CfaArg = DAG.getNode(ISD::SIGN_EXTEND, + TLI.getPointerTy(), getValue(I.getOperand(1))); + + SDOperand Offset = DAG.getNode(ISD::ADD, + TLI.getPointerTy(), + DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, + TLI.getPointerTy()), + CfaArg); + setValue(&I, DAG.getNode(ISD::ADD, + TLI.getPointerTy(), + DAG.getNode(ISD::FRAMEADDR, + TLI.getPointerTy(), + DAG.getConstant(0, + TLI.getPointerTy())), + Offset)); return 0; } @@ -2960,21 +3395,21 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { return 0; case Intrinsic::cttz: { SDOperand Arg = getValue(I.getOperand(1)); - MVT::ValueType Ty = Arg.getValueType(); + MVT Ty = Arg.getValueType(); SDOperand result = DAG.getNode(ISD::CTTZ, Ty, Arg); setValue(&I, result); return 0; } case Intrinsic::ctlz: { SDOperand Arg = getValue(I.getOperand(1)); - MVT::ValueType Ty = Arg.getValueType(); + MVT Ty = Arg.getValueType(); SDOperand result = DAG.getNode(ISD::CTLZ, Ty, Arg); setValue(&I, result); return 0; } case Intrinsic::ctpop: { SDOperand Arg = getValue(I.getOperand(1)); - MVT::ValueType Ty = Arg.getValueType(); + MVT Ty = Arg.getValueType(); SDOperand result = DAG.getNode(ISD::CTPOP, Ty, Arg); setValue(&I, result); return 0; @@ -2992,17 +3427,12 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, MVT::Other, getRoot(), Tmp)); return 0; } - case Intrinsic::prefetch: - // FIXME: Currently discarding prefetches. - return 0; - case Intrinsic::var_annotation: // Discard annotate attributes return 0; case Intrinsic::init_trampoline: { - const Function *F = - cast(IntrinsicInst::StripPointerCasts(I.getOperand(2))); + const Function *F = cast(I.getOperand(2)->stripPointerCasts()); SDOperand Ops[6]; Ops[0] = getRoot(); @@ -3046,6 +3476,58 @@ SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { DAG.setRoot(DAG.getNode(ISD::TRAP, MVT::Other, getRoot())); return 0; } + case Intrinsic::prefetch: { + SDOperand Ops[4]; + Ops[0] = getRoot(); + Ops[1] = getValue(I.getOperand(1)); + Ops[2] = getValue(I.getOperand(2)); + Ops[3] = getValue(I.getOperand(3)); + DAG.setRoot(DAG.getNode(ISD::PREFETCH, MVT::Other, &Ops[0], 4)); + return 0; + } + + case Intrinsic::memory_barrier: { + SDOperand Ops[6]; + Ops[0] = getRoot(); + for (int x = 1; x < 6; ++x) + Ops[x] = getValue(I.getOperand(x)); + + DAG.setRoot(DAG.getNode(ISD::MEMBARRIER, MVT::Other, &Ops[0], 6)); + return 0; + } + case Intrinsic::atomic_cmp_swap: { + SDOperand Root = getRoot(); + SDOperand L = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, Root, + getValue(I.getOperand(1)), + getValue(I.getOperand(2)), + getValue(I.getOperand(3)), + I.getOperand(1)); + setValue(&I, L); + DAG.setRoot(L.getValue(1)); + return 0; + } + case Intrinsic::atomic_load_add: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_ADD); + case Intrinsic::atomic_load_sub: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_SUB); + case Intrinsic::atomic_load_and: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_AND); + case Intrinsic::atomic_load_or: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_OR); + case Intrinsic::atomic_load_xor: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_XOR); + case Intrinsic::atomic_load_nand: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_NAND); + case Intrinsic::atomic_load_min: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MIN); + case Intrinsic::atomic_load_max: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_MAX); + case Intrinsic::atomic_load_umin: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMIN); + case Intrinsic::atomic_load_umax: + return implVisitBinaryAtomic(I, ISD::ATOMIC_LOAD_UMAX); + case Intrinsic::atomic_swap: + return implVisitBinaryAtomic(I, ISD::ATOMIC_SWAP); } } @@ -3073,38 +3555,35 @@ void SelectionDAGLowering::LowerCallTo(CallSite CS, SDOperand Callee, Entry.isSRet = CS.paramHasAttr(attrInd, ParamAttr::StructRet); Entry.isNest = CS.paramHasAttr(attrInd, ParamAttr::Nest); Entry.isByVal = CS.paramHasAttr(attrInd, ParamAttr::ByVal); + Entry.Alignment = CS.getParamAlignment(attrInd); Args.push_back(Entry); } - bool MarkTryRange = LandingPad || - // C++ requires special handling of 'nounwind' calls. - (CS.doesNotThrow()); - - if (MarkTryRange && ExceptionHandling && MMI) { + if (LandingPad && MMI) { // Insert a label before the invoke call to mark the try range. This can be // used to detect deletion of the invoke via the MachineModuleInfo. BeginLabel = MMI->NextLabelID(); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(BeginLabel, MVT::i32), - DAG.getConstant(1, MVT::i32))); + // Both PendingLoads and PendingExports must be flushed here; + // this call might not return. + (void)getRoot(); + DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, getControlRoot(), BeginLabel)); } std::pair Result = TLI.LowerCallTo(getRoot(), CS.getType(), CS.paramHasAttr(0, ParamAttr::SExt), + CS.paramHasAttr(0, ParamAttr::ZExt), FTy->isVarArg(), CS.getCallingConv(), IsTailCall, Callee, Args, DAG); if (CS.getType() != Type::VoidTy) setValue(CS.getInstruction(), Result.first); DAG.setRoot(Result.second); - if (MarkTryRange && ExceptionHandling && MMI) { + if (LandingPad && MMI) { // Insert a label at the end of the invoke call to mark the try range. This // can be used to detect deletion of the invoke via the MachineModuleInfo. EndLabel = MMI->NextLabelID(); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, getRoot(), - DAG.getConstant(EndLabel, MVT::i32), - DAG.getConstant(1, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, getRoot(), EndLabel)); // Inform MachineModuleInfo of range. MMI->addInvoke(LandingPad, BeginLabel, EndLabel); @@ -3191,27 +3670,103 @@ void SelectionDAGLowering::visitCall(CallInst &I) { } +void SelectionDAGLowering::visitGetResult(GetResultInst &I) { + if (isa(I.getOperand(0))) { + SDOperand Undef = DAG.getNode(ISD::UNDEF, TLI.getValueType(I.getType())); + setValue(&I, Undef); + return; + } + + // To add support for individual return values with aggregate types, + // we'd need a way to take a getresult index and determine which + // values of the Call SDNode are associated with it. + assert(TLI.getValueType(I.getType(), true) != MVT::Other && + "Individual return values must not be aggregates!"); + + SDOperand Call = getValue(I.getOperand(0)); + setValue(&I, SDOperand(Call.Val, I.getIndex())); +} + + /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from /// this value and returns the result as a ValueVT value. This uses /// Chain/Flag as the input and updates them for the output Chain/Flag. /// If the Flag pointer is NULL, no flag is used. -SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG, - SDOperand &Chain, SDOperand *Flag)const{ - // Copy the legal parts from the registers. - unsigned NumParts = Regs.size(); - SmallVector Parts(NumParts); - for (unsigned i = 0; i != NumParts; ++i) { - SDOperand Part = Flag ? - DAG.getCopyFromReg(Chain, Regs[i], RegVT, *Flag) : - DAG.getCopyFromReg(Chain, Regs[i], RegVT); - Chain = Part.getValue(1); - if (Flag) - *Flag = Part.getValue(2); - Parts[i] = Part; - } +SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG, + SDOperand &Chain, + SDOperand *Flag) const { + // Assemble the legal parts into the final values. + SmallVector Values(ValueVTs.size()); + SmallVector Parts; + for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { + // Copy the legal parts from the registers. + MVT ValueVT = ValueVTs[Value]; + unsigned NumRegs = TLI->getNumRegisters(ValueVT); + MVT RegisterVT = RegVTs[Value]; + + Parts.resize(NumRegs); + for (unsigned i = 0; i != NumRegs; ++i) { + SDOperand P; + if (Flag == 0) + P = DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT); + else { + P = DAG.getCopyFromReg(Chain, Regs[Part+i], RegisterVT, *Flag); + *Flag = P.getValue(2); + } + Chain = P.getValue(1); + + // If the source register was virtual and if we know something about it, + // add an assert node. + if (TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) && + RegisterVT.isInteger() && !RegisterVT.isVector()) { + unsigned SlotNo = Regs[Part+i]-TargetRegisterInfo::FirstVirtualRegister; + FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); + if (FLI.LiveOutRegInfo.size() > SlotNo) { + FunctionLoweringInfo::LiveOutInfo &LOI = FLI.LiveOutRegInfo[SlotNo]; + + unsigned RegSize = RegisterVT.getSizeInBits(); + unsigned NumSignBits = LOI.NumSignBits; + unsigned NumZeroBits = LOI.KnownZero.countLeadingOnes(); + + // FIXME: We capture more information than the dag can represent. For + // now, just use the tightest assertzext/assertsext possible. + bool isSExt = true; + MVT FromVT(MVT::Other); + if (NumSignBits == RegSize) + isSExt = true, FromVT = MVT::i1; // ASSERT SEXT 1 + else if (NumZeroBits >= RegSize-1) + isSExt = false, FromVT = MVT::i1; // ASSERT ZEXT 1 + else if (NumSignBits > RegSize-8) + isSExt = true, FromVT = MVT::i8; // ASSERT SEXT 8 + else if (NumZeroBits >= RegSize-9) + isSExt = false, FromVT = MVT::i8; // ASSERT ZEXT 8 + else if (NumSignBits > RegSize-16) + isSExt = true, FromVT = MVT::i16; // ASSERT SEXT 16 + else if (NumZeroBits >= RegSize-17) + isSExt = false, FromVT = MVT::i16; // ASSERT ZEXT 16 + else if (NumSignBits > RegSize-32) + isSExt = true, FromVT = MVT::i32; // ASSERT SEXT 32 + else if (NumZeroBits >= RegSize-33) + isSExt = false, FromVT = MVT::i32; // ASSERT ZEXT 32 + + if (FromVT != MVT::Other) { + P = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, + RegisterVT, P, DAG.getValueType(FromVT)); + + } + } + } + + Parts[Part+i] = P; + } - // Assemble the legal parts into the final value. - return getCopyFromParts(DAG, &Parts[0], NumParts, RegVT, ValueVT); + Values[Value] = getCopyFromParts(DAG, &Parts[Part], NumRegs, RegisterVT, + ValueVT); + Part += NumRegs; + } + + return DAG.getMergeValues(DAG.getVTList(&ValueVTs[0], ValueVTs.size()), + &Values[0], ValueVTs.size()); } /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the @@ -3221,19 +3776,45 @@ SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG, void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG, SDOperand &Chain, SDOperand *Flag) const { // Get the list of the values's legal parts. - unsigned NumParts = Regs.size(); - SmallVector Parts(NumParts); - getCopyToParts(DAG, Val, &Parts[0], NumParts, RegVT); + unsigned NumRegs = Regs.size(); + SmallVector Parts(NumRegs); + for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { + MVT ValueVT = ValueVTs[Value]; + unsigned NumParts = TLI->getNumRegisters(ValueVT); + MVT RegisterVT = RegVTs[Value]; + + getCopyToParts(DAG, Val.getValue(Val.ResNo + Value), + &Parts[Part], NumParts, RegisterVT); + Part += NumParts; + } // Copy the parts into the registers. - for (unsigned i = 0; i != NumParts; ++i) { - SDOperand Part = Flag ? - DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag) : - DAG.getCopyToReg(Chain, Regs[i], Parts[i]); - Chain = Part.getValue(0); - if (Flag) + SmallVector Chains(NumRegs); + for (unsigned i = 0; i != NumRegs; ++i) { + SDOperand Part; + if (Flag == 0) + Part = DAG.getCopyToReg(Chain, Regs[i], Parts[i]); + else { + Part = DAG.getCopyToReg(Chain, Regs[i], Parts[i], *Flag); *Flag = Part.getValue(1); + } + Chains[i] = Part.getValue(0); } + + if (NumRegs == 1 || Flag) + // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is + // flagged to it. That is the CopyToReg nodes and the user are considered + // a single scheduling unit. If we create a TokenFactor and return it as + // chain, then the TokenFactor is both a predecessor (operand) of the + // user as well as a successor (the TF operands are flagged to the user). + // c1, f1 = CopyToReg + // c2, f2 = CopyToReg + // c3 = TokenFactor c1, c2 + // ... + // = op c3, ..., f2 + Chain = Chains[NumRegs-1]; + else + Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs); } /// AddInlineAsmOperands - Add this value to the specified inlineasm node @@ -3241,10 +3822,14 @@ void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG, /// values added into it. void RegsForValue::AddInlineAsmOperands(unsigned Code, SelectionDAG &DAG, std::vector &Ops) const { - MVT::ValueType IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); + MVT IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); Ops.push_back(DAG.getTargetConstant(Code | (Regs.size() << 3), IntPtrTy)); - for (unsigned i = 0, e = Regs.size(); i != e; ++i) - Ops.push_back(DAG.getRegister(Regs[i], RegVT)); + for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { + unsigned NumRegs = TLI->getNumRegisters(ValueVTs[Value]); + MVT RegisterVT = RegVTs[Value]; + for (unsigned i = 0; i != NumRegs; ++i) + Ops.push_back(DAG.getRegister(Regs[Reg++], RegisterVT)); + } } /// isAllocatableRegister - If the specified register is safe to allocate, @@ -3254,11 +3839,11 @@ static const TargetRegisterClass * isAllocatableRegister(unsigned Reg, MachineFunction &MF, const TargetLowering &TLI, const TargetRegisterInfo *TRI) { - MVT::ValueType FoundVT = MVT::Other; + MVT FoundVT = MVT::Other; const TargetRegisterClass *FoundRC = 0; for (TargetRegisterInfo::regclass_iterator RCI = TRI->regclass_begin(), E = TRI->regclass_end(); RCI != E; ++RCI) { - MVT::ValueType ThisVT = MVT::Other; + MVT ThisVT = MVT::Other; const TargetRegisterClass *RC = *RCI; // If none of the the value types for this register class are valid, we @@ -3269,8 +3854,7 @@ isAllocatableRegister(unsigned Reg, MachineFunction &MF, // If we have already found this register in a different register class, // choose the one with the largest VT specified. For example, on // PowerPC, we favor f64 register classes over f32. - if (FoundVT == MVT::Other || - MVT::getSizeInBits(FoundVT) < MVT::getSizeInBits(*I)) { + if (FoundVT == MVT::Other || FoundVT.bitsLT(*I)) { ThisVT = *I; break; } @@ -3300,112 +3884,64 @@ isAllocatableRegister(unsigned Reg, MachineFunction &MF, namespace { /// AsmOperandInfo - This contains information for each constraint that we are /// lowering. -struct AsmOperandInfo : public InlineAsm::ConstraintInfo { - /// ConstraintCode - This contains the actual string for the code, like "m". - std::string ConstraintCode; - - /// ConstraintType - Information about the constraint code, e.g. Register, - /// RegisterClass, Memory, Other, Unknown. - TargetLowering::ConstraintType ConstraintType; - - /// CallOperand/CallOperandval - If this is the result output operand or a - /// clobber, this is null, otherwise it is the incoming operand to the - /// CallInst. This gets modified as the asm is processed. +struct SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { + /// CallOperand - If this is the result output operand or a clobber + /// this is null, otherwise it is the incoming operand to the CallInst. + /// This gets modified as the asm is processed. SDOperand CallOperand; - Value *CallOperandVal; - - /// ConstraintVT - The ValueType for the operand value. - MVT::ValueType ConstraintVT; - + /// AssignedRegs - If this is a register or register class operand, this /// contains the set of register corresponding to the operand. RegsForValue AssignedRegs; - AsmOperandInfo(const InlineAsm::ConstraintInfo &info) - : InlineAsm::ConstraintInfo(info), - ConstraintType(TargetLowering::C_Unknown), - CallOperand(0,0), CallOperandVal(0), ConstraintVT(MVT::Other) { + explicit SDISelAsmOperandInfo(const InlineAsm::ConstraintInfo &info) + : TargetLowering::AsmOperandInfo(info), CallOperand(0,0) { } - void ComputeConstraintToUse(const TargetLowering &TLI); - /// MarkAllocatedRegs - Once AssignedRegs is set, mark the assigned registers /// busy in OutputRegs/InputRegs. void MarkAllocatedRegs(bool isOutReg, bool isInReg, std::set &OutputRegs, - std::set &InputRegs) const { - if (isOutReg) - OutputRegs.insert(AssignedRegs.Regs.begin(), AssignedRegs.Regs.end()); - if (isInReg) - InputRegs.insert(AssignedRegs.Regs.begin(), AssignedRegs.Regs.end()); - } -}; -} // end anon namespace. - -/// getConstraintGenerality - Return an integer indicating how general CT is. -static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) { - switch (CT) { - default: assert(0 && "Unknown constraint type!"); - case TargetLowering::C_Other: - case TargetLowering::C_Unknown: - return 0; - case TargetLowering::C_Register: - return 1; - case TargetLowering::C_RegisterClass: - return 2; - case TargetLowering::C_Memory: - return 3; - } -} - -void AsmOperandInfo::ComputeConstraintToUse(const TargetLowering &TLI) { - assert(!Codes.empty() && "Must have at least one constraint"); - - std::string *Current = &Codes[0]; - TargetLowering::ConstraintType CurType = TLI.getConstraintType(*Current); - if (Codes.size() == 1) { // Single-letter constraints ('r') are very common. - ConstraintCode = *Current; - ConstraintType = CurType; - } else { - unsigned CurGenerality = getConstraintGenerality(CurType); - - // If we have multiple constraints, try to pick the most general one ahead - // of time. This isn't a wonderful solution, but handles common cases. - for (unsigned j = 1, e = Codes.size(); j != e; ++j) { - TargetLowering::ConstraintType ThisType = TLI.getConstraintType(Codes[j]); - unsigned ThisGenerality = getConstraintGenerality(ThisType); - if (ThisGenerality > CurGenerality) { - // This constraint letter is more general than the previous one, - // use it. - CurType = ThisType; - Current = &Codes[j]; - CurGenerality = ThisGenerality; - } + std::set &InputRegs, + const TargetRegisterInfo &TRI) const { + if (isOutReg) { + for (unsigned i = 0, e = AssignedRegs.Regs.size(); i != e; ++i) + MarkRegAndAliases(AssignedRegs.Regs[i], OutputRegs, TRI); } - - ConstraintCode = *Current; - ConstraintType = CurType; - } - - if (ConstraintCode == "X") { - if (isa(CallOperandVal) || isa(CallOperandVal)) - return; - // This matches anything. Labels and constants we handle elsewhere - // ('X' is the only thing that matches labels). Otherwise, try to - // resolve it to something we know about by looking at the actual - // operand type. - std::string s = ""; - TLI.lowerXConstraint(ConstraintVT, s); - if (s!="") { - ConstraintCode = s; - ConstraintType = TLI.getConstraintType(ConstraintCode); + if (isInReg) { + for (unsigned i = 0, e = AssignedRegs.Regs.size(); i != e; ++i) + MarkRegAndAliases(AssignedRegs.Regs[i], InputRegs, TRI); } } -} + +private: + /// MarkRegAndAliases - Mark the specified register and all aliases in the + /// specified set. + static void MarkRegAndAliases(unsigned Reg, std::set &Regs, + const TargetRegisterInfo &TRI) { + assert(TargetRegisterInfo::isPhysicalRegister(Reg) && "Isn't a physreg"); + Regs.insert(Reg); + if (const unsigned *Aliases = TRI.getAliasSet(Reg)) + for (; *Aliases; ++Aliases) + Regs.insert(*Aliases); + } +}; +} // end anon namespace. +/// GetRegistersForValue - Assign registers (virtual or physical) for the +/// specified operand. We prefer to assign virtual registers, to allow the +/// register allocator handle the assignment process. However, if the asm uses +/// features that we can't model on machineinstrs, we have SDISel do the +/// allocation. This produces generally horrible, but correct, code. +/// +/// OpInfo describes the operand. +/// HasEarlyClobber is true if there are any early clobber constraints (=&r) +/// or any explicitly clobbered registers. +/// Input and OutputRegs are the set of already allocated physical registers. +/// void SelectionDAGLowering:: -GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber, +GetRegistersForValue(SDISelAsmOperandInfo &OpInfo, bool HasEarlyClobber, std::set &OutputRegs, std::set &InputRegs) { // Compute whether this value requires an input register, an output register, @@ -3433,7 +3969,7 @@ GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber, MachineFunction &MF = DAG.getMachineFunction(); - std::vector Regs; + SmallVector Regs; // If this is a constraint for a single physreg, or a constraint for a // register class, find it. @@ -3444,8 +3980,8 @@ GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber, unsigned NumRegs = 1; if (OpInfo.ConstraintVT != MVT::Other) NumRegs = TLI.getNumRegisters(OpInfo.ConstraintVT); - MVT::ValueType RegVT; - MVT::ValueType ValueVT = OpInfo.ConstraintVT; + MVT RegVT; + MVT ValueVT = OpInfo.ConstraintVT; // If this is a constraint for a specific physical register, like {r17}, @@ -3465,19 +4001,19 @@ GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber, // If this is an expanded reference, add the rest of the regs to Regs. if (NumRegs != 1) { TargetRegisterClass::iterator I = PhysReg.second->begin(); - TargetRegisterClass::iterator E = PhysReg.second->end(); for (; *I != PhysReg.first; ++I) - assert(I != E && "Didn't find reg!"); + assert(I != PhysReg.second->end() && "Didn't find reg!"); // Already added the first reg. --NumRegs; ++I; for (; NumRegs; --NumRegs, ++I) { - assert(I != E && "Ran out of registers to allocate!"); + assert(I != PhysReg.second->end() && "Ran out of registers to allocate!"); Regs.push_back(*I); } } - OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); - OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs); + OpInfo.AssignedRegs = RegsForValue(TLI, Regs, RegVT, ValueVT); + const TargetRegisterInfo *TRI = DAG.getTarget().getRegisterInfo(); + OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs, *TRI); return; } @@ -3504,8 +4040,7 @@ GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber, for (; NumRegs; --NumRegs) Regs.push_back(RegInfo.createVirtualRegister(PhysReg.second)); - OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); - OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs); + OpInfo.AssignedRegs = RegsForValue(TLI, Regs, RegVT, ValueVT); return; } @@ -3555,15 +4090,14 @@ GetRegistersForValue(AsmOperandInfo &OpInfo, bool HasEarlyClobber, for (unsigned i = RegStart; i != RegEnd; ++i) Regs.push_back(RegClassRegs[i]); - OpInfo.AssignedRegs = RegsForValue(Regs, *RC->vt_begin(), + OpInfo.AssignedRegs = RegsForValue(TLI, Regs, *RC->vt_begin(), OpInfo.ConstraintVT); - OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs); + OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs, *TRI); return; } } // Otherwise, we couldn't allocate enough registers for this. - return; } @@ -3573,7 +4107,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { InlineAsm *IA = cast(CS.getCalledValue()); /// ConstraintOperands - Information about all of the constraints. - std::vector ConstraintOperands; + std::vector ConstraintOperands; SDOperand Chain = getRoot(); SDOperand Flag; @@ -3591,23 +4125,31 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { bool SawEarlyClobber = false; unsigned ArgNo = 0; // ArgNo - The argument of the CallInst. + unsigned ResNo = 0; // ResNo - The result number of the next output. for (unsigned i = 0, e = ConstraintInfos.size(); i != e; ++i) { - ConstraintOperands.push_back(AsmOperandInfo(ConstraintInfos[i])); - AsmOperandInfo &OpInfo = ConstraintOperands.back(); + ConstraintOperands.push_back(SDISelAsmOperandInfo(ConstraintInfos[i])); + SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back(); - MVT::ValueType OpVT = MVT::Other; + MVT OpVT = MVT::Other; // Compute the value type for each operand. switch (OpInfo.Type) { case InlineAsm::isOutput: - if (!OpInfo.isIndirect) { - // The return value of the call is this value. As such, there is no - // corresponding argument. - assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); - OpVT = TLI.getValueType(CS.getType()); - } else { + // Indirect outputs just consume an argument. + if (OpInfo.isIndirect) { OpInfo.CallOperandVal = CS.getArgument(ArgNo++); + break; + } + // The return value of the call is this value. As such, there is no + // corresponding argument. + assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); + if (const StructType *STy = dyn_cast(CS.getType())) { + OpVT = TLI.getValueType(STy->getElementType(ResNo)); + } else { + assert(ResNo == 0 && "Asm only has one result!"); + OpVT = TLI.getValueType(CS.getType()); } + ++ResNo; break; case InlineAsm::isInput: OpInfo.CallOperandVal = CS.getArgument(ArgNo++); @@ -3620,10 +4162,8 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // If this is an input or an indirect output, process the call argument. // BasicBlocks are labels, currently appearing only in asm's. if (OpInfo.CallOperandVal) { - if (isa(OpInfo.CallOperandVal)) - OpInfo.CallOperand = - DAG.getBasicBlock(FuncInfo.MBBMap[cast( - OpInfo.CallOperandVal)]); + if (BasicBlock *BB = dyn_cast(OpInfo.CallOperandVal)) + OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]); else { OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); const Type *OpTy = OpInfo.CallOperandVal->getType(); @@ -3632,9 +4172,9 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { if (OpInfo.isIndirect) OpTy = cast(OpTy)->getElementType(); - // If OpTy is not a first-class value, it may be a struct/union that we + // If OpTy is not a single value, it may be a struct/union that we // can tile with integers. - if (!OpTy->isFirstClassType() && OpTy->isSized()) { + if (!OpTy->isSingleValueType() && OpTy->isSized()) { unsigned BitSize = TD->getTypeSizeInBits(OpTy); switch (BitSize) { default: break; @@ -3655,11 +4195,26 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { OpInfo.ConstraintVT = OpVT; // Compute the constraint code and ConstraintType to use. - OpInfo.ComputeConstraintToUse(TLI); + TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); // Keep track of whether we see an earlyclobber. SawEarlyClobber |= OpInfo.isEarlyClobber; + // If we see a clobber of a register, it is an early clobber. + if (!SawEarlyClobber && + OpInfo.Type == InlineAsm::isClobber && + OpInfo.ConstraintType == TargetLowering::C_Register) { + // Note that we want to ignore things that we don't trick here, like + // dirflag, fpsr, flags, etc. + std::pair PhysReg = + TLI.getRegForInlineAsmConstraint(OpInfo.ConstraintCode, + OpInfo.ConstraintVT); + if (PhysReg.first || PhysReg.second) { + // This is a register we know of. + SawEarlyClobber = true; + } + } + // If this is a memory input, and if the operand is not indirect, do what we // need to to provide an address for the memory input. if (OpInfo.ConstraintType == TargetLowering::C_Memory && @@ -3708,7 +4263,7 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // Second pass - Loop over all of the operands, assigning virtual or physregs // to registerclass operands. for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { - AsmOperandInfo &OpInfo = ConstraintOperands[i]; + SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; // C_Register operands have already been allocated, Other/Memory don't need // to be. @@ -3726,12 +4281,12 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // Loop over all of the inputs, copying the operand values into the // appropriate registers and processing the output regs. RegsForValue RetValRegs; - + // IndirectStoresToEmit - The set of stores to emit after the inline asm node. std::vector > IndirectStoresToEmit; for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) { - AsmOperandInfo &OpInfo = ConstraintOperands[i]; + SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i]; switch (OpInfo.Type) { case InlineAsm::isOutput: { @@ -3753,20 +4308,21 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // Copy the output from the appropriate register. Find a register that // we can use. if (OpInfo.AssignedRegs.Regs.empty()) { - cerr << "Couldn't allocate output reg for contraint '" + cerr << "Couldn't allocate output reg for constraint '" << OpInfo.ConstraintCode << "'!\n"; exit(1); } - if (!OpInfo.isIndirect) { - // This is the result value of the call. - assert(RetValRegs.Regs.empty() && - "Cannot have multiple output constraints yet!"); - assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); - RetValRegs = OpInfo.AssignedRegs; - } else { + // If this is an indirect operand, store through the pointer after the + // asm. + if (OpInfo.isIndirect) { IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs, OpInfo.CallOperandVal)); + } else { + // This is the result value of the call. + assert(CS.getType() != Type::VoidTy && "Bad inline asm!"); + // Concatenate this output onto the outputs list. + RetValRegs.append(OpInfo.AssignedRegs); } // Add information to the INLINEASM node to know that this register is @@ -3801,8 +4357,9 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { if ((NumOps & 7) == 2 /*REGDEF*/) { // Add NumOps>>3 registers to MatchedRegs. RegsForValue MatchedRegs; - MatchedRegs.ValueVT = InOperandVal.getValueType(); - MatchedRegs.RegVT = AsmNodeOperands[CurOp+1].getValueType(); + MatchedRegs.TLI = &TLI; + MatchedRegs.ValueVTs.push_back(InOperandVal.getValueType()); + MatchedRegs.RegVTs.push_back(AsmNodeOperands[CurOp+1].getValueType()); for (unsigned i = 0, e = NumOps>>3; i != e; ++i) { unsigned Reg = cast(AsmNodeOperands[++CurOp])->getReg(); @@ -3815,7 +4372,13 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { break; } else { assert((NumOps & 7) == 4/*MEM*/ && "Unknown matching constraint!"); - assert(0 && "matching constraints for memory operands unimp"); + assert((NumOps >> 3) == 1 && "Unexpected number of operands"); + // Add information to the INLINEASM node to know about this input. + unsigned ResOpType = 4/*MEM*/ | (1 << 3); + AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, + TLI.getPointerTy())); + AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]); + break; } } @@ -3891,17 +4454,24 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { // and set it as the value of the call. if (!RetValRegs.Regs.empty()) { SDOperand Val = RetValRegs.getCopyFromRegs(DAG, Chain, &Flag); - - // If the result of the inline asm is a vector, it may have the wrong - // width/num elts. Make sure to convert it to the right type with + + // If any of the results of the inline asm is a vector, it may have the + // wrong width/num elts. This can happen for register classes that can + // contain multiple different value types. The preg or vreg allocated may + // not have the same VT as was expected. Convert it to the right type with // bit_convert. - if (MVT::isVector(Val.getValueType())) { - const VectorType *VTy = cast(CS.getType()); - MVT::ValueType DesiredVT = TLI.getValueType(VTy); - - Val = DAG.getNode(ISD::BIT_CONVERT, DesiredVT, Val); + if (const StructType *ResSTy = dyn_cast(CS.getType())) { + for (unsigned i = 0, e = ResSTy->getNumElements(); i != e; ++i) { + if (Val.Val->getValueType(i).isVector()) + Val = DAG.getNode(ISD::BIT_CONVERT, + TLI.getValueType(ResSTy->getElementType(i)), Val); + } + } else { + if (Val.getValueType().isVector()) + Val = DAG.getNode(ISD::BIT_CONVERT, TLI.getValueType(CS.getType()), + Val); } - + setValue(CS.getInstruction(), Val); } @@ -3932,11 +4502,11 @@ void SelectionDAGLowering::visitInlineAsm(CallSite CS) { void SelectionDAGLowering::visitMalloc(MallocInst &I) { SDOperand Src = getValue(I.getOperand(0)); - MVT::ValueType IntPtr = TLI.getPointerTy(); + MVT IntPtr = TLI.getPointerTy(); - if (IntPtr < Src.getValueType()) + if (IntPtr.bitsLT(Src.getValueType())) Src = DAG.getNode(ISD::TRUNCATE, IntPtr, Src); - else if (IntPtr > Src.getValueType()) + else if (IntPtr.bitsGT(Src.getValueType())) Src = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, Src); // Scale the source by the type size. @@ -3951,9 +4521,8 @@ void SelectionDAGLowering::visitMalloc(MallocInst &I) { Args.push_back(Entry); std::pair Result = - TLI.LowerCallTo(getRoot(), I.getType(), false, false, CallingConv::C, true, - DAG.getExternalSymbol("malloc", IntPtr), - Args, DAG); + TLI.LowerCallTo(getRoot(), I.getType(), false, false, false, CallingConv::C, + true, DAG.getExternalSymbol("malloc", IntPtr), Args, DAG); setValue(&I, Result.first); // Pointers always fit in registers DAG.setRoot(Result.second); } @@ -3964,9 +4533,10 @@ void SelectionDAGLowering::visitFree(FreeInst &I) { Entry.Node = getValue(I.getOperand(0)); Entry.Ty = TLI.getTargetData()->getIntPtrType(); Args.push_back(Entry); - MVT::ValueType IntPtr = TLI.getPointerTy(); + MVT IntPtr = TLI.getPointerTy(); std::pair Result = - TLI.LowerCallTo(getRoot(), Type::VoidTy, false, false, CallingConv::C, true, + TLI.LowerCallTo(getRoot(), Type::VoidTy, false, false, false, + CallingConv::C, true, DAG.getExternalSymbol("free", IntPtr), Args, DAG); DAG.setRoot(Result.second); } @@ -4017,56 +4587,66 @@ void SelectionDAGLowering::visitVACopy(CallInst &I) { /// implementation, which just inserts a FORMAL_ARGUMENTS node. FIXME: When all /// targets are migrated to using FORMAL_ARGUMENTS, this hook should be /// integrated into SDISel. -std::vector -TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { +void TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG, + SmallVectorImpl &ArgValues) { // Add CC# and isVararg as operands to the FORMAL_ARGUMENTS node. - std::vector Ops; + SmallVector Ops; Ops.push_back(DAG.getRoot()); Ops.push_back(DAG.getConstant(F.getCallingConv(), getPointerTy())); Ops.push_back(DAG.getConstant(F.isVarArg(), getPointerTy())); // Add one result value for each formal argument. - std::vector RetVals; + SmallVector RetVals; unsigned j = 1; for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++j) { - MVT::ValueType VT = getValueType(I->getType()); - unsigned Flags = ISD::ParamFlags::NoFlagSet; - unsigned OriginalAlignment = - getTargetData()->getABITypeAlignment(I->getType()); - - // FIXME: Distinguish between a formal with no [sz]ext attribute from one - // that is zero extended! - if (F.paramHasAttr(j, ParamAttr::ZExt)) - Flags &= ~(ISD::ParamFlags::SExt); - if (F.paramHasAttr(j, ParamAttr::SExt)) - Flags |= ISD::ParamFlags::SExt; - if (F.paramHasAttr(j, ParamAttr::InReg)) - Flags |= ISD::ParamFlags::InReg; - if (F.paramHasAttr(j, ParamAttr::StructRet)) - Flags |= ISD::ParamFlags::StructReturn; - if (F.paramHasAttr(j, ParamAttr::ByVal)) { - Flags |= ISD::ParamFlags::ByVal; - const PointerType *Ty = cast(I->getType()); - const Type *ElementTy = Ty->getElementType(); - unsigned FrameAlign = Log2_32(getByValTypeAlignment(ElementTy)); - unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); - Flags |= (FrameAlign << ISD::ParamFlags::ByValAlignOffs); - Flags |= (FrameSize << ISD::ParamFlags::ByValSizeOffs); - } - if (F.paramHasAttr(j, ParamAttr::Nest)) - Flags |= ISD::ParamFlags::Nest; - Flags |= (OriginalAlignment << ISD::ParamFlags::OrigAlignmentOffs); - - MVT::ValueType RegisterVT = getRegisterType(VT); - unsigned NumRegs = getNumRegisters(VT); - for (unsigned i = 0; i != NumRegs; ++i) { - RetVals.push_back(RegisterVT); - // if it isn't first piece, alignment must be 1 - if (i > 0) - Flags = (Flags & (~ISD::ParamFlags::OrigAlignment)) | - (1 << ISD::ParamFlags::OrigAlignmentOffs); - Ops.push_back(DAG.getConstant(Flags, MVT::i32)); + SmallVector ValueVTs; + ComputeValueVTs(*this, I->getType(), ValueVTs); + for (unsigned Value = 0, NumValues = ValueVTs.size(); + Value != NumValues; ++Value) { + MVT VT = ValueVTs[Value]; + const Type *ArgTy = VT.getTypeForMVT(); + ISD::ArgFlagsTy Flags; + unsigned OriginalAlignment = + getTargetData()->getABITypeAlignment(ArgTy); + + if (F.paramHasAttr(j, ParamAttr::ZExt)) + Flags.setZExt(); + if (F.paramHasAttr(j, ParamAttr::SExt)) + Flags.setSExt(); + if (F.paramHasAttr(j, ParamAttr::InReg)) + Flags.setInReg(); + if (F.paramHasAttr(j, ParamAttr::StructRet)) + Flags.setSRet(); + if (F.paramHasAttr(j, ParamAttr::ByVal)) { + Flags.setByVal(); + const PointerType *Ty = cast(I->getType()); + const Type *ElementTy = Ty->getElementType(); + unsigned FrameAlign = getByValTypeAlignment(ElementTy); + unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); + // For ByVal, alignment should be passed from FE. BE will guess if + // this info is not there but there are cases it cannot get right. + if (F.getParamAlignment(j)) + FrameAlign = F.getParamAlignment(j); + Flags.setByValAlign(FrameAlign); + Flags.setByValSize(FrameSize); + } + if (F.paramHasAttr(j, ParamAttr::Nest)) + Flags.setNest(); + Flags.setOrigAlign(OriginalAlignment); + + MVT RegisterVT = getRegisterType(VT); + unsigned NumRegs = getNumRegisters(VT); + for (unsigned i = 0; i != NumRegs; ++i) { + RetVals.push_back(RegisterVT); + ISD::ArgFlagsTy MyFlags = Flags; + if (NumRegs > 1 && i == 0) + MyFlags.setSplit(); + // if it isn't first piece, alignment must be 1 + else if (i > 0) + MyFlags.setOrigAlign(1); + Ops.push_back(DAG.getArgFlags(MyFlags)); + } } } @@ -4094,30 +4674,33 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { DAG.setRoot(SDOperand(Result, NumArgRegs)); // Set up the return result vector. - Ops.clear(); unsigned i = 0; unsigned Idx = 1; for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I, ++Idx) { - MVT::ValueType VT = getValueType(I->getType()); - MVT::ValueType PartVT = getRegisterType(VT); - - unsigned NumParts = getNumRegisters(VT); - SmallVector Parts(NumParts); - for (unsigned j = 0; j != NumParts; ++j) - Parts[j] = SDOperand(Result, i++); - - ISD::NodeType AssertOp = ISD::DELETED_NODE; - if (F.paramHasAttr(Idx, ParamAttr::SExt)) - AssertOp = ISD::AssertSext; - else if (F.paramHasAttr(Idx, ParamAttr::ZExt)) - AssertOp = ISD::AssertZext; - - Ops.push_back(getCopyFromParts(DAG, &Parts[0], NumParts, PartVT, VT, - AssertOp, true)); + SmallVector ValueVTs; + ComputeValueVTs(*this, I->getType(), ValueVTs); + for (unsigned Value = 0, NumValues = ValueVTs.size(); + Value != NumValues; ++Value) { + MVT VT = ValueVTs[Value]; + MVT PartVT = getRegisterType(VT); + + unsigned NumParts = getNumRegisters(VT); + SmallVector Parts(NumParts); + for (unsigned j = 0; j != NumParts; ++j) + Parts[j] = SDOperand(Result, i++); + + ISD::NodeType AssertOp = ISD::DELETED_NODE; + if (F.paramHasAttr(Idx, ParamAttr::SExt)) + AssertOp = ISD::AssertSext; + else if (F.paramHasAttr(Idx, ParamAttr::ZExt)) + AssertOp = ISD::AssertZext; + + ArgValues.push_back(getCopyFromParts(DAG, &Parts[0], NumParts, PartVT, VT, + AssertOp)); + } } assert(i == NumArgRegs && "Argument register count mismatch!"); - return Ops; } @@ -4126,9 +4709,9 @@ TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) { /// lowered by the target to something concrete. FIXME: When all targets are /// migrated to using ISD::CALL, this hook should be integrated into SDISel. std::pair -TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, - bool RetTyIsSigned, bool isVarArg, - unsigned CallingConv, bool isTailCall, +TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, + bool RetSExt, bool RetZExt, bool isVarArg, + unsigned CallingConv, bool isTailCall, SDOperand Callee, ArgListTy &Args, SelectionDAG &DAG) { SmallVector Ops; @@ -4140,82 +4723,118 @@ TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy, // Handle all of the outgoing arguments. for (unsigned i = 0, e = Args.size(); i != e; ++i) { - MVT::ValueType VT = getValueType(Args[i].Ty); - SDOperand Op = Args[i].Node; - unsigned Flags = ISD::ParamFlags::NoFlagSet; - unsigned OriginalAlignment = - getTargetData()->getABITypeAlignment(Args[i].Ty); - - if (Args[i].isSExt) - Flags |= ISD::ParamFlags::SExt; - if (Args[i].isZExt) - Flags |= ISD::ParamFlags::ZExt; - if (Args[i].isInReg) - Flags |= ISD::ParamFlags::InReg; - if (Args[i].isSRet) - Flags |= ISD::ParamFlags::StructReturn; - if (Args[i].isByVal) { - Flags |= ISD::ParamFlags::ByVal; - const PointerType *Ty = cast(Args[i].Ty); - const Type *ElementTy = Ty->getElementType(); - unsigned FrameAlign = Log2_32(getByValTypeAlignment(ElementTy)); - unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); - Flags |= (FrameAlign << ISD::ParamFlags::ByValAlignOffs); - Flags |= (FrameSize << ISD::ParamFlags::ByValSizeOffs); - } - if (Args[i].isNest) - Flags |= ISD::ParamFlags::Nest; - Flags |= OriginalAlignment << ISD::ParamFlags::OrigAlignmentOffs; - - MVT::ValueType PartVT = getRegisterType(VT); - unsigned NumParts = getNumRegisters(VT); - SmallVector Parts(NumParts); - ISD::NodeType ExtendKind = ISD::ANY_EXTEND; - - if (Args[i].isSExt) - ExtendKind = ISD::SIGN_EXTEND; - else if (Args[i].isZExt) - ExtendKind = ISD::ZERO_EXTEND; - - getCopyToParts(DAG, Op, &Parts[0], NumParts, PartVT, ExtendKind); - - for (unsigned i = 0; i != NumParts; ++i) { - // if it isn't first piece, alignment must be 1 - unsigned MyFlags = Flags; - if (i != 0) - MyFlags = (MyFlags & (~ISD::ParamFlags::OrigAlignment)) | - (1 << ISD::ParamFlags::OrigAlignmentOffs); - - Ops.push_back(Parts[i]); - Ops.push_back(DAG.getConstant(MyFlags, MVT::i32)); + SmallVector ValueVTs; + ComputeValueVTs(*this, Args[i].Ty, ValueVTs); + for (unsigned Value = 0, NumValues = ValueVTs.size(); + Value != NumValues; ++Value) { + MVT VT = ValueVTs[Value]; + const Type *ArgTy = VT.getTypeForMVT(); + SDOperand Op = SDOperand(Args[i].Node.Val, Args[i].Node.ResNo + Value); + ISD::ArgFlagsTy Flags; + unsigned OriginalAlignment = + getTargetData()->getABITypeAlignment(ArgTy); + + if (Args[i].isZExt) + Flags.setZExt(); + if (Args[i].isSExt) + Flags.setSExt(); + if (Args[i].isInReg) + Flags.setInReg(); + if (Args[i].isSRet) + Flags.setSRet(); + if (Args[i].isByVal) { + Flags.setByVal(); + const PointerType *Ty = cast(Args[i].Ty); + const Type *ElementTy = Ty->getElementType(); + unsigned FrameAlign = getByValTypeAlignment(ElementTy); + unsigned FrameSize = getTargetData()->getABITypeSize(ElementTy); + // For ByVal, alignment should come from FE. BE will guess if this + // info is not there but there are cases it cannot get right. + if (Args[i].Alignment) + FrameAlign = Args[i].Alignment; + Flags.setByValAlign(FrameAlign); + Flags.setByValSize(FrameSize); + } + if (Args[i].isNest) + Flags.setNest(); + Flags.setOrigAlign(OriginalAlignment); + + MVT PartVT = getRegisterType(VT); + unsigned NumParts = getNumRegisters(VT); + SmallVector Parts(NumParts); + ISD::NodeType ExtendKind = ISD::ANY_EXTEND; + + if (Args[i].isSExt) + ExtendKind = ISD::SIGN_EXTEND; + else if (Args[i].isZExt) + ExtendKind = ISD::ZERO_EXTEND; + + getCopyToParts(DAG, Op, &Parts[0], NumParts, PartVT, ExtendKind); + + for (unsigned i = 0; i != NumParts; ++i) { + // if it isn't first piece, alignment must be 1 + ISD::ArgFlagsTy MyFlags = Flags; + if (NumParts > 1 && i == 0) + MyFlags.setSplit(); + else if (i != 0) + MyFlags.setOrigAlign(1); + + Ops.push_back(Parts[i]); + Ops.push_back(DAG.getArgFlags(MyFlags)); + } } } - // Figure out the result value types. - MVT::ValueType VT = getValueType(RetTy); - MVT::ValueType RegisterVT = getRegisterType(VT); - unsigned NumRegs = getNumRegisters(VT); - SmallVector RetTys(NumRegs); - for (unsigned i = 0; i != NumRegs; ++i) - RetTys[i] = RegisterVT; + // Figure out the result value types. We start by making a list of + // the potentially illegal return value types. + SmallVector LoweredRetTys; + SmallVector RetTys; + ComputeValueVTs(*this, RetTy, RetTys); + + // Then we translate that to a list of legal types. + for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { + MVT VT = RetTys[I]; + MVT RegisterVT = getRegisterType(VT); + unsigned NumRegs = getNumRegisters(VT); + for (unsigned i = 0; i != NumRegs; ++i) + LoweredRetTys.push_back(RegisterVT); + } - RetTys.push_back(MVT::Other); // Always has a chain. + LoweredRetTys.push_back(MVT::Other); // Always has a chain. // Create the CALL node. SDOperand Res = DAG.getNode(ISD::CALL, - DAG.getVTList(&RetTys[0], NumRegs + 1), + DAG.getVTList(&LoweredRetTys[0], + LoweredRetTys.size()), &Ops[0], Ops.size()); - Chain = Res.getValue(NumRegs); + Chain = Res.getValue(LoweredRetTys.size() - 1); // Gather up the call result into a single value. if (RetTy != Type::VoidTy) { - ISD::NodeType AssertOp = ISD::AssertSext; - if (!RetTyIsSigned) + ISD::NodeType AssertOp = ISD::DELETED_NODE; + + if (RetSExt) + AssertOp = ISD::AssertSext; + else if (RetZExt) AssertOp = ISD::AssertZext; - SmallVector Results(NumRegs); - for (unsigned i = 0; i != NumRegs; ++i) - Results[i] = Res.getValue(i); - Res = getCopyFromParts(DAG, &Results[0], NumRegs, RegisterVT, VT, AssertOp); + + SmallVector ReturnValues; + unsigned RegNo = 0; + for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { + MVT VT = RetTys[I]; + MVT RegisterVT = getRegisterType(VT); + unsigned NumRegs = getNumRegisters(VT); + unsigned RegNoEnd = NumRegs + RegNo; + SmallVector Results; + for (; RegNo != RegNoEnd; ++RegNo) + Results.push_back(Res.getValue(RegNo)); + SDOperand ReturnValue = + getCopyFromParts(DAG, &Results[0], NumRegs, RegisterVT, VT, + AssertOp); + ReturnValues.push_back(ReturnValue); + } + Res = DAG.getMergeValues(DAG.getVTList(&RetTys[0], RetTys.size()), + &ReturnValues[0], ReturnValues.size()); } return std::make_pair(Res, Chain); @@ -4227,254 +4846,12 @@ SDOperand TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { return SDOperand(); } -SDOperand TargetLowering::CustomPromoteOperation(SDOperand Op, - SelectionDAG &DAG) { - assert(0 && "CustomPromoteOperation not implemented for this target!"); - abort(); - return SDOperand(); -} - -/// getMemsetValue - Vectorized representation of the memset value -/// operand. -static SDOperand getMemsetValue(SDOperand Value, MVT::ValueType VT, - SelectionDAG &DAG) { - MVT::ValueType CurVT = VT; - if (ConstantSDNode *C = dyn_cast(Value)) { - uint64_t Val = C->getValue() & 255; - unsigned Shift = 8; - while (CurVT != MVT::i8) { - Val = (Val << Shift) | Val; - Shift <<= 1; - CurVT = (MVT::ValueType)((unsigned)CurVT - 1); - } - return DAG.getConstant(Val, VT); - } else { - Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value); - unsigned Shift = 8; - while (CurVT != MVT::i8) { - Value = - DAG.getNode(ISD::OR, VT, - DAG.getNode(ISD::SHL, VT, Value, - DAG.getConstant(Shift, MVT::i8)), Value); - Shift <<= 1; - CurVT = (MVT::ValueType)((unsigned)CurVT - 1); - } - - return Value; - } -} - -/// getMemsetStringVal - Similar to getMemsetValue. Except this is only -/// used when a memcpy is turned into a memset when the source is a constant -/// string ptr. -static SDOperand getMemsetStringVal(MVT::ValueType VT, - SelectionDAG &DAG, TargetLowering &TLI, - std::string &Str, unsigned Offset) { - uint64_t Val = 0; - unsigned MSB = MVT::getSizeInBits(VT) / 8; - if (TLI.isLittleEndian()) - Offset = Offset + MSB - 1; - for (unsigned i = 0; i != MSB; ++i) { - Val = (Val << 8) | (unsigned char)Str[Offset]; - Offset += TLI.isLittleEndian() ? -1 : 1; - } - return DAG.getConstant(Val, VT); -} - -/// getMemBasePlusOffset - Returns base and offset node for the -static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset, - SelectionDAG &DAG, TargetLowering &TLI) { - MVT::ValueType VT = Base.getValueType(); - return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT)); -} - -/// MeetsMaxMemopRequirement - Determines if the number of memory ops required -/// to replace the memset / memcpy is below the threshold. It also returns the -/// types of the sequence of memory ops to perform memset / memcpy. -static bool MeetsMaxMemopRequirement(std::vector &MemOps, - unsigned Limit, uint64_t Size, - unsigned Align, TargetLowering &TLI) { - MVT::ValueType VT; - - if (TLI.allowsUnalignedMemoryAccesses()) { - VT = MVT::i64; - } else { - switch (Align & 7) { - case 0: - VT = MVT::i64; - break; - case 4: - VT = MVT::i32; - break; - case 2: - VT = MVT::i16; - break; - default: - VT = MVT::i8; - break; - } - } - - MVT::ValueType LVT = MVT::i64; - while (!TLI.isTypeLegal(LVT)) - LVT = (MVT::ValueType)((unsigned)LVT - 1); - assert(MVT::isInteger(LVT)); - - if (VT > LVT) - VT = LVT; - - unsigned NumMemOps = 0; - while (Size != 0) { - unsigned VTSize = MVT::getSizeInBits(VT) / 8; - while (VTSize > Size) { - VT = (MVT::ValueType)((unsigned)VT - 1); - VTSize >>= 1; - } - assert(MVT::isInteger(VT)); - - if (++NumMemOps > Limit) - return false; - MemOps.push_back(VT); - Size -= VTSize; - } - - return true; -} - -void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) { - SDOperand Op1 = getValue(I.getOperand(1)); - SDOperand Op2 = getValue(I.getOperand(2)); - SDOperand Op3 = getValue(I.getOperand(3)); - SDOperand Op4 = getValue(I.getOperand(4)); - unsigned Align = (unsigned)cast(Op4)->getValue(); - if (Align == 0) Align = 1; - - // If the source and destination are known to not be aliases, we can - // lower memmove as memcpy. - if (Op == ISD::MEMMOVE) { - uint64_t Size = -1ULL; - if (ConstantSDNode *C = dyn_cast(Op3)) - Size = C->getValue(); - if (AA.alias(I.getOperand(1), Size, I.getOperand(2), Size) == - AliasAnalysis::NoAlias) - Op = ISD::MEMCPY; - } - - if (ConstantSDNode *Size = dyn_cast(Op3)) { - std::vector MemOps; - - // Expand memset / memcpy to a series of load / store ops - // if the size operand falls below a certain threshold. - SmallVector OutChains; - switch (Op) { - default: break; // Do nothing for now. - case ISD::MEMSET: { - if (MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemset(), - Size->getValue(), Align, TLI)) { - unsigned NumMemOps = MemOps.size(); - unsigned Offset = 0; - for (unsigned i = 0; i < NumMemOps; i++) { - MVT::ValueType VT = MemOps[i]; - unsigned VTSize = MVT::getSizeInBits(VT) / 8; - SDOperand Value = getMemsetValue(Op2, VT, DAG); - SDOperand Store = DAG.getStore(getRoot(), Value, - getMemBasePlusOffset(Op1, Offset, DAG, TLI), - I.getOperand(1), Offset); - OutChains.push_back(Store); - Offset += VTSize; - } - } - break; - } - case ISD::MEMCPY: { - if (MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemcpy(), - Size->getValue(), Align, TLI)) { - unsigned NumMemOps = MemOps.size(); - unsigned SrcOff = 0, DstOff = 0, SrcDelta = 0; - GlobalAddressSDNode *G = NULL; - std::string Str; - bool CopyFromStr = false; - - if (Op2.getOpcode() == ISD::GlobalAddress) - G = cast(Op2); - else if (Op2.getOpcode() == ISD::ADD && - Op2.getOperand(0).getOpcode() == ISD::GlobalAddress && - Op2.getOperand(1).getOpcode() == ISD::Constant) { - G = cast(Op2.getOperand(0)); - SrcDelta = cast(Op2.getOperand(1))->getValue(); - } - if (G) { - GlobalVariable *GV = dyn_cast(G->getGlobal()); - if (GV && GV->isConstant()) { - Str = GV->getStringValue(false); - if (!Str.empty()) { - CopyFromStr = true; - SrcOff += SrcDelta; - } - } - } - - for (unsigned i = 0; i < NumMemOps; i++) { - MVT::ValueType VT = MemOps[i]; - unsigned VTSize = MVT::getSizeInBits(VT) / 8; - SDOperand Value, Chain, Store; - - if (CopyFromStr) { - Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff); - Chain = getRoot(); - Store = - DAG.getStore(Chain, Value, - getMemBasePlusOffset(Op1, DstOff, DAG, TLI), - I.getOperand(1), DstOff); - } else { - Value = DAG.getLoad(VT, getRoot(), - getMemBasePlusOffset(Op2, SrcOff, DAG, TLI), - I.getOperand(2), SrcOff, false, Align); - Chain = Value.getValue(1); - Store = - DAG.getStore(Chain, Value, - getMemBasePlusOffset(Op1, DstOff, DAG, TLI), - I.getOperand(1), DstOff, false, Align); - } - OutChains.push_back(Store); - SrcOff += VTSize; - DstOff += VTSize; - } - } - break; - } - } - - if (!OutChains.empty()) { - DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other, - &OutChains[0], OutChains.size())); - return; - } - } - - SDOperand AlwaysInline = DAG.getConstant(0, MVT::i1); - SDOperand Node; - switch(Op) { - default: - assert(0 && "Unknown Op"); - case ISD::MEMCPY: - Node = DAG.getMemcpy(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline); - break; - case ISD::MEMMOVE: - Node = DAG.getMemmove(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline); - break; - case ISD::MEMSET: - Node = DAG.getMemset(getRoot(), Op1, Op2, Op3, Op4, AlwaysInline); - break; - } - DAG.setRoot(Node); -} //===----------------------------------------------------------------------===// // SelectionDAGISel code //===----------------------------------------------------------------------===// -unsigned SelectionDAGISel::MakeReg(MVT::ValueType VT) { +unsigned SelectionDAGISel::MakeReg(MVT VT) { return RegInfo->createVirtualRegister(TLI.getRegClassFor(VT)); } @@ -4484,8 +4861,6 @@ void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); } - - bool SelectionDAGISel::runOnFunction(Function &Fn) { // Get alias analysis for load/store combining. AA = &getAnalysis(); @@ -4500,14 +4875,12 @@ bool SelectionDAGISel::runOnFunction(Function &Fn) { FunctionLoweringInfo FuncInfo(TLI, Fn, MF); - if (ExceptionHandling) - for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) - if (InvokeInst *Invoke = dyn_cast(I->getTerminator())) - // Mark landing pad. - FuncInfo.MBBMap[Invoke->getSuccessor(1)]->setIsLandingPad(); - for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) - SelectBasicBlock(I, MF, FuncInfo); + if (InvokeInst *Invoke = dyn_cast(I->getTerminator())) + // Mark landing pad. + FuncInfo.MBBMap[Invoke->getSuccessor(1)]->setIsLandingPad(); + + SelectAllBasicBlocks(Fn, MF, FuncInfo); // Add function live-ins to entry block live-in set. BasicBlock *EntryBB = &Fn.getEntryBlock(); @@ -4525,49 +4898,45 @@ bool SelectionDAGISel::runOnFunction(Function &Fn) { return true; } -SDOperand SelectionDAGLowering::CopyValueToVirtualRegister(Value *V, - unsigned Reg) { +void SelectionDAGLowering::CopyValueToVirtualRegister(Value *V, unsigned Reg) { SDOperand Op = getValue(V); assert((Op.getOpcode() != ISD::CopyFromReg || cast(Op.getOperand(1))->getReg() != Reg) && "Copy from a reg to the same reg!"); - - MVT::ValueType SrcVT = Op.getValueType(); - MVT::ValueType RegisterVT = TLI.getRegisterType(SrcVT); - unsigned NumRegs = TLI.getNumRegisters(SrcVT); - SmallVector Regs(NumRegs); - SmallVector Chains(NumRegs); + assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg"); - // Copy the value by legal parts into sequential virtual registers. - getCopyToParts(DAG, Op, &Regs[0], NumRegs, RegisterVT); - for (unsigned i = 0; i != NumRegs; ++i) - Chains[i] = DAG.getCopyToReg(getRoot(), Reg + i, Regs[i]); - return DAG.getNode(ISD::TokenFactor, MVT::Other, &Chains[0], NumRegs); + RegsForValue RFV(TLI, Reg, V->getType()); + SDOperand Chain = DAG.getEntryNode(); + RFV.getCopyToRegs(Op, DAG, Chain, 0); + PendingExports.push_back(Chain); } void SelectionDAGISel:: -LowerArguments(BasicBlock *LLVMBB, SelectionDAGLowering &SDL, - std::vector &UnorderedChains) { +LowerArguments(BasicBlock *LLVMBB, SelectionDAGLowering &SDL) { // If this is the entry block, emit arguments. Function &F = *LLVMBB->getParent(); FunctionLoweringInfo &FuncInfo = SDL.FuncInfo; SDOperand OldRoot = SDL.DAG.getRoot(); - std::vector Args = TLI.LowerArguments(F, SDL.DAG); + SmallVector Args; + TLI.LowerArguments(F, SDL.DAG, Args); unsigned a = 0; for (Function::arg_iterator AI = F.arg_begin(), E = F.arg_end(); - AI != E; ++AI, ++a) + AI != E; ++AI) { + SmallVector ValueVTs; + ComputeValueVTs(TLI, AI->getType(), ValueVTs); + unsigned NumValues = ValueVTs.size(); if (!AI->use_empty()) { - SDL.setValue(AI, Args[a]); - + SDL.setValue(AI, SDL.DAG.getMergeValues(&Args[a], NumValues)); // If this argument is live outside of the entry block, insert a copy from // whereever we got it to the vreg that other BB's will reference it as. DenseMap::iterator VMI=FuncInfo.ValueMap.find(AI); if (VMI != FuncInfo.ValueMap.end()) { - SDOperand Copy = SDL.CopyValueToVirtualRegister(AI, VMI->second); - UnorderedChains.push_back(Copy); + SDL.CopyValueToVirtualRegister(AI, VMI->second); } } + a += NumValues; + } // Finally, if the target has anything special to do, allow it to do so. // FIXME: this should insert code into the DAG! @@ -4587,6 +4956,40 @@ static void copyCatchInfo(BasicBlock *SrcBB, BasicBlock *DestBB, } } +/// IsFixedFrameObjectWithPosOffset - Check if object is a fixed frame object and +/// whether object offset >= 0. +static bool +IsFixedFrameObjectWithPosOffset(MachineFrameInfo * MFI, SDOperand Op) { + if (!isa(Op)) return false; + + FrameIndexSDNode * FrameIdxNode = dyn_cast(Op); + int FrameIdx = FrameIdxNode->getIndex(); + return MFI->isFixedObjectIndex(FrameIdx) && + MFI->getObjectOffset(FrameIdx) >= 0; +} + +/// IsPossiblyOverwrittenArgumentOfTailCall - Check if the operand could +/// possibly be overwritten when lowering the outgoing arguments in a tail +/// call. Currently the implementation of this call is very conservative and +/// assumes all arguments sourcing from FORMAL_ARGUMENTS or a CopyFromReg with +/// virtual registers would be overwritten by direct lowering. +static bool IsPossiblyOverwrittenArgumentOfTailCall(SDOperand Op, + MachineFrameInfo * MFI) { + RegisterSDNode * OpReg = NULL; + if (Op.getOpcode() == ISD::FORMAL_ARGUMENTS || + (Op.getOpcode()== ISD::CopyFromReg && + (OpReg = dyn_cast(Op.getOperand(1))) && + (OpReg->getReg() >= TargetRegisterInfo::FirstVirtualRegister)) || + (Op.getOpcode() == ISD::LOAD && + IsFixedFrameObjectWithPosOffset(MFI, Op.getOperand(1))) || + (Op.getOpcode() == ISD::MERGE_VALUES && + Op.getOperand(Op.ResNo).getOpcode() == ISD::LOAD && + IsFixedFrameObjectWithPosOffset(MFI, Op.getOperand(Op.ResNo). + getOperand(1)))) + return true; + return false; +} + /// CheckDAGForTailCallsAndFixThem - This Function looks for CALL nodes in the /// DAG and fixes their tailcall attribute operand. static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG, @@ -4601,29 +5004,62 @@ static void CheckDAGForTailCallsAndFixThem(SelectionDAG &DAG, // Fix tail call attribute of CALL nodes. for (SelectionDAG::allnodes_iterator BE = DAG.allnodes_begin(), - BI = prior(DAG.allnodes_end()); BI != BE; --BI) { + BI = DAG.allnodes_end(); BI != BE; ) { + --BI; if (BI->getOpcode() == ISD::CALL) { SDOperand OpRet(Ret, 0); - SDOperand OpCall(static_cast(BI), 0); + SDOperand OpCall(BI, 0); bool isMarkedTailCall = cast(OpCall.getOperand(3))->getValue() != 0; // If CALL node has tail call attribute set to true and the call is not // eligible (no RET or the target rejects) the attribute is fixed to // false. The TargetLowering::IsEligibleForTailCallOptimization function // must correctly identify tail call optimizable calls. - if (isMarkedTailCall && - (Ret==NULL || - !TLI.IsEligibleForTailCallOptimization(OpCall, OpRet, DAG))) { + if (!isMarkedTailCall) continue; + if (Ret==NULL || + !TLI.IsEligibleForTailCallOptimization(OpCall, OpRet, DAG)) { + // Not eligible. Mark CALL node as non tail call. SmallVector Ops; unsigned idx=0; - for(SDNode::op_iterator I =OpCall.Val->op_begin(), - E=OpCall.Val->op_end(); I!=E; I++, idx++) { + for(SDNode::op_iterator I =OpCall.Val->op_begin(), + E = OpCall.Val->op_end(); I != E; I++, idx++) { if (idx!=3) Ops.push_back(*I); - else + else Ops.push_back(DAG.getConstant(false, TLI.getPointerTy())); } DAG.UpdateNodeOperands(OpCall, Ops.begin(), Ops.size()); + } else { + // Look for tail call clobbered arguments. Emit a series of + // copyto/copyfrom virtual register nodes to protect them. + SmallVector Ops; + SDOperand Chain = OpCall.getOperand(0), InFlag; + unsigned idx=0; + for(SDNode::op_iterator I = OpCall.Val->op_begin(), + E = OpCall.Val->op_end(); I != E; I++, idx++) { + SDOperand Arg = *I; + if (idx > 4 && (idx % 2)) { + bool isByVal = cast(OpCall.getOperand(idx+1))-> + getArgFlags().isByVal(); + MachineFunction &MF = DAG.getMachineFunction(); + MachineFrameInfo *MFI = MF.getFrameInfo(); + if (!isByVal && + IsPossiblyOverwrittenArgumentOfTailCall(Arg, MFI)) { + MVT VT = Arg.getValueType(); + unsigned VReg = MF.getRegInfo(). + createVirtualRegister(TLI.getRegClassFor(VT)); + Chain = DAG.getCopyToReg(Chain, VReg, Arg, InFlag); + InFlag = Chain.getValue(1); + Arg = DAG.getCopyFromReg(Chain, VReg, VT, InFlag); + Chain = Arg.getValue(1); + InFlag = Arg.getValue(2); + } + } + Ops.push_back(Arg); + } + // Link in chain of CopyTo/CopyFromReg. + Ops[0] = Chain; + DAG.UpdateNodeOperands(OpCall, Ops.begin(), Ops.size()); } } } @@ -4634,24 +5070,20 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, FunctionLoweringInfo &FuncInfo) { SelectionDAGLowering SDL(DAG, TLI, *AA, FuncInfo, GCI); - std::vector UnorderedChains; - // Lower any arguments needed in this block if this is the entry block. if (LLVMBB == &LLVMBB->getParent()->getEntryBlock()) - LowerArguments(LLVMBB, SDL, UnorderedChains); + LowerArguments(LLVMBB, SDL); BB = FuncInfo.MBBMap[LLVMBB]; SDL.setCurrentBasicBlock(BB); MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); - if (ExceptionHandling && MMI && BB->isLandingPad()) { + if (MMI && BB->isLandingPad()) { // Add a label to mark the beginning of the landing pad. Deletion of the // landing pad can thus be detected via the MachineModuleInfo. unsigned LabelID = MMI->addLandingPad(BB); - DAG.setRoot(DAG.getNode(ISD::LABEL, MVT::Other, DAG.getEntryNode(), - DAG.getConstant(LabelID, MVT::i32), - DAG.getConstant(1, MVT::i32))); + DAG.setRoot(DAG.getLabel(ISD::EH_LABEL, DAG.getEntryNode(), LabelID)); // Mark exception register as live in. unsigned Reg = TLI.getExceptionAddressRegister(); @@ -4697,8 +5129,7 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, if (!I->use_empty() && !isa(I) && !isa(I)) { DenseMap::iterator VMI =FuncInfo.ValueMap.find(I); if (VMI != FuncInfo.ValueMap.end()) - UnorderedChains.push_back( - SDL.CopyValueToVirtualRegister(I, VMI->second)); + SDL.CopyValueToVirtualRegister(I, VMI->second); } // Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to @@ -4749,8 +5180,7 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, unsigned &RegOut = ConstantsOut[C]; if (RegOut == 0) { RegOut = FuncInfo.CreateRegForValue(C); - UnorderedChains.push_back( - SDL.CopyValueToVirtualRegister(C, RegOut)); + SDL.CopyValueToVirtualRegister(C, RegOut); } Reg = RegOut; } else { @@ -4760,14 +5190,13 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, FuncInfo.StaticAllocaMap.count(cast(PHIOp)) && "Didn't codegen value into a register!??"); Reg = FuncInfo.CreateRegForValue(PHIOp); - UnorderedChains.push_back( - SDL.CopyValueToVirtualRegister(PHIOp, Reg)); + SDL.CopyValueToVirtualRegister(PHIOp, Reg); } } // Remember that this register needs to added to the machine PHI node as // the input for this MBB. - MVT::ValueType VT = TLI.getValueType(PN->getType()); + MVT VT = TLI.getValueType(PN->getType()); unsigned NumRegisters = TLI.getNumRegisters(VT); for (unsigned i = 0, e = NumRegisters; i != e; ++i) PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); @@ -4775,24 +5204,6 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, } ConstantsOut.clear(); - // Turn all of the unordered chains into one factored node. - if (!UnorderedChains.empty()) { - SDOperand Root = SDL.getRoot(); - if (Root.getOpcode() != ISD::EntryToken) { - unsigned i = 0, e = UnorderedChains.size(); - for (; i != e; ++i) { - assert(UnorderedChains[i].Val->getNumOperands() > 1); - if (UnorderedChains[i].Val->getOperand(0) == Root) - break; // Don't add the root if we already indirectly depend on it. - } - - if (i == e) - UnorderedChains.push_back(Root); - } - DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other, - &UnorderedChains[0], UnorderedChains.size())); - } - // Lower the terminator after the copies are emitted. SDL.visit(*LLVMBB->getTerminator()); @@ -4806,7 +5217,7 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, BitTestCases = SDL.BitTestCases; // Make sure the root of the DAG is up-to-date. - DAG.setRoot(SDL.getRoot()); + DAG.setRoot(SDL.getControlRoot()); // Check whether calls in this block are real tail calls. Fix up CALL nodes // with correct tailcall attribute so that the target can rely on the tailcall @@ -4815,48 +5226,177 @@ void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, CheckDAGForTailCallsAndFixThem(DAG, TLI); } +void SelectionDAGISel::ComputeLiveOutVRegInfo(SelectionDAG &DAG) { + SmallPtrSet VisitedNodes; + SmallVector Worklist; + + Worklist.push_back(DAG.getRoot().Val); + + APInt Mask; + APInt KnownZero; + APInt KnownOne; + + while (!Worklist.empty()) { + SDNode *N = Worklist.back(); + Worklist.pop_back(); + + // If we've already seen this node, ignore it. + if (!VisitedNodes.insert(N)) + continue; + + // Otherwise, add all chain operands to the worklist. + for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) + if (N->getOperand(i).getValueType() == MVT::Other) + Worklist.push_back(N->getOperand(i).Val); + + // If this is a CopyToReg with a vreg dest, process it. + if (N->getOpcode() != ISD::CopyToReg) + continue; + + unsigned DestReg = cast(N->getOperand(1))->getReg(); + if (!TargetRegisterInfo::isVirtualRegister(DestReg)) + continue; + + // Ignore non-scalar or non-integer values. + SDOperand Src = N->getOperand(2); + MVT SrcVT = Src.getValueType(); + if (!SrcVT.isInteger() || SrcVT.isVector()) + continue; + + unsigned NumSignBits = DAG.ComputeNumSignBits(Src); + Mask = APInt::getAllOnesValue(SrcVT.getSizeInBits()); + DAG.ComputeMaskedBits(Src, Mask, KnownZero, KnownOne); + + // Only install this information if it tells us something. + if (NumSignBits != 1 || KnownZero != 0 || KnownOne != 0) { + DestReg -= TargetRegisterInfo::FirstVirtualRegister; + FunctionLoweringInfo &FLI = DAG.getFunctionLoweringInfo(); + if (DestReg >= FLI.LiveOutRegInfo.size()) + FLI.LiveOutRegInfo.resize(DestReg+1); + FunctionLoweringInfo::LiveOutInfo &LOI = FLI.LiveOutRegInfo[DestReg]; + LOI.NumSignBits = NumSignBits; + LOI.KnownOne = NumSignBits; + LOI.KnownZero = NumSignBits; + } + } +} + void SelectionDAGISel::CodeGenAndEmitDAG(SelectionDAG &DAG) { DOUT << "Lowered selection DAG:\n"; DEBUG(DAG.dump()); + std::string GroupName = "Instruction Selection and Scheduling"; // Run the DAG combiner in pre-legalize mode. - DAG.Combine(false, *AA); + if (TimePassesIsEnabled) { + NamedRegionTimer T("DAG Combining 1", GroupName); + DAG.Combine(false, *AA); + } else { + DAG.Combine(false, *AA); + } DOUT << "Optimized lowered selection DAG:\n"; DEBUG(DAG.dump()); // Second step, hack on the DAG until it only uses operations and types that // the target supports. -#if 0 // Enable this some day. - DAG.LegalizeTypes(); - // Someday even later, enable a dag combine pass here. -#endif - DAG.Legalize(); + if (EnableLegalizeTypes) {// Enable this some day. + DAG.LegalizeTypes(); + // TODO: enable a dag combine pass here. + } + + if (TimePassesIsEnabled) { + NamedRegionTimer T("DAG Legalization", GroupName); + DAG.Legalize(); + } else { + DAG.Legalize(); + } DOUT << "Legalized selection DAG:\n"; DEBUG(DAG.dump()); // Run the DAG combiner in post-legalize mode. - DAG.Combine(true, *AA); + if (TimePassesIsEnabled) { + NamedRegionTimer T("DAG Combining 2", GroupName); + DAG.Combine(true, *AA); + } else { + DAG.Combine(true, *AA); + } DOUT << "Optimized legalized selection DAG:\n"; DEBUG(DAG.dump()); if (ViewISelDAGs) DAG.viewGraph(); + + if (!FastISel && EnableValueProp) + ComputeLiveOutVRegInfo(DAG); // Third, instruction select all of the operations to machine code, adding the // code to the MachineBasicBlock. - InstructionSelectBasicBlock(DAG); + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Selection", GroupName); + InstructionSelect(DAG); + } else { + InstructionSelect(DAG); + } + + // Schedule machine code. + ScheduleDAG *Scheduler; + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Scheduling", GroupName); + Scheduler = Schedule(DAG); + } else { + Scheduler = Schedule(DAG); + } + + // Emit machine code to BB. This can change 'BB' to the last block being + // inserted into. + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Creation", GroupName); + BB = Scheduler->EmitSchedule(); + } else { + BB = Scheduler->EmitSchedule(); + } + + // Free the scheduler state. + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Scheduling Cleanup", GroupName); + delete Scheduler; + } else { + delete Scheduler; + } + + // Perform target specific isel post processing. + if (TimePassesIsEnabled) { + NamedRegionTimer T("Instruction Selection Post Processing", GroupName); + InstructionSelectPostProcessing(DAG); + } else { + InstructionSelectPostProcessing(DAG); + } DOUT << "Selected machine code:\n"; DEBUG(BB->dump()); } +void SelectionDAGISel::SelectAllBasicBlocks(Function &Fn, MachineFunction &MF, + FunctionLoweringInfo &FuncInfo) { + // Define AllNodes here so that memory allocation is reused for + // each basic block. + alist AllNodes; + + for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) { + SelectBasicBlock(I, MF, FuncInfo, AllNodes); + AllNodes.clear(); + } +} + void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, - FunctionLoweringInfo &FuncInfo) { + FunctionLoweringInfo &FuncInfo, + alist &AllNodes) { std::vector > PHINodesToUpdate; { - SelectionDAG DAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG DAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + AllNodes); CurDAG = &DAG; // First step, lower LLVM code to some DAG. This DAG may use operations and @@ -4890,7 +5430,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i) { // Lower header first, if it wasn't already lowered if (!BitTestCases[i].Emitted) { - SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG HSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + AllNodes); CurDAG = &HSDAG; SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -4903,7 +5445,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, } for (unsigned j = 0, ej = BitTestCases[i].Cases.size(); j != ej; ++j) { - SelectionDAG BSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG BSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + AllNodes); CurDAG = &BSDAG; SelectionDAGLowering BSDL(BSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -4960,7 +5504,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, for (unsigned i = 0, e = JTCases.size(); i != e; ++i) { // Lower header first, if it wasn't already lowered if (!JTCases[i].first.Emitted) { - SelectionDAG HSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG HSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + AllNodes); CurDAG = &HSDAG; SelectionDAGLowering HSDL(HSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -4972,7 +5518,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, CodeGenAndEmitDAG(HSDAG); } - SelectionDAG JSDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG JSDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + AllNodes); CurDAG = &JSDAG; SelectionDAGLowering JSDL(JSDAG, TLI, *AA, FuncInfo, GCI); // Set the current basic block to the mbb we wish to insert the code into @@ -5020,7 +5568,9 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, // If we generated any switch lowering information, build and codegen any // additional DAGs necessary. for (unsigned i = 0, e = SwitchCases.size(); i != e; ++i) { - SelectionDAG SDAG(TLI, MF, getAnalysisToUpdate()); + SelectionDAG SDAG(TLI, MF, FuncInfo, + getAnalysisToUpdate(), + AllNodes); CurDAG = &SDAG; SelectionDAGLowering SDL(SDAG, TLI, *AA, FuncInfo, GCI); @@ -5065,10 +5615,10 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, } -//===----------------------------------------------------------------------===// -/// ScheduleAndEmitDAG - Pick a safe ordering and emit instructions for each +/// Schedule - Pick a safe ordering for instructions for each /// target node in the graph. -void SelectionDAGISel::ScheduleAndEmitDAG(SelectionDAG &DAG) { +/// +ScheduleDAG *SelectionDAGISel::Schedule(SelectionDAG &DAG) { if (ViewSchedDAGs) DAG.viewGraph(); RegisterScheduler::FunctionPassCtor Ctor = RegisterScheduler::getDefault(); @@ -5078,12 +5628,11 @@ void SelectionDAGISel::ScheduleAndEmitDAG(SelectionDAG &DAG) { RegisterScheduler::setDefault(Ctor); } - ScheduleDAG *SL = Ctor(this, &DAG, BB); - BB = SL->Run(); + ScheduleDAG *Scheduler = Ctor(this, &DAG, BB, FastISel); + Scheduler->Run(); - if (ViewSUnitDAGs) SL->viewGraph(); - - delete SL; + if (ViewSUnitDAGs) Scheduler->viewGraph(); + return Scheduler; } @@ -5102,20 +5651,20 @@ HazardRecognizer *SelectionDAGISel::CreateTargetHazardRecognizer() { /// specified in the .td file (e.g. 255). bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const { - uint64_t ActualMask = RHS->getValue(); - uint64_t DesiredMask =DesiredMaskS & MVT::getIntVTBitMask(LHS.getValueType()); + const APInt &ActualMask = RHS->getAPIntValue(); + const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS); // If the actual mask exactly matches, success! if (ActualMask == DesiredMask) return true; // If the actual AND mask is allowing unallowed bits, this doesn't match. - if (ActualMask & ~DesiredMask) + if (ActualMask.intersects(~DesiredMask)) return false; // Otherwise, the DAG Combiner may have proven that the value coming in is // either already zero or is not demanded. Check for known zero input bits. - uint64_t NeededMask = DesiredMask & ~ActualMask; + APInt NeededMask = DesiredMask & ~ActualMask; if (CurDAG->MaskedValueIsZero(LHS, NeededMask)) return true; @@ -5130,23 +5679,23 @@ bool SelectionDAGISel::CheckAndMask(SDOperand LHS, ConstantSDNode *RHS, /// actual value in the DAG on the RHS of an OR, and DesiredMaskS is the value /// specified in the .td file (e.g. 255). bool SelectionDAGISel::CheckOrMask(SDOperand LHS, ConstantSDNode *RHS, - int64_t DesiredMaskS) const { - uint64_t ActualMask = RHS->getValue(); - uint64_t DesiredMask =DesiredMaskS & MVT::getIntVTBitMask(LHS.getValueType()); + int64_t DesiredMaskS) const { + const APInt &ActualMask = RHS->getAPIntValue(); + const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS); // If the actual mask exactly matches, success! if (ActualMask == DesiredMask) return true; // If the actual AND mask is allowing unallowed bits, this doesn't match. - if (ActualMask & ~DesiredMask) + if (ActualMask.intersects(~DesiredMask)) return false; // Otherwise, the DAG Combiner may have proven that the value coming in is // either already zero or is not demanded. Check for known zero input bits. - uint64_t NeededMask = DesiredMask & ~ActualMask; + APInt NeededMask = DesiredMask & ~ActualMask; - uint64_t KnownZero, KnownOne; + APInt KnownZero, KnownOne; CurDAG->ComputeMaskedBits(LHS, NeededMask, KnownZero, KnownOne); // If all the missing bits in the or are already known to be set, match! @@ -5190,7 +5739,7 @@ SelectInlineAsmMemoryOperands(std::vector &Ops, SelectionDAG &DAG) { } // Add this to the output node. - MVT::ValueType IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); + MVT IntPtrTy = DAG.getTargetLoweringInfo().getPointerTy(); Ops.push_back(DAG.getTargetConstant(4/*MEM*/ | (SelOps.size() << 3), IntPtrTy)); Ops.insert(Ops.end(), SelOps.begin(), SelOps.end());