X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FLegalizeDAG.cpp;h=5866e1a9f97f233f0cb80b22991aef1eb6c4a7f5;hb=9a6b92de4c2207b427f3b9cd67cd122dafc5b6c6;hp=3be33770bc88fee9615dbd13a68c3a8635c9fd63;hpb=3461cc9e9842c7bce46b9d6e114ff8e44d8ba7b9;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 3be33770bc8..5866e1a9f97 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -35,14 +35,6 @@ #include using namespace llvm; -#ifndef NDEBUG -static cl::opt -ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, - cl::desc("Pop up a window to show dags before legalize")); -#else -static const bool ViewLegalizeDAGs = 0; -#endif - //===----------------------------------------------------------------------===// /// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and /// hacks on it until the target machine can handle it. This involves @@ -65,7 +57,7 @@ class VISIBILITY_HIDDEN SelectionDAGLegalize { /// LastCALLSEQ_END - This keeps track of the CALLSEQ_END node that has been /// legalized. We use this to ensure that calls are properly serialized /// against each other, including inserted libcalls. - SDOperand LastCALLSEQ_END; + SDValue LastCALLSEQ_END; /// IsLegalizingCall - This member is used *only* for purposes of providing /// helpful assertions that a libcall isn't created while another call is @@ -86,44 +78,54 @@ class VISIBILITY_HIDDEN SelectionDAGLegalize { /// LegalizedNodes - For nodes that are of legal width, and that have more /// than one use, this map indicates what regularized operand to use. This /// allows us to avoid legalizing the same thing more than once. - DenseMap LegalizedNodes; + DenseMap LegalizedNodes; /// PromotedNodes - For nodes that are below legal width, and that have more /// than one use, this map indicates what promoted value to use. This allows /// us to avoid promoting the same thing more than once. - DenseMap PromotedNodes; + DenseMap PromotedNodes; /// ExpandedNodes - For nodes that need to be expanded this map indicates - /// which which operands are the expanded version of the input. This allows + /// which operands are the expanded version of the input. This allows /// us to avoid expanding the same node more than once. - DenseMap > ExpandedNodes; + DenseMap > ExpandedNodes; /// SplitNodes - For vector nodes that need to be split, this map indicates - /// which which operands are the split version of the input. This allows us + /// which operands are the split version of the input. This allows us /// to avoid splitting the same node more than once. - std::map > SplitNodes; + std::map > SplitNodes; /// ScalarizedNodes - For nodes that need to be converted from vector types to /// scalar types, this contains the mapping of ones we have already /// processed to the result. - std::map ScalarizedNodes; + std::map ScalarizedNodes; - void AddLegalizedOperand(SDOperand From, SDOperand To) { + /// WidenNodes - For nodes that need to be widened from one vector type to + /// another, this contains the mapping of those that we have already widen. + /// This allows us to avoid widening more than once. + std::map WidenNodes; + + void AddLegalizedOperand(SDValue From, SDValue To) { LegalizedNodes.insert(std::make_pair(From, To)); // If someone requests legalization of the new node, return itself. if (From != To) LegalizedNodes.insert(std::make_pair(To, To)); } - void AddPromotedOperand(SDOperand From, SDOperand To) { - bool isNew = PromotedNodes.insert(std::make_pair(From, To)); + void AddPromotedOperand(SDValue From, SDValue To) { + bool isNew = PromotedNodes.insert(std::make_pair(From, To)).second; + assert(isNew && "Got into the map somehow?"); + // If someone requests legalization of the new node, return itself. + LegalizedNodes.insert(std::make_pair(To, To)); + } + void AddWidenedOperand(SDValue From, SDValue To) { + bool isNew = WidenNodes.insert(std::make_pair(From, To)).second; assert(isNew && "Got into the map somehow?"); // If someone requests legalization of the new node, return itself. LegalizedNodes.insert(std::make_pair(To, To)); } public: - - SelectionDAGLegalize(SelectionDAG &DAG); + explicit SelectionDAGLegalize(SelectionDAG &DAG); /// getTypeAction - Return how we should legalize values of this type, either /// it is already legal or we need to expand it into multiple registers of @@ -143,51 +145,115 @@ public: private: /// HandleOp - Legalize, Promote, or Expand the specified operand as /// appropriate for its type. - void HandleOp(SDOperand Op); + void HandleOp(SDValue Op); /// LegalizeOp - We know that the specified value has a legal type. /// Recursively ensure that the operands have legal types, then return the /// result. - SDOperand LegalizeOp(SDOperand O); + SDValue LegalizeOp(SDValue O); /// UnrollVectorOp - We know that the given vector has a legal type, however /// the operation it performs is not legal and is an operation that we have /// no way of lowering. "Unroll" the vector, splitting out the scalars and /// operating on each element individually. - SDOperand UnrollVectorOp(SDOperand O); + SDValue UnrollVectorOp(SDValue O); /// PerformInsertVectorEltInMemory - Some target cannot handle a variable /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it /// is necessary to spill the vector being inserted into to memory, perform /// the insert there, and then read the result back. - SDOperand PerformInsertVectorEltInMemory(SDOperand Vec, SDOperand Val, - SDOperand Idx); + SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, + SDValue Idx); /// PromoteOp - Given an operation that produces a value in an invalid type, /// promote it to compute the value into a larger type. The produced value /// will have the correct bits for the low portion of the register, but no /// guarantee is made about the top bits: it may be zero, sign-extended, or /// garbage. - SDOperand PromoteOp(SDOperand O); + SDValue PromoteOp(SDValue O); - /// ExpandOp - Expand the specified SDOperand into its two component pieces + /// ExpandOp - Expand the specified SDValue into its two component pieces /// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, - /// the LegalizeNodes map is filled in for any results that are not expanded, + /// the LegalizedNodes map is filled in for any results that are not expanded, /// the ExpandedNodes map is filled in for any results that are expanded, and /// the Lo/Hi values are returned. This applies to integer types and Vector /// types. - void ExpandOp(SDOperand O, SDOperand &Lo, SDOperand &Hi); + void ExpandOp(SDValue O, SDValue &Lo, SDValue &Hi); + + /// WidenVectorOp - Widen a vector operation to a wider type given by WidenVT + /// (e.g., v3i32 to v4i32). The produced value will have the correct value + /// for the existing elements but no guarantee is made about the new elements + /// at the end of the vector: it may be zero, ones, or garbage. This is useful + /// when we have an instruction operating on an illegal vector type and we + /// want to widen it to do the computation on a legal wider vector type. + SDValue WidenVectorOp(SDValue Op, MVT WidenVT); /// SplitVectorOp - Given an operand of vector type, break it down into /// two smaller values. - void SplitVectorOp(SDOperand O, SDOperand &Lo, SDOperand &Hi); + void SplitVectorOp(SDValue O, SDValue &Lo, SDValue &Hi); /// ScalarizeVectorOp - Given an operand of single-element vector type /// (e.g. v1f32), convert it into the equivalent operation that returns a /// scalar (e.g. f32) value. - SDOperand ScalarizeVectorOp(SDOperand O); + SDValue ScalarizeVectorOp(SDValue O); + + /// Useful 16 element vector type that is used to pass operands for widening. + typedef SmallVector SDValueVector; + + /// LoadWidenVectorOp - Load a vector for a wider type. Returns true if + /// the LdChain contains a single load and false if it contains a token + /// factor for multiple loads. It takes + /// Result: location to return the result + /// LdChain: location to return the load chain + /// Op: load operation to widen + /// NVT: widen vector result type we want for the load + bool LoadWidenVectorOp(SDValue& Result, SDValue& LdChain, + SDValue Op, MVT NVT); + + /// Helper genWidenVectorLoads - Helper function to generate a set of + /// loads to load a vector with a resulting wider type. It takes + /// LdChain: list of chains for the load we have generated + /// Chain: incoming chain for the ld vector + /// BasePtr: base pointer to load from + /// SV: memory disambiguation source value + /// SVOffset: memory disambiugation offset + /// Alignment: alignment of the memory + /// isVolatile: volatile load + /// LdWidth: width of memory that we want to load + /// ResType: the wider result result type for the resulting loaded vector + SDValue genWidenVectorLoads(SDValueVector& LdChain, SDValue Chain, + SDValue BasePtr, const Value *SV, + int SVOffset, unsigned Alignment, + bool isVolatile, unsigned LdWidth, + MVT ResType); - /// isShuffleLegal - Return true if a vector shuffle is legal with the + /// StoreWidenVectorOp - Stores a widen vector into non widen memory + /// location. It takes + /// ST: store node that we want to replace + /// Chain: incoming store chain + /// BasePtr: base address of where we want to store into + SDValue StoreWidenVectorOp(StoreSDNode *ST, SDValue Chain, + SDValue BasePtr); + + /// Helper genWidenVectorStores - Helper function to generate a set of + /// stores to store a widen vector into non widen memory + // It takes + // StChain: list of chains for the stores we have generated + // Chain: incoming chain for the ld vector + // BasePtr: base pointer to load from + // SV: memory disambiguation source value + // SVOffset: memory disambiugation offset + // Alignment: alignment of the memory + // isVolatile: volatile lod + // ValOp: value to store + // StWidth: width of memory that we want to store + void genWidenVectorStores(SDValueVector& StChain, SDValue Chain, + SDValue BasePtr, const Value *SV, + int SVOffset, unsigned Alignment, + bool isVolatile, SDValue ValOp, + unsigned StWidth); + + /// isShuffleLegal - Return non-null if a vector shuffle is legal with the /// specified mask and type. Targets can specify exactly which masks they /// support and the code generator is tasked with not creating illegal masks. /// @@ -196,33 +262,39 @@ private: /// /// If this is a legal shuffle, this method returns the (possibly promoted) /// build_vector Mask. If it's not a legal shuffle, it returns null. - SDNode *isShuffleLegal(MVT VT, SDOperand Mask) const; + SDNode *isShuffleLegal(MVT VT, SDValue Mask) const; bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, SmallPtrSet &NodesLeadingTo); - void LegalizeSetCCOperands(SDOperand &LHS, SDOperand &RHS, SDOperand &CC); + void LegalizeSetCCOperands(SDValue &LHS, SDValue &RHS, SDValue &CC); + void LegalizeSetCCCondCode(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC); + void LegalizeSetCC(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC) { + LegalizeSetCCOperands(LHS, RHS, CC); + LegalizeSetCCCondCode(VT, LHS, RHS, CC); + } - SDOperand ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned, - SDOperand &Hi); - SDOperand ExpandIntToFP(bool isSigned, MVT DestTy, SDOperand Source); - - SDOperand EmitStackConvert(SDOperand SrcOp, MVT SlotVT, MVT DestVT); - SDOperand ExpandBUILD_VECTOR(SDNode *Node); - SDOperand ExpandSCALAR_TO_VECTOR(SDNode *Node); - SDOperand ExpandLegalINT_TO_FP(bool isSigned, SDOperand LegalOp, MVT DestVT); - SDOperand PromoteLegalINT_TO_FP(SDOperand LegalOp, MVT DestVT, bool isSigned); - SDOperand PromoteLegalFP_TO_INT(SDOperand LegalOp, MVT DestVT, bool isSigned); - - SDOperand ExpandBSWAP(SDOperand Op); - SDOperand ExpandBitCount(unsigned Opc, SDOperand Op); - bool ExpandShift(unsigned Opc, SDOperand Op, SDOperand Amt, - SDOperand &Lo, SDOperand &Hi); - void ExpandShiftParts(unsigned NodeOp, SDOperand Op, SDOperand Amt, - SDOperand &Lo, SDOperand &Hi); - - SDOperand ExpandEXTRACT_SUBVECTOR(SDOperand Op); - SDOperand ExpandEXTRACT_VECTOR_ELT(SDOperand Op); + SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned, + SDValue &Hi); + SDValue ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source); + + SDValue EmitStackConvert(SDValue SrcOp, MVT SlotVT, MVT DestVT); + SDValue ExpandBUILD_VECTOR(SDNode *Node); + SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); + SDValue LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, SDValue Op); + SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, MVT DestVT); + SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, MVT DestVT, bool isSigned); + SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, MVT DestVT, bool isSigned); + + SDValue ExpandBSWAP(SDValue Op); + SDValue ExpandBitCount(unsigned Opc, SDValue Op); + bool ExpandShift(unsigned Opc, SDValue Op, SDValue Amt, + SDValue &Lo, SDValue &Hi); + void ExpandShiftParts(unsigned NodeOp, SDValue Op, SDValue Amt, + SDValue &Lo, SDValue &Hi); + + SDValue ExpandEXTRACT_SUBVECTOR(SDValue Op); + SDValue ExpandEXTRACT_VECTOR_ELT(SDValue Op); }; } @@ -232,7 +304,7 @@ private: /// /// Note that this will also return true for shuffles that are promoted to a /// different type. -SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDOperand Mask) const { +SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDValue Mask) const { switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) { default: return 0; case TargetLowering::Legal: @@ -242,6 +314,7 @@ SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDOperand Mask) const { // If this is promoted to a different type, convert the shuffle mask and // ask if it is legal in the promoted type! MVT NVT = TLI.getTypeToPromoteTo(ISD::VECTOR_SHUFFLE, VT); + MVT EltVT = NVT.getVectorElementType(); // If we changed # elements, change the shuffle mask. unsigned NumEltsGrowth = @@ -249,15 +322,15 @@ SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDOperand Mask) const { assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); if (NumEltsGrowth > 1) { // Renumber the elements. - SmallVector Ops; + SmallVector Ops; for (unsigned i = 0, e = Mask.getNumOperands(); i != e; ++i) { - SDOperand InOp = Mask.getOperand(i); + SDValue InOp = Mask.getOperand(i); for (unsigned j = 0; j != NumEltsGrowth; ++j) { if (InOp.getOpcode() == ISD::UNDEF) - Ops.push_back(DAG.getNode(ISD::UNDEF, MVT::i32)); + Ops.push_back(DAG.getNode(ISD::UNDEF, EltVT)); else { - unsigned InEltNo = cast(InOp)->getValue(); - Ops.push_back(DAG.getConstant(InEltNo*NumEltsGrowth+j, MVT::i32)); + unsigned InEltNo = cast(InOp)->getZExtValue(); + Ops.push_back(DAG.getConstant(InEltNo*NumEltsGrowth+j, EltVT)); } } } @@ -267,7 +340,7 @@ SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDOperand Mask) const { break; } } - return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.Val : 0; + return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.getNode() : 0; } SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag) @@ -277,47 +350,6 @@ SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag) "Too many value types for ValueTypeActions to hold!"); } -/// ComputeTopDownOrdering - Compute a top-down ordering of the dag, where Order -/// contains all of a nodes operands before it contains the node. -static void ComputeTopDownOrdering(SelectionDAG &DAG, - SmallVector &Order) { - - DenseMap Visited; - std::vector Worklist; - Worklist.reserve(128); - - // Compute ordering from all of the leaves in the graphs, those (like the - // entry node) that have no operands. - for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), - E = DAG.allnodes_end(); I != E; ++I) { - if (I->getNumOperands() == 0) { - Visited[I] = 0 - 1U; - Worklist.push_back(I); - } - } - - while (!Worklist.empty()) { - SDNode *N = Worklist.back(); - Worklist.pop_back(); - - if (++Visited[N] != N->getNumOperands()) - continue; // Haven't visited all operands yet - - Order.push_back(N); - - // Now that we have N in, add anything that uses it if all of their operands - // are now done. - for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); - UI != E; ++UI) - Worklist.push_back(UI->getUser()); - } - - assert(Order.size() == Visited.size() && - Order.size() == DAG.allnodes_size() && - "Error: DAG is cyclic!"); -} - - void SelectionDAGLegalize::LegalizeDAG() { LastCALLSEQ_END = DAG.getEntryNode(); IsLegalizingCall = false; @@ -328,14 +360,13 @@ void SelectionDAGLegalize::LegalizeDAG() { // practice however, this causes us to run out of stack space on large basic // blocks. To avoid this problem, compute an ordering of the nodes where each // node is only legalized after all of its operands are legalized. - SmallVector Order; - ComputeTopDownOrdering(DAG, Order); - - for (unsigned i = 0, e = Order.size(); i != e; ++i) - HandleOp(SDOperand(Order[i], 0)); + DAG.AssignTopologicalOrder(); + for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), + E = prior(DAG.allnodes_end()); I != next(E); ++I) + HandleOp(SDValue(I, 0)); // Finally, it's possible the root changed. Get the new root. - SDOperand OldRoot = DAG.getRoot(); + SDValue OldRoot = DAG.getRoot(); assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?"); DAG.setRoot(LegalizedNodes[OldRoot]); @@ -344,6 +375,7 @@ void SelectionDAGLegalize::LegalizeDAG() { PromotedNodes.clear(); SplitNodes.clear(); ScalarizedNodes.clear(); + WidenNodes.clear(); // Remove dead nodes now. DAG.RemoveDeadNodes(); @@ -359,15 +391,15 @@ static SDNode *FindCallEndFromCallStart(SDNode *Node) { return 0; // No CallSeqEnd // The chain is usually at the end. - SDOperand TheChain(Node, Node->getNumValues()-1); + SDValue TheChain(Node, Node->getNumValues()-1); if (TheChain.getValueType() != MVT::Other) { // Sometimes it's at the beginning. - TheChain = SDOperand(Node, 0); + TheChain = SDValue(Node, 0); if (TheChain.getValueType() != MVT::Other) { // Otherwise, hunt for it. for (unsigned i = 1, e = Node->getNumValues(); i != e; ++i) if (Node->getValueType(i) == MVT::Other) { - TheChain = SDOperand(Node, i); + TheChain = SDValue(Node, i); break; } @@ -381,7 +413,7 @@ static SDNode *FindCallEndFromCallStart(SDNode *Node) { E = Node->use_end(); UI != E; ++UI) { // Make sure to only follow users of our token chain. - SDNode *User = UI->getUser(); + SDNode *User = *UI; for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) if (User->getOperand(i) == TheChain) if (SDNode *Result = FindCallEndFromCallStart(User)) @@ -398,7 +430,7 @@ static SDNode *FindCallStartFromCallEnd(SDNode *Node) { assert(Node->getOperand(0).getValueType() == MVT::Other && "Node doesn't have a token chain argument!"); - return FindCallStartFromCallEnd(Node->getOperand(0).Val); + return FindCallStartFromCallEnd(Node->getOperand(0).getNode()); } /// LegalizeAllNodesNotLeadingTo - Recursively walk the uses of N, looking to @@ -420,13 +452,13 @@ bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, // reach N. switch (getTypeAction(N->getValueType(0))) { case Legal: - if (LegalizedNodes.count(SDOperand(N, 0))) return false; + if (LegalizedNodes.count(SDValue(N, 0))) return false; break; case Promote: - if (PromotedNodes.count(SDOperand(N, 0))) return false; + if (PromotedNodes.count(SDValue(N, 0))) return false; break; case Expand: - if (ExpandedNodes.count(SDOperand(N, 0))) return false; + if (ExpandedNodes.count(SDValue(N, 0))) return false; break; } @@ -435,7 +467,7 @@ bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, bool OperandsLeadToDest = false; for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) OperandsLeadToDest |= // If an operand leads to Dest, so do we. - LegalizeAllNodesNotLeadingTo(N->getOperand(i).Val, Dest, NodesLeadingTo); + LegalizeAllNodesNotLeadingTo(N->getOperand(i).getNode(), Dest, NodesLeadingTo); if (OperandsLeadToDest) { NodesLeadingTo.insert(N); @@ -443,23 +475,36 @@ bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, } // Okay, this node looks safe, legalize it and return false. - HandleOp(SDOperand(N, 0)); + HandleOp(SDValue(N, 0)); return false; } -/// HandleOp - Legalize, Promote, or Expand the specified operand as +/// HandleOp - Legalize, Promote, Widen, or Expand the specified operand as /// appropriate for its type. -void SelectionDAGLegalize::HandleOp(SDOperand Op) { +void SelectionDAGLegalize::HandleOp(SDValue Op) { MVT VT = Op.getValueType(); switch (getTypeAction(VT)) { default: assert(0 && "Bad type action!"); case Legal: (void)LegalizeOp(Op); break; - case Promote: (void)PromoteOp(Op); break; + case Promote: + if (!VT.isVector()) { + (void)PromoteOp(Op); + break; + } + else { + // See if we can widen otherwise use Expand to either scalarize or split + MVT WidenVT = TLI.getWidenVectorType(VT); + if (WidenVT != MVT::Other) { + (void) WidenVectorOp(Op, WidenVT); + break; + } + // else fall thru to expand since we can't widen the vector + } case Expand: if (!VT.isVector()) { // If this is an illegal scalar, expand it into its two component // pieces. - SDOperand X, Y; + SDValue X, Y; if (Op.getOpcode() == ISD::TargetConstant) break; // Allow illegal target nodes. ExpandOp(Op, X, Y); @@ -468,9 +513,9 @@ void SelectionDAGLegalize::HandleOp(SDOperand Op) { // scalar operation. (void)ScalarizeVectorOp(Op); } else { - // Otherwise, this is an illegal multiple element vector. + // This is an illegal multiple element vector. // Split it in half and legalize both parts. - SDOperand X, Y; + SDValue X, Y; SplitVectorOp(Op, X, Y); } break; @@ -479,7 +524,7 @@ void SelectionDAGLegalize::HandleOp(SDOperand Op) { /// ExpandConstantFP - Expands the ConstantFP node to an integer constant or /// a load from the constant pool. -static SDOperand ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, +static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, SelectionDAG &DAG, TargetLowering &TLI) { bool Extend = false; @@ -490,11 +535,11 @@ static SDOperand ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, // an FP extending load is the same cost as a normal load (such as on the x87 // fp stack or PPC FP unit). MVT VT = CFP->getValueType(0); - ConstantFP *LLVMC = ConstantFP::get(CFP->getValueAPF()); + ConstantFP *LLVMC = const_cast(CFP->getConstantFPValue()); if (!UseCP) { if (VT!=MVT::f64 && VT!=MVT::f32) assert(0 && "Invalid type expansion"); - return DAG.getConstant(LLVMC->getValueAPF().convertToAPInt(), + return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), (VT == MVT::f64) ? MVT::i64 : MVT::i32); } @@ -505,7 +550,7 @@ static SDOperand ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, if (CFP->isValueValidForType(SVT, CFP->getValueAPF()) && // Only do this if the target has a native EXTLOAD instruction from // smaller type. - TLI.isLoadXLegal(ISD::EXTLOAD, SVT) && + TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && TLI.ShouldShrinkFPConstant(OrigVT)) { const Type *SType = SVT.getTypeForMVT(); LLVMC = cast(ConstantExpr::getFPTrunc(LLVMC, SType)); @@ -514,21 +559,22 @@ static SDOperand ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, } } - SDOperand CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); + SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); + unsigned Alignment = 1 << cast(CPIdx)->getAlignment(); if (Extend) return DAG.getExtLoad(ISD::EXTLOAD, OrigVT, DAG.getEntryNode(), CPIdx, PseudoSourceValue::getConstantPool(), - 0, VT); + 0, VT, false, Alignment); return DAG.getLoad(OrigVT, DAG.getEntryNode(), CPIdx, - PseudoSourceValue::getConstantPool(), 0); + PseudoSourceValue::getConstantPool(), 0, false, Alignment); } /// ExpandFCOPYSIGNToBitwiseOps - Expands fcopysign to a series of bitwise /// operations. static -SDOperand ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT, - SelectionDAG &DAG, TargetLowering &TLI) { +SDValue ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT, + SelectionDAG &DAG, TargetLowering &TLI) { MVT VT = Node->getValueType(0); MVT SrcVT = Node->getOperand(1).getValueType(); assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) && @@ -536,11 +582,11 @@ SDOperand ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT, MVT SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32; // First get the sign bit of second operand. - SDOperand Mask1 = (SrcVT == MVT::f64) + SDValue Mask1 = (SrcVT == MVT::f64) ? DAG.getConstantFP(BitsToDouble(1ULL << 63), SrcVT) : DAG.getConstantFP(BitsToFloat(1U << 31), SrcVT); Mask1 = DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Mask1); - SDOperand SignBit= DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Node->getOperand(1)); + SDValue SignBit= DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Node->getOperand(1)); SignBit = DAG.getNode(ISD::AND, SrcNVT, SignBit, Mask1); // Shift right or sign-extend it if the two operands have different types. int SizeDiff = SrcNVT.getSizeInBits() - NVT.getSizeInBits(); @@ -548,15 +594,18 @@ SDOperand ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT, SignBit = DAG.getNode(ISD::SRL, SrcNVT, SignBit, DAG.getConstant(SizeDiff, TLI.getShiftAmountTy())); SignBit = DAG.getNode(ISD::TRUNCATE, NVT, SignBit); - } else if (SizeDiff < 0) - SignBit = DAG.getNode(ISD::SIGN_EXTEND, NVT, SignBit); + } else if (SizeDiff < 0) { + SignBit = DAG.getNode(ISD::ZERO_EXTEND, NVT, SignBit); + SignBit = DAG.getNode(ISD::SHL, NVT, SignBit, + DAG.getConstant(-SizeDiff, TLI.getShiftAmountTy())); + } // Clear the sign bit of first operand. - SDOperand Mask2 = (VT == MVT::f64) + SDValue Mask2 = (VT == MVT::f64) ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT) : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT); Mask2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask2); - SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0)); + SDValue Result = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0)); Result = DAG.getNode(ISD::AND, NVT, Result, Mask2); // Or the value with the sign bit. @@ -566,11 +615,11 @@ SDOperand ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT, /// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores. static -SDOperand ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, - TargetLowering &TLI) { - SDOperand Chain = ST->getChain(); - SDOperand Ptr = ST->getBasePtr(); - SDOperand Val = ST->getValue(); +SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, + TargetLowering &TLI) { + SDValue Chain = ST->getChain(); + SDValue Ptr = ST->getBasePtr(); + SDValue Val = ST->getValue(); MVT VT = Val.getValueType(); int Alignment = ST->getAlignment(); int SVOffset = ST->getSrcValueOffset(); @@ -588,7 +637,7 @@ SDOperand ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, else assert(0 && "Unaligned store of unsupported type"); - SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, intVT, Val); + SDValue Result = DAG.getNode(ISD::BIT_CONVERT, intVT, Val); return DAG.getStore(Chain, Result, Ptr, ST->getSrcValue(), SVOffset, ST->isVolatile(), Alignment); } @@ -602,12 +651,12 @@ SDOperand ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, int IncrementSize = NumBits / 8; // Divide the stored value in two parts. - SDOperand ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy()); - SDOperand Lo = Val; - SDOperand Hi = DAG.getNode(ISD::SRL, VT, Val, ShiftAmount); + SDValue ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy()); + SDValue Lo = Val; + SDValue Hi = DAG.getNode(ISD::SRL, VT, Val, ShiftAmount); // Store the two parts - SDOperand Store1, Store2; + SDValue Store1, Store2; Store1 = DAG.getTruncStore(Chain, TLI.isLittleEndian()?Lo:Hi, Ptr, ST->getSrcValue(), SVOffset, NewStoredVT, ST->isVolatile(), Alignment); @@ -623,11 +672,11 @@ SDOperand ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, /// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads. static -SDOperand ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, - TargetLowering &TLI) { +SDValue ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, + TargetLowering &TLI) { int SVOffset = LD->getSrcValueOffset(); - SDOperand Chain = LD->getChain(); - SDOperand Ptr = LD->getBasePtr(); + SDValue Chain = LD->getChain(); + SDValue Ptr = LD->getBasePtr(); MVT VT = LD->getValueType(0); MVT LoadedVT = LD->getMemoryVT(); if (VT.isFloatingPoint() || VT.isVector()) { @@ -644,16 +693,15 @@ SDOperand ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, else assert(0 && "Unaligned load of unsupported type"); - SDOperand newLoad = DAG.getLoad(intVT, Chain, Ptr, LD->getSrcValue(), + SDValue newLoad = DAG.getLoad(intVT, Chain, Ptr, LD->getSrcValue(), SVOffset, LD->isVolatile(), LD->getAlignment()); - SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, LoadedVT, newLoad); + SDValue Result = DAG.getNode(ISD::BIT_CONVERT, LoadedVT, newLoad); if (VT.isFloatingPoint() && LoadedVT != VT) Result = DAG.getNode(ISD::FP_EXTEND, VT, Result); - SDOperand Ops[] = { Result, Chain }; - return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other), - Ops, 2); + SDValue Ops[] = { Result, Chain }; + return DAG.getMergeValues(Ops, 2); } assert(LoadedVT.isInteger() && !LoadedVT.isVector() && "Unaligned load of unsupported type."); @@ -674,7 +722,7 @@ SDOperand ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, HiExtType = ISD::ZEXTLOAD; // Load the value in two parts - SDOperand Lo, Hi; + SDValue Lo, Hi; if (TLI.isLittleEndian()) { Lo = DAG.getExtLoad(ISD::ZEXTLOAD, VT, Chain, Ptr, LD->getSrcValue(), SVOffset, NewLoadedVT, LD->isVolatile(), Alignment); @@ -694,35 +742,35 @@ SDOperand ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, } // aggregate the two parts - SDOperand ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy()); - SDOperand Result = DAG.getNode(ISD::SHL, VT, Hi, ShiftAmount); + SDValue ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy()); + SDValue Result = DAG.getNode(ISD::SHL, VT, Hi, ShiftAmount); Result = DAG.getNode(ISD::OR, VT, Result, Lo); - SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), + SDValue TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), Hi.getValue(1)); - SDOperand Ops[] = { Result, TF }; - return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other), Ops, 2); + SDValue Ops[] = { Result, TF }; + return DAG.getMergeValues(Ops, 2); } /// UnrollVectorOp - We know that the given vector has a legal type, however /// the operation it performs is not legal and is an operation that we have /// no way of lowering. "Unroll" the vector, splitting out the scalars and /// operating on each element individually. -SDOperand SelectionDAGLegalize::UnrollVectorOp(SDOperand Op) { +SDValue SelectionDAGLegalize::UnrollVectorOp(SDValue Op) { MVT VT = Op.getValueType(); assert(isTypeLegal(VT) && "Caller should expand or promote operands that are not legal!"); - assert(Op.Val->getNumValues() == 1 && + assert(Op.getNode()->getNumValues() == 1 && "Can't unroll a vector with multiple results!"); unsigned NE = VT.getVectorNumElements(); MVT EltVT = VT.getVectorElementType(); - SmallVector Scalars; - SmallVector Operands(Op.getNumOperands()); + SmallVector Scalars; + SmallVector Operands(Op.getNumOperands()); for (unsigned i = 0; i != NE; ++i) { for (unsigned j = 0; j != Op.getNumOperands(); ++j) { - SDOperand Operand = Op.getOperand(j); + SDValue Operand = Op.getOperand(j); MVT OperandVT = Operand.getValueType(); if (OperandVT.isVector()) { // A vector operand; extract a single element. @@ -761,11 +809,11 @@ static RTLIB::Libcall GetFPLibCall(MVT VT, /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it /// is necessary to spill the vector being inserted into to memory, perform /// the insert there, and then read the result back. -SDOperand SelectionDAGLegalize:: -PerformInsertVectorEltInMemory(SDOperand Vec, SDOperand Val, SDOperand Idx) { - SDOperand Tmp1 = Vec; - SDOperand Tmp2 = Val; - SDOperand Tmp3 = Idx; +SDValue SelectionDAGLegalize:: +PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx) { + SDValue Tmp1 = Vec; + SDValue Tmp2 = Val; + SDValue Tmp3 = Idx; // If the target doesn't support this, we have to spill the input vector // to a temporary stack slot, update the element, then reload it. This is @@ -777,15 +825,13 @@ PerformInsertVectorEltInMemory(SDOperand Vec, SDOperand Val, SDOperand Idx) { MVT EltVT = VT.getVectorElementType(); MVT IdxVT = Tmp3.getValueType(); MVT PtrVT = TLI.getPointerTy(); - SDOperand StackPtr = DAG.CreateStackTemporary(VT); + SDValue StackPtr = DAG.CreateStackTemporary(VT); - FrameIndexSDNode *StackPtrFI = cast(StackPtr.Val); - int SPFI = StackPtrFI->getIndex(); + int SPFI = cast(StackPtr.getNode())->getIndex(); // Store the vector. - SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Tmp1, StackPtr, - PseudoSourceValue::getFixedStack(), - SPFI); + SDValue Ch = DAG.getStore(DAG.getEntryNode(), Tmp1, StackPtr, + PseudoSourceValue::getFixedStack(SPFI), 0); // Truncate or zero extend offset to target pointer type. unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; @@ -793,25 +839,26 @@ PerformInsertVectorEltInMemory(SDOperand Vec, SDOperand Val, SDOperand Idx) { // Add the offset to the index. unsigned EltSize = EltVT.getSizeInBits()/8; Tmp3 = DAG.getNode(ISD::MUL, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); - SDOperand StackPtr2 = DAG.getNode(ISD::ADD, IdxVT, Tmp3, StackPtr); + SDValue StackPtr2 = DAG.getNode(ISD::ADD, IdxVT, Tmp3, StackPtr); // Store the scalar value. Ch = DAG.getTruncStore(Ch, Tmp2, StackPtr2, - PseudoSourceValue::getFixedStack(), SPFI, EltVT); + PseudoSourceValue::getFixedStack(SPFI), 0, EltVT); // Load the updated vector. - return DAG.getLoad(VT, Ch, StackPtr, PseudoSourceValue::getFixedStack(),SPFI); + return DAG.getLoad(VT, Ch, StackPtr, + PseudoSourceValue::getFixedStack(SPFI), 0); } /// LegalizeOp - We know that the specified value has a legal type, and /// that its operands are legal. Now ensure that the operation itself /// is legal, recursively ensuring that the operands' operations remain /// legal. -SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { +SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { if (Op.getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. return Op; assert(isTypeLegal(Op.getValueType()) && "Caller should expand or promote operands that are not legal!"); - SDNode *Node = Op.Val; + SDNode *Node = Op.getNode(); // If this operation defines any values that cannot be represented in a // register on this target, make sure to expand or promote them. @@ -827,11 +874,11 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // Note that LegalizeOp may be reentered even from single-use nodes, which // means that we always must cache transformed nodes. - DenseMap::iterator I = LegalizedNodes.find(Op); + DenseMap::iterator I = LegalizedNodes.find(Op); if (I != LegalizedNodes.end()) return I->second; - SDOperand Tmp1, Tmp2, Tmp3, Tmp4; - SDOperand Result = Op; + SDValue Tmp1, Tmp2, Tmp3, Tmp4; + SDValue Result = Op; bool isCustom = false; switch (Node->getOpcode()) { @@ -850,7 +897,6 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case ISD::VALUETYPE: case ISD::SRCVALUE: case ISD::MEMOPERAND: - case ISD::STRING: case ISD::CONDCODE: case ISD::ARG_FLAGS: // Primitives must all be legal. @@ -861,7 +907,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { // If this is a target node, legalize it by legalizing the operands then // passing it through. - SmallVector Ops; + SmallVector Ops; for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) Ops.push_back(LegalizeOp(Node->getOperand(i))); @@ -869,7 +915,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) AddLegalizedOperand(Op.getValue(i), Result.getValue(i)); - return Result.getValue(Op.ResNo); + return Result.getValue(Op.getResNo()); } // Otherwise this is an unhandled builtin node. splat. #ifndef NDEBUG @@ -887,7 +933,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { default: assert(0 && "This action is not supported yet!"); case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Op, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; // FALLTHROUGH if the target doesn't want to lower this op after all. case TargetLowering::Legal: break; @@ -898,7 +944,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // The only option for these nodes is to custom lower them. If the target // does not custom lower them, then return zero. Tmp1 = TLI.LowerOperation(Op, DAG); - if (Tmp1.Val) + if (Tmp1.getNode()) Result = Tmp1; else Result = DAG.getConstant(0, TLI.getPointerTy()); @@ -909,7 +955,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { default: assert(0 && "This action is not supported yet!"); case TargetLowering::Custom: Result = TLI.LowerOperation(Op, DAG); - if (Result.Val) break; + if (Result.getNode()) break; // Fall Thru case TargetLowering::Legal: Result = DAG.getConstant(0, VT); @@ -929,19 +975,18 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Custom: Result = TLI.LowerOperation(Op, DAG); - if (Result.Val) break; + if (Result.getNode()) break; // Fall Thru case TargetLowering::Legal: { - SDOperand Ops[] = { DAG.getConstant(0, VT), Tmp1 }; - Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other), - Ops, 2); + SDValue Ops[] = { DAG.getConstant(0, VT), Tmp1 }; + Result = DAG.getMergeValues(Ops, 2); break; } } } - if (Result.Val->getNumValues() == 1) break; + if (Result.getNode()->getNumValues() == 1) break; - assert(Result.Val->getNumValues() == 2 && + assert(Result.getNode()->getNumValues() == 2 && "Cannot return more than two values!"); // Since we produced two values, make sure to remember that we @@ -950,7 +995,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp2 = LegalizeOp(Result.getValue(1)); AddLegalizedOperand(Op.getValue(0), Tmp1); AddLegalizedOperand(Op.getValue(1), Tmp2); - return Op.ResNo ? Tmp2 : Tmp1; + return Op.getResNo() ? Tmp2 : Tmp1; case ISD::EHSELECTION: { Tmp1 = LegalizeOp(Node->getOperand(0)); Tmp2 = LegalizeOp(Node->getOperand(1)); @@ -964,19 +1009,18 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Custom: Result = TLI.LowerOperation(Op, DAG); - if (Result.Val) break; + if (Result.getNode()) break; // Fall Thru case TargetLowering::Legal: { - SDOperand Ops[] = { DAG.getConstant(0, VT), Tmp2 }; - Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other), - Ops, 2); + SDValue Ops[] = { DAG.getConstant(0, VT), Tmp2 }; + Result = DAG.getMergeValues(Ops, 2); break; } } } - if (Result.Val->getNumValues() == 1) break; + if (Result.getNode()->getNumValues() == 1) break; - assert(Result.Val->getNumValues() == 2 && + assert(Result.getNode()->getNumValues() == 2 && "Cannot return more than two values!"); // Since we produced two values, make sure to remember that we @@ -985,7 +1029,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp2 = LegalizeOp(Result.getValue(1)); AddLegalizedOperand(Op.getValue(0), Tmp1); AddLegalizedOperand(Op.getValue(1), Tmp2); - return Op.ResNo ? Tmp2 : Tmp1; + return Op.getResNo() ? Tmp2 : Tmp1; case ISD::EH_RETURN: { MVT VT = Node->getValueType(0); // The only "good" option for this node is to custom lower it. @@ -993,7 +1037,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { default: assert(0 && "This action is not supported at all!"); case TargetLowering::Custom: Result = TLI.LowerOperation(Op, DAG); - if (Result.Val) break; + if (Result.getNode()) break; // Fall Thru case TargetLowering::Legal: // Target does not know, how to lower this, lower to noop @@ -1009,7 +1053,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case ISD::MERGE_VALUES: // Legalize eliminates MERGE_VALUES nodes. - Result = Node->getOperand(Op.ResNo); + Result = Node->getOperand(Op.getResNo()); break; case ISD::CopyFromReg: Tmp1 = LegalizeOp(Node->getOperand(0)); @@ -1030,7 +1074,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // legalized both of them. AddLegalizedOperand(Op.getValue(0), Result); AddLegalizedOperand(Op.getValue(1), Result.getValue(1)); - return Result.getValue(Op.ResNo); + return Result.getValue(Op.getResNo()); case ISD::UNDEF: { MVT VT = Op.getValueType(); switch (TLI.getOperationAction(ISD::UNDEF, VT)) { @@ -1053,7 +1097,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case ISD::INTRINSIC_W_CHAIN: case ISD::INTRINSIC_WO_CHAIN: case ISD::INTRINSIC_VOID: { - SmallVector Ops; + SmallVector Ops; for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) Ops.push_back(LegalizeOp(Node->getOperand(i))); Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); @@ -1062,83 +1106,77 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { if (TLI.getOperationAction(Node->getOpcode(), MVT::Other) == TargetLowering::Custom) { Tmp3 = TLI.LowerOperation(Result, DAG); - if (Tmp3.Val) Result = Tmp3; + if (Tmp3.getNode()) Result = Tmp3; } - if (Result.Val->getNumValues() == 1) break; + if (Result.getNode()->getNumValues() == 1) break; // Must have return value and chain result. - assert(Result.Val->getNumValues() == 2 && + assert(Result.getNode()->getNumValues() == 2 && "Cannot return more than two values!"); // Since loads produce two values, make sure to remember that we // legalized both of them. - AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0)); - AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1)); - return Result.getValue(Op.ResNo); + AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); + AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); + return Result.getValue(Op.getResNo()); } - case ISD::LOCATION: - assert(Node->getNumOperands() == 5 && "Invalid LOCATION node!"); + case ISD::DBG_STOPPOINT: + assert(Node->getNumOperands() == 1 && "Invalid DBG_STOPPOINT node!"); Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the input chain. - switch (TLI.getOperationAction(ISD::LOCATION, MVT::Other)) { + switch (TLI.getOperationAction(ISD::DBG_STOPPOINT, MVT::Other)) { case TargetLowering::Promote: default: assert(0 && "This action is not supported yet!"); case TargetLowering::Expand: { MachineModuleInfo *MMI = DAG.getMachineModuleInfo(); bool useDEBUG_LOC = TLI.isOperationLegal(ISD::DEBUG_LOC, MVT::Other); - bool useLABEL = TLI.isOperationLegal(ISD::LABEL, MVT::Other); + bool useLABEL = TLI.isOperationLegal(ISD::DBG_LABEL, MVT::Other); + const DbgStopPointSDNode *DSP = cast(Node); if (MMI && (useDEBUG_LOC || useLABEL)) { - const std::string &FName = - cast(Node->getOperand(3))->getValue(); - const std::string &DirName = - cast(Node->getOperand(4))->getValue(); - unsigned SrcFile = MMI->RecordSource(DirName, FName); - - SmallVector Ops; - Ops.push_back(Tmp1); // chain - SDOperand LineOp = Node->getOperand(1); - SDOperand ColOp = Node->getOperand(2); + const CompileUnitDesc *CompileUnit = DSP->getCompileUnit(); + unsigned SrcFile = MMI->RecordSource(CompileUnit); + + unsigned Line = DSP->getLine(); + unsigned Col = DSP->getColumn(); if (useDEBUG_LOC) { - Ops.push_back(LineOp); // line # - Ops.push_back(ColOp); // col # - Ops.push_back(DAG.getConstant(SrcFile, MVT::i32)); // source file id - Result = DAG.getNode(ISD::DEBUG_LOC, MVT::Other, &Ops[0], Ops.size()); + SDValue Ops[] = { Tmp1, DAG.getConstant(Line, MVT::i32), + DAG.getConstant(Col, MVT::i32), + DAG.getConstant(SrcFile, MVT::i32) }; + Result = DAG.getNode(ISD::DEBUG_LOC, MVT::Other, Ops, 4); } else { - unsigned Line = cast(LineOp)->getValue(); - unsigned Col = cast(ColOp)->getValue(); unsigned ID = MMI->RecordSourceLine(Line, Col, SrcFile); - Ops.push_back(DAG.getConstant(ID, MVT::i32)); - Ops.push_back(DAG.getConstant(0, MVT::i32)); // a debug label - Result = DAG.getNode(ISD::LABEL, MVT::Other, &Ops[0], Ops.size()); + Result = DAG.getLabel(ISD::DBG_LABEL, Tmp1, ID); } } else { Result = Tmp1; // chain } break; } - case TargetLowering::Legal: - if (Tmp1 != Node->getOperand(0) || - getTypeAction(Node->getOperand(1).getValueType()) == Promote) { - SmallVector Ops; - Ops.push_back(Tmp1); - if (getTypeAction(Node->getOperand(1).getValueType()) == Legal) { - Ops.push_back(Node->getOperand(1)); // line # must be legal. - Ops.push_back(Node->getOperand(2)); // col # must be legal. - } else { - // Otherwise promote them. - Ops.push_back(PromoteOp(Node->getOperand(1))); - Ops.push_back(PromoteOp(Node->getOperand(2))); - } - Ops.push_back(Node->getOperand(3)); // filename must be legal. - Ops.push_back(Node->getOperand(4)); // working dir # must be legal. - Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); + case TargetLowering::Legal: { + LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType()); + if (Action == Legal && Tmp1 == Node->getOperand(0)) + break; + + SmallVector Ops; + Ops.push_back(Tmp1); + if (Action == Legal) { + Ops.push_back(Node->getOperand(1)); // line # must be legal. + Ops.push_back(Node->getOperand(2)); // col # must be legal. + } else { + // Otherwise promote them. + Ops.push_back(PromoteOp(Node->getOperand(1))); + Ops.push_back(PromoteOp(Node->getOperand(2))); } + Ops.push_back(Node->getOperand(3)); // filename must be legal. + Ops.push_back(Node->getOperand(4)); // working dir # must be legal. + Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); break; } + } break; case ISD::DECLARE: @@ -1161,25 +1199,34 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { assert(Node->getNumOperands() == 4 && "Invalid DEBUG_LOC node!"); switch (TLI.getOperationAction(ISD::DEBUG_LOC, MVT::Other)) { default: assert(0 && "This action is not supported yet!"); - case TargetLowering::Legal: + case TargetLowering::Legal: { + LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType()); Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. - Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the line #. - Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the col #. - Tmp4 = LegalizeOp(Node->getOperand(3)); // Legalize the source file id. + if (Action == Legal && Tmp1 == Node->getOperand(0)) + break; + if (Action == Legal) { + Tmp2 = Node->getOperand(1); + Tmp3 = Node->getOperand(2); + Tmp4 = Node->getOperand(3); + } else { + Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the line #. + Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the col #. + Tmp4 = LegalizeOp(Node->getOperand(3)); // Legalize the source file id. + } Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4); break; } + } break; - case ISD::LABEL: - assert(Node->getNumOperands() == 3 && "Invalid LABEL node!"); - switch (TLI.getOperationAction(ISD::LABEL, MVT::Other)) { + case ISD::DBG_LABEL: + case ISD::EH_LABEL: + assert(Node->getNumOperands() == 1 && "Invalid LABEL node!"); + switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) { default: assert(0 && "This action is not supported yet!"); case TargetLowering::Legal: Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. - Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the label id. - Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the "flavor" operand. - Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); + Result = DAG.UpdateNodeOperands(Result, Tmp1); break; case TargetLowering::Expand: Result = LegalizeOp(Node->getOperand(0)); @@ -1210,7 +1257,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { switch (TLI.getOperationAction(ISD::MEMBARRIER, MVT::Other)) { default: assert(0 && "This action is not supported yet!"); case TargetLowering::Legal: { - SDOperand Ops[6]; + SDValue Ops[6]; Ops[0] = LegalizeOp(Node->getOperand(0)); // Legalize the chain. for (int x = 1; x < 6; ++x) { Ops[x] = Node->getOperand(x); @@ -1228,10 +1275,13 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; } - case ISD::ATOMIC_LCS: { + case ISD::ATOMIC_CMP_SWAP_8: + case ISD::ATOMIC_CMP_SWAP_16: + case ISD::ATOMIC_CMP_SWAP_32: + case ISD::ATOMIC_CMP_SWAP_64: { unsigned int num_operands = 4; assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!"); - SDOperand Ops[4]; + SDValue Ops[4]; for (unsigned int x = 0; x < num_operands; ++x) Ops[x] = LegalizeOp(Node->getOperand(x)); Result = DAG.UpdateNodeOperands(Result, &Ops[0], num_operands); @@ -1244,43 +1294,73 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; } - AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0)); - AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1)); - return Result.getValue(Op.ResNo); - } - case ISD::ATOMIC_LAS: - case ISD::ATOMIC_LSS: - case ISD::ATOMIC_LOAD_AND: - case ISD::ATOMIC_LOAD_OR: - case ISD::ATOMIC_LOAD_XOR: - case ISD::ATOMIC_LOAD_NAND: - case ISD::ATOMIC_LOAD_MIN: - case ISD::ATOMIC_LOAD_MAX: - case ISD::ATOMIC_LOAD_UMIN: - case ISD::ATOMIC_LOAD_UMAX: - case ISD::ATOMIC_SWAP: { + AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); + AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); + return Result.getValue(Op.getResNo()); + } + case ISD::ATOMIC_LOAD_ADD_8: + case ISD::ATOMIC_LOAD_SUB_8: + case ISD::ATOMIC_LOAD_AND_8: + case ISD::ATOMIC_LOAD_OR_8: + case ISD::ATOMIC_LOAD_XOR_8: + case ISD::ATOMIC_LOAD_NAND_8: + case ISD::ATOMIC_LOAD_MIN_8: + case ISD::ATOMIC_LOAD_MAX_8: + case ISD::ATOMIC_LOAD_UMIN_8: + case ISD::ATOMIC_LOAD_UMAX_8: + case ISD::ATOMIC_SWAP_8: + case ISD::ATOMIC_LOAD_ADD_16: + case ISD::ATOMIC_LOAD_SUB_16: + case ISD::ATOMIC_LOAD_AND_16: + case ISD::ATOMIC_LOAD_OR_16: + case ISD::ATOMIC_LOAD_XOR_16: + case ISD::ATOMIC_LOAD_NAND_16: + case ISD::ATOMIC_LOAD_MIN_16: + case ISD::ATOMIC_LOAD_MAX_16: + case ISD::ATOMIC_LOAD_UMIN_16: + case ISD::ATOMIC_LOAD_UMAX_16: + case ISD::ATOMIC_SWAP_16: + case ISD::ATOMIC_LOAD_ADD_32: + case ISD::ATOMIC_LOAD_SUB_32: + case ISD::ATOMIC_LOAD_AND_32: + case ISD::ATOMIC_LOAD_OR_32: + case ISD::ATOMIC_LOAD_XOR_32: + case ISD::ATOMIC_LOAD_NAND_32: + case ISD::ATOMIC_LOAD_MIN_32: + case ISD::ATOMIC_LOAD_MAX_32: + case ISD::ATOMIC_LOAD_UMIN_32: + case ISD::ATOMIC_LOAD_UMAX_32: + case ISD::ATOMIC_SWAP_32: + case ISD::ATOMIC_LOAD_ADD_64: + case ISD::ATOMIC_LOAD_SUB_64: + case ISD::ATOMIC_LOAD_AND_64: + case ISD::ATOMIC_LOAD_OR_64: + case ISD::ATOMIC_LOAD_XOR_64: + case ISD::ATOMIC_LOAD_NAND_64: + case ISD::ATOMIC_LOAD_MIN_64: + case ISD::ATOMIC_LOAD_MAX_64: + case ISD::ATOMIC_LOAD_UMIN_64: + case ISD::ATOMIC_LOAD_UMAX_64: + case ISD::ATOMIC_SWAP_64: { unsigned int num_operands = 3; assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!"); - SDOperand Ops[3]; + SDValue Ops[3]; for (unsigned int x = 0; x < num_operands; ++x) Ops[x] = LegalizeOp(Node->getOperand(x)); Result = DAG.UpdateNodeOperands(Result, &Ops[0], num_operands); - + switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { default: assert(0 && "This action is not supported yet!"); case TargetLowering::Custom: Result = TLI.LowerOperation(Result, DAG); break; - case TargetLowering::Expand: - Result = SDOperand(TLI.ExpandOperationResult(Op.Val, DAG),0); - break; case TargetLowering::Legal: break; } - AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0)); - AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1)); - return Result.getValue(Op.ResNo); - } + AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); + AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); + return Result.getValue(Op.getResNo()); + } case ISD::Constant: { ConstantSDNode *CN = cast(Node); unsigned opAction = @@ -1291,7 +1371,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { if (opAction == TargetLowering::Custom) { Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) + if (Tmp1.getNode()) Result = Tmp1; } break; @@ -1309,7 +1389,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Custom: Tmp3 = TLI.LowerOperation(Result, DAG); - if (Tmp3.Val) { + if (Tmp3.getNode()) { Result = Tmp3; break; } @@ -1343,7 +1423,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp3 = LegalizeOp(Node->getOperand(2)); Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); } else { - SmallVector Ops; + SmallVector Ops; // Legalize the operands. for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) Ops.push_back(LegalizeOp(Node->getOperand(i))); @@ -1355,39 +1435,39 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case ISD::CALL: // The only option for this is to custom lower it. Tmp3 = TLI.LowerOperation(Result.getValue(0), DAG); - assert(Tmp3.Val && "Target didn't custom lower this node!"); + assert(Tmp3.getNode() && "Target didn't custom lower this node!"); // A call within a calling sequence must be legalized to something // other than the normal CALLSEQ_END. Violating this gets Legalize // into an infinite loop. assert ((!IsLegalizingCall || Node->getOpcode() != ISD::CALL || - Tmp3.Val->getOpcode() != ISD::CALLSEQ_END) && + Tmp3.getNode()->getOpcode() != ISD::CALLSEQ_END) && "Nested CALLSEQ_START..CALLSEQ_END not supported."); // The number of incoming and outgoing values should match; unless the final // outgoing value is a flag. - assert((Tmp3.Val->getNumValues() == Result.Val->getNumValues() || - (Tmp3.Val->getNumValues() == Result.Val->getNumValues() + 1 && - Tmp3.Val->getValueType(Tmp3.Val->getNumValues() - 1) == + assert((Tmp3.getNode()->getNumValues() == Result.getNode()->getNumValues() || + (Tmp3.getNode()->getNumValues() == Result.getNode()->getNumValues() + 1 && + Tmp3.getNode()->getValueType(Tmp3.getNode()->getNumValues() - 1) == MVT::Flag)) && "Lowering call/formal_arguments produced unexpected # results!"); // Since CALL/FORMAL_ARGUMENTS nodes produce multiple values, make sure to // remember that we legalized all of them, so it doesn't get relegalized. - for (unsigned i = 0, e = Tmp3.Val->getNumValues(); i != e; ++i) { - if (Tmp3.Val->getValueType(i) == MVT::Flag) + for (unsigned i = 0, e = Tmp3.getNode()->getNumValues(); i != e; ++i) { + if (Tmp3.getNode()->getValueType(i) == MVT::Flag) continue; Tmp1 = LegalizeOp(Tmp3.getValue(i)); - if (Op.ResNo == i) + if (Op.getResNo() == i) Tmp2 = Tmp1; - AddLegalizedOperand(SDOperand(Node, i), Tmp1); + AddLegalizedOperand(SDValue(Node, i), Tmp1); } return Tmp2; case ISD::EXTRACT_SUBREG: { Tmp1 = LegalizeOp(Node->getOperand(0)); ConstantSDNode *idx = dyn_cast(Node->getOperand(1)); assert(idx && "Operand must be a constant"); - Tmp2 = DAG.getTargetConstant(idx->getValue(), idx->getValueType(0)); + Tmp2 = DAG.getTargetConstant(idx->getAPIntValue(), idx->getValueType(0)); Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); } break; @@ -1396,7 +1476,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp2 = LegalizeOp(Node->getOperand(1)); ConstantSDNode *idx = dyn_cast(Node->getOperand(2)); assert(idx && "Operand must be a constant"); - Tmp3 = DAG.getTargetConstant(idx->getValue(), idx->getValueType(0)); + Tmp3 = DAG.getTargetConstant(idx->getAPIntValue(), idx->getValueType(0)); Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); } break; @@ -1405,13 +1485,13 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { default: assert(0 && "This action is not supported yet!"); case TargetLowering::Custom: Tmp3 = TLI.LowerOperation(Result, DAG); - if (Tmp3.Val) { + if (Tmp3.getNode()) { Result = Tmp3; break; } // FALLTHROUGH case TargetLowering::Expand: - Result = ExpandBUILD_VECTOR(Result.Val); + Result = ExpandBUILD_VECTOR(Result.getNode()); break; } break; @@ -1426,6 +1506,12 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { default: assert(0 && "Cannot expand insert element operand"); case Legal: Tmp2 = LegalizeOp(Node->getOperand(1)); break; case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break; + case Expand: + // FIXME: An alternative would be to check to see if the target is not + // going to custom lower this operation, we could bitcast to half elt + // width and perform two inserts at that width, if that is legal. + Tmp2 = Node->getOperand(1); + break; } Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); @@ -1436,11 +1522,13 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Custom: Tmp4 = TLI.LowerOperation(Result, DAG); - if (Tmp4.Val) { + if (Tmp4.getNode()) { Result = Tmp4; break; } // FALLTHROUGH + case TargetLowering::Promote: + // Fall thru for vector case case TargetLowering::Expand: { // If the insert index is a constant, codegen this as a scalar_to_vector, // then a shuffle that inserts it into the right position in the vector. @@ -1449,7 +1537,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // match the element type of the vector being created. if (Tmp2.getValueType() == Op.getValueType().getVectorElementType()) { - SDOperand ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, + SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, Tmp1.getValueType(), Tmp2); unsigned NumElts = Tmp1.getValueType().getVectorNumElements(); @@ -1460,14 +1548,14 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // We generate a shuffle of InVec and ScVec, so the shuffle mask // should be 0,1,2,3,4,5... with the appropriate element replaced with // elt 0 of the RHS. - SmallVector ShufOps; + SmallVector ShufOps; for (unsigned i = 0; i != NumElts; ++i) { - if (i != InsertPos->getValue()) + if (i != InsertPos->getZExtValue()) ShufOps.push_back(DAG.getConstant(i, ShufMaskEltVT)); else ShufOps.push_back(DAG.getConstant(NumElts, ShufMaskEltVT)); } - SDOperand ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMaskVT, + SDValue ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMaskVT, &ShufOps[0], ShufOps.size()); Result = DAG.getNode(ISD::VECTOR_SHUFFLE, Tmp1.getValueType(), @@ -1496,7 +1584,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Custom: Tmp3 = TLI.LowerOperation(Result, DAG); - if (Tmp3.Val) { + if (Tmp3.getNode()) { Result = Tmp3; break; } @@ -1520,7 +1608,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Custom: Tmp3 = TLI.LowerOperation(Result, DAG); - if (Tmp3.Val) { + if (Tmp3.getNode()) { Result = Tmp3; break; } @@ -1529,16 +1617,16 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { MVT VT = Node->getValueType(0); MVT EltVT = VT.getVectorElementType(); MVT PtrVT = TLI.getPointerTy(); - SDOperand Mask = Node->getOperand(2); + SDValue Mask = Node->getOperand(2); unsigned NumElems = Mask.getNumOperands(); - SmallVector Ops; + SmallVector Ops; for (unsigned i = 0; i != NumElems; ++i) { - SDOperand Arg = Mask.getOperand(i); + SDValue Arg = Mask.getOperand(i); if (Arg.getOpcode() == ISD::UNDEF) { Ops.push_back(DAG.getNode(ISD::UNDEF, EltVT)); } else { assert(isa(Arg) && "Invalid VECTOR_SHUFFLE mask!"); - unsigned Idx = cast(Arg)->getValue(); + unsigned Idx = cast(Arg)->getZExtValue(); if (Idx < NumElems) Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp1, DAG.getConstant(Idx, PtrVT))); @@ -1560,8 +1648,8 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp2); // Convert the shuffle mask to the right # elements. - Tmp3 = SDOperand(isShuffleLegal(OVT, Node->getOperand(2)), 0); - assert(Tmp3.Val && "Shuffle not legal?"); + Tmp3 = SDValue(isShuffleLegal(OVT, Node->getOperand(2)), 0); + assert(Tmp3.getNode() && "Shuffle not legal?"); Result = DAG.getNode(ISD::VECTOR_SHUFFLE, NVT, Tmp1, Tmp2, Tmp3); Result = DAG.getNode(ISD::BIT_CONVERT, OVT, Result); break; @@ -1583,6 +1671,26 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = ExpandEXTRACT_SUBVECTOR(Result); break; + case ISD::CONCAT_VECTORS: { + // Use extract/insert/build vector for now. We might try to be + // more clever later. + MVT PtrVT = TLI.getPointerTy(); + SmallVector Ops; + unsigned NumOperands = Node->getNumOperands(); + for (unsigned i=0; i < NumOperands; ++i) { + SDValue SubOp = Node->getOperand(i); + MVT VVT = SubOp.getNode()->getValueType(0); + MVT EltVT = VVT.getVectorElementType(); + unsigned NumSubElem = VVT.getVectorNumElements(); + for (unsigned j=0; j < NumSubElem; ++j) { + Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, SubOp, + DAG.getConstant(j, PtrVT))); + } + } + return LegalizeOp(DAG.getNode(ISD::BUILD_VECTOR, Node->getValueType(0), + &Ops[0], Ops.size())); + } + case ISD::CALLSEQ_START: { SDNode *CallEnd = FindCallEndFromCallStart(Node); @@ -1591,7 +1699,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // are inserted *before* the CALLSEQ_START. {SmallPtrSet NodesLeadingTo; for (unsigned i = 0, e = CallEnd->getNumOperands(); i != e; ++i) - LegalizeAllNodesNotLeadingTo(CallEnd->getOperand(i).Val, Node, + LegalizeAllNodesNotLeadingTo(CallEnd->getOperand(i).getNode(), Node, NodesLeadingTo); } @@ -1608,7 +1716,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // Do not try to legalize the target-specific arguments (#1+). if (Tmp1 != Node->getOperand(0)) { - SmallVector Ops(Node->op_begin(), Node->op_end()); + SmallVector Ops(Node->op_begin(), Node->op_end()); Ops[0] = Tmp1; Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); } @@ -1624,7 +1732,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // can overlap. assert(!IsLegalizingCall && "Inconsistent sequentialization of calls!"); // Note that we are selecting this call! - LastCALLSEQ_END = SDOperand(CallEnd, 0); + LastCALLSEQ_END = SDValue(CallEnd, 0); IsLegalizingCall = true; // Legalize the call, starting from the CALLSEQ_END. @@ -1635,9 +1743,9 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case ISD::CALLSEQ_END: // If the CALLSEQ_START node hasn't been legalized first, legalize it. This // will cause this node to be legalized as well as handling libcalls right. - if (LastCALLSEQ_END.Val != Node) { - LegalizeOp(SDOperand(FindCallStartFromCallEnd(Node), 0)); - DenseMap::iterator I = LegalizedNodes.find(Op); + if (LastCALLSEQ_END.getNode() != Node) { + LegalizeOp(SDValue(FindCallStartFromCallEnd(Node), 0)); + DenseMap::iterator I = LegalizedNodes.find(Op); assert(I != LegalizedNodes.end() && "Legalizing the call start should have legalized this node!"); return I->second; @@ -1650,7 +1758,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // an optional flag input. if (Node->getOperand(Node->getNumOperands()-1).getValueType() != MVT::Flag){ if (Tmp1 != Node->getOperand(0)) { - SmallVector Ops(Node->op_begin(), Node->op_end()); + SmallVector Ops(Node->op_begin(), Node->op_end()); Ops[0] = Tmp1; Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); } @@ -1658,7 +1766,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp2 = LegalizeOp(Node->getOperand(Node->getNumOperands()-1)); if (Tmp1 != Node->getOperand(0) || Tmp2 != Node->getOperand(Node->getNumOperands()-1)) { - SmallVector Ops(Node->op_begin(), Node->op_end()); + SmallVector Ops(Node->op_begin(), Node->op_end()); Ops[0] = Tmp1; Ops.back() = Tmp2; Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); @@ -1669,10 +1777,10 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { IsLegalizingCall = false; // If the CALLSEQ_END node has a flag, remember that we legalized it. - AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0)); + AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); if (Node->getNumValues() == 2) - AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1)); - return Result.getValue(Op.ResNo); + AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); + return Result.getValue(Op.getResNo()); case ISD::DYNAMIC_STACKALLOC: { MVT VT = Node->getValueType(0); Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. @@ -1688,17 +1796,16 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" " not tell us which reg is the stack pointer!"); - SDOperand Chain = Tmp1.getOperand(0); + SDValue Chain = Tmp1.getOperand(0); // Chain the dynamic stack allocation so that it doesn't modify the stack // pointer when other instructions are using the stack. - Chain = DAG.getCALLSEQ_START(Chain, - DAG.getConstant(0, TLI.getPointerTy())); + Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true)); - SDOperand Size = Tmp2.getOperand(1); - SDOperand SP = DAG.getCopyFromReg(Chain, SPReg, VT); + SDValue Size = Tmp2.getOperand(1); + SDValue SP = DAG.getCopyFromReg(Chain, SPReg, VT); Chain = SP.getValue(1); - unsigned Align = cast(Tmp3)->getValue(); + unsigned Align = cast(Tmp3)->getZExtValue(); unsigned StackAlign = TLI.getTargetMachine().getFrameInfo()->getStackAlignment(); if (Align > StackAlign) @@ -1707,11 +1814,8 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp1 = DAG.getNode(ISD::SUB, VT, SP, Size); // Value Chain = DAG.getCopyToReg(Chain, SPReg, Tmp1); // Output chain - Tmp2 = - DAG.getCALLSEQ_END(Chain, - DAG.getConstant(0, TLI.getPointerTy()), - DAG.getConstant(0, TLI.getPointerTy()), - SDOperand()); + Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), + DAG.getIntPtrConstant(0, true), SDValue()); Tmp1 = LegalizeOp(Tmp1); Tmp2 = LegalizeOp(Tmp2); @@ -1719,7 +1823,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } case TargetLowering::Custom: Tmp3 = TLI.LowerOperation(Tmp1, DAG); - if (Tmp3.Val) { + if (Tmp3.getNode()) { Tmp1 = LegalizeOp(Tmp3); Tmp2 = LegalizeOp(Tmp3.getValue(1)); } @@ -1729,25 +1833,25 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } // Since this op produce two values, make sure to remember that we // legalized both of them. - AddLegalizedOperand(SDOperand(Node, 0), Tmp1); - AddLegalizedOperand(SDOperand(Node, 1), Tmp2); - return Op.ResNo ? Tmp2 : Tmp1; + AddLegalizedOperand(SDValue(Node, 0), Tmp1); + AddLegalizedOperand(SDValue(Node, 1), Tmp2); + return Op.getResNo() ? Tmp2 : Tmp1; } case ISD::INLINEASM: { - SmallVector Ops(Node->op_begin(), Node->op_end()); + SmallVector Ops(Node->op_begin(), Node->op_end()); bool Changed = false; // Legalize all of the operands of the inline asm, in case they are nodes // that need to be expanded or something. Note we skip the asm string and // all of the TargetConstant flags. - SDOperand Op = LegalizeOp(Ops[0]); + SDValue Op = LegalizeOp(Ops[0]); Changed = Op != Ops[0]; Ops[0] = Op; bool HasInFlag = Ops.back().getValueType() == MVT::Flag; for (unsigned i = 2, e = Ops.size()-HasInFlag; i < e; ) { - unsigned NumVals = cast(Ops[i])->getValue() >> 3; + unsigned NumVals = cast(Ops[i])->getZExtValue() >> 3; for (++i; NumVals; ++i, --NumVals) { - SDOperand Op = LegalizeOp(Ops[i]); + SDValue Op = LegalizeOp(Ops[i]); if (Op != Ops[i]) { Changed = true; Ops[i] = Op; @@ -1765,9 +1869,9 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); // INLINE asm returns a chain and flag, make sure to add both to the map. - AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0)); - AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1)); - return Result.getValue(Op.ResNo); + AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); + AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); + return Result.getValue(Op.getResNo()); } case ISD::BR: Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. @@ -1808,20 +1912,20 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; case TargetLowering::Expand: { - SDOperand Chain = Result.getOperand(0); - SDOperand Table = Result.getOperand(1); - SDOperand Index = Result.getOperand(2); + SDValue Chain = Result.getOperand(0); + SDValue Table = Result.getOperand(1); + SDValue Index = Result.getOperand(2); MVT PTy = TLI.getPointerTy(); MachineFunction &MF = DAG.getMachineFunction(); unsigned EntrySize = MF.getJumpTableInfo()->getEntrySize(); Index= DAG.getNode(ISD::MUL, PTy, Index, DAG.getConstant(EntrySize, PTy)); - SDOperand Addr = DAG.getNode(ISD::ADD, PTy, Index, Table); + SDValue Addr = DAG.getNode(ISD::ADD, PTy, Index, Table); - SDOperand LD; + SDValue LD; switch (EntrySize) { default: assert(0 && "Size of jump table not supported yet."); break; case 4: LD = DAG.getLoad(MVT::i32, Chain, Addr, @@ -1877,7 +1981,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; case TargetLowering::Expand: // Expand brcond's setcc into its constituent parts and create a BR_CC @@ -1904,13 +2008,13 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp3 = Node->getOperand(3); // RHS Tmp4 = Node->getOperand(1); // CC - LegalizeSetCCOperands(Tmp2, Tmp3, Tmp4); + LegalizeSetCC(TLI.getSetCCResultType(Tmp2), Tmp2, Tmp3, Tmp4); LastCALLSEQ_END = DAG.getEntryNode(); - // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands, + // If we didn't get both a LHS and RHS back from LegalizeSetCC, // the LHS is a legal SETCC itself. In this case, we need to compare // the result against zero to select between true and false values. - if (Tmp3.Val == 0) { + if (Tmp3.getNode() == 0) { Tmp3 = DAG.getConstant(0, Tmp2.getValueType()); Tmp4 = DAG.getCondCode(ISD::SETNE); } @@ -1923,7 +2027,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp4 = TLI.LowerOperation(Result, DAG); - if (Tmp4.Val) Result = Tmp4; + if (Tmp4.getNode()) Result = Tmp4; break; } break; @@ -1948,7 +2052,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { unsigned ABIAlignment = TLI.getTargetData()-> getABITypeAlignment(LD->getMemoryVT().getTypeForMVT()); if (LD->getAlignment() < ABIAlignment){ - Result = ExpandUnalignedLoad(cast(Result.Val), DAG, + Result = ExpandUnalignedLoad(cast(Result.getNode()), DAG, TLI); Tmp3 = Result.getOperand(0); Tmp4 = Result.getOperand(1); @@ -1959,7 +2063,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Tmp3, DAG); - if (Tmp1.Val) { + if (Tmp1.getNode()) { Tmp3 = LegalizeOp(Tmp1); Tmp4 = LegalizeOp(Tmp1.getValue(1)); } @@ -1980,9 +2084,9 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } // Since loads produce two values, make sure to remember that we // legalized both of them. - AddLegalizedOperand(SDOperand(Node, 0), Tmp3); - AddLegalizedOperand(SDOperand(Node, 1), Tmp4); - return Op.ResNo ? Tmp4 : Tmp3; + AddLegalizedOperand(SDValue(Node, 0), Tmp3); + AddLegalizedOperand(SDValue(Node, 1), Tmp4); + return Op.getResNo() ? Tmp4 : Tmp3; } else { MVT SrcVT = LD->getMemoryVT(); unsigned SrcWidth = SrcVT.getSizeInBits(); @@ -1999,12 +2103,12 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // nice to have an effective generic way of getting these benefits... // Until such a way is found, don't insist on promoting i1 here. (SrcVT != MVT::i1 || - TLI.getLoadXAction(ExtType, MVT::i1) == TargetLowering::Promote)) { + TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) { // Promote to a byte-sized load if not loading an integral number of // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. unsigned NewWidth = SrcVT.getStoreSizeInBits(); MVT NVT = MVT::getIntegerVT(NewWidth); - SDOperand Ch; + SDValue Ch; // The extra bits are guaranteed to be zero, since we stored them that // way. A zext load from NVT thus automatically gives zext from SrcVT. @@ -2041,7 +2145,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { "Load size not an integral number of bytes!"); MVT RoundVT = MVT::getIntegerVT(RoundWidth); MVT ExtraVT = MVT::getIntegerVT(ExtraWidth); - SDOperand Lo, Hi, Ch; + SDValue Lo, Hi, Ch; unsigned IncrementSize; if (TLI.isLittleEndian()) { @@ -2104,7 +2208,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp1 = LegalizeOp(Result); Tmp2 = LegalizeOp(Ch); } else { - switch (TLI.getLoadXAction(ExtType, SrcVT)) { + switch (TLI.getLoadExtAction(ExtType, SrcVT)) { default: assert(0 && "This action is not supported yet!"); case TargetLowering::Custom: isCustom = true; @@ -2116,7 +2220,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { if (isCustom) { Tmp3 = TLI.LowerOperation(Result, DAG); - if (Tmp3.Val) { + if (Tmp3.getNode()) { Tmp1 = LegalizeOp(Tmp3); Tmp2 = LegalizeOp(Tmp3.getValue(1)); } @@ -2127,7 +2231,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { unsigned ABIAlignment = TLI.getTargetData()-> getABITypeAlignment(LD->getMemoryVT().getTypeForMVT()); if (LD->getAlignment() < ABIAlignment){ - Result = ExpandUnalignedLoad(cast(Result.Val), DAG, + Result = ExpandUnalignedLoad(cast(Result.getNode()), DAG, TLI); Tmp1 = Result.getOperand(0); Tmp2 = Result.getOperand(1); @@ -2140,7 +2244,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Expand: // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND if (SrcVT == MVT::f32 && Node->getValueType(0) == MVT::f64) { - SDOperand Load = DAG.getLoad(SrcVT, Tmp1, Tmp2, LD->getSrcValue(), + SDValue Load = DAG.getLoad(SrcVT, Tmp1, Tmp2, LD->getSrcValue(), LD->getSrcValueOffset(), LD->isVolatile(), LD->getAlignment()); Result = DAG.getNode(ISD::FP_EXTEND, Node->getValueType(0), Load); @@ -2155,7 +2259,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp1, Tmp2, LD->getSrcValue(), LD->getSrcValueOffset(), SrcVT, LD->isVolatile(), LD->getAlignment()); - SDOperand ValRes; + SDValue ValRes; if (ExtType == ISD::SEXTLOAD) ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(), Result, DAG.getValueType(SrcVT)); @@ -2169,9 +2273,9 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // Since loads produce two values, make sure to remember that we legalized // both of them. - AddLegalizedOperand(SDOperand(Node, 0), Tmp1); - AddLegalizedOperand(SDOperand(Node, 1), Tmp2); - return Op.ResNo ? Tmp2 : Tmp1; + AddLegalizedOperand(SDValue(Node, 0), Tmp1); + AddLegalizedOperand(SDValue(Node, 1), Tmp2); + return Op.getResNo() ? Tmp2 : Tmp1; } } case ISD::EXTRACT_ELEMENT: { @@ -2179,7 +2283,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { switch (getTypeAction(OpTy)) { default: assert(0 && "EXTRACT_ELEMENT action for type unimplemented!"); case Legal: - if (cast(Node->getOperand(1))->getValue()) { + if (cast(Node->getOperand(1))->getZExtValue()) { // 1 -> Hi Result = DAG.getNode(ISD::SRL, OpTy, Node->getOperand(0), DAG.getConstant(OpTy.getSizeInBits()/2, @@ -2194,7 +2298,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case Expand: // Get both the low and high parts. ExpandOp(Node->getOperand(0), Tmp1, Tmp2); - if (cast(Node->getOperand(1))->getValue()) + if (cast(Node->getOperand(1))->getZExtValue()) Result = Tmp2; // 1 -> Hi else Result = Tmp1; // 0 -> Lo @@ -2224,8 +2328,8 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // Since this produces two values, make sure to remember that we legalized // both of them. - AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0)); - AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1)); + AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); + AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); return Result; } break; @@ -2248,21 +2352,21 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case Expand: if (!Tmp2.getValueType().isVector()) { - SDOperand Lo, Hi; + SDValue Lo, Hi; ExpandOp(Tmp2, Lo, Hi); // Big endian systems want the hi reg first. if (TLI.isBigEndian()) std::swap(Lo, Hi); - if (Hi.Val) + if (Hi.getNode()) Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3); else Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3); Result = LegalizeOp(Result); } else { - SDNode *InVal = Tmp2.Val; - int InIx = Tmp2.ResNo; + SDNode *InVal = Tmp2.getNode(); + int InIx = Tmp2.getResNo(); unsigned NumElems = InVal->getValueType(InIx).getVectorNumElements(); MVT EVT = InVal->getValueType(InIx).getVectorElementType(); @@ -2288,7 +2392,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } else { // FIXME: Returns of gcc generic vectors larger than a legal vector // type should be returned by reference! - SDOperand Lo, Hi; + SDValue Lo, Hi; SplitVectorOp(Tmp2, Lo, Hi); Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3); Result = LegalizeOp(Result); @@ -2306,7 +2410,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = DAG.UpdateNodeOperands(Result, Tmp1); break; default: { // ret - SmallVector NewValues; + SmallVector NewValues; NewValues.push_back(Tmp1); for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) switch (getTypeAction(Node->getOperand(i).getValueType())) { @@ -2315,13 +2419,13 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { NewValues.push_back(Node->getOperand(i+1)); break; case Expand: { - SDOperand Lo, Hi; + SDValue Lo, Hi; assert(!Node->getOperand(i).getValueType().isExtended() && "FIXME: TODO: implement returning non-legal vector types!"); ExpandOp(Node->getOperand(i), Lo, Hi); NewValues.push_back(Lo); NewValues.push_back(Node->getOperand(i+1)); - if (Hi.Val) { + if (Hi.getNode()) { NewValues.push_back(Hi); NewValues.push_back(Node->getOperand(i+1)); } @@ -2346,7 +2450,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; } } @@ -2371,7 +2475,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { if (CFP->getValueType(0) == MVT::f32 && getTypeAction(MVT::i32) == Legal) { Tmp3 = DAG.getConstant(CFP->getValueAPF(). - convertToAPInt().zextOrTrunc(32), + bitcastToAPInt().zextOrTrunc(32), MVT::i32); Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), SVOffset, isVolatile, Alignment); @@ -2379,7 +2483,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } else if (CFP->getValueType(0) == MVT::f64) { // If this target supports 64-bit registers, do a single 64-bit store. if (getTypeAction(MVT::i64) == Legal) { - Tmp3 = DAG.getConstant(CFP->getValueAPF().convertToAPInt(). + Tmp3 = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). zextOrTrunc(64), MVT::i64); Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), SVOffset, isVolatile, Alignment); @@ -2388,9 +2492,9 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // Otherwise, if the target supports 32-bit registers, use 2 32-bit // stores. If the target supports neither 32- nor 64-bits, this // xform is certainly not worth it. - const APInt &IntVal =CFP->getValueAPF().convertToAPInt(); - SDOperand Lo = DAG.getConstant(APInt(IntVal).trunc(32), MVT::i32); - SDOperand Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); + const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); + SDValue Lo = DAG.getConstant(APInt(IntVal).trunc(32), MVT::i32); + SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); if (TLI.isBigEndian()) std::swap(Lo, Hi); Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(), @@ -2422,13 +2526,13 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { unsigned ABIAlignment = TLI.getTargetData()-> getABITypeAlignment(ST->getMemoryVT().getTypeForMVT()); if (ST->getAlignment() < ABIAlignment) - Result = ExpandUnalignedStore(cast(Result.Val), DAG, + Result = ExpandUnalignedStore(cast(Result.getNode()), DAG, TLI); } break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; case TargetLowering::Promote: assert(VT.isVector() && "Unknown legal promote case!"); @@ -2442,23 +2546,25 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; } case Promote: - // Truncate the value and store the result. - Tmp3 = PromoteOp(ST->getValue()); - Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), - SVOffset, ST->getMemoryVT(), - isVolatile, Alignment); - break; - - case Expand: + if (!ST->getMemoryVT().isVector()) { + // Truncate the value and store the result. + Tmp3 = PromoteOp(ST->getValue()); + Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), + SVOffset, ST->getMemoryVT(), + isVolatile, Alignment); + break; + } + // Fall thru to expand for vector + case Expand: { unsigned IncrementSize = 0; - SDOperand Lo, Hi; + SDValue Lo, Hi; // If this is a vector type, then we have to calculate the increment as // the product of the element size in bytes, and the number of elements // in the high half of the vector. if (ST->getValue().getValueType().isVector()) { - SDNode *InVal = ST->getValue().Val; - int InIx = ST->getValue().ResNo; + SDNode *InVal = ST->getValue().getNode(); + int InIx = ST->getValue().getResNo(); MVT InVT = InVal->getValueType(InIx); unsigned NumElems = InVT.getVectorNumElements(); MVT EVT = InVT.getVectorElementType(); @@ -2483,22 +2589,31 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = LegalizeOp(Result); break; } else { - SplitVectorOp(ST->getValue(), Lo, Hi); - IncrementSize = Lo.Val->getValueType(0).getVectorNumElements() * - EVT.getSizeInBits()/8; + // Check if we have widen this node with another value + std::map::iterator I = + WidenNodes.find(ST->getValue()); + if (I != WidenNodes.end()) { + Result = StoreWidenVectorOp(ST, Tmp1, Tmp2); + break; + } + else { + SplitVectorOp(ST->getValue(), Lo, Hi); + IncrementSize = Lo.getNode()->getValueType(0).getVectorNumElements() * + EVT.getSizeInBits()/8; + } } } else { ExpandOp(ST->getValue(), Lo, Hi); - IncrementSize = Hi.Val ? Hi.getValueType().getSizeInBits()/8 : 0; + IncrementSize = Hi.getNode() ? Hi.getValueType().getSizeInBits()/8 : 0; - if (TLI.isBigEndian()) + if (Hi.getNode() && TLI.isBigEndian()) std::swap(Lo, Hi); } Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(), SVOffset, isVolatile, Alignment); - if (Hi.Val == NULL) { + if (Hi.getNode() == NULL) { // Must be int <-> float one-to-one expansion. Result = Lo; break; @@ -2514,6 +2629,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { SVOffset, isVolatile, Alignment); Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); break; + } // case Expand } } else { switch (getTypeAction(ST->getValue().getValueType())) { @@ -2521,9 +2637,12 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp3 = LegalizeOp(ST->getValue()); break; case Promote: - // We can promote the value, the truncstore will still take care of it. - Tmp3 = PromoteOp(ST->getValue()); - break; + if (!ST->getValue().getValueType().isVector()) { + // We can promote the value, the truncstore will still take care of it. + Tmp3 = PromoteOp(ST->getValue()); + break; + } + // Vector case falls through to expand case Expand: // Just store the low part. This may become a non-trunc store, so make // sure to use getTruncStore, not UpdateNodeOperands below. @@ -2555,7 +2674,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { "Store size not an integral number of bytes!"); MVT RoundVT = MVT::getIntegerVT(RoundWidth); MVT ExtraVT = MVT::getIntegerVT(ExtraWidth); - SDOperand Lo, Hi; + SDValue Lo, Hi; unsigned IncrementSize; if (TLI.isLittleEndian()) { @@ -2609,7 +2728,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { unsigned ABIAlignment = TLI.getTargetData()-> getABITypeAlignment(ST->getMemoryVT().getTypeForMVT()); if (ST->getAlignment() < ABIAlignment) - Result = ExpandUnalignedStore(cast(Result.Val), DAG, + Result = ExpandUnalignedStore(cast(Result.getNode()), DAG, TLI); } break; @@ -2643,7 +2762,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp3 = TLI.LowerOperation(Result, DAG); - if (Tmp3.Val) { + if (Tmp3.getNode()) { Tmp1 = LegalizeOp(Tmp3); Tmp2 = LegalizeOp(Tmp3.getValue(1)); } @@ -2664,9 +2783,9 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // Since stacksave produce two values, make sure to remember that we // legalized both of them. - AddLegalizedOperand(SDOperand(Node, 0), Tmp1); - AddLegalizedOperand(SDOperand(Node, 1), Tmp2); - return Op.ResNo ? Tmp2 : Tmp1; + AddLegalizedOperand(SDValue(Node, 0), Tmp1); + AddLegalizedOperand(SDValue(Node, 1), Tmp2); + return Op.getResNo() ? Tmp2 : Tmp1; case ISD::STACKRESTORE: Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain. @@ -2678,7 +2797,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; case TargetLowering::Expand: // Expand to CopyToReg if the target set @@ -2711,8 +2830,8 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // Since rdcc produce two values, make sure to remember that we legalized // both of them. - AddLegalizedOperand(SDOperand(Node, 0), Tmp1); - AddLegalizedOperand(SDOperand(Node, 1), Tmp2); + AddLegalizedOperand(SDValue(Node, 0), Tmp1); + AddLegalizedOperand(SDValue(Node, 1), Tmp2); return Result; case ISD::SELECT: @@ -2722,6 +2841,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the condition. break; case Promote: { + assert(!Node->getOperand(0).getValueType().isVector() && "not possible"); Tmp1 = PromoteOp(Node->getOperand(0)); // Promote the condition. // Make sure the condition is either zero or one. unsigned BitWidth = Tmp1.getValueSizeInBits(); @@ -2741,7 +2861,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: { Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; } case TargetLowering::Expand: @@ -2788,14 +2908,14 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp2 = Node->getOperand(1); // RHS Tmp3 = LegalizeOp(Node->getOperand(2)); // True Tmp4 = LegalizeOp(Node->getOperand(3)); // False - SDOperand CC = Node->getOperand(4); + SDValue CC = Node->getOperand(4); - LegalizeSetCCOperands(Tmp1, Tmp2, CC); + LegalizeSetCC(TLI.getSetCCResultType(Tmp1), Tmp1, Tmp2, CC); - // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands, + // If we didn't get both a LHS and RHS back from LegalizeSetCC, // the LHS is a legal SETCC itself. In this case, we need to compare // the result against zero to select between true and false values. - if (Tmp2.Val == 0) { + if (Tmp2.getNode() == 0) { Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); CC = DAG.getCondCode(ISD::SETNE); } @@ -2807,7 +2927,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; } break; @@ -2816,12 +2936,12 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp1 = Node->getOperand(0); Tmp2 = Node->getOperand(1); Tmp3 = Node->getOperand(2); - LegalizeSetCCOperands(Tmp1, Tmp2, Tmp3); + LegalizeSetCC(Node->getValueType(0), Tmp1, Tmp2, Tmp3); // If we had to Expand the SetCC operands into a SELECT node, then it may // not always be possible to return a true LHS & RHS. In this case, just // return the value we legalized, returned in the LHS - if (Tmp2.Val == 0) { + if (Tmp2.getNode() == 0) { Result = Tmp1; break; } @@ -2835,7 +2955,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); if (isCustom) { Tmp4 = TLI.LowerOperation(Result, DAG); - if (Tmp4.Val) Result = Tmp4; + if (Tmp4.getNode()) Result = Tmp4; } break; case TargetLowering::Promote: { @@ -2883,7 +3003,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case ISD::VSETCC: { Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS - SDOperand CC = Node->getOperand(2); + SDValue CC = Node->getOperand(2); Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, CC); @@ -2893,7 +3013,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; } break; @@ -2902,7 +3022,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case ISD::SHL_PARTS: case ISD::SRA_PARTS: case ISD::SRL_PARTS: { - SmallVector Ops; + SmallVector Ops; bool Changed = false; for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { Ops.push_back(LegalizeOp(Node->getOperand(i))); @@ -2917,15 +3037,15 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) { - SDOperand Tmp2, RetVal(0, 0); + if (Tmp1.getNode()) { + SDValue Tmp2, RetVal(0, 0); for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { Tmp2 = LegalizeOp(Tmp1.getValue(i)); - AddLegalizedOperand(SDOperand(Node, i), Tmp2); - if (i == Op.ResNo) + AddLegalizedOperand(SDValue(Node, i), Tmp2); + if (i == Op.getResNo()) RetVal = Tmp2; } - assert(RetVal.Val && "Illegal result number"); + assert(RetVal.getNode() && "Illegal result number"); return RetVal; } break; @@ -2934,8 +3054,8 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // Since these produce multiple values, make sure to remember that we // legalized all of them. for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) - AddLegalizedOperand(SDOperand(Node, i), Result.getValue(i)); - return Result.getValue(Op.ResNo); + AddLegalizedOperand(SDValue(Node, i), Result.getValue(i)); + return Result.getValue(Op.getResNo()); } // Binary operators @@ -2967,19 +3087,32 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Tmp2 = PromoteOp(Node->getOperand(1)); // Promote the RHS. break; } - + + if ((Node->getOpcode() == ISD::SHL || + Node->getOpcode() == ISD::SRL || + Node->getOpcode() == ISD::SRA) && + !Node->getValueType(0).isVector()) { + if (TLI.getShiftAmountTy().bitsLT(Tmp2.getValueType())) + Tmp2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Tmp2); + else if (TLI.getShiftAmountTy().bitsGT(Tmp2.getValueType())) + Tmp2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Tmp2); + } + Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); - + switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { default: assert(0 && "BinOp legalize operation not supported"); case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; - break; + if (Tmp1.getNode()) { + Result = Tmp1; + break; + } + // Fall through if the custom lower can't deal with the operation case TargetLowering::Expand: { MVT VT = Op.getValueType(); - + // See if multiply or divide can be lowered using two-result operations. SDVTList VTs = DAG.getVTList(VT, VT); if (Node->getOpcode() == ISD::MUL) { @@ -3002,31 +3135,35 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { OpToUse = ISD::UMUL_LOHI; } if (OpToUse) { - Result = SDOperand(DAG.getNode(OpToUse, VTs, Tmp1, Tmp2).Val, 0); + Result = SDValue(DAG.getNode(OpToUse, VTs, Tmp1, Tmp2).getNode(), 0); break; } } if (Node->getOpcode() == ISD::MULHS && TLI.isOperationLegal(ISD::SMUL_LOHI, VT)) { - Result = SDOperand(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).Val, 1); + Result = SDValue(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).getNode(), + 1); break; } if (Node->getOpcode() == ISD::MULHU && TLI.isOperationLegal(ISD::UMUL_LOHI, VT)) { - Result = SDOperand(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).Val, 1); + Result = SDValue(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).getNode(), + 1); break; } if (Node->getOpcode() == ISD::SDIV && TLI.isOperationLegal(ISD::SDIVREM, VT)) { - Result = SDOperand(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).Val, 0); + Result = SDValue(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).getNode(), + 0); break; } if (Node->getOpcode() == ISD::UDIV && TLI.isOperationLegal(ISD::UDIVREM, VT)) { - Result = SDOperand(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).Val, 0); + Result = SDValue(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).getNode(), + 0); break; } - + // Check to see if we have a libcall for this operator. RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; bool isSigned = false; @@ -3035,10 +3172,14 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case ISD::SDIV: if (VT == MVT::i32) { LC = Node->getOpcode() == ISD::UDIV - ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32; + ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32; isSigned = Node->getOpcode() == ISD::SDIV; } break; + case ISD::MUL: + if (VT == MVT::i32) + LC = RTLIB::MUL_I32; + break; case ISD::FPOW: LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, RTLIB::POW_PPCF128); @@ -3046,11 +3187,11 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { default: break; } if (LC != RTLIB::UNKNOWN_LIBCALL) { - SDOperand Dummy; + SDValue Dummy; Result = ExpandLibCall(LC, Node, isSigned, Dummy); break; } - + assert(Node->getValueType(0).isVector() && "Cannot expand this binary operator!"); // Expand the operation into a bunch of nasty scalar code. @@ -3111,7 +3252,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { default: assert(0 && "Operation not supported"); case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; case TargetLowering::Legal: break; case TargetLowering::Expand: { @@ -3125,11 +3266,11 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // Get the sign bit of the RHS. MVT IVT = Tmp2.getValueType() == MVT::f32 ? MVT::i32 : MVT::i64; - SDOperand SignBit = DAG.getNode(ISD::BIT_CONVERT, IVT, Tmp2); + SDValue SignBit = DAG.getNode(ISD::BIT_CONVERT, IVT, Tmp2); SignBit = DAG.getSetCC(TLI.getSetCCResultType(SignBit), SignBit, DAG.getConstant(0, IVT), ISD::SETLT); // Get the absolute value of the result. - SDOperand AbsVal = DAG.getNode(ISD::FABS, Tmp1.getValueType(), Tmp1); + SDValue AbsVal = DAG.getNode(ISD::FABS, Tmp1.getValueType(), Tmp1); // Select between the nabs and abs value based on the sign bit of // the input. Result = DAG.getNode(ISD::SELECT, AbsVal.getValueType(), SignBit, @@ -3158,8 +3299,8 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); // Since this produces two values, make sure to remember that we legalized // both of them. - AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0)); - AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1)); + AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); + AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); return Result; case ISD::ADDE: @@ -3170,8 +3311,8 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); // Since this produces two values, make sure to remember that we legalized // both of them. - AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0)); - AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1)); + AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0)); + AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1)); return Result; case ISD::BUILD_PAIR: { @@ -3214,7 +3355,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); if (isCustom) { Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; } break; case TargetLowering::Expand: { @@ -3226,12 +3367,12 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { SDVTList VTs = DAG.getVTList(VT, VT); if (Node->getOpcode() == ISD::SREM && TLI.isOperationLegal(ISD::SDIVREM, VT)) { - Result = SDOperand(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).Val, 1); + Result = SDValue(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).getNode(), 1); break; } if (Node->getOpcode() == ISD::UREM && TLI.isOperationLegal(ISD::UDIVREM, VT)) { - Result = SDOperand(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).Val, 1); + Result = SDValue(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).getNode(), 1); break; } @@ -3249,7 +3390,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { "Cannot expand this binary operator!"); RTLIB::Libcall LC = Node->getOpcode() == ISD::UREM ? RTLIB::UREM_I32 : RTLIB::SREM_I32; - SDOperand Dummy; + SDValue Dummy; Result = ExpandLibCall(LC, Node, isSigned, Dummy); } } else { @@ -3261,7 +3402,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // Floating point mod -> fmod libcall. RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::REM_F32, RTLIB::REM_F64, RTLIB::REM_F80, RTLIB::REM_PPCF128); - SDOperand Dummy; + SDValue Dummy; Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); } } @@ -3286,7 +3427,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { if (isCustom) { Tmp2 = TLI.LowerOperation(Result, DAG); - if (Tmp2.Val) { + if (Tmp2.getNode()) { Result = LegalizeOp(Tmp2); Tmp1 = LegalizeOp(Tmp2.getValue(1)); } @@ -3294,11 +3435,11 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Expand: { const Value *V = cast(Node->getOperand(2))->getValue(); - SDOperand VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2, V, 0); + SDValue VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2, V, 0); // Increment the pointer, VAList, to the next vaarg - Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList, - DAG.getConstant(VT.getSizeInBits()/8, - TLI.getPointerTy())); + Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList, + DAG.getConstant(TLI.getTargetData()->getABITypeSize(VT.getTypeForMVT()), + TLI.getPointerTy())); // Store the incremented VAList to the legalized pointer Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, V, 0); // Load the actual argument out of the pointer VAList @@ -3310,9 +3451,9 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } // Since VAARG produces two values, make sure to remember that we // legalized both of them. - AddLegalizedOperand(SDOperand(Node, 0), Result); - AddLegalizedOperand(SDOperand(Node, 1), Tmp1); - return Op.ResNo ? Tmp1 : Result; + AddLegalizedOperand(SDValue(Node, 0), Result); + AddLegalizedOperand(SDValue(Node, 1), Tmp1); + return Op.getResNo() ? Tmp1 : Result; } case ISD::VACOPY: @@ -3330,7 +3471,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Node->getOperand(3), Node->getOperand(4)); if (isCustom) { Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; } break; case TargetLowering::Expand: @@ -3357,7 +3498,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2)); if (isCustom) { Tmp1 = TLI.LowerOperation(Tmp1, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; } break; case TargetLowering::Expand: @@ -3377,7 +3518,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case TargetLowering::Legal: break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; } break; @@ -3395,7 +3536,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Custom: Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; break; case TargetLowering::Promote: assert(0 && "Do not know how to promote ROTL/ROTR"); @@ -3442,7 +3583,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == TargetLowering::Custom) { Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) { + if (Tmp1.getNode()) { Result = Tmp1; } } @@ -3488,6 +3629,16 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { case ISD::FSQRT: case ISD::FSIN: case ISD::FCOS: + case ISD::FLOG: + case ISD::FLOG2: + case ISD::FLOG10: + case ISD::FEXP: + case ISD::FEXP2: + case ISD::FTRUNC: + case ISD::FFLOOR: + case ISD::FCEIL: + case ISD::FRINT: + case ISD::FNEARBYINT: Tmp1 = LegalizeOp(Node->getOperand(0)); switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { case TargetLowering::Promote: @@ -3498,7 +3649,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = DAG.UpdateNodeOperands(Result, Tmp1); if (isCustom) { Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; } break; case TargetLowering::Expand: @@ -3521,7 +3672,17 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } case ISD::FSQRT: case ISD::FSIN: - case ISD::FCOS: { + case ISD::FCOS: + case ISD::FLOG: + case ISD::FLOG2: + case ISD::FLOG10: + case ISD::FEXP: + case ISD::FEXP2: + case ISD::FTRUNC: + case ISD::FFLOOR: + case ISD::FCEIL: + case ISD::FRINT: + case ISD::FNEARBYINT: { MVT VT = Node->getValueType(0); // Expand unsupported unary vector operators by unrolling them. @@ -3544,9 +3705,50 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64, RTLIB::COS_F80, RTLIB::COS_PPCF128); break; + case ISD::FLOG: + LC = GetFPLibCall(VT, RTLIB::LOG_F32, RTLIB::LOG_F64, + RTLIB::LOG_F80, RTLIB::LOG_PPCF128); + break; + case ISD::FLOG2: + LC = GetFPLibCall(VT, RTLIB::LOG2_F32, RTLIB::LOG2_F64, + RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128); + break; + case ISD::FLOG10: + LC = GetFPLibCall(VT, RTLIB::LOG10_F32, RTLIB::LOG10_F64, + RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128); + break; + case ISD::FEXP: + LC = GetFPLibCall(VT, RTLIB::EXP_F32, RTLIB::EXP_F64, + RTLIB::EXP_F80, RTLIB::EXP_PPCF128); + break; + case ISD::FEXP2: + LC = GetFPLibCall(VT, RTLIB::EXP2_F32, RTLIB::EXP2_F64, + RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128); + break; + case ISD::FTRUNC: + LC = GetFPLibCall(VT, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, + RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128); + break; + case ISD::FFLOOR: + LC = GetFPLibCall(VT, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, + RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128); + break; + case ISD::FCEIL: + LC = GetFPLibCall(VT, RTLIB::CEIL_F32, RTLIB::CEIL_F64, + RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128); + break; + case ISD::FRINT: + LC = GetFPLibCall(VT, RTLIB::RINT_F32, RTLIB::RINT_F64, + RTLIB::RINT_F80, RTLIB::RINT_PPCF128); + break; + case ISD::FNEARBYINT: + LC = GetFPLibCall(VT, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, + RTLIB::NEARBYINT_F80, RTLIB::NEARBYINT_PPCF128); + break; + break; default: assert(0 && "Unreachable!"); } - SDOperand Dummy; + SDValue Dummy; Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); break; } @@ -3566,7 +3768,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // We always lower FPOWI into a libcall. No target support for it yet. RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64, RTLIB::POWI_F80, RTLIB::POWI_PPCF128); - SDOperand Dummy; + SDValue Dummy; Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); break; } @@ -3577,8 +3779,8 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } else if (Op.getOperand(0).getValueType().isVector()) { // The input has to be a vector type, we have to either scalarize it, pack // it, or convert it based on whether the input vector type is legal. - SDNode *InVal = Node->getOperand(0).Val; - int InIx = Node->getOperand(0).ResNo; + SDNode *InVal = Node->getOperand(0).getNode(); + int InIx = Node->getOperand(0).getResNo(); unsigned NumElems = InVal->getValueType(InIx).getVectorNumElements(); MVT EVT = InVal->getValueType(InIx).getVectorElementType(); @@ -3614,56 +3816,55 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } } break; - - // Conversion operators. The source and destination have different types. - case ISD::SINT_TO_FP: - case ISD::UINT_TO_FP: { - bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP; - switch (getTypeAction(Node->getOperand(0).getValueType())) { - case Legal: - switch (TLI.getOperationAction(Node->getOpcode(), - Node->getOperand(0).getValueType())) { - default: assert(0 && "Unknown operation action!"); - case TargetLowering::Custom: - isCustom = true; - // FALLTHROUGH - case TargetLowering::Legal: + case ISD::CONVERT_RNDSAT: { + ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); + switch (CvtCode) { + default: assert(0 && "Unknown cvt code!"); + case ISD::CVT_SF: + case ISD::CVT_UF: + break; + case ISD::CVT_FF: + case ISD::CVT_FS: + case ISD::CVT_FU: + case ISD::CVT_SS: + case ISD::CVT_SU: + case ISD::CVT_US: + case ISD::CVT_UU: { + SDValue DTyOp = Node->getOperand(1); + SDValue STyOp = Node->getOperand(2); + SDValue RndOp = Node->getOperand(3); + SDValue SatOp = Node->getOperand(4); + switch (getTypeAction(Node->getOperand(0).getValueType())) { + case Expand: assert(0 && "Shouldn't need to expand other operators here!"); + case Legal: Tmp1 = LegalizeOp(Node->getOperand(0)); - Result = DAG.UpdateNodeOperands(Result, Tmp1); - if (isCustom) { + Result = DAG.UpdateNodeOperands(Result, Tmp1, DTyOp, STyOp, + RndOp, SatOp); + if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == + TargetLowering::Custom) { Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; } break; - case TargetLowering::Expand: - Result = ExpandLegalINT_TO_FP(isSigned, - LegalizeOp(Node->getOperand(0)), - Node->getValueType(0)); - break; - case TargetLowering::Promote: - Result = PromoteLegalINT_TO_FP(LegalizeOp(Node->getOperand(0)), - Node->getValueType(0), - isSigned); + case Promote: + Result = PromoteOp(Node->getOperand(0)); + // For FP, make Op1 a i32 + + Result = DAG.getConvertRndSat(Result.getValueType(), Result, + DTyOp, STyOp, RndOp, SatOp, CvtCode); break; } break; - case Expand: - Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, - Node->getValueType(0), Node->getOperand(0)); - break; - case Promote: - Tmp1 = PromoteOp(Node->getOperand(0)); - if (isSigned) { - Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp1.getValueType(), - Tmp1, DAG.getValueType(Node->getOperand(0).getValueType())); - } else { - Tmp1 = DAG.getZeroExtendInReg(Tmp1, - Node->getOperand(0).getValueType()); - } - Result = DAG.UpdateNodeOperands(Result, Tmp1); - Result = LegalizeOp(Result); // The 'op' is not necessarily legal! - break; } + } // end switch CvtCode + break; + } + // Conversion operators. The source and destination have different types. + case ISD::SINT_TO_FP: + case ISD::UINT_TO_FP: { + bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP; + Result = LegalizeINT_TO_FP(Result, isSigned, + Node->getValueType(0), Node->getOperand(0)); break; } case ISD::TRUNCATE: @@ -3701,7 +3902,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { Result = DAG.UpdateNodeOperands(Result, Tmp1); if (isCustom) { Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; } break; case TargetLowering::Promote: @@ -3710,7 +3911,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Expand: if (Node->getOpcode() == ISD::FP_TO_UINT) { - SDOperand True, False; + SDValue True, False; MVT VT = Node->getOperand(0).getValueType(); MVT NVT = Node->getValueType(0); const uint64_t zero[] = {0, 0}; @@ -3764,83 +3965,10 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; } // Convert f32 / f64 to i32 / i64 / i128. - RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; - switch (Node->getOpcode()) { - case ISD::FP_TO_SINT: { - if (VT == MVT::i32) { - if (OVT == MVT::f32) - LC = RTLIB::FPTOSINT_F32_I32; - else if (OVT == MVT::f64) - LC = RTLIB::FPTOSINT_F64_I32; - else - assert(0 && "Unexpected i32-to-fp conversion!"); - } else if (VT == MVT::i64) { - if (OVT == MVT::f32) - LC = RTLIB::FPTOSINT_F32_I64; - else if (OVT == MVT::f64) - LC = RTLIB::FPTOSINT_F64_I64; - else if (OVT == MVT::f80) - LC = RTLIB::FPTOSINT_F80_I64; - else if (OVT == MVT::ppcf128) - LC = RTLIB::FPTOSINT_PPCF128_I64; - else - assert(0 && "Unexpected i64-to-fp conversion!"); - } else if (VT == MVT::i128) { - if (OVT == MVT::f32) - LC = RTLIB::FPTOSINT_F32_I128; - else if (OVT == MVT::f64) - LC = RTLIB::FPTOSINT_F64_I128; - else if (OVT == MVT::f80) - LC = RTLIB::FPTOSINT_F80_I128; - else if (OVT == MVT::ppcf128) - LC = RTLIB::FPTOSINT_PPCF128_I128; - else - assert(0 && "Unexpected i128-to-fp conversion!"); - } else { - assert(0 && "Unexpectd int-to-fp conversion!"); - } - break; - } - case ISD::FP_TO_UINT: { - if (VT == MVT::i32) { - if (OVT == MVT::f32) - LC = RTLIB::FPTOUINT_F32_I32; - else if (OVT == MVT::f64) - LC = RTLIB::FPTOUINT_F64_I32; - else if (OVT == MVT::f80) - LC = RTLIB::FPTOUINT_F80_I32; - else - assert(0 && "Unexpected i32-to-fp conversion!"); - } else if (VT == MVT::i64) { - if (OVT == MVT::f32) - LC = RTLIB::FPTOUINT_F32_I64; - else if (OVT == MVT::f64) - LC = RTLIB::FPTOUINT_F64_I64; - else if (OVT == MVT::f80) - LC = RTLIB::FPTOUINT_F80_I64; - else if (OVT == MVT::ppcf128) - LC = RTLIB::FPTOUINT_PPCF128_I64; - else - assert(0 && "Unexpected i64-to-fp conversion!"); - } else if (VT == MVT::i128) { - if (OVT == MVT::f32) - LC = RTLIB::FPTOUINT_F32_I128; - else if (OVT == MVT::f64) - LC = RTLIB::FPTOUINT_F64_I128; - else if (OVT == MVT::f80) - LC = RTLIB::FPTOUINT_F80_I128; - else if (OVT == MVT::ppcf128) - LC = RTLIB::FPTOUINT_PPCF128_I128; - else - assert(0 && "Unexpected i128-to-fp conversion!"); - } else { - assert(0 && "Unexpectd int-to-fp conversion!"); - } - break; - } - default: assert(0 && "Unreachable!"); - } - SDOperand Dummy; + RTLIB::Libcall LC = (Node->getOpcode() == ISD::FP_TO_SINT) ? + RTLIB::getFPTOSINT(OVT, VT) : RTLIB::getFPTOUINT(OVT, VT); + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpectd fp-to-int conversion!"); + SDValue Dummy; Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy); break; } @@ -3879,7 +4007,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { MVT SrcVT = Op.getOperand(0).getValueType(); if (TLI.getConvertAction(SrcVT, DstVT) == TargetLowering::Expand) { if (SrcVT == MVT::ppcf128) { - SDOperand Lo; + SDValue Lo; ExpandOp(Node->getOperand(0), Lo, Result); // Round it the rest of the way (e.g. to f32) if needed. if (DstVT!=MVT::f64) @@ -3916,7 +4044,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == TargetLowering::Custom) { Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.Val) Result = Tmp1; + if (Tmp1.getNode()) Result = Tmp1; } break; case Promote: @@ -3960,7 +4088,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { // SAR. However, it is doubtful that any exist. unsigned BitsDiff = Node->getValueType(0).getSizeInBits() - ExtraVT.getSizeInBits(); - SDOperand ShiftCst = DAG.getConstant(BitsDiff, TLI.getShiftAmountTy()); + SDValue ShiftCst = DAG.getConstant(BitsDiff, TLI.getShiftAmountTy()); Result = DAG.getNode(ISD::SHL, Node->getValueType(0), Node->getOperand(0), ShiftCst); Result = DAG.getNode(ISD::SRA, Node->getValueType(0), @@ -3982,21 +4110,21 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; } case ISD::TRAMPOLINE: { - SDOperand Ops[6]; + SDValue Ops[6]; for (unsigned i = 0; i != 6; ++i) Ops[i] = LegalizeOp(Node->getOperand(i)); Result = DAG.UpdateNodeOperands(Result, Ops, 6); // The only option for this node is to custom lower it. Result = TLI.LowerOperation(Result, DAG); - assert(Result.Val && "Should always custom lower!"); + assert(Result.getNode() && "Should always custom lower!"); // Since trampoline produces two values, make sure to remember that we // legalized both of them. Tmp1 = LegalizeOp(Result.getValue(1)); Result = LegalizeOp(Result); - AddLegalizedOperand(SDOperand(Node, 0), Result); - AddLegalizedOperand(SDOperand(Node, 1), Tmp1); - return Op.ResNo ? Tmp1 : Result; + AddLegalizedOperand(SDValue(Node, 0), Result); + AddLegalizedOperand(SDValue(Node, 1), Tmp1); + return Op.getResNo() ? Tmp1 : Result; } case ISD::FLT_ROUNDS_: { MVT VT = Node->getValueType(0); @@ -4004,7 +4132,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { default: assert(0 && "This action not supported for this op yet!"); case TargetLowering::Custom: Result = TLI.LowerOperation(Op, DAG); - if (Result.Val) break; + if (Result.getNode()) break; // Fall Thru case TargetLowering::Legal: // If this operation is not supported, lower it to constant 1 @@ -4023,15 +4151,15 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; case TargetLowering::Custom: Result = TLI.LowerOperation(Op, DAG); - if (Result.Val) break; + if (Result.getNode()) break; // Fall Thru case TargetLowering::Expand: // If this operation is not supported, lower it to 'abort()' call Tmp1 = LegalizeOp(Node->getOperand(0)); TargetLowering::ArgListTy Args; - std::pair CallResult = + std::pair CallResult = TLI.LowerCallTo(Tmp1, Type::VoidTy, - false, false, false, CallingConv::C, false, + false, false, false, false, CallingConv::C, false, DAG.getExternalSymbol("abort", TLI.getPointerTy()), Args, DAG); Result = CallResult.second; @@ -4058,7 +4186,7 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { /// promote it to compute the value into a larger type. The produced value will /// have the correct bits for the low portion of the register, but no guarantee /// is made about the top bits: it may be zero, sign-extended, or garbage. -SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) { +SDValue SelectionDAGLegalize::PromoteOp(SDValue Op) { MVT VT = Op.getValueType(); MVT NVT = TLI.getTypeToTransformTo(VT); assert(getTypeAction(VT) == Promote && @@ -4066,11 +4194,11 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) { assert(NVT.bitsGT(VT) && NVT.isInteger() == VT.isInteger() && "Cannot promote to smaller type!"); - SDOperand Tmp1, Tmp2, Tmp3; - SDOperand Result; - SDNode *Node = Op.Val; + SDValue Tmp1, Tmp2, Tmp3; + SDValue Result; + SDNode *Node = Op.getNode(); - DenseMap::iterator I = PromotedNodes.find(Op); + DenseMap::iterator I = PromotedNodes.find(Op); if (I != PromotedNodes.end()) return I->second; switch (Node->getOpcode()) { @@ -4148,6 +4276,19 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) { break; } break; + case ISD::CONVERT_RNDSAT: { + ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); + assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || + CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || + CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && + "can only promote integers"); + Result = DAG.getConvertRndSat(NVT, Node->getOperand(0), + Node->getOperand(1), Node->getOperand(2), + Node->getOperand(3), Node->getOperand(4), + CvtCode); + break; + + } case ISD::BIT_CONVERT: Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), Node->getValueType(0)); @@ -4247,9 +4388,19 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) { // precision, and these operations don't modify precision at all. break; + case ISD::FLOG: + case ISD::FLOG2: + case ISD::FLOG10: + case ISD::FEXP: + case ISD::FEXP2: case ISD::FSQRT: case ISD::FSIN: case ISD::FCOS: + case ISD::FTRUNC: + case ISD::FFLOOR: + case ISD::FCEIL: + case ISD::FRINT: + case ISD::FNEARBYINT: Tmp1 = PromoteOp(Node->getOperand(0)); assert(Tmp1.getValueType() == NVT); Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1); @@ -4258,43 +4409,87 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) { DAG.getValueType(VT)); break; + case ISD::FPOW: case ISD::FPOWI: { - // Promote f32 powi to f64 powi. Note that this could insert a libcall + // Promote f32 pow(i) to f64 pow(i). Note that this could insert a libcall // directly as well, which may be better. Tmp1 = PromoteOp(Node->getOperand(0)); + Tmp2 = Node->getOperand(1); + if (Node->getOpcode() == ISD::FPOW) + Tmp2 = PromoteOp(Tmp2); assert(Tmp1.getValueType() == NVT); - Result = DAG.getNode(ISD::FPOWI, NVT, Tmp1, Node->getOperand(1)); + Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2); if (NoExcessFPPrecision) Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result, DAG.getValueType(VT)); break; } - case ISD::ATOMIC_LCS: { + case ISD::ATOMIC_CMP_SWAP_8: + case ISD::ATOMIC_CMP_SWAP_16: + case ISD::ATOMIC_CMP_SWAP_32: + case ISD::ATOMIC_CMP_SWAP_64: { + AtomicSDNode* AtomNode = cast(Node); Tmp2 = PromoteOp(Node->getOperand(2)); Tmp3 = PromoteOp(Node->getOperand(3)); - Result = DAG.getAtomic(Node->getOpcode(), Node->getOperand(0), - Node->getOperand(1), Tmp2, Tmp3, - cast(Node)->getVT()); + Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getChain(), + AtomNode->getBasePtr(), Tmp2, Tmp3, + AtomNode->getSrcValue(), + AtomNode->getAlignment()); // Remember that we legalized the chain. AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); break; } - case ISD::ATOMIC_LAS: - case ISD::ATOMIC_LSS: - case ISD::ATOMIC_LOAD_AND: - case ISD::ATOMIC_LOAD_OR: - case ISD::ATOMIC_LOAD_XOR: - case ISD::ATOMIC_LOAD_NAND: - case ISD::ATOMIC_LOAD_MIN: - case ISD::ATOMIC_LOAD_MAX: - case ISD::ATOMIC_LOAD_UMIN: - case ISD::ATOMIC_LOAD_UMAX: - case ISD::ATOMIC_SWAP: { + case ISD::ATOMIC_LOAD_ADD_8: + case ISD::ATOMIC_LOAD_SUB_8: + case ISD::ATOMIC_LOAD_AND_8: + case ISD::ATOMIC_LOAD_OR_8: + case ISD::ATOMIC_LOAD_XOR_8: + case ISD::ATOMIC_LOAD_NAND_8: + case ISD::ATOMIC_LOAD_MIN_8: + case ISD::ATOMIC_LOAD_MAX_8: + case ISD::ATOMIC_LOAD_UMIN_8: + case ISD::ATOMIC_LOAD_UMAX_8: + case ISD::ATOMIC_SWAP_8: + case ISD::ATOMIC_LOAD_ADD_16: + case ISD::ATOMIC_LOAD_SUB_16: + case ISD::ATOMIC_LOAD_AND_16: + case ISD::ATOMIC_LOAD_OR_16: + case ISD::ATOMIC_LOAD_XOR_16: + case ISD::ATOMIC_LOAD_NAND_16: + case ISD::ATOMIC_LOAD_MIN_16: + case ISD::ATOMIC_LOAD_MAX_16: + case ISD::ATOMIC_LOAD_UMIN_16: + case ISD::ATOMIC_LOAD_UMAX_16: + case ISD::ATOMIC_SWAP_16: + case ISD::ATOMIC_LOAD_ADD_32: + case ISD::ATOMIC_LOAD_SUB_32: + case ISD::ATOMIC_LOAD_AND_32: + case ISD::ATOMIC_LOAD_OR_32: + case ISD::ATOMIC_LOAD_XOR_32: + case ISD::ATOMIC_LOAD_NAND_32: + case ISD::ATOMIC_LOAD_MIN_32: + case ISD::ATOMIC_LOAD_MAX_32: + case ISD::ATOMIC_LOAD_UMIN_32: + case ISD::ATOMIC_LOAD_UMAX_32: + case ISD::ATOMIC_SWAP_32: + case ISD::ATOMIC_LOAD_ADD_64: + case ISD::ATOMIC_LOAD_SUB_64: + case ISD::ATOMIC_LOAD_AND_64: + case ISD::ATOMIC_LOAD_OR_64: + case ISD::ATOMIC_LOAD_XOR_64: + case ISD::ATOMIC_LOAD_NAND_64: + case ISD::ATOMIC_LOAD_MIN_64: + case ISD::ATOMIC_LOAD_MAX_64: + case ISD::ATOMIC_LOAD_UMIN_64: + case ISD::ATOMIC_LOAD_UMAX_64: + case ISD::ATOMIC_SWAP_64: { + AtomicSDNode* AtomNode = cast(Node); Tmp2 = PromoteOp(Node->getOperand(2)); - Result = DAG.getAtomic(Node->getOpcode(), Node->getOperand(0), - Node->getOperand(1), Tmp2, - cast(Node)->getVT()); + Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getChain(), + AtomNode->getBasePtr(), Tmp2, + AtomNode->getSrcValue(), + AtomNode->getAlignment()); // Remember that we legalized the chain. AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); break; @@ -4406,10 +4601,10 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) { Tmp2 = Node->getOperand(1); // Get the pointer. if (TLI.getOperationAction(ISD::VAARG, VT) == TargetLowering::Custom) { Tmp3 = DAG.getVAArg(VT, Tmp1, Tmp2, Node->getOperand(2)); - Result = TLI.CustomPromoteOperation(Tmp3, DAG); + Result = TLI.LowerOperation(Tmp3, DAG); } else { const Value *V = cast(Node->getOperand(2))->getValue(); - SDOperand VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2, V, 0); + SDValue VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2, V, 0); // Increment the pointer, VAList, to the next vaarg Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList, DAG.getConstant(VT.getSizeInBits()/8, @@ -4500,7 +4695,7 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) { break; } - assert(Result.Val && "Didn't set a result!"); + assert(Result.getNode() && "Didn't set a result!"); // Make sure the result is itself legal. Result = LegalizeOp(Result); @@ -4514,12 +4709,12 @@ SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) { /// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic, /// based on the vector type. The return type of this matches the element type /// of the vector, which may not be legal for the target. -SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) { +SDValue SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDValue Op) { // We know that operand #0 is the Vec vector. If the index is a constant // or if the invec is a supported hardware type, we can use it. Otherwise, // lower to a store then an indexed load. - SDOperand Vec = Op.getOperand(0); - SDOperand Idx = Op.getOperand(1); + SDValue Vec = Op.getOperand(0); + SDValue Idx = Op.getOperand(1); MVT TVT = Vec.getValueType(); unsigned NumElems = TVT.getVectorNumElements(); @@ -4529,8 +4724,8 @@ SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) { case TargetLowering::Custom: { Vec = LegalizeOp(Vec); Op = DAG.UpdateNodeOperands(Op, Vec, Idx); - SDOperand Tmp3 = TLI.LowerOperation(Op, DAG); - if (Tmp3.Val) + SDValue Tmp3 = TLI.LowerOperation(Op, DAG); + if (Tmp3.getNode()) return Tmp3; break; } @@ -4541,6 +4736,9 @@ SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) { return Op; } break; + case TargetLowering::Promote: + assert(TVT.isVector() && "not vector type"); + // fall thru to expand since vectors are by default are promote case TargetLowering::Expand: break; } @@ -4551,13 +4749,13 @@ SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) { } else if (!TLI.isTypeLegal(TVT) && isa(Idx)) { unsigned NumLoElts = 1 << Log2_32(NumElems-1); ConstantSDNode *CIdx = cast(Idx); - SDOperand Lo, Hi; + SDValue Lo, Hi; SplitVectorOp(Vec, Lo, Hi); - if (CIdx->getValue() < NumLoElts) { + if (CIdx->getZExtValue() < NumLoElts) { Vec = Lo; } else { Vec = Hi; - Idx = DAG.getConstant(CIdx->getValue() - NumLoElts, + Idx = DAG.getConstant(CIdx->getZExtValue() - NumLoElts, Idx.getValueType()); } @@ -4567,8 +4765,8 @@ SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) { } else { // Store the value to a temporary stack slot, then LOAD the scalar // element back out. - SDOperand StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); - SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0); + SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType()); + SDValue Ch = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0); // Add the offset to the index. unsigned EltSize = Op.getValueType().getSizeInBits()/8; @@ -4589,11 +4787,11 @@ SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) { /// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation. For now /// we assume the operation can be split if it is not already legal. -SDOperand SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDOperand Op) { +SDValue SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDValue Op) { // We know that operand #0 is the Vec vector. For now we assume the index // is a constant and that the extracted result is a supported hardware type. - SDOperand Vec = Op.getOperand(0); - SDOperand Idx = LegalizeOp(Op.getOperand(1)); + SDValue Vec = Op.getOperand(0); + SDValue Idx = LegalizeOp(Op.getOperand(1)); unsigned NumElems = Vec.getValueType().getVectorNumElements(); @@ -4603,13 +4801,14 @@ SDOperand SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDOperand Op) { } ConstantSDNode *CIdx = cast(Idx); - SDOperand Lo, Hi; + SDValue Lo, Hi; SplitVectorOp(Vec, Lo, Hi); - if (CIdx->getValue() < NumElems/2) { + if (CIdx->getZExtValue() < NumElems/2) { Vec = Lo; } else { Vec = Hi; - Idx = DAG.getConstant(CIdx->getValue() - NumElems/2, Idx.getValueType()); + Idx = DAG.getConstant(CIdx->getZExtValue() - NumElems/2, + Idx.getValueType()); } // It's now an extract from the appropriate high or low part. Recurse. @@ -4622,11 +4821,11 @@ SDOperand SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDOperand Op) { /// or promoting the arguments. In the case where LHS and RHS must be expanded, /// there may be no choice but to create a new SetCC node to represent the /// legalized value of setcc lhs, rhs. In this case, the value is returned in -/// LHS, and the SDOperand returned in RHS has a nil SDNode value. -void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS, - SDOperand &RHS, - SDOperand &CC) { - SDOperand Tmp1, Tmp2, Tmp3, Result; +/// LHS, and the SDValue returned in RHS has a nil SDNode value. +void SelectionDAGLegalize::LegalizeSetCCOperands(SDValue &LHS, + SDValue &RHS, + SDValue &CC) { + SDValue Tmp1, Tmp2, Tmp3, Result; switch (getTypeAction(LHS.getValueType())) { case Legal: @@ -4668,6 +4867,8 @@ void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS, DAG.getValueType(VT)); Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2, DAG.getValueType(VT)); + Tmp1 = LegalizeOp(Tmp1); // Relegalize new nodes. + Tmp2 = LegalizeOp(Tmp2); // Relegalize new nodes. break; } } @@ -4676,7 +4877,7 @@ void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS, MVT VT = LHS.getValueType(); if (VT == MVT::f32 || VT == MVT::f64) { // Expand into one or more soft-fp libcall(s). - RTLIB::Libcall LC1, LC2 = RTLIB::UNKNOWN_LIBCALL; + RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL; switch (cast(CC)->get()) { case ISD::SETEQ: case ISD::SETOEQ: @@ -4733,48 +4934,49 @@ void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS, default: assert(0 && "Unsupported FP setcc!"); } } - - SDOperand Dummy; - Tmp1 = ExpandLibCall(LC1, - DAG.getNode(ISD::MERGE_VALUES, VT, LHS, RHS).Val, + + SDValue Dummy; + SDValue Ops[2] = { LHS, RHS }; + Tmp1 = ExpandLibCall(LC1, DAG.getMergeValues(Ops, 2).getNode(), false /*sign irrelevant*/, Dummy); Tmp2 = DAG.getConstant(0, MVT::i32); CC = DAG.getCondCode(TLI.getCmpLibcallCC(LC1)); if (LC2 != RTLIB::UNKNOWN_LIBCALL) { Tmp1 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(Tmp1), Tmp1, Tmp2, CC); - LHS = ExpandLibCall(LC2, - DAG.getNode(ISD::MERGE_VALUES, VT, LHS, RHS).Val, + LHS = ExpandLibCall(LC2, DAG.getMergeValues(Ops, 2).getNode(), false /*sign irrelevant*/, Dummy); Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHS), LHS, Tmp2, DAG.getCondCode(TLI.getCmpLibcallCC(LC2))); Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp2); - Tmp2 = SDOperand(); + Tmp2 = SDValue(); } - LHS = Tmp1; + LHS = LegalizeOp(Tmp1); RHS = Tmp2; return; } - SDOperand LHSLo, LHSHi, RHSLo, RHSHi; + SDValue LHSLo, LHSHi, RHSLo, RHSHi; ExpandOp(LHS, LHSLo, LHSHi); ExpandOp(RHS, RHSLo, RHSHi); ISD::CondCode CCCode = cast(CC)->get(); if (VT==MVT::ppcf128) { // FIXME: This generated code sucks. We want to generate - // FCMP crN, hi1, hi2 + // FCMPU crN, hi1, hi2 // BNE crN, L: - // FCMP crN, lo1, lo2 + // FCMPU crN, lo1, lo2 // The following can be improved, but not that much. - Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, ISD::SETEQ); + Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, + ISD::SETOEQ); Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, CCCode); Tmp3 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2); - Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, ISD::SETNE); + Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, + ISD::SETUNE); Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, CCCode); Tmp1 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2); Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp3); - Tmp2 = SDOperand(); + Tmp2 = SDValue(); break; } @@ -4831,16 +5033,16 @@ void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS, TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL); Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC, false, DagCombineInfo); - if (!Tmp1.Val) + if (!Tmp1.getNode()) Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC); Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, CCCode, false, DagCombineInfo); - if (!Tmp2.Val) + if (!Tmp2.getNode()) Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,CC); - ConstantSDNode *Tmp1C = dyn_cast(Tmp1.Val); - ConstantSDNode *Tmp2C = dyn_cast(Tmp2.Val); + ConstantSDNode *Tmp1C = dyn_cast(Tmp1.getNode()); + ConstantSDNode *Tmp2C = dyn_cast(Tmp2.getNode()); if ((Tmp1C && Tmp1C->isNullValue()) || (Tmp2C && Tmp2C->isNullValue() && (CCCode == ISD::SETLE || CCCode == ISD::SETGE || @@ -4852,17 +5054,17 @@ void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS, // For LE / GE, if high part is known false, ignore the low part. // For LT / GT, if high part is known true, ignore the low part. Tmp1 = Tmp2; - Tmp2 = SDOperand(); + Tmp2 = SDValue(); } else { Result = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, ISD::SETEQ, false, DagCombineInfo); - if (!Result.Val) + if (!Result.getNode()) Result=DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, ISD::SETEQ); Result = LegalizeOp(DAG.getNode(ISD::SELECT, Tmp1.getValueType(), Result, Tmp1, Tmp2)); Tmp1 = Result; - Tmp2 = SDOperand(); + Tmp2 = SDValue(); } } } @@ -4871,73 +5073,123 @@ void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS, RHS = Tmp2; } +/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and +/// condition code CC on the current target. This routine assumes LHS and rHS +/// have already been legalized by LegalizeSetCCOperands. It expands SETCC with +/// illegal condition code into AND / OR of multiple SETCC values. +void SelectionDAGLegalize::LegalizeSetCCCondCode(MVT VT, + SDValue &LHS, SDValue &RHS, + SDValue &CC) { + MVT OpVT = LHS.getValueType(); + ISD::CondCode CCCode = cast(CC)->get(); + switch (TLI.getCondCodeAction(CCCode, OpVT)) { + default: assert(0 && "Unknown condition code action!"); + case TargetLowering::Legal: + // Nothing to do. + break; + case TargetLowering::Expand: { + ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; + unsigned Opc = 0; + switch (CCCode) { + default: assert(0 && "Don't know how to expand this condition!"); abort(); + case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETOLT: CC1 = ISD::SETLT; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETOLE: CC1 = ISD::SETLE; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETONE: CC1 = ISD::SETNE; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETUEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETUO; Opc = ISD::OR; break; + case ISD::SETUGT: CC1 = ISD::SETGT; CC2 = ISD::SETUO; Opc = ISD::OR; break; + case ISD::SETUGE: CC1 = ISD::SETGE; CC2 = ISD::SETUO; Opc = ISD::OR; break; + case ISD::SETULT: CC1 = ISD::SETLT; CC2 = ISD::SETUO; Opc = ISD::OR; break; + case ISD::SETULE: CC1 = ISD::SETLE; CC2 = ISD::SETUO; Opc = ISD::OR; break; + case ISD::SETUNE: CC1 = ISD::SETNE; CC2 = ISD::SETUO; Opc = ISD::OR; break; + // FIXME: Implement more expansions. + } + + SDValue SetCC1 = DAG.getSetCC(VT, LHS, RHS, CC1); + SDValue SetCC2 = DAG.getSetCC(VT, LHS, RHS, CC2); + LHS = DAG.getNode(Opc, VT, SetCC1, SetCC2); + RHS = SDValue(); + CC = SDValue(); + break; + } + } +} + /// EmitStackConvert - Emit a store/load combination to the stack. This stores /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does /// a load from the stack slot to DestVT, extending it if needed. /// The resultant code need not be legal. -SDOperand SelectionDAGLegalize::EmitStackConvert(SDOperand SrcOp, - MVT SlotVT, - MVT DestVT) { +SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, + MVT SlotVT, + MVT DestVT) { // Create the stack frame object. - SDOperand FIPtr = DAG.CreateStackTemporary(SlotVT); - + unsigned SrcAlign = TLI.getTargetData()->getPrefTypeAlignment( + SrcOp.getValueType().getTypeForMVT()); + SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); + FrameIndexSDNode *StackPtrFI = cast(FIPtr); int SPFI = StackPtrFI->getIndex(); - + unsigned SrcSize = SrcOp.getValueType().getSizeInBits(); unsigned SlotSize = SlotVT.getSizeInBits(); unsigned DestSize = DestVT.getSizeInBits(); + unsigned DestAlign = TLI.getTargetData()->getPrefTypeAlignment( + DestVT.getTypeForMVT()); // Emit a store to the stack slot. Use a truncstore if the input value is // later than DestVT. - SDOperand Store; + SDValue Store; + if (SrcSize > SlotSize) Store = DAG.getTruncStore(DAG.getEntryNode(), SrcOp, FIPtr, - PseudoSourceValue::getFixedStack(), - SPFI, SlotVT); + PseudoSourceValue::getFixedStack(SPFI), 0, + SlotVT, false, SrcAlign); else { assert(SrcSize == SlotSize && "Invalid store"); Store = DAG.getStore(DAG.getEntryNode(), SrcOp, FIPtr, - PseudoSourceValue::getFixedStack(), - SPFI); + PseudoSourceValue::getFixedStack(SPFI), 0, + false, SrcAlign); } // Result is a load from the stack slot. if (SlotSize == DestSize) - return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0); + return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0, false, DestAlign); assert(SlotSize < DestSize && "Unknown extension!"); - return DAG.getExtLoad(ISD::EXTLOAD, DestVT, Store, FIPtr, NULL, 0, SlotVT); + return DAG.getExtLoad(ISD::EXTLOAD, DestVT, Store, FIPtr, NULL, 0, SlotVT, + false, DestAlign); } -SDOperand SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { +SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { // Create a vector sized/aligned stack slot, store the value to element #0, // then load the whole vector back out. - SDOperand StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); + SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); FrameIndexSDNode *StackPtrFI = cast(StackPtr); int SPFI = StackPtrFI->getIndex(); - SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Node->getOperand(0), StackPtr, - PseudoSourceValue::getFixedStack(), SPFI); + SDValue Ch = DAG.getStore(DAG.getEntryNode(), Node->getOperand(0), StackPtr, + PseudoSourceValue::getFixedStack(SPFI), 0); return DAG.getLoad(Node->getValueType(0), Ch, StackPtr, - PseudoSourceValue::getFixedStack(), SPFI); + PseudoSourceValue::getFixedStack(SPFI), 0); } /// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't /// support the operation, but do support the resultant vector type. -SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { +SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { // If the only non-undef value is the low element, turn this into a // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. unsigned NumElems = Node->getNumOperands(); bool isOnlyLowElement = true; - SDOperand SplatValue = Node->getOperand(0); + SDValue SplatValue = Node->getOperand(0); - // FIXME: it would be far nicer to change this into map + // FIXME: it would be far nicer to change this into map // and use a bitmask instead of a list of elements. - std::map > Values; + std::map > Values; Values[SplatValue].push_back(0); bool isConstant = true; if (!isa(SplatValue) && !isa(SplatValue) && @@ -4945,12 +5197,12 @@ SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { isConstant = false; for (unsigned i = 1; i < NumElems; ++i) { - SDOperand V = Node->getOperand(i); + SDValue V = Node->getOperand(i); Values[V].push_back(i); if (V.getOpcode() != ISD::UNDEF) isOnlyLowElement = false; if (SplatValue != V) - SplatValue = SDOperand(0,0); + SplatValue = SDValue(0,0); // If this isn't a constant element or an undef, we can't use a constant // pool load. @@ -4975,10 +5227,10 @@ SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { for (unsigned i = 0, e = NumElems; i != e; ++i) { if (ConstantFPSDNode *V = dyn_cast(Node->getOperand(i))) { - CV.push_back(ConstantFP::get(V->getValueAPF())); + CV.push_back(const_cast(V->getConstantFPValue())); } else if (ConstantSDNode *V = dyn_cast(Node->getOperand(i))) { - CV.push_back(ConstantInt::get(V->getAPIntValue())); + CV.push_back(const_cast(V->getConstantIntValue())); } else { assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); const Type *OpNTy = @@ -4987,23 +5239,25 @@ SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { } } Constant *CP = ConstantVector::get(CV); - SDOperand CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); + SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); + unsigned Alignment = 1 << cast(CPIdx)->getAlignment(); return DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, - PseudoSourceValue::getConstantPool(), 0); + PseudoSourceValue::getConstantPool(), 0, + false, Alignment); } - if (SplatValue.Val) { // Splat of one value? + if (SplatValue.getNode()) { // Splat of one value? // Build the shuffle constant vector: <0, 0, 0, 0> MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); - SDOperand Zero = DAG.getConstant(0, MaskVT.getVectorElementType()); - std::vector ZeroVec(NumElems, Zero); - SDOperand SplatMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, + SDValue Zero = DAG.getConstant(0, MaskVT.getVectorElementType()); + std::vector ZeroVec(NumElems, Zero); + SDValue SplatMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &ZeroVec[0], ZeroVec.size()); // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it. if (isShuffleLegal(Node->getValueType(0), SplatMask)) { // Get the splatted value into the low element of a vector register. - SDOperand LowValVec = + SDValue LowValVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), SplatValue); // Return shuffle(LowValVec, undef, <0,0,0,0>) @@ -5017,9 +5271,9 @@ SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { // vector shuffle. if (Values.size() == 2) { // Get the two values in deterministic order. - SDOperand Val1 = Node->getOperand(1); - SDOperand Val2; - std::map >::iterator MI = Values.begin(); + SDValue Val1 = Node->getOperand(1); + SDValue Val2; + std::map >::iterator MI = Values.begin(); if (MI->first != Val1) Val2 = MI->first; else @@ -5033,7 +5287,7 @@ SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { // Build the shuffle constant vector: e.g. <0, 4, 0, 4> MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems); MVT MaskEltVT = MaskVT.getVectorElementType(); - std::vector MaskVec(NumElems); + std::vector MaskVec(NumElems); // Set elements of the shuffle mask for Val1. std::vector &Val1Elts = Values[Val1]; @@ -5048,7 +5302,7 @@ SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { else MaskVec[Val2Elts[i]] = DAG.getNode(ISD::UNDEF, MaskEltVT); - SDOperand ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, + SDValue ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &MaskVec[0], MaskVec.size()); // If the target supports SCALAR_TO_VECTOR and this shuffle mask, use it. @@ -5056,7 +5310,7 @@ SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { isShuffleLegal(Node->getValueType(0), ShuffleMask)) { Val1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), Val1); Val2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), Val2); - SDOperand Ops[] = { Val1, Val2, ShuffleMask }; + SDValue Ops[] = { Val1, Val2, ShuffleMask }; // Return shuffle(LoValVec, HiValVec, <0,1,0,1>) return DAG.getNode(ISD::VECTOR_SHUFFLE, Node->getValueType(0), Ops, 3); @@ -5068,10 +5322,10 @@ SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { // the result as a vector. MVT VT = Node->getValueType(0); // Create the stack frame object. - SDOperand FIPtr = DAG.CreateStackTemporary(VT); + SDValue FIPtr = DAG.CreateStackTemporary(VT); // Emit a store of each element to the stack slot. - SmallVector Stores; + SmallVector Stores; unsigned TypeByteSize = Node->getOperand(0).getValueType().getSizeInBits()/8; // Store (in the right endianness) the elements to memory. for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { @@ -5080,14 +5334,14 @@ SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { unsigned Offset = TypeByteSize*i; - SDOperand Idx = DAG.getConstant(Offset, FIPtr.getValueType()); + SDValue Idx = DAG.getConstant(Offset, FIPtr.getValueType()); Idx = DAG.getNode(ISD::ADD, FIPtr.getValueType(), FIPtr, Idx); Stores.push_back(DAG.getStore(DAG.getEntryNode(), Node->getOperand(i), Idx, NULL, 0)); } - SDOperand StoreChain; + SDValue StoreChain; if (!Stores.empty()) // Not all undef elements? StoreChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Stores[0], Stores.size()); @@ -5099,13 +5353,13 @@ SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { } void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp, - SDOperand Op, SDOperand Amt, - SDOperand &Lo, SDOperand &Hi) { + SDValue Op, SDValue Amt, + SDValue &Lo, SDValue &Hi) { // Expand the subcomponents. - SDOperand LHSL, LHSH; + SDValue LHSL, LHSH; ExpandOp(Op, LHSL, LHSH); - SDOperand Ops[] = { LHSL, LHSH, Amt }; + SDValue Ops[] = { LHSL, LHSH, Amt }; MVT VT = LHSL.getValueType(); Lo = DAG.getNode(NodeOp, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3); Hi = Lo.getValue(1); @@ -5116,23 +5370,23 @@ void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp, /// smaller elements. If we can't find a way that is more efficient than a /// libcall on this target, return false. Otherwise, return true with the /// low-parts expanded into Lo and Hi. -bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDOperand Op,SDOperand Amt, - SDOperand &Lo, SDOperand &Hi) { +bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDValue Op,SDValue Amt, + SDValue &Lo, SDValue &Hi) { assert((Opc == ISD::SHL || Opc == ISD::SRA || Opc == ISD::SRL) && "This is not a shift!"); MVT NVT = TLI.getTypeToTransformTo(Op.getValueType()); - SDOperand ShAmt = LegalizeOp(Amt); + SDValue ShAmt = LegalizeOp(Amt); MVT ShTy = ShAmt.getValueType(); unsigned ShBits = ShTy.getSizeInBits(); unsigned VTBits = Op.getValueType().getSizeInBits(); unsigned NVTBits = NVT.getSizeInBits(); // Handle the case when Amt is an immediate. - if (ConstantSDNode *CN = dyn_cast(Amt.Val)) { - unsigned Cst = CN->getValue(); + if (ConstantSDNode *CN = dyn_cast(Amt.getNode())) { + unsigned Cst = CN->getZExtValue(); // Expand the incoming operand to be shifted, so that we have its parts - SDOperand InL, InH; + SDValue InL, InH; ExpandOp(Op, InL, InH); switch(Opc) { case ISD::SHL: @@ -5206,7 +5460,7 @@ bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDOperand Op,SDOperand Amt, DAG.getConstant(~Mask, Amt.getValueType())); // Expand the incoming operand to be shifted, so that we have its parts - SDOperand InL, InH; + SDValue InL, InH; ExpandOp(Op, InL, InH); switch(Opc) { case ISD::SHL: @@ -5229,12 +5483,12 @@ bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDOperand Op,SDOperand Amt, // do this as a couple of simple shifts. if ((KnownZero & Mask) == Mask) { // Compute 32-amt. - SDOperand Amt2 = DAG.getNode(ISD::SUB, Amt.getValueType(), + SDValue Amt2 = DAG.getNode(ISD::SUB, Amt.getValueType(), DAG.getConstant(NVTBits, Amt.getValueType()), Amt); // Expand the incoming operand to be shifted, so that we have its parts - SDOperand InL, InH; + SDValue InL, InH; ExpandOp(Op, InL, InH); switch(Opc) { case ISD::SHL: @@ -5266,13 +5520,13 @@ bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDOperand Op,SDOperand Amt, // does not fit into a register, return the lo part and set the hi part to the // by-reg argument. If it does fit into a single register, return the result // and leave the Hi part unset. -SDOperand SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, - bool isSigned, SDOperand &Hi) { +SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, + bool isSigned, SDValue &Hi) { assert(!IsLegalizingCall && "Cannot overlap legalization of calls!"); // The input chain to this libcall is the entry node of the function. // Legalizing the call will automatically add the previous call to the // dependence. - SDOperand InChain = DAG.getEntryNode(); + SDValue InChain = DAG.getEntryNode(); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; @@ -5284,20 +5538,20 @@ SDOperand SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, Entry.isZExt = !isSigned; Args.push_back(Entry); } - SDOperand Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), - TLI.getPointerTy()); + SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), + TLI.getPointerTy()); // Splice the libcall in wherever FindInputOutputChains tells us to. const Type *RetTy = Node->getValueType(0).getTypeForMVT(); - std::pair CallInfo = - TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, CallingConv::C, - false, Callee, Args, DAG); + std::pair CallInfo = + TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false, + CallingConv::C, false, Callee, Args, DAG); // Legalize the call sequence, starting with the chain. This will advance // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that // was added by LowerCallTo (guaranteeing proper serialization of calls). LegalizeOp(CallInfo.second); - SDOperand Result; + SDValue Result; switch (getTypeAction(CallInfo.first.getValueType())) { default: assert(0 && "Unknown thing"); case Legal: @@ -5310,22 +5564,99 @@ SDOperand SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, return Result; } +/// LegalizeINT_TO_FP - Legalize a [US]INT_TO_FP operation. +/// +SDValue SelectionDAGLegalize:: +LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, SDValue Op) { + bool isCustom = false; + SDValue Tmp1; + switch (getTypeAction(Op.getValueType())) { + case Legal: + switch (TLI.getOperationAction(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, + Op.getValueType())) { + default: assert(0 && "Unknown operation action!"); + case TargetLowering::Custom: + isCustom = true; + // FALLTHROUGH + case TargetLowering::Legal: + Tmp1 = LegalizeOp(Op); + if (Result.getNode()) + Result = DAG.UpdateNodeOperands(Result, Tmp1); + else + Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, + DestTy, Tmp1); + if (isCustom) { + Tmp1 = TLI.LowerOperation(Result, DAG); + if (Tmp1.getNode()) Result = Tmp1; + } + break; + case TargetLowering::Expand: + Result = ExpandLegalINT_TO_FP(isSigned, LegalizeOp(Op), DestTy); + break; + case TargetLowering::Promote: + Result = PromoteLegalINT_TO_FP(LegalizeOp(Op), DestTy, isSigned); + break; + } + break; + case Expand: + Result = ExpandIntToFP(isSigned, DestTy, Op); + break; + case Promote: + Tmp1 = PromoteOp(Op); + if (isSigned) { + Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp1.getValueType(), + Tmp1, DAG.getValueType(Op.getValueType())); + } else { + Tmp1 = DAG.getZeroExtendInReg(Tmp1, + Op.getValueType()); + } + if (Result.getNode()) + Result = DAG.UpdateNodeOperands(Result, Tmp1); + else + Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, + DestTy, Tmp1); + Result = LegalizeOp(Result); // The 'op' is not necessarily legal! + break; + } + return Result; +} /// ExpandIntToFP - Expand a [US]INT_TO_FP operation. /// -SDOperand SelectionDAGLegalize:: -ExpandIntToFP(bool isSigned, MVT DestTy, SDOperand Source) { +SDValue SelectionDAGLegalize:: +ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source) { MVT SourceVT = Source.getValueType(); bool ExpandSource = getTypeAction(SourceVT) == Expand; + // Expand unsupported int-to-fp vector casts by unrolling them. + if (DestTy.isVector()) { + if (!ExpandSource) + return LegalizeOp(UnrollVectorOp(Source)); + MVT DestEltTy = DestTy.getVectorElementType(); + if (DestTy.getVectorNumElements() == 1) { + SDValue Scalar = ScalarizeVectorOp(Source); + SDValue Result = LegalizeINT_TO_FP(SDValue(), isSigned, + DestEltTy, Scalar); + return DAG.getNode(ISD::BUILD_VECTOR, DestTy, Result); + } + SDValue Lo, Hi; + SplitVectorOp(Source, Lo, Hi); + MVT SplitDestTy = MVT::getVectorVT(DestEltTy, + DestTy.getVectorNumElements() / 2); + SDValue LoResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, Lo); + SDValue HiResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, Hi); + return LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, DestTy, LoResult, + HiResult)); + } + // Special case for i32 source to take advantage of UINTTOFP_I32_F32, etc. if (!isSigned && SourceVT != MVT::i32) { // The integer value loaded will be incorrectly if the 'sign bit' of the // incoming integer is set. To handle this, we dynamically test to see if // it is set, and, if so, add a fudge factor. - SDOperand Hi; + SDValue Hi; if (ExpandSource) { - SDOperand Lo; + SDValue Lo; ExpandOp(Source, Lo, Hi); Source = DAG.getNode(ISD::BUILD_PAIR, SourceVT, Lo, Hi); } else { @@ -5333,32 +5664,51 @@ ExpandIntToFP(bool isSigned, MVT DestTy, SDOperand Source) { Hi = Source; } + // Check to see if the target has a custom way to lower this. If so, use + // it. (Note we've already expanded the operand in this case.) + switch (TLI.getOperationAction(ISD::UINT_TO_FP, SourceVT)) { + default: assert(0 && "This action not implemented for this operation!"); + case TargetLowering::Legal: + case TargetLowering::Expand: + break; // This case is handled below. + case TargetLowering::Custom: { + SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::UINT_TO_FP, DestTy, + Source), DAG); + if (NV.getNode()) + return LegalizeOp(NV); + break; // The target decided this was legal after all + } + } + // If this is unsigned, and not supported, first perform the conversion to // signed, then adjust the result if the sign bit is set. - SDOperand SignedConv = ExpandIntToFP(true, DestTy, Source); + SDValue SignedConv = ExpandIntToFP(true, DestTy, Source); - SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi, + SDValue SignSet = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi, DAG.getConstant(0, Hi.getValueType()), ISD::SETLT); - SDOperand Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); - SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(), + SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); + SDValue CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(), SignSet, Four, Zero); uint64_t FF = 0x5f800000ULL; if (TLI.isLittleEndian()) FF <<= 32; static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); - SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); + SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); + unsigned Alignment = 1 << cast(CPIdx)->getAlignment(); CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset); - SDOperand FudgeInReg; + Alignment = std::min(Alignment, 4u); + SDValue FudgeInReg; if (DestTy == MVT::f32) FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, - PseudoSourceValue::getConstantPool(), 0); + PseudoSourceValue::getConstantPool(), 0, + false, Alignment); else if (DestTy.bitsGT(MVT::f32)) // FIXME: Avoid the extend by construction the right constantpool? FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, DestTy, DAG.getEntryNode(), CPIdx, PseudoSourceValue::getConstantPool(), 0, - MVT::f32); + MVT::f32, false, Alignment); else assert(0 && "Unexpected conversion"); @@ -5383,9 +5733,9 @@ ExpandIntToFP(bool isSigned, MVT DestTy, SDOperand Source) { case TargetLowering::Expand: break; // This case is handled below. case TargetLowering::Custom: { - SDOperand NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, DestTy, + SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, DestTy, Source), DAG); - if (NV.Val) + if (NV.getNode()) return LegalizeOp(NV); break; // The target decided this was legal after all } @@ -5394,50 +5744,20 @@ ExpandIntToFP(bool isSigned, MVT DestTy, SDOperand Source) { // Expand the source, then glue it back together for the call. We must expand // the source in case it is shared (this pass of legalize must traverse it). if (ExpandSource) { - SDOperand SrcLo, SrcHi; + SDValue SrcLo, SrcHi; ExpandOp(Source, SrcLo, SrcHi); Source = DAG.getNode(ISD::BUILD_PAIR, SourceVT, SrcLo, SrcHi); } - RTLIB::Libcall LC; - if (SourceVT == MVT::i32) { - if (DestTy == MVT::f32) - LC = isSigned ? RTLIB::SINTTOFP_I32_F32 : RTLIB::UINTTOFP_I32_F32; - else { - assert(DestTy == MVT::f64 && "Unknown fp value type!"); - LC = isSigned ? RTLIB::SINTTOFP_I32_F64 : RTLIB::UINTTOFP_I32_F64; - } - } else if (SourceVT == MVT::i64) { - if (DestTy == MVT::f32) - LC = RTLIB::SINTTOFP_I64_F32; - else if (DestTy == MVT::f64) - LC = RTLIB::SINTTOFP_I64_F64; - else if (DestTy == MVT::f80) - LC = RTLIB::SINTTOFP_I64_F80; - else { - assert(DestTy == MVT::ppcf128 && "Unknown fp value type!"); - LC = RTLIB::SINTTOFP_I64_PPCF128; - } - } else if (SourceVT == MVT::i128) { - if (DestTy == MVT::f32) - LC = RTLIB::SINTTOFP_I128_F32; - else if (DestTy == MVT::f64) - LC = RTLIB::SINTTOFP_I128_F64; - else if (DestTy == MVT::f80) - LC = RTLIB::SINTTOFP_I128_F80; - else { - assert(DestTy == MVT::ppcf128 && "Unknown fp value type!"); - LC = RTLIB::SINTTOFP_I128_PPCF128; - } - } else { - assert(0 && "Unknown int value type"); - } - - assert(TLI.getLibcallName(LC) && "Don't know how to expand this SINT_TO_FP!"); + RTLIB::Libcall LC = isSigned ? + RTLIB::getSINTTOFP(SourceVT, DestTy) : + RTLIB::getUINTTOFP(SourceVT, DestTy); + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unknown int value type"); + Source = DAG.getNode(ISD::SINT_TO_FP, DestTy, Source); - SDOperand HiPart; - SDOperand Result = ExpandLibCall(LC, Source.Val, isSigned, HiPart); - if (Result.getValueType() != DestTy && HiPart.Val) + SDValue HiPart; + SDValue Result = ExpandLibCall(LC, Source.getNode(), isSigned, HiPart); + if (Result.getValueType() != DestTy && HiPart.getNode()) Result = DAG.getNode(ISD::BUILD_PAIR, DestTy, Result, HiPart); return Result; } @@ -5446,50 +5766,50 @@ ExpandIntToFP(bool isSigned, MVT DestTy, SDOperand Source) { /// INT_TO_FP operation of the specified operand when the target requests that /// we expand it. At this point, we know that the result and operand types are /// legal for the target. -SDOperand SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, - SDOperand Op0, - MVT DestVT) { +SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, + SDValue Op0, + MVT DestVT) { if (Op0.getValueType() == MVT::i32) { // simple 32-bit [signed|unsigned] integer to float/double expansion // Get the stack frame index of a 8 byte buffer. - SDOperand StackSlot = DAG.CreateStackTemporary(MVT::f64); + SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); // word offset constant for Hi/Lo address computation - SDOperand WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy()); + SDValue WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy()); // set up Hi and Lo (into buffer) address based on endian - SDOperand Hi = StackSlot; - SDOperand Lo = DAG.getNode(ISD::ADD, TLI.getPointerTy(), StackSlot,WordOff); + SDValue Hi = StackSlot; + SDValue Lo = DAG.getNode(ISD::ADD, TLI.getPointerTy(), StackSlot,WordOff); if (TLI.isLittleEndian()) std::swap(Hi, Lo); // if signed map to unsigned space - SDOperand Op0Mapped; + SDValue Op0Mapped; if (isSigned) { // constant used to invert sign bit (signed to unsigned mapping) - SDOperand SignBit = DAG.getConstant(0x80000000u, MVT::i32); + SDValue SignBit = DAG.getConstant(0x80000000u, MVT::i32); Op0Mapped = DAG.getNode(ISD::XOR, MVT::i32, Op0, SignBit); } else { Op0Mapped = Op0; } // store the lo of the constructed double - based on integer input - SDOperand Store1 = DAG.getStore(DAG.getEntryNode(), + SDValue Store1 = DAG.getStore(DAG.getEntryNode(), Op0Mapped, Lo, NULL, 0); // initial hi portion of constructed double - SDOperand InitialHi = DAG.getConstant(0x43300000u, MVT::i32); + SDValue InitialHi = DAG.getConstant(0x43300000u, MVT::i32); // store the hi of the constructed double - biased exponent - SDOperand Store2=DAG.getStore(Store1, InitialHi, Hi, NULL, 0); + SDValue Store2=DAG.getStore(Store1, InitialHi, Hi, NULL, 0); // load the constructed double - SDOperand Load = DAG.getLoad(MVT::f64, Store2, StackSlot, NULL, 0); + SDValue Load = DAG.getLoad(MVT::f64, Store2, StackSlot, NULL, 0); // FP constant to bias correct the final result - SDOperand Bias = DAG.getConstantFP(isSigned ? + SDValue Bias = DAG.getConstantFP(isSigned ? BitsToDouble(0x4330000080000000ULL) : BitsToDouble(0x4330000000000000ULL), MVT::f64); // subtract the bias - SDOperand Sub = DAG.getNode(ISD::FSUB, MVT::f64, Load, Bias); + SDValue Sub = DAG.getNode(ISD::FSUB, MVT::f64, Load, Bias); // final result - SDOperand Result; + SDValue Result; // handle final rounding if (DestVT == MVT::f64) { // do nothing @@ -5503,13 +5823,13 @@ SDOperand SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, return Result; } assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); - SDOperand Tmp1 = DAG.getNode(ISD::SINT_TO_FP, DestVT, Op0); + SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, DestVT, Op0); - SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultType(Op0), Op0, + SDValue SignSet = DAG.getSetCC(TLI.getSetCCResultType(Op0), Op0, DAG.getConstant(0, Op0.getValueType()), ISD::SETLT); - SDOperand Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); - SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(), + SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); + SDValue CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(), SignSet, Four, Zero); // If the sign bit of the integer is set, the large number will be treated @@ -5526,18 +5846,21 @@ SDOperand SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, if (TLI.isLittleEndian()) FF <<= 32; static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); - SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); + SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); + unsigned Alignment = 1 << cast(CPIdx)->getAlignment(); CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset); - SDOperand FudgeInReg; + Alignment = std::min(Alignment, 4u); + SDValue FudgeInReg; if (DestVT == MVT::f32) FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, - PseudoSourceValue::getConstantPool(), 0); + PseudoSourceValue::getConstantPool(), 0, + false, Alignment); else { FudgeInReg = LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, DestVT, DAG.getEntryNode(), CPIdx, PseudoSourceValue::getConstantPool(), 0, - MVT::f32)); + MVT::f32, false, Alignment)); } return DAG.getNode(ISD::FADD, DestVT, Tmp1, FudgeInReg); @@ -5548,9 +5871,9 @@ SDOperand SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, /// we promote it. At this point, we know that the result and operand types are /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP /// operation that takes a larger input. -SDOperand SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDOperand LegalOp, - MVT DestVT, - bool isSigned) { +SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, + MVT DestVT, + bool isSigned) { // First step, figure out the appropriate *INT_TO_FP operation to use. MVT NewInTy = LegalOp.getValueType(); @@ -5603,9 +5926,9 @@ SDOperand SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDOperand LegalOp, /// we promote it. At this point, we know that the result and operand types are /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT /// operation that returns a larger result. -SDOperand SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDOperand LegalOp, - MVT DestVT, - bool isSigned) { +SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, + MVT DestVT, + bool isSigned) { // First step, figure out the appropriate FP_TO*INT operation to use. MVT NewOutTy = DestVT; @@ -5647,16 +5970,16 @@ SDOperand SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDOperand LegalOp, // Okay, we found the operation and type to use. - SDOperand Operation = DAG.getNode(OpToUse, NewOutTy, LegalOp); - + SDValue Operation = DAG.getNode(OpToUse, NewOutTy, LegalOp); + // If the operation produces an invalid type, it must be custom lowered. Use // the target lowering hooks to expand it. Just keep the low part of the // expanded operation, we know that we're truncating anyway. if (getTypeAction(NewOutTy) == Expand) { - Operation = SDOperand(TLI.ExpandOperationResult(Operation.Val, DAG), 0); - assert(Operation.Val && "Didn't return anything"); + Operation = SDValue(TLI.ReplaceNodeResults(Operation.getNode(), DAG), 0); + assert(Operation.getNode() && "Didn't return anything"); } - + // Truncate the result of the extended FP_TO_*INT operation to the desired // size. return DAG.getNode(ISD::TRUNCATE, DestVT, Operation); @@ -5664,10 +5987,10 @@ SDOperand SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDOperand LegalOp, /// ExpandBSWAP - Open code the operations for BSWAP of the specified operation. /// -SDOperand SelectionDAGLegalize::ExpandBSWAP(SDOperand Op) { +SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op) { MVT VT = Op.getValueType(); MVT SHVT = TLI.getShiftAmountTy(); - SDOperand Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; + SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; switch (VT.getSimpleVT()) { default: assert(0 && "Unhandled Expand type in BSWAP!"); abort(); case MVT::i16: @@ -5711,7 +6034,7 @@ SDOperand SelectionDAGLegalize::ExpandBSWAP(SDOperand Op) { /// ExpandBitCount - Expand the specified bitcount instruction into operations. /// -SDOperand SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDOperand Op) { +SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op) { switch (Opc) { default: assert(0 && "Cannot expand this yet!"); case ISD::CTPOP: { @@ -5725,8 +6048,8 @@ SDOperand SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDOperand Op) { unsigned len = VT.getSizeInBits(); for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { //x = (x & mask[i][len/8]) + (x >> (1 << i) & mask[i][len/8]) - SDOperand Tmp2 = DAG.getConstant(mask[i], VT); - SDOperand Tmp3 = DAG.getConstant(1ULL << i, ShVT); + SDValue Tmp2 = DAG.getConstant(mask[i], VT); + SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); Op = DAG.getNode(ISD::ADD, VT, DAG.getNode(ISD::AND, VT, Op, Tmp2), DAG.getNode(ISD::AND, VT, DAG.getNode(ISD::SRL, VT, Op, Tmp3),Tmp2)); @@ -5747,7 +6070,7 @@ SDOperand SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDOperand Op) { MVT ShVT = TLI.getShiftAmountTy(); unsigned len = VT.getSizeInBits(); for (unsigned i = 0; (1U << i) <= (len / 2); ++i) { - SDOperand Tmp3 = DAG.getConstant(1ULL << i, ShVT); + SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT); Op = DAG.getNode(ISD::OR, VT, Op, DAG.getNode(ISD::SRL, VT, Op, Tmp3)); } Op = DAG.getNode(ISD::XOR, VT, Op, DAG.getConstant(~0ULL, VT)); @@ -5759,8 +6082,8 @@ SDOperand SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDOperand Op) { // { return 32 - nlz(~x & (x-1)); } // see also http://www.hackersdelight.org/HDcode/ntz.cc MVT VT = Op.getValueType(); - SDOperand Tmp2 = DAG.getConstant(~0ULL, VT); - SDOperand Tmp3 = DAG.getNode(ISD::AND, VT, + SDValue Tmp2 = DAG.getConstant(~0ULL, VT); + SDValue Tmp3 = DAG.getNode(ISD::AND, VT, DAG.getNode(ISD::XOR, VT, Op, Tmp2), DAG.getNode(ISD::SUB, VT, Op, DAG.getConstant(1, VT))); // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. @@ -5774,21 +6097,21 @@ SDOperand SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDOperand Op) { } } -/// ExpandOp - Expand the specified SDOperand into its two component pieces +/// ExpandOp - Expand the specified SDValue into its two component pieces /// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, the -/// LegalizeNodes map is filled in for any results that are not expanded, the +/// LegalizedNodes map is filled in for any results that are not expanded, the /// ExpandedNodes map is filled in for any results that are expanded, and the /// Lo/Hi values are returned. -void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ +void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ MVT VT = Op.getValueType(); MVT NVT = TLI.getTypeToTransformTo(VT); - SDNode *Node = Op.Val; + SDNode *Node = Op.getNode(); assert(getTypeAction(VT) == Expand && "Not an expanded type!"); assert(((NVT.isInteger() && NVT.bitsLT(VT)) || VT.isFloatingPoint() || VT.isVector()) && "Cannot expand to FP value or to larger int value!"); // See if we already expanded it. - DenseMap >::iterator I + DenseMap >::iterator I = ExpandedNodes.find(Op); if (I != ExpandedNodes.end()) { Lo = I->second.first; @@ -5803,14 +6126,14 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ if (VT == MVT::ppcf128 && TLI.getOperationAction(ISD::FP_ROUND_INREG, VT) == TargetLowering::Custom) { - SDOperand SrcLo, SrcHi, Src; + SDValue SrcLo, SrcHi, Src; ExpandOp(Op.getOperand(0), SrcLo, SrcHi); Src = DAG.getNode(ISD::BUILD_PAIR, VT, SrcLo, SrcHi); - SDOperand Result = TLI.LowerOperation( + SDValue Result = TLI.LowerOperation( DAG.getNode(ISD::FP_ROUND_INREG, VT, Src, Op.getOperand(1)), DAG); - assert(Result.Val->getOpcode() == ISD::BUILD_PAIR); - Lo = Result.Val->getOperand(0); - Hi = Result.Val->getOperand(1); + assert(Result.getNode()->getOpcode() == ISD::BUILD_PAIR); + Lo = Result.getNode()->getOperand(0); + Hi = Result.getNode()->getOperand(1); break; } // fall through @@ -5822,11 +6145,10 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ abort(); case ISD::EXTRACT_ELEMENT: ExpandOp(Node->getOperand(0), Lo, Hi); - if (cast(Node->getOperand(1))->getValue()) + if (cast(Node->getOperand(1))->getZExtValue()) return ExpandOp(Hi, Lo, Hi); return ExpandOp(Lo, Lo, Hi); case ISD::EXTRACT_VECTOR_ELT: - assert(VT==MVT::i64 && "Do not know how to expand this operator!"); // ExpandEXTRACT_VECTOR_ELT tolerates invalid result types. Lo = ExpandEXTRACT_VECTOR_ELT(Op); return ExpandOp(Lo, Lo, Hi); @@ -5844,7 +6166,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::ConstantFP: { ConstantFPSDNode *CFP = cast(Node); if (CFP->getValueType(0) == MVT::ppcf128) { - APInt api = CFP->getValueAPF().convertToAPInt(); + APInt api = CFP->getValueAPF().bitcastToAPInt(); Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[1])), MVT::f64); Hi = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[0])), @@ -5868,7 +6190,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ break; } // FIXME: For now only expand i64,chain = MERGE_VALUES (x, y) - assert(Op.ResNo == 0 && Node->getNumValues() == 2 && + assert(Op.getResNo() == 0 && Node->getNumValues() == 2 && Op.getValue(1).getValueType() == MVT::Other && "unhandled MERGE_VALUES"); ExpandOp(Op.getOperand(0), Lo, Hi); @@ -5890,7 +6212,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::BSWAP: { ExpandOp(Node->getOperand(0), Lo, Hi); - SDOperand TempLo = DAG.getNode(ISD::BSWAP, NVT, Hi); + SDValue TempLo = DAG.getNode(ISD::BSWAP, NVT, Hi); Hi = DAG.getNode(ISD::BSWAP, NVT, Lo); Lo = TempLo; break; @@ -5907,11 +6229,11 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::CTLZ: { // ctlz (HL) -> ctlz(H) != 32 ? ctlz(H) : (ctlz(L)+32) ExpandOp(Node->getOperand(0), Lo, Hi); - SDOperand BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); - SDOperand HLZ = DAG.getNode(ISD::CTLZ, NVT, Hi); - SDOperand TopNotZero = DAG.getSetCC(TLI.getSetCCResultType(HLZ), HLZ, BitsC, + SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); + SDValue HLZ = DAG.getNode(ISD::CTLZ, NVT, Hi); + SDValue TopNotZero = DAG.getSetCC(TLI.getSetCCResultType(HLZ), HLZ, BitsC, ISD::SETNE); - SDOperand LowPart = DAG.getNode(ISD::CTLZ, NVT, Lo); + SDValue LowPart = DAG.getNode(ISD::CTLZ, NVT, Lo); LowPart = DAG.getNode(ISD::ADD, NVT, LowPart, BitsC); Lo = DAG.getNode(ISD::SELECT, NVT, TopNotZero, HLZ, LowPart); @@ -5922,11 +6244,11 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::CTTZ: { // cttz (HL) -> cttz(L) != 32 ? cttz(L) : (cttz(H)+32) ExpandOp(Node->getOperand(0), Lo, Hi); - SDOperand BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); - SDOperand LTZ = DAG.getNode(ISD::CTTZ, NVT, Lo); - SDOperand BotNotZero = DAG.getSetCC(TLI.getSetCCResultType(LTZ), LTZ, BitsC, + SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); + SDValue LTZ = DAG.getNode(ISD::CTTZ, NVT, Lo); + SDValue BotNotZero = DAG.getSetCC(TLI.getSetCCResultType(LTZ), LTZ, BitsC, ISD::SETNE); - SDOperand HiPart = DAG.getNode(ISD::CTTZ, NVT, Hi); + SDValue HiPart = DAG.getNode(ISD::CTTZ, NVT, Hi); HiPart = DAG.getNode(ISD::ADD, NVT, HiPart, BitsC); Lo = DAG.getNode(ISD::SELECT, NVT, BotNotZero, LTZ, HiPart); @@ -5935,8 +6257,8 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ } case ISD::VAARG: { - SDOperand Ch = Node->getOperand(0); // Legalize the chain. - SDOperand Ptr = Node->getOperand(1); // Legalize the pointer. + SDValue Ch = Node->getOperand(0); // Legalize the chain. + SDValue Ptr = Node->getOperand(1); // Legalize the pointer. Lo = DAG.getVAArg(NVT, Ch, Ptr, Node->getOperand(2)); Hi = DAG.getVAArg(NVT, Lo.getValue(1), Ptr, Node->getOperand(2)); @@ -5950,20 +6272,21 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::LOAD: { LoadSDNode *LD = cast(Node); - SDOperand Ch = LD->getChain(); // Legalize the chain. - SDOperand Ptr = LD->getBasePtr(); // Legalize the pointer. + SDValue Ch = LD->getChain(); // Legalize the chain. + SDValue Ptr = LD->getBasePtr(); // Legalize the pointer. ISD::LoadExtType ExtType = LD->getExtensionType(); + const Value *SV = LD->getSrcValue(); int SVOffset = LD->getSrcValueOffset(); unsigned Alignment = LD->getAlignment(); bool isVolatile = LD->isVolatile(); if (ExtType == ISD::NON_EXTLOAD) { - Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset, + Lo = DAG.getLoad(NVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment); if (VT == MVT::f32 || VT == MVT::f64) { // f32->i32 or f64->i64 one to one expansion. // Remember that we legalized the chain. - AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1))); + AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Lo.getValue(1))); // Recursively expand the new load. if (getTypeAction(NVT) == Expand) ExpandOp(Lo, Lo, Hi); @@ -5976,12 +6299,12 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ DAG.getIntPtrConstant(IncrementSize)); SVOffset += IncrementSize; Alignment = MinAlign(Alignment, IncrementSize); - Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset, + Hi = DAG.getLoad(NVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment); // Build a factor node to remember that this load is independent of the // other one. - SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), + SDValue TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), Hi.getValue(1)); // Remember that we legalized the chain. @@ -5994,24 +6317,24 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ if ((VT == MVT::f64 && EVT == MVT::f32) || (VT == MVT::ppcf128 && (EVT==MVT::f64 || EVT==MVT::f32))) { // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND - SDOperand Load = DAG.getLoad(EVT, Ch, Ptr, LD->getSrcValue(), + SDValue Load = DAG.getLoad(EVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment); // Remember that we legalized the chain. - AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Load.getValue(1))); + AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Load.getValue(1))); ExpandOp(DAG.getNode(ISD::FP_EXTEND, VT, Load), Lo, Hi); break; } if (EVT == NVT) - Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), + Lo = DAG.getLoad(NVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment); else - Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, LD->getSrcValue(), + Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, SV, SVOffset, EVT, isVolatile, Alignment); // Remember that we legalized the chain. - AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1))); + AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Lo.getValue(1))); if (ExtType == ISD::SEXTLOAD) { // The high part is obtained by SRA'ing all but one of the bits of the @@ -6032,7 +6355,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::AND: case ISD::OR: case ISD::XOR: { // Simple logical operators -> two trivial pieces. - SDOperand LL, LH, RL, RH; + SDValue LL, LH, RL, RH; ExpandOp(Node->getOperand(0), LL, LH); ExpandOp(Node->getOperand(1), RL, RH); Lo = DAG.getNode(Node->getOpcode(), NVT, LL, RL); @@ -6040,7 +6363,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ break; } case ISD::SELECT: { - SDOperand LL, LH, RL, RH; + SDValue LL, LH, RL, RH; ExpandOp(Node->getOperand(1), LL, LH); ExpandOp(Node->getOperand(2), RL, RH); if (getTypeAction(NVT) == Expand) @@ -6051,7 +6374,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ break; } case ISD::SELECT_CC: { - SDOperand TL, TH, FL, FH; + SDValue TL, TH, FL, FH; ExpandOp(Node->getOperand(2), TL, TH); ExpandOp(Node->getOperand(3), FL, FH); if (getTypeAction(NVT) == Expand) @@ -6092,7 +6415,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::TRUNCATE: { // The input value must be larger than this value. Expand *it*. - SDOperand NewLo; + SDValue NewLo; ExpandOp(Node->getOperand(0), NewLo, Hi); // The low part is now either the right size, or it is closer. If not the @@ -6104,7 +6427,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ } case ISD::BIT_CONVERT: { - SDOperand Tmp; + SDValue Tmp; if (TLI.getOperationAction(ISD::BIT_CONVERT, VT) == TargetLowering::Custom){ // If the target wants to, allow it to lower this itself. switch (getTypeAction(Node->getOperand(0).getValueType())) { @@ -6132,7 +6455,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ } // Turn this into a load/store pair by default. - if (Tmp.Val == 0) + if (Tmp.getNode() == 0) Tmp = EmitStackConvert(Node->getOperand(0), VT, VT); ExpandOp(Tmp, Lo, Hi); @@ -6143,30 +6466,52 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ assert(TLI.getOperationAction(ISD::READCYCLECOUNTER, VT) == TargetLowering::Custom && "Must custom expand ReadCycleCounter"); - SDOperand Tmp = TLI.LowerOperation(Op, DAG); - assert(Tmp.Val && "Node must be custom expanded!"); + SDValue Tmp = TLI.LowerOperation(Op, DAG); + assert(Tmp.getNode() && "Node must be custom expanded!"); ExpandOp(Tmp.getValue(0), Lo, Hi); - AddLegalizedOperand(SDOperand(Node, 1), // Remember we legalized the chain. + AddLegalizedOperand(SDValue(Node, 1), // Remember we legalized the chain. LegalizeOp(Tmp.getValue(1))); break; } - case ISD::ATOMIC_LCS: { - SDOperand Tmp = TLI.LowerOperation(Op, DAG); - assert(Tmp.Val && "Node must be custom expanded!"); + case ISD::ATOMIC_CMP_SWAP_64: { + // This operation does not need a loop. + SDValue Tmp = TLI.LowerOperation(Op, DAG); + assert(Tmp.getNode() && "Node must be custom expanded!"); ExpandOp(Tmp.getValue(0), Lo, Hi); - AddLegalizedOperand(SDOperand(Node, 1), // Remember we legalized the chain. + AddLegalizedOperand(SDValue(Node, 1), // Remember we legalized the chain. LegalizeOp(Tmp.getValue(1))); break; } - + case ISD::ATOMIC_LOAD_ADD_64: + case ISD::ATOMIC_LOAD_SUB_64: + case ISD::ATOMIC_LOAD_AND_64: + case ISD::ATOMIC_LOAD_OR_64: + case ISD::ATOMIC_LOAD_XOR_64: + case ISD::ATOMIC_LOAD_NAND_64: + case ISD::ATOMIC_SWAP_64: { + // These operations require a loop to be generated. We can't do that yet, + // so substitute a target-dependent pseudo and expand that later. + SDValue In2Lo, In2Hi, In2; + ExpandOp(Op.getOperand(2), In2Lo, In2Hi); + In2 = DAG.getNode(ISD::BUILD_PAIR, VT, In2Lo, In2Hi); + AtomicSDNode* Anode = cast(Node); + SDValue Replace = + DAG.getAtomic(Op.getOpcode(), Op.getOperand(0), Op.getOperand(1), In2, + Anode->getSrcValue(), Anode->getAlignment()); + SDValue Result = TLI.LowerOperation(Replace, DAG); + ExpandOp(Result.getValue(0), Lo, Hi); + // Remember that we legalized the chain. + AddLegalizedOperand(SDValue(Node,1), LegalizeOp(Result.getValue(1))); + break; + } // These operators cannot be expanded directly, emit them as calls to // library functions. case ISD::FP_TO_SINT: { if (TLI.getOperationAction(ISD::FP_TO_SINT, VT) == TargetLowering::Custom) { - SDOperand Op; + SDValue Op; switch (getTypeAction(Node->getOperand(0).getValueType())) { case Expand: assert(0 && "cannot expand FP!"); case Legal: Op = LegalizeOp(Node->getOperand(0)); break; @@ -6177,42 +6522,22 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ // Now that the custom expander is done, expand the result, which is still // VT. - if (Op.Val) { + if (Op.getNode()) { ExpandOp(Op, Lo, Hi); break; } } - RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; - if (VT == MVT::i64) { - if (Node->getOperand(0).getValueType() == MVT::f32) - LC = RTLIB::FPTOSINT_F32_I64; - else if (Node->getOperand(0).getValueType() == MVT::f64) - LC = RTLIB::FPTOSINT_F64_I64; - else if (Node->getOperand(0).getValueType() == MVT::f80) - LC = RTLIB::FPTOSINT_F80_I64; - else if (Node->getOperand(0).getValueType() == MVT::ppcf128) - LC = RTLIB::FPTOSINT_PPCF128_I64; - Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); - } else if (VT == MVT::i128) { - if (Node->getOperand(0).getValueType() == MVT::f32) - LC = RTLIB::FPTOSINT_F32_I128; - else if (Node->getOperand(0).getValueType() == MVT::f64) - LC = RTLIB::FPTOSINT_F64_I128; - else if (Node->getOperand(0).getValueType() == MVT::f80) - LC = RTLIB::FPTOSINT_F80_I128; - else if (Node->getOperand(0).getValueType() == MVT::ppcf128) - LC = RTLIB::FPTOSINT_PPCF128_I128; - Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); - } else { - assert(0 && "Unexpected uint-to-fp conversion!"); - } + RTLIB::Libcall LC = RTLIB::getFPTOSINT(Node->getOperand(0).getValueType(), + VT); + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected uint-to-fp conversion!"); + Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); break; } case ISD::FP_TO_UINT: { if (TLI.getOperationAction(ISD::FP_TO_UINT, VT) == TargetLowering::Custom) { - SDOperand Op; + SDValue Op; switch (getTypeAction(Node->getOperand(0).getValueType())) { case Expand: assert(0 && "cannot expand FP!"); case Legal: Op = LegalizeOp(Node->getOperand(0)); break; @@ -6222,46 +6547,26 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_UINT, VT, Op), DAG); // Now that the custom expander is done, expand the result. - if (Op.Val) { + if (Op.getNode()) { ExpandOp(Op, Lo, Hi); break; } } - RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; - if (VT == MVT::i64) { - if (Node->getOperand(0).getValueType() == MVT::f32) - LC = RTLIB::FPTOUINT_F32_I64; - else if (Node->getOperand(0).getValueType() == MVT::f64) - LC = RTLIB::FPTOUINT_F64_I64; - else if (Node->getOperand(0).getValueType() == MVT::f80) - LC = RTLIB::FPTOUINT_F80_I64; - else if (Node->getOperand(0).getValueType() == MVT::ppcf128) - LC = RTLIB::FPTOUINT_PPCF128_I64; - Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); - } else if (VT == MVT::i128) { - if (Node->getOperand(0).getValueType() == MVT::f32) - LC = RTLIB::FPTOUINT_F32_I128; - else if (Node->getOperand(0).getValueType() == MVT::f64) - LC = RTLIB::FPTOUINT_F64_I128; - else if (Node->getOperand(0).getValueType() == MVT::f80) - LC = RTLIB::FPTOUINT_F80_I128; - else if (Node->getOperand(0).getValueType() == MVT::ppcf128) - LC = RTLIB::FPTOUINT_PPCF128_I128; - Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); - } else { - assert(0 && "Unexpected uint-to-fp conversion!"); - } + RTLIB::Libcall LC = RTLIB::getFPTOUINT(Node->getOperand(0).getValueType(), + VT); + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); + Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); break; } case ISD::SHL: { // If the target wants custom lowering, do so. - SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1)); + SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); if (TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Custom) { - SDOperand Op = DAG.getNode(ISD::SHL, VT, Node->getOperand(0), ShiftAmt); + SDValue Op = DAG.getNode(ISD::SHL, VT, Node->getOperand(0), ShiftAmt); Op = TLI.LowerOperation(Op, DAG); - if (Op.Val) { + if (Op.getNode()) { // Now that the custom expander is done, expand the result, which is // still VT. ExpandOp(Op, Lo, Hi); @@ -6274,7 +6579,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ if (ConstantSDNode *ShAmt = dyn_cast(ShiftAmt)) { if (ShAmt->getAPIntValue() == 1 && TLI.isOperationLegal(ISD::ADDC, NVT) && TLI.isOperationLegal(ISD::ADDE, NVT)) { - SDOperand LoOps[2], HiOps[3]; + SDValue LoOps[2], HiOps[3]; ExpandOp(Node->getOperand(0), LoOps[0], HiOps[0]); SDVTList VTList = DAG.getVTList(LoOps[0].getValueType(), MVT::Flag); LoOps[1] = LoOps[0]; @@ -6307,11 +6612,11 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::SRA: { // If the target wants custom lowering, do so. - SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1)); + SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Custom) { - SDOperand Op = DAG.getNode(ISD::SRA, VT, Node->getOperand(0), ShiftAmt); + SDValue Op = DAG.getNode(ISD::SRA, VT, Node->getOperand(0), ShiftAmt); Op = TLI.LowerOperation(Op, DAG); - if (Op.Val) { + if (Op.getNode()) { // Now that the custom expander is done, expand the result, which is // still VT. ExpandOp(Op, Lo, Hi); @@ -6339,11 +6644,11 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::SRL: { // If the target wants custom lowering, do so. - SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1)); + SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); if (TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Custom) { - SDOperand Op = DAG.getNode(ISD::SRL, VT, Node->getOperand(0), ShiftAmt); + SDValue Op = DAG.getNode(ISD::SRL, VT, Node->getOperand(0), ShiftAmt); Op = TLI.LowerOperation(Op, DAG); - if (Op.Val) { + if (Op.getNode()) { // Now that the custom expander is done, expand the result, which is // still VT. ExpandOp(Op, Lo, Hi); @@ -6374,44 +6679,82 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ // If the target wants to custom expand this, let them. if (TLI.getOperationAction(Node->getOpcode(), VT) == TargetLowering::Custom) { - Op = TLI.LowerOperation(Op, DAG); - if (Op.Val) { - ExpandOp(Op, Lo, Hi); + SDValue Result = TLI.LowerOperation(Op, DAG); + if (Result.getNode()) { + ExpandOp(Result, Lo, Hi); break; } } - // Expand the subcomponents. - SDOperand LHSL, LHSH, RHSL, RHSH; + SDValue LHSL, LHSH, RHSL, RHSH; ExpandOp(Node->getOperand(0), LHSL, LHSH); ExpandOp(Node->getOperand(1), RHSL, RHSH); SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); - SDOperand LoOps[2], HiOps[3]; + SDValue LoOps[2], HiOps[3]; LoOps[0] = LHSL; LoOps[1] = RHSL; HiOps[0] = LHSH; HiOps[1] = RHSH; - if (Node->getOpcode() == ISD::ADD) { - Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); - HiOps[2] = Lo.getValue(1); - Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); + + //cascaded check to see if any smaller size has a a carry flag. + unsigned OpV = Node->getOpcode() == ISD::ADD ? ISD::ADDC : ISD::SUBC; + bool hasCarry = false; + for (unsigned BitSize = NVT.getSizeInBits(); BitSize != 0; BitSize /= 2) { + MVT AVT = MVT::getIntegerVT(BitSize); + if (TLI.isOperationLegal(OpV, AVT)) { + hasCarry = true; + break; + } + } + + if(hasCarry) { + if (Node->getOpcode() == ISD::ADD) { + Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); + HiOps[2] = Lo.getValue(1); + Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); + } else { + Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); + HiOps[2] = Lo.getValue(1); + Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); + } + break; } else { - Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); - HiOps[2] = Lo.getValue(1); - Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); + if (Node->getOpcode() == ISD::ADD) { + Lo = DAG.getNode(ISD::ADD, VTList, LoOps, 2); + Hi = DAG.getNode(ISD::ADD, VTList, HiOps, 2); + SDValue Cmp1 = DAG.getSetCC(TLI.getSetCCResultType(Lo), + Lo, LoOps[0], ISD::SETULT); + SDValue Carry1 = DAG.getNode(ISD::SELECT, NVT, Cmp1, + DAG.getConstant(1, NVT), + DAG.getConstant(0, NVT)); + SDValue Cmp2 = DAG.getSetCC(TLI.getSetCCResultType(Lo), + Lo, LoOps[1], ISD::SETULT); + SDValue Carry2 = DAG.getNode(ISD::SELECT, NVT, Cmp2, + DAG.getConstant(1, NVT), + Carry1); + Hi = DAG.getNode(ISD::ADD, NVT, Hi, Carry2); + } else { + Lo = DAG.getNode(ISD::SUB, VTList, LoOps, 2); + Hi = DAG.getNode(ISD::SUB, VTList, HiOps, 2); + SDValue Cmp = DAG.getSetCC(NVT, LoOps[0], LoOps[1], ISD::SETULT); + SDValue Borrow = DAG.getNode(ISD::SELECT, NVT, Cmp, + DAG.getConstant(1, NVT), + DAG.getConstant(0, NVT)); + Hi = DAG.getNode(ISD::SUB, NVT, Hi, Borrow); + } + break; } - break; } case ISD::ADDC: case ISD::SUBC: { // Expand the subcomponents. - SDOperand LHSL, LHSH, RHSL, RHSH; + SDValue LHSL, LHSH, RHSL, RHSH; ExpandOp(Node->getOperand(0), LHSL, LHSH); ExpandOp(Node->getOperand(1), RHSL, RHSH); SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); - SDOperand LoOps[2] = { LHSL, RHSL }; - SDOperand HiOps[3] = { LHSH, RHSH }; + SDValue LoOps[2] = { LHSL, RHSL }; + SDValue HiOps[3] = { LHSH, RHSH }; if (Node->getOpcode() == ISD::ADDC) { Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); @@ -6429,12 +6772,12 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::ADDE: case ISD::SUBE: { // Expand the subcomponents. - SDOperand LHSL, LHSH, RHSL, RHSH; + SDValue LHSL, LHSH, RHSL, RHSH; ExpandOp(Node->getOperand(0), LHSL, LHSH); ExpandOp(Node->getOperand(1), RHSL, RHSH); SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); - SDOperand LoOps[3] = { LHSL, RHSL, Node->getOperand(2) }; - SDOperand HiOps[3] = { LHSH, RHSH }; + SDValue LoOps[3] = { LHSL, RHSL, Node->getOperand(2) }; + SDValue HiOps[3] = { LHSH, RHSH }; Lo = DAG.getNode(Node->getOpcode(), VTList, LoOps, 3); HiOps[2] = Lo.getValue(1); @@ -6447,8 +6790,8 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ case ISD::MUL: { // If the target wants to custom expand this, let them. if (TLI.getOperationAction(ISD::MUL, VT) == TargetLowering::Custom) { - SDOperand New = TLI.LowerOperation(Op, DAG); - if (New.Val) { + SDValue New = TLI.LowerOperation(Op, DAG); + if (New.getNode()) { ExpandOp(New, Lo, Hi); break; } @@ -6459,7 +6802,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT); bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT); if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { - SDOperand LL, LH, RL, RH; + SDValue LL, LH, RL, RH; ExpandOp(Node->getOperand(0), LL, LH); ExpandOp(Node->getOperand(1), RL, RH); unsigned OuterBitSize = Op.getValueSizeInBits(); @@ -6473,7 +6816,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ if (HasUMUL_LOHI) { // We can emit a umul_lohi. Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); - Hi = SDOperand(Lo.Val, 1); + Hi = SDValue(Lo.getNode(), 1); break; } if (HasMULHU) { @@ -6488,7 +6831,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ if (HasSMUL_LOHI) { // We can emit a smul_lohi. Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); - Hi = SDOperand(Lo.Val, 1); + Hi = SDValue(Lo.getNode(), 1); break; } if (HasMULHS) { @@ -6500,7 +6843,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ } if (HasUMUL_LOHI) { // Lo,Hi = umul LHS, RHS. - SDOperand UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, + SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL); Lo = UMulLOHI; Hi = UMulLOHI.getValue(1); @@ -6566,7 +6909,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ RTLIB::DIV_PPCF128), Node, false, Hi); break; - case ISD::FP_EXTEND: + case ISD::FP_EXTEND: { if (VT == MVT::ppcf128) { assert(Node->getOperand(0).getValueType()==MVT::f32 || Node->getOperand(0).getValueType()==MVT::f64); @@ -6578,21 +6921,33 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); break; } - Lo = ExpandLibCall(RTLIB::FPEXT_F32_F64, Node, true, Hi); - break; - case ISD::FP_ROUND: - Lo = ExpandLibCall(RTLIB::FPROUND_F64_F32, Node, true, Hi); + RTLIB::Libcall LC = RTLIB::getFPEXT(Node->getOperand(0).getValueType(), VT); + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); + Lo = ExpandLibCall(LC, Node, true, Hi); break; - case ISD::FPOWI: - Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::POWI_F32, - RTLIB::POWI_F64, - RTLIB::POWI_F80, - RTLIB::POWI_PPCF128), - Node, false, Hi); + } + case ISD::FP_ROUND: { + RTLIB::Libcall LC = RTLIB::getFPROUND(Node->getOperand(0).getValueType(), + VT); + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); + Lo = ExpandLibCall(LC, Node, true, Hi); break; + } case ISD::FSQRT: case ISD::FSIN: - case ISD::FCOS: { + case ISD::FCOS: + case ISD::FLOG: + case ISD::FLOG2: + case ISD::FLOG10: + case ISD::FEXP: + case ISD::FEXP2: + case ISD::FTRUNC: + case ISD::FFLOOR: + case ISD::FCEIL: + case ISD::FRINT: + case ISD::FNEARBYINT: + case ISD::FPOW: + case ISD::FPOWI: { RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; switch(Node->getOpcode()) { case ISD::FSQRT: @@ -6607,6 +6962,54 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64, RTLIB::COS_F80, RTLIB::COS_PPCF128); break; + case ISD::FLOG: + LC = GetFPLibCall(VT, RTLIB::LOG_F32, RTLIB::LOG_F64, + RTLIB::LOG_F80, RTLIB::LOG_PPCF128); + break; + case ISD::FLOG2: + LC = GetFPLibCall(VT, RTLIB::LOG2_F32, RTLIB::LOG2_F64, + RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128); + break; + case ISD::FLOG10: + LC = GetFPLibCall(VT, RTLIB::LOG10_F32, RTLIB::LOG10_F64, + RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128); + break; + case ISD::FEXP: + LC = GetFPLibCall(VT, RTLIB::EXP_F32, RTLIB::EXP_F64, + RTLIB::EXP_F80, RTLIB::EXP_PPCF128); + break; + case ISD::FEXP2: + LC = GetFPLibCall(VT, RTLIB::EXP2_F32, RTLIB::EXP2_F64, + RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128); + break; + case ISD::FTRUNC: + LC = GetFPLibCall(VT, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, + RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128); + break; + case ISD::FFLOOR: + LC = GetFPLibCall(VT, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, + RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128); + break; + case ISD::FCEIL: + LC = GetFPLibCall(VT, RTLIB::CEIL_F32, RTLIB::CEIL_F64, + RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128); + break; + case ISD::FRINT: + LC = GetFPLibCall(VT, RTLIB::RINT_F32, RTLIB::RINT_F64, + RTLIB::RINT_F80, RTLIB::RINT_PPCF128); + break; + case ISD::FNEARBYINT: + LC = GetFPLibCall(VT, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, + RTLIB::NEARBYINT_F80, RTLIB::NEARBYINT_PPCF128); + break; + case ISD::FPOW: + LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, + RTLIB::POW_PPCF128); + break; + case ISD::FPOWI: + LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64, RTLIB::POWI_F80, + RTLIB::POWI_PPCF128); + break; default: assert(0 && "Unreachable!"); } Lo = ExpandLibCall(LC, Node, false, Hi); @@ -6614,7 +7017,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ } case ISD::FABS: { if (VT == MVT::ppcf128) { - SDOperand Tmp; + SDValue Tmp; ExpandOp(Node->getOperand(0), Lo, Tmp); Hi = DAG.getNode(ISD::FABS, NVT, Tmp); // lo = hi==fabs(hi) ? lo : -lo; @@ -6623,7 +7026,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ DAG.getCondCode(ISD::SETEQ)); break; } - SDOperand Mask = (VT == MVT::f64) + SDValue Mask = (VT == MVT::f64) ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT) : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT); Mask = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask); @@ -6640,7 +7043,7 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ Hi = DAG.getNode(ISD::FNEG, MVT::f64, Hi); break; } - SDOperand Mask = (VT == MVT::f64) + SDValue Mask = (VT == MVT::f64) ? DAG.getConstantFP(BitsToDouble(1ULL << 63), VT) : DAG.getConstantFP(BitsToFloat(1U << 31), VT); Mask = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask); @@ -6664,12 +7067,12 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ // Promote the operand if needed. Do this before checking for // ppcf128 so conversions of i16 and i8 work. if (getTypeAction(SrcVT) == Promote) { - SDOperand Tmp = PromoteOp(Node->getOperand(0)); + SDValue Tmp = PromoteOp(Node->getOperand(0)); Tmp = isSigned ? DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp.getValueType(), Tmp, DAG.getValueType(SrcVT)) : DAG.getZeroExtendInReg(Tmp, SrcVT); - Node = DAG.UpdateNodeOperands(Op, Tmp).Val; + Node = DAG.UpdateNodeOperands(Op, Tmp).getNode(); SrcVT = Node->getOperand(0).getValueType(); } @@ -6730,22 +7133,23 @@ void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){ // is a type that requires multi-step expansion. if (getTypeAction(NVT) != Expand && NVT != MVT::isVoid) { Lo = LegalizeOp(Lo); - if (Hi.Val) + if (Hi.getNode()) // Don't legalize the high part if it is expanded to a single node. Hi = LegalizeOp(Hi); } // Remember in a map if the values will be reused later. - bool isNew = ExpandedNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))); + bool isNew = + ExpandedNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second; assert(isNew && "Value already expanded?!?"); } /// SplitVectorOp - Given an operand of vector type, break it down into /// two smaller values, still of vector type. -void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, - SDOperand &Hi) { +void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo, + SDValue &Hi) { assert(Op.getValueType().isVector() && "Cannot split non-vector type!"); - SDNode *Node = Op.Val; + SDNode *Node = Op.getNode(); unsigned NumElements = Op.getValueType().getVectorNumElements(); assert(NumElements > 1 && "Cannot split a single element vector!"); @@ -6758,7 +7162,7 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, MVT NewVT_Hi = MVT::getVectorVT(NewEltVT, NewNumElts_Hi); // See if we already split it. - std::map >::iterator I + std::map >::iterator I = SplitNodes.find(Op); if (I != SplitNodes.end()) { Lo = I->second.first; @@ -6783,8 +7187,8 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, case ISD::INSERT_VECTOR_ELT: { if (ConstantSDNode *Idx = dyn_cast(Node->getOperand(2))) { SplitVectorOp(Node->getOperand(0), Lo, Hi); - unsigned Index = Idx->getValue(); - SDOperand ScalarOp = Node->getOperand(1); + unsigned Index = Idx->getZExtValue(); + SDValue ScalarOp = Node->getOperand(1); if (Index < NewNumElts_Lo) Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT_Lo, Lo, ScalarOp, DAG.getIntPtrConstant(Index)); @@ -6793,7 +7197,7 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, DAG.getIntPtrConstant(Index - NewNumElts_Lo)); break; } - SDOperand Tmp = PerformInsertVectorEltInMemory(Node->getOperand(0), + SDValue Tmp = PerformInsertVectorEltInMemory(Node->getOperand(0), Node->getOperand(1), Node->getOperand(2)); SplitVectorOp(Tmp, Lo, Hi); @@ -6801,21 +7205,21 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, } case ISD::VECTOR_SHUFFLE: { // Build the low part. - SDOperand Mask = Node->getOperand(2); - SmallVector Ops; + SDValue Mask = Node->getOperand(2); + SmallVector Ops; MVT PtrVT = TLI.getPointerTy(); // Insert all of the elements from the input that are needed. We use // buildvector of extractelement here because the input vectors will have // to be legalized, so this makes the code simpler. for (unsigned i = 0; i != NewNumElts_Lo; ++i) { - SDOperand IdxNode = Mask.getOperand(i); + SDValue IdxNode = Mask.getOperand(i); if (IdxNode.getOpcode() == ISD::UNDEF) { Ops.push_back(DAG.getNode(ISD::UNDEF, NewEltVT)); continue; } - unsigned Idx = cast(IdxNode)->getValue(); - SDOperand InVec = Node->getOperand(0); + unsigned Idx = cast(IdxNode)->getZExtValue(); + SDValue InVec = Node->getOperand(0); if (Idx >= NumElements) { InVec = Node->getOperand(1); Idx -= NumElements; @@ -6827,13 +7231,13 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, Ops.clear(); for (unsigned i = NewNumElts_Lo; i != NumElements; ++i) { - SDOperand IdxNode = Mask.getOperand(i); + SDValue IdxNode = Mask.getOperand(i); if (IdxNode.getOpcode() == ISD::UNDEF) { Ops.push_back(DAG.getNode(ISD::UNDEF, NewEltVT)); continue; } - unsigned Idx = cast(IdxNode)->getValue(); - SDOperand InVec = Node->getOperand(0); + unsigned Idx = cast(IdxNode)->getZExtValue(); + SDValue InVec = Node->getOperand(0); if (Idx >= NumElements) { InVec = Node->getOperand(1); Idx -= NumElements; @@ -6841,15 +7245,15 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewEltVT, InVec, DAG.getConstant(Idx, PtrVT))); } - Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Lo, &Ops[0], Ops.size()); + Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Hi, &Ops[0], Ops.size()); break; } case ISD::BUILD_VECTOR: { - SmallVector LoOps(Node->op_begin(), + SmallVector LoOps(Node->op_begin(), Node->op_begin()+NewNumElts_Lo); Lo = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Lo, &LoOps[0], LoOps.size()); - SmallVector HiOps(Node->op_begin()+NewNumElts_Lo, + SmallVector HiOps(Node->op_begin()+NewNumElts_Lo, Node->op_end()); Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Hi, &HiOps[0], HiOps.size()); break; @@ -6861,26 +7265,44 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, Lo = Node->getOperand(0); Hi = Node->getOperand(1); } else { - SmallVector LoOps(Node->op_begin(), - Node->op_begin()+NewNumSubvectors); + SmallVector LoOps(Node->op_begin(), + Node->op_begin()+NewNumSubvectors); Lo = DAG.getNode(ISD::CONCAT_VECTORS, NewVT_Lo, &LoOps[0], LoOps.size()); - SmallVector HiOps(Node->op_begin()+NewNumSubvectors, + SmallVector HiOps(Node->op_begin()+NewNumSubvectors, Node->op_end()); Hi = DAG.getNode(ISD::CONCAT_VECTORS, NewVT_Hi, &HiOps[0], HiOps.size()); } break; } + case ISD::EXTRACT_SUBVECTOR: { + SDValue Vec = Op.getOperand(0); + SDValue Idx = Op.getOperand(1); + MVT IdxVT = Idx.getValueType(); + + Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Lo, Vec, Idx); + ConstantSDNode *CIdx = dyn_cast(Idx); + if (CIdx) { + Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Hi, Vec, + DAG.getConstant(CIdx->getZExtValue() + NewNumElts_Lo, + IdxVT)); + } else { + Idx = DAG.getNode(ISD::ADD, IdxVT, Idx, + DAG.getConstant(NewNumElts_Lo, IdxVT)); + Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Hi, Vec, Idx); + } + break; + } case ISD::SELECT: { - SDOperand Cond = Node->getOperand(0); + SDValue Cond = Node->getOperand(0); - SDOperand LL, LH, RL, RH; + SDValue LL, LH, RL, RH; SplitVectorOp(Node->getOperand(1), LL, LH); SplitVectorOp(Node->getOperand(2), RL, RH); if (Cond.getValueType().isVector()) { // Handle a vector merge. - SDOperand CL, CH; + SDValue CL, CH; SplitVectorOp(Cond, CL, CH); Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, CL, LL, RL); Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, CH, LH, RH); @@ -6891,8 +7313,24 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, } break; } + case ISD::SELECT_CC: { + SDValue CondLHS = Node->getOperand(0); + SDValue CondRHS = Node->getOperand(1); + SDValue CondCode = Node->getOperand(4); + + SDValue LL, LH, RL, RH; + SplitVectorOp(Node->getOperand(2), LL, LH); + SplitVectorOp(Node->getOperand(3), RL, RH); + + // Handle a simple select with vector operands. + Lo = DAG.getNode(ISD::SELECT_CC, NewVT_Lo, CondLHS, CondRHS, + LL, RL, CondCode); + Hi = DAG.getNode(ISD::SELECT_CC, NewVT_Hi, CondLHS, CondRHS, + LH, RH, CondCode); + break; + } case ISD::VSETCC: { - SDOperand LL, LH, RL, RH; + SDValue LL, LH, RL, RH; SplitVectorOp(Node->getOperand(0), LL, LH); SplitVectorOp(Node->getOperand(1), RL, RH); Lo = DAG.getNode(ISD::VSETCC, NewVT_Lo, LL, RL, Node->getOperand(2)); @@ -6915,7 +7353,7 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, case ISD::UREM: case ISD::SREM: case ISD::FREM: { - SDOperand LL, LH, RL, RH; + SDValue LL, LH, RL, RH; SplitVectorOp(Node->getOperand(0), LL, LH); SplitVectorOp(Node->getOperand(1), RL, RH); @@ -6923,8 +7361,9 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, LH, RH); break; } + case ISD::FP_ROUND: case ISD::FPOWI: { - SDOperand L, H; + SDValue L, H; SplitVectorOp(Node->getOperand(0), L, H); Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, L, Node->getOperand(1)); @@ -6939,37 +7378,78 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, case ISD::FSQRT: case ISD::FSIN: case ISD::FCOS: + case ISD::FLOG: + case ISD::FLOG2: + case ISD::FLOG10: + case ISD::FEXP: + case ISD::FEXP2: case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: case ISD::SINT_TO_FP: - case ISD::UINT_TO_FP: { - SDOperand L, H; + case ISD::UINT_TO_FP: + case ISD::TRUNCATE: + case ISD::ANY_EXTEND: + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: + case ISD::FP_EXTEND: { + SDValue L, H; SplitVectorOp(Node->getOperand(0), L, H); Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, L); Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, H); break; } + case ISD::CONVERT_RNDSAT: { + ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); + SDValue L, H; + SplitVectorOp(Node->getOperand(0), L, H); + SDValue DTyOpL = DAG.getValueType(NewVT_Lo); + SDValue DTyOpH = DAG.getValueType(NewVT_Hi); + SDValue STyOpL = DAG.getValueType(L.getValueType()); + SDValue STyOpH = DAG.getValueType(H.getValueType()); + + SDValue RndOp = Node->getOperand(3); + SDValue SatOp = Node->getOperand(4); + + Lo = DAG.getConvertRndSat(NewVT_Lo, L, DTyOpL, STyOpL, + RndOp, SatOp, CvtCode); + Hi = DAG.getConvertRndSat(NewVT_Hi, H, DTyOpH, STyOpH, + RndOp, SatOp, CvtCode); + break; + } case ISD::LOAD: { LoadSDNode *LD = cast(Node); - SDOperand Ch = LD->getChain(); - SDOperand Ptr = LD->getBasePtr(); + SDValue Ch = LD->getChain(); + SDValue Ptr = LD->getBasePtr(); + ISD::LoadExtType ExtType = LD->getExtensionType(); const Value *SV = LD->getSrcValue(); int SVOffset = LD->getSrcValueOffset(); + MVT MemoryVT = LD->getMemoryVT(); unsigned Alignment = LD->getAlignment(); bool isVolatile = LD->isVolatile(); - Lo = DAG.getLoad(NewVT_Lo, Ch, Ptr, SV, SVOffset, isVolatile, Alignment); - unsigned IncrementSize = NewNumElts_Lo * NewEltVT.getSizeInBits()/8; + assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!"); + SDValue Offset = DAG.getNode(ISD::UNDEF, Ptr.getValueType()); + + MVT MemNewEltVT = MemoryVT.getVectorElementType(); + MVT MemNewVT_Lo = MVT::getVectorVT(MemNewEltVT, NewNumElts_Lo); + MVT MemNewVT_Hi = MVT::getVectorVT(MemNewEltVT, NewNumElts_Hi); + + Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, + NewVT_Lo, Ch, Ptr, Offset, + SV, SVOffset, MemNewVT_Lo, isVolatile, Alignment); + unsigned IncrementSize = NewNumElts_Lo * MemNewEltVT.getSizeInBits()/8; Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, DAG.getIntPtrConstant(IncrementSize)); SVOffset += IncrementSize; Alignment = MinAlign(Alignment, IncrementSize); - Hi = DAG.getLoad(NewVT_Hi, Ch, Ptr, SV, SVOffset, isVolatile, Alignment); + Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, + NewVT_Hi, Ch, Ptr, Offset, + SV, SVOffset, MemNewVT_Hi, isVolatile, Alignment); // Build a factor node to remember that this load is independent of the // other one. - SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), + SDValue TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1), Hi.getValue(1)); // Remember that we legalized the chain. @@ -6979,22 +7459,22 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, case ISD::BIT_CONVERT: { // We know the result is a vector. The input may be either a vector or a // scalar value. - SDOperand InOp = Node->getOperand(0); + SDValue InOp = Node->getOperand(0); if (!InOp.getValueType().isVector() || InOp.getValueType().getVectorNumElements() == 1) { // The input is a scalar or single-element vector. // Lower to a store/load so that it can be split. // FIXME: this could be improved probably. - SDOperand Ptr = DAG.CreateStackTemporary(InOp.getValueType()); - FrameIndexSDNode *FI = cast(Ptr.Val); + unsigned LdAlign = TLI.getTargetData()->getPrefTypeAlignment( + Op.getValueType().getTypeForMVT()); + SDValue Ptr = DAG.CreateStackTemporary(InOp.getValueType(), LdAlign); + int FI = cast(Ptr.getNode())->getIndex(); - SDOperand St = DAG.getStore(DAG.getEntryNode(), + SDValue St = DAG.getStore(DAG.getEntryNode(), InOp, Ptr, - PseudoSourceValue::getFixedStack(), - FI->getIndex()); + PseudoSourceValue::getFixedStack(FI), 0); InOp = DAG.getLoad(Op.getValueType(), St, Ptr, - PseudoSourceValue::getFixedStack(), - FI->getIndex()); + PseudoSourceValue::getFixedStack(FI), 0); } // Split the vector and convert each of the pieces now. SplitVectorOp(InOp, Lo, Hi); @@ -7014,17 +7494,17 @@ void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo, /// ScalarizeVectorOp - Given an operand of single-element vector type /// (e.g. v1f32), convert it into the equivalent operation that returns a /// scalar (e.g. f32) value. -SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) { +SDValue SelectionDAGLegalize::ScalarizeVectorOp(SDValue Op) { assert(Op.getValueType().isVector() && "Bad ScalarizeVectorOp invocation!"); - SDNode *Node = Op.Val; + SDNode *Node = Op.getNode(); MVT NewVT = Op.getValueType().getVectorElementType(); assert(Op.getValueType().getVectorNumElements() == 1); // See if we already scalarized it. - std::map::iterator I = ScalarizedNodes.find(Op); + std::map::iterator I = ScalarizedNodes.find(Op); if (I != ScalarizedNodes.end()) return I->second; - SDOperand Result; + SDValue Result; switch (Node->getOpcode()) { default: #ifndef NDEBUG @@ -7057,11 +7537,36 @@ SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) { case ISD::FSQRT: case ISD::FSIN: case ISD::FCOS: + case ISD::FLOG: + case ISD::FLOG2: + case ISD::FLOG10: + case ISD::FEXP: + case ISD::FEXP2: + case ISD::FP_TO_SINT: + case ISD::FP_TO_UINT: + case ISD::SINT_TO_FP: + case ISD::UINT_TO_FP: + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: + case ISD::ANY_EXTEND: + case ISD::TRUNCATE: + case ISD::FP_EXTEND: Result = DAG.getNode(Node->getOpcode(), NewVT, ScalarizeVectorOp(Node->getOperand(0))); break; + case ISD::CONVERT_RNDSAT: { + SDValue Op0 = ScalarizeVectorOp(Node->getOperand(0)); + Result = DAG.getConvertRndSat(NewVT, Op0, + DAG.getValueType(NewVT), + DAG.getValueType(Op0.getValueType()), + Node->getOperand(3), + Node->getOperand(4), + cast(Node)->getCvtCode()); + break; + } case ISD::FPOWI: + case ISD::FP_ROUND: Result = DAG.getNode(Node->getOpcode(), NewVT, ScalarizeVectorOp(Node->getOperand(0)), @@ -7069,13 +7574,22 @@ SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) { break; case ISD::LOAD: { LoadSDNode *LD = cast(Node); - SDOperand Ch = LegalizeOp(LD->getChain()); // Legalize the chain. - SDOperand Ptr = LegalizeOp(LD->getBasePtr()); // Legalize the pointer. - + SDValue Ch = LegalizeOp(LD->getChain()); // Legalize the chain. + SDValue Ptr = LegalizeOp(LD->getBasePtr()); // Legalize the pointer. + ISD::LoadExtType ExtType = LD->getExtensionType(); const Value *SV = LD->getSrcValue(); int SVOffset = LD->getSrcValueOffset(); - Result = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, - LD->isVolatile(), LD->getAlignment()); + MVT MemoryVT = LD->getMemoryVT(); + unsigned Alignment = LD->getAlignment(); + bool isVolatile = LD->isVolatile(); + + assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!"); + SDValue Offset = DAG.getNode(ISD::UNDEF, Ptr.getValueType()); + + Result = DAG.getLoad(ISD::UNINDEXED, ExtType, + NewVT, Ch, Ptr, Offset, SV, SVOffset, + MemoryVT.getVectorElementType(), + isVolatile, Alignment); // Remember that we legalized the chain. AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); @@ -7095,19 +7609,19 @@ SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) { break; case ISD::VECTOR_SHUFFLE: { // Figure out if the scalar is the LHS or RHS and return it. - SDOperand EltNum = Node->getOperand(2).getOperand(0); - if (cast(EltNum)->getValue()) + SDValue EltNum = Node->getOperand(2).getOperand(0); + if (cast(EltNum)->getZExtValue()) Result = ScalarizeVectorOp(Node->getOperand(1)); else Result = ScalarizeVectorOp(Node->getOperand(0)); break; } case ISD::EXTRACT_SUBVECTOR: - Result = Node->getOperand(0); - assert(Result.getValueType() == NewVT); + Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, Node->getOperand(0), + Node->getOperand(1)); break; case ISD::BIT_CONVERT: { - SDOperand Op0 = Op.getOperand(0); + SDValue Op0 = Op.getOperand(0); if (Op0.getValueType().getVectorNumElements() == 1) Op0 = ScalarizeVectorOp(Op0); Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op0); @@ -7118,9 +7632,16 @@ SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) { ScalarizeVectorOp(Op.getOperand(1)), ScalarizeVectorOp(Op.getOperand(2))); break; + case ISD::SELECT_CC: + Result = DAG.getNode(ISD::SELECT_CC, NewVT, Node->getOperand(0), + Node->getOperand(1), + ScalarizeVectorOp(Op.getOperand(2)), + ScalarizeVectorOp(Op.getOperand(3)), + Node->getOperand(4)); + break; case ISD::VSETCC: { - SDOperand Op0 = ScalarizeVectorOp(Op.getOperand(0)); - SDOperand Op1 = ScalarizeVectorOp(Op.getOperand(1)); + SDValue Op0 = ScalarizeVectorOp(Op.getOperand(0)); + SDValue Op1 = ScalarizeVectorOp(Op.getOperand(1)); Result = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(Op0), Op0, Op1, Op.getOperand(2)); Result = DAG.getNode(ISD::SELECT, NewVT, Result, @@ -7138,11 +7659,769 @@ SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) { } +SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) { + std::map::iterator I = WidenNodes.find(Op); + if (I != WidenNodes.end()) return I->second; + + MVT VT = Op.getValueType(); + assert(VT.isVector() && "Cannot widen non-vector type!"); + + SDValue Result; + SDNode *Node = Op.getNode(); + MVT EVT = VT.getVectorElementType(); + + unsigned NumElts = VT.getVectorNumElements(); + unsigned NewNumElts = WidenVT.getVectorNumElements(); + assert(NewNumElts > NumElts && "Cannot widen to smaller type!"); + assert(NewNumElts < 17); + + // When widen is called, it is assumed that it is more efficient to use a + // wide type. The default action is to widen to operation to a wider legal + // vector type and then do the operation if it is legal by calling LegalizeOp + // again. If there is no vector equivalent, we will unroll the operation, do + // it, and rebuild the vector. If most of the operations are vectorizible to + // the legal type, the resulting code will be more efficient. If this is not + // the case, the resulting code will preform badly as we end up generating + // code to pack/unpack the results. It is the function that calls widen + // that is responsible for seeing this doesn't happen. + switch (Node->getOpcode()) { + default: +#ifndef NDEBUG + Node->dump(&DAG); +#endif + assert(0 && "Unexpected operation in WidenVectorOp!"); + break; + case ISD::CopyFromReg: + assert(0 && "CopyFromReg doesn't need widening!"); + case ISD::Constant: + case ISD::ConstantFP: + // To build a vector of these elements, clients should call BuildVector + // and with each element instead of creating a node with a vector type + assert(0 && "Unexpected operation in WidenVectorOp!"); + case ISD::VAARG: + // Variable Arguments with vector types doesn't make any sense to me + assert(0 && "Unexpected operation in WidenVectorOp!"); + break; + case ISD::UNDEF: + Result = DAG.getNode(ISD::UNDEF, WidenVT); + break; + case ISD::BUILD_VECTOR: { + // Build a vector with undefined for the new nodes + SDValueVector NewOps(Node->op_begin(), Node->op_end()); + for (unsigned i = NumElts; i < NewNumElts; ++i) { + NewOps.push_back(DAG.getNode(ISD::UNDEF,EVT)); + } + Result = DAG.getNode(ISD::BUILD_VECTOR, WidenVT, &NewOps[0], NewOps.size()); + break; + } + case ISD::INSERT_VECTOR_ELT: { + SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, WidenVT, Tmp1, + Node->getOperand(1), Node->getOperand(2)); + break; + } + case ISD::VECTOR_SHUFFLE: { + SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); + // VECTOR_SHUFFLE 3rd operand must be a constant build vector that is + // used as permutation array. We build the vector here instead of widening + // because we don't want to legalize and have it turned to something else. + SDValue PermOp = Node->getOperand(2); + SDValueVector NewOps; + MVT PVT = PermOp.getValueType().getVectorElementType(); + for (unsigned i = 0; i < NumElts; ++i) { + if (PermOp.getOperand(i).getOpcode() == ISD::UNDEF) { + NewOps.push_back(PermOp.getOperand(i)); + } else { + unsigned Idx = + cast(PermOp.getOperand(i))->getZExtValue(); + if (Idx < NumElts) { + NewOps.push_back(PermOp.getOperand(i)); + } + else { + NewOps.push_back(DAG.getConstant(Idx + NewNumElts - NumElts, + PermOp.getOperand(i).getValueType())); + } + } + } + for (unsigned i = NumElts; i < NewNumElts; ++i) { + NewOps.push_back(DAG.getNode(ISD::UNDEF,PVT)); + } + + SDValue Tmp3 = DAG.getNode(ISD::BUILD_VECTOR, + MVT::getVectorVT(PVT, NewOps.size()), + &NewOps[0], NewOps.size()); + + Result = DAG.getNode(ISD::VECTOR_SHUFFLE, WidenVT, Tmp1, Tmp2, Tmp3); + break; + } + case ISD::LOAD: { + // If the load widen returns true, we can use a single load for the + // vector. Otherwise, it is returning a token factor for multiple + // loads. + SDValue TFOp; + if (LoadWidenVectorOp(Result, TFOp, Op, WidenVT)) + AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(1))); + else + AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(0))); + break; + } + + case ISD::BIT_CONVERT: { + SDValue Tmp1 = Node->getOperand(0); + // Converts between two different types so we need to determine + // the correct widen type for the input operand. + MVT TVT = Tmp1.getValueType(); + assert(TVT.isVector() && "can not widen non vector type"); + MVT TEVT = TVT.getVectorElementType(); + assert(WidenVT.getSizeInBits() % EVT.getSizeInBits() == 0 && + "can not widen bit bit convert that are not multiple of element type"); + MVT TWidenVT = MVT::getVectorVT(TEVT, + WidenVT.getSizeInBits()/EVT.getSizeInBits()); + Tmp1 = WidenVectorOp(Tmp1, TWidenVT); + assert(Tmp1.getValueType().getSizeInBits() == WidenVT.getSizeInBits()); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1); + + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + + case ISD::SINT_TO_FP: + case ISD::UINT_TO_FP: + case ISD::FP_TO_SINT: + case ISD::FP_TO_UINT: { + SDValue Tmp1 = Node->getOperand(0); + // Converts between two different types so we need to determine + // the correct widen type for the input operand. + MVT TVT = Tmp1.getValueType(); + assert(TVT.isVector() && "can not widen non vector type"); + MVT TEVT = TVT.getVectorElementType(); + MVT TWidenVT = MVT::getVectorVT(TEVT, NewNumElts); + Tmp1 = WidenVectorOp(Tmp1, TWidenVT); + assert(Tmp1.getValueType().getVectorNumElements() == NewNumElts); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1); + + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + + case ISD::FP_EXTEND: + assert(0 && "Case not implemented. Dynamically dead with 2 FP types!"); + case ISD::TRUNCATE: + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: + case ISD::ANY_EXTEND: + case ISD::FP_ROUND: + case ISD::SIGN_EXTEND_INREG: + case ISD::FABS: + case ISD::FNEG: + case ISD::FSQRT: + case ISD::FSIN: + case ISD::FCOS: + case ISD::CTPOP: + case ISD::CTTZ: + case ISD::CTLZ: { + // Unary op widening + SDValue Tmp1; + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + assert(Tmp1.getValueType() == WidenVT); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + case ISD::CONVERT_RNDSAT: { + SDValue RndOp = Node->getOperand(3); + SDValue SatOp = Node->getOperand(4); + + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + SDValue SrcOp = Node->getOperand(0); + + // Converts between two different types so we need to determine + // the correct widen type for the input operand. + MVT SVT = SrcOp.getValueType(); + assert(SVT.isVector() && "can not widen non vector type"); + MVT SEVT = SVT.getVectorElementType(); + MVT SWidenVT = MVT::getVectorVT(SEVT, NewNumElts); + + SrcOp = WidenVectorOp(SrcOp, SWidenVT); + assert(SrcOp.getValueType() == WidenVT); + SDValue DTyOp = DAG.getValueType(WidenVT); + SDValue STyOp = DAG.getValueType(SrcOp.getValueType()); + ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); + + Result = DAG.getConvertRndSat(WidenVT, SrcOp, DTyOp, STyOp, + RndOp, SatOp, CvtCode); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + case ISD::FPOW: + case ISD::FPOWI: + case ISD::ADD: + case ISD::SUB: + case ISD::MUL: + case ISD::MULHS: + case ISD::MULHU: + case ISD::AND: + case ISD::OR: + case ISD::XOR: + case ISD::FADD: + case ISD::FSUB: + case ISD::FMUL: + case ISD::SDIV: + case ISD::SREM: + case ISD::FDIV: + case ISD::FREM: + case ISD::FCOPYSIGN: + case ISD::UDIV: + case ISD::UREM: + case ISD::BSWAP: { + // Binary op widening + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); + assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code by first + // Widening to the right type and then unroll the beast. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + + case ISD::SHL: + case ISD::SRA: + case ISD::SRL: { + // Binary op with one non vector operand + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + assert(Tmp1.getValueType() == WidenVT); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Node->getOperand(1)); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + case ISD::EXTRACT_VECTOR_ELT: { + SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + assert(Tmp1.getValueType() == WidenVT); + Result = DAG.getNode(Node->getOpcode(), EVT, Tmp1, Node->getOperand(1)); + break; + } + case ISD::CONCAT_VECTORS: { + // We concurrently support only widen on a multiple of the incoming vector. + // We could widen on a multiple of the incoming operand if necessary. + unsigned NumConcat = NewNumElts / NumElts; + assert(NewNumElts % NumElts == 0 && "Can widen only a multiple of vector"); + std::vector UnOps(NumElts, DAG.getNode(ISD::UNDEF, + VT.getVectorElementType())); + SDValue UndefVal = DAG.getNode(ISD::BUILD_VECTOR, VT, + &UnOps[0], UnOps.size()); + SmallVector MOps; + MOps.push_back(Op); + for (unsigned i = 1; i != NumConcat; ++i) { + MOps.push_back(UndefVal); + } + Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, WidenVT, + &MOps[0], MOps.size())); + break; + } + case ISD::EXTRACT_SUBVECTOR: { + SDValue Tmp1 = Node->getOperand(0); + SDValue Idx = Node->getOperand(1); + ConstantSDNode *CIdx = dyn_cast(Idx); + if (CIdx && CIdx->getZExtValue() == 0) { + // Since we are access the start of the vector, the incoming + // vector type might be the proper. + MVT Tmp1VT = Tmp1.getValueType(); + if (Tmp1VT == WidenVT) + return Tmp1; + else { + unsigned Tmp1VTNumElts = Tmp1VT.getVectorNumElements(); + if (Tmp1VTNumElts < NewNumElts) + Result = WidenVectorOp(Tmp1, WidenVT); + else + Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, WidenVT, Tmp1, Idx); + } + } else if (NewNumElts % NumElts == 0) { + // Widen the extracted subvector. + unsigned NumConcat = NewNumElts / NumElts; + SDValue UndefVal = DAG.getNode(ISD::UNDEF, VT); + SmallVector MOps; + MOps.push_back(Op); + for (unsigned i = 1; i != NumConcat; ++i) { + MOps.push_back(UndefVal); + } + Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, WidenVT, + &MOps[0], MOps.size())); + } else { + assert(0 && "can not widen extract subvector"); + // This could be implemented using insert and build vector but I would + // like to see when this happens. + } + break; + } + + case ISD::SELECT: { + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + // Determine new condition widen type and widen + SDValue Cond1 = Node->getOperand(0); + MVT CondVT = Cond1.getValueType(); + assert(CondVT.isVector() && "can not widen non vector type"); + MVT CondEVT = CondVT.getVectorElementType(); + MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); + Cond1 = WidenVectorOp(Cond1, CondWidenVT); + assert(Cond1.getValueType() == CondWidenVT && "Condition not widen"); + + SDValue Tmp1 = WidenVectorOp(Node->getOperand(1), WidenVT); + SDValue Tmp2 = WidenVectorOp(Node->getOperand(2), WidenVT); + assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Cond1, Tmp1, Tmp2); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code by first + // Widening to the right type and then unroll the beast. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + + case ISD::SELECT_CC: { + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + // Determine new condition widen type and widen + SDValue Cond1 = Node->getOperand(0); + SDValue Cond2 = Node->getOperand(1); + MVT CondVT = Cond1.getValueType(); + assert(CondVT.isVector() && "can not widen non vector type"); + assert(CondVT == Cond2.getValueType() && "mismatch lhs/rhs"); + MVT CondEVT = CondVT.getVectorElementType(); + MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); + Cond1 = WidenVectorOp(Cond1, CondWidenVT); + Cond2 = WidenVectorOp(Cond2, CondWidenVT); + assert(Cond1.getValueType() == CondWidenVT && + Cond2.getValueType() == CondWidenVT && "condition not widen"); + + SDValue Tmp1 = WidenVectorOp(Node->getOperand(2), WidenVT); + SDValue Tmp2 = WidenVectorOp(Node->getOperand(3), WidenVT); + assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT && + "operands not widen"); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Cond1, Cond2, Tmp1, + Tmp2, Node->getOperand(4)); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code by first + // Widening to the right type and then unroll the beast. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + case ISD::VSETCC: { + // Determine widen for the operand + SDValue Tmp1 = Node->getOperand(0); + MVT TmpVT = Tmp1.getValueType(); + assert(TmpVT.isVector() && "can not widen non vector type"); + MVT TmpEVT = TmpVT.getVectorElementType(); + MVT TmpWidenVT = MVT::getVectorVT(TmpEVT, NewNumElts); + Tmp1 = WidenVectorOp(Tmp1, TmpWidenVT); + SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), TmpWidenVT); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2, + Node->getOperand(2)); + break; + } + case ISD::ATOMIC_CMP_SWAP_8: + case ISD::ATOMIC_CMP_SWAP_16: + case ISD::ATOMIC_CMP_SWAP_32: + case ISD::ATOMIC_CMP_SWAP_64: + case ISD::ATOMIC_LOAD_ADD_8: + case ISD::ATOMIC_LOAD_SUB_8: + case ISD::ATOMIC_LOAD_AND_8: + case ISD::ATOMIC_LOAD_OR_8: + case ISD::ATOMIC_LOAD_XOR_8: + case ISD::ATOMIC_LOAD_NAND_8: + case ISD::ATOMIC_LOAD_MIN_8: + case ISD::ATOMIC_LOAD_MAX_8: + case ISD::ATOMIC_LOAD_UMIN_8: + case ISD::ATOMIC_LOAD_UMAX_8: + case ISD::ATOMIC_SWAP_8: + case ISD::ATOMIC_LOAD_ADD_16: + case ISD::ATOMIC_LOAD_SUB_16: + case ISD::ATOMIC_LOAD_AND_16: + case ISD::ATOMIC_LOAD_OR_16: + case ISD::ATOMIC_LOAD_XOR_16: + case ISD::ATOMIC_LOAD_NAND_16: + case ISD::ATOMIC_LOAD_MIN_16: + case ISD::ATOMIC_LOAD_MAX_16: + case ISD::ATOMIC_LOAD_UMIN_16: + case ISD::ATOMIC_LOAD_UMAX_16: + case ISD::ATOMIC_SWAP_16: + case ISD::ATOMIC_LOAD_ADD_32: + case ISD::ATOMIC_LOAD_SUB_32: + case ISD::ATOMIC_LOAD_AND_32: + case ISD::ATOMIC_LOAD_OR_32: + case ISD::ATOMIC_LOAD_XOR_32: + case ISD::ATOMIC_LOAD_NAND_32: + case ISD::ATOMIC_LOAD_MIN_32: + case ISD::ATOMIC_LOAD_MAX_32: + case ISD::ATOMIC_LOAD_UMIN_32: + case ISD::ATOMIC_LOAD_UMAX_32: + case ISD::ATOMIC_SWAP_32: + case ISD::ATOMIC_LOAD_ADD_64: + case ISD::ATOMIC_LOAD_SUB_64: + case ISD::ATOMIC_LOAD_AND_64: + case ISD::ATOMIC_LOAD_OR_64: + case ISD::ATOMIC_LOAD_XOR_64: + case ISD::ATOMIC_LOAD_NAND_64: + case ISD::ATOMIC_LOAD_MIN_64: + case ISD::ATOMIC_LOAD_MAX_64: + case ISD::ATOMIC_LOAD_UMIN_64: + case ISD::ATOMIC_LOAD_UMAX_64: + case ISD::ATOMIC_SWAP_64: { + // For now, we assume that using vectors for these operations don't make + // much sense so we just split it. We return an empty result + SDValue X, Y; + SplitVectorOp(Op, X, Y); + return Result; + break; + } + + } // end switch (Node->getOpcode()) + + assert(Result.getNode() && "Didn't set a result!"); + if (Result != Op) + Result = LegalizeOp(Result); + + AddWidenedOperand(Op, Result); + return Result; +} + +// Utility function to find a legal vector type and its associated element +// type from a preferred width and whose vector type must be the same size +// as the VVT. +// TLI: Target lowering used to determine legal types +// Width: Preferred width of element type +// VVT: Vector value type whose size we must match. +// Returns VecEVT and EVT - the vector type and its associated element type +static void FindWidenVecType(TargetLowering &TLI, unsigned Width, MVT VVT, + MVT& EVT, MVT& VecEVT) { + // We start with the preferred width, make it a power of 2 and see if + // we can find a vector type of that width. If not, we reduce it by + // another power of 2. If we have widen the type, a vector of bytes should + // always be legal. + assert(TLI.isTypeLegal(VVT)); + unsigned EWidth = Width + 1; + do { + assert(EWidth > 0); + EWidth = (1 << Log2_32(EWidth-1)); + EVT = MVT::getIntegerVT(EWidth); + unsigned NumEVT = VVT.getSizeInBits()/EWidth; + VecEVT = MVT::getVectorVT(EVT, NumEVT); + } while (!TLI.isTypeLegal(VecEVT) || + VVT.getSizeInBits() != VecEVT.getSizeInBits()); +} + +SDValue SelectionDAGLegalize::genWidenVectorLoads(SDValueVector& LdChain, + SDValue Chain, + SDValue BasePtr, + const Value *SV, + int SVOffset, + unsigned Alignment, + bool isVolatile, + unsigned LdWidth, + MVT ResType) { + // We assume that we have good rules to handle loading power of two loads so + // we break down the operations to power of 2 loads. The strategy is to + // load the largest power of 2 that we can easily transform to a legal vector + // and then insert into that vector, and the cast the result into the legal + // vector that we want. This avoids unnecessary stack converts. + // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and + // the load is nonvolatile, we an use a wider load for the value. + // Find a vector length we can load a large chunk + MVT EVT, VecEVT; + unsigned EVTWidth; + FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); + EVTWidth = EVT.getSizeInBits(); + + SDValue LdOp = DAG.getLoad(EVT, Chain, BasePtr, SV, SVOffset, + isVolatile, Alignment); + SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecEVT, LdOp); + LdChain.push_back(LdOp.getValue(1)); + + // Check if we can load the element with one instruction + if (LdWidth == EVTWidth) { + return DAG.getNode(ISD::BIT_CONVERT, ResType, VecOp); + } + + // The vector element order is endianness dependent. + unsigned Idx = 1; + LdWidth -= EVTWidth; + unsigned Offset = 0; + + while (LdWidth > 0) { + unsigned Increment = EVTWidth / 8; + Offset += Increment; + BasePtr = DAG.getNode(ISD::ADD, BasePtr.getValueType(), BasePtr, + DAG.getIntPtrConstant(Increment)); + + if (LdWidth < EVTWidth) { + // Our current type we are using is too large, use a smaller size by + // using a smaller power of 2 + unsigned oEVTWidth = EVTWidth; + FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); + EVTWidth = EVT.getSizeInBits(); + // Readjust position and vector position based on new load type + Idx = Idx * (oEVTWidth/EVTWidth); + VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, VecOp); + } + + SDValue LdOp = DAG.getLoad(EVT, Chain, BasePtr, SV, + SVOffset+Offset, isVolatile, + MinAlign(Alignment, Offset)); + LdChain.push_back(LdOp.getValue(1)); + VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, VecEVT, VecOp, LdOp, + DAG.getIntPtrConstant(Idx++)); + + LdWidth -= EVTWidth; + } + + return DAG.getNode(ISD::BIT_CONVERT, ResType, VecOp); +} + +bool SelectionDAGLegalize::LoadWidenVectorOp(SDValue& Result, + SDValue& TFOp, + SDValue Op, + MVT NVT) { + // TODO: Add support for ConcatVec and the ability to load many vector + // types (e.g., v4i8). This will not work when a vector register + // to memory mapping is strange (e.g., vector elements are not + // stored in some sequential order). + + // It must be true that the widen vector type is bigger than where + // we need to load from. + LoadSDNode *LD = cast(Op.getNode()); + MVT LdVT = LD->getMemoryVT(); + assert(LdVT.isVector() && NVT.isVector()); + assert(LdVT.getVectorElementType() == NVT.getVectorElementType()); + + // Load information + SDValue Chain = LD->getChain(); + SDValue BasePtr = LD->getBasePtr(); + int SVOffset = LD->getSrcValueOffset(); + unsigned Alignment = LD->getAlignment(); + bool isVolatile = LD->isVolatile(); + const Value *SV = LD->getSrcValue(); + unsigned int LdWidth = LdVT.getSizeInBits(); + + // Load value as a large register + SDValueVector LdChain; + Result = genWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, + Alignment, isVolatile, LdWidth, NVT); + + if (LdChain.size() == 1) { + TFOp = LdChain[0]; + return true; + } + else { + TFOp=DAG.getNode(ISD::TokenFactor, MVT::Other, &LdChain[0], LdChain.size()); + return false; + } +} + + +void SelectionDAGLegalize::genWidenVectorStores(SDValueVector& StChain, + SDValue Chain, + SDValue BasePtr, + const Value *SV, + int SVOffset, + unsigned Alignment, + bool isVolatile, + SDValue ValOp, + unsigned StWidth) { + // Breaks the stores into a series of power of 2 width stores. For any + // width, we convert the vector to the vector of element size that we + // want to store. This avoids requiring a stack convert. + + // Find a width of the element type we can store with + MVT VVT = ValOp.getValueType(); + MVT EVT, VecEVT; + unsigned EVTWidth; + FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); + EVTWidth = EVT.getSizeInBits(); + + SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, ValOp); + SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EVT, VecOp, + DAG.getIntPtrConstant(0)); + SDValue StOp = DAG.getStore(Chain, EOp, BasePtr, SV, SVOffset, + isVolatile, Alignment); + StChain.push_back(StOp); + + // Check if we are done + if (StWidth == EVTWidth) { + return; + } + + unsigned Idx = 1; + StWidth -= EVTWidth; + unsigned Offset = 0; + + while (StWidth > 0) { + unsigned Increment = EVTWidth / 8; + Offset += Increment; + BasePtr = DAG.getNode(ISD::ADD, BasePtr.getValueType(), BasePtr, + DAG.getIntPtrConstant(Increment)); + + if (StWidth < EVTWidth) { + // Our current type we are using is too large, use a smaller size by + // using a smaller power of 2 + unsigned oEVTWidth = EVTWidth; + FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); + EVTWidth = EVT.getSizeInBits(); + // Readjust position and vector position based on new load type + Idx = Idx * (oEVTWidth/EVTWidth); + VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, VecOp); + } + + EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EVT, VecOp, + DAG.getIntPtrConstant(Idx++)); + StChain.push_back(DAG.getStore(Chain, EOp, BasePtr, SV, + SVOffset + Offset, isVolatile, + MinAlign(Alignment, Offset))); + StWidth -= EVTWidth; + } +} + + +SDValue SelectionDAGLegalize::StoreWidenVectorOp(StoreSDNode *ST, + SDValue Chain, + SDValue BasePtr) { + // TODO: It might be cleaner if we can use SplitVector and have more legal + // vector types that can be stored into memory (e.g., v4xi8 can + // be stored as a word). This will not work when a vector register + // to memory mapping is strange (e.g., vector elements are not + // stored in some sequential order). + + MVT StVT = ST->getMemoryVT(); + SDValue ValOp = ST->getValue(); + + // Check if we have widen this node with another value + std::map::iterator I = WidenNodes.find(ValOp); + if (I != WidenNodes.end()) + ValOp = I->second; + + MVT VVT = ValOp.getValueType(); + + // It must be true that we the widen vector type is bigger than where + // we need to store. + assert(StVT.isVector() && VVT.isVector()); + assert(StVT.getSizeInBits() < VVT.getSizeInBits()); + assert(StVT.getVectorElementType() == VVT.getVectorElementType()); + + // Store value + SDValueVector StChain; + genWidenVectorStores(StChain, Chain, BasePtr, ST->getSrcValue(), + ST->getSrcValueOffset(), ST->getAlignment(), + ST->isVolatile(), ValOp, StVT.getSizeInBits()); + if (StChain.size() == 1) + return StChain[0]; + else + return DAG.getNode(ISD::TokenFactor, MVT::Other,&StChain[0],StChain.size()); +} + + // SelectionDAG::Legalize - This is the entry point for the file. // void SelectionDAG::Legalize() { - if (ViewLegalizeDAGs) viewGraph(); - /// run - This is the main entry point to this class. /// SelectionDAGLegalize(*this).LegalizeDAG();