X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FLegalizeDAG.cpp;h=5866e1a9f97f233f0cb80b22991aef1eb6c4a7f5;hb=9a6b92de4c2207b427f3b9cd67cd122dafc5b6c6;hp=741e3f78c6cc2b78bc319dda12c38ef63817e46b;hpb=9468a9b6beed640eca64274c8dcc5aed3b94450b;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 741e3f78c6c..5866e1a9f97 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -86,12 +86,12 @@ class VISIBILITY_HIDDEN SelectionDAGLegalize { DenseMap PromotedNodes; /// ExpandedNodes - For nodes that need to be expanded this map indicates - /// which which operands are the expanded version of the input. This allows + /// which operands are the expanded version of the input. This allows /// us to avoid expanding the same node more than once. DenseMap > ExpandedNodes; /// SplitNodes - For vector nodes that need to be split, this map indicates - /// which which operands are the split version of the input. This allows us + /// which operands are the split version of the input. This allows us /// to avoid splitting the same node more than once. std::map > SplitNodes; @@ -100,6 +100,11 @@ class VISIBILITY_HIDDEN SelectionDAGLegalize { /// processed to the result. std::map ScalarizedNodes; + /// WidenNodes - For nodes that need to be widened from one vector type to + /// another, this contains the mapping of those that we have already widen. + /// This allows us to avoid widening more than once. + std::map WidenNodes; + void AddLegalizedOperand(SDValue From, SDValue To) { LegalizedNodes.insert(std::make_pair(From, To)); // If someone requests legalization of the new node, return itself. @@ -112,6 +117,12 @@ class VISIBILITY_HIDDEN SelectionDAGLegalize { // If someone requests legalization of the new node, return itself. LegalizedNodes.insert(std::make_pair(To, To)); } + void AddWidenedOperand(SDValue From, SDValue To) { + bool isNew = WidenNodes.insert(std::make_pair(From, To)).second; + assert(isNew && "Got into the map somehow?"); + // If someone requests legalization of the new node, return itself. + LegalizedNodes.insert(std::make_pair(To, To)); + } public: explicit SelectionDAGLegalize(SelectionDAG &DAG); @@ -163,12 +174,20 @@ private: /// ExpandOp - Expand the specified SDValue into its two component pieces /// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, - /// the LegalizeNodes map is filled in for any results that are not expanded, + /// the LegalizedNodes map is filled in for any results that are not expanded, /// the ExpandedNodes map is filled in for any results that are expanded, and /// the Lo/Hi values are returned. This applies to integer types and Vector /// types. void ExpandOp(SDValue O, SDValue &Lo, SDValue &Hi); + /// WidenVectorOp - Widen a vector operation to a wider type given by WidenVT + /// (e.g., v3i32 to v4i32). The produced value will have the correct value + /// for the existing elements but no guarantee is made about the new elements + /// at the end of the vector: it may be zero, ones, or garbage. This is useful + /// when we have an instruction operating on an illegal vector type and we + /// want to widen it to do the computation on a legal wider vector type. + SDValue WidenVectorOp(SDValue Op, MVT WidenVT); + /// SplitVectorOp - Given an operand of vector type, break it down into /// two smaller values. void SplitVectorOp(SDValue O, SDValue &Lo, SDValue &Hi); @@ -178,6 +197,62 @@ private: /// scalar (e.g. f32) value. SDValue ScalarizeVectorOp(SDValue O); + /// Useful 16 element vector type that is used to pass operands for widening. + typedef SmallVector SDValueVector; + + /// LoadWidenVectorOp - Load a vector for a wider type. Returns true if + /// the LdChain contains a single load and false if it contains a token + /// factor for multiple loads. It takes + /// Result: location to return the result + /// LdChain: location to return the load chain + /// Op: load operation to widen + /// NVT: widen vector result type we want for the load + bool LoadWidenVectorOp(SDValue& Result, SDValue& LdChain, + SDValue Op, MVT NVT); + + /// Helper genWidenVectorLoads - Helper function to generate a set of + /// loads to load a vector with a resulting wider type. It takes + /// LdChain: list of chains for the load we have generated + /// Chain: incoming chain for the ld vector + /// BasePtr: base pointer to load from + /// SV: memory disambiguation source value + /// SVOffset: memory disambiugation offset + /// Alignment: alignment of the memory + /// isVolatile: volatile load + /// LdWidth: width of memory that we want to load + /// ResType: the wider result result type for the resulting loaded vector + SDValue genWidenVectorLoads(SDValueVector& LdChain, SDValue Chain, + SDValue BasePtr, const Value *SV, + int SVOffset, unsigned Alignment, + bool isVolatile, unsigned LdWidth, + MVT ResType); + + /// StoreWidenVectorOp - Stores a widen vector into non widen memory + /// location. It takes + /// ST: store node that we want to replace + /// Chain: incoming store chain + /// BasePtr: base address of where we want to store into + SDValue StoreWidenVectorOp(StoreSDNode *ST, SDValue Chain, + SDValue BasePtr); + + /// Helper genWidenVectorStores - Helper function to generate a set of + /// stores to store a widen vector into non widen memory + // It takes + // StChain: list of chains for the stores we have generated + // Chain: incoming chain for the ld vector + // BasePtr: base pointer to load from + // SV: memory disambiguation source value + // SVOffset: memory disambiugation offset + // Alignment: alignment of the memory + // isVolatile: volatile lod + // ValOp: value to store + // StWidth: width of memory that we want to store + void genWidenVectorStores(SDValueVector& StChain, SDValue Chain, + SDValue BasePtr, const Value *SV, + int SVOffset, unsigned Alignment, + bool isVolatile, SDValue ValOp, + unsigned StWidth); + /// isShuffleLegal - Return non-null if a vector shuffle is legal with the /// specified mask and type. Targets can specify exactly which masks they /// support and the code generator is tasked with not creating illegal masks. @@ -193,6 +268,11 @@ private: SmallPtrSet &NodesLeadingTo); void LegalizeSetCCOperands(SDValue &LHS, SDValue &RHS, SDValue &CC); + void LegalizeSetCCCondCode(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC); + void LegalizeSetCC(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC) { + LegalizeSetCCOperands(LHS, RHS, CC); + LegalizeSetCCCondCode(VT, LHS, RHS, CC); + } SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned, SDValue &Hi); @@ -280,11 +360,10 @@ void SelectionDAGLegalize::LegalizeDAG() { // practice however, this causes us to run out of stack space on large basic // blocks. To avoid this problem, compute an ordering of the nodes where each // node is only legalized after all of its operands are legalized. - std::vector TopOrder; - unsigned N = DAG.AssignTopologicalOrder(TopOrder); - for (unsigned i = N; i != 0; --i) - HandleOp(SDValue(TopOrder[i-1], 0)); - TopOrder.clear(); + DAG.AssignTopologicalOrder(); + for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), + E = prior(DAG.allnodes_end()); I != next(E); ++I) + HandleOp(SDValue(I, 0)); // Finally, it's possible the root changed. Get the new root. SDValue OldRoot = DAG.getRoot(); @@ -296,6 +375,7 @@ void SelectionDAGLegalize::LegalizeDAG() { PromotedNodes.clear(); SplitNodes.clear(); ScalarizedNodes.clear(); + WidenNodes.clear(); // Remove dead nodes now. DAG.RemoveDeadNodes(); @@ -399,14 +479,27 @@ bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, return false; } -/// HandleOp - Legalize, Promote, or Expand the specified operand as +/// HandleOp - Legalize, Promote, Widen, or Expand the specified operand as /// appropriate for its type. void SelectionDAGLegalize::HandleOp(SDValue Op) { MVT VT = Op.getValueType(); switch (getTypeAction(VT)) { default: assert(0 && "Bad type action!"); case Legal: (void)LegalizeOp(Op); break; - case Promote: (void)PromoteOp(Op); break; + case Promote: + if (!VT.isVector()) { + (void)PromoteOp(Op); + break; + } + else { + // See if we can widen otherwise use Expand to either scalarize or split + MVT WidenVT = TLI.getWidenVectorType(VT); + if (WidenVT != MVT::Other) { + (void) WidenVectorOp(Op, WidenVT); + break; + } + // else fall thru to expand since we can't widen the vector + } case Expand: if (!VT.isVector()) { // If this is an illegal scalar, expand it into its two component @@ -420,7 +513,7 @@ void SelectionDAGLegalize::HandleOp(SDValue Op) { // scalar operation. (void)ScalarizeVectorOp(Op); } else { - // Otherwise, this is an illegal multiple element vector. + // This is an illegal multiple element vector. // Split it in half and legalize both parts. SDValue X, Y; SplitVectorOp(Op, X, Y); @@ -446,7 +539,7 @@ static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, if (!UseCP) { if (VT!=MVT::f64 && VT!=MVT::f32) assert(0 && "Invalid type expansion"); - return DAG.getConstant(LLVMC->getValueAPF().convertToAPInt(), + return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), (VT == MVT::f64) ? MVT::i64 : MVT::i32); } @@ -457,7 +550,7 @@ static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, if (CFP->isValueValidForType(SVT, CFP->getValueAPF()) && // Only do this if the target has a native EXTLOAD instruction from // smaller type. - TLI.isLoadXLegal(ISD::EXTLOAD, SVT) && + TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) && TLI.ShouldShrinkFPConstant(OrigVT)) { const Type *SType = SVT.getTypeForMVT(); LLVMC = cast(ConstantExpr::getFPTrunc(LLVMC, SType)); @@ -467,12 +560,13 @@ static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, } SDValue CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy()); + unsigned Alignment = 1 << cast(CPIdx)->getAlignment(); if (Extend) return DAG.getExtLoad(ISD::EXTLOAD, OrigVT, DAG.getEntryNode(), CPIdx, PseudoSourceValue::getConstantPool(), - 0, VT); + 0, VT, false, Alignment); return DAG.getLoad(OrigVT, DAG.getEntryNode(), CPIdx, - PseudoSourceValue::getConstantPool(), 0); + PseudoSourceValue::getConstantPool(), 0, false, Alignment); } @@ -737,7 +831,7 @@ PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx) { // Store the vector. SDValue Ch = DAG.getStore(DAG.getEntryNode(), Tmp1, StackPtr, - PseudoSourceValue::getFixedStack(SPFI), 0); + PseudoSourceValue::getFixedStack(SPFI), 0); // Truncate or zero extend offset to target pointer type. unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; @@ -799,7 +893,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { case ISD::TargetConstantPool: case ISD::TargetGlobalAddress: case ISD::TargetGlobalTLSAddress: - case ISD::TargetSymbol: + case ISD::TargetExternalSymbol: case ISD::VALUETYPE: case ISD::SRCVALUE: case ISD::MEMOPERAND: @@ -832,7 +926,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { case ISD::GLOBAL_OFFSET_TABLE: case ISD::GlobalAddress: case ISD::GlobalTLSAddress: - case ISD::Symbol: + case ISD::ExternalSymbol: case ISD::ConstantPool: case ISD::JumpTable: // Nothing to do. switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { @@ -1260,9 +1354,6 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { case TargetLowering::Custom: Result = TLI.LowerOperation(Result, DAG); break; - case TargetLowering::Expand: - Result = SDValue(TLI.ReplaceNodeResults(Op.getNode(), DAG),0); - break; case TargetLowering::Legal: break; } @@ -1415,6 +1506,12 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { default: assert(0 && "Cannot expand insert element operand"); case Legal: Tmp2 = LegalizeOp(Node->getOperand(1)); break; case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break; + case Expand: + // FIXME: An alternative would be to check to see if the target is not + // going to custom lower this operation, we could bitcast to half elt + // width and perform two inserts at that width, if that is legal. + Tmp2 = Node->getOperand(1); + break; } Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3); @@ -1430,6 +1527,8 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { break; } // FALLTHROUGH + case TargetLowering::Promote: + // Fall thru for vector case case TargetLowering::Expand: { // If the insert index is a constant, codegen this as a scalar_to_vector, // then a shuffle that inserts it into the right position in the vector. @@ -1572,6 +1671,26 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Result = ExpandEXTRACT_SUBVECTOR(Result); break; + case ISD::CONCAT_VECTORS: { + // Use extract/insert/build vector for now. We might try to be + // more clever later. + MVT PtrVT = TLI.getPointerTy(); + SmallVector Ops; + unsigned NumOperands = Node->getNumOperands(); + for (unsigned i=0; i < NumOperands; ++i) { + SDValue SubOp = Node->getOperand(i); + MVT VVT = SubOp.getNode()->getValueType(0); + MVT EltVT = VVT.getVectorElementType(); + unsigned NumSubElem = VVT.getVectorNumElements(); + for (unsigned j=0; j < NumSubElem; ++j) { + Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, SubOp, + DAG.getConstant(j, PtrVT))); + } + } + return LegalizeOp(DAG.getNode(ISD::BUILD_VECTOR, Node->getValueType(0), + &Ops[0], Ops.size())); + } + case ISD::CALLSEQ_START: { SDNode *CallEnd = FindCallEndFromCallStart(Node); @@ -1681,8 +1800,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // Chain the dynamic stack allocation so that it doesn't modify the stack // pointer when other instructions are using the stack. - Chain = DAG.getCALLSEQ_START(Chain, - DAG.getConstant(0, TLI.getPointerTy())); + Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true)); SDValue Size = Tmp2.getOperand(1); SDValue SP = DAG.getCopyFromReg(Chain, SPReg, VT); @@ -1696,11 +1814,8 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp1 = DAG.getNode(ISD::SUB, VT, SP, Size); // Value Chain = DAG.getCopyToReg(Chain, SPReg, Tmp1); // Output chain - Tmp2 = - DAG.getCALLSEQ_END(Chain, - DAG.getConstant(0, TLI.getPointerTy()), - DAG.getConstant(0, TLI.getPointerTy()), - SDValue()); + Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true), + DAG.getIntPtrConstant(0, true), SDValue()); Tmp1 = LegalizeOp(Tmp1); Tmp2 = LegalizeOp(Tmp2); @@ -1893,10 +2008,10 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp3 = Node->getOperand(3); // RHS Tmp4 = Node->getOperand(1); // CC - LegalizeSetCCOperands(Tmp2, Tmp3, Tmp4); + LegalizeSetCC(TLI.getSetCCResultType(Tmp2), Tmp2, Tmp3, Tmp4); LastCALLSEQ_END = DAG.getEntryNode(); - // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands, + // If we didn't get both a LHS and RHS back from LegalizeSetCC, // the LHS is a legal SETCC itself. In this case, we need to compare // the result against zero to select between true and false values. if (Tmp3.getNode() == 0) { @@ -1988,7 +2103,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // nice to have an effective generic way of getting these benefits... // Until such a way is found, don't insist on promoting i1 here. (SrcVT != MVT::i1 || - TLI.getLoadXAction(ExtType, MVT::i1) == TargetLowering::Promote)) { + TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) { // Promote to a byte-sized load if not loading an integral number of // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. unsigned NewWidth = SrcVT.getStoreSizeInBits(); @@ -2093,7 +2208,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp1 = LegalizeOp(Result); Tmp2 = LegalizeOp(Ch); } else { - switch (TLI.getLoadXAction(ExtType, SrcVT)) { + switch (TLI.getLoadExtAction(ExtType, SrcVT)) { default: assert(0 && "This action is not supported yet!"); case TargetLowering::Custom: isCustom = true; @@ -2360,7 +2475,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { if (CFP->getValueType(0) == MVT::f32 && getTypeAction(MVT::i32) == Legal) { Tmp3 = DAG.getConstant(CFP->getValueAPF(). - convertToAPInt().zextOrTrunc(32), + bitcastToAPInt().zextOrTrunc(32), MVT::i32); Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), SVOffset, isVolatile, Alignment); @@ -2368,7 +2483,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { } else if (CFP->getValueType(0) == MVT::f64) { // If this target supports 64-bit registers, do a single 64-bit store. if (getTypeAction(MVT::i64) == Legal) { - Tmp3 = DAG.getConstant(CFP->getValueAPF().convertToAPInt(). + Tmp3 = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). zextOrTrunc(64), MVT::i64); Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), SVOffset, isVolatile, Alignment); @@ -2377,7 +2492,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // Otherwise, if the target supports 32-bit registers, use 2 32-bit // stores. If the target supports neither 32- nor 64-bits, this // xform is certainly not worth it. - const APInt &IntVal =CFP->getValueAPF().convertToAPInt(); + const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt(); SDValue Lo = DAG.getConstant(APInt(IntVal).trunc(32), MVT::i32); SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32); if (TLI.isBigEndian()) std::swap(Lo, Hi); @@ -2431,14 +2546,16 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { break; } case Promote: - // Truncate the value and store the result. - Tmp3 = PromoteOp(ST->getValue()); - Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), - SVOffset, ST->getMemoryVT(), - isVolatile, Alignment); - break; - - case Expand: + if (!ST->getMemoryVT().isVector()) { + // Truncate the value and store the result. + Tmp3 = PromoteOp(ST->getValue()); + Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), + SVOffset, ST->getMemoryVT(), + isVolatile, Alignment); + break; + } + // Fall thru to expand for vector + case Expand: { unsigned IncrementSize = 0; SDValue Lo, Hi; @@ -2472,15 +2589,24 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Result = LegalizeOp(Result); break; } else { - SplitVectorOp(ST->getValue(), Lo, Hi); - IncrementSize = Lo.getNode()->getValueType(0).getVectorNumElements() * - EVT.getSizeInBits()/8; + // Check if we have widen this node with another value + std::map::iterator I = + WidenNodes.find(ST->getValue()); + if (I != WidenNodes.end()) { + Result = StoreWidenVectorOp(ST, Tmp1, Tmp2); + break; + } + else { + SplitVectorOp(ST->getValue(), Lo, Hi); + IncrementSize = Lo.getNode()->getValueType(0).getVectorNumElements() * + EVT.getSizeInBits()/8; + } } } else { ExpandOp(ST->getValue(), Lo, Hi); IncrementSize = Hi.getNode() ? Hi.getValueType().getSizeInBits()/8 : 0; - if (TLI.isBigEndian()) + if (Hi.getNode() && TLI.isBigEndian()) std::swap(Lo, Hi); } @@ -2503,6 +2629,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { SVOffset, isVolatile, Alignment); Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi); break; + } // case Expand } } else { switch (getTypeAction(ST->getValue().getValueType())) { @@ -2510,9 +2637,12 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp3 = LegalizeOp(ST->getValue()); break; case Promote: - // We can promote the value, the truncstore will still take care of it. - Tmp3 = PromoteOp(ST->getValue()); - break; + if (!ST->getValue().getValueType().isVector()) { + // We can promote the value, the truncstore will still take care of it. + Tmp3 = PromoteOp(ST->getValue()); + break; + } + // Vector case falls through to expand case Expand: // Just store the low part. This may become a non-trunc store, so make // sure to use getTruncStore, not UpdateNodeOperands below. @@ -2711,6 +2841,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the condition. break; case Promote: { + assert(!Node->getOperand(0).getValueType().isVector() && "not possible"); Tmp1 = PromoteOp(Node->getOperand(0)); // Promote the condition. // Make sure the condition is either zero or one. unsigned BitWidth = Tmp1.getValueSizeInBits(); @@ -2779,9 +2910,9 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp4 = LegalizeOp(Node->getOperand(3)); // False SDValue CC = Node->getOperand(4); - LegalizeSetCCOperands(Tmp1, Tmp2, CC); + LegalizeSetCC(TLI.getSetCCResultType(Tmp1), Tmp1, Tmp2, CC); - // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands, + // If we didn't get both a LHS and RHS back from LegalizeSetCC, // the LHS is a legal SETCC itself. In this case, we need to compare // the result against zero to select between true and false values. if (Tmp2.getNode() == 0) { @@ -2805,7 +2936,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Tmp1 = Node->getOperand(0); Tmp2 = Node->getOperand(1); Tmp3 = Node->getOperand(2); - LegalizeSetCCOperands(Tmp1, Tmp2, Tmp3); + LegalizeSetCC(Node->getValueType(0), Tmp1, Tmp2, Tmp3); // If we had to Expand the SetCC operands into a SELECT node, then it may // not always be possible to return a true LHS & RHS. In this case, just @@ -2968,7 +3099,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { } Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2); - + switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) { default: assert(0 && "BinOp legalize operation not supported"); case TargetLowering::Legal: break; @@ -2981,7 +3112,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // Fall through if the custom lower can't deal with the operation case TargetLowering::Expand: { MVT VT = Op.getValueType(); - + // See if multiply or divide can be lowered using two-result operations. SDVTList VTs = DAG.getVTList(VT, VT); if (Node->getOpcode() == ISD::MUL) { @@ -3010,25 +3141,29 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { } if (Node->getOpcode() == ISD::MULHS && TLI.isOperationLegal(ISD::SMUL_LOHI, VT)) { - Result = SDValue(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).getNode(), 1); + Result = SDValue(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).getNode(), + 1); break; } if (Node->getOpcode() == ISD::MULHU && TLI.isOperationLegal(ISD::UMUL_LOHI, VT)) { - Result = SDValue(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).getNode(), 1); + Result = SDValue(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).getNode(), + 1); break; } if (Node->getOpcode() == ISD::SDIV && TLI.isOperationLegal(ISD::SDIVREM, VT)) { - Result = SDValue(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).getNode(), 0); + Result = SDValue(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).getNode(), + 0); break; } if (Node->getOpcode() == ISD::UDIV && TLI.isOperationLegal(ISD::UDIVREM, VT)) { - Result = SDValue(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).getNode(), 0); + Result = SDValue(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).getNode(), + 0); break; } - + // Check to see if we have a libcall for this operator. RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; bool isSigned = false; @@ -3037,10 +3172,14 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { case ISD::SDIV: if (VT == MVT::i32) { LC = Node->getOpcode() == ISD::UDIV - ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32; + ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32; isSigned = Node->getOpcode() == ISD::SDIV; } break; + case ISD::MUL: + if (VT == MVT::i32) + LC = RTLIB::MUL_I32; + break; case ISD::FPOW: LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, RTLIB::POW_PPCF128); @@ -3052,7 +3191,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Result = ExpandLibCall(LC, Node, isSigned, Dummy); break; } - + assert(Node->getValueType(0).isVector() && "Cannot expand this binary operator!"); // Expand the operation into a bunch of nasty scalar code. @@ -3298,9 +3437,9 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { const Value *V = cast(Node->getOperand(2))->getValue(); SDValue VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2, V, 0); // Increment the pointer, VAList, to the next vaarg - Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList, - DAG.getConstant(VT.getSizeInBits()/8, - TLI.getPointerTy())); + Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList, + DAG.getConstant(TLI.getTargetData()->getABITypeSize(VT.getTypeForMVT()), + TLI.getPointerTy())); // Store the incremented VAList to the legalized pointer Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, V, 0); // Load the actual argument out of the pointer VAList @@ -3677,7 +3816,49 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { } } break; - + case ISD::CONVERT_RNDSAT: { + ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); + switch (CvtCode) { + default: assert(0 && "Unknown cvt code!"); + case ISD::CVT_SF: + case ISD::CVT_UF: + break; + case ISD::CVT_FF: + case ISD::CVT_FS: + case ISD::CVT_FU: + case ISD::CVT_SS: + case ISD::CVT_SU: + case ISD::CVT_US: + case ISD::CVT_UU: { + SDValue DTyOp = Node->getOperand(1); + SDValue STyOp = Node->getOperand(2); + SDValue RndOp = Node->getOperand(3); + SDValue SatOp = Node->getOperand(4); + switch (getTypeAction(Node->getOperand(0).getValueType())) { + case Expand: assert(0 && "Shouldn't need to expand other operators here!"); + case Legal: + Tmp1 = LegalizeOp(Node->getOperand(0)); + Result = DAG.UpdateNodeOperands(Result, Tmp1, DTyOp, STyOp, + RndOp, SatOp); + if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) == + TargetLowering::Custom) { + Tmp1 = TLI.LowerOperation(Result, DAG); + if (Tmp1.getNode()) Result = Tmp1; + } + break; + case Promote: + Result = PromoteOp(Node->getOperand(0)); + // For FP, make Op1 a i32 + + Result = DAG.getConvertRndSat(Result.getValueType(), Result, + DTyOp, STyOp, RndOp, SatOp, CvtCode); + break; + } + break; + } + } // end switch CvtCode + break; + } // Conversion operators. The source and destination have different types. case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: { @@ -3978,8 +4159,8 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { TargetLowering::ArgListTy Args; std::pair CallResult = TLI.LowerCallTo(Tmp1, Type::VoidTy, - false, false, false, CallingConv::C, false, - DAG.getSymbol("abort", TLI.getPointerTy()), + false, false, false, false, CallingConv::C, false, + DAG.getExternalSymbol("abort", TLI.getPointerTy()), Args, DAG); Result = CallResult.second; break; @@ -4095,6 +4276,19 @@ SDValue SelectionDAGLegalize::PromoteOp(SDValue Op) { break; } break; + case ISD::CONVERT_RNDSAT: { + ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); + assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || + CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || + CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && + "can only promote integers"); + Result = DAG.getConvertRndSat(NVT, Node->getOperand(0), + Node->getOperand(1), Node->getOperand(2), + Node->getOperand(3), Node->getOperand(4), + CvtCode); + break; + + } case ISD::BIT_CONVERT: Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), Node->getValueType(0)); @@ -4542,6 +4736,9 @@ SDValue SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDValue Op) { return Op; } break; + case TargetLowering::Promote: + assert(TVT.isVector() && "not vector type"); + // fall thru to expand since vectors are by default are promote case TargetLowering::Expand: break; } @@ -4670,6 +4867,8 @@ void SelectionDAGLegalize::LegalizeSetCCOperands(SDValue &LHS, DAG.getValueType(VT)); Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2, DAG.getValueType(VT)); + Tmp1 = LegalizeOp(Tmp1); // Relegalize new nodes. + Tmp2 = LegalizeOp(Tmp2); // Relegalize new nodes. break; } } @@ -4874,6 +5073,50 @@ void SelectionDAGLegalize::LegalizeSetCCOperands(SDValue &LHS, RHS = Tmp2; } +/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and +/// condition code CC on the current target. This routine assumes LHS and rHS +/// have already been legalized by LegalizeSetCCOperands. It expands SETCC with +/// illegal condition code into AND / OR of multiple SETCC values. +void SelectionDAGLegalize::LegalizeSetCCCondCode(MVT VT, + SDValue &LHS, SDValue &RHS, + SDValue &CC) { + MVT OpVT = LHS.getValueType(); + ISD::CondCode CCCode = cast(CC)->get(); + switch (TLI.getCondCodeAction(CCCode, OpVT)) { + default: assert(0 && "Unknown condition code action!"); + case TargetLowering::Legal: + // Nothing to do. + break; + case TargetLowering::Expand: { + ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; + unsigned Opc = 0; + switch (CCCode) { + default: assert(0 && "Don't know how to expand this condition!"); abort(); + case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETOLT: CC1 = ISD::SETLT; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETOLE: CC1 = ISD::SETLE; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETONE: CC1 = ISD::SETNE; CC2 = ISD::SETO; Opc = ISD::AND; break; + case ISD::SETUEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETUO; Opc = ISD::OR; break; + case ISD::SETUGT: CC1 = ISD::SETGT; CC2 = ISD::SETUO; Opc = ISD::OR; break; + case ISD::SETUGE: CC1 = ISD::SETGE; CC2 = ISD::SETUO; Opc = ISD::OR; break; + case ISD::SETULT: CC1 = ISD::SETLT; CC2 = ISD::SETUO; Opc = ISD::OR; break; + case ISD::SETULE: CC1 = ISD::SETLE; CC2 = ISD::SETUO; Opc = ISD::OR; break; + case ISD::SETUNE: CC1 = ISD::SETNE; CC2 = ISD::SETUO; Opc = ISD::OR; break; + // FIXME: Implement more expansions. + } + + SDValue SetCC1 = DAG.getSetCC(VT, LHS, RHS, CC1); + SDValue SetCC2 = DAG.getSetCC(VT, LHS, RHS, CC2); + LHS = DAG.getNode(Opc, VT, SetCC1, SetCC2); + RHS = SDValue(); + CC = SDValue(); + break; + } + } +} + /// EmitStackConvert - Emit a store/load combination to the stack. This stores /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does /// a load from the stack slot to DestVT, extending it if needed. @@ -4997,8 +5240,10 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { } Constant *CP = ConstantVector::get(CV); SDValue CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); + unsigned Alignment = 1 << cast(CPIdx)->getAlignment(); return DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, - PseudoSourceValue::getConstantPool(), 0); + PseudoSourceValue::getConstantPool(), 0, + false, Alignment); } if (SplatValue.getNode()) { // Splat of one value? @@ -5293,15 +5538,14 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, Entry.isZExt = !isSigned; Args.push_back(Entry); } - - SDValue Callee = DAG.getSymbol(TLI.getLibcallName(LC), - TLI.getPointerTy()); + SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), + TLI.getPointerTy()); // Splice the libcall in wherever FindInputOutputChains tells us to. const Type *RetTy = Node->getValueType(0).getTypeForMVT(); std::pair CallInfo = - TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, CallingConv::C, - false, Callee, Args, DAG); + TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, false, + CallingConv::C, false, Callee, Args, DAG); // Legalize the call sequence, starting with the chain. This will advance // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that @@ -5401,7 +5645,8 @@ ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source) { DestTy.getVectorNumElements() / 2); SDValue LoResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, Lo); SDValue HiResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, Hi); - return LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, DestTy, LoResult, HiResult)); + return LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, DestTy, LoResult, + HiResult)); } // Special case for i32 source to take advantage of UINTTOFP_I32_F32, etc. @@ -5419,6 +5664,22 @@ ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source) { Hi = Source; } + // Check to see if the target has a custom way to lower this. If so, use + // it. (Note we've already expanded the operand in this case.) + switch (TLI.getOperationAction(ISD::UINT_TO_FP, SourceVT)) { + default: assert(0 && "This action not implemented for this operation!"); + case TargetLowering::Legal: + case TargetLowering::Expand: + break; // This case is handled below. + case TargetLowering::Custom: { + SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::UINT_TO_FP, DestTy, + Source), DAG); + if (NV.getNode()) + return LegalizeOp(NV); + break; // The target decided this was legal after all + } + } + // If this is unsigned, and not supported, first perform the conversion to // signed, then adjust the result if the sign bit is set. SDValue SignedConv = ExpandIntToFP(true, DestTy, Source); @@ -5434,17 +5695,20 @@ ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source) { static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); + unsigned Alignment = 1 << cast(CPIdx)->getAlignment(); CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset); + Alignment = std::min(Alignment, 4u); SDValue FudgeInReg; if (DestTy == MVT::f32) FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, - PseudoSourceValue::getConstantPool(), 0); + PseudoSourceValue::getConstantPool(), 0, + false, Alignment); else if (DestTy.bitsGT(MVT::f32)) // FIXME: Avoid the extend by construction the right constantpool? FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, DestTy, DAG.getEntryNode(), CPIdx, PseudoSourceValue::getConstantPool(), 0, - MVT::f32); + MVT::f32, false, Alignment); else assert(0 && "Unexpected conversion"); @@ -5583,17 +5847,20 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); + unsigned Alignment = 1 << cast(CPIdx)->getAlignment(); CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset); + Alignment = std::min(Alignment, 4u); SDValue FudgeInReg; if (DestVT == MVT::f32) FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, - PseudoSourceValue::getConstantPool(), 0); + PseudoSourceValue::getConstantPool(), 0, + false, Alignment); else { FudgeInReg = LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, DestVT, DAG.getEntryNode(), CPIdx, PseudoSourceValue::getConstantPool(), 0, - MVT::f32)); + MVT::f32, false, Alignment)); } return DAG.getNode(ISD::FADD, DestVT, Tmp1, FudgeInReg); @@ -5832,7 +6099,7 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op) { /// ExpandOp - Expand the specified SDValue into its two component pieces /// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, the -/// LegalizeNodes map is filled in for any results that are not expanded, the +/// LegalizedNodes map is filled in for any results that are not expanded, the /// ExpandedNodes map is filled in for any results that are expanded, and the /// Lo/Hi values are returned. void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ @@ -5882,7 +6149,6 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ return ExpandOp(Hi, Lo, Hi); return ExpandOp(Lo, Lo, Hi); case ISD::EXTRACT_VECTOR_ELT: - assert(VT==MVT::i64 && "Do not know how to expand this operator!"); // ExpandEXTRACT_VECTOR_ELT tolerates invalid result types. Lo = ExpandEXTRACT_VECTOR_ELT(Op); return ExpandOp(Lo, Lo, Hi); @@ -5900,7 +6166,7 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ case ISD::ConstantFP: { ConstantFPSDNode *CFP = cast(Node); if (CFP->getValueType(0) == MVT::ppcf128) { - APInt api = CFP->getValueAPF().convertToAPInt(); + APInt api = CFP->getValueAPF().bitcastToAPInt(); Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[1])), MVT::f64); Hi = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[0])), @@ -6208,11 +6474,8 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ break; } - // FIXME: should the LOAD_BIN and SWAP atomics get here too? Probably. - case ISD::ATOMIC_CMP_SWAP_8: - case ISD::ATOMIC_CMP_SWAP_16: - case ISD::ATOMIC_CMP_SWAP_32: case ISD::ATOMIC_CMP_SWAP_64: { + // This operation does not need a loop. SDValue Tmp = TLI.LowerOperation(Op, DAG); assert(Tmp.getNode() && "Node must be custom expanded!"); ExpandOp(Tmp.getValue(0), Lo, Hi); @@ -6221,7 +6484,28 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ break; } - + case ISD::ATOMIC_LOAD_ADD_64: + case ISD::ATOMIC_LOAD_SUB_64: + case ISD::ATOMIC_LOAD_AND_64: + case ISD::ATOMIC_LOAD_OR_64: + case ISD::ATOMIC_LOAD_XOR_64: + case ISD::ATOMIC_LOAD_NAND_64: + case ISD::ATOMIC_SWAP_64: { + // These operations require a loop to be generated. We can't do that yet, + // so substitute a target-dependent pseudo and expand that later. + SDValue In2Lo, In2Hi, In2; + ExpandOp(Op.getOperand(2), In2Lo, In2Hi); + In2 = DAG.getNode(ISD::BUILD_PAIR, VT, In2Lo, In2Hi); + AtomicSDNode* Anode = cast(Node); + SDValue Replace = + DAG.getAtomic(Op.getOpcode(), Op.getOperand(0), Op.getOperand(1), In2, + Anode->getSrcValue(), Anode->getAlignment()); + SDValue Result = TLI.LowerOperation(Replace, DAG); + ExpandOp(Result.getValue(0), Lo, Hi); + // Remember that we legalized the chain. + AddLegalizedOperand(SDValue(Node,1), LegalizeOp(Result.getValue(1))); + break; + } // These operators cannot be expanded directly, emit them as calls to // library functions. @@ -6401,7 +6685,6 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ break; } } - // Expand the subcomponents. SDValue LHSL, LHSH, RHSL, RHSH; ExpandOp(Node->getOperand(0), LHSL, LHSH); @@ -6412,16 +6695,55 @@ void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ LoOps[1] = RHSL; HiOps[0] = LHSH; HiOps[1] = RHSH; - if (Node->getOpcode() == ISD::ADD) { - Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); - HiOps[2] = Lo.getValue(1); - Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); + + //cascaded check to see if any smaller size has a a carry flag. + unsigned OpV = Node->getOpcode() == ISD::ADD ? ISD::ADDC : ISD::SUBC; + bool hasCarry = false; + for (unsigned BitSize = NVT.getSizeInBits(); BitSize != 0; BitSize /= 2) { + MVT AVT = MVT::getIntegerVT(BitSize); + if (TLI.isOperationLegal(OpV, AVT)) { + hasCarry = true; + break; + } + } + + if(hasCarry) { + if (Node->getOpcode() == ISD::ADD) { + Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2); + HiOps[2] = Lo.getValue(1); + Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3); + } else { + Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); + HiOps[2] = Lo.getValue(1); + Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); + } + break; } else { - Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2); - HiOps[2] = Lo.getValue(1); - Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3); + if (Node->getOpcode() == ISD::ADD) { + Lo = DAG.getNode(ISD::ADD, VTList, LoOps, 2); + Hi = DAG.getNode(ISD::ADD, VTList, HiOps, 2); + SDValue Cmp1 = DAG.getSetCC(TLI.getSetCCResultType(Lo), + Lo, LoOps[0], ISD::SETULT); + SDValue Carry1 = DAG.getNode(ISD::SELECT, NVT, Cmp1, + DAG.getConstant(1, NVT), + DAG.getConstant(0, NVT)); + SDValue Cmp2 = DAG.getSetCC(TLI.getSetCCResultType(Lo), + Lo, LoOps[1], ISD::SETULT); + SDValue Carry2 = DAG.getNode(ISD::SELECT, NVT, Cmp2, + DAG.getConstant(1, NVT), + Carry1); + Hi = DAG.getNode(ISD::ADD, NVT, Hi, Carry2); + } else { + Lo = DAG.getNode(ISD::SUB, VTList, LoOps, 2); + Hi = DAG.getNode(ISD::SUB, VTList, HiOps, 2); + SDValue Cmp = DAG.getSetCC(NVT, LoOps[0], LoOps[1], ISD::SETULT); + SDValue Borrow = DAG.getNode(ISD::SELECT, NVT, Cmp, + DAG.getConstant(1, NVT), + DAG.getConstant(0, NVT)); + Hi = DAG.getNode(ISD::SUB, NVT, Hi, Borrow); + } + break; } - break; } case ISD::ADDC: @@ -6943,16 +7265,34 @@ void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo, Lo = Node->getOperand(0); Hi = Node->getOperand(1); } else { - SmallVector LoOps(Node->op_begin(), - Node->op_begin()+NewNumSubvectors); + SmallVector LoOps(Node->op_begin(), + Node->op_begin()+NewNumSubvectors); Lo = DAG.getNode(ISD::CONCAT_VECTORS, NewVT_Lo, &LoOps[0], LoOps.size()); - SmallVector HiOps(Node->op_begin()+NewNumSubvectors, + SmallVector HiOps(Node->op_begin()+NewNumSubvectors, Node->op_end()); Hi = DAG.getNode(ISD::CONCAT_VECTORS, NewVT_Hi, &HiOps[0], HiOps.size()); } break; } + case ISD::EXTRACT_SUBVECTOR: { + SDValue Vec = Op.getOperand(0); + SDValue Idx = Op.getOperand(1); + MVT IdxVT = Idx.getValueType(); + + Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Lo, Vec, Idx); + ConstantSDNode *CIdx = dyn_cast(Idx); + if (CIdx) { + Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Hi, Vec, + DAG.getConstant(CIdx->getZExtValue() + NewNumElts_Lo, + IdxVT)); + } else { + Idx = DAG.getNode(ISD::ADD, IdxVT, Idx, + DAG.getConstant(NewNumElts_Lo, IdxVT)); + Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Hi, Vec, Idx); + } + break; + } case ISD::SELECT: { SDValue Cond = Node->getOperand(0); @@ -7059,6 +7399,24 @@ void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo, Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, H); break; } + case ISD::CONVERT_RNDSAT: { + ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); + SDValue L, H; + SplitVectorOp(Node->getOperand(0), L, H); + SDValue DTyOpL = DAG.getValueType(NewVT_Lo); + SDValue DTyOpH = DAG.getValueType(NewVT_Hi); + SDValue STyOpL = DAG.getValueType(L.getValueType()); + SDValue STyOpH = DAG.getValueType(H.getValueType()); + + SDValue RndOp = Node->getOperand(3); + SDValue SatOp = Node->getOperand(4); + + Lo = DAG.getConvertRndSat(NewVT_Lo, L, DTyOpL, STyOpL, + RndOp, SatOp, CvtCode); + Hi = DAG.getConvertRndSat(NewVT_Hi, H, DTyOpH, STyOpH, + RndOp, SatOp, CvtCode); + break; + } case ISD::LOAD: { LoadSDNode *LD = cast(Node); SDValue Ch = LD->getChain(); @@ -7197,6 +7555,16 @@ SDValue SelectionDAGLegalize::ScalarizeVectorOp(SDValue Op) { NewVT, ScalarizeVectorOp(Node->getOperand(0))); break; + case ISD::CONVERT_RNDSAT: { + SDValue Op0 = ScalarizeVectorOp(Node->getOperand(0)); + Result = DAG.getConvertRndSat(NewVT, Op0, + DAG.getValueType(NewVT), + DAG.getValueType(Op0.getValueType()), + Node->getOperand(3), + Node->getOperand(4), + cast(Node)->getCvtCode()); + break; + } case ISD::FPOWI: case ISD::FP_ROUND: Result = DAG.getNode(Node->getOpcode(), @@ -7249,8 +7617,8 @@ SDValue SelectionDAGLegalize::ScalarizeVectorOp(SDValue Op) { break; } case ISD::EXTRACT_SUBVECTOR: - Result = Node->getOperand(0); - assert(Result.getValueType() == NewVT); + Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, Node->getOperand(0), + Node->getOperand(1)); break; case ISD::BIT_CONVERT: { SDValue Op0 = Op.getOperand(0); @@ -7291,6 +7659,766 @@ SDValue SelectionDAGLegalize::ScalarizeVectorOp(SDValue Op) { } +SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) { + std::map::iterator I = WidenNodes.find(Op); + if (I != WidenNodes.end()) return I->second; + + MVT VT = Op.getValueType(); + assert(VT.isVector() && "Cannot widen non-vector type!"); + + SDValue Result; + SDNode *Node = Op.getNode(); + MVT EVT = VT.getVectorElementType(); + + unsigned NumElts = VT.getVectorNumElements(); + unsigned NewNumElts = WidenVT.getVectorNumElements(); + assert(NewNumElts > NumElts && "Cannot widen to smaller type!"); + assert(NewNumElts < 17); + + // When widen is called, it is assumed that it is more efficient to use a + // wide type. The default action is to widen to operation to a wider legal + // vector type and then do the operation if it is legal by calling LegalizeOp + // again. If there is no vector equivalent, we will unroll the operation, do + // it, and rebuild the vector. If most of the operations are vectorizible to + // the legal type, the resulting code will be more efficient. If this is not + // the case, the resulting code will preform badly as we end up generating + // code to pack/unpack the results. It is the function that calls widen + // that is responsible for seeing this doesn't happen. + switch (Node->getOpcode()) { + default: +#ifndef NDEBUG + Node->dump(&DAG); +#endif + assert(0 && "Unexpected operation in WidenVectorOp!"); + break; + case ISD::CopyFromReg: + assert(0 && "CopyFromReg doesn't need widening!"); + case ISD::Constant: + case ISD::ConstantFP: + // To build a vector of these elements, clients should call BuildVector + // and with each element instead of creating a node with a vector type + assert(0 && "Unexpected operation in WidenVectorOp!"); + case ISD::VAARG: + // Variable Arguments with vector types doesn't make any sense to me + assert(0 && "Unexpected operation in WidenVectorOp!"); + break; + case ISD::UNDEF: + Result = DAG.getNode(ISD::UNDEF, WidenVT); + break; + case ISD::BUILD_VECTOR: { + // Build a vector with undefined for the new nodes + SDValueVector NewOps(Node->op_begin(), Node->op_end()); + for (unsigned i = NumElts; i < NewNumElts; ++i) { + NewOps.push_back(DAG.getNode(ISD::UNDEF,EVT)); + } + Result = DAG.getNode(ISD::BUILD_VECTOR, WidenVT, &NewOps[0], NewOps.size()); + break; + } + case ISD::INSERT_VECTOR_ELT: { + SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, WidenVT, Tmp1, + Node->getOperand(1), Node->getOperand(2)); + break; + } + case ISD::VECTOR_SHUFFLE: { + SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); + // VECTOR_SHUFFLE 3rd operand must be a constant build vector that is + // used as permutation array. We build the vector here instead of widening + // because we don't want to legalize and have it turned to something else. + SDValue PermOp = Node->getOperand(2); + SDValueVector NewOps; + MVT PVT = PermOp.getValueType().getVectorElementType(); + for (unsigned i = 0; i < NumElts; ++i) { + if (PermOp.getOperand(i).getOpcode() == ISD::UNDEF) { + NewOps.push_back(PermOp.getOperand(i)); + } else { + unsigned Idx = + cast(PermOp.getOperand(i))->getZExtValue(); + if (Idx < NumElts) { + NewOps.push_back(PermOp.getOperand(i)); + } + else { + NewOps.push_back(DAG.getConstant(Idx + NewNumElts - NumElts, + PermOp.getOperand(i).getValueType())); + } + } + } + for (unsigned i = NumElts; i < NewNumElts; ++i) { + NewOps.push_back(DAG.getNode(ISD::UNDEF,PVT)); + } + + SDValue Tmp3 = DAG.getNode(ISD::BUILD_VECTOR, + MVT::getVectorVT(PVT, NewOps.size()), + &NewOps[0], NewOps.size()); + + Result = DAG.getNode(ISD::VECTOR_SHUFFLE, WidenVT, Tmp1, Tmp2, Tmp3); + break; + } + case ISD::LOAD: { + // If the load widen returns true, we can use a single load for the + // vector. Otherwise, it is returning a token factor for multiple + // loads. + SDValue TFOp; + if (LoadWidenVectorOp(Result, TFOp, Op, WidenVT)) + AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(1))); + else + AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(0))); + break; + } + + case ISD::BIT_CONVERT: { + SDValue Tmp1 = Node->getOperand(0); + // Converts between two different types so we need to determine + // the correct widen type for the input operand. + MVT TVT = Tmp1.getValueType(); + assert(TVT.isVector() && "can not widen non vector type"); + MVT TEVT = TVT.getVectorElementType(); + assert(WidenVT.getSizeInBits() % EVT.getSizeInBits() == 0 && + "can not widen bit bit convert that are not multiple of element type"); + MVT TWidenVT = MVT::getVectorVT(TEVT, + WidenVT.getSizeInBits()/EVT.getSizeInBits()); + Tmp1 = WidenVectorOp(Tmp1, TWidenVT); + assert(Tmp1.getValueType().getSizeInBits() == WidenVT.getSizeInBits()); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1); + + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + + case ISD::SINT_TO_FP: + case ISD::UINT_TO_FP: + case ISD::FP_TO_SINT: + case ISD::FP_TO_UINT: { + SDValue Tmp1 = Node->getOperand(0); + // Converts between two different types so we need to determine + // the correct widen type for the input operand. + MVT TVT = Tmp1.getValueType(); + assert(TVT.isVector() && "can not widen non vector type"); + MVT TEVT = TVT.getVectorElementType(); + MVT TWidenVT = MVT::getVectorVT(TEVT, NewNumElts); + Tmp1 = WidenVectorOp(Tmp1, TWidenVT); + assert(Tmp1.getValueType().getVectorNumElements() == NewNumElts); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1); + + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + + case ISD::FP_EXTEND: + assert(0 && "Case not implemented. Dynamically dead with 2 FP types!"); + case ISD::TRUNCATE: + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: + case ISD::ANY_EXTEND: + case ISD::FP_ROUND: + case ISD::SIGN_EXTEND_INREG: + case ISD::FABS: + case ISD::FNEG: + case ISD::FSQRT: + case ISD::FSIN: + case ISD::FCOS: + case ISD::CTPOP: + case ISD::CTTZ: + case ISD::CTLZ: { + // Unary op widening + SDValue Tmp1; + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + assert(Tmp1.getValueType() == WidenVT); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + case ISD::CONVERT_RNDSAT: { + SDValue RndOp = Node->getOperand(3); + SDValue SatOp = Node->getOperand(4); + + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + SDValue SrcOp = Node->getOperand(0); + + // Converts between two different types so we need to determine + // the correct widen type for the input operand. + MVT SVT = SrcOp.getValueType(); + assert(SVT.isVector() && "can not widen non vector type"); + MVT SEVT = SVT.getVectorElementType(); + MVT SWidenVT = MVT::getVectorVT(SEVT, NewNumElts); + + SrcOp = WidenVectorOp(SrcOp, SWidenVT); + assert(SrcOp.getValueType() == WidenVT); + SDValue DTyOp = DAG.getValueType(WidenVT); + SDValue STyOp = DAG.getValueType(SrcOp.getValueType()); + ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); + + Result = DAG.getConvertRndSat(WidenVT, SrcOp, DTyOp, STyOp, + RndOp, SatOp, CvtCode); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + case ISD::FPOW: + case ISD::FPOWI: + case ISD::ADD: + case ISD::SUB: + case ISD::MUL: + case ISD::MULHS: + case ISD::MULHU: + case ISD::AND: + case ISD::OR: + case ISD::XOR: + case ISD::FADD: + case ISD::FSUB: + case ISD::FMUL: + case ISD::SDIV: + case ISD::SREM: + case ISD::FDIV: + case ISD::FREM: + case ISD::FCOPYSIGN: + case ISD::UDIV: + case ISD::UREM: + case ISD::BSWAP: { + // Binary op widening + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); + assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code by first + // Widening to the right type and then unroll the beast. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + + case ISD::SHL: + case ISD::SRA: + case ISD::SRL: { + // Binary op with one non vector operand + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + assert(Tmp1.getValueType() == WidenVT); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Node->getOperand(1)); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + case ISD::EXTRACT_VECTOR_ELT: { + SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); + assert(Tmp1.getValueType() == WidenVT); + Result = DAG.getNode(Node->getOpcode(), EVT, Tmp1, Node->getOperand(1)); + break; + } + case ISD::CONCAT_VECTORS: { + // We concurrently support only widen on a multiple of the incoming vector. + // We could widen on a multiple of the incoming operand if necessary. + unsigned NumConcat = NewNumElts / NumElts; + assert(NewNumElts % NumElts == 0 && "Can widen only a multiple of vector"); + std::vector UnOps(NumElts, DAG.getNode(ISD::UNDEF, + VT.getVectorElementType())); + SDValue UndefVal = DAG.getNode(ISD::BUILD_VECTOR, VT, + &UnOps[0], UnOps.size()); + SmallVector MOps; + MOps.push_back(Op); + for (unsigned i = 1; i != NumConcat; ++i) { + MOps.push_back(UndefVal); + } + Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, WidenVT, + &MOps[0], MOps.size())); + break; + } + case ISD::EXTRACT_SUBVECTOR: { + SDValue Tmp1 = Node->getOperand(0); + SDValue Idx = Node->getOperand(1); + ConstantSDNode *CIdx = dyn_cast(Idx); + if (CIdx && CIdx->getZExtValue() == 0) { + // Since we are access the start of the vector, the incoming + // vector type might be the proper. + MVT Tmp1VT = Tmp1.getValueType(); + if (Tmp1VT == WidenVT) + return Tmp1; + else { + unsigned Tmp1VTNumElts = Tmp1VT.getVectorNumElements(); + if (Tmp1VTNumElts < NewNumElts) + Result = WidenVectorOp(Tmp1, WidenVT); + else + Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, WidenVT, Tmp1, Idx); + } + } else if (NewNumElts % NumElts == 0) { + // Widen the extracted subvector. + unsigned NumConcat = NewNumElts / NumElts; + SDValue UndefVal = DAG.getNode(ISD::UNDEF, VT); + SmallVector MOps; + MOps.push_back(Op); + for (unsigned i = 1; i != NumConcat; ++i) { + MOps.push_back(UndefVal); + } + Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, WidenVT, + &MOps[0], MOps.size())); + } else { + assert(0 && "can not widen extract subvector"); + // This could be implemented using insert and build vector but I would + // like to see when this happens. + } + break; + } + + case ISD::SELECT: { + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + // Determine new condition widen type and widen + SDValue Cond1 = Node->getOperand(0); + MVT CondVT = Cond1.getValueType(); + assert(CondVT.isVector() && "can not widen non vector type"); + MVT CondEVT = CondVT.getVectorElementType(); + MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); + Cond1 = WidenVectorOp(Cond1, CondWidenVT); + assert(Cond1.getValueType() == CondWidenVT && "Condition not widen"); + + SDValue Tmp1 = WidenVectorOp(Node->getOperand(1), WidenVT); + SDValue Tmp2 = WidenVectorOp(Node->getOperand(2), WidenVT); + assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Cond1, Tmp1, Tmp2); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code by first + // Widening to the right type and then unroll the beast. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + + case ISD::SELECT_CC: { + TargetLowering::LegalizeAction action = + TLI.getOperationAction(Node->getOpcode(), WidenVT); + + // Determine new condition widen type and widen + SDValue Cond1 = Node->getOperand(0); + SDValue Cond2 = Node->getOperand(1); + MVT CondVT = Cond1.getValueType(); + assert(CondVT.isVector() && "can not widen non vector type"); + assert(CondVT == Cond2.getValueType() && "mismatch lhs/rhs"); + MVT CondEVT = CondVT.getVectorElementType(); + MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); + Cond1 = WidenVectorOp(Cond1, CondWidenVT); + Cond2 = WidenVectorOp(Cond2, CondWidenVT); + assert(Cond1.getValueType() == CondWidenVT && + Cond2.getValueType() == CondWidenVT && "condition not widen"); + + SDValue Tmp1 = WidenVectorOp(Node->getOperand(2), WidenVT); + SDValue Tmp2 = WidenVectorOp(Node->getOperand(3), WidenVT); + assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT && + "operands not widen"); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Cond1, Cond2, Tmp1, + Tmp2, Node->getOperand(4)); + switch (action) { + default: assert(0 && "action not supported"); + case TargetLowering::Legal: + break; + case TargetLowering::Promote: + // We defer the promotion to when we legalize the op + break; + case TargetLowering::Expand: + // Expand the operation into a bunch of nasty scalar code by first + // Widening to the right type and then unroll the beast. + Result = LegalizeOp(UnrollVectorOp(Result)); + break; + } + break; + } + case ISD::VSETCC: { + // Determine widen for the operand + SDValue Tmp1 = Node->getOperand(0); + MVT TmpVT = Tmp1.getValueType(); + assert(TmpVT.isVector() && "can not widen non vector type"); + MVT TmpEVT = TmpVT.getVectorElementType(); + MVT TmpWidenVT = MVT::getVectorVT(TmpEVT, NewNumElts); + Tmp1 = WidenVectorOp(Tmp1, TmpWidenVT); + SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), TmpWidenVT); + Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2, + Node->getOperand(2)); + break; + } + case ISD::ATOMIC_CMP_SWAP_8: + case ISD::ATOMIC_CMP_SWAP_16: + case ISD::ATOMIC_CMP_SWAP_32: + case ISD::ATOMIC_CMP_SWAP_64: + case ISD::ATOMIC_LOAD_ADD_8: + case ISD::ATOMIC_LOAD_SUB_8: + case ISD::ATOMIC_LOAD_AND_8: + case ISD::ATOMIC_LOAD_OR_8: + case ISD::ATOMIC_LOAD_XOR_8: + case ISD::ATOMIC_LOAD_NAND_8: + case ISD::ATOMIC_LOAD_MIN_8: + case ISD::ATOMIC_LOAD_MAX_8: + case ISD::ATOMIC_LOAD_UMIN_8: + case ISD::ATOMIC_LOAD_UMAX_8: + case ISD::ATOMIC_SWAP_8: + case ISD::ATOMIC_LOAD_ADD_16: + case ISD::ATOMIC_LOAD_SUB_16: + case ISD::ATOMIC_LOAD_AND_16: + case ISD::ATOMIC_LOAD_OR_16: + case ISD::ATOMIC_LOAD_XOR_16: + case ISD::ATOMIC_LOAD_NAND_16: + case ISD::ATOMIC_LOAD_MIN_16: + case ISD::ATOMIC_LOAD_MAX_16: + case ISD::ATOMIC_LOAD_UMIN_16: + case ISD::ATOMIC_LOAD_UMAX_16: + case ISD::ATOMIC_SWAP_16: + case ISD::ATOMIC_LOAD_ADD_32: + case ISD::ATOMIC_LOAD_SUB_32: + case ISD::ATOMIC_LOAD_AND_32: + case ISD::ATOMIC_LOAD_OR_32: + case ISD::ATOMIC_LOAD_XOR_32: + case ISD::ATOMIC_LOAD_NAND_32: + case ISD::ATOMIC_LOAD_MIN_32: + case ISD::ATOMIC_LOAD_MAX_32: + case ISD::ATOMIC_LOAD_UMIN_32: + case ISD::ATOMIC_LOAD_UMAX_32: + case ISD::ATOMIC_SWAP_32: + case ISD::ATOMIC_LOAD_ADD_64: + case ISD::ATOMIC_LOAD_SUB_64: + case ISD::ATOMIC_LOAD_AND_64: + case ISD::ATOMIC_LOAD_OR_64: + case ISD::ATOMIC_LOAD_XOR_64: + case ISD::ATOMIC_LOAD_NAND_64: + case ISD::ATOMIC_LOAD_MIN_64: + case ISD::ATOMIC_LOAD_MAX_64: + case ISD::ATOMIC_LOAD_UMIN_64: + case ISD::ATOMIC_LOAD_UMAX_64: + case ISD::ATOMIC_SWAP_64: { + // For now, we assume that using vectors for these operations don't make + // much sense so we just split it. We return an empty result + SDValue X, Y; + SplitVectorOp(Op, X, Y); + return Result; + break; + } + + } // end switch (Node->getOpcode()) + + assert(Result.getNode() && "Didn't set a result!"); + if (Result != Op) + Result = LegalizeOp(Result); + + AddWidenedOperand(Op, Result); + return Result; +} + +// Utility function to find a legal vector type and its associated element +// type from a preferred width and whose vector type must be the same size +// as the VVT. +// TLI: Target lowering used to determine legal types +// Width: Preferred width of element type +// VVT: Vector value type whose size we must match. +// Returns VecEVT and EVT - the vector type and its associated element type +static void FindWidenVecType(TargetLowering &TLI, unsigned Width, MVT VVT, + MVT& EVT, MVT& VecEVT) { + // We start with the preferred width, make it a power of 2 and see if + // we can find a vector type of that width. If not, we reduce it by + // another power of 2. If we have widen the type, a vector of bytes should + // always be legal. + assert(TLI.isTypeLegal(VVT)); + unsigned EWidth = Width + 1; + do { + assert(EWidth > 0); + EWidth = (1 << Log2_32(EWidth-1)); + EVT = MVT::getIntegerVT(EWidth); + unsigned NumEVT = VVT.getSizeInBits()/EWidth; + VecEVT = MVT::getVectorVT(EVT, NumEVT); + } while (!TLI.isTypeLegal(VecEVT) || + VVT.getSizeInBits() != VecEVT.getSizeInBits()); +} + +SDValue SelectionDAGLegalize::genWidenVectorLoads(SDValueVector& LdChain, + SDValue Chain, + SDValue BasePtr, + const Value *SV, + int SVOffset, + unsigned Alignment, + bool isVolatile, + unsigned LdWidth, + MVT ResType) { + // We assume that we have good rules to handle loading power of two loads so + // we break down the operations to power of 2 loads. The strategy is to + // load the largest power of 2 that we can easily transform to a legal vector + // and then insert into that vector, and the cast the result into the legal + // vector that we want. This avoids unnecessary stack converts. + // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and + // the load is nonvolatile, we an use a wider load for the value. + // Find a vector length we can load a large chunk + MVT EVT, VecEVT; + unsigned EVTWidth; + FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); + EVTWidth = EVT.getSizeInBits(); + + SDValue LdOp = DAG.getLoad(EVT, Chain, BasePtr, SV, SVOffset, + isVolatile, Alignment); + SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecEVT, LdOp); + LdChain.push_back(LdOp.getValue(1)); + + // Check if we can load the element with one instruction + if (LdWidth == EVTWidth) { + return DAG.getNode(ISD::BIT_CONVERT, ResType, VecOp); + } + + // The vector element order is endianness dependent. + unsigned Idx = 1; + LdWidth -= EVTWidth; + unsigned Offset = 0; + + while (LdWidth > 0) { + unsigned Increment = EVTWidth / 8; + Offset += Increment; + BasePtr = DAG.getNode(ISD::ADD, BasePtr.getValueType(), BasePtr, + DAG.getIntPtrConstant(Increment)); + + if (LdWidth < EVTWidth) { + // Our current type we are using is too large, use a smaller size by + // using a smaller power of 2 + unsigned oEVTWidth = EVTWidth; + FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); + EVTWidth = EVT.getSizeInBits(); + // Readjust position and vector position based on new load type + Idx = Idx * (oEVTWidth/EVTWidth); + VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, VecOp); + } + + SDValue LdOp = DAG.getLoad(EVT, Chain, BasePtr, SV, + SVOffset+Offset, isVolatile, + MinAlign(Alignment, Offset)); + LdChain.push_back(LdOp.getValue(1)); + VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, VecEVT, VecOp, LdOp, + DAG.getIntPtrConstant(Idx++)); + + LdWidth -= EVTWidth; + } + + return DAG.getNode(ISD::BIT_CONVERT, ResType, VecOp); +} + +bool SelectionDAGLegalize::LoadWidenVectorOp(SDValue& Result, + SDValue& TFOp, + SDValue Op, + MVT NVT) { + // TODO: Add support for ConcatVec and the ability to load many vector + // types (e.g., v4i8). This will not work when a vector register + // to memory mapping is strange (e.g., vector elements are not + // stored in some sequential order). + + // It must be true that the widen vector type is bigger than where + // we need to load from. + LoadSDNode *LD = cast(Op.getNode()); + MVT LdVT = LD->getMemoryVT(); + assert(LdVT.isVector() && NVT.isVector()); + assert(LdVT.getVectorElementType() == NVT.getVectorElementType()); + + // Load information + SDValue Chain = LD->getChain(); + SDValue BasePtr = LD->getBasePtr(); + int SVOffset = LD->getSrcValueOffset(); + unsigned Alignment = LD->getAlignment(); + bool isVolatile = LD->isVolatile(); + const Value *SV = LD->getSrcValue(); + unsigned int LdWidth = LdVT.getSizeInBits(); + + // Load value as a large register + SDValueVector LdChain; + Result = genWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, + Alignment, isVolatile, LdWidth, NVT); + + if (LdChain.size() == 1) { + TFOp = LdChain[0]; + return true; + } + else { + TFOp=DAG.getNode(ISD::TokenFactor, MVT::Other, &LdChain[0], LdChain.size()); + return false; + } +} + + +void SelectionDAGLegalize::genWidenVectorStores(SDValueVector& StChain, + SDValue Chain, + SDValue BasePtr, + const Value *SV, + int SVOffset, + unsigned Alignment, + bool isVolatile, + SDValue ValOp, + unsigned StWidth) { + // Breaks the stores into a series of power of 2 width stores. For any + // width, we convert the vector to the vector of element size that we + // want to store. This avoids requiring a stack convert. + + // Find a width of the element type we can store with + MVT VVT = ValOp.getValueType(); + MVT EVT, VecEVT; + unsigned EVTWidth; + FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); + EVTWidth = EVT.getSizeInBits(); + + SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, ValOp); + SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EVT, VecOp, + DAG.getIntPtrConstant(0)); + SDValue StOp = DAG.getStore(Chain, EOp, BasePtr, SV, SVOffset, + isVolatile, Alignment); + StChain.push_back(StOp); + + // Check if we are done + if (StWidth == EVTWidth) { + return; + } + + unsigned Idx = 1; + StWidth -= EVTWidth; + unsigned Offset = 0; + + while (StWidth > 0) { + unsigned Increment = EVTWidth / 8; + Offset += Increment; + BasePtr = DAG.getNode(ISD::ADD, BasePtr.getValueType(), BasePtr, + DAG.getIntPtrConstant(Increment)); + + if (StWidth < EVTWidth) { + // Our current type we are using is too large, use a smaller size by + // using a smaller power of 2 + unsigned oEVTWidth = EVTWidth; + FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); + EVTWidth = EVT.getSizeInBits(); + // Readjust position and vector position based on new load type + Idx = Idx * (oEVTWidth/EVTWidth); + VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, VecOp); + } + + EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EVT, VecOp, + DAG.getIntPtrConstant(Idx++)); + StChain.push_back(DAG.getStore(Chain, EOp, BasePtr, SV, + SVOffset + Offset, isVolatile, + MinAlign(Alignment, Offset))); + StWidth -= EVTWidth; + } +} + + +SDValue SelectionDAGLegalize::StoreWidenVectorOp(StoreSDNode *ST, + SDValue Chain, + SDValue BasePtr) { + // TODO: It might be cleaner if we can use SplitVector and have more legal + // vector types that can be stored into memory (e.g., v4xi8 can + // be stored as a word). This will not work when a vector register + // to memory mapping is strange (e.g., vector elements are not + // stored in some sequential order). + + MVT StVT = ST->getMemoryVT(); + SDValue ValOp = ST->getValue(); + + // Check if we have widen this node with another value + std::map::iterator I = WidenNodes.find(ValOp); + if (I != WidenNodes.end()) + ValOp = I->second; + + MVT VVT = ValOp.getValueType(); + + // It must be true that we the widen vector type is bigger than where + // we need to store. + assert(StVT.isVector() && VVT.isVector()); + assert(StVT.getSizeInBits() < VVT.getSizeInBits()); + assert(StVT.getVectorElementType() == VVT.getVectorElementType()); + + // Store value + SDValueVector StChain; + genWidenVectorStores(StChain, Chain, BasePtr, ST->getSrcValue(), + ST->getSrcValueOffset(), ST->getAlignment(), + ST->isVolatile(), ValOp, StVT.getSizeInBits()); + if (StChain.size() == 1) + return StChain[0]; + else + return DAG.getNode(ISD::TokenFactor, MVT::Other,&StChain[0],StChain.size()); +} + + // SelectionDAG::Legalize - This is the entry point for the file. // void SelectionDAG::Legalize() {