X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FDAGCombiner.cpp;h=31cbdb9c787f46b304b02fcb5326ff05a667911f;hb=14036c00c0b3a83805695afb50b6d42430b70979;hp=0e719f844e990da7de3304e71af47aac94fe70f6;hpb=c8e3b147eea6155eb047340205730b5332259bb6;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 0e719f844e9..fc3ddbe319d 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -29,6 +29,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" #include +#include using namespace llvm; STATISTIC(NodesCombined , "Number of dag nodes combined"); @@ -36,20 +37,6 @@ STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created"); STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created"); namespace { -#ifndef NDEBUG - static cl::opt - ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, - cl::desc("Pop up a window to show dags before the first " - "dag combine pass")); - static cl::opt - ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, - cl::desc("Pop up a window to show dags before the second " - "dag combine pass")); -#else - static const bool ViewDAGCombine1 = false; - static const bool ViewDAGCombine2 = false; -#endif - static cl::opt CombinerAA("combiner-alias-analysis", cl::Hidden, cl::desc("Turn on alias analysis during testing")); @@ -62,8 +49,11 @@ namespace { class VISIBILITY_HIDDEN DAGCombiner { SelectionDAG &DAG; - TargetLowering &TLI; - bool AfterLegalize; + const TargetLowering &TLI; + CombineLevel Level; + bool LegalOperations; + bool LegalTypes; + bool Fast; // Worklist of all of the nodes that need to be simplified. std::vector WorkList; @@ -83,7 +73,7 @@ namespace { /// visit - call the node-specific routine that knows how to fold each /// particular type of node. - SDOperand visit(SDNode *N); + SDValue visit(SDNode *N); public: /// AddToWorkList - Add to the work list making sure it's instance is at the @@ -100,30 +90,32 @@ namespace { WorkList.end()); } - SDOperand CombineTo(SDNode *N, const SDOperand *To, unsigned NumTo, + SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo, bool AddTo = true); - SDOperand CombineTo(SDNode *N, SDOperand Res, bool AddTo = true) { + SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) { return CombineTo(N, &Res, 1, AddTo); } - SDOperand CombineTo(SDNode *N, SDOperand Res0, SDOperand Res1, + SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo = true) { - SDOperand To[] = { Res0, Res1 }; + SDValue To[] = { Res0, Res1 }; return CombineTo(N, To, 2, AddTo); } + + void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO); private: /// SimplifyDemandedBits - Check the specified integer node value to see if /// it can be simplified or if things it uses can be simplified by bit /// propagation. If so, return true. - bool SimplifyDemandedBits(SDOperand Op) { + bool SimplifyDemandedBits(SDValue Op) { APInt Demanded = APInt::getAllOnesValue(Op.getValueSizeInBits()); return SimplifyDemandedBits(Op, Demanded); } - bool SimplifyDemandedBits(SDOperand Op, const APInt &Demanded); + bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded); bool CombineToPreIndexedLoadStore(SDNode *N); bool CombineToPostIndexedLoadStore(SDNode *N); @@ -132,130 +124,135 @@ namespace { /// combine - call the node-specific routine that knows how to fold each /// particular type of node. If that doesn't do anything, try the /// target-specific DAG combines. - SDOperand combine(SDNode *N); + SDValue combine(SDNode *N); // Visitation implementation - Implement dag node combining for different // node types. The semantics are as follows: // Return Value: - // SDOperand.Val == 0 - No change was made - // SDOperand.Val == N - N was replaced, is dead, and is already handled. - // otherwise - N should be replaced by the returned Operand. + // SDValue.getNode() == 0 - No change was made + // SDValue.getNode() == N - N was replaced, is dead and has been handled. + // otherwise - N should be replaced by the returned Operand. // - SDOperand visitTokenFactor(SDNode *N); - SDOperand visitMERGE_VALUES(SDNode *N); - SDOperand visitADD(SDNode *N); - SDOperand visitSUB(SDNode *N); - SDOperand visitADDC(SDNode *N); - SDOperand visitADDE(SDNode *N); - SDOperand visitMUL(SDNode *N); - SDOperand visitSDIV(SDNode *N); - SDOperand visitUDIV(SDNode *N); - SDOperand visitSREM(SDNode *N); - SDOperand visitUREM(SDNode *N); - SDOperand visitMULHU(SDNode *N); - SDOperand visitMULHS(SDNode *N); - SDOperand visitSMUL_LOHI(SDNode *N); - SDOperand visitUMUL_LOHI(SDNode *N); - SDOperand visitSDIVREM(SDNode *N); - SDOperand visitUDIVREM(SDNode *N); - SDOperand visitAND(SDNode *N); - SDOperand visitOR(SDNode *N); - SDOperand visitXOR(SDNode *N); - SDOperand SimplifyVBinOp(SDNode *N); - SDOperand visitSHL(SDNode *N); - SDOperand visitSRA(SDNode *N); - SDOperand visitSRL(SDNode *N); - SDOperand visitCTLZ(SDNode *N); - SDOperand visitCTTZ(SDNode *N); - SDOperand visitCTPOP(SDNode *N); - SDOperand visitSELECT(SDNode *N); - SDOperand visitSELECT_CC(SDNode *N); - SDOperand visitSETCC(SDNode *N); - SDOperand visitSIGN_EXTEND(SDNode *N); - SDOperand visitZERO_EXTEND(SDNode *N); - SDOperand visitANY_EXTEND(SDNode *N); - SDOperand visitSIGN_EXTEND_INREG(SDNode *N); - SDOperand visitTRUNCATE(SDNode *N); - SDOperand visitBIT_CONVERT(SDNode *N); - SDOperand visitFADD(SDNode *N); - SDOperand visitFSUB(SDNode *N); - SDOperand visitFMUL(SDNode *N); - SDOperand visitFDIV(SDNode *N); - SDOperand visitFREM(SDNode *N); - SDOperand visitFCOPYSIGN(SDNode *N); - SDOperand visitSINT_TO_FP(SDNode *N); - SDOperand visitUINT_TO_FP(SDNode *N); - SDOperand visitFP_TO_SINT(SDNode *N); - SDOperand visitFP_TO_UINT(SDNode *N); - SDOperand visitFP_ROUND(SDNode *N); - SDOperand visitFP_ROUND_INREG(SDNode *N); - SDOperand visitFP_EXTEND(SDNode *N); - SDOperand visitFNEG(SDNode *N); - SDOperand visitFABS(SDNode *N); - SDOperand visitBRCOND(SDNode *N); - SDOperand visitBR_CC(SDNode *N); - SDOperand visitLOAD(SDNode *N); - SDOperand visitSTORE(SDNode *N); - SDOperand visitINSERT_VECTOR_ELT(SDNode *N); - SDOperand visitEXTRACT_VECTOR_ELT(SDNode *N); - SDOperand visitBUILD_VECTOR(SDNode *N); - SDOperand visitCONCAT_VECTORS(SDNode *N); - SDOperand visitVECTOR_SHUFFLE(SDNode *N); - - SDOperand XformToShuffleWithZero(SDNode *N); - SDOperand ReassociateOps(unsigned Opc, SDOperand LHS, SDOperand RHS); + SDValue visitTokenFactor(SDNode *N); + SDValue visitMERGE_VALUES(SDNode *N); + SDValue visitADD(SDNode *N); + SDValue visitSUB(SDNode *N); + SDValue visitADDC(SDNode *N); + SDValue visitADDE(SDNode *N); + SDValue visitMUL(SDNode *N); + SDValue visitSDIV(SDNode *N); + SDValue visitUDIV(SDNode *N); + SDValue visitSREM(SDNode *N); + SDValue visitUREM(SDNode *N); + SDValue visitMULHU(SDNode *N); + SDValue visitMULHS(SDNode *N); + SDValue visitSMUL_LOHI(SDNode *N); + SDValue visitUMUL_LOHI(SDNode *N); + SDValue visitSDIVREM(SDNode *N); + SDValue visitUDIVREM(SDNode *N); + SDValue visitAND(SDNode *N); + SDValue visitOR(SDNode *N); + SDValue visitXOR(SDNode *N); + SDValue SimplifyVBinOp(SDNode *N); + SDValue visitSHL(SDNode *N); + SDValue visitSRA(SDNode *N); + SDValue visitSRL(SDNode *N); + SDValue visitCTLZ(SDNode *N); + SDValue visitCTTZ(SDNode *N); + SDValue visitCTPOP(SDNode *N); + SDValue visitSELECT(SDNode *N); + SDValue visitSELECT_CC(SDNode *N); + SDValue visitSETCC(SDNode *N); + SDValue visitSIGN_EXTEND(SDNode *N); + SDValue visitZERO_EXTEND(SDNode *N); + SDValue visitANY_EXTEND(SDNode *N); + SDValue visitSIGN_EXTEND_INREG(SDNode *N); + SDValue visitTRUNCATE(SDNode *N); + SDValue visitBIT_CONVERT(SDNode *N); + SDValue visitBUILD_PAIR(SDNode *N); + SDValue visitFADD(SDNode *N); + SDValue visitFSUB(SDNode *N); + SDValue visitFMUL(SDNode *N); + SDValue visitFDIV(SDNode *N); + SDValue visitFREM(SDNode *N); + SDValue visitFCOPYSIGN(SDNode *N); + SDValue visitSINT_TO_FP(SDNode *N); + SDValue visitUINT_TO_FP(SDNode *N); + SDValue visitFP_TO_SINT(SDNode *N); + SDValue visitFP_TO_UINT(SDNode *N); + SDValue visitFP_ROUND(SDNode *N); + SDValue visitFP_ROUND_INREG(SDNode *N); + SDValue visitFP_EXTEND(SDNode *N); + SDValue visitFNEG(SDNode *N); + SDValue visitFABS(SDNode *N); + SDValue visitBRCOND(SDNode *N); + SDValue visitBR_CC(SDNode *N); + SDValue visitLOAD(SDNode *N); + SDValue visitSTORE(SDNode *N); + SDValue visitINSERT_VECTOR_ELT(SDNode *N); + SDValue visitEXTRACT_VECTOR_ELT(SDNode *N); + SDValue visitBUILD_VECTOR(SDNode *N); + SDValue visitCONCAT_VECTORS(SDNode *N); + SDValue visitVECTOR_SHUFFLE(SDNode *N); + + SDValue XformToShuffleWithZero(SDNode *N); + SDValue ReassociateOps(unsigned Opc, DebugLoc DL, SDValue LHS, SDValue RHS); - SDOperand visitShiftByConstant(SDNode *N, unsigned Amt); + SDValue visitShiftByConstant(SDNode *N, unsigned Amt); - bool SimplifySelectOps(SDNode *SELECT, SDOperand LHS, SDOperand RHS); - SDOperand SimplifyBinOpWithSameOpcodeHands(SDNode *N); - SDOperand SimplifySelect(SDOperand N0, SDOperand N1, SDOperand N2); - SDOperand SimplifySelectCC(SDOperand N0, SDOperand N1, SDOperand N2, - SDOperand N3, ISD::CondCode CC, + bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS); + SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N); + SDValue SimplifySelect(SDValue N0, SDValue N1, SDValue N2); + SDValue SimplifySelectCC(SDValue N0, SDValue N1, SDValue N2, + SDValue N3, ISD::CondCode CC, bool NotExtCompare = false); - SDOperand SimplifySetCC(MVT::ValueType VT, SDOperand N0, SDOperand N1, - ISD::CondCode Cond, bool foldBooleans = true); - SDOperand SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp, + SDValue SimplifySetCC(MVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, + bool foldBooleans = true); + SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp, unsigned HiOp); - SDOperand ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *, MVT::ValueType); - SDOperand BuildSDIV(SDNode *N); - SDOperand BuildUDIV(SDNode *N); - SDNode *MatchRotate(SDOperand LHS, SDOperand RHS); - SDOperand ReduceLoadWidth(SDNode *N); + SDValue CombineConsecutiveLoads(SDNode *N, MVT VT); + SDValue ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *, MVT); + SDValue BuildSDIV(SDNode *N); + SDValue BuildUDIV(SDNode *N); + SDNode *MatchRotate(SDValue LHS, SDValue RHS); + SDValue ReduceLoadWidth(SDNode *N); - SDOperand GetDemandedBits(SDOperand V, const APInt &Mask); + SDValue GetDemandedBits(SDValue V, const APInt &Mask); /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes, /// looking for aliasing nodes and adding them to the Aliases vector. - void GatherAllAliases(SDNode *N, SDOperand OriginalChain, - SmallVector &Aliases); + void GatherAllAliases(SDNode *N, SDValue OriginalChain, + SmallVector &Aliases); /// isAlias - Return true if there is any possibility that the two addresses /// overlap. - bool isAlias(SDOperand Ptr1, int64_t Size1, + bool isAlias(SDValue Ptr1, int64_t Size1, const Value *SrcValue1, int SrcValueOffset1, - SDOperand Ptr2, int64_t Size2, + SDValue Ptr2, int64_t Size2, const Value *SrcValue2, int SrcValueOffset2); /// FindAliasInfo - Extracts the relevant alias information from the memory /// node. Returns true if the operand was a load. bool FindAliasInfo(SDNode *N, - SDOperand &Ptr, int64_t &Size, + SDValue &Ptr, int64_t &Size, const Value *&SrcValue, int &SrcValueOffset); /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes, /// looking for a better chain (aliasing node.) - SDOperand FindBetterChain(SDNode *N, SDOperand Chain); + SDValue FindBetterChain(SDNode *N, SDValue Chain); public: - DAGCombiner(SelectionDAG &D, AliasAnalysis &A) + DAGCombiner(SelectionDAG &D, AliasAnalysis &A, bool fast) : DAG(D), TLI(D.getTargetLoweringInfo()), - AfterLegalize(false), + Level(Unrestricted), + LegalOperations(false), + LegalTypes(false), + Fast(fast), AA(A) {} /// Run - runs the dag combiner on all nodes in the work list - void Run(bool RunningAfterLegalize); + void Run(CombineLevel AtLevel); }; } @@ -269,7 +266,7 @@ class VISIBILITY_HIDDEN WorkListRemover : public: explicit WorkListRemover(DAGCombiner &dc) : DC(dc) {} - virtual void NodeDeleted(SDNode *N) { + virtual void NodeDeleted(SDNode *N, SDNode *E) { DC.removeFromWorkList(N); } @@ -287,22 +284,26 @@ void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) { ((DAGCombiner*)DC)->AddToWorkList(N); } -SDOperand TargetLowering::DAGCombinerInfo:: -CombineTo(SDNode *N, const std::vector &To) { +SDValue TargetLowering::DAGCombinerInfo:: +CombineTo(SDNode *N, const std::vector &To) { return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size()); } -SDOperand TargetLowering::DAGCombinerInfo:: -CombineTo(SDNode *N, SDOperand Res) { +SDValue TargetLowering::DAGCombinerInfo:: +CombineTo(SDNode *N, SDValue Res) { return ((DAGCombiner*)DC)->CombineTo(N, Res); } -SDOperand TargetLowering::DAGCombinerInfo:: -CombineTo(SDNode *N, SDOperand Res0, SDOperand Res1) { +SDValue TargetLowering::DAGCombinerInfo:: +CombineTo(SDNode *N, SDValue Res0, SDValue Res1) { return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1); } +void TargetLowering::DAGCombinerInfo:: +CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) { + return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO); +} //===----------------------------------------------------------------------===// // Helper Functions @@ -311,7 +312,7 @@ CombineTo(SDNode *N, SDOperand Res0, SDOperand Res1) { /// isNegatibleForFree - Return 1 if we can compute the negated form of the /// specified expression for the same cost as the expression itself, or 2 if we /// can compute the negated form more cheaply than the expression itself. -static char isNegatibleForFree(SDOperand Op, bool AfterLegalize, +static char isNegatibleForFree(SDValue Op, bool LegalOperations, unsigned Depth = 0) { // No compile time optimizations on this type. if (Op.getValueType() == MVT::ppcf128) @@ -331,16 +332,16 @@ static char isNegatibleForFree(SDOperand Op, bool AfterLegalize, case ISD::ConstantFP: // Don't invert constant FP values after legalize. The negated constant // isn't necessarily legal. - return AfterLegalize ? 0 : 1; + return LegalOperations ? 0 : 1; case ISD::FADD: // FIXME: determine better conditions for this xform. if (!UnsafeFPMath) return 0; // -(A+B) -> -A - B - if (char V = isNegatibleForFree(Op.getOperand(0), AfterLegalize, Depth+1)) + if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, Depth+1)) return V; // -(A+B) -> -B - A - return isNegatibleForFree(Op.getOperand(1), AfterLegalize, Depth+1); + return isNegatibleForFree(Op.getOperand(1), LegalOperations, Depth+1); case ISD::FSUB: // We can't turn -(A-B) into B-A when we honor signed zeros. if (!UnsafeFPMath) return 0; @@ -353,22 +354,22 @@ static char isNegatibleForFree(SDOperand Op, bool AfterLegalize, if (HonorSignDependentRoundingFPMath()) return 0; // -(X*Y) -> (-X * Y) or (X*-Y) - if (char V = isNegatibleForFree(Op.getOperand(0), AfterLegalize, Depth+1)) + if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, Depth+1)) return V; - return isNegatibleForFree(Op.getOperand(1), AfterLegalize, Depth+1); + return isNegatibleForFree(Op.getOperand(1), LegalOperations, Depth+1); case ISD::FP_EXTEND: case ISD::FP_ROUND: case ISD::FSIN: - return isNegatibleForFree(Op.getOperand(0), AfterLegalize, Depth+1); + return isNegatibleForFree(Op.getOperand(0), LegalOperations, Depth+1); } } /// GetNegatedExpression - If isNegatibleForFree returns true, this function /// returns the newly negated expression. -static SDOperand GetNegatedExpression(SDOperand Op, SelectionDAG &DAG, - bool AfterLegalize, unsigned Depth = 0) { +static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, + bool LegalOperations, unsigned Depth = 0) { // fneg is removable even if it has multiple uses. if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0); @@ -388,15 +389,15 @@ static SDOperand GetNegatedExpression(SDOperand Op, SelectionDAG &DAG, assert(UnsafeFPMath); // -(A+B) -> -A - B - if (isNegatibleForFree(Op.getOperand(0), AfterLegalize, Depth+1)) - return DAG.getNode(ISD::FSUB, Op.getValueType(), + if (isNegatibleForFree(Op.getOperand(0), LegalOperations, Depth+1)) + return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(), GetNegatedExpression(Op.getOperand(0), DAG, - AfterLegalize, Depth+1), + LegalOperations, Depth+1), Op.getOperand(1)); // -(A+B) -> -B - A - return DAG.getNode(ISD::FSUB, Op.getValueType(), + return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(), GetNegatedExpression(Op.getOperand(1), DAG, - AfterLegalize, Depth+1), + LegalOperations, Depth+1), Op.getOperand(0)); case ISD::FSUB: // We can't turn -(A-B) into B-A when we honor signed zeros. @@ -408,35 +409,35 @@ static SDOperand GetNegatedExpression(SDOperand Op, SelectionDAG &DAG, return Op.getOperand(1); // -(A-B) -> B-A - return DAG.getNode(ISD::FSUB, Op.getValueType(), Op.getOperand(1), - Op.getOperand(0)); + return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(), + Op.getOperand(1), Op.getOperand(0)); case ISD::FMUL: case ISD::FDIV: assert(!HonorSignDependentRoundingFPMath()); // -(X*Y) -> -X * Y - if (isNegatibleForFree(Op.getOperand(0), AfterLegalize, Depth+1)) - return DAG.getNode(Op.getOpcode(), Op.getValueType(), + if (isNegatibleForFree(Op.getOperand(0), LegalOperations, Depth+1)) + return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(), GetNegatedExpression(Op.getOperand(0), DAG, - AfterLegalize, Depth+1), + LegalOperations, Depth+1), Op.getOperand(1)); // -(X*Y) -> X * -Y - return DAG.getNode(Op.getOpcode(), Op.getValueType(), + return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(), Op.getOperand(0), GetNegatedExpression(Op.getOperand(1), DAG, - AfterLegalize, Depth+1)); + LegalOperations, Depth+1)); case ISD::FP_EXTEND: case ISD::FSIN: - return DAG.getNode(Op.getOpcode(), Op.getValueType(), + return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(), GetNegatedExpression(Op.getOperand(0), DAG, - AfterLegalize, Depth+1)); + LegalOperations, Depth+1)); case ISD::FP_ROUND: - return DAG.getNode(ISD::FP_ROUND, Op.getValueType(), + return DAG.getNode(ISD::FP_ROUND, Op.getDebugLoc(), Op.getValueType(), GetNegatedExpression(Op.getOperand(0), DAG, - AfterLegalize, Depth+1), + LegalOperations, Depth+1), Op.getOperand(1)); } } @@ -447,8 +448,8 @@ static SDOperand GetNegatedExpression(SDOperand Op, SelectionDAG &DAG, // Also, set the incoming LHS, RHS, and CC references to the appropriate // nodes based on the type of node we are checking. This simplifies life a // bit for the callers. -static bool isSetCCEquivalent(SDOperand N, SDOperand &LHS, SDOperand &RHS, - SDOperand &CC) { +static bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS, + SDValue &CC) { if (N.getOpcode() == ISD::SETCC) { LHS = N.getOperand(0); RHS = N.getOperand(1); @@ -458,7 +459,7 @@ static bool isSetCCEquivalent(SDOperand N, SDOperand &LHS, SDOperand &RHS, if (N.getOpcode() == ISD::SELECT_CC && N.getOperand(2).getOpcode() == ISD::Constant && N.getOperand(3).getOpcode() == ISD::Constant && - cast(N.getOperand(2))->getValue() == 1 && + cast(N.getOperand(2))->getAPIntValue() == 1 && cast(N.getOperand(3))->isNullValue()) { LHS = N.getOperand(0); RHS = N.getOperand(1); @@ -471,112 +472,133 @@ static bool isSetCCEquivalent(SDOperand N, SDOperand &LHS, SDOperand &RHS, // isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only // one use. If this is true, it allows the users to invert the operation for // free when it is profitable to do so. -static bool isOneUseSetCC(SDOperand N) { - SDOperand N0, N1, N2; - if (isSetCCEquivalent(N, N0, N1, N2) && N.Val->hasOneUse()) +static bool isOneUseSetCC(SDValue N) { + SDValue N0, N1, N2; + if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse()) return true; return false; } -SDOperand DAGCombiner::ReassociateOps(unsigned Opc, SDOperand N0, SDOperand N1){ - MVT::ValueType VT = N0.getValueType(); - // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use - // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2)) +SDValue DAGCombiner::ReassociateOps(unsigned Opc, DebugLoc DL, + SDValue N0, SDValue N1) { + MVT VT = N0.getValueType(); if (N0.getOpcode() == Opc && isa(N0.getOperand(1))) { if (isa(N1)) { - SDOperand OpNode = DAG.getNode(Opc, VT, N0.getOperand(1), N1); - AddToWorkList(OpNode.Val); - return DAG.getNode(Opc, VT, OpNode, N0.getOperand(0)); + // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2)) + SDValue OpNode = DAG.getNode(Opc, N1.getDebugLoc(), VT, + N0.getOperand(1), N1); + AddToWorkList(OpNode.getNode()); + return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode); } else if (N0.hasOneUse()) { - SDOperand OpNode = DAG.getNode(Opc, VT, N0.getOperand(0), N1); - AddToWorkList(OpNode.Val); - return DAG.getNode(Opc, VT, OpNode, N0.getOperand(1)); + // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one use + SDValue OpNode = DAG.getNode(Opc, N0.getDebugLoc(), VT, + N0.getOperand(0), N1); + AddToWorkList(OpNode.getNode()); + return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1)); } } - // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use - // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2)) + if (N1.getOpcode() == Opc && isa(N1.getOperand(1))) { if (isa(N0)) { - SDOperand OpNode = DAG.getNode(Opc, VT, N1.getOperand(1), N0); - AddToWorkList(OpNode.Val); - return DAG.getNode(Opc, VT, OpNode, N1.getOperand(0)); + // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2)) + SDValue OpNode = DAG.getNode(Opc, N1.getDebugLoc(), VT, + N1.getOperand(1), N0); + AddToWorkList(OpNode.getNode()); + return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode); } else if (N1.hasOneUse()) { - SDOperand OpNode = DAG.getNode(Opc, VT, N1.getOperand(0), N0); - AddToWorkList(OpNode.Val); - return DAG.getNode(Opc, VT, OpNode, N1.getOperand(1)); + // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one use + SDValue OpNode = DAG.getNode(Opc, N0.getDebugLoc(), VT, + N1.getOperand(0), N0); + AddToWorkList(OpNode.getNode()); + return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1)); } } - return SDOperand(); + + return SDValue(); } -SDOperand DAGCombiner::CombineTo(SDNode *N, const SDOperand *To, unsigned NumTo, - bool AddTo) { +SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo, + bool AddTo) { assert(N->getNumValues() == NumTo && "Broken CombineTo call!"); ++NodesCombined; DOUT << "\nReplacing.1 "; DEBUG(N->dump(&DAG)); - DOUT << "\nWith: "; DEBUG(To[0].Val->dump(&DAG)); + DOUT << "\nWith: "; DEBUG(To[0].getNode()->dump(&DAG)); DOUT << " and " << NumTo-1 << " other values\n"; + DEBUG(for (unsigned i = 0, e = NumTo; i != e; ++i) + assert(N->getValueType(i) == To[i].getValueType() && + "Cannot combine value to value of different type!")); WorkListRemover DeadNodes(*this); DAG.ReplaceAllUsesWith(N, To, &DeadNodes); if (AddTo) { // Push the new nodes and any users onto the worklist for (unsigned i = 0, e = NumTo; i != e; ++i) { - AddToWorkList(To[i].Val); - AddUsersToWorkList(To[i].Val); + AddToWorkList(To[i].getNode()); + AddUsersToWorkList(To[i].getNode()); } } - // Nodes can be reintroduced into the worklist. Make sure we do not - // process a node that has been replaced. - removeFromWorkList(N); + // Finally, if the node is now dead, remove it from the graph. The node + // may not be dead if the replacement process recursively simplified to + // something else needing this node. + if (N->use_empty()) { + // Nodes can be reintroduced into the worklist. Make sure we do not + // process a node that has been replaced. + removeFromWorkList(N); - // Finally, since the node is now dead, remove it from the graph. - DAG.DeleteNode(N); - return SDOperand(N, 0); + // Finally, since the node is now dead, remove it from the graph. + DAG.DeleteNode(N); + } + return SDValue(N, 0); } -/// SimplifyDemandedBits - Check the specified integer node value to see if -/// it can be simplified or if things it uses can be simplified by bit -/// propagation. If so, return true. -bool DAGCombiner::SimplifyDemandedBits(SDOperand Op, const APInt &Demanded) { - TargetLowering::TargetLoweringOpt TLO(DAG, AfterLegalize); - APInt KnownZero, KnownOne; - if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO)) - return false; - - // Revisit the node. - AddToWorkList(Op.Val); - - // Replace the old value with the new one. - ++NodesCombined; - DOUT << "\nReplacing.2 "; DEBUG(TLO.Old.Val->dump(&DAG)); - DOUT << "\nWith: "; DEBUG(TLO.New.Val->dump(&DAG)); - DOUT << '\n'; - +void +DAGCombiner::CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt & + TLO) { // Replace all uses. If any nodes become isomorphic to other nodes and // are deleted, make sure to remove them from our worklist. WorkListRemover DeadNodes(*this); DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New, &DeadNodes); - + // Push the new node and any (possibly new) users onto the worklist. - AddToWorkList(TLO.New.Val); - AddUsersToWorkList(TLO.New.Val); + AddToWorkList(TLO.New.getNode()); + AddUsersToWorkList(TLO.New.getNode()); // Finally, if the node is now dead, remove it from the graph. The node // may not be dead if the replacement process recursively simplified to // something else needing this node. - if (TLO.Old.Val->use_empty()) { - removeFromWorkList(TLO.Old.Val); + if (TLO.Old.getNode()->use_empty()) { + removeFromWorkList(TLO.Old.getNode()); // If the operands of this node are only used by the node, they will now // be dead. Make sure to visit them first to delete dead nodes early. - for (unsigned i = 0, e = TLO.Old.Val->getNumOperands(); i != e; ++i) - if (TLO.Old.Val->getOperand(i).Val->hasOneUse()) - AddToWorkList(TLO.Old.Val->getOperand(i).Val); + for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); i != e; ++i) + if (TLO.Old.getNode()->getOperand(i).getNode()->hasOneUse()) + AddToWorkList(TLO.Old.getNode()->getOperand(i).getNode()); - DAG.DeleteNode(TLO.Old.Val); + DAG.DeleteNode(TLO.Old.getNode()); } +} + +/// SimplifyDemandedBits - Check the specified integer node value to see if +/// it can be simplified or if things it uses can be simplified by bit +/// propagation. If so, return true. +bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) { + TargetLowering::TargetLoweringOpt TLO(DAG); + APInt KnownZero, KnownOne; + if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO)) + return false; + + // Revisit the node. + AddToWorkList(Op.getNode()); + + // Replace the old value with the new one. + ++NodesCombined; + DOUT << "\nReplacing.2 "; DEBUG(TLO.Old.getNode()->dump(&DAG)); + DOUT << "\nWith: "; DEBUG(TLO.New.getNode()->dump(&DAG)); + DOUT << '\n'; + + CommitTargetLoweringOpt(TLO); return true; } @@ -584,15 +606,18 @@ bool DAGCombiner::SimplifyDemandedBits(SDOperand Op, const APInt &Demanded) { // Main DAG Combiner implementation //===----------------------------------------------------------------------===// -void DAGCombiner::Run(bool RunningAfterLegalize) { - // set the instance variable, so that the various visit routines may use it. - AfterLegalize = RunningAfterLegalize; +void DAGCombiner::Run(CombineLevel AtLevel) { + // set the instance variables, so that the various visit routines may use it. + Level = AtLevel; + LegalOperations = Level >= NoIllegalOperations; + LegalTypes = Level >= NoIllegalTypes; // Add all the dag nodes to the worklist. + WorkList.reserve(DAG.allnodes_size()); for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), E = DAG.allnodes_end(); I != E; ++I) WorkList.push_back(I); - + // Create a dummy node (which is not added to allnodes), that adds a reference // to the root node, preventing it from being deleted, and tracking any // changes of the root. @@ -600,7 +625,7 @@ void DAGCombiner::Run(bool RunningAfterLegalize) { // The root of the dag may dangle to deleted nodes until the dag combiner is // done. Set it to null to avoid confusion. - DAG.setRoot(SDOperand()); + DAG.setRoot(SDValue()); // while the worklist isn't empty, inspect the node on the end of it and // try and combine it. @@ -613,15 +638,15 @@ void DAGCombiner::Run(bool RunningAfterLegalize) { // reduced number of uses, allowing other xforms. if (N->use_empty() && N != &Dummy) { for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) - AddToWorkList(N->getOperand(i).Val); + AddToWorkList(N->getOperand(i).getNode()); DAG.DeleteNode(N); continue; } - SDOperand RV = combine(N); + SDValue RV = combine(N); - if (RV.Val == 0) + if (RV.getNode() == 0) continue; ++NodesCombined; @@ -630,49 +655,54 @@ void DAGCombiner::Run(bool RunningAfterLegalize) { // zero, we know that the node must have defined multiple values and // CombineTo was used. Since CombineTo takes care of the worklist // mechanics for us, we have no work to do in this case. - if (RV.Val == N) + if (RV.getNode() == N) continue; assert(N->getOpcode() != ISD::DELETED_NODE && - RV.Val->getOpcode() != ISD::DELETED_NODE && + RV.getNode()->getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned new node!"); DOUT << "\nReplacing.3 "; DEBUG(N->dump(&DAG)); - DOUT << "\nWith: "; DEBUG(RV.Val->dump(&DAG)); + DOUT << "\nWith: "; DEBUG(RV.getNode()->dump(&DAG)); DOUT << '\n'; WorkListRemover DeadNodes(*this); - if (N->getNumValues() == RV.Val->getNumValues()) - DAG.ReplaceAllUsesWith(N, RV.Val, &DeadNodes); + if (N->getNumValues() == RV.getNode()->getNumValues()) + DAG.ReplaceAllUsesWith(N, RV.getNode(), &DeadNodes); else { assert(N->getValueType(0) == RV.getValueType() && N->getNumValues() == 1 && "Type mismatch"); - SDOperand OpV = RV; + SDValue OpV = RV; DAG.ReplaceAllUsesWith(N, &OpV, &DeadNodes); } // Push the new node and any users onto the worklist - AddToWorkList(RV.Val); - AddUsersToWorkList(RV.Val); + AddToWorkList(RV.getNode()); + AddUsersToWorkList(RV.getNode()); // Add any uses of the old node to the worklist in case this node is the // last one that uses them. They may become dead after this node is // deleted. for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) - AddToWorkList(N->getOperand(i).Val); + AddToWorkList(N->getOperand(i).getNode()); - // Nodes can be reintroduced into the worklist. Make sure we do not - // process a node that has been replaced. - removeFromWorkList(N); + // Finally, if the node is now dead, remove it from the graph. The node + // may not be dead if the replacement process recursively simplified to + // something else needing this node. + if (N->use_empty()) { + // Nodes can be reintroduced into the worklist. Make sure we do not + // process a node that has been replaced. + removeFromWorkList(N); - // Finally, since the node is now dead, remove it from the graph. - DAG.DeleteNode(N); + // Finally, since the node is now dead, remove it from the graph. + DAG.DeleteNode(N); + } } // If the root changed (e.g. it was a dead load, update the root). DAG.setRoot(Dummy.getValue()); } -SDOperand DAGCombiner::visit(SDNode *N) { +SDValue DAGCombiner::visit(SDNode *N) { switch(N->getOpcode()) { default: break; case ISD::TokenFactor: return visitTokenFactor(N); @@ -710,6 +740,7 @@ SDOperand DAGCombiner::visit(SDNode *N) { case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N); case ISD::TRUNCATE: return visitTRUNCATE(N); case ISD::BIT_CONVERT: return visitBIT_CONVERT(N); + case ISD::BUILD_PAIR: return visitBUILD_PAIR(N); case ISD::FADD: return visitFADD(N); case ISD::FSUB: return visitFSUB(N); case ISD::FMUL: return visitFMUL(N); @@ -735,15 +766,14 @@ SDOperand DAGCombiner::visit(SDNode *N) { case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N); case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::combine(SDNode *N) { - - SDOperand RV = visit(N); +SDValue DAGCombiner::combine(SDNode *N) { + SDValue RV = visit(N); // If nothing happened, try a target-specific DAG combine. - if (RV.Val == 0) { + if (RV.getNode() == 0) { assert(N->getOpcode() != ISD::DELETED_NODE && "Node was deleted but visit returned NULL!"); @@ -752,18 +782,36 @@ SDOperand DAGCombiner::combine(SDNode *N) { // Expose the DAG combiner to the target combiner impls. TargetLowering::DAGCombinerInfo - DagCombineInfo(DAG, !AfterLegalize, false, this); + DagCombineInfo(DAG, Level == Unrestricted, false, this); RV = TLI.PerformDAGCombine(N, DagCombineInfo); } } + // If N is a commutative binary node, try commuting it to enable more + // sdisel CSE. + if (RV.getNode() == 0 && + SelectionDAG::isCommutativeBinOp(N->getOpcode()) && + N->getNumValues() == 1) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + + // Constant operands are canonicalized to RHS. + if (isa(N0) || !isa(N1)) { + SDValue Ops[] = { N1, N0 }; + SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), + Ops, 2); + if (CSENode) + return SDValue(CSENode, 0); + } + } + return RV; } /// getInputChainForNode - Given a node, return its input chain if it has one, /// otherwise return a null sd operand. -static SDOperand getInputChainForNode(SDNode *N) { +static SDValue getInputChainForNode(SDNode *N) { if (unsigned NumOps = N->getNumOperands()) { if (N->getOperand(0).getValueType() == MVT::Other) return N->getOperand(0); @@ -773,21 +821,21 @@ static SDOperand getInputChainForNode(SDNode *N) { if (N->getOperand(i).getValueType() == MVT::Other) return N->getOperand(i); } - return SDOperand(0, 0); + return SDValue(); } -SDOperand DAGCombiner::visitTokenFactor(SDNode *N) { +SDValue DAGCombiner::visitTokenFactor(SDNode *N) { // If N has two operands, where one has an input chain equal to the other, // the 'other' chain is redundant. if (N->getNumOperands() == 2) { - if (getInputChainForNode(N->getOperand(0).Val) == N->getOperand(1)) + if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1)) return N->getOperand(0); - if (getInputChainForNode(N->getOperand(1).Val) == N->getOperand(0)) + if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0)) return N->getOperand(1); } SmallVector TFs; // List of token factors to visit. - SmallVector Ops; // Ops for replacing token factor. + SmallVector Ops; // Ops for replacing token factor. SmallPtrSet SeenOps; bool Changed = false; // If we should replace this token factor. @@ -801,7 +849,7 @@ SDOperand DAGCombiner::visitTokenFactor(SDNode *N) { // Check each of the operands. for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) { - SDOperand Op = TF->getOperand(i); + SDValue Op = TF->getOperand(i); switch (Op.getOpcode()) { case ISD::EntryToken: @@ -812,11 +860,11 @@ SDOperand DAGCombiner::visitTokenFactor(SDNode *N) { case ISD::TokenFactor: if ((CombinerAA || Op.hasOneUse()) && - std::find(TFs.begin(), TFs.end(), Op.Val) == TFs.end()) { + std::find(TFs.begin(), TFs.end(), Op.getNode()) == TFs.end()) { // Queue up for processing. - TFs.push_back(Op.Val); + TFs.push_back(Op.getNode()); // Clean up in case the token factor is removed. - AddToWorkList(Op.Val); + AddToWorkList(Op.getNode()); Changed = true; break; } @@ -824,7 +872,7 @@ SDOperand DAGCombiner::visitTokenFactor(SDNode *N) { default: // Only add if it isn't already in the list. - if (SeenOps.insert(Op.Val)) + if (SeenOps.insert(Op.getNode())) Ops.push_back(Op); else Changed = true; @@ -833,7 +881,7 @@ SDOperand DAGCombiner::visitTokenFactor(SDNode *N) { } } - SDOperand Result; + SDValue Result; // If we've change things around then replace token factor. if (Changed) { @@ -842,9 +890,10 @@ SDOperand DAGCombiner::visitTokenFactor(SDNode *N) { Result = DAG.getEntryNode(); } else { // New and improved token factor. - Result = DAG.getNode(ISD::TokenFactor, MVT::Other, &Ops[0], Ops.size()); + Result = DAG.getNode(ISD::TokenFactor, N->getDebugLoc(), + MVT::Other, &Ops[0], Ops.size()); } - + // Don't add users to work list. return CombineTo(N, Result, false); } @@ -853,46 +902,53 @@ SDOperand DAGCombiner::visitTokenFactor(SDNode *N) { } /// MERGE_VALUES can always be eliminated. -SDOperand DAGCombiner::visitMERGE_VALUES(SDNode *N) { +SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) { WorkListRemover DeadNodes(*this); for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, i), N->getOperand(i), + DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i), &DeadNodes); removeFromWorkList(N); DAG.DeleteNode(N); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + return SDValue(N, 0); // Return N so it doesn't get rechecked! } - static -SDOperand combineShlAddConstant(SDOperand N0, SDOperand N1, SelectionDAG &DAG) { - MVT::ValueType VT = N0.getValueType(); - SDOperand N00 = N0.getOperand(0); - SDOperand N01 = N0.getOperand(1); +SDValue combineShlAddConstant(DebugLoc DL, SDValue N0, SDValue N1, + SelectionDAG &DAG) { + MVT VT = N0.getValueType(); + SDValue N00 = N0.getOperand(0); + SDValue N01 = N0.getOperand(1); ConstantSDNode *N01C = dyn_cast(N01); - if (N01C && N00.getOpcode() == ISD::ADD && N00.Val->hasOneUse() && + + if (N01C && N00.getOpcode() == ISD::ADD && N00.getNode()->hasOneUse() && isa(N00.getOperand(1))) { - N0 = DAG.getNode(ISD::ADD, VT, - DAG.getNode(ISD::SHL, VT, N00.getOperand(0), N01), - DAG.getNode(ISD::SHL, VT, N00.getOperand(1), N01)); - return DAG.getNode(ISD::ADD, VT, N0, N1); + // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<getValueType(0); +SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, + SelectionDAG &DAG, const TargetLowering &TLI, + bool LegalOperations) { + MVT VT = N->getValueType(0); unsigned Opc = N->getOpcode(); bool isSlctCC = Slct.getOpcode() == ISD::SELECT_CC; - SDOperand LHS = isSlctCC ? Slct.getOperand(2) : Slct.getOperand(1); - SDOperand RHS = isSlctCC ? Slct.getOperand(3) : Slct.getOperand(2); + SDValue LHS = isSlctCC ? Slct.getOperand(2) : Slct.getOperand(1); + SDValue RHS = isSlctCC ? Slct.getOperand(3) : Slct.getOperand(2); ISD::CondCode CC = ISD::SETCC_INVALID; - if (isSlctCC) + + if (isSlctCC) { CC = cast(Slct.getOperand(4))->get(); - else { - SDOperand CCOp = Slct.getOperand(0); + } else { + SDValue CCOp = Slct.getOperand(0); if (CCOp.getOpcode() == ISD::SETCC) CC = cast(CCOp.getOperand(2))->get(); } @@ -901,48 +957,55 @@ SDOperand combineSelectAndUse(SDNode *N, SDOperand Slct, SDOperand OtherOp, bool InvCC = false; assert ((Opc == ISD::ADD || (Opc == ISD::SUB && Slct == N->getOperand(1))) && "Bad input!"); + if (LHS.getOpcode() == ISD::Constant && - cast(LHS)->isNullValue()) + cast(LHS)->isNullValue()) { DoXform = true; - else if (CC != ISD::SETCC_INVALID && - RHS.getOpcode() == ISD::Constant && - cast(RHS)->isNullValue()) { + } else if (CC != ISD::SETCC_INVALID && + RHS.getOpcode() == ISD::Constant && + cast(RHS)->isNullValue()) { std::swap(LHS, RHS); - SDOperand Op0 = Slct.getOperand(0); - bool isInt = MVT::isInteger(isSlctCC ? Op0.getValueType() - : Op0.getOperand(0).getValueType()); + SDValue Op0 = Slct.getOperand(0); + MVT OpVT = isSlctCC ? Op0.getValueType() : + Op0.getOperand(0).getValueType(); + bool isInt = OpVT.isInteger(); CC = ISD::getSetCCInverse(CC, isInt); + + if (LegalOperations && !TLI.isCondCodeLegal(CC, OpVT)) + return SDValue(); // Inverse operator isn't legal. + DoXform = true; InvCC = true; } if (DoXform) { - SDOperand Result = DAG.getNode(Opc, VT, OtherOp, RHS); + SDValue Result = DAG.getNode(Opc, RHS.getDebugLoc(), VT, OtherOp, RHS); if (isSlctCC) - return DAG.getSelectCC(OtherOp, Result, + return DAG.getSelectCC(N->getDebugLoc(), OtherOp, Result, Slct.getOperand(0), Slct.getOperand(1), CC); - SDOperand CCOp = Slct.getOperand(0); + SDValue CCOp = Slct.getOperand(0); if (InvCC) - CCOp = DAG.getSetCC(CCOp.getValueType(), CCOp.getOperand(0), - CCOp.getOperand(1), CC); - return DAG.getNode(ISD::SELECT, VT, CCOp, OtherOp, Result); + CCOp = DAG.getSetCC(Slct.getDebugLoc(), CCOp.getValueType(), + CCOp.getOperand(0), CCOp.getOperand(1), CC); + return DAG.getNode(ISD::SELECT, N->getDebugLoc(), VT, + CCOp, OtherOp, Result); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitADD(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitADD(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N0.getValueType(); + MVT VT = N0.getValueType(); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } - + // fold (add x, undef) -> undef if (N0.getOpcode() == ISD::UNDEF) return N0; @@ -950,44 +1013,85 @@ SDOperand DAGCombiner::visitADD(SDNode *N) { return N1; // fold (add c1, c2) -> c1+c2 if (N0C && N1C) - return DAG.getConstant(N0C->getValue() + N1C->getValue(), VT); + return DAG.FoldConstantArithmetic(ISD::ADD, VT, N0C, N1C); // canonicalize constant to RHS if (N0C && !N1C) - return DAG.getNode(ISD::ADD, VT, N1, N0); + return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N1, N0); // fold (add x, 0) -> x if (N1C && N1C->isNullValue()) return N0; + // fold (add Sym, c) -> Sym+c + if (GlobalAddressSDNode *GA = dyn_cast(N0)) + if (!LegalOperations && TLI.isOffsetFoldingLegal(GA) && N1C && + GA->getOpcode() == ISD::GlobalAddress) + return DAG.getGlobalAddress(GA->getGlobal(), VT, + GA->getOffset() + + (uint64_t)N1C->getSExtValue()); // fold ((c1-A)+c2) -> (c1+c2)-A if (N1C && N0.getOpcode() == ISD::SUB) if (ConstantSDNode *N0C = dyn_cast(N0.getOperand(0))) - return DAG.getNode(ISD::SUB, VT, - DAG.getConstant(N1C->getValue()+N0C->getValue(), VT), + return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, + DAG.getConstant(N1C->getAPIntValue()+ + N0C->getAPIntValue(), VT), N0.getOperand(1)); // reassociate add - SDOperand RADD = ReassociateOps(ISD::ADD, N0, N1); - if (RADD.Val != 0) + SDValue RADD = ReassociateOps(ISD::ADD, N->getDebugLoc(), N0, N1); + if (RADD.getNode() != 0) return RADD; // fold ((0-A) + B) -> B-A if (N0.getOpcode() == ISD::SUB && isa(N0.getOperand(0)) && cast(N0.getOperand(0))->isNullValue()) - return DAG.getNode(ISD::SUB, VT, N1, N0.getOperand(1)); + return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1, N0.getOperand(1)); // fold (A + (0-B)) -> A-B if (N1.getOpcode() == ISD::SUB && isa(N1.getOperand(0)) && cast(N1.getOperand(0))->isNullValue()) - return DAG.getNode(ISD::SUB, VT, N0, N1.getOperand(1)); + return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N0, N1.getOperand(1)); // fold (A+(B-A)) -> B if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1)) return N1.getOperand(0); - - if (!MVT::isVector(VT) && SimplifyDemandedBits(SDOperand(N, 0))) - return SDOperand(N, 0); + // fold ((B-A)+A) -> B + if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1)) + return N0.getOperand(0); + // fold (A+(B-(A+C))) to (B-C) + if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && + N0 == N1.getOperand(1).getOperand(0)) + return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1.getOperand(0), + N1.getOperand(1).getOperand(1)); + // fold (A+(B-(C+A))) to (B-C) + if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && + N0 == N1.getOperand(1).getOperand(1)) + return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, N1.getOperand(0), + N1.getOperand(1).getOperand(0)); + // fold (A+((B-A)+or-C)) to (B+or-C) + if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) && + N1.getOperand(0).getOpcode() == ISD::SUB && + N0 == N1.getOperand(0).getOperand(1)) + return DAG.getNode(N1.getOpcode(), N->getDebugLoc(), VT, + N1.getOperand(0).getOperand(0), N1.getOperand(1)); + + // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant + if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) { + SDValue N00 = N0.getOperand(0); + SDValue N01 = N0.getOperand(1); + SDValue N10 = N1.getOperand(0); + SDValue N11 = N1.getOperand(1); + + if (isa(N00) || isa(N10)) + return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, + DAG.getNode(ISD::ADD, N0.getDebugLoc(), VT, N00, N10), + DAG.getNode(ISD::ADD, N1.getDebugLoc(), VT, N01, N11)); + } + + if (!VT.isVector() && SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); // fold (a+b) -> (a|b) iff a and b share no bits. - if (MVT::isInteger(VT) && !MVT::isVector(VT)) { + if (VT.isInteger() && !VT.isVector()) { APInt LHSZero, LHSOne; APInt RHSZero, RHSOne; - APInt Mask = APInt::getAllOnesValue(MVT::getSizeInBits(VT)); + APInt Mask = APInt::getAllOnesValue(VT.getSizeInBits()); DAG.ComputeMaskedBits(N0, Mask, LHSZero, LHSOne); + if (LHSZero.getBoolValue()) { DAG.ComputeMaskedBits(N1, Mask, RHSZero, RHSOne); @@ -995,60 +1099,61 @@ SDOperand DAGCombiner::visitADD(SDNode *N) { // If all possibly-set bits on the RHS are clear on the LHS, return an OR. if ((RHSZero & (~LHSZero & Mask)) == (~LHSZero & Mask) || (LHSZero & (~RHSZero & Mask)) == (~RHSZero & Mask)) - return DAG.getNode(ISD::OR, VT, N0, N1); + return DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N1); } } // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<hasOneUse()) { - SDOperand Result = combineShlAddConstant(N0, N1, DAG); - if (Result.Val) return Result; + if (N0.getOpcode() == ISD::SHL && N0.getNode()->hasOneUse()) { + SDValue Result = combineShlAddConstant(N->getDebugLoc(), N0, N1, DAG); + if (Result.getNode()) return Result; } - if (N1.getOpcode() == ISD::SHL && N1.Val->hasOneUse()) { - SDOperand Result = combineShlAddConstant(N1, N0, DAG); - if (Result.Val) return Result; + if (N1.getOpcode() == ISD::SHL && N1.getNode()->hasOneUse()) { + SDValue Result = combineShlAddConstant(N->getDebugLoc(), N1, N0, DAG); + if (Result.getNode()) return Result; } // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c)) - if (N0.getOpcode() == ISD::SELECT && N0.Val->hasOneUse()) { - SDOperand Result = combineSelectAndUse(N, N0, N1, DAG); - if (Result.Val) return Result; + if (N0.getOpcode() == ISD::SELECT && N0.getNode()->hasOneUse()) { + SDValue Result = combineSelectAndUse(N, N0, N1, DAG, TLI, LegalOperations); + if (Result.getNode()) return Result; } - if (N1.getOpcode() == ISD::SELECT && N1.Val->hasOneUse()) { - SDOperand Result = combineSelectAndUse(N, N1, N0, DAG); - if (Result.Val) return Result; + if (N1.getOpcode() == ISD::SELECT && N1.getNode()->hasOneUse()) { + SDValue Result = combineSelectAndUse(N, N1, N0, DAG, TLI, LegalOperations); + if (Result.getNode()) return Result; } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitADDC(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitADDC(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N0.getValueType(); + MVT VT = N0.getValueType(); // If the flag result is dead, turn this into an ADD. if (N->hasNUsesOfValue(0, 1)) - return CombineTo(N, DAG.getNode(ISD::ADD, VT, N1, N0), - DAG.getNode(ISD::CARRY_FALSE, MVT::Flag)); + return CombineTo(N, DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N1, N0), + DAG.getNode(ISD::CARRY_FALSE, + N->getDebugLoc(), MVT::Flag)); // canonicalize constant to RHS. - if (N0C && !N1C) { - SDOperand Ops[] = { N1, N0 }; - return DAG.getNode(ISD::ADDC, N->getVTList(), Ops, 2); - } + if (N0C && !N1C) + return DAG.getNode(ISD::ADDC, N->getDebugLoc(), N->getVTList(), N1, N0); // fold (addc x, 0) -> x + no carry out if (N1C && N1C->isNullValue()) - return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, MVT::Flag)); + return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, + N->getDebugLoc(), MVT::Flag)); // fold (addc a, b) -> (or a, b), CARRY_FALSE iff a and b share no bits. APInt LHSZero, LHSOne; APInt RHSZero, RHSOne; - APInt Mask = APInt::getAllOnesValue(MVT::getSizeInBits(VT)); + APInt Mask = APInt::getAllOnesValue(VT.getSizeInBits()); DAG.ComputeMaskedBits(N0, Mask, LHSZero, LHSOne); + if (LHSZero.getBoolValue()) { DAG.ComputeMaskedBits(N1, Mask, RHSZero, RHSOne); @@ -1056,70 +1161,85 @@ SDOperand DAGCombiner::visitADDC(SDNode *N) { // If all possibly-set bits on the RHS are clear on the LHS, return an OR. if ((RHSZero & (~LHSZero & Mask)) == (~LHSZero & Mask) || (LHSZero & (~RHSZero & Mask)) == (~RHSZero & Mask)) - return CombineTo(N, DAG.getNode(ISD::OR, VT, N0, N1), - DAG.getNode(ISD::CARRY_FALSE, MVT::Flag)); + return CombineTo(N, DAG.getNode(ISD::OR, N->getDebugLoc(), VT, N0, N1), + DAG.getNode(ISD::CARRY_FALSE, + N->getDebugLoc(), MVT::Flag)); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitADDE(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - SDOperand CarryIn = N->getOperand(2); +SDValue DAGCombiner::visitADDE(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + SDValue CarryIn = N->getOperand(2); ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - //MVT::ValueType VT = N0.getValueType(); // canonicalize constant to RHS - if (N0C && !N1C) { - SDOperand Ops[] = { N1, N0, CarryIn }; - return DAG.getNode(ISD::ADDE, N->getVTList(), Ops, 3); - } + if (N0C && !N1C) + return DAG.getNode(ISD::ADDE, N->getDebugLoc(), N->getVTList(), + N1, N0, CarryIn); // fold (adde x, y, false) -> (addc x, y) - if (CarryIn.getOpcode() == ISD::CARRY_FALSE) { - SDOperand Ops[] = { N1, N0 }; - return DAG.getNode(ISD::ADDC, N->getVTList(), Ops, 2); - } + if (CarryIn.getOpcode() == ISD::CARRY_FALSE) + return DAG.getNode(ISD::ADDC, N->getDebugLoc(), N->getVTList(), N1, N0); - return SDOperand(); + return SDValue(); } - - -SDOperand DAGCombiner::visitSUB(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - ConstantSDNode *N0C = dyn_cast(N0.Val); - ConstantSDNode *N1C = dyn_cast(N1.Val); - MVT::ValueType VT = N0.getValueType(); +SDValue DAGCombiner::visitSUB(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + ConstantSDNode *N0C = dyn_cast(N0.getNode()); + ConstantSDNode *N1C = dyn_cast(N1.getNode()); + MVT VT = N0.getValueType(); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } - + // fold (sub x, x) -> 0 if (N0 == N1) return DAG.getConstant(0, N->getValueType(0)); // fold (sub c1, c2) -> c1-c2 if (N0C && N1C) - return DAG.getNode(ISD::SUB, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::SUB, VT, N0C, N1C); // fold (sub x, c) -> (add x, -c) if (N1C) - return DAG.getNode(ISD::ADD, VT, N0, DAG.getConstant(-N1C->getValue(), VT)); + return DAG.getNode(ISD::ADD, VT, N0, + DAG.getConstant(-N1C->getAPIntValue(), VT)); // fold (A+B)-A -> B if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1) return N0.getOperand(1); // fold (A+B)-B -> A if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1) - return N0.getOperand(0); + return N0.getOperand(0); + // fold ((A+(B+or-C))-B) -> A+or-C + if (N0.getOpcode() == ISD::ADD && + (N0.getOperand(1).getOpcode() == ISD::SUB || + N0.getOperand(1).getOpcode() == ISD::ADD) && + N0.getOperand(1).getOperand(0) == N1) + return DAG.getNode(N0.getOperand(1).getOpcode(), VT, N0.getOperand(0), + N0.getOperand(1).getOperand(1)); + // fold ((A+(C+B))-B) -> A+C + if (N0.getOpcode() == ISD::ADD && + N0.getOperand(1).getOpcode() == ISD::ADD && + N0.getOperand(1).getOperand(1) == N1) + return DAG.getNode(ISD::ADD, VT, N0.getOperand(0), + N0.getOperand(1).getOperand(0)); + // fold ((A-(B-C))-C) -> A-B + if (N0.getOpcode() == ISD::SUB && + N0.getOperand(1).getOpcode() == ISD::SUB && + N0.getOperand(1).getOperand(1) == N1) + return DAG.getNode(ISD::SUB, VT, N0.getOperand(0), + N0.getOperand(1).getOperand(0)); // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c)) - if (N1.getOpcode() == ISD::SELECT && N1.Val->hasOneUse()) { - SDOperand Result = combineSelectAndUse(N, N1, N0, DAG); - if (Result.Val) return Result; + if (N1.getOpcode() == ISD::SELECT && N1.getNode()->hasOneUse()) { + SDValue Result = combineSelectAndUse(N, N1, N0, DAG, TLI, LegalOperations); + if (Result.getNode()) return Result; } // If either operand of a sub is undef, the result is undef if (N0.getOpcode() == ISD::UNDEF) @@ -1127,20 +1247,35 @@ SDOperand DAGCombiner::visitSUB(SDNode *N) { if (N1.getOpcode() == ISD::UNDEF) return N1; - return SDOperand(); + // If the relocation model supports it, consider symbol offsets. + if (GlobalAddressSDNode *GA = dyn_cast(N0)) + if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) { + // fold (sub Sym, c) -> Sym-c + if (N1C && GA->getOpcode() == ISD::GlobalAddress) + return DAG.getGlobalAddress(GA->getGlobal(), VT, + GA->getOffset() - + (uint64_t)N1C->getSExtValue()); + // fold (sub Sym+c1, Sym+c2) -> c1-c2 + if (GlobalAddressSDNode *GB = dyn_cast(N1)) + if (GA->getGlobal() == GB->getGlobal()) + return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(), + VT); + } + + return SDValue(); } -SDOperand DAGCombiner::visitMUL(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitMUL(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N0.getValueType(); + MVT VT = N0.getValueType(); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } // fold (mul x, undef) -> 0 @@ -1148,7 +1283,7 @@ SDOperand DAGCombiner::visitMUL(SDNode *N) { return DAG.getConstant(0, VT); // fold (mul c1, c2) -> c1*c2 if (N0C && N1C) - return DAG.getNode(ISD::MUL, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::MUL, VT, N0C, N1C); // canonicalize constant to RHS if (N0C && !N1C) return DAG.getNode(ISD::MUL, VT, N1, N0); @@ -1159,47 +1294,48 @@ SDOperand DAGCombiner::visitMUL(SDNode *N) { if (N1C && N1C->isAllOnesValue()) return DAG.getNode(ISD::SUB, VT, DAG.getConstant(0, VT), N0); // fold (mul x, (1 << c)) -> x << c - if (N1C && isPowerOf2_64(N1C->getValue())) + if (N1C && N1C->getAPIntValue().isPowerOf2()) return DAG.getNode(ISD::SHL, VT, N0, - DAG.getConstant(Log2_64(N1C->getValue()), + DAG.getConstant(N1C->getAPIntValue().logBase2(), TLI.getShiftAmountTy())); // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c - if (N1C && isPowerOf2_64(-N1C->getSignExtended())) { + if (N1C && isPowerOf2_64(-N1C->getSExtValue())) { // FIXME: If the input is something that is easily negated (e.g. a // single-use add), we should put the negate there. return DAG.getNode(ISD::SUB, VT, DAG.getConstant(0, VT), DAG.getNode(ISD::SHL, VT, N0, - DAG.getConstant(Log2_64(-N1C->getSignExtended()), + DAG.getConstant(Log2_64(-N1C->getSExtValue()), TLI.getShiftAmountTy()))); } // (mul (shl X, c1), c2) -> (mul X, c2 << c1) if (N1C && N0.getOpcode() == ISD::SHL && isa(N0.getOperand(1))) { - SDOperand C3 = DAG.getNode(ISD::SHL, VT, N1, N0.getOperand(1)); - AddToWorkList(C3.Val); + SDValue C3 = DAG.getNode(ISD::SHL, VT, N1, N0.getOperand(1)); + AddToWorkList(C3.getNode()); return DAG.getNode(ISD::MUL, VT, N0.getOperand(0), C3); } // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one // use. { - SDOperand Sh(0,0), Y(0,0); + SDValue Sh(0,0), Y(0,0); // Check for both (mul (shl X, C), Y) and (mul Y, (shl X, C)). if (N0.getOpcode() == ISD::SHL && isa(N0.getOperand(1)) && - N0.Val->hasOneUse()) { + N0.getNode()->hasOneUse()) { Sh = N0; Y = N1; } else if (N1.getOpcode() == ISD::SHL && - isa(N1.getOperand(1)) && N1.Val->hasOneUse()) { + isa(N1.getOperand(1)) && + N1.getNode()->hasOneUse()) { Sh = N1; Y = N0; } - if (Sh.Val) { - SDOperand Mul = DAG.getNode(ISD::MUL, VT, Sh.getOperand(0), Y); + if (Sh.getNode()) { + SDValue Mul = DAG.getNode(ISD::MUL, VT, Sh.getOperand(0), Y); return DAG.getNode(ISD::SHL, VT, Mul, Sh.getOperand(1)); } } // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2) - if (N1C && N0.getOpcode() == ISD::ADD && N0.Val->hasOneUse() && + if (N1C && N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse() && isa(N0.getOperand(1))) { return DAG.getNode(ISD::ADD, VT, DAG.getNode(ISD::MUL, VT, N0.getOperand(0), N1), @@ -1207,79 +1343,79 @@ SDOperand DAGCombiner::visitMUL(SDNode *N) { } // reassociate mul - SDOperand RMUL = ReassociateOps(ISD::MUL, N0, N1); - if (RMUL.Val != 0) + SDValue RMUL = ReassociateOps(ISD::MUL, N->getDebugLoc(), N0, N1); + if (RMUL.getNode() != 0) return RMUL; - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitSDIV(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - ConstantSDNode *N0C = dyn_cast(N0.Val); - ConstantSDNode *N1C = dyn_cast(N1.Val); - MVT::ValueType VT = N->getValueType(0); +SDValue DAGCombiner::visitSDIV(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + ConstantSDNode *N0C = dyn_cast(N0.getNode()); + ConstantSDNode *N1C = dyn_cast(N1.getNode()); + MVT VT = N->getValueType(0); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } // fold (sdiv c1, c2) -> c1/c2 if (N0C && N1C && !N1C->isNullValue()) - return DAG.getNode(ISD::SDIV, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::SDIV, VT, N0C, N1C); // fold (sdiv X, 1) -> X - if (N1C && N1C->getSignExtended() == 1LL) + if (N1C && N1C->getSExtValue() == 1LL) return N0; // fold (sdiv X, -1) -> 0-X if (N1C && N1C->isAllOnesValue()) return DAG.getNode(ISD::SUB, VT, DAG.getConstant(0, VT), N0); // If we know the sign bits of both operands are zero, strength reduce to a // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2 - if (!MVT::isVector(VT)) { + if (!VT.isVector()) { if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0)) return DAG.getNode(ISD::UDIV, N1.getValueType(), N0, N1); } // fold (sdiv X, pow2) -> simple ops after legalize - if (N1C && N1C->getValue() && !TLI.isIntDivCheap() && - (isPowerOf2_64(N1C->getSignExtended()) || - isPowerOf2_64(-N1C->getSignExtended()))) { + if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap() && + (isPowerOf2_64(N1C->getSExtValue()) || + isPowerOf2_64(-N1C->getSExtValue()))) { // If dividing by powers of two is cheap, then don't perform the following // fold. if (TLI.isPow2DivCheap()) - return SDOperand(); - int64_t pow2 = N1C->getSignExtended(); + return SDValue(); + int64_t pow2 = N1C->getSExtValue(); int64_t abs2 = pow2 > 0 ? pow2 : -pow2; unsigned lg2 = Log2_64(abs2); // Splat the sign bit into the register - SDOperand SGN = DAG.getNode(ISD::SRA, VT, N0, - DAG.getConstant(MVT::getSizeInBits(VT)-1, + SDValue SGN = DAG.getNode(ISD::SRA, VT, N0, + DAG.getConstant(VT.getSizeInBits()-1, TLI.getShiftAmountTy())); - AddToWorkList(SGN.Val); + AddToWorkList(SGN.getNode()); // Add (N0 < 0) ? abs2 - 1 : 0; - SDOperand SRL = DAG.getNode(ISD::SRL, VT, SGN, - DAG.getConstant(MVT::getSizeInBits(VT)-lg2, + SDValue SRL = DAG.getNode(ISD::SRL, VT, SGN, + DAG.getConstant(VT.getSizeInBits()-lg2, TLI.getShiftAmountTy())); - SDOperand ADD = DAG.getNode(ISD::ADD, VT, N0, SRL); - AddToWorkList(SRL.Val); - AddToWorkList(ADD.Val); // Divide by pow2 - SDOperand SRA = DAG.getNode(ISD::SRA, VT, ADD, + SDValue ADD = DAG.getNode(ISD::ADD, VT, N0, SRL); + AddToWorkList(SRL.getNode()); + AddToWorkList(ADD.getNode()); // Divide by pow2 + SDValue SRA = DAG.getNode(ISD::SRA, VT, ADD, DAG.getConstant(lg2, TLI.getShiftAmountTy())); // If we're dividing by a positive value, we're done. Otherwise, we must // negate the result. if (pow2 > 0) return SRA; - AddToWorkList(SRA.Val); + AddToWorkList(SRA.getNode()); return DAG.getNode(ISD::SUB, VT, DAG.getConstant(0, VT), SRA); } // if integer divide is expensive and we satisfy the requirements, emit an // alternate sequence. - if (N1C && (N1C->getSignExtended() < -1 || N1C->getSignExtended() > 1) && + if (N1C && (N1C->getSExtValue() < -1 || N1C->getSExtValue() > 1) && !TLI.isIntDivCheap()) { - SDOperand Op = BuildSDIV(N); - if (Op.Val) return Op; + SDValue Op = BuildSDIV(N); + if (Op.getNode()) return Op; } // undef / X -> 0 @@ -1289,47 +1425,48 @@ SDOperand DAGCombiner::visitSDIV(SDNode *N) { if (N1.getOpcode() == ISD::UNDEF) return N1; - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitUDIV(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - ConstantSDNode *N0C = dyn_cast(N0.Val); - ConstantSDNode *N1C = dyn_cast(N1.Val); - MVT::ValueType VT = N->getValueType(0); +SDValue DAGCombiner::visitUDIV(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + ConstantSDNode *N0C = dyn_cast(N0.getNode()); + ConstantSDNode *N1C = dyn_cast(N1.getNode()); + MVT VT = N->getValueType(0); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } // fold (udiv c1, c2) -> c1/c2 if (N0C && N1C && !N1C->isNullValue()) - return DAG.getNode(ISD::UDIV, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::UDIV, VT, N0C, N1C); // fold (udiv x, (1 << c)) -> x >>u c - if (N1C && isPowerOf2_64(N1C->getValue())) + if (N1C && N1C->getAPIntValue().isPowerOf2()) return DAG.getNode(ISD::SRL, VT, N0, - DAG.getConstant(Log2_64(N1C->getValue()), + DAG.getConstant(N1C->getAPIntValue().logBase2(), TLI.getShiftAmountTy())); // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2 if (N1.getOpcode() == ISD::SHL) { if (ConstantSDNode *SHC = dyn_cast(N1.getOperand(0))) { - if (isPowerOf2_64(SHC->getValue())) { - MVT::ValueType ADDVT = N1.getOperand(1).getValueType(); - SDOperand Add = DAG.getNode(ISD::ADD, ADDVT, N1.getOperand(1), - DAG.getConstant(Log2_64(SHC->getValue()), + if (SHC->getAPIntValue().isPowerOf2()) { + MVT ADDVT = N1.getOperand(1).getValueType(); + SDValue Add = DAG.getNode(ISD::ADD, ADDVT, N1.getOperand(1), + DAG.getConstant(SHC->getAPIntValue() + .logBase2(), ADDVT)); - AddToWorkList(Add.Val); + AddToWorkList(Add.getNode()); return DAG.getNode(ISD::SRL, VT, N0, Add); } } } // fold (udiv x, c) -> alternate - if (N1C && N1C->getValue() && !TLI.isIntDivCheap()) { - SDOperand Op = BuildUDIV(N); - if (Op.Val) return Op; + if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap()) { + SDValue Op = BuildUDIV(N); + if (Op.getNode()) return Op; } // undef / X -> 0 @@ -1339,22 +1476,22 @@ SDOperand DAGCombiner::visitUDIV(SDNode *N) { if (N1.getOpcode() == ISD::UNDEF) return N1; - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitSREM(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitSREM(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold (srem c1, c2) -> c1%c2 if (N0C && N1C && !N1C->isNullValue()) - return DAG.getNode(ISD::SREM, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::SREM, VT, N0C, N1C); // If we know the sign bits of both operands are zero, strength reduce to a // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15 - if (!MVT::isVector(VT)) { + if (!VT.isVector()) { if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0)) return DAG.getNode(ISD::UREM, VT, N0, N1); } @@ -1362,13 +1499,13 @@ SDOperand DAGCombiner::visitSREM(SDNode *N) { // If X/C can be simplified by the division-by-constant logic, lower // X%C to the equivalent of X-X/C*C. if (N1C && !N1C->isNullValue()) { - SDOperand Div = DAG.getNode(ISD::SDIV, VT, N0, N1); - AddToWorkList(Div.Val); - SDOperand OptimizedDiv = combine(Div.Val); - if (OptimizedDiv.Val && OptimizedDiv.Val != Div.Val) { - SDOperand Mul = DAG.getNode(ISD::MUL, VT, OptimizedDiv, N1); - SDOperand Sub = DAG.getNode(ISD::SUB, VT, N0, Mul); - AddToWorkList(Mul.Val); + SDValue Div = DAG.getNode(ISD::SDIV, VT, N0, N1); + AddToWorkList(Div.getNode()); + SDValue OptimizedDiv = combine(Div.getNode()); + if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) { + SDValue Mul = DAG.getNode(ISD::MUL, VT, OptimizedDiv, N1); + SDValue Sub = DAG.getNode(ISD::SUB, VT, N0, Mul); + AddToWorkList(Mul.getNode()); return Sub; } } @@ -1380,28 +1517,32 @@ SDOperand DAGCombiner::visitSREM(SDNode *N) { if (N1.getOpcode() == ISD::UNDEF) return N1; - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitUREM(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitUREM(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold (urem c1, c2) -> c1%c2 if (N0C && N1C && !N1C->isNullValue()) - return DAG.getNode(ISD::UREM, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::UREM, VT, N0C, N1C); // fold (urem x, pow2) -> (and x, pow2-1) - if (N1C && !N1C->isNullValue() && isPowerOf2_64(N1C->getValue())) - return DAG.getNode(ISD::AND, VT, N0, DAG.getConstant(N1C->getValue()-1,VT)); + if (N1C && !N1C->isNullValue() && N1C->getAPIntValue().isPowerOf2()) + return DAG.getNode(ISD::AND, VT, N0, + DAG.getConstant(N1C->getAPIntValue()-1,VT)); // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1)) if (N1.getOpcode() == ISD::SHL) { if (ConstantSDNode *SHC = dyn_cast(N1.getOperand(0))) { - if (isPowerOf2_64(SHC->getValue())) { - SDOperand Add = DAG.getNode(ISD::ADD, VT, N1,DAG.getConstant(~0ULL,VT)); - AddToWorkList(Add.Val); + if (SHC->getAPIntValue().isPowerOf2()) { + SDValue Add = + DAG.getNode(ISD::ADD, VT, N1, + DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), + VT)); + AddToWorkList(Add.getNode()); return DAG.getNode(ISD::AND, VT, N0, Add); } } @@ -1410,12 +1551,13 @@ SDOperand DAGCombiner::visitUREM(SDNode *N) { // If X/C can be simplified by the division-by-constant logic, lower // X%C to the equivalent of X-X/C*C. if (N1C && !N1C->isNullValue()) { - SDOperand Div = DAG.getNode(ISD::UDIV, VT, N0, N1); - SDOperand OptimizedDiv = combine(Div.Val); - if (OptimizedDiv.Val && OptimizedDiv.Val != Div.Val) { - SDOperand Mul = DAG.getNode(ISD::MUL, VT, OptimizedDiv, N1); - SDOperand Sub = DAG.getNode(ISD::SUB, VT, N0, Mul); - AddToWorkList(Mul.Val); + SDValue Div = DAG.getNode(ISD::UDIV, VT, N0, N1); + AddToWorkList(Div.getNode()); + SDValue OptimizedDiv = combine(Div.getNode()); + if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) { + SDValue Mul = DAG.getNode(ISD::MUL, VT, OptimizedDiv, N1); + SDValue Sub = DAG.getNode(ISD::SUB, VT, N0, Mul); + AddToWorkList(Mul.getNode()); return Sub; } } @@ -1427,135 +1569,137 @@ SDOperand DAGCombiner::visitUREM(SDNode *N) { if (N1.getOpcode() == ISD::UNDEF) return N1; - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitMULHS(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitMULHS(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold (mulhs x, 0) -> 0 if (N1C && N1C->isNullValue()) return N1; // fold (mulhs x, 1) -> (sra x, size(x)-1) - if (N1C && N1C->getValue() == 1) + if (N1C && N1C->getAPIntValue() == 1) return DAG.getNode(ISD::SRA, N0.getValueType(), N0, - DAG.getConstant(MVT::getSizeInBits(N0.getValueType())-1, + DAG.getConstant(N0.getValueType().getSizeInBits()-1, TLI.getShiftAmountTy())); // fold (mulhs x, undef) -> 0 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) return DAG.getConstant(0, VT); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitMULHU(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitMULHU(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold (mulhu x, 0) -> 0 if (N1C && N1C->isNullValue()) return N1; // fold (mulhu x, 1) -> 0 - if (N1C && N1C->getValue() == 1) + if (N1C && N1C->getAPIntValue() == 1) return DAG.getConstant(0, N0.getValueType()); // fold (mulhu x, undef) -> 0 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) return DAG.getConstant(0, VT); - return SDOperand(); + return SDValue(); } /// SimplifyNodeWithTwoResults - Perform optimizations common to nodes that /// compute two values. LoOp and HiOp give the opcodes for the two computations /// that are being performed. Return true if a simplification was made. /// -SDOperand DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp, - unsigned HiOp) { +SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp, + unsigned HiOp) { // If the high half is not needed, just compute the low half. bool HiExists = N->hasAnyUseOfValue(1); if (!HiExists && - (!AfterLegalize || + (!LegalOperations || TLI.isOperationLegal(LoOp, N->getValueType(0)))) { - SDOperand Res = DAG.getNode(LoOp, N->getValueType(0), N->op_begin(), - N->getNumOperands()); + SDValue Res = DAG.getNode(LoOp, N->getValueType(0), N->op_begin(), + N->getNumOperands()); return CombineTo(N, Res, Res); } // If the low half is not needed, just compute the high half. bool LoExists = N->hasAnyUseOfValue(0); if (!LoExists && - (!AfterLegalize || + (!LegalOperations || TLI.isOperationLegal(HiOp, N->getValueType(1)))) { - SDOperand Res = DAG.getNode(HiOp, N->getValueType(1), N->op_begin(), - N->getNumOperands()); + SDValue Res = DAG.getNode(HiOp, N->getValueType(1), N->op_begin(), + N->getNumOperands()); return CombineTo(N, Res, Res); } // If both halves are used, return as it is. if (LoExists && HiExists) - return SDOperand(); + return SDValue(); // If the two computed results can be simplified separately, separate them. if (LoExists) { - SDOperand Lo = DAG.getNode(LoOp, N->getValueType(0), + SDValue Lo = DAG.getNode(LoOp, N->getValueType(0), N->op_begin(), N->getNumOperands()); - AddToWorkList(Lo.Val); - SDOperand LoOpt = combine(Lo.Val); - if (LoOpt.Val && LoOpt.Val != Lo.Val && - TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())) + AddToWorkList(Lo.getNode()); + SDValue LoOpt = combine(Lo.getNode()); + if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() && + (!LegalOperations || + TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType()))) return CombineTo(N, LoOpt, LoOpt); } if (HiExists) { - SDOperand Hi = DAG.getNode(HiOp, N->getValueType(1), - N->op_begin(), N->getNumOperands()); - AddToWorkList(Hi.Val); - SDOperand HiOpt = combine(Hi.Val); - if (HiOpt.Val && HiOpt != Hi && - TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())) + SDValue Hi = DAG.getNode(HiOp, N->getValueType(1), + N->op_begin(), N->getNumOperands()); + AddToWorkList(Hi.getNode()); + SDValue HiOpt = combine(Hi.getNode()); + if (HiOpt.getNode() && HiOpt != Hi && + (!LegalOperations || + TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType()))) return CombineTo(N, HiOpt, HiOpt); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitSMUL_LOHI(SDNode *N) { - SDOperand Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS); - if (Res.Val) return Res; +SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) { + SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS); + if (Res.getNode()) return Res; - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitUMUL_LOHI(SDNode *N) { - SDOperand Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU); - if (Res.Val) return Res; +SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) { + SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU); + if (Res.getNode()) return Res; - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitSDIVREM(SDNode *N) { - SDOperand Res = SimplifyNodeWithTwoResults(N, ISD::SDIV, ISD::SREM); - if (Res.Val) return Res; +SDValue DAGCombiner::visitSDIVREM(SDNode *N) { + SDValue Res = SimplifyNodeWithTwoResults(N, ISD::SDIV, ISD::SREM); + if (Res.getNode()) return Res; - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitUDIVREM(SDNode *N) { - SDOperand Res = SimplifyNodeWithTwoResults(N, ISD::UDIV, ISD::UREM); - if (Res.Val) return Res; +SDValue DAGCombiner::visitUDIVREM(SDNode *N) { + SDValue Res = SimplifyNodeWithTwoResults(N, ISD::UDIV, ISD::UREM); + if (Res.getNode()) return Res; - return SDOperand(); + return SDValue(); } /// SimplifyBinOpWithSameOpcodeHands - If this is a binary operator with /// two operands of the same opcode, try to simplify it. -SDOperand DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) { - SDOperand N0 = N->getOperand(0), N1 = N->getOperand(1); - MVT::ValueType VT = N0.getValueType(); +SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) { + SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); + MVT VT = N0.getValueType(); assert(N0.getOpcode() == N1.getOpcode() && "Bad input!"); // For each of OP in AND/OR/XOR: @@ -1566,10 +1710,10 @@ SDOperand DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) { if ((N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND|| N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::TRUNCATE) && N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType()) { - SDOperand ORNode = DAG.getNode(N->getOpcode(), + SDValue ORNode = DAG.getNode(N->getOpcode(), N0.getOperand(0).getValueType(), N0.getOperand(0), N1.getOperand(0)); - AddToWorkList(ORNode.Val); + AddToWorkList(ORNode.getNode()); return DAG.getNode(N0.getOpcode(), VT, ORNode); } @@ -1580,29 +1724,29 @@ SDOperand DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) { if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) && N0.getOperand(1) == N1.getOperand(1)) { - SDOperand ORNode = DAG.getNode(N->getOpcode(), + SDValue ORNode = DAG.getNode(N->getOpcode(), N0.getOperand(0).getValueType(), N0.getOperand(0), N1.getOperand(0)); - AddToWorkList(ORNode.Val); + AddToWorkList(ORNode.getNode()); return DAG.getNode(N0.getOpcode(), VT, ORNode, N0.getOperand(1)); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitAND(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - SDOperand LL, LR, RL, RR, CC0, CC1; +SDValue DAGCombiner::visitAND(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + SDValue LL, LR, RL, RR, CC0, CC1; ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N1.getValueType(); - unsigned BitWidth = MVT::getSizeInBits(VT); + MVT VT = N1.getValueType(); + unsigned BitWidth = VT.getSizeInBits(); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } // fold (and x, undef) -> 0 @@ -1610,7 +1754,7 @@ SDOperand DAGCombiner::visitAND(SDNode *N) { return DAG.getConstant(0, VT); // fold (and c1, c2) -> c1&c2 if (N0C && N1C) - return DAG.getNode(ISD::AND, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::AND, VT, N0C, N1C); // canonicalize constant to RHS if (N0C && !N1C) return DAG.getNode(ISD::AND, VT, N1, N0); @@ -1618,25 +1762,25 @@ SDOperand DAGCombiner::visitAND(SDNode *N) { if (N1C && N1C->isAllOnesValue()) return N0; // if (and x, c) is known to be zero, return 0 - if (N1C && DAG.MaskedValueIsZero(SDOperand(N, 0), + if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnesValue(BitWidth))) return DAG.getConstant(0, VT); // reassociate and - SDOperand RAND = ReassociateOps(ISD::AND, N0, N1); - if (RAND.Val != 0) + SDValue RAND = ReassociateOps(ISD::AND, N->getDebugLoc(), N0, N1); + if (RAND.getNode() != 0) return RAND; // fold (and (or x, 0xFFFF), 0xFF) -> 0xFF if (N1C && N0.getOpcode() == ISD::OR) if (ConstantSDNode *ORI = dyn_cast(N0.getOperand(1))) - if ((ORI->getValue() & N1C->getValue()) == N1C->getValue()) + if ((ORI->getAPIntValue() & N1C->getAPIntValue()) == N1C->getAPIntValue()) return N1; // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits. if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { - SDOperand N0Op0 = N0.getOperand(0); + SDValue N0Op0 = N0.getOperand(0); APInt Mask = ~N1C->getAPIntValue(); Mask.trunc(N0Op0.getValueSizeInBits()); if (DAG.MaskedValueIsZero(N0Op0, Mask)) { - SDOperand Zext = DAG.getNode(ISD::ZERO_EXTEND, N0.getValueType(), + SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, N0.getValueType(), N0Op0); // Replace uses of the AND with uses of the Zero extend node. @@ -1645,8 +1789,8 @@ SDOperand DAGCombiner::visitAND(SDNode *N) { // We actually want to replace all uses of the any_extend with the // zero_extend, to avoid duplicating things. This will later cause this // AND to be folded. - CombineTo(N0.Val, Zext); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + CombineTo(N0.getNode(), Zext); + return SDValue(N, 0); // Return N so it doesn't get rechecked! } } // fold (and (setcc x), (setcc y)) -> (setcc (and x, y)) @@ -1655,23 +1799,23 @@ SDOperand DAGCombiner::visitAND(SDNode *N) { ISD::CondCode Op1 = cast(CC1)->get(); if (LR == RR && isa(LR) && Op0 == Op1 && - MVT::isInteger(LL.getValueType())) { + LL.getValueType().isInteger()) { // fold (X == 0) & (Y == 0) -> (X|Y == 0) - if (cast(LR)->getValue() == 0 && Op1 == ISD::SETEQ) { - SDOperand ORNode = DAG.getNode(ISD::OR, LR.getValueType(), LL, RL); - AddToWorkList(ORNode.Val); + if (cast(LR)->isNullValue() && Op1 == ISD::SETEQ) { + SDValue ORNode = DAG.getNode(ISD::OR, LR.getValueType(), LL, RL); + AddToWorkList(ORNode.getNode()); return DAG.getSetCC(VT, ORNode, LR, Op1); } // fold (X == -1) & (Y == -1) -> (X&Y == -1) if (cast(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) { - SDOperand ANDNode = DAG.getNode(ISD::AND, LR.getValueType(), LL, RL); - AddToWorkList(ANDNode.Val); + SDValue ANDNode = DAG.getNode(ISD::AND, LR.getValueType(), LL, RL); + AddToWorkList(ANDNode.getNode()); return DAG.getSetCC(VT, ANDNode, LR, Op1); } // fold (X > -1) & (Y > -1) -> (X|Y > -1) if (cast(LR)->isAllOnesValue() && Op1 == ISD::SETGT) { - SDOperand ORNode = DAG.getNode(ISD::OR, LR.getValueType(), LL, RL); - AddToWorkList(ORNode.Val); + SDValue ORNode = DAG.getNode(ISD::OR, LR.getValueType(), LL, RL); + AddToWorkList(ORNode.getNode()); return DAG.getSetCC(VT, ORNode, LR, Op1); } } @@ -1681,63 +1825,64 @@ SDOperand DAGCombiner::visitAND(SDNode *N) { std::swap(RL, RR); } if (LL == RL && LR == RR) { - bool isInteger = MVT::isInteger(LL.getValueType()); + bool isInteger = LL.getValueType().isInteger(); ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger); - if (Result != ISD::SETCC_INVALID) + if (Result != ISD::SETCC_INVALID && + (!LegalOperations || TLI.isCondCodeLegal(Result, LL.getValueType()))) return DAG.getSetCC(N0.getValueType(), LL, LR, Result); } } // Simplify: and (op x...), (op y...) -> (op (and x, y)) if (N0.getOpcode() == N1.getOpcode()) { - SDOperand Tmp = SimplifyBinOpWithSameOpcodeHands(N); - if (Tmp.Val) return Tmp; + SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N); + if (Tmp.getNode()) return Tmp; } // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1) // fold (and (sra)) -> (and (srl)) when possible. - if (!MVT::isVector(VT) && - SimplifyDemandedBits(SDOperand(N, 0))) - return SDOperand(N, 0); + if (!VT.isVector() && + SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); // fold (zext_inreg (extload x)) -> (zextload x) - if (ISD::isEXTLoad(N0.Val) && ISD::isUNINDEXEDLoad(N0.Val)) { + if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) { LoadSDNode *LN0 = cast(N0); - MVT::ValueType EVT = LN0->getMemoryVT(); + MVT EVT = LN0->getMemoryVT(); // If we zero all the possible extended bits, then we can turn this into // a zextload if we are running before legalize or the operation is legal. unsigned BitWidth = N1.getValueSizeInBits(); if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth, - BitWidth - MVT::getSizeInBits(EVT))) && - (!AfterLegalize || TLI.isLoadXLegal(ISD::ZEXTLOAD, EVT))) { - SDOperand ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(), - LN0->getBasePtr(), LN0->getSrcValue(), - LN0->getSrcValueOffset(), EVT, - LN0->isVolatile(), - LN0->getAlignment()); + BitWidth - EVT.getSizeInBits())) && + ((!LegalOperations && !LN0->isVolatile()) || + TLI.isLoadExtLegal(ISD::ZEXTLOAD, EVT))) { + SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(), + LN0->getBasePtr(), LN0->getSrcValue(), + LN0->getSrcValueOffset(), EVT, + LN0->isVolatile(), LN0->getAlignment()); AddToWorkList(N); - CombineTo(N0.Val, ExtLoad, ExtLoad.getValue(1)); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! } } // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use - if (ISD::isSEXTLoad(N0.Val) && ISD::isUNINDEXEDLoad(N0.Val) && + if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { LoadSDNode *LN0 = cast(N0); - MVT::ValueType EVT = LN0->getMemoryVT(); + MVT EVT = LN0->getMemoryVT(); // If we zero all the possible extended bits, then we can turn this into // a zextload if we are running before legalize or the operation is legal. unsigned BitWidth = N1.getValueSizeInBits(); if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth, - BitWidth - MVT::getSizeInBits(EVT))) && - (!AfterLegalize || TLI.isLoadXLegal(ISD::ZEXTLOAD, EVT))) { - SDOperand ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(), - LN0->getBasePtr(), LN0->getSrcValue(), - LN0->getSrcValueOffset(), EVT, - LN0->isVolatile(), - LN0->getAlignment()); + BitWidth - EVT.getSizeInBits())) && + ((!LegalOperations && !LN0->isVolatile()) || + TLI.isLoadExtLegal(ISD::ZEXTLOAD, EVT))) { + SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(), + LN0->getBasePtr(), LN0->getSrcValue(), + LN0->getSrcValueOffset(), EVT, + LN0->isVolatile(), LN0->getAlignment()); AddToWorkList(N); - CombineTo(N0.Val, ExtLoad, ExtLoad.getValue(1)); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! } } @@ -1746,61 +1891,60 @@ SDOperand DAGCombiner::visitAND(SDNode *N) { if (N1C && N0.getOpcode() == ISD::LOAD) { LoadSDNode *LN0 = cast(N0); if (LN0->getExtensionType() != ISD::SEXTLOAD && - LN0->isUnindexed() && N0.hasOneUse()) { - MVT::ValueType EVT, LoadedVT; - if (N1C->getValue() == 255) - EVT = MVT::i8; - else if (N1C->getValue() == 65535) - EVT = MVT::i16; - else if (N1C->getValue() == ~0U) - EVT = MVT::i32; - else - EVT = MVT::Other; - - LoadedVT = LN0->getMemoryVT(); - if (EVT != MVT::Other && LoadedVT > EVT && - (!AfterLegalize || TLI.isLoadXLegal(ISD::ZEXTLOAD, EVT))) { - MVT::ValueType PtrType = N0.getOperand(1).getValueType(); + LN0->isUnindexed() && N0.hasOneUse() && + // Do not change the width of a volatile load. + !LN0->isVolatile()) { + MVT EVT = MVT::Other; + uint32_t ActiveBits = N1C->getAPIntValue().getActiveBits(); + if (ActiveBits > 0 && APIntOps::isMask(ActiveBits, N1C->getAPIntValue())) + EVT = MVT::getIntegerVT(ActiveBits); + + MVT LoadedVT = LN0->getMemoryVT(); + // Do not generate loads of non-round integer types since these can + // be expensive (and would be wrong if the type is not byte sized). + if (EVT != MVT::Other && LoadedVT.bitsGT(EVT) && EVT.isRound() && + (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, EVT))) { + MVT PtrType = N0.getOperand(1).getValueType(); // For big endian targets, we need to add an offset to the pointer to // load the correct bytes. For little endian systems, we merely need to // read fewer bytes from the same pointer. - unsigned LVTStoreBytes = MVT::getStoreSizeInBits(LoadedVT)/8; - unsigned EVTStoreBytes = MVT::getStoreSizeInBits(EVT)/8; + unsigned LVTStoreBytes = LoadedVT.getStoreSizeInBits()/8; + unsigned EVTStoreBytes = EVT.getStoreSizeInBits()/8; unsigned PtrOff = LVTStoreBytes - EVTStoreBytes; unsigned Alignment = LN0->getAlignment(); - SDOperand NewPtr = LN0->getBasePtr(); + SDValue NewPtr = LN0->getBasePtr(); if (TLI.isBigEndian()) { NewPtr = DAG.getNode(ISD::ADD, PtrType, NewPtr, DAG.getConstant(PtrOff, PtrType)); Alignment = MinAlign(Alignment, PtrOff); } - AddToWorkList(NewPtr.Val); - SDOperand Load = + AddToWorkList(NewPtr.getNode()); + SDValue Load = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(), NewPtr, LN0->getSrcValue(), LN0->getSrcValueOffset(), EVT, LN0->isVolatile(), Alignment); AddToWorkList(N); - CombineTo(N0.Val, Load, Load.getValue(1)); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + CombineTo(N0.getNode(), Load, Load.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! } } } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitOR(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - SDOperand LL, LR, RL, RR, CC0, CC1; +SDValue DAGCombiner::visitOR(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + SDValue LL, LR, RL, RR, CC0, CC1; ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N1.getValueType(); + MVT VT = N1.getValueType(); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } // fold (or x, undef) -> -1 @@ -1808,7 +1952,7 @@ SDOperand DAGCombiner::visitOR(SDNode *N) { return DAG.getConstant(~0ULL, VT); // fold (or c1, c2) -> c1|c2 if (N0C && N1C) - return DAG.getNode(ISD::OR, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::OR, VT, N0C, N1C); // canonicalize constant to RHS if (N0C && !N1C) return DAG.getNode(ISD::OR, VT, N1, N0); @@ -1822,16 +1966,17 @@ SDOperand DAGCombiner::visitOR(SDNode *N) { if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue())) return N1; // reassociate or - SDOperand ROR = ReassociateOps(ISD::OR, N0, N1); - if (ROR.Val != 0) + SDValue ROR = ReassociateOps(ISD::OR, N->getDebugLoc(), N0, N1); + if (ROR.getNode() != 0) return ROR; // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2) - if (N1C && N0.getOpcode() == ISD::AND && N0.Val->hasOneUse() && + if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && isa(N0.getOperand(1))) { ConstantSDNode *C1 = cast(N0.getOperand(1)); return DAG.getNode(ISD::AND, VT, DAG.getNode(ISD::OR, VT, N0.getOperand(0), N1), - DAG.getConstant(N1C->getValue() | C1->getValue(), VT)); + DAG.getConstant(N1C->getAPIntValue() | + C1->getAPIntValue(), VT)); } // fold (or (setcc x), (setcc y)) -> (setcc (or x, y)) if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){ @@ -1839,21 +1984,21 @@ SDOperand DAGCombiner::visitOR(SDNode *N) { ISD::CondCode Op1 = cast(CC1)->get(); if (LR == RR && isa(LR) && Op0 == Op1 && - MVT::isInteger(LL.getValueType())) { + LL.getValueType().isInteger()) { // fold (X != 0) | (Y != 0) -> (X|Y != 0) // fold (X < 0) | (Y < 0) -> (X|Y < 0) - if (cast(LR)->getValue() == 0 && + if (cast(LR)->isNullValue() && (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) { - SDOperand ORNode = DAG.getNode(ISD::OR, LR.getValueType(), LL, RL); - AddToWorkList(ORNode.Val); + SDValue ORNode = DAG.getNode(ISD::OR, LR.getValueType(), LL, RL); + AddToWorkList(ORNode.getNode()); return DAG.getSetCC(VT, ORNode, LR, Op1); } // fold (X != -1) | (Y != -1) -> (X&Y != -1) // fold (X > -1) | (Y > -1) -> (X&Y > -1) if (cast(LR)->isAllOnesValue() && (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) { - SDOperand ANDNode = DAG.getNode(ISD::AND, LR.getValueType(), LL, RL); - AddToWorkList(ANDNode.Val); + SDValue ANDNode = DAG.getNode(ISD::AND, LR.getValueType(), LL, RL); + AddToWorkList(ANDNode.getNode()); return DAG.getSetCC(VT, ANDNode, LR, Op1); } } @@ -1863,17 +2008,18 @@ SDOperand DAGCombiner::visitOR(SDNode *N) { std::swap(RL, RR); } if (LL == RL && LR == RR) { - bool isInteger = MVT::isInteger(LL.getValueType()); + bool isInteger = LL.getValueType().isInteger(); ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger); - if (Result != ISD::SETCC_INVALID) + if (Result != ISD::SETCC_INVALID && + (!LegalOperations || TLI.isCondCodeLegal(Result, LL.getValueType()))) return DAG.getSetCC(N0.getValueType(), LL, LR, Result); } } // Simplify: or (op x...), (op y...) -> (op (or x, y)) if (N0.getOpcode() == N1.getOpcode()) { - SDOperand Tmp = SimplifyBinOpWithSameOpcodeHands(N); - if (Tmp.Val) return Tmp; + SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N); + if (Tmp.getNode()) return Tmp; } // (X & C1) | (Y & C2) -> (X|Y) & C3 if possible. @@ -1882,7 +2028,7 @@ SDOperand DAGCombiner::visitOR(SDNode *N) { N0.getOperand(1).getOpcode() == ISD::Constant && N1.getOperand(1).getOpcode() == ISD::Constant && // Don't increase # computations. - (N0.Val->hasOneUse() || N1.Val->hasOneUse())) { + (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) { // We can only do this xform if we know that bits from X that are set in C2 // but not in C1 are already zero. Likewise for Y. const APInt &LHSMask = @@ -1892,7 +2038,7 @@ SDOperand DAGCombiner::visitOR(SDNode *N) { if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) && DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) { - SDOperand X =DAG.getNode(ISD::OR, VT, N0.getOperand(0), N1.getOperand(0)); + SDValue X =DAG.getNode(ISD::OR, VT, N0.getOperand(0), N1.getOperand(0)); return DAG.getNode(ISD::AND, VT, X, DAG.getConstant(LHSMask|RHSMask, VT)); } } @@ -1900,14 +2046,14 @@ SDOperand DAGCombiner::visitOR(SDNode *N) { // See if this is some rotate idiom. if (SDNode *Rot = MatchRotate(N0, N1)) - return SDOperand(Rot, 0); + return SDValue(Rot, 0); - return SDOperand(); + return SDValue(); } /// MatchRotateHalf - Match "(X shl/srl V1) & V2" where V2 may not be present. -static bool MatchRotateHalf(SDOperand Op, SDOperand &Shift, SDOperand &Mask) { +static bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) { if (Op.getOpcode() == ISD::AND) { if (isa(Op.getOperand(1))) { Mask = Op.getOperand(1); @@ -1928,24 +2074,24 @@ static bool MatchRotateHalf(SDOperand Op, SDOperand &Shift, SDOperand &Mask) { // MatchRotate - Handle an 'or' of two operands. If this is one of the many // idioms for rotate, and if the target supports rotation instructions, generate // a rot[lr]. -SDNode *DAGCombiner::MatchRotate(SDOperand LHS, SDOperand RHS) { - // Must be a legal type. Expanded an promoted things won't work with rotates. - MVT::ValueType VT = LHS.getValueType(); +SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS) { + // Must be a legal type. Expanded 'n promoted things won't work with rotates. + MVT VT = LHS.getValueType(); if (!TLI.isTypeLegal(VT)) return 0; // The target must have at least one rotate flavor. - bool HasROTL = TLI.isOperationLegal(ISD::ROTL, VT); - bool HasROTR = TLI.isOperationLegal(ISD::ROTR, VT); + bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT); + bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT); if (!HasROTL && !HasROTR) return 0; - + // Match "(X shl/srl V1) & V2" where V2 may not be present. - SDOperand LHSShift; // The shift. - SDOperand LHSMask; // AND value if any. + SDValue LHSShift; // The shift. + SDValue LHSMask; // AND value if any. if (!MatchRotateHalf(LHS, LHSShift, LHSMask)) return 0; // Not part of a rotate. - SDOperand RHSShift; // The shift. - SDOperand RHSMask; // AND value if any. + SDValue RHSShift; // The shift. + SDValue RHSMask; // AND value if any. if (!MatchRotateHalf(RHS, RHSShift, RHSMask)) return 0; // Not part of a rotate. @@ -1962,35 +2108,35 @@ SDNode *DAGCombiner::MatchRotate(SDOperand LHS, SDOperand RHS) { std::swap(LHSMask , RHSMask ); } - unsigned OpSizeInBits = MVT::getSizeInBits(VT); - SDOperand LHSShiftArg = LHSShift.getOperand(0); - SDOperand LHSShiftAmt = LHSShift.getOperand(1); - SDOperand RHSShiftAmt = RHSShift.getOperand(1); + unsigned OpSizeInBits = VT.getSizeInBits(); + SDValue LHSShiftArg = LHSShift.getOperand(0); + SDValue LHSShiftAmt = LHSShift.getOperand(1); + SDValue RHSShiftAmt = RHSShift.getOperand(1); // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1) // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2) if (LHSShiftAmt.getOpcode() == ISD::Constant && RHSShiftAmt.getOpcode() == ISD::Constant) { - uint64_t LShVal = cast(LHSShiftAmt)->getValue(); - uint64_t RShVal = cast(RHSShiftAmt)->getValue(); + uint64_t LShVal = cast(LHSShiftAmt)->getZExtValue(); + uint64_t RShVal = cast(RHSShiftAmt)->getZExtValue(); if ((LShVal + RShVal) != OpSizeInBits) return 0; - SDOperand Rot; + SDValue Rot; if (HasROTL) Rot = DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt); else Rot = DAG.getNode(ISD::ROTR, VT, LHSShiftArg, RHSShiftAmt); // If there is an AND of either shifted operand, apply it to the result. - if (LHSMask.Val || RHSMask.Val) { + if (LHSMask.getNode() || RHSMask.getNode()) { APInt Mask = APInt::getAllOnesValue(OpSizeInBits); - if (LHSMask.Val) { + if (LHSMask.getNode()) { APInt RHSBits = APInt::getLowBitsSet(OpSizeInBits, LShVal); Mask &= cast(LHSMask)->getAPIntValue() | RHSBits; } - if (RHSMask.Val) { + if (RHSMask.getNode()) { APInt LHSBits = APInt::getHighBitsSet(OpSizeInBits, RShVal); Mask &= cast(RHSMask)->getAPIntValue() | LHSBits; } @@ -1998,12 +2144,12 @@ SDNode *DAGCombiner::MatchRotate(SDOperand LHS, SDOperand RHS) { Rot = DAG.getNode(ISD::AND, VT, Rot, DAG.getConstant(Mask, VT)); } - return Rot.Val; + return Rot.getNode(); } // If there is a mask here, and we have a variable shift, we can't be sure // that we're masking out the right stuff. - if (LHSMask.Val || RHSMask.Val) + if (LHSMask.getNode() || RHSMask.getNode()) return 0; // fold (or (shl x, y), (srl x, (sub 32, y))) -> (rotl x, y) @@ -2012,11 +2158,11 @@ SDNode *DAGCombiner::MatchRotate(SDOperand LHS, SDOperand RHS) { LHSShiftAmt == RHSShiftAmt.getOperand(1)) { if (ConstantSDNode *SUBC = dyn_cast(RHSShiftAmt.getOperand(0))) { - if (SUBC->getValue() == OpSizeInBits) { + if (SUBC->getAPIntValue() == OpSizeInBits) { if (HasROTL) - return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt).Val; + return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt).getNode(); else - return DAG.getNode(ISD::ROTR, VT, LHSShiftArg, RHSShiftAmt).Val; + return DAG.getNode(ISD::ROTR, VT, LHSShiftArg, RHSShiftAmt).getNode(); } } } @@ -2027,50 +2173,50 @@ SDNode *DAGCombiner::MatchRotate(SDOperand LHS, SDOperand RHS) { RHSShiftAmt == LHSShiftAmt.getOperand(1)) { if (ConstantSDNode *SUBC = dyn_cast(LHSShiftAmt.getOperand(0))) { - if (SUBC->getValue() == OpSizeInBits) { - if (HasROTL) - return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt).Val; + if (SUBC->getAPIntValue() == OpSizeInBits) { + if (HasROTR) + return DAG.getNode(ISD::ROTR, VT, LHSShiftArg, RHSShiftAmt).getNode(); else - return DAG.getNode(ISD::ROTR, VT, LHSShiftArg, RHSShiftAmt).Val; + return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt).getNode(); } } } - // Look for sign/zext/any-extended cases: + // Look for sign/zext/any-extended or truncate cases: if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND - || LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND) && + || LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND + || LHSShiftAmt.getOpcode() == ISD::TRUNCATE) && (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND - || RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND)) { - SDOperand LExtOp0 = LHSShiftAmt.getOperand(0); - SDOperand RExtOp0 = RHSShiftAmt.getOperand(0); + || RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND + || RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) { + SDValue LExtOp0 = LHSShiftAmt.getOperand(0); + SDValue RExtOp0 = RHSShiftAmt.getOperand(0); if (RExtOp0.getOpcode() == ISD::SUB && RExtOp0.getOperand(1) == LExtOp0) { // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) -> - // (rotr x, y) + // (rotl x, y) // fold (or (shl x, (*ext y)), (srl x, (*ext (sub 32, y)))) -> - // (rotl x, (sub 32, y)) - if (ConstantSDNode *SUBC = cast(RExtOp0.getOperand(0))) { - if (SUBC->getValue() == OpSizeInBits) { - if (HasROTL) - return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt).Val; - else - return DAG.getNode(ISD::ROTR, VT, LHSShiftArg, RHSShiftAmt).Val; + // (rotr x, (sub 32, y)) + if (ConstantSDNode *SUBC = + dyn_cast(RExtOp0.getOperand(0))) { + if (SUBC->getAPIntValue() == OpSizeInBits) { + return DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, VT, LHSShiftArg, + HasROTL ? LHSShiftAmt : RHSShiftAmt).getNode(); } } } else if (LExtOp0.getOpcode() == ISD::SUB && RExtOp0 == LExtOp0.getOperand(1)) { - // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext r))) -> - // (rotl x, y) - // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext r))) -> - // (rotr x, (sub 32, y)) - if (ConstantSDNode *SUBC = cast(LExtOp0.getOperand(0))) { - if (SUBC->getValue() == OpSizeInBits) { - if (HasROTL) - return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, RHSShiftAmt).Val; - else - return DAG.getNode(ISD::ROTL, VT, LHSShiftArg, LHSShiftAmt).Val; + // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext y))) -> + // (rotr x, y) + // fold (or (shl x, (*ext (sub 32, y))), (srl x, (*ext y))) -> + // (rotl x, (sub 32, y)) + if (ConstantSDNode *SUBC = + dyn_cast(LExtOp0.getOperand(0))) { + if (SUBC->getAPIntValue() == OpSizeInBits) { + return DAG.getNode(HasROTR ? ISD::ROTR : ISD::ROTL, VT, LHSShiftArg, + HasROTR ? RHSShiftAmt : LHSShiftAmt).getNode(); } } } @@ -2080,20 +2226,23 @@ SDNode *DAGCombiner::MatchRotate(SDOperand LHS, SDOperand RHS) { } -SDOperand DAGCombiner::visitXOR(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - SDOperand LHS, RHS, CC; +SDValue DAGCombiner::visitXOR(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + SDValue LHS, RHS, CC; ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N0.getValueType(); + MVT VT = N0.getValueType(); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } + // fold (xor undef, undef) -> 0. This is a common idiom (misuse). + if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF) + return DAG.getConstant(0, VT); // fold (xor x, undef) -> undef if (N0.getOpcode() == ISD::UNDEF) return N0; @@ -2101,7 +2250,7 @@ SDOperand DAGCombiner::visitXOR(SDNode *N) { return N1; // fold (xor c1, c2) -> c1^c2 if (N0C && N1C) - return DAG.getNode(ISD::XOR, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::XOR, VT, N0C, N1C); // canonicalize constant to RHS if (N0C && !N1C) return DAG.getNode(ISD::XOR, VT, N1, N0); @@ -2109,52 +2258,62 @@ SDOperand DAGCombiner::visitXOR(SDNode *N) { if (N1C && N1C->isNullValue()) return N0; // reassociate xor - SDOperand RXOR = ReassociateOps(ISD::XOR, N0, N1); - if (RXOR.Val != 0) + SDValue RXOR = ReassociateOps(ISD::XOR, N->getDebugLoc(), N0, N1); + if (RXOR.getNode() != 0) return RXOR; + // fold !(x cc y) -> (x !cc y) - if (N1C && N1C->getValue() == 1 && isSetCCEquivalent(N0, LHS, RHS, CC)) { - bool isInt = MVT::isInteger(LHS.getValueType()); + if (N1C && N1C->getAPIntValue() == 1 && isSetCCEquivalent(N0, LHS, RHS, CC)) { + bool isInt = LHS.getValueType().isInteger(); ISD::CondCode NotCC = ISD::getSetCCInverse(cast(CC)->get(), isInt); - if (N0.getOpcode() == ISD::SETCC) - return DAG.getSetCC(VT, LHS, RHS, NotCC); - if (N0.getOpcode() == ISD::SELECT_CC) - return DAG.getSelectCC(LHS, RHS, N0.getOperand(2),N0.getOperand(3),NotCC); - assert(0 && "Unhandled SetCC Equivalent!"); - abort(); + + if (!LegalOperations || TLI.isCondCodeLegal(NotCC, LHS.getValueType())) { + switch (N0.getOpcode()) { + default: + assert(0 && "Unhandled SetCC Equivalent!"); + abort(); + case ISD::SETCC: + return DAG.getSetCC(VT, LHS, RHS, NotCC); + case ISD::SELECT_CC: + return DAG.getSelectCC(LHS, RHS, N0.getOperand(2), + N0.getOperand(3), NotCC); + } + } } + // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y))) - if (N1C && N1C->getValue() == 1 && N0.getOpcode() == ISD::ZERO_EXTEND && - N0.Val->hasOneUse() && isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){ - SDOperand V = N0.getOperand(0); + if (N1C && N1C->getAPIntValue() == 1 && N0.getOpcode() == ISD::ZERO_EXTEND && + N0.getNode()->hasOneUse() && + isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){ + SDValue V = N0.getOperand(0); V = DAG.getNode(ISD::XOR, V.getValueType(), V, DAG.getConstant(1, V.getValueType())); - AddToWorkList(V.Val); + AddToWorkList(V.getNode()); return DAG.getNode(ISD::ZERO_EXTEND, VT, V); } // fold !(x or y) -> (!x and !y) iff x or y are setcc - if (N1C && N1C->getValue() == 1 && VT == MVT::i1 && + if (N1C && N1C->getAPIntValue() == 1 && VT == MVT::i1 && (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) { - SDOperand LHS = N0.getOperand(0), RHS = N0.getOperand(1); + SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1); if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) { unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND; LHS = DAG.getNode(ISD::XOR, VT, LHS, N1); // RHS = ~LHS RHS = DAG.getNode(ISD::XOR, VT, RHS, N1); // RHS = ~RHS - AddToWorkList(LHS.Val); AddToWorkList(RHS.Val); + AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode()); return DAG.getNode(NewOpcode, VT, LHS, RHS); } } // fold !(x or y) -> (!x and !y) iff x or y are constants if (N1C && N1C->isAllOnesValue() && (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) { - SDOperand LHS = N0.getOperand(0), RHS = N0.getOperand(1); + SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1); if (isa(RHS) || isa(LHS)) { unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND; LHS = DAG.getNode(ISD::XOR, VT, LHS, N1); // RHS = ~LHS RHS = DAG.getNode(ISD::XOR, VT, RHS, N1); // RHS = ~RHS - AddToWorkList(LHS.Val); AddToWorkList(RHS.Val); + AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode()); return DAG.getNode(NewOpcode, VT, LHS, RHS); } } @@ -2164,42 +2323,44 @@ SDOperand DAGCombiner::visitXOR(SDNode *N) { ConstantSDNode *N01C = dyn_cast(N0.getOperand(1)); if (N00C) return DAG.getNode(ISD::XOR, VT, N0.getOperand(1), - DAG.getConstant(N1C->getValue()^N00C->getValue(), VT)); + DAG.getConstant(N1C->getAPIntValue()^ + N00C->getAPIntValue(), VT)); if (N01C) return DAG.getNode(ISD::XOR, VT, N0.getOperand(0), - DAG.getConstant(N1C->getValue()^N01C->getValue(), VT)); + DAG.getConstant(N1C->getAPIntValue()^ + N01C->getAPIntValue(), VT)); } // fold (xor x, x) -> 0 if (N0 == N1) { - if (!MVT::isVector(VT)) { + if (!VT.isVector()) { return DAG.getConstant(0, VT); - } else if (!AfterLegalize || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) { + } else if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)){ // Produce a vector of zeros. - SDOperand El = DAG.getConstant(0, MVT::getVectorElementType(VT)); - std::vector Ops(MVT::getVectorNumElements(VT), El); + SDValue El = DAG.getConstant(0, VT.getVectorElementType()); + std::vector Ops(VT.getVectorNumElements(), El); return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } } // Simplify: xor (op x...), (op y...) -> (op (xor x, y)) if (N0.getOpcode() == N1.getOpcode()) { - SDOperand Tmp = SimplifyBinOpWithSameOpcodeHands(N); - if (Tmp.Val) return Tmp; + SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N); + if (Tmp.getNode()) return Tmp; } // Simplify the expression using non-local knowledge. - if (!MVT::isVector(VT) && - SimplifyDemandedBits(SDOperand(N, 0))) - return SDOperand(N, 0); + if (!VT.isVector() && + SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); - return SDOperand(); + return SDValue(); } /// visitShiftByConstant - Handle transforms common to the three shifts, when /// the shift amount is a constant. -SDOperand DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) { - SDNode *LHS = N->getOperand(0).Val; - if (!LHS->hasOneUse()) return SDOperand(); +SDValue DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) { + SDNode *LHS = N->getOperand(0).getNode(); + if (!LHS->hasOneUse()) return SDValue(); // We want to pull some binops through shifts, so that we have (and (shift)) // instead of (shift (and)), likewise for add, or, xor, etc. This sort of @@ -2208,7 +2369,7 @@ SDOperand DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) { bool HighBitSet = false; // Can we transform this if the high bit is set? switch (LHS->getOpcode()) { - default: return SDOperand(); + default: return SDValue(); case ISD::OR: case ISD::XOR: HighBitSet = false; // We can only transform sra if the high bit is clear. @@ -2218,14 +2379,14 @@ SDOperand DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) { break; case ISD::ADD: if (N->getOpcode() != ISD::SHL) - return SDOperand(); // only shl(add) not sr[al](add). + return SDValue(); // only shl(add) not sr[al](add). HighBitSet = false; // We can only transform sra if the high bit is clear. break; } // We require the RHS of the binop to be a constant as well. ConstantSDNode *BinOpCst = dyn_cast(LHS->getOperand(1)); - if (!BinOpCst) return SDOperand(); + if (!BinOpCst) return SDValue(); // FIXME: disable this for unless the input to the binop is a shift by a @@ -2233,14 +2394,14 @@ SDOperand DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) { // //void foo(int *X, int i) { X[i & 1235] = 1; } //int bar(int *X, int i) { return X[i & 255]; } - SDNode *BinOpLHSVal = LHS->getOperand(0).Val; + SDNode *BinOpLHSVal = LHS->getOperand(0).getNode(); if ((BinOpLHSVal->getOpcode() != ISD::SHL && BinOpLHSVal->getOpcode() != ISD::SRA && BinOpLHSVal->getOpcode() != ISD::SRL) || !isa(BinOpLHSVal->getOperand(1))) - return SDOperand(); + return SDValue(); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // If this is a signed shift right, and the high bit is modified // by the logical operation, do not perform the transformation. @@ -2250,15 +2411,15 @@ SDOperand DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) { if (N->getOpcode() == ISD::SRA) { bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative(); if (BinOpRHSSignSet != HighBitSet) - return SDOperand(); + return SDValue(); } // Fold the constants, shifting the binop RHS by the shift amount. - SDOperand NewRHS = DAG.getNode(N->getOpcode(), N->getValueType(0), + SDValue NewRHS = DAG.getNode(N->getOpcode(), N->getValueType(0), LHS->getOperand(1), N->getOperand(1)); // Create the new shift. - SDOperand NewShift = DAG.getNode(N->getOpcode(), VT, LHS->getOperand(0), + SDValue NewShift = DAG.getNode(N->getOpcode(), VT, LHS->getOperand(0), N->getOperand(1)); // Create the new binop. @@ -2266,37 +2427,55 @@ SDOperand DAGCombiner::visitShiftByConstant(SDNode *N, unsigned Amt) { } -SDOperand DAGCombiner::visitSHL(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitSHL(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N0.getValueType(); - unsigned OpSizeInBits = MVT::getSizeInBits(VT); + MVT VT = N0.getValueType(); + unsigned OpSizeInBits = VT.getSizeInBits(); // fold (shl c1, c2) -> c1< 0 if (N0C && N0C->isNullValue()) return N0; // fold (shl x, c >= size(x)) -> undef - if (N1C && N1C->getValue() >= OpSizeInBits) + if (N1C && N1C->getZExtValue() >= OpSizeInBits) return DAG.getNode(ISD::UNDEF, VT); // fold (shl x, 0) -> x if (N1C && N1C->isNullValue()) return N0; // if (shl x, c) is known to be zero, return 0 - if (DAG.MaskedValueIsZero(SDOperand(N, 0), - APInt::getAllOnesValue(MVT::getSizeInBits(VT)))) + if (DAG.MaskedValueIsZero(SDValue(N, 0), + APInt::getAllOnesValue(VT.getSizeInBits()))) return DAG.getConstant(0, VT); - if (N1C && SimplifyDemandedBits(SDOperand(N, 0))) - return SDOperand(N, 0); + // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), c)) + // iff (trunc c) == c + if (N1.getOpcode() == ISD::TRUNCATE && + N1.getOperand(0).getOpcode() == ISD::AND && + N1.hasOneUse() && N1.getOperand(0).hasOneUse()) { + SDValue N101 = N1.getOperand(0).getOperand(1); + if (ConstantSDNode *N101C = dyn_cast(N101)) { + MVT TruncVT = N1.getValueType(); + SDValue N100 = N1.getOperand(0).getOperand(0); + uint64_t TruncC = TruncVT.getIntegerVTBitMask() & + N101C->getZExtValue(); + return DAG.getNode(ISD::SHL, VT, N0, + DAG.getNode(ISD::AND, TruncVT, + DAG.getNode(ISD::TRUNCATE, TruncVT, N100), + DAG.getConstant(TruncC, TruncVT))); + } + } + + if (N1C && SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); // fold (shl (shl x, c1), c2) -> 0 or (shl x, c1+c2) if (N1C && N0.getOpcode() == ISD::SHL && N0.getOperand(1).getOpcode() == ISD::Constant) { - uint64_t c1 = cast(N0.getOperand(1))->getValue(); - uint64_t c2 = N1C->getValue(); + uint64_t c1 = cast(N0.getOperand(1))->getZExtValue(); + uint64_t c2 = N1C->getZExtValue(); if (c1 + c2 > OpSizeInBits) return DAG.getConstant(0, VT); return DAG.getNode(ISD::SHL, VT, N0.getOperand(0), @@ -2306,9 +2485,9 @@ SDOperand DAGCombiner::visitSHL(SDNode *N) { // (srl (and x, -1 << c1), c1-c2) if (N1C && N0.getOpcode() == ISD::SRL && N0.getOperand(1).getOpcode() == ISD::Constant) { - uint64_t c1 = cast(N0.getOperand(1))->getValue(); - uint64_t c2 = N1C->getValue(); - SDOperand Mask = DAG.getNode(ISD::AND, VT, N0.getOperand(0), + uint64_t c1 = cast(N0.getOperand(1))->getZExtValue(); + uint64_t c2 = N1C->getZExtValue(); + SDValue Mask = DAG.getNode(ISD::AND, VT, N0.getOperand(0), DAG.getConstant(~0ULL << c1, VT)); if (c2 > c1) return DAG.getNode(ISD::SHL, VT, Mask, @@ -2320,21 +2499,21 @@ SDOperand DAGCombiner::visitSHL(SDNode *N) { // fold (shl (sra x, c1), c1) -> (and x, -1 << c1) if (N1C && N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1)) return DAG.getNode(ISD::AND, VT, N0.getOperand(0), - DAG.getConstant(~0ULL << N1C->getValue(), VT)); + DAG.getConstant(~0ULL << N1C->getZExtValue(), VT)); - return N1C ? visitShiftByConstant(N, N1C->getValue()) : SDOperand(); + return N1C ? visitShiftByConstant(N, N1C->getZExtValue()) : SDValue(); } -SDOperand DAGCombiner::visitSRA(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitSRA(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N0.getValueType(); + MVT VT = N0.getValueType(); // fold (sra c1, c2) -> c1>>c2 if (N0C && N1C) - return DAG.getNode(ISD::SRA, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::SRA, VT, N0C, N1C); // fold (sra 0, x) -> 0 if (N0C && N0C->isNullValue()) return N0; @@ -2342,7 +2521,7 @@ SDOperand DAGCombiner::visitSRA(SDNode *N) { if (N0C && N0C->isAllOnesValue()) return N0; // fold (sra x, c >= size(x)) -> undef - if (N1C && N1C->getValue() >= MVT::getSizeInBits(VT)) + if (N1C && N1C->getZExtValue() >= VT.getSizeInBits()) return DAG.getNode(ISD::UNDEF, VT); // fold (sra x, 0) -> x if (N1C && N1C->isNullValue()) @@ -2350,72 +2529,116 @@ SDOperand DAGCombiner::visitSRA(SDNode *N) { // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports // sext_inreg. if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) { - unsigned LowBits = MVT::getSizeInBits(VT) - (unsigned)N1C->getValue(); - MVT::ValueType EVT; - switch (LowBits) { - default: EVT = MVT::Other; break; - case 1: EVT = MVT::i1; break; - case 8: EVT = MVT::i8; break; - case 16: EVT = MVT::i16; break; - case 32: EVT = MVT::i32; break; - } - if (EVT > MVT::Other && TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, EVT)) + unsigned LowBits = VT.getSizeInBits() - (unsigned)N1C->getZExtValue(); + MVT EVT = MVT::getIntegerVT(LowBits); + if ((!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, EVT))) return DAG.getNode(ISD::SIGN_EXTEND_INREG, VT, N0.getOperand(0), DAG.getValueType(EVT)); } - + // fold (sra (sra x, c1), c2) -> (sra x, c1+c2) if (N1C && N0.getOpcode() == ISD::SRA) { if (ConstantSDNode *C1 = dyn_cast(N0.getOperand(1))) { - unsigned Sum = N1C->getValue() + C1->getValue(); - if (Sum >= MVT::getSizeInBits(VT)) Sum = MVT::getSizeInBits(VT)-1; + unsigned Sum = N1C->getZExtValue() + C1->getZExtValue(); + if (Sum >= VT.getSizeInBits()) Sum = VT.getSizeInBits()-1; return DAG.getNode(ISD::SRA, VT, N0.getOperand(0), DAG.getConstant(Sum, N1C->getValueType(0))); } } + + // fold sra (shl X, m), result_size - n + // -> (sign_extend (trunc (shl X, result_size - n - m))) for + // result_size - n != m. + // If truncate is free for the target sext(shl) is likely to result in better + // code. + if (N0.getOpcode() == ISD::SHL) { + // Get the two constanst of the shifts, CN0 = m, CN = n. + const ConstantSDNode *N01C = dyn_cast(N0.getOperand(1)); + if (N01C && N1C) { + // Determine what the truncate's result bitsize and type would be. + unsigned VTValSize = VT.getSizeInBits(); + MVT TruncVT = + MVT::getIntegerVT(VTValSize - N1C->getZExtValue()); + // Determine the residual right-shift amount. + unsigned ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue(); + + // If the shift is not a no-op (in which case this should be just a sign + // extend already), the truncated to type is legal, sign_extend is legal + // on that type, and the the truncate to that type is both legal and free, + // perform the transform. + if (ShiftAmt && + TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) && + TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) && + TLI.isTruncateFree(VT, TruncVT)) { + + SDValue Amt = DAG.getConstant(ShiftAmt, TLI.getShiftAmountTy()); + SDValue Shift = DAG.getNode(ISD::SRL, VT, N0.getOperand(0), Amt); + SDValue Trunc = DAG.getNode(ISD::TRUNCATE, TruncVT, Shift); + return DAG.getNode(ISD::SIGN_EXTEND, N->getValueType(0), Trunc); + } + } + } + // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), c)) + // iff (trunc c) == c + if (N1.getOpcode() == ISD::TRUNCATE && + N1.getOperand(0).getOpcode() == ISD::AND && + N1.hasOneUse() && N1.getOperand(0).hasOneUse()) { + SDValue N101 = N1.getOperand(0).getOperand(1); + if (ConstantSDNode *N101C = dyn_cast(N101)) { + MVT TruncVT = N1.getValueType(); + SDValue N100 = N1.getOperand(0).getOperand(0); + uint64_t TruncC = TruncVT.getIntegerVTBitMask() & + N101C->getZExtValue(); + return DAG.getNode(ISD::SRA, VT, N0, + DAG.getNode(ISD::AND, TruncVT, + DAG.getNode(ISD::TRUNCATE, TruncVT, N100), + DAG.getConstant(TruncC, TruncVT))); + } + } + // Simplify, based on bits shifted out of the LHS. - if (N1C && SimplifyDemandedBits(SDOperand(N, 0))) - return SDOperand(N, 0); + if (N1C && SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); // If the sign bit is known to be zero, switch this to a SRL. if (DAG.SignBitIsZero(N0)) return DAG.getNode(ISD::SRL, VT, N0, N1); - return N1C ? visitShiftByConstant(N, N1C->getValue()) : SDOperand(); + return N1C ? visitShiftByConstant(N, N1C->getZExtValue()) : SDValue(); } -SDOperand DAGCombiner::visitSRL(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitSRL(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); - MVT::ValueType VT = N0.getValueType(); - unsigned OpSizeInBits = MVT::getSizeInBits(VT); + MVT VT = N0.getValueType(); + unsigned OpSizeInBits = VT.getSizeInBits(); // fold (srl c1, c2) -> c1 >>u c2 if (N0C && N1C) - return DAG.getNode(ISD::SRL, VT, N0, N1); + return DAG.FoldConstantArithmetic(ISD::SRL, VT, N0C, N1C); // fold (srl 0, x) -> 0 if (N0C && N0C->isNullValue()) return N0; // fold (srl x, c >= size(x)) -> undef - if (N1C && N1C->getValue() >= OpSizeInBits) + if (N1C && N1C->getZExtValue() >= OpSizeInBits) return DAG.getNode(ISD::UNDEF, VT); // fold (srl x, 0) -> x if (N1C && N1C->isNullValue()) return N0; // if (srl x, c) is known to be zero, return 0 - if (N1C && DAG.MaskedValueIsZero(SDOperand(N, 0), + if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnesValue(OpSizeInBits))) return DAG.getConstant(0, VT); // fold (srl (srl x, c1), c2) -> 0 or (srl x, c1+c2) if (N1C && N0.getOpcode() == ISD::SRL && N0.getOperand(1).getOpcode() == ISD::Constant) { - uint64_t c1 = cast(N0.getOperand(1))->getValue(); - uint64_t c2 = N1C->getValue(); + uint64_t c1 = cast(N0.getOperand(1))->getZExtValue(); + uint64_t c2 = N1C->getZExtValue(); if (c1 + c2 > OpSizeInBits) return DAG.getConstant(0, VT); return DAG.getNode(ISD::SRL, VT, N0.getOperand(0), @@ -2425,27 +2648,27 @@ SDOperand DAGCombiner::visitSRL(SDNode *N) { // fold (srl (anyextend x), c) -> (anyextend (srl x, c)) if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { // Shifting in all undef bits? - MVT::ValueType SmallVT = N0.getOperand(0).getValueType(); - if (N1C->getValue() >= MVT::getSizeInBits(SmallVT)) + MVT SmallVT = N0.getOperand(0).getValueType(); + if (N1C->getZExtValue() >= SmallVT.getSizeInBits()) return DAG.getNode(ISD::UNDEF, VT); - SDOperand SmallShift = DAG.getNode(ISD::SRL, SmallVT, N0.getOperand(0), N1); - AddToWorkList(SmallShift.Val); + SDValue SmallShift = DAG.getNode(ISD::SRL, SmallVT, N0.getOperand(0), N1); + AddToWorkList(SmallShift.getNode()); return DAG.getNode(ISD::ANY_EXTEND, VT, SmallShift); } // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign // bit, which is unmodified by sra. - if (N1C && N1C->getValue()+1 == MVT::getSizeInBits(VT)) { + if (N1C && N1C->getZExtValue()+1 == VT.getSizeInBits()) { if (N0.getOpcode() == ISD::SRA) return DAG.getNode(ISD::SRL, VT, N0.getOperand(0), N1); } // fold (srl (ctlz x), "5") -> x iff x has one bit set (the low bit). if (N1C && N0.getOpcode() == ISD::CTLZ && - N1C->getValue() == Log2_32(MVT::getSizeInBits(VT))) { + N1C->getAPIntValue() == Log2_32(VT.getSizeInBits())) { APInt KnownZero, KnownOne; - APInt Mask = APInt::getAllOnesValue(MVT::getSizeInBits(VT)); + APInt Mask = APInt::getAllOnesValue(VT.getSizeInBits()); DAG.ComputeMaskedBits(N0.getOperand(0), Mask, KnownZero, KnownOne); // If any of the input bits are KnownOne, then the input couldn't be all @@ -2464,63 +2687,81 @@ SDOperand DAGCombiner::visitSRL(SDNode *N) { // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair // to an SRL,XOR pair, which is likely to simplify more. unsigned ShAmt = UnknownBits.countTrailingZeros(); - SDOperand Op = N0.getOperand(0); + SDValue Op = N0.getOperand(0); if (ShAmt) { Op = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(ShAmt, TLI.getShiftAmountTy())); - AddToWorkList(Op.Val); + AddToWorkList(Op.getNode()); } return DAG.getNode(ISD::XOR, VT, Op, DAG.getConstant(1, VT)); } } + + // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), c)) + // iff (trunc c) == c + if (N1.getOpcode() == ISD::TRUNCATE && + N1.getOperand(0).getOpcode() == ISD::AND && + N1.hasOneUse() && N1.getOperand(0).hasOneUse()) { + SDValue N101 = N1.getOperand(0).getOperand(1); + if (ConstantSDNode *N101C = dyn_cast(N101)) { + MVT TruncVT = N1.getValueType(); + SDValue N100 = N1.getOperand(0).getOperand(0); + uint64_t TruncC = TruncVT.getIntegerVTBitMask() & + N101C->getZExtValue(); + return DAG.getNode(ISD::SRL, VT, N0, + DAG.getNode(ISD::AND, TruncVT, + DAG.getNode(ISD::TRUNCATE, TruncVT, N100), + DAG.getConstant(TruncC, TruncVT))); + } + } // fold operands of srl based on knowledge that the low bits are not // demanded. - if (N1C && SimplifyDemandedBits(SDOperand(N, 0))) - return SDOperand(N, 0); + if (N1C && SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); - return N1C ? visitShiftByConstant(N, N1C->getValue()) : SDOperand(); + return N1C ? visitShiftByConstant(N, N1C->getZExtValue()) : SDValue(); } -SDOperand DAGCombiner::visitCTLZ(SDNode *N) { - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); +SDValue DAGCombiner::visitCTLZ(SDNode *N) { + SDValue N0 = N->getOperand(0); + MVT VT = N->getValueType(0); // fold (ctlz c1) -> c2 if (isa(N0)) return DAG.getNode(ISD::CTLZ, VT, N0); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitCTTZ(SDNode *N) { - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); +SDValue DAGCombiner::visitCTTZ(SDNode *N) { + SDValue N0 = N->getOperand(0); + MVT VT = N->getValueType(0); // fold (cttz c1) -> c2 if (isa(N0)) return DAG.getNode(ISD::CTTZ, VT, N0); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitCTPOP(SDNode *N) { - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); +SDValue DAGCombiner::visitCTPOP(SDNode *N) { + SDValue N0 = N->getOperand(0); + MVT VT = N->getValueType(0); // fold (ctpop c1) -> c2 if (isa(N0)) return DAG.getNode(ISD::CTPOP, VT, N0); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitSELECT(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - SDOperand N2 = N->getOperand(2); +SDValue DAGCombiner::visitSELECT(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + SDValue N2 = N->getOperand(2); ConstantSDNode *N0C = dyn_cast(N0); ConstantSDNode *N1C = dyn_cast(N1); ConstantSDNode *N2C = dyn_cast(N2); - MVT::ValueType VT = N->getValueType(0); - MVT::ValueType VT0 = N0.getValueType(); + MVT VT = N->getValueType(0); + MVT VT0 = N0.getValueType(); // fold select C, X, X -> X if (N1 == N2) @@ -2532,67 +2773,69 @@ SDOperand DAGCombiner::visitSELECT(SDNode *N) { if (N0C && N0C->isNullValue()) return N2; // fold select C, 1, X -> C | X - if (MVT::i1 == VT && N1C && N1C->getValue() == 1) + if (VT == MVT::i1 && N1C && N1C->getAPIntValue() == 1) return DAG.getNode(ISD::OR, VT, N0, N2); - // fold select C, 0, 1 -> ~C - if (MVT::isInteger(VT) && MVT::isInteger(VT0) && - N1C && N2C && N1C->isNullValue() && N2C->getValue() == 1) { - SDOperand XORNode = DAG.getNode(ISD::XOR, VT0, N0, DAG.getConstant(1, VT0)); + // fold select C, 0, 1 -> C ^ 1 + if (VT.isInteger() && + (VT0 == MVT::i1 || + (VT0.isInteger() && + TLI.getBooleanContents() == TargetLowering::ZeroOrOneBooleanContent)) && + N1C && N2C && N1C->isNullValue() && N2C->getAPIntValue() == 1) { + SDValue XORNode = DAG.getNode(ISD::XOR, VT0, N0, DAG.getConstant(1, VT0)); if (VT == VT0) return XORNode; - AddToWorkList(XORNode.Val); - if (MVT::getSizeInBits(VT) > MVT::getSizeInBits(VT0)) + AddToWorkList(XORNode.getNode()); + if (VT.bitsGT(VT0)) return DAG.getNode(ISD::ZERO_EXTEND, VT, XORNode); return DAG.getNode(ISD::TRUNCATE, VT, XORNode); } // fold select C, 0, X -> ~C & X if (VT == VT0 && VT == MVT::i1 && N1C && N1C->isNullValue()) { - SDOperand XORNode = DAG.getNode(ISD::XOR, VT, N0, DAG.getConstant(1, VT)); - AddToWorkList(XORNode.Val); - return DAG.getNode(ISD::AND, VT, XORNode, N2); + SDValue NOTNode = DAG.getNOT(N0, VT); + AddToWorkList(NOTNode.getNode()); + return DAG.getNode(ISD::AND, VT, NOTNode, N2); } // fold select C, X, 1 -> ~C | X - if (VT == VT0 && VT == MVT::i1 && N2C && N2C->getValue() == 1) { - SDOperand XORNode = DAG.getNode(ISD::XOR, VT, N0, DAG.getConstant(1, VT)); - AddToWorkList(XORNode.Val); - return DAG.getNode(ISD::OR, VT, XORNode, N1); + if (VT == VT0 && VT == MVT::i1 && N2C && N2C->getAPIntValue() == 1) { + SDValue NOTNode = DAG.getNOT(N0, VT); + AddToWorkList(NOTNode.getNode()); + return DAG.getNode(ISD::OR, VT, NOTNode, N1); } // fold select C, X, 0 -> C & X - // FIXME: this should check for C type == X type, not i1? - if (MVT::i1 == VT && N2C && N2C->isNullValue()) + if (VT == MVT::i1 && N2C && N2C->isNullValue()) return DAG.getNode(ISD::AND, VT, N0, N1); // fold X ? X : Y --> X ? 1 : Y --> X | Y - if (MVT::i1 == VT && N0 == N1) + if (VT == MVT::i1 && N0 == N1) return DAG.getNode(ISD::OR, VT, N0, N2); // fold X ? Y : X --> X ? Y : 0 --> X & Y - if (MVT::i1 == VT && N0 == N2) + if (VT == MVT::i1 && N0 == N2) return DAG.getNode(ISD::AND, VT, N0, N1); // If we can fold this based on the true/false value, do so. if (SimplifySelectOps(N, N1, N2)) - return SDOperand(N, 0); // Don't revisit N. - + return SDValue(N, 0); // Don't revisit N. + // fold selects based on a setcc into other things, such as min/max/abs if (N0.getOpcode() == ISD::SETCC) { // FIXME: // Check against MVT::Other for SELECT_CC, which is a workaround for targets // having to say they don't support SELECT_CC on every type the DAG knows // about, since there is no way to mark an opcode illegal at all value types - if (TLI.isOperationLegal(ISD::SELECT_CC, MVT::Other)) + if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, MVT::Other)) return DAG.getNode(ISD::SELECT_CC, VT, N0.getOperand(0), N0.getOperand(1), N1, N2, N0.getOperand(2)); else return SimplifySelect(N0, N1, N2); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitSELECT_CC(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - SDOperand N2 = N->getOperand(2); - SDOperand N3 = N->getOperand(3); - SDOperand N4 = N->getOperand(4); +SDValue DAGCombiner::visitSELECT_CC(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + SDValue N2 = N->getOperand(2); + SDValue N3 = N->getOperand(3); + SDValue N4 = N->getOperand(4); ISD::CondCode CC = cast(N4)->get(); // fold select_cc lhs, rhs, x, x, cc -> x @@ -2600,31 +2843,32 @@ SDOperand DAGCombiner::visitSELECT_CC(SDNode *N) { return N2; // Determine if the condition we're dealing with is constant - SDOperand SCC = SimplifySetCC(TLI.getSetCCResultType(N0), N0, N1, CC, false); - if (SCC.Val) AddToWorkList(SCC.Val); + SDValue SCC = SimplifySetCC(TLI.getSetCCResultType(N0.getValueType()), + N0, N1, CC, false); + if (SCC.getNode()) AddToWorkList(SCC.getNode()); - if (ConstantSDNode *SCCC = dyn_cast_or_null(SCC.Val)) { - if (SCCC->getValue()) + if (ConstantSDNode *SCCC = dyn_cast_or_null(SCC.getNode())) { + if (!SCCC->isNullValue()) return N2; // cond always true -> true val else return N3; // cond always false -> false val } // Fold to a simpler select_cc - if (SCC.Val && SCC.getOpcode() == ISD::SETCC) + if (SCC.getNode() && SCC.getOpcode() == ISD::SETCC) return DAG.getNode(ISD::SELECT_CC, N2.getValueType(), SCC.getOperand(0), SCC.getOperand(1), N2, N3, SCC.getOperand(2)); // If we can fold this based on the true/false value, do so. if (SimplifySelectOps(N, N2, N3)) - return SDOperand(N, 0); // Don't revisit N. + return SDValue(N, 0); // Don't revisit N. // fold select_cc into other things, such as min/max/abs return SimplifySelectCC(N0, N1, N2, N3, CC); } -SDOperand DAGCombiner::visitSETCC(SDNode *N) { +SDValue DAGCombiner::visitSETCC(SDNode *N) { return SimplifySetCC(N->getValueType(0), N->getOperand(0), N->getOperand(1), cast(N->getOperand(2))->get()); } @@ -2633,13 +2877,14 @@ SDOperand DAGCombiner::visitSETCC(SDNode *N) { // "fold ({s|z}ext (load x)) -> ({s|z}ext (truncate ({s|z}extload x)))" // transformation. Returns true if extension are possible and the above // mentioned transformation is profitable. -static bool ExtendUsesToFormExtLoad(SDNode *N, SDOperand N0, +static bool ExtendUsesToFormExtLoad(SDNode *N, SDValue N0, unsigned ExtOpc, SmallVector &ExtendNodes, - TargetLowering &TLI) { + const TargetLowering &TLI) { bool HasCopyToRegUses = false; bool isTruncFree = TLI.isTruncateFree(N->getValueType(0), N0.getValueType()); - for (SDNode::use_iterator UI = N0.Val->use_begin(), UE = N0.Val->use_end(); + for (SDNode::use_iterator UI = N0.getNode()->use_begin(), + UE = N0.getNode()->use_end(); UI != UE; ++UI) { SDNode *User = *UI; if (User == N) @@ -2652,7 +2897,7 @@ static bool ExtendUsesToFormExtLoad(SDNode *N, SDOperand N0, return false; bool Add = false; for (unsigned i = 0; i != 2; ++i) { - SDOperand UseOp = User->getOperand(i); + SDValue UseOp = User->getOperand(i); if (UseOp == N0) continue; if (!isa(UseOp)) @@ -2663,7 +2908,7 @@ static bool ExtendUsesToFormExtLoad(SDNode *N, SDOperand N0, ExtendNodes.push_back(User); } else { for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { - SDOperand UseOp = User->getOperand(i); + SDValue UseOp = User->getOperand(i); if (UseOp == N0) { // If truncate from extended type to original load type is free // on this target, then it's ok to extend a CopyToReg. @@ -2682,8 +2927,8 @@ static bool ExtendUsesToFormExtLoad(SDNode *N, SDOperand N0, UI != UE; ++UI) { SDNode *User = *UI; for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { - SDOperand UseOp = User->getOperand(i); - if (UseOp.Val == N && UseOp.ResNo == 0) { + SDValue UseOp = User->getOperand(i); + if (UseOp.getNode() == N && UseOp.getResNo() == 0) { BothLiveOut = true; break; } @@ -2697,9 +2942,9 @@ static bool ExtendUsesToFormExtLoad(SDNode *N, SDOperand N0, return true; } -SDOperand DAGCombiner::visitSIGN_EXTEND(SDNode *N) { - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); +SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { + SDValue N0 = N->getOperand(0); + MVT VT = N->getValueType(0); // fold (sext c1) -> c1 if (isa(N0)) @@ -2710,24 +2955,22 @@ SDOperand DAGCombiner::visitSIGN_EXTEND(SDNode *N) { if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) return DAG.getNode(ISD::SIGN_EXTEND, VT, N0.getOperand(0)); - // fold (sext (truncate (load x))) -> (sext (smaller load x)) - // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n))) if (N0.getOpcode() == ISD::TRUNCATE) { - SDOperand NarrowLoad = ReduceLoadWidth(N0.Val); - if (NarrowLoad.Val) { - if (NarrowLoad.Val != N0.Val) - CombineTo(N0.Val, NarrowLoad); + // fold (sext (truncate (load x))) -> (sext (smaller load x)) + // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n))) + SDValue NarrowLoad = ReduceLoadWidth(N0.getNode()); + if (NarrowLoad.getNode()) { + if (NarrowLoad.getNode() != N0.getNode()) + CombineTo(N0.getNode(), NarrowLoad); return DAG.getNode(ISD::SIGN_EXTEND, VT, NarrowLoad); } - } - // See if the value being truncated is already sign extended. If so, just - // eliminate the trunc/sext pair. - if (N0.getOpcode() == ISD::TRUNCATE) { - SDOperand Op = N0.getOperand(0); - unsigned OpBits = MVT::getSizeInBits(Op.getValueType()); - unsigned MidBits = MVT::getSizeInBits(N0.getValueType()); - unsigned DestBits = MVT::getSizeInBits(VT); + // See if the value being truncated is already sign extended. If so, just + // eliminate the trunc/sext pair. + SDValue Op = N0.getOperand(0); + unsigned OpBits = Op.getValueType().getSizeInBits(); + unsigned MidBits = N0.getValueType().getSizeInBits(); + unsigned DestBits = VT.getSizeInBits(); unsigned NumSignBits = DAG.ComputeNumSignBits(Op); if (OpBits == DestBits) { @@ -2748,11 +2991,11 @@ SDOperand DAGCombiner::visitSIGN_EXTEND(SDNode *N) { } // fold (sext (truncate x)) -> (sextinreg x). - if (!AfterLegalize || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, - N0.getValueType())) { - if (Op.getValueType() < VT) + if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, + N0.getValueType())) { + if (Op.getValueType().bitsLT(VT)) Op = DAG.getNode(ISD::ANY_EXTEND, VT, Op); - else if (Op.getValueType() > VT) + else if (Op.getValueType().bitsGT(VT)) Op = DAG.getNode(ISD::TRUNCATE, VT, Op); return DAG.getNode(ISD::SIGN_EXTEND_INREG, VT, Op, DAG.getValueType(N0.getValueType())); @@ -2760,29 +3003,29 @@ SDOperand DAGCombiner::visitSIGN_EXTEND(SDNode *N) { } // fold (sext (load x)) -> (sext (truncate (sextload x))) - if (ISD::isNON_EXTLoad(N0.Val) && - (!AfterLegalize||TLI.isLoadXLegal(ISD::SEXTLOAD, N0.getValueType()))){ + if (ISD::isNON_EXTLoad(N0.getNode()) && + ((!LegalOperations && !cast(N0)->isVolatile()) || + TLI.isLoadExtLegal(ISD::SEXTLOAD, N0.getValueType()))) { bool DoXform = true; SmallVector SetCCs; if (!N0.hasOneUse()) DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::SIGN_EXTEND, SetCCs, TLI); if (DoXform) { LoadSDNode *LN0 = cast(N0); - SDOperand ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(), - LN0->getBasePtr(), LN0->getSrcValue(), - LN0->getSrcValueOffset(), - N0.getValueType(), - LN0->isVolatile(), - LN0->getAlignment()); + SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(), + LN0->getBasePtr(), LN0->getSrcValue(), + LN0->getSrcValueOffset(), + N0.getValueType(), + LN0->isVolatile(), LN0->getAlignment()); CombineTo(N, ExtLoad); - SDOperand Trunc = DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad); - CombineTo(N0.Val, Trunc, ExtLoad.getValue(1)); + SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad); + CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1)); // Extend SetCC uses if necessary. for (unsigned i = 0, e = SetCCs.size(); i != e; ++i) { SDNode *SetCC = SetCCs[i]; - SmallVector Ops; + SmallVector Ops; for (unsigned j = 0; j != 2; ++j) { - SDOperand SOp = SetCC->getOperand(j); + SDValue SOp = SetCC->getOperand(j); if (SOp == Trunc) Ops.push_back(ExtLoad); else @@ -2792,44 +3035,50 @@ SDOperand DAGCombiner::visitSIGN_EXTEND(SDNode *N) { CombineTo(SetCC, DAG.getNode(ISD::SETCC, SetCC->getValueType(0), &Ops[0], Ops.size())); } - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + return SDValue(N, 0); // Return N so it doesn't get rechecked! } } // fold (sext (sextload x)) -> (sext (truncate (sextload x))) // fold (sext ( extload x)) -> (sext (truncate (sextload x))) - if ((ISD::isSEXTLoad(N0.Val) || ISD::isEXTLoad(N0.Val)) && - ISD::isUNINDEXEDLoad(N0.Val) && N0.hasOneUse()) { + if ((ISD::isSEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) && + ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { LoadSDNode *LN0 = cast(N0); - MVT::ValueType EVT = LN0->getMemoryVT(); - if (!AfterLegalize || TLI.isLoadXLegal(ISD::SEXTLOAD, EVT)) { - SDOperand ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(), - LN0->getBasePtr(), LN0->getSrcValue(), - LN0->getSrcValueOffset(), EVT, - LN0->isVolatile(), - LN0->getAlignment()); + MVT EVT = LN0->getMemoryVT(); + if ((!LegalOperations && !LN0->isVolatile()) || + TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT)) { + SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(), + LN0->getBasePtr(), LN0->getSrcValue(), + LN0->getSrcValueOffset(), EVT, + LN0->isVolatile(), LN0->getAlignment()); CombineTo(N, ExtLoad); - CombineTo(N0.Val, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad), + CombineTo(N0.getNode(), + DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad), ExtLoad.getValue(1)); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + return SDValue(N, 0); // Return N so it doesn't get rechecked! } } // sext(setcc x,y,cc) -> select_cc x, y, -1, 0, cc if (N0.getOpcode() == ISD::SETCC) { - SDOperand SCC = + SDValue SCC = SimplifySelectCC(N0.getOperand(0), N0.getOperand(1), DAG.getConstant(~0ULL, VT), DAG.getConstant(0, VT), cast(N0.getOperand(2))->get(), true); - if (SCC.Val) return SCC; + if (SCC.getNode()) return SCC; } - return SDOperand(); + // fold (sext x) -> (zext x) if the sign bit is known zero. + if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && + DAG.SignBitIsZero(N0)) + return DAG.getNode(ISD::ZERO_EXTEND, VT, N0); + + return SDValue(); } -SDOperand DAGCombiner::visitZERO_EXTEND(SDNode *N) { - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); +SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { + SDValue N0 = N->getOperand(0); + MVT VT = N->getValueType(0); // fold (zext c1) -> c1 if (isa(N0)) @@ -2842,21 +3091,21 @@ SDOperand DAGCombiner::visitZERO_EXTEND(SDNode *N) { // fold (zext (truncate (load x))) -> (zext (smaller load x)) // fold (zext (truncate (srl (load x), c))) -> (zext (small load (x+c/n))) if (N0.getOpcode() == ISD::TRUNCATE) { - SDOperand NarrowLoad = ReduceLoadWidth(N0.Val); - if (NarrowLoad.Val) { - if (NarrowLoad.Val != N0.Val) - CombineTo(N0.Val, NarrowLoad); + SDValue NarrowLoad = ReduceLoadWidth(N0.getNode()); + if (NarrowLoad.getNode()) { + if (NarrowLoad.getNode() != N0.getNode()) + CombineTo(N0.getNode(), NarrowLoad); return DAG.getNode(ISD::ZERO_EXTEND, VT, NarrowLoad); } } // fold (zext (truncate x)) -> (and x, mask) if (N0.getOpcode() == ISD::TRUNCATE && - (!AfterLegalize || TLI.isOperationLegal(ISD::AND, VT))) { - SDOperand Op = N0.getOperand(0); - if (Op.getValueType() < VT) { + (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT))) { + SDValue Op = N0.getOperand(0); + if (Op.getValueType().bitsLT(VT)) { Op = DAG.getNode(ISD::ANY_EXTEND, VT, Op); - } else if (Op.getValueType() > VT) { + } else if (Op.getValueType().bitsGT(VT)) { Op = DAG.getNode(ISD::TRUNCATE, VT, Op); } return DAG.getZeroExtendInReg(Op, N0.getValueType()); @@ -2866,41 +3115,41 @@ SDOperand DAGCombiner::visitZERO_EXTEND(SDNode *N) { if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::TRUNCATE && N0.getOperand(1).getOpcode() == ISD::Constant) { - SDOperand X = N0.getOperand(0).getOperand(0); - if (X.getValueType() < VT) { + SDValue X = N0.getOperand(0).getOperand(0); + if (X.getValueType().bitsLT(VT)) { X = DAG.getNode(ISD::ANY_EXTEND, VT, X); - } else if (X.getValueType() > VT) { + } else if (X.getValueType().bitsGT(VT)) { X = DAG.getNode(ISD::TRUNCATE, VT, X); } APInt Mask = cast(N0.getOperand(1))->getAPIntValue(); - Mask.zext(MVT::getSizeInBits(VT)); + Mask.zext(VT.getSizeInBits()); return DAG.getNode(ISD::AND, VT, X, DAG.getConstant(Mask, VT)); } // fold (zext (load x)) -> (zext (truncate (zextload x))) - if (ISD::isNON_EXTLoad(N0.Val) && - (!AfterLegalize||TLI.isLoadXLegal(ISD::ZEXTLOAD, N0.getValueType()))) { + if (ISD::isNON_EXTLoad(N0.getNode()) && + ((!LegalOperations && !cast(N0)->isVolatile()) || + TLI.isLoadExtLegal(ISD::ZEXTLOAD, N0.getValueType()))) { bool DoXform = true; SmallVector SetCCs; if (!N0.hasOneUse()) DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ZERO_EXTEND, SetCCs, TLI); if (DoXform) { LoadSDNode *LN0 = cast(N0); - SDOperand ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(), - LN0->getBasePtr(), LN0->getSrcValue(), - LN0->getSrcValueOffset(), - N0.getValueType(), - LN0->isVolatile(), - LN0->getAlignment()); + SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(), + LN0->getBasePtr(), LN0->getSrcValue(), + LN0->getSrcValueOffset(), + N0.getValueType(), + LN0->isVolatile(), LN0->getAlignment()); CombineTo(N, ExtLoad); - SDOperand Trunc = DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad); - CombineTo(N0.Val, Trunc, ExtLoad.getValue(1)); + SDValue Trunc = DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad); + CombineTo(N0.getNode(), Trunc, ExtLoad.getValue(1)); // Extend SetCC uses if necessary. for (unsigned i = 0, e = SetCCs.size(); i != e; ++i) { SDNode *SetCC = SetCCs[i]; - SmallVector Ops; + SmallVector Ops; for (unsigned j = 0; j != 2; ++j) { - SDOperand SOp = SetCC->getOperand(j); + SDValue SOp = SetCC->getOperand(j); if (SOp == Trunc) Ops.push_back(ExtLoad); else @@ -2910,42 +3159,45 @@ SDOperand DAGCombiner::visitZERO_EXTEND(SDNode *N) { CombineTo(SetCC, DAG.getNode(ISD::SETCC, SetCC->getValueType(0), &Ops[0], Ops.size())); } - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + return SDValue(N, 0); // Return N so it doesn't get rechecked! } } // fold (zext (zextload x)) -> (zext (truncate (zextload x))) // fold (zext ( extload x)) -> (zext (truncate (zextload x))) - if ((ISD::isZEXTLoad(N0.Val) || ISD::isEXTLoad(N0.Val)) && - ISD::isUNINDEXEDLoad(N0.Val) && N0.hasOneUse()) { + if ((ISD::isZEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) && + ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { LoadSDNode *LN0 = cast(N0); - MVT::ValueType EVT = LN0->getMemoryVT(); - SDOperand ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(), + MVT EVT = LN0->getMemoryVT(); + if ((!LegalOperations && !LN0->isVolatile()) || + TLI.isLoadExtLegal(ISD::ZEXTLOAD, EVT)) { + SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, VT, LN0->getChain(), LN0->getBasePtr(), LN0->getSrcValue(), LN0->getSrcValueOffset(), EVT, - LN0->isVolatile(), - LN0->getAlignment()); - CombineTo(N, ExtLoad); - CombineTo(N0.Val, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad), - ExtLoad.getValue(1)); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + LN0->isVolatile(), LN0->getAlignment()); + CombineTo(N, ExtLoad); + CombineTo(N0.getNode(), + DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad), + ExtLoad.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! + } } // zext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc if (N0.getOpcode() == ISD::SETCC) { - SDOperand SCC = + SDValue SCC = SimplifySelectCC(N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, VT), DAG.getConstant(0, VT), cast(N0.getOperand(2))->get(), true); - if (SCC.Val) return SCC; + if (SCC.getNode()) return SCC; } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitANY_EXTEND(SDNode *N) { - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); +SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) { + SDValue N0 = N->getOperand(0); + MVT VT = N->getValueType(0); // fold (aext c1) -> c1 if (isa(N0)) @@ -2961,20 +3213,20 @@ SDOperand DAGCombiner::visitANY_EXTEND(SDNode *N) { // fold (aext (truncate (load x))) -> (aext (smaller load x)) // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n))) if (N0.getOpcode() == ISD::TRUNCATE) { - SDOperand NarrowLoad = ReduceLoadWidth(N0.Val); - if (NarrowLoad.Val) { - if (NarrowLoad.Val != N0.Val) - CombineTo(N0.Val, NarrowLoad); + SDValue NarrowLoad = ReduceLoadWidth(N0.getNode()); + if (NarrowLoad.getNode()) { + if (NarrowLoad.getNode() != N0.getNode()) + CombineTo(N0.getNode(), NarrowLoad); return DAG.getNode(ISD::ANY_EXTEND, VT, NarrowLoad); } } // fold (aext (truncate x)) if (N0.getOpcode() == ISD::TRUNCATE) { - SDOperand TruncOp = N0.getOperand(0); + SDValue TruncOp = N0.getOperand(0); if (TruncOp.getValueType() == VT) return TruncOp; // x iff x size == zext size. - if (TruncOp.getValueType() > VT) + if (TruncOp.getValueType().bitsGT(VT)) return DAG.getNode(ISD::TRUNCATE, VT, TruncOp); return DAG.getNode(ISD::ANY_EXTEND, VT, TruncOp); } @@ -2983,70 +3235,75 @@ SDOperand DAGCombiner::visitANY_EXTEND(SDNode *N) { if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::TRUNCATE && N0.getOperand(1).getOpcode() == ISD::Constant) { - SDOperand X = N0.getOperand(0).getOperand(0); - if (X.getValueType() < VT) { + SDValue X = N0.getOperand(0).getOperand(0); + if (X.getValueType().bitsLT(VT)) { X = DAG.getNode(ISD::ANY_EXTEND, VT, X); - } else if (X.getValueType() > VT) { + } else if (X.getValueType().bitsGT(VT)) { X = DAG.getNode(ISD::TRUNCATE, VT, X); } APInt Mask = cast(N0.getOperand(1))->getAPIntValue(); - Mask.zext(MVT::getSizeInBits(VT)); + Mask.zext(VT.getSizeInBits()); return DAG.getNode(ISD::AND, VT, X, DAG.getConstant(Mask, VT)); } // fold (aext (load x)) -> (aext (truncate (extload x))) - if (ISD::isNON_EXTLoad(N0.Val) && N0.hasOneUse() && - (!AfterLegalize||TLI.isLoadXLegal(ISD::EXTLOAD, N0.getValueType()))) { + if (ISD::isNON_EXTLoad(N0.getNode()) && N0.hasOneUse() && + ((!LegalOperations && !cast(N0)->isVolatile()) || + TLI.isLoadExtLegal(ISD::EXTLOAD, N0.getValueType()))) { LoadSDNode *LN0 = cast(N0); - SDOperand ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, VT, LN0->getChain(), - LN0->getBasePtr(), LN0->getSrcValue(), - LN0->getSrcValueOffset(), - N0.getValueType(), - LN0->isVolatile(), - LN0->getAlignment()); + SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, VT, LN0->getChain(), + LN0->getBasePtr(), LN0->getSrcValue(), + LN0->getSrcValueOffset(), + N0.getValueType(), + LN0->isVolatile(), LN0->getAlignment()); CombineTo(N, ExtLoad); - CombineTo(N0.Val, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad), - ExtLoad.getValue(1)); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + // Redirect any chain users to the new load. + DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), + SDValue(ExtLoad.getNode(), 1)); + // If any node needs the original loaded value, recompute it. + if (!LN0->use_empty()) + CombineTo(LN0, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad), + ExtLoad.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! } // fold (aext (zextload x)) -> (aext (truncate (zextload x))) // fold (aext (sextload x)) -> (aext (truncate (sextload x))) // fold (aext ( extload x)) -> (aext (truncate (extload x))) if (N0.getOpcode() == ISD::LOAD && - !ISD::isNON_EXTLoad(N0.Val) && ISD::isUNINDEXEDLoad(N0.Val) && + !ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { LoadSDNode *LN0 = cast(N0); - MVT::ValueType EVT = LN0->getMemoryVT(); - SDOperand ExtLoad = DAG.getExtLoad(LN0->getExtensionType(), VT, - LN0->getChain(), LN0->getBasePtr(), - LN0->getSrcValue(), - LN0->getSrcValueOffset(), EVT, - LN0->isVolatile(), - LN0->getAlignment()); + MVT EVT = LN0->getMemoryVT(); + SDValue ExtLoad = DAG.getExtLoad(LN0->getExtensionType(), VT, + LN0->getChain(), LN0->getBasePtr(), + LN0->getSrcValue(), + LN0->getSrcValueOffset(), EVT, + LN0->isVolatile(), LN0->getAlignment()); CombineTo(N, ExtLoad); - CombineTo(N0.Val, DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad), + CombineTo(N0.getNode(), + DAG.getNode(ISD::TRUNCATE, N0.getValueType(), ExtLoad), ExtLoad.getValue(1)); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + return SDValue(N, 0); // Return N so it doesn't get rechecked! } // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc if (N0.getOpcode() == ISD::SETCC) { - SDOperand SCC = + SDValue SCC = SimplifySelectCC(N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, VT), DAG.getConstant(0, VT), cast(N0.getOperand(2))->get(), true); - if (SCC.Val) + if (SCC.getNode()) return SCC; } - return SDOperand(); + return SDValue(); } /// GetDemandedBits - See if the specified operand can be simplified with the /// knowledge that only the bits specified by Mask are used. If so, return the -/// simpler operand, otherwise return a null SDOperand. -SDOperand DAGCombiner::GetDemandedBits(SDOperand V, const APInt &Mask) { +/// simpler operand, otherwise return a null SDValue. +SDValue DAGCombiner::GetDemandedBits(SDValue V, const APInt &Mask) { switch (V.getOpcode()) { default: break; case ISD::OR: @@ -3059,20 +3316,22 @@ SDOperand DAGCombiner::GetDemandedBits(SDOperand V, const APInt &Mask) { break; case ISD::SRL: // Only look at single-use SRLs. - if (!V.Val->hasOneUse()) + if (!V.getNode()->hasOneUse()) break; if (ConstantSDNode *RHSC = dyn_cast(V.getOperand(1))) { // See if we can recursively simplify the LHS. - unsigned Amt = RHSC->getValue(); + unsigned Amt = RHSC->getZExtValue(); + // Watch out for shift count overflow though. + if (Amt >= Mask.getBitWidth()) break; APInt NewMask = Mask << Amt; - SDOperand SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask); - if (SimplifyLHS.Val) { + SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask); + if (SimplifyLHS.getNode()) { return DAG.getNode(ISD::SRL, V.getValueType(), SimplifyLHS, V.getOperand(1)); } } } - return SDOperand(); + return SDValue(); } /// ReduceLoadWidth - If the result of a wider load is shifted to right of N @@ -3080,123 +3339,123 @@ SDOperand DAGCombiner::GetDemandedBits(SDOperand V, const APInt &Mask) { /// of number of bits of the narrower type, transform it to a narrower load /// from address + N / num of bits of new type. If the result is to be /// extended, also fold the extension to form a extending load. -SDOperand DAGCombiner::ReduceLoadWidth(SDNode *N) { +SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) { unsigned Opc = N->getOpcode(); ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); - MVT::ValueType EVT = N->getValueType(0); + SDValue N0 = N->getOperand(0); + MVT VT = N->getValueType(0); + MVT EVT = VT; + + // This transformation isn't valid for vector loads. + if (VT.isVector()) + return SDValue(); // Special case: SIGN_EXTEND_INREG is basically truncating to EVT then // extended to VT. if (Opc == ISD::SIGN_EXTEND_INREG) { ExtType = ISD::SEXTLOAD; EVT = cast(N->getOperand(1))->getVT(); - if (AfterLegalize && !TLI.isLoadXLegal(ISD::SEXTLOAD, EVT)) - return SDOperand(); + if (LegalOperations && !TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT)) + return SDValue(); } - unsigned EVTBits = MVT::getSizeInBits(EVT); + unsigned EVTBits = EVT.getSizeInBits(); unsigned ShAmt = 0; - bool CombineSRL = false; if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) { if (ConstantSDNode *N01 = dyn_cast(N0.getOperand(1))) { - ShAmt = N01->getValue(); + ShAmt = N01->getZExtValue(); // Is the shift amount a multiple of size of VT? if ((ShAmt & (EVTBits-1)) == 0) { N0 = N0.getOperand(0); - if (MVT::getSizeInBits(N0.getValueType()) <= EVTBits) - return SDOperand(); - CombineSRL = true; + if (N0.getValueType().getSizeInBits() <= EVTBits) + return SDValue(); } } } - if (ISD::isNON_EXTLoad(N0.Val) && N0.hasOneUse() && - // Do not allow folding to i1 here. i1 is implicitly stored in memory in - // zero extended form: by shrinking the load, we lose track of the fact - // that it is already zero extended. - // FIXME: This should be reevaluated. - VT != MVT::i1) { - assert(MVT::getSizeInBits(N0.getValueType()) > EVTBits && - "Cannot truncate to larger type!"); + // Do not generate loads of non-round integer types since these can + // be expensive (and would be wrong if the type is not byte sized). + if (isa(N0) && N0.hasOneUse() && EVT.isRound() && + cast(N0)->getMemoryVT().getSizeInBits() > EVTBits && + // Do not change the width of a volatile load. + !cast(N0)->isVolatile()) { LoadSDNode *LN0 = cast(N0); - MVT::ValueType PtrType = N0.getOperand(1).getValueType(); + MVT PtrType = N0.getOperand(1).getValueType(); // For big endian targets, we need to adjust the offset to the pointer to // load the correct bytes. if (TLI.isBigEndian()) { - unsigned LVTStoreBits = MVT::getStoreSizeInBits(N0.getValueType()); - unsigned EVTStoreBits = MVT::getStoreSizeInBits(EVT); + unsigned LVTStoreBits = LN0->getMemoryVT().getStoreSizeInBits(); + unsigned EVTStoreBits = EVT.getStoreSizeInBits(); ShAmt = LVTStoreBits - EVTStoreBits - ShAmt; } uint64_t PtrOff = ShAmt / 8; unsigned NewAlign = MinAlign(LN0->getAlignment(), PtrOff); - SDOperand NewPtr = DAG.getNode(ISD::ADD, PtrType, LN0->getBasePtr(), - DAG.getConstant(PtrOff, PtrType)); - AddToWorkList(NewPtr.Val); - SDOperand Load = (ExtType == ISD::NON_EXTLOAD) + SDValue NewPtr = DAG.getNode(ISD::ADD, PtrType, LN0->getBasePtr(), + DAG.getConstant(PtrOff, PtrType)); + AddToWorkList(NewPtr.getNode()); + SDValue Load = (ExtType == ISD::NON_EXTLOAD) ? DAG.getLoad(VT, LN0->getChain(), NewPtr, - LN0->getSrcValue(), LN0->getSrcValueOffset(), + LN0->getSrcValue(), LN0->getSrcValueOffset() + PtrOff, LN0->isVolatile(), NewAlign) : DAG.getExtLoad(ExtType, VT, LN0->getChain(), NewPtr, - LN0->getSrcValue(), LN0->getSrcValueOffset(), EVT, - LN0->isVolatile(), NewAlign); - AddToWorkList(N); - if (CombineSRL) { - WorkListRemover DeadNodes(*this); - DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1), - &DeadNodes); - CombineTo(N->getOperand(0).Val, Load); - } else - CombineTo(N0.Val, Load, Load.getValue(1)); - if (ShAmt) { - if (Opc == ISD::SIGN_EXTEND_INREG) - return DAG.getNode(Opc, VT, Load, N->getOperand(1)); - else - return DAG.getNode(Opc, VT, Load); - } - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + LN0->getSrcValue(), LN0->getSrcValueOffset() + PtrOff, + EVT, LN0->isVolatile(), NewAlign); + // Replace the old load's chain with the new load's chain. + WorkListRemover DeadNodes(*this); + DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1), + &DeadNodes); + // Return the new loaded value. + return Load; } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - MVT::ValueType VT = N->getValueType(0); - MVT::ValueType EVT = cast(N1)->getVT(); - unsigned VTBits = MVT::getSizeInBits(VT); - unsigned EVTBits = MVT::getSizeInBits(EVT); +SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + MVT VT = N->getValueType(0); + MVT EVT = cast(N1)->getVT(); + unsigned VTBits = VT.getSizeInBits(); + unsigned EVTBits = EVT.getSizeInBits(); // fold (sext_in_reg c1) -> c1 if (isa(N0) || N0.getOpcode() == ISD::UNDEF) return DAG.getNode(ISD::SIGN_EXTEND_INREG, VT, N0, N1); // If the input is already sign extended, just drop the extension. - if (DAG.ComputeNumSignBits(N0) >= MVT::getSizeInBits(VT)-EVTBits+1) + if (DAG.ComputeNumSignBits(N0) >= VT.getSizeInBits()-EVTBits+1) return N0; // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && - EVT < cast(N0.getOperand(1))->getVT()) { + EVT.bitsLT(cast(N0.getOperand(1))->getVT())) { return DAG.getNode(ISD::SIGN_EXTEND_INREG, VT, N0.getOperand(0), N1); } + // fold (sext_in_reg (sext x)) -> (sext x) + // fold (sext_in_reg (aext x)) -> (sext x) + // if x is small enough. + if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { + SDValue N00 = N0.getOperand(0); + if (N00.getValueType().getSizeInBits() < EVTBits) + return DAG.getNode(ISD::SIGN_EXTEND, VT, N00, N1); + } + // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero. if (DAG.MaskedValueIsZero(N0, APInt::getBitsSet(VTBits, EVTBits-1, EVTBits))) return DAG.getZeroExtendInReg(N0, EVT); // fold operands of sext_in_reg based on knowledge that the top bits are not // demanded. - if (SimplifyDemandedBits(SDOperand(N, 0))) - return SDOperand(N, 0); + if (SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); // fold (sext_in_reg (load x)) -> (smaller sextload x) // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits)) - SDOperand NarrowLoad = ReduceLoadWidth(N); - if (NarrowLoad.Val) + SDValue NarrowLoad = ReduceLoadWidth(N); + if (NarrowLoad.getNode()) return NarrowLoad; // fold (sext_in_reg (srl X, 24), i8) -> sra X, 24 @@ -3204,51 +3463,51 @@ SDOperand DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) { // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above. if (N0.getOpcode() == ISD::SRL) { if (ConstantSDNode *ShAmt = dyn_cast(N0.getOperand(1))) - if (ShAmt->getValue()+EVTBits <= MVT::getSizeInBits(VT)) { + if (ShAmt->getZExtValue()+EVTBits <= VT.getSizeInBits()) { // We can turn this into an SRA iff the input to the SRL is already sign // extended enough. unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0)); - if (MVT::getSizeInBits(VT)-(ShAmt->getValue()+EVTBits) < InSignBits) + if (VT.getSizeInBits()-(ShAmt->getZExtValue()+EVTBits) < InSignBits) return DAG.getNode(ISD::SRA, VT, N0.getOperand(0), N0.getOperand(1)); } } // fold (sext_inreg (extload x)) -> (sextload x) - if (ISD::isEXTLoad(N0.Val) && - ISD::isUNINDEXEDLoad(N0.Val) && + if (ISD::isEXTLoad(N0.getNode()) && + ISD::isUNINDEXEDLoad(N0.getNode()) && EVT == cast(N0)->getMemoryVT() && - (!AfterLegalize || TLI.isLoadXLegal(ISD::SEXTLOAD, EVT))) { + ((!LegalOperations && !cast(N0)->isVolatile()) || + TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT))) { LoadSDNode *LN0 = cast(N0); - SDOperand ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(), - LN0->getBasePtr(), LN0->getSrcValue(), - LN0->getSrcValueOffset(), EVT, - LN0->isVolatile(), - LN0->getAlignment()); + SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(), + LN0->getBasePtr(), LN0->getSrcValue(), + LN0->getSrcValueOffset(), EVT, + LN0->isVolatile(), LN0->getAlignment()); CombineTo(N, ExtLoad); - CombineTo(N0.Val, ExtLoad, ExtLoad.getValue(1)); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! } // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use - if (ISD::isZEXTLoad(N0.Val) && ISD::isUNINDEXEDLoad(N0.Val) && + if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse() && EVT == cast(N0)->getMemoryVT() && - (!AfterLegalize || TLI.isLoadXLegal(ISD::SEXTLOAD, EVT))) { + ((!LegalOperations && !cast(N0)->isVolatile()) || + TLI.isLoadExtLegal(ISD::SEXTLOAD, EVT))) { LoadSDNode *LN0 = cast(N0); - SDOperand ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(), - LN0->getBasePtr(), LN0->getSrcValue(), - LN0->getSrcValueOffset(), EVT, - LN0->isVolatile(), - LN0->getAlignment()); + SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, VT, LN0->getChain(), + LN0->getBasePtr(), LN0->getSrcValue(), + LN0->getSrcValueOffset(), EVT, + LN0->isVolatile(), LN0->getAlignment()); CombineTo(N, ExtLoad); - CombineTo(N0.Val, ExtLoad, ExtLoad.getValue(1)); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); + return SDValue(N, 0); // Return N so it doesn't get rechecked! } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitTRUNCATE(SDNode *N) { - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); +SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { + SDValue N0 = N->getOperand(0); + MVT VT = N->getValueType(0); // noop truncate if (N0.getValueType() == N->getValueType(0)) @@ -3262,10 +3521,10 @@ SDOperand DAGCombiner::visitTRUNCATE(SDNode *N) { // fold (truncate (ext x)) -> (ext x) or (truncate x) or x if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::SIGN_EXTEND|| N0.getOpcode() == ISD::ANY_EXTEND) { - if (N0.getOperand(0).getValueType() < VT) + if (N0.getOperand(0).getValueType().bitsLT(VT)) // if the source is smaller than the dest, we still need an extend return DAG.getNode(N0.getOpcode(), VT, N0.getOperand(0)); - else if (N0.getOperand(0).getValueType() > VT) + else if (N0.getOperand(0).getValueType().bitsGT(VT)) // if the source is larger than the dest, than we just need the truncate return DAG.getNode(ISD::TRUNCATE, VT, N0.getOperand(0)); else @@ -3277,10 +3536,10 @@ SDOperand DAGCombiner::visitTRUNCATE(SDNode *N) { // See if we can simplify the input to this truncate through knowledge that // only the low bits are being used. For example "trunc (or (shl x, 8), y)" // -> trunc y - SDOperand Shorter = + SDValue Shorter = GetDemandedBits(N0, APInt::getLowBitsSet(N0.getValueSizeInBits(), - MVT::getSizeInBits(VT))); - if (Shorter.Val) + VT.getSizeInBits())); + if (Shorter.getNode()) return DAG.getNode(ISD::TRUNCATE, VT, Shorter); // fold (truncate (load x)) -> (smaller load x) @@ -3288,17 +3547,55 @@ SDOperand DAGCombiner::visitTRUNCATE(SDNode *N) { return ReduceLoadWidth(N); } -SDOperand DAGCombiner::visitBIT_CONVERT(SDNode *N) { - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); +static SDNode *getBuildPairElt(SDNode *N, unsigned i) { + SDValue Elt = N->getOperand(i); + if (Elt.getOpcode() != ISD::MERGE_VALUES) + return Elt.getNode(); + return Elt.getOperand(Elt.getResNo()).getNode(); +} + +/// CombineConsecutiveLoads - build_pair (load, load) -> load +/// if load locations are consecutive. +SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, MVT VT) { + assert(N->getOpcode() == ISD::BUILD_PAIR); + + SDNode *LD1 = getBuildPairElt(N, 0); + if (!ISD::isNON_EXTLoad(LD1) || !LD1->hasOneUse()) + return SDValue(); + MVT LD1VT = LD1->getValueType(0); + SDNode *LD2 = getBuildPairElt(N, 1); + const MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); + if (ISD::isNON_EXTLoad(LD2) && + LD2->hasOneUse() && + // If both are volatile this would reduce the number of volatile loads. + // If one is volatile it might be ok, but play conservative and bail out. + !cast(LD1)->isVolatile() && + !cast(LD2)->isVolatile() && + TLI.isConsecutiveLoad(LD2, LD1, LD1VT.getSizeInBits()/8, 1, MFI)) { + LoadSDNode *LD = cast(LD1); + unsigned Align = LD->getAlignment(); + unsigned NewAlign = TLI.getTargetData()-> + getABITypeAlignment(VT.getTypeForMVT()); + if (NewAlign <= Align && + (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT))) + return DAG.getLoad(VT, LD->getChain(), LD->getBasePtr(), + LD->getSrcValue(), LD->getSrcValueOffset(), + false, Align); + } + return SDValue(); +} + +SDValue DAGCombiner::visitBIT_CONVERT(SDNode *N) { + SDValue N0 = N->getOperand(0); + MVT VT = N->getValueType(0); // If the input is a BUILD_VECTOR with all constant elements, fold this now. // Only do this before legalize, since afterward the target may be depending // on the bitconvert. // First check to see if this is all constant. - if (!AfterLegalize && - N0.getOpcode() == ISD::BUILD_VECTOR && N0.Val->hasOneUse() && - MVT::isVector(VT)) { + if (!LegalTypes && + N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() && + VT.isVector()) { bool isSimple = true; for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) if (N0.getOperand(i).getOpcode() != ISD::UNDEF && @@ -3308,18 +3605,18 @@ SDOperand DAGCombiner::visitBIT_CONVERT(SDNode *N) { break; } - MVT::ValueType DestEltVT = MVT::getVectorElementType(N->getValueType(0)); - assert(!MVT::isVector(DestEltVT) && + MVT DestEltVT = N->getValueType(0).getVectorElementType(); + assert(!DestEltVT.isVector() && "Element type of vector ValueType must not be vector!"); if (isSimple) { - return ConstantFoldBIT_CONVERTofBUILD_VECTOR(N0.Val, DestEltVT); + return ConstantFoldBIT_CONVERTofBUILD_VECTOR(N0.getNode(), DestEltVT); } } - // If the input is a constant, let getNode() fold it. + // If the input is a constant, let getNode fold it. if (isa(N0) || isa(N0)) { - SDOperand Res = DAG.getNode(ISD::BIT_CONVERT, VT, N0); - if (Res.Val != N) return Res; + SDValue Res = DAG.getNode(ISD::BIT_CONVERT, VT, N0); + if (Res.getNode() != N) return Res; } if (N0.getOpcode() == ISD::BIT_CONVERT) // conv(conv(x,t1),t2) -> conv(x,t2) @@ -3327,32 +3624,35 @@ SDOperand DAGCombiner::visitBIT_CONVERT(SDNode *N) { // fold (conv (load x)) -> (load (conv*)x) // If the resultant load doesn't need a higher alignment than the original! - if (ISD::isNormalLoad(N0.Val) && N0.hasOneUse() && - TLI.isOperationLegal(ISD::LOAD, VT)) { + if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && + // Do not change the width of a volatile load. + !cast(N0)->isVolatile() && + (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT))) { LoadSDNode *LN0 = cast(N0); - unsigned Align = TLI.getTargetMachine().getTargetData()-> - getABITypeAlignment(MVT::getTypeForValueType(VT)); + unsigned Align = TLI.getTargetData()-> + getABITypeAlignment(VT.getTypeForMVT()); unsigned OrigAlign = LN0->getAlignment(); if (Align <= OrigAlign) { - SDOperand Load = DAG.getLoad(VT, LN0->getChain(), LN0->getBasePtr(), - LN0->getSrcValue(), LN0->getSrcValueOffset(), - LN0->isVolatile(), Align); + SDValue Load = DAG.getLoad(VT, LN0->getChain(), LN0->getBasePtr(), + LN0->getSrcValue(), LN0->getSrcValueOffset(), + LN0->isVolatile(), OrigAlign); AddToWorkList(N); - CombineTo(N0.Val, DAG.getNode(ISD::BIT_CONVERT, N0.getValueType(), Load), + CombineTo(N0.getNode(), + DAG.getNode(ISD::BIT_CONVERT, N0.getValueType(), Load), Load.getValue(1)); return Load; } } - + // Fold bitconvert(fneg(x)) -> xor(bitconvert(x), signbit) // Fold bitconvert(fabs(x)) -> and(bitconvert(x), ~signbit) // This often reduces constant pool loads. if ((N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FABS) && - N0.Val->hasOneUse() && MVT::isInteger(VT) && !MVT::isVector(VT)) { - SDOperand NewConv = DAG.getNode(ISD::BIT_CONVERT, VT, N0.getOperand(0)); - AddToWorkList(NewConv.Val); + N0.getNode()->hasOneUse() && VT.isInteger() && !VT.isVector()) { + SDValue NewConv = DAG.getNode(ISD::BIT_CONVERT, VT, N0.getOperand(0)); + AddToWorkList(NewConv.getNode()); - APInt SignBit = APInt::getSignBit(MVT::getSizeInBits(VT)); + APInt SignBit = APInt::getSignBit(VT.getSizeInBits()); if (N0.getOpcode() == ISD::FNEG) return DAG.getNode(ISD::XOR, VT, NewConv, DAG.getConstant(SignBit, VT)); assert(N0.getOpcode() == ISD::FABS); @@ -3362,88 +3662,101 @@ SDOperand DAGCombiner::visitBIT_CONVERT(SDNode *N) { // Fold bitconvert(fcopysign(cst, x)) -> bitconvert(x)&sign | cst&~sign' // Note that we don't handle copysign(x,cst) because this can always be folded // to an fneg or fabs. - if (N0.getOpcode() == ISD::FCOPYSIGN && N0.Val->hasOneUse() && + if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse() && isa(N0.getOperand(0)) && - MVT::isInteger(VT) && !MVT::isVector(VT)) { - unsigned OrigXWidth = MVT::getSizeInBits(N0.getOperand(1).getValueType()); - SDOperand X = DAG.getNode(ISD::BIT_CONVERT, MVT::getIntegerType(OrigXWidth), - N0.getOperand(1)); - AddToWorkList(X.Val); - - // If X has a different width than the result/lhs, sext it or truncate it. - unsigned VTWidth = MVT::getSizeInBits(VT); - if (OrigXWidth < VTWidth) { - X = DAG.getNode(ISD::SIGN_EXTEND, VT, X); - AddToWorkList(X.Val); - } else if (OrigXWidth > VTWidth) { - // To get the sign bit in the right place, we have to shift it right - // before truncating. - X = DAG.getNode(ISD::SRL, X.getValueType(), X, - DAG.getConstant(OrigXWidth-VTWidth, X.getValueType())); - AddToWorkList(X.Val); - X = DAG.getNode(ISD::TRUNCATE, VT, X); - AddToWorkList(X.Val); - } + VT.isInteger() && !VT.isVector()) { + unsigned OrigXWidth = N0.getOperand(1).getValueType().getSizeInBits(); + MVT IntXVT = MVT::getIntegerVT(OrigXWidth); + if (TLI.isTypeLegal(IntXVT) || !LegalTypes) { + SDValue X = DAG.getNode(ISD::BIT_CONVERT, IntXVT, N0.getOperand(1)); + AddToWorkList(X.getNode()); + + // If X has a different width than the result/lhs, sext it or truncate it. + unsigned VTWidth = VT.getSizeInBits(); + if (OrigXWidth < VTWidth) { + X = DAG.getNode(ISD::SIGN_EXTEND, VT, X); + AddToWorkList(X.getNode()); + } else if (OrigXWidth > VTWidth) { + // To get the sign bit in the right place, we have to shift it right + // before truncating. + X = DAG.getNode(ISD::SRL, X.getValueType(), X, + DAG.getConstant(OrigXWidth-VTWidth, X.getValueType())); + AddToWorkList(X.getNode()); + X = DAG.getNode(ISD::TRUNCATE, VT, X); + AddToWorkList(X.getNode()); + } - APInt SignBit = APInt::getSignBit(MVT::getSizeInBits(VT)); - X = DAG.getNode(ISD::AND, VT, X, DAG.getConstant(SignBit, VT)); - AddToWorkList(X.Val); + APInt SignBit = APInt::getSignBit(VT.getSizeInBits()); + X = DAG.getNode(ISD::AND, VT, X, DAG.getConstant(SignBit, VT)); + AddToWorkList(X.getNode()); - SDOperand Cst = DAG.getNode(ISD::BIT_CONVERT, VT, N0.getOperand(0)); - Cst = DAG.getNode(ISD::AND, VT, Cst, DAG.getConstant(~SignBit, VT)); - AddToWorkList(Cst.Val); + SDValue Cst = DAG.getNode(ISD::BIT_CONVERT, VT, N0.getOperand(0)); + Cst = DAG.getNode(ISD::AND, VT, Cst, DAG.getConstant(~SignBit, VT)); + AddToWorkList(Cst.getNode()); - return DAG.getNode(ISD::OR, VT, X, Cst); + return DAG.getNode(ISD::OR, VT, X, Cst); + } + } + + // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive. + if (N0.getOpcode() == ISD::BUILD_PAIR) { + SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT); + if (CombineLD.getNode()) + return CombineLD; } - return SDOperand(); + return SDValue(); +} + +SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) { + MVT VT = N->getValueType(0); + return CombineConsecutiveLoads(N, VT); } /// ConstantFoldBIT_CONVERTofBUILD_VECTOR - We know that BV is a build_vector /// node with Constant, ConstantFP or Undef operands. DstEltVT indicates the /// destination element value type. -SDOperand DAGCombiner:: -ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { - MVT::ValueType SrcEltVT = BV->getOperand(0).getValueType(); +SDValue DAGCombiner:: +ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT DstEltVT) { + MVT SrcEltVT = BV->getOperand(0).getValueType(); // If this is already the right type, we're done. - if (SrcEltVT == DstEltVT) return SDOperand(BV, 0); + if (SrcEltVT == DstEltVT) return SDValue(BV, 0); - unsigned SrcBitSize = MVT::getSizeInBits(SrcEltVT); - unsigned DstBitSize = MVT::getSizeInBits(DstEltVT); + unsigned SrcBitSize = SrcEltVT.getSizeInBits(); + unsigned DstBitSize = DstEltVT.getSizeInBits(); // If this is a conversion of N elements of one type to N elements of another // type, convert each element. This handles FP<->INT cases. if (SrcBitSize == DstBitSize) { - SmallVector Ops; + SmallVector Ops; for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { Ops.push_back(DAG.getNode(ISD::BIT_CONVERT, DstEltVT, BV->getOperand(i))); - AddToWorkList(Ops.back().Val); + AddToWorkList(Ops.back().getNode()); } - MVT::ValueType VT = - MVT::getVectorType(DstEltVT, - MVT::getVectorNumElements(BV->getValueType(0))); + MVT VT = MVT::getVectorVT(DstEltVT, + BV->getValueType(0).getVectorNumElements()); return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } // Otherwise, we're growing or shrinking the elements. To avoid having to // handle annoying details of growing/shrinking FP values, we convert them to // int first. - if (MVT::isFloatingPoint(SrcEltVT)) { + if (SrcEltVT.isFloatingPoint()) { // Convert the input float vector to a int vector where the elements are the // same sizes. assert((SrcEltVT == MVT::f32 || SrcEltVT == MVT::f64) && "Unknown FP VT!"); - MVT::ValueType IntVT = SrcEltVT == MVT::f32 ? MVT::i32 : MVT::i64; - BV = ConstantFoldBIT_CONVERTofBUILD_VECTOR(BV, IntVT).Val; + MVT IntVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits()); + BV = ConstantFoldBIT_CONVERTofBUILD_VECTOR(BV, IntVT).getNode(); SrcEltVT = IntVT; } // Now we know the input is an integer vector. If the output is a FP type, // convert to integer first, then to FP of the right size. - if (MVT::isFloatingPoint(DstEltVT)) { + if (DstEltVT.isFloatingPoint()) { assert((DstEltVT == MVT::f32 || DstEltVT == MVT::f64) && "Unknown FP VT!"); - MVT::ValueType TmpVT = DstEltVT == MVT::f32 ? MVT::i32 : MVT::i64; - SDNode *Tmp = ConstantFoldBIT_CONVERTofBUILD_VECTOR(BV, TmpVT).Val; + MVT TmpVT = MVT::getIntegerVT(DstEltVT.getSizeInBits()); + SDNode *Tmp = ConstantFoldBIT_CONVERTofBUILD_VECTOR(BV, TmpVT).getNode(); // Next, convert to FP elements of the same size. return ConstantFoldBIT_CONVERTofBUILD_VECTOR(Tmp, DstEltVT); @@ -3451,11 +3764,11 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { // Okay, we know the src/dst types are both integers of differing types. // Handling growing first. - assert(MVT::isInteger(SrcEltVT) && MVT::isInteger(DstEltVT)); + assert(SrcEltVT.isInteger() && DstEltVT.isInteger()); if (SrcBitSize < DstBitSize) { unsigned NumInputsPerOutput = DstBitSize/SrcBitSize; - SmallVector Ops; + SmallVector Ops; for (unsigned i = 0, e = BV->getNumOperands(); i != e; i += NumInputsPerOutput) { bool isLE = TLI.isLittleEndian(); @@ -3464,7 +3777,7 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { for (unsigned j = 0; j != NumInputsPerOutput; ++j) { // Shift the previously computed bits over. NewBits <<= SrcBitSize; - SDOperand Op = BV->getOperand(i+ (isLE ? (NumInputsPerOutput-j-1) : j)); + SDValue Op = BV->getOperand(i+ (isLE ? (NumInputsPerOutput-j-1) : j)); if (Op.getOpcode() == ISD::UNDEF) continue; EltIsUndef = false; @@ -3478,7 +3791,7 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { Ops.push_back(DAG.getConstant(NewBits, DstEltVT)); } - MVT::ValueType VT = MVT::getVectorType(DstEltVT, Ops.size()); + MVT VT = MVT::getVectorVT(DstEltVT, Ops.size()); return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } @@ -3486,9 +3799,8 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { // turns into multiple outputs. bool isS2V = ISD::isScalarToVector(BV); unsigned NumOutputsPerInput = SrcBitSize/DstBitSize; - MVT::ValueType VT = MVT::getVectorType(DstEltVT, - NumOutputsPerInput * BV->getNumOperands()); - SmallVector Ops; + MVT VT = MVT::getVectorVT(DstEltVT, NumOutputsPerInput*BV->getNumOperands()); + SmallVector Ops; for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { if (BV->getOperand(i).getOpcode() == ISD::UNDEF) { for (unsigned j = 0; j != NumOutputsPerInput; ++j) @@ -3514,17 +3826,17 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT::ValueType DstEltVT) { -SDOperand DAGCombiner::visitFADD(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitFADD(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantFPSDNode *N0CFP = dyn_cast(N0); ConstantFPSDNode *N1CFP = dyn_cast(N1); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } // fold (fadd c1, c2) -> c1+c2 @@ -3533,65 +3845,72 @@ SDOperand DAGCombiner::visitFADD(SDNode *N) { // canonicalize constant to RHS if (N0CFP && !N1CFP) return DAG.getNode(ISD::FADD, VT, N1, N0); + // fold (A + 0) -> A + if (UnsafeFPMath && N1CFP && N1CFP->getValueAPF().isZero()) + return N0; // fold (A + (-B)) -> A-B - if (isNegatibleForFree(N1, AfterLegalize) == 2) + if (isNegatibleForFree(N1, LegalOperations) == 2) return DAG.getNode(ISD::FSUB, VT, N0, - GetNegatedExpression(N1, DAG, AfterLegalize)); + GetNegatedExpression(N1, DAG, LegalOperations)); // fold ((-A) + B) -> B-A - if (isNegatibleForFree(N0, AfterLegalize) == 2) + if (isNegatibleForFree(N0, LegalOperations) == 2) return DAG.getNode(ISD::FSUB, VT, N1, - GetNegatedExpression(N0, DAG, AfterLegalize)); + GetNegatedExpression(N0, DAG, LegalOperations)); // If allowed, fold (fadd (fadd x, c1), c2) -> (fadd x, (fadd c1, c2)) if (UnsafeFPMath && N1CFP && N0.getOpcode() == ISD::FADD && - N0.Val->hasOneUse() && isa(N0.getOperand(1))) + N0.getNode()->hasOneUse() && isa(N0.getOperand(1))) return DAG.getNode(ISD::FADD, VT, N0.getOperand(0), DAG.getNode(ISD::FADD, VT, N0.getOperand(1), N1)); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitFSUB(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitFSUB(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantFPSDNode *N0CFP = dyn_cast(N0); ConstantFPSDNode *N1CFP = dyn_cast(N1); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } // fold (fsub c1, c2) -> c1-c2 if (N0CFP && N1CFP && VT != MVT::ppcf128) return DAG.getNode(ISD::FSUB, VT, N0, N1); + // fold (A-0) -> A + if (UnsafeFPMath && N1CFP && N1CFP->getValueAPF().isZero()) + return N0; // fold (0-B) -> -B if (UnsafeFPMath && N0CFP && N0CFP->getValueAPF().isZero()) { - if (isNegatibleForFree(N1, AfterLegalize)) - return GetNegatedExpression(N1, DAG, AfterLegalize); - return DAG.getNode(ISD::FNEG, VT, N1); + if (isNegatibleForFree(N1, LegalOperations)) + return GetNegatedExpression(N1, DAG, LegalOperations); + if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) + return DAG.getNode(ISD::FNEG, VT, N1); } // fold (A-(-B)) -> A+B - if (isNegatibleForFree(N1, AfterLegalize)) + if (isNegatibleForFree(N1, LegalOperations)) return DAG.getNode(ISD::FADD, VT, N0, - GetNegatedExpression(N1, DAG, AfterLegalize)); + GetNegatedExpression(N1, DAG, LegalOperations)); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitFMUL(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitFMUL(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantFPSDNode *N0CFP = dyn_cast(N0); ConstantFPSDNode *N1CFP = dyn_cast(N1); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } // fold (fmul c1, c2) -> c1*c2 @@ -3600,45 +3919,49 @@ SDOperand DAGCombiner::visitFMUL(SDNode *N) { // canonicalize constant to RHS if (N0CFP && !N1CFP) return DAG.getNode(ISD::FMUL, VT, N1, N0); + // fold (A * 0) -> 0 + if (UnsafeFPMath && N1CFP && N1CFP->getValueAPF().isZero()) + return N1; // fold (fmul X, 2.0) -> (fadd X, X) if (N1CFP && N1CFP->isExactlyValue(+2.0)) return DAG.getNode(ISD::FADD, VT, N0, N0); // fold (fmul X, -1.0) -> (fneg X) if (N1CFP && N1CFP->isExactlyValue(-1.0)) - return DAG.getNode(ISD::FNEG, VT, N0); + if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) + return DAG.getNode(ISD::FNEG, VT, N0); // -X * -Y -> X*Y - if (char LHSNeg = isNegatibleForFree(N0, AfterLegalize)) { - if (char RHSNeg = isNegatibleForFree(N1, AfterLegalize)) { + if (char LHSNeg = isNegatibleForFree(N0, LegalOperations)) { + if (char RHSNeg = isNegatibleForFree(N1, LegalOperations)) { // Both can be negated for free, check to see if at least one is cheaper // negated. if (LHSNeg == 2 || RHSNeg == 2) return DAG.getNode(ISD::FMUL, VT, - GetNegatedExpression(N0, DAG, AfterLegalize), - GetNegatedExpression(N1, DAG, AfterLegalize)); + GetNegatedExpression(N0, DAG, LegalOperations), + GetNegatedExpression(N1, DAG, LegalOperations)); } } // If allowed, fold (fmul (fmul x, c1), c2) -> (fmul x, (fmul c1, c2)) if (UnsafeFPMath && N1CFP && N0.getOpcode() == ISD::FMUL && - N0.Val->hasOneUse() && isa(N0.getOperand(1))) + N0.getNode()->hasOneUse() && isa(N0.getOperand(1))) return DAG.getNode(ISD::FMUL, VT, N0.getOperand(0), DAG.getNode(ISD::FMUL, VT, N0.getOperand(1), N1)); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitFDIV(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitFDIV(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantFPSDNode *N0CFP = dyn_cast(N0); ConstantFPSDNode *N1CFP = dyn_cast(N1); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold vector ops - if (MVT::isVector(VT)) { - SDOperand FoldedVOp = SimplifyVBinOp(N); - if (FoldedVOp.Val) return FoldedVOp; + if (VT.isVector()) { + SDValue FoldedVOp = SimplifyVBinOp(N); + if (FoldedVOp.getNode()) return FoldedVOp; } // fold (fdiv c1, c2) -> c1/c2 @@ -3647,40 +3970,40 @@ SDOperand DAGCombiner::visitFDIV(SDNode *N) { // -X / -Y -> X*Y - if (char LHSNeg = isNegatibleForFree(N0, AfterLegalize)) { - if (char RHSNeg = isNegatibleForFree(N1, AfterLegalize)) { + if (char LHSNeg = isNegatibleForFree(N0, LegalOperations)) { + if (char RHSNeg = isNegatibleForFree(N1, LegalOperations)) { // Both can be negated for free, check to see if at least one is cheaper // negated. if (LHSNeg == 2 || RHSNeg == 2) return DAG.getNode(ISD::FDIV, VT, - GetNegatedExpression(N0, DAG, AfterLegalize), - GetNegatedExpression(N1, DAG, AfterLegalize)); + GetNegatedExpression(N0, DAG, LegalOperations), + GetNegatedExpression(N1, DAG, LegalOperations)); } } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitFREM(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitFREM(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantFPSDNode *N0CFP = dyn_cast(N0); ConstantFPSDNode *N1CFP = dyn_cast(N1); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold (frem c1, c2) -> fmod(c1,c2) if (N0CFP && N1CFP && VT != MVT::ppcf128) return DAG.getNode(ISD::FREM, VT, N0, N1); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitFCOPYSIGN(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantFPSDNode *N0CFP = dyn_cast(N0); ConstantFPSDNode *N1CFP = dyn_cast(N1); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); if (N0CFP && N1CFP && VT != MVT::ppcf128) // Constant fold return DAG.getNode(ISD::FCOPYSIGN, VT, N0, N1); @@ -3689,10 +4012,13 @@ SDOperand DAGCombiner::visitFCOPYSIGN(SDNode *N) { const APFloat& V = N1CFP->getValueAPF(); // copysign(x, c1) -> fabs(x) iff ispos(c1) // copysign(x, c1) -> fneg(fabs(x)) iff isneg(c1) - if (!V.isNegative()) - return DAG.getNode(ISD::FABS, VT, N0); - else - return DAG.getNode(ISD::FNEG, VT, DAG.getNode(ISD::FABS, VT, N0)); + if (!V.isNegative()) { + if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT)) + return DAG.getNode(ISD::FABS, VT, N0); + } else { + if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) + return DAG.getNode(ISD::FNEG, VT, DAG.getNode(ISD::FABS, VT, N0)); + } } // copysign(fabs(x), y) -> copysign(x, y) @@ -3715,60 +4041,83 @@ SDOperand DAGCombiner::visitFCOPYSIGN(SDNode *N) { if (N1.getOpcode() == ISD::FP_EXTEND || N1.getOpcode() == ISD::FP_ROUND) return DAG.getNode(ISD::FCOPYSIGN, VT, N0, N1.getOperand(0)); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitSINT_TO_FP(SDNode *N) { - SDOperand N0 = N->getOperand(0); +SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) { + SDValue N0 = N->getOperand(0); ConstantSDNode *N0C = dyn_cast(N0); - MVT::ValueType VT = N->getValueType(0); - + MVT VT = N->getValueType(0); + MVT OpVT = N0.getValueType(); + // fold (sint_to_fp c1) -> c1fp - if (N0C && N0.getValueType() != MVT::ppcf128) + if (N0C && OpVT != MVT::ppcf128) return DAG.getNode(ISD::SINT_TO_FP, VT, N0); - return SDOperand(); + + // If the input is a legal type, and SINT_TO_FP is not legal on this target, + // but UINT_TO_FP is legal on this target, try to convert. + if (!TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT) && + TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT)) { + // If the sign bit is known to be zero, we can change this to UINT_TO_FP. + if (DAG.SignBitIsZero(N0)) + return DAG.getNode(ISD::UINT_TO_FP, VT, N0); + } + + + return SDValue(); } -SDOperand DAGCombiner::visitUINT_TO_FP(SDNode *N) { - SDOperand N0 = N->getOperand(0); +SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) { + SDValue N0 = N->getOperand(0); ConstantSDNode *N0C = dyn_cast(N0); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); + MVT OpVT = N0.getValueType(); // fold (uint_to_fp c1) -> c1fp - if (N0C && N0.getValueType() != MVT::ppcf128) + if (N0C && OpVT != MVT::ppcf128) return DAG.getNode(ISD::UINT_TO_FP, VT, N0); - return SDOperand(); + + // If the input is a legal type, and UINT_TO_FP is not legal on this target, + // but SINT_TO_FP is legal on this target, try to convert. + if (!TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT) && + TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT)) { + // If the sign bit is known to be zero, we can change this to SINT_TO_FP. + if (DAG.SignBitIsZero(N0)) + return DAG.getNode(ISD::SINT_TO_FP, VT, N0); + } + + return SDValue(); } -SDOperand DAGCombiner::visitFP_TO_SINT(SDNode *N) { - SDOperand N0 = N->getOperand(0); +SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) { + SDValue N0 = N->getOperand(0); ConstantFPSDNode *N0CFP = dyn_cast(N0); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold (fp_to_sint c1fp) -> c1 if (N0CFP) return DAG.getNode(ISD::FP_TO_SINT, VT, N0); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitFP_TO_UINT(SDNode *N) { - SDOperand N0 = N->getOperand(0); +SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) { + SDValue N0 = N->getOperand(0); ConstantFPSDNode *N0CFP = dyn_cast(N0); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold (fp_to_uint c1fp) -> c1 if (N0CFP && VT != MVT::ppcf128) return DAG.getNode(ISD::FP_TO_UINT, VT, N0); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitFP_ROUND(SDNode *N) { - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); +SDValue DAGCombiner::visitFP_ROUND(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); ConstantFPSDNode *N0CFP = dyn_cast(N0); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold (fp_round c1fp) -> c1fp if (N0CFP && N0.getValueType() != MVT::ppcf128) @@ -3782,43 +4131,44 @@ SDOperand DAGCombiner::visitFP_ROUND(SDNode *N) { if (N0.getOpcode() == ISD::FP_ROUND) { // This is a value preserving truncation if both round's are. bool IsTrunc = N->getConstantOperandVal(1) == 1 && - N0.Val->getConstantOperandVal(1) == 1; + N0.getNode()->getConstantOperandVal(1) == 1; return DAG.getNode(ISD::FP_ROUND, VT, N0.getOperand(0), DAG.getIntPtrConstant(IsTrunc)); } // fold (fp_round (copysign X, Y)) -> (copysign (fp_round X), Y) - if (N0.getOpcode() == ISD::FCOPYSIGN && N0.Val->hasOneUse()) { - SDOperand Tmp = DAG.getNode(ISD::FP_ROUND, VT, N0.getOperand(0), N1); - AddToWorkList(Tmp.Val); + if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse()) { + SDValue Tmp = DAG.getNode(ISD::FP_ROUND, VT, N0.getOperand(0), N1); + AddToWorkList(Tmp.getNode()); return DAG.getNode(ISD::FCOPYSIGN, VT, Tmp, N0.getOperand(1)); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitFP_ROUND_INREG(SDNode *N) { - SDOperand N0 = N->getOperand(0); - MVT::ValueType VT = N->getValueType(0); - MVT::ValueType EVT = cast(N->getOperand(1))->getVT(); +SDValue DAGCombiner::visitFP_ROUND_INREG(SDNode *N) { + SDValue N0 = N->getOperand(0); + MVT VT = N->getValueType(0); + MVT EVT = cast(N->getOperand(1))->getVT(); ConstantFPSDNode *N0CFP = dyn_cast(N0); // fold (fp_round_inreg c1fp) -> c1fp - if (N0CFP) { - SDOperand Round = DAG.getConstantFP(N0CFP->getValueAPF(), EVT); + if (N0CFP && (TLI.isTypeLegal(EVT) || !LegalTypes)) { + SDValue Round = DAG.getConstantFP(*N0CFP->getConstantFPValue(), EVT); return DAG.getNode(ISD::FP_EXTEND, VT, Round); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitFP_EXTEND(SDNode *N) { - SDOperand N0 = N->getOperand(0); +SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) { + SDValue N0 = N->getOperand(0); ConstantFPSDNode *N0CFP = dyn_cast(N0); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // If this is fp_round(fpextend), don't fold it, allow ourselves to be folded. - if (N->hasOneUse() && (*N->use_begin())->getOpcode() == ISD::FP_ROUND) - return SDOperand(); + if (N->hasOneUse() && + N->use_begin()->getOpcode() == ISD::FP_ROUND) + return SDValue(); // fold (fp_extend c1fp) -> c1fp if (N0CFP && VT != MVT::ppcf128) @@ -3826,63 +4176,63 @@ SDOperand DAGCombiner::visitFP_EXTEND(SDNode *N) { // Turn fp_extend(fp_round(X, 1)) -> x since the fp_round doesn't affect the // value of X. - if (N0.getOpcode() == ISD::FP_ROUND && N0.Val->getConstantOperandVal(1) == 1){ - SDOperand In = N0.getOperand(0); + if (N0.getOpcode() == ISD::FP_ROUND + && N0.getNode()->getConstantOperandVal(1) == 1) { + SDValue In = N0.getOperand(0); if (In.getValueType() == VT) return In; - if (VT < In.getValueType()) + if (VT.bitsLT(In.getValueType())) return DAG.getNode(ISD::FP_ROUND, VT, In, N0.getOperand(1)); return DAG.getNode(ISD::FP_EXTEND, VT, In); } // fold (fpext (load x)) -> (fpext (fptrunc (extload x))) - if (ISD::isNON_EXTLoad(N0.Val) && N0.hasOneUse() && - (!AfterLegalize||TLI.isLoadXLegal(ISD::EXTLOAD, N0.getValueType()))) { + if (ISD::isNON_EXTLoad(N0.getNode()) && N0.hasOneUse() && + ((!LegalOperations && !cast(N0)->isVolatile()) || + TLI.isLoadExtLegal(ISD::EXTLOAD, N0.getValueType()))) { LoadSDNode *LN0 = cast(N0); - SDOperand ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, VT, LN0->getChain(), - LN0->getBasePtr(), LN0->getSrcValue(), - LN0->getSrcValueOffset(), - N0.getValueType(), - LN0->isVolatile(), - LN0->getAlignment()); + SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, VT, LN0->getChain(), + LN0->getBasePtr(), LN0->getSrcValue(), + LN0->getSrcValueOffset(), + N0.getValueType(), + LN0->isVolatile(), LN0->getAlignment()); CombineTo(N, ExtLoad); - CombineTo(N0.Val, DAG.getNode(ISD::FP_ROUND, N0.getValueType(), ExtLoad, - DAG.getIntPtrConstant(1)), + CombineTo(N0.getNode(), DAG.getNode(ISD::FP_ROUND, N0.getValueType(), + ExtLoad, DAG.getIntPtrConstant(1)), ExtLoad.getValue(1)); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + return SDValue(N, 0); // Return N so it doesn't get rechecked! } - - - return SDOperand(); + + return SDValue(); } -SDOperand DAGCombiner::visitFNEG(SDNode *N) { - SDOperand N0 = N->getOperand(0); +SDValue DAGCombiner::visitFNEG(SDNode *N) { + SDValue N0 = N->getOperand(0); - if (isNegatibleForFree(N0, AfterLegalize)) - return GetNegatedExpression(N0, DAG, AfterLegalize); + if (isNegatibleForFree(N0, LegalOperations)) + return GetNegatedExpression(N0, DAG, LegalOperations); // Transform fneg(bitconvert(x)) -> bitconvert(x^sign) to avoid loading // constant pool values. - if (N0.getOpcode() == ISD::BIT_CONVERT && N0.Val->hasOneUse() && - MVT::isInteger(N0.getOperand(0).getValueType()) && - !MVT::isVector(N0.getOperand(0).getValueType())) { - SDOperand Int = N0.getOperand(0); - MVT::ValueType IntVT = Int.getValueType(); - if (MVT::isInteger(IntVT) && !MVT::isVector(IntVT)) { + if (N0.getOpcode() == ISD::BIT_CONVERT && N0.getNode()->hasOneUse() && + N0.getOperand(0).getValueType().isInteger() && + !N0.getOperand(0).getValueType().isVector()) { + SDValue Int = N0.getOperand(0); + MVT IntVT = Int.getValueType(); + if (IntVT.isInteger() && !IntVT.isVector()) { Int = DAG.getNode(ISD::XOR, IntVT, Int, - DAG.getConstant(MVT::getIntVTSignBit(IntVT), IntVT)); - AddToWorkList(Int.Val); + DAG.getConstant(IntVT.getIntegerVTSignBit(), IntVT)); + AddToWorkList(Int.getNode()); return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), Int); } } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitFABS(SDNode *N) { - SDOperand N0 = N->getOperand(0); +SDValue DAGCombiner::visitFABS(SDNode *N) { + SDValue N0 = N->getOperand(0); ConstantFPSDNode *N0CFP = dyn_cast(N0); - MVT::ValueType VT = N->getValueType(0); + MVT VT = N->getValueType(0); // fold (fabs c1) -> fabs(c1) if (N0CFP && VT != MVT::ppcf128) @@ -3897,58 +4247,59 @@ SDOperand DAGCombiner::visitFABS(SDNode *N) { // Transform fabs(bitconvert(x)) -> bitconvert(x&~sign) to avoid loading // constant pool values. - if (N0.getOpcode() == ISD::BIT_CONVERT && N0.Val->hasOneUse() && - MVT::isInteger(N0.getOperand(0).getValueType()) && - !MVT::isVector(N0.getOperand(0).getValueType())) { - SDOperand Int = N0.getOperand(0); - MVT::ValueType IntVT = Int.getValueType(); - if (MVT::isInteger(IntVT) && !MVT::isVector(IntVT)) { + if (N0.getOpcode() == ISD::BIT_CONVERT && N0.getNode()->hasOneUse() && + N0.getOperand(0).getValueType().isInteger() && + !N0.getOperand(0).getValueType().isVector()) { + SDValue Int = N0.getOperand(0); + MVT IntVT = Int.getValueType(); + if (IntVT.isInteger() && !IntVT.isVector()) { Int = DAG.getNode(ISD::AND, IntVT, Int, - DAG.getConstant(~MVT::getIntVTSignBit(IntVT), IntVT)); - AddToWorkList(Int.Val); + DAG.getConstant(~IntVT.getIntegerVTSignBit(), IntVT)); + AddToWorkList(Int.getNode()); return DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), Int); } } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitBRCOND(SDNode *N) { - SDOperand Chain = N->getOperand(0); - SDOperand N1 = N->getOperand(1); - SDOperand N2 = N->getOperand(2); +SDValue DAGCombiner::visitBRCOND(SDNode *N) { + SDValue Chain = N->getOperand(0); + SDValue N1 = N->getOperand(1); + SDValue N2 = N->getOperand(2); ConstantSDNode *N1C = dyn_cast(N1); // never taken branch, fold to chain if (N1C && N1C->isNullValue()) return Chain; // unconditional branch - if (N1C && N1C->getValue() == 1) + if (N1C && N1C->getAPIntValue() == 1) return DAG.getNode(ISD::BR, MVT::Other, Chain, N2); // fold a brcond with a setcc condition into a BR_CC node if BR_CC is legal // on the target. if (N1.getOpcode() == ISD::SETCC && - TLI.isOperationLegal(ISD::BR_CC, MVT::Other)) { + TLI.isOperationLegalOrCustom(ISD::BR_CC, MVT::Other)) { return DAG.getNode(ISD::BR_CC, MVT::Other, Chain, N1.getOperand(2), N1.getOperand(0), N1.getOperand(1), N2); } - return SDOperand(); + return SDValue(); } // Operand List for BR_CC: Chain, CondCC, CondLHS, CondRHS, DestBB. // -SDOperand DAGCombiner::visitBR_CC(SDNode *N) { +SDValue DAGCombiner::visitBR_CC(SDNode *N) { CondCodeSDNode *CC = cast(N->getOperand(1)); - SDOperand CondLHS = N->getOperand(2), CondRHS = N->getOperand(3); + SDValue CondLHS = N->getOperand(2), CondRHS = N->getOperand(3); - // Use SimplifySetCC to simplify SETCC's. - SDOperand Simp = SimplifySetCC(MVT::i1, CondLHS, CondRHS, CC->get(), false); - if (Simp.Val) AddToWorkList(Simp.Val); + // Use SimplifySetCC to simplify SETCC's. + SDValue Simp = SimplifySetCC(TLI.getSetCCResultType(CondLHS.getValueType()), + CondLHS, CondRHS, CC->get(), false); + if (Simp.getNode()) AddToWorkList(Simp.getNode()); - ConstantSDNode *SCCC = dyn_cast_or_null(Simp.Val); + ConstantSDNode *SCCC = dyn_cast_or_null(Simp.getNode()); // fold br_cc true, dest -> br dest (unconditional branch) - if (SCCC && SCCC->getValue()) + if (SCCC && !SCCC->isNullValue()) return DAG.getNode(ISD::BR, MVT::Other, N->getOperand(0), N->getOperand(4)); // fold br_cc false, dest -> unconditional fall through @@ -3956,27 +4307,27 @@ SDOperand DAGCombiner::visitBR_CC(SDNode *N) { return N->getOperand(0); // fold to a simpler setcc - if (Simp.Val && Simp.getOpcode() == ISD::SETCC) + if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC) return DAG.getNode(ISD::BR_CC, MVT::Other, N->getOperand(0), Simp.getOperand(2), Simp.getOperand(0), Simp.getOperand(1), N->getOperand(4)); - return SDOperand(); + return SDValue(); } -/// CombineToPreIndexedLoadStore - Try turning a load / store and a -/// pre-indexed load / store when the base pointer is a add or subtract +/// CombineToPreIndexedLoadStore - Try turning a load / store into a +/// pre-indexed load / store when the base pointer is an add or subtract /// and it has other uses besides the load / store. After the /// transformation, the new indexed load / store has effectively folded /// the add / subtract in and all of its other uses are redirected to the /// new load / store. bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { - if (!AfterLegalize) + if (!LegalOperations) return false; bool isLoad = true; - SDOperand Ptr; - MVT::ValueType VT; + SDValue Ptr; + MVT VT; if (LoadSDNode *LD = dyn_cast(N)) { if (LD->isIndexed()) return false; @@ -4000,18 +4351,18 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { // If the pointer is not an add/sub, or if it doesn't have multiple uses, bail // out. There is no reason to make this a preinc/predec. if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) || - Ptr.Val->hasOneUse()) + Ptr.getNode()->hasOneUse()) return false; // Ask the target to do addressing mode selection. - SDOperand BasePtr; - SDOperand Offset; + SDValue BasePtr; + SDValue Offset; ISD::MemIndexedMode AM = ISD::UNINDEXED; if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG)) return false; // Don't create a indexed load / store with zero offset. if (isa(Offset) && - cast(Offset)->getValue() == 0) + cast(Offset)->isNullValue()) return false; // Try turning it into a pre-indexed load / store except when: @@ -4029,15 +4380,15 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { // Check #2. if (!isLoad) { - SDOperand Val = cast(N)->getValue(); - if (Val == BasePtr || BasePtr.Val->isPredecessorOf(Val.Val)) + SDValue Val = cast(N)->getValue(); + if (Val == BasePtr || BasePtr.getNode()->isPredecessorOf(Val.getNode())) return false; } // Now check for #3 and #4. bool RealUse = false; - for (SDNode::use_iterator I = Ptr.Val->use_begin(), - E = Ptr.Val->use_end(); I != E; ++I) { + for (SDNode::use_iterator I = Ptr.getNode()->use_begin(), + E = Ptr.getNode()->use_end(); I != E; ++I) { SDNode *Use = *I; if (Use == N) continue; @@ -4053,24 +4404,24 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { if (!RealUse) return false; - SDOperand Result; + SDValue Result; if (isLoad) - Result = DAG.getIndexedLoad(SDOperand(N,0), BasePtr, Offset, AM); + Result = DAG.getIndexedLoad(SDValue(N,0), BasePtr, Offset, AM); else - Result = DAG.getIndexedStore(SDOperand(N,0), BasePtr, Offset, AM); + Result = DAG.getIndexedStore(SDValue(N,0), BasePtr, Offset, AM); ++PreIndexedNodes; ++NodesCombined; DOUT << "\nReplacing.4 "; DEBUG(N->dump(&DAG)); - DOUT << "\nWith: "; DEBUG(Result.Val->dump(&DAG)); + DOUT << "\nWith: "; DEBUG(Result.getNode()->dump(&DAG)); DOUT << '\n'; WorkListRemover DeadNodes(*this); if (isLoad) { - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 0), Result.getValue(0), + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0), &DeadNodes); - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 1), Result.getValue(2), + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2), &DeadNodes); } else { - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 0), Result.getValue(1), + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1), &DeadNodes); } @@ -4080,24 +4431,24 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { // Replace the uses of Ptr with uses of the updated base value. DAG.ReplaceAllUsesOfValueWith(Ptr, Result.getValue(isLoad ? 1 : 0), &DeadNodes); - removeFromWorkList(Ptr.Val); - DAG.DeleteNode(Ptr.Val); + removeFromWorkList(Ptr.getNode()); + DAG.DeleteNode(Ptr.getNode()); return true; } -/// CombineToPostIndexedLoadStore - Try combine a load / store with a +/// CombineToPostIndexedLoadStore - Try to combine a load / store with a /// add / sub of the base pointer node into a post-indexed load / store. /// The transformation folded the add / subtract into the new indexed /// load / store effectively and all of its uses are redirected to the /// new load / store. bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) { - if (!AfterLegalize) + if (!LegalOperations) return false; bool isLoad = true; - SDOperand Ptr; - MVT::ValueType VT; + SDValue Ptr; + MVT VT; if (LoadSDNode *LD = dyn_cast(N)) { if (LD->isIndexed()) return false; @@ -4118,18 +4469,18 @@ bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) { } else return false; - if (Ptr.Val->hasOneUse()) + if (Ptr.getNode()->hasOneUse()) return false; - for (SDNode::use_iterator I = Ptr.Val->use_begin(), - E = Ptr.Val->use_end(); I != E; ++I) { + for (SDNode::use_iterator I = Ptr.getNode()->use_begin(), + E = Ptr.getNode()->use_end(); I != E; ++I) { SDNode *Op = *I; if (Op == N || (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)) continue; - SDOperand BasePtr; - SDOperand Offset; + SDValue BasePtr; + SDValue Offset; ISD::MemIndexedMode AM = ISD::UNINDEXED; if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) { if (Ptr == Offset) @@ -4138,7 +4489,7 @@ bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) { continue; // Don't create a indexed load / store with zero offset. if (isa(Offset) && - cast(Offset)->getValue() == 0) + cast(Offset)->isNullValue()) continue; // Try turning it into a post-indexed load / store except when @@ -4149,10 +4500,10 @@ bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) { // Check for #1. bool TryNext = false; - for (SDNode::use_iterator II = BasePtr.Val->use_begin(), - EE = BasePtr.Val->use_end(); II != EE; ++II) { + for (SDNode::use_iterator II = BasePtr.getNode()->use_begin(), + EE = BasePtr.getNode()->use_end(); II != EE; ++II) { SDNode *Use = *II; - if (Use == Ptr.Val) + if (Use == Ptr.getNode()) continue; // If all the uses are load / store addresses, then don't do the @@ -4163,9 +4514,9 @@ bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) { EEE = Use->use_end(); III != EEE; ++III) { SDNode *UseUse = *III; if (!((UseUse->getOpcode() == ISD::LOAD && - cast(UseUse)->getBasePtr().Val == Use) || + cast(UseUse)->getBasePtr().getNode() == Use) || (UseUse->getOpcode() == ISD::STORE && - cast(UseUse)->getBasePtr().Val == Use))) + cast(UseUse)->getBasePtr().getNode() == Use))) RealUse = true; } @@ -4180,22 +4531,22 @@ bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) { // Check for #2 if (!Op->isPredecessorOf(N) && !N->isPredecessorOf(Op)) { - SDOperand Result = isLoad - ? DAG.getIndexedLoad(SDOperand(N,0), BasePtr, Offset, AM) - : DAG.getIndexedStore(SDOperand(N,0), BasePtr, Offset, AM); + SDValue Result = isLoad + ? DAG.getIndexedLoad(SDValue(N,0), BasePtr, Offset, AM) + : DAG.getIndexedStore(SDValue(N,0), BasePtr, Offset, AM); ++PostIndexedNodes; ++NodesCombined; DOUT << "\nReplacing.5 "; DEBUG(N->dump(&DAG)); - DOUT << "\nWith: "; DEBUG(Result.Val->dump(&DAG)); + DOUT << "\nWith: "; DEBUG(Result.getNode()->dump(&DAG)); DOUT << '\n'; WorkListRemover DeadNodes(*this); if (isLoad) { - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 0), Result.getValue(0), + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(0), &DeadNodes); - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 1), Result.getValue(2), + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Result.getValue(2), &DeadNodes); } else { - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 0), Result.getValue(1), + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result.getValue(1), &DeadNodes); } @@ -4203,7 +4554,7 @@ bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) { DAG.DeleteNode(N); // Replace the uses of Use with uses of the updated base value. - DAG.ReplaceAllUsesOfValueWith(SDOperand(Op, 0), + DAG.ReplaceAllUsesOfValueWith(SDValue(Op, 0), Result.getValue(isLoad ? 1 : 0), &DeadNodes); removeFromWorkList(Op); @@ -4217,7 +4568,7 @@ bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) { /// InferAlignment - If we can infer some alignment information from this /// pointer, return it. -static unsigned InferAlignment(SDOperand Ptr, SelectionDAG &DAG) { +static unsigned InferAlignment(SDValue Ptr, SelectionDAG &DAG) { // If this is a direct reference to a stack slot, use information about the // stack slot's alignment. int FrameIdx = 1 << 31; @@ -4235,7 +4586,7 @@ static unsigned InferAlignment(SDOperand Ptr, SelectionDAG &DAG) { // FIXME: Handle FI+CST. const MachineFrameInfo &MFI = *DAG.getMachineFunction().getFrameInfo(); if (MFI.isFixedObjectIndex(FrameIdx)) { - int64_t ObjectOffset = MFI.getObjectOffset(FrameIdx); + int64_t ObjectOffset = MFI.getObjectOffset(FrameIdx) + FrameOffset; // The alignment of the frame index can be determined from its offset from // the incoming frame position. If the frame object is at offset 32 and @@ -4258,13 +4609,13 @@ static unsigned InferAlignment(SDOperand Ptr, SelectionDAG &DAG) { return 0; } -SDOperand DAGCombiner::visitLOAD(SDNode *N) { +SDValue DAGCombiner::visitLOAD(SDNode *N) { LoadSDNode *LD = cast(N); - SDOperand Chain = LD->getChain(); - SDOperand Ptr = LD->getBasePtr(); + SDValue Chain = LD->getChain(); + SDValue Ptr = LD->getBasePtr(); // Try to infer better alignment information than the load already has. - if (LD->isUnindexed()) { + if (!Fast && LD->isUnindexed()) { if (unsigned Align = InferAlignment(Ptr, DAG)) { if (Align > LD->getAlignment()) return DAG.getExtLoad(LD->getExtensionType(), LD->getValueType(0), @@ -4289,33 +4640,33 @@ SDOperand DAGCombiner::visitLOAD(SDNode *N) { // Now we replace use of chain2 with chain1. This makes the second load // isomorphic to the one we are deleting, and thus makes this load live. DOUT << "\nReplacing.6 "; DEBUG(N->dump(&DAG)); - DOUT << "\nWith chain: "; DEBUG(Chain.Val->dump(&DAG)); + DOUT << "\nWith chain: "; DEBUG(Chain.getNode()->dump(&DAG)); DOUT << "\n"; WorkListRemover DeadNodes(*this); - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 1), Chain, &DeadNodes); + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), Chain, &DeadNodes); if (N->use_empty()) { removeFromWorkList(N); DAG.DeleteNode(N); } - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + return SDValue(N, 0); // Return N so it doesn't get rechecked! } } else { // Indexed loads. assert(N->getValueType(2) == MVT::Other && "Malformed indexed loads?"); if (N->hasNUsesOfValue(0, 0) && N->hasNUsesOfValue(0, 1)) { - SDOperand Undef = DAG.getNode(ISD::UNDEF, N->getValueType(0)); + SDValue Undef = DAG.getNode(ISD::UNDEF, N->getValueType(0)); DOUT << "\nReplacing.6 "; DEBUG(N->dump(&DAG)); - DOUT << "\nWith: "; DEBUG(Undef.Val->dump(&DAG)); + DOUT << "\nWith: "; DEBUG(Undef.getNode()->dump(&DAG)); DOUT << " and 2 other values\n"; WorkListRemover DeadNodes(*this); - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 0), Undef, &DeadNodes); - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 1), + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Undef, &DeadNodes); + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), DAG.getNode(ISD::UNDEF, N->getValueType(1)), &DeadNodes); - DAG.ReplaceAllUsesOfValueWith(SDOperand(N, 2), Chain, &DeadNodes); + DAG.ReplaceAllUsesOfValueWith(SDValue(N, 2), Chain, &DeadNodes); removeFromWorkList(N); DAG.DeleteNode(N); - return SDOperand(N, 0); // Return N so it doesn't get rechecked! + return SDValue(N, 0); // Return N so it doesn't get rechecked! } } } @@ -4324,8 +4675,9 @@ SDOperand DAGCombiner::visitLOAD(SDNode *N) { // value. // TODO: Handle store large -> read small portion. // TODO: Handle TRUNCSTORE/LOADEXT - if (LD->getExtensionType() == ISD::NON_EXTLOAD) { - if (ISD::isNON_TRUNCStore(Chain.Val)) { + if (LD->getExtensionType() == ISD::NON_EXTLOAD && + !LD->isVolatile()) { + if (ISD::isNON_TRUNCStore(Chain.getNode())) { StoreSDNode *PrevST = cast(Chain); if (PrevST->getBasePtr() == Ptr && PrevST->getValue().getValueType() == N->getValueType(0)) @@ -4335,11 +4687,11 @@ SDOperand DAGCombiner::visitLOAD(SDNode *N) { if (CombinerAA) { // Walk up chain skipping non-aliasing memory nodes. - SDOperand BetterChain = FindBetterChain(N, Chain); + SDValue BetterChain = FindBetterChain(N, Chain); // If there is a better chain. if (Chain != BetterChain) { - SDOperand ReplLoad; + SDValue ReplLoad; // Replace the chain to void dependency. if (LD->getExtensionType() == ISD::NON_EXTLOAD) { @@ -4357,7 +4709,7 @@ SDOperand DAGCombiner::visitLOAD(SDNode *N) { } // Create token factor to keep old chain connected. - SDOperand Token = DAG.getNode(ISD::TokenFactor, MVT::Other, + SDValue Token = DAG.getNode(ISD::TokenFactor, MVT::Other, Chain, ReplLoad.getValue(1)); // Replace uses with load result and token factor. Don't add users @@ -4368,20 +4720,20 @@ SDOperand DAGCombiner::visitLOAD(SDNode *N) { // Try transforming N to an indexed load. if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N)) - return SDOperand(N, 0); + return SDValue(N, 0); - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitSTORE(SDNode *N) { +SDValue DAGCombiner::visitSTORE(SDNode *N) { StoreSDNode *ST = cast(N); - SDOperand Chain = ST->getChain(); - SDOperand Value = ST->getValue(); - SDOperand Ptr = ST->getBasePtr(); + SDValue Chain = ST->getChain(); + SDValue Value = ST->getValue(); + SDValue Ptr = ST->getBasePtr(); // Try to infer better alignment information than the store already has. - if (ST->isUnindexed()) { + if (!Fast && ST->isUnindexed()) { if (unsigned Align = InferAlignment(Ptr, DAG)) { if (Align > ST->getAlignment()) return DAG.getTruncStore(Chain, Value, Ptr, ST->getSrcValue(), @@ -4389,67 +4741,78 @@ SDOperand DAGCombiner::visitSTORE(SDNode *N) { ST->isVolatile(), Align); } } - + // If this is a store of a bit convert, store the input value if the // resultant store does not need a higher alignment than the original. if (Value.getOpcode() == ISD::BIT_CONVERT && !ST->isTruncatingStore() && ST->isUnindexed()) { unsigned Align = ST->getAlignment(); - MVT::ValueType SVT = Value.getOperand(0).getValueType(); - unsigned OrigAlign = TLI.getTargetMachine().getTargetData()-> - getABITypeAlignment(MVT::getTypeForValueType(SVT)); - if (Align <= OrigAlign && TLI.isOperationLegal(ISD::STORE, SVT)) + MVT SVT = Value.getOperand(0).getValueType(); + unsigned OrigAlign = TLI.getTargetData()-> + getABITypeAlignment(SVT.getTypeForMVT()); + if (Align <= OrigAlign && + ((!LegalOperations && !ST->isVolatile()) || + TLI.isOperationLegalOrCustom(ISD::STORE, SVT))) return DAG.getStore(Chain, Value.getOperand(0), Ptr, ST->getSrcValue(), - ST->getSrcValueOffset(), ST->isVolatile(), Align); + ST->getSrcValueOffset(), ST->isVolatile(), OrigAlign); } - + // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' if (ConstantFPSDNode *CFP = dyn_cast(Value)) { + // NOTE: If the original store is volatile, this transform must not increase + // the number of stores. For example, on x86-32 an f64 can be stored in one + // processor operation but an i64 (which is not legal) requires two. So the + // transform should not be done in this case. if (Value.getOpcode() != ISD::TargetConstantFP) { - SDOperand Tmp; - switch (CFP->getValueType(0)) { + SDValue Tmp; + switch (CFP->getValueType(0).getSimpleVT()) { default: assert(0 && "Unknown FP type"); case MVT::f80: // We don't do this for these yet. case MVT::f128: case MVT::ppcf128: break; case MVT::f32: - if (!AfterLegalize || TLI.isTypeLegal(MVT::i32)) { + if (((TLI.isTypeLegal(MVT::i32) || !LegalTypes) && !LegalOperations && + !ST->isVolatile()) || + TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { Tmp = DAG.getConstant((uint32_t)CFP->getValueAPF(). - convertToAPInt().getZExtValue(), MVT::i32); + bitcastToAPInt().getZExtValue(), MVT::i32); return DAG.getStore(Chain, Tmp, Ptr, ST->getSrcValue(), ST->getSrcValueOffset(), ST->isVolatile(), ST->getAlignment()); } break; case MVT::f64: - if (!AfterLegalize || TLI.isTypeLegal(MVT::i64)) { - Tmp = DAG.getConstant(CFP->getValueAPF().convertToAPInt(). + if (((TLI.isTypeLegal(MVT::i64) || !LegalTypes) && !LegalOperations && + !ST->isVolatile()) || + TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) { + Tmp = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). getZExtValue(), MVT::i64); return DAG.getStore(Chain, Tmp, Ptr, ST->getSrcValue(), ST->getSrcValueOffset(), ST->isVolatile(), ST->getAlignment()); - } else if (TLI.isTypeLegal(MVT::i32)) { + } else if (!ST->isVolatile() && + TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { // Many FP stores are not made apparent until after legalize, e.g. for // argument passing. Since this is so common, custom legalize the // 64-bit integer store into two 32-bit stores. - uint64_t Val = CFP->getValueAPF().convertToAPInt().getZExtValue(); - SDOperand Lo = DAG.getConstant(Val & 0xFFFFFFFF, MVT::i32); - SDOperand Hi = DAG.getConstant(Val >> 32, MVT::i32); + uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue(); + SDValue Lo = DAG.getConstant(Val & 0xFFFFFFFF, MVT::i32); + SDValue Hi = DAG.getConstant(Val >> 32, MVT::i32); if (TLI.isBigEndian()) std::swap(Lo, Hi); int SVOffset = ST->getSrcValueOffset(); unsigned Alignment = ST->getAlignment(); bool isVolatile = ST->isVolatile(); - SDOperand St0 = DAG.getStore(Chain, Lo, Ptr, ST->getSrcValue(), + SDValue St0 = DAG.getStore(Chain, Lo, Ptr, ST->getSrcValue(), ST->getSrcValueOffset(), isVolatile, ST->getAlignment()); Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, DAG.getConstant(4, Ptr.getValueType())); SVOffset += 4; Alignment = MinAlign(Alignment, 4U); - SDOperand St1 = DAG.getStore(Chain, Hi, Ptr, ST->getSrcValue(), + SDValue St1 = DAG.getStore(Chain, Hi, Ptr, ST->getSrcValue(), SVOffset, isVolatile, Alignment); return DAG.getNode(ISD::TokenFactor, MVT::Other, St0, St1); } @@ -4460,12 +4823,12 @@ SDOperand DAGCombiner::visitSTORE(SDNode *N) { if (CombinerAA) { // Walk up chain skipping non-aliasing memory nodes. - SDOperand BetterChain = FindBetterChain(N, Chain); + SDValue BetterChain = FindBetterChain(N, Chain); // If there is a better chain. if (Chain != BetterChain) { // Replace the chain to avoid dependency. - SDOperand ReplStore; + SDValue ReplStore; if (ST->isTruncatingStore()) { ReplStore = DAG.getTruncStore(BetterChain, Value, Ptr, ST->getSrcValue(),ST->getSrcValueOffset(), @@ -4478,7 +4841,7 @@ SDOperand DAGCombiner::visitSTORE(SDNode *N) { } // Create token to keep both nodes around. - SDOperand Token = + SDValue Token = DAG.getNode(ISD::TokenFactor, MVT::Other, Chain, ReplStore); // Don't add users to work list. @@ -4488,20 +4851,20 @@ SDOperand DAGCombiner::visitSTORE(SDNode *N) { // Try transforming N to an indexed store. if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N)) - return SDOperand(N, 0); + return SDValue(N, 0); // FIXME: is there such a thing as a truncating indexed store? if (ST->isTruncatingStore() && ST->isUnindexed() && - MVT::isInteger(Value.getValueType())) { + Value.getValueType().isInteger()) { // See if we can simplify the input to this truncstore with knowledge that // only the low bits are being used. For example: // "truncstore (or (shl x, 8), y), i8" -> "truncstore y, i8" - SDOperand Shorter = + SDValue Shorter = GetDemandedBits(Value, APInt::getLowBitsSet(Value.getValueSizeInBits(), - MVT::getSizeInBits(ST->getMemoryVT()))); - AddToWorkList(Value.Val); - if (Shorter.Val) + ST->getMemoryVT().getSizeInBits())); + AddToWorkList(Value.getNode()); + if (Shorter.getNode()) return DAG.getTruncStore(Chain, Shorter, Ptr, ST->getSrcValue(), ST->getSrcValueOffset(), ST->getMemoryVT(), ST->isVolatile(), ST->getAlignment()); @@ -4511,8 +4874,8 @@ SDOperand DAGCombiner::visitSTORE(SDNode *N) { if (SimplifyDemandedBits(Value, APInt::getLowBitsSet( Value.getValueSizeInBits(), - MVT::getSizeInBits(ST->getMemoryVT())))) - return SDOperand(N, 0); + ST->getMemoryVT().getSizeInBits()))) + return SDValue(N, 0); } // If this is a load followed by a store to the same location, then the store @@ -4522,105 +4885,148 @@ SDOperand DAGCombiner::visitSTORE(SDNode *N) { ST->isUnindexed() && !ST->isVolatile() && // There can't be any side effects between the load and store, such as // a call or store. - Chain.reachesChainWithoutSideEffects(SDOperand(Ld, 1))) { + Chain.reachesChainWithoutSideEffects(SDValue(Ld, 1))) { // The store is dead, remove it. return Chain; } } - + // If this is an FP_ROUND or TRUNC followed by a store, fold this into a // truncating store. We can do this even if this is already a truncstore. if ((Value.getOpcode() == ISD::FP_ROUND || Value.getOpcode() == ISD::TRUNCATE) - && TLI.isTypeLegal(Value.getOperand(0).getValueType()) && - Value.Val->hasOneUse() && ST->isUnindexed() && + && Value.getNode()->hasOneUse() && ST->isUnindexed() && TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(), ST->getMemoryVT())) { return DAG.getTruncStore(Chain, Value.getOperand(0), Ptr, ST->getSrcValue(), ST->getSrcValueOffset(), ST->getMemoryVT(), ST->isVolatile(), ST->getAlignment()); } - - return SDOperand(); + + return SDValue(); } -SDOperand DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { - SDOperand InVec = N->getOperand(0); - SDOperand InVal = N->getOperand(1); - SDOperand EltNo = N->getOperand(2); +SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) { + SDValue InVec = N->getOperand(0); + SDValue InVal = N->getOperand(1); + SDValue EltNo = N->getOperand(2); // If the invec is a BUILD_VECTOR and if EltNo is a constant, build a new // vector with the inserted element. if (InVec.getOpcode() == ISD::BUILD_VECTOR && isa(EltNo)) { - unsigned Elt = cast(EltNo)->getValue(); - SmallVector Ops(InVec.Val->op_begin(), InVec.Val->op_end()); + unsigned Elt = cast(EltNo)->getZExtValue(); + SmallVector Ops(InVec.getNode()->op_begin(), + InVec.getNode()->op_end()); if (Elt < Ops.size()) Ops[Elt] = InVal; return DAG.getNode(ISD::BUILD_VECTOR, InVec.getValueType(), &Ops[0], Ops.size()); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { - SDOperand InVec = N->getOperand(0); - SDOperand EltNo = N->getOperand(1); +SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { + // (vextract (scalar_to_vector val, 0) -> val + SDValue InVec = N->getOperand(0); + + if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR) + return InVec.getOperand(0); + + // Perform only after legalization to ensure build_vector / vector_shuffle + // optimizations have already been done. + if (!LegalOperations) return SDValue(); + + // (vextract (v4f32 load $addr), c) -> (f32 load $addr+c*size) + // (vextract (v4f32 s2v (f32 load $addr)), c) -> (f32 load $addr+c*size) + // (vextract (v4f32 shuffle (load $addr), <1,u,u,u>), 0) -> (f32 load $addr) + SDValue EltNo = N->getOperand(1); - // (vextract (v4f32 s2v (f32 load $addr)), 0) -> (f32 load $addr) - // (vextract (v4i32 bc (v4f32 s2v (f32 load $addr))), 0) -> (i32 load $addr) if (isa(EltNo)) { - unsigned Elt = cast(EltNo)->getValue(); + unsigned Elt = cast(EltNo)->getZExtValue(); bool NewLoad = false; - if (Elt == 0) { - MVT::ValueType VT = InVec.getValueType(); - MVT::ValueType EVT = MVT::getVectorElementType(VT); - MVT::ValueType LVT = EVT; - unsigned NumElts = MVT::getVectorNumElements(VT); - if (InVec.getOpcode() == ISD::BIT_CONVERT) { - MVT::ValueType BCVT = InVec.getOperand(0).getValueType(); - if (!MVT::isVector(BCVT) || - NumElts != MVT::getVectorNumElements(BCVT)) - return SDOperand(); + bool BCNumEltsChanged = false; + MVT VT = InVec.getValueType(); + MVT EVT = VT.getVectorElementType(); + MVT LVT = EVT; + if (InVec.getOpcode() == ISD::BIT_CONVERT) { + MVT BCVT = InVec.getOperand(0).getValueType(); + if (!BCVT.isVector() || EVT.bitsGT(BCVT.getVectorElementType())) + return SDValue(); + if (VT.getVectorNumElements() != BCVT.getVectorNumElements()) + BCNumEltsChanged = true; + InVec = InVec.getOperand(0); + EVT = BCVT.getVectorElementType(); + NewLoad = true; + } + + LoadSDNode *LN0 = NULL; + if (ISD::isNormalLoad(InVec.getNode())) + LN0 = cast(InVec); + else if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR && + InVec.getOperand(0).getValueType() == EVT && + ISD::isNormalLoad(InVec.getOperand(0).getNode())) { + LN0 = cast(InVec.getOperand(0)); + } else if (InVec.getOpcode() == ISD::VECTOR_SHUFFLE) { + // (vextract (vector_shuffle (load $addr), v2, <1, u, u, u>), 1) + // => + // (load $addr+1*size) + + // If the bit convert changed the number of elements, it is unsafe + // to examine the mask. + if (BCNumEltsChanged) + return SDValue(); + unsigned Idx = cast(InVec.getOperand(2). + getOperand(Elt))->getZExtValue(); + unsigned NumElems = InVec.getOperand(2).getNumOperands(); + InVec = (Idx < NumElems) ? InVec.getOperand(0) : InVec.getOperand(1); + if (InVec.getOpcode() == ISD::BIT_CONVERT) InVec = InVec.getOperand(0); - EVT = MVT::getVectorElementType(BCVT); - NewLoad = true; + if (ISD::isNormalLoad(InVec.getNode())) { + LN0 = cast(InVec); + Elt = (Idx < NumElems) ? Idx : Idx - NumElems; } - if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR && - InVec.getOperand(0).getValueType() == EVT && - ISD::isNormalLoad(InVec.getOperand(0).Val) && - InVec.getOperand(0).hasOneUse()) { - LoadSDNode *LN0 = cast(InVec.getOperand(0)); - unsigned Align = LN0->getAlignment(); - if (NewLoad) { - // Check the resultant load doesn't need a higher alignment than the - // original load. - unsigned NewAlign = TLI.getTargetMachine().getTargetData()-> - getABITypeAlignment(MVT::getTypeForValueType(LVT)); - if (!TLI.isOperationLegal(ISD::LOAD, LVT) || NewAlign > Align) - return SDOperand(); - Align = NewAlign; - } + } + if (!LN0 || !LN0->hasOneUse() || LN0->isVolatile()) + return SDValue(); + + unsigned Align = LN0->getAlignment(); + if (NewLoad) { + // Check the resultant load doesn't need a higher alignment than the + // original load. + unsigned NewAlign = TLI.getTargetData()-> + getABITypeAlignment(LVT.getTypeForMVT()); + if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, LVT)) + return SDValue(); + Align = NewAlign; + } - return DAG.getLoad(LVT, LN0->getChain(), LN0->getBasePtr(), - LN0->getSrcValue(), LN0->getSrcValueOffset(), - LN0->isVolatile(), Align); - } + SDValue NewPtr = LN0->getBasePtr(); + if (Elt) { + unsigned PtrOff = LVT.getSizeInBits() * Elt / 8; + MVT PtrType = NewPtr.getValueType(); + if (TLI.isBigEndian()) + PtrOff = VT.getSizeInBits() / 8 - PtrOff; + NewPtr = DAG.getNode(ISD::ADD, PtrType, NewPtr, + DAG.getConstant(PtrOff, PtrType)); } + return DAG.getLoad(LVT, LN0->getChain(), NewPtr, + LN0->getSrcValue(), LN0->getSrcValueOffset(), + LN0->isVolatile(), Align); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitBUILD_VECTOR(SDNode *N) { +SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { unsigned NumInScalars = N->getNumOperands(); - MVT::ValueType VT = N->getValueType(0); - unsigned NumElts = MVT::getVectorNumElements(VT); - MVT::ValueType EltType = MVT::getVectorElementType(VT); + MVT VT = N->getValueType(0); + unsigned NumElts = VT.getVectorNumElements(); + MVT EltType = VT.getVectorElementType(); // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT // operations. If so, and if the EXTRACT_VECTOR_ELT vector inputs come from // at most two distinct vectors, turn this into a shuffle node. - SDOperand VecIn1, VecIn2; + SDValue VecIn1, VecIn2; for (unsigned i = 0; i != NumInScalars; ++i) { // Ignore undef inputs. if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue; @@ -4629,15 +5035,15 @@ SDOperand DAGCombiner::visitBUILD_VECTOR(SDNode *N) { // constant index, bail out. if (N->getOperand(i).getOpcode() != ISD::EXTRACT_VECTOR_ELT || !isa(N->getOperand(i).getOperand(1))) { - VecIn1 = VecIn2 = SDOperand(0, 0); + VecIn1 = VecIn2 = SDValue(0, 0); break; } // If the input vector type disagrees with the result of the build_vector, // we can't make a shuffle. - SDOperand ExtractedFromVec = N->getOperand(i).getOperand(0); + SDValue ExtractedFromVec = N->getOperand(i).getOperand(0); if (ExtractedFromVec.getValueType() != VT) { - VecIn1 = VecIn2 = SDOperand(0, 0); + VecIn1 = VecIn2 = SDValue(0, 0); break; } @@ -4645,27 +5051,27 @@ SDOperand DAGCombiner::visitBUILD_VECTOR(SDNode *N) { if (ExtractedFromVec == VecIn1 || ExtractedFromVec == VecIn2) continue; - if (VecIn1.Val == 0) { + if (VecIn1.getNode() == 0) { VecIn1 = ExtractedFromVec; - } else if (VecIn2.Val == 0) { + } else if (VecIn2.getNode() == 0) { VecIn2 = ExtractedFromVec; } else { // Too many inputs. - VecIn1 = VecIn2 = SDOperand(0, 0); + VecIn1 = VecIn2 = SDValue(0, 0); break; } } // If everything is good, we can make a shuffle operation. - if (VecIn1.Val) { - SmallVector BuildVecIndices; + if (VecIn1.getNode()) { + SmallVector BuildVecIndices; for (unsigned i = 0; i != NumInScalars; ++i) { if (N->getOperand(i).getOpcode() == ISD::UNDEF) { BuildVecIndices.push_back(DAG.getNode(ISD::UNDEF, TLI.getPointerTy())); continue; } - SDOperand Extract = N->getOperand(i); + SDValue Extract = N->getOperand(i); // If extracting from the first vector, just use the index directly. if (Extract.getOperand(0) == VecIn1) { @@ -4674,36 +5080,39 @@ SDOperand DAGCombiner::visitBUILD_VECTOR(SDNode *N) { } // Otherwise, use InIdx + VecSize - unsigned Idx = cast(Extract.getOperand(1))->getValue(); + unsigned Idx = + cast(Extract.getOperand(1))->getZExtValue(); BuildVecIndices.push_back(DAG.getIntPtrConstant(Idx+NumInScalars)); } // Add count and size info. - MVT::ValueType BuildVecVT = MVT::getVectorType(TLI.getPointerTy(), NumElts); - + MVT BuildVecVT = MVT::getVectorVT(TLI.getPointerTy(), NumElts); + if (!TLI.isTypeLegal(BuildVecVT) && LegalTypes) + return SDValue(); + // Return the new VECTOR_SHUFFLE node. - SDOperand Ops[5]; + SDValue Ops[5]; Ops[0] = VecIn1; - if (VecIn2.Val) { + if (VecIn2.getNode()) { Ops[1] = VecIn2; } else { // Use an undef build_vector as input for the second operand. - std::vector UnOps(NumInScalars, + std::vector UnOps(NumInScalars, DAG.getNode(ISD::UNDEF, EltType)); Ops[1] = DAG.getNode(ISD::BUILD_VECTOR, VT, &UnOps[0], UnOps.size()); - AddToWorkList(Ops[1].Val); + AddToWorkList(Ops[1].getNode()); } Ops[2] = DAG.getNode(ISD::BUILD_VECTOR, BuildVecVT, &BuildVecIndices[0], BuildVecIndices.size()); return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, Ops, 3); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { +SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { // TODO: Check to see if this is a CONCAT_VECTORS of a bunch of // EXTRACT_SUBVECTOR operations. If so, and if the EXTRACT_SUBVECTOR vector // inputs come from at most two distinct vectors, turn this into a shuffle @@ -4714,18 +5123,24 @@ SDOperand DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { return N->getOperand(0); } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { - SDOperand ShufMask = N->getOperand(2); +SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { + SDValue ShufMask = N->getOperand(2); unsigned NumElts = ShufMask.getNumOperands(); + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + + assert(N0.getValueType().getVectorNumElements() == NumElts && + "Vector shuffle must be normalized in DAG"); + // If the shuffle mask is an identity operation on the LHS, return the LHS. bool isIdentity = true; for (unsigned i = 0; i != NumElts; ++i) { if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF && - cast(ShufMask.getOperand(i))->getValue() != i) { + cast(ShufMask.getOperand(i))->getZExtValue() != i) { isIdentity = false; break; } @@ -4736,7 +5151,8 @@ SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { isIdentity = true; for (unsigned i = 0; i != NumElts; ++i) { if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF && - cast(ShufMask.getOperand(i))->getValue() != i+NumElts) { + cast(ShufMask.getOperand(i))->getZExtValue() != + i+NumElts) { isIdentity = false; break; } @@ -4751,7 +5167,7 @@ SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { unsigned BaseIdx = 0; for (unsigned i = 0; i != NumElts; ++i) if (ShufMask.getOperand(i).getOpcode() != ISD::UNDEF) { - unsigned Idx = cast(ShufMask.getOperand(i))->getValue(); + unsigned Idx=cast(ShufMask.getOperand(i))->getZExtValue(); int V = (Idx < NumElts) ? 0 : 1; if (VecNum == -1) { VecNum = V; @@ -4766,8 +5182,6 @@ SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { } } - SDOperand N0 = N->getOperand(0); - SDOperand N1 = N->getOperand(1); // Normalize unary shuffle so the RHS is undef. if (isUnary && VecNum == 1) std::swap(N0, N1); @@ -4775,21 +5189,22 @@ SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { // If it is a splat, check if the argument vector is a build_vector with // all scalar elements the same. if (isSplat) { - SDNode *V = N0.Val; + SDNode *V = N0.getNode(); // If this is a bit convert that changes the element type of the vector but // not the number of vector elements, look through it. Be careful not to // look though conversions that change things like v4f32 to v2f64. if (V->getOpcode() == ISD::BIT_CONVERT) { - SDOperand ConvInput = V->getOperand(0); - if (MVT::getVectorNumElements(ConvInput.getValueType()) == NumElts) - V = ConvInput.Val; + SDValue ConvInput = V->getOperand(0); + if (ConvInput.getValueType().isVector() && + ConvInput.getValueType().getVectorNumElements() == NumElts) + V = ConvInput.getNode(); } if (V->getOpcode() == ISD::BUILD_VECTOR) { unsigned NumElems = V->getNumOperands(); if (NumElems > BaseIdx) { - SDOperand Base; + SDValue Base; bool AllSame = true; for (unsigned i = 0; i != NumElems; ++i) { if (V->getOperand(i).getOpcode() != ISD::UNDEF) { @@ -4798,7 +5213,7 @@ SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { } } // Splat of , return - if (!Base.Val) + if (!Base.getNode()) return N0; for (unsigned i = 0; i != NumElems; ++i) { if (V->getOperand(i) != Base) { @@ -4818,106 +5233,109 @@ SDOperand DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { if (isUnary || N0 == N1) { // Check the SHUFFLE mask, mapping any inputs from the 2nd operand into the // first operand. - SmallVector MappedOps; + SmallVector MappedOps; for (unsigned i = 0; i != NumElts; ++i) { if (ShufMask.getOperand(i).getOpcode() == ISD::UNDEF || - cast(ShufMask.getOperand(i))->getValue() < NumElts) { + cast(ShufMask.getOperand(i))->getZExtValue() < + NumElts) { MappedOps.push_back(ShufMask.getOperand(i)); } else { unsigned NewIdx = - cast(ShufMask.getOperand(i))->getValue() - NumElts; - MappedOps.push_back(DAG.getConstant(NewIdx, MVT::i32)); + cast(ShufMask.getOperand(i))->getZExtValue() - + NumElts; + MappedOps.push_back(DAG.getConstant(NewIdx, + ShufMask.getOperand(i).getValueType())); } } ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMask.getValueType(), &MappedOps[0], MappedOps.size()); - AddToWorkList(ShufMask.Val); + AddToWorkList(ShufMask.getNode()); return DAG.getNode(ISD::VECTOR_SHUFFLE, N->getValueType(0), N0, DAG.getNode(ISD::UNDEF, N->getValueType(0)), ShufMask); } - return SDOperand(); + return SDValue(); } /// XformToShuffleWithZero - Returns a vector_shuffle if it able to transform /// an AND to a vector_shuffle with the destination vector and a zero vector. /// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==> /// vector_shuffle V, Zero, <0, 4, 2, 4> -SDOperand DAGCombiner::XformToShuffleWithZero(SDNode *N) { - SDOperand LHS = N->getOperand(0); - SDOperand RHS = N->getOperand(1); +SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) { + SDValue LHS = N->getOperand(0); + SDValue RHS = N->getOperand(1); if (N->getOpcode() == ISD::AND) { if (RHS.getOpcode() == ISD::BIT_CONVERT) RHS = RHS.getOperand(0); if (RHS.getOpcode() == ISD::BUILD_VECTOR) { - std::vector IdxOps; + std::vector IdxOps; unsigned NumOps = RHS.getNumOperands(); unsigned NumElts = NumOps; - MVT::ValueType EVT = MVT::getVectorElementType(RHS.getValueType()); for (unsigned i = 0; i != NumElts; ++i) { - SDOperand Elt = RHS.getOperand(i); + SDValue Elt = RHS.getOperand(i); if (!isa(Elt)) - return SDOperand(); + return SDValue(); else if (cast(Elt)->isAllOnesValue()) - IdxOps.push_back(DAG.getConstant(i, EVT)); + IdxOps.push_back(DAG.getIntPtrConstant(i)); else if (cast(Elt)->isNullValue()) - IdxOps.push_back(DAG.getConstant(NumElts, EVT)); + IdxOps.push_back(DAG.getIntPtrConstant(NumElts)); else - return SDOperand(); + return SDValue(); } // Let's see if the target supports this vector_shuffle. - if (!TLI.isVectorClearMaskLegal(IdxOps, EVT, DAG)) - return SDOperand(); + if (!TLI.isVectorClearMaskLegal(IdxOps, TLI.getPointerTy(), DAG)) + return SDValue(); // Return the new VECTOR_SHUFFLE node. - MVT::ValueType VT = MVT::getVectorType(EVT, NumElts); - std::vector Ops; + MVT EVT = RHS.getValueType().getVectorElementType(); + MVT VT = MVT::getVectorVT(EVT, NumElts); + MVT MaskVT = MVT::getVectorVT(TLI.getPointerTy(), NumElts); + std::vector Ops; LHS = DAG.getNode(ISD::BIT_CONVERT, VT, LHS); Ops.push_back(LHS); - AddToWorkList(LHS.Val); - std::vector ZeroOps(NumElts, DAG.getConstant(0, EVT)); + AddToWorkList(LHS.getNode()); + std::vector ZeroOps(NumElts, DAG.getConstant(0, EVT)); Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, VT, &ZeroOps[0], ZeroOps.size())); - Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, VT, + Ops.push_back(DAG.getNode(ISD::BUILD_VECTOR, MaskVT, &IdxOps[0], IdxOps.size())); - SDOperand Result = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, + SDValue Result = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, &Ops[0], Ops.size()); - if (VT != LHS.getValueType()) { - Result = DAG.getNode(ISD::BIT_CONVERT, LHS.getValueType(), Result); - } + if (VT != N->getValueType(0)) + Result = DAG.getNode(ISD::BIT_CONVERT, N->getValueType(0), Result); return Result; } } - return SDOperand(); + return SDValue(); } /// SimplifyVBinOp - Visit a binary vector operation, like ADD. -SDOperand DAGCombiner::SimplifyVBinOp(SDNode *N) { +SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) { // After legalize, the target may be depending on adds and other // binary ops to provide legal ways to construct constants or other // things. Simplifying them may result in a loss of legality. - if (AfterLegalize) return SDOperand(); + if (LegalOperations) return SDValue(); - MVT::ValueType VT = N->getValueType(0); - assert(MVT::isVector(VT) && "SimplifyVBinOp only works on vectors!"); + MVT VT = N->getValueType(0); + assert(VT.isVector() && "SimplifyVBinOp only works on vectors!"); - MVT::ValueType EltType = MVT::getVectorElementType(VT); - SDOperand LHS = N->getOperand(0); - SDOperand RHS = N->getOperand(1); - SDOperand Shuffle = XformToShuffleWithZero(N); - if (Shuffle.Val) return Shuffle; + MVT EltType = VT.getVectorElementType(); + SDValue LHS = N->getOperand(0); + SDValue RHS = N->getOperand(1); + SDValue Shuffle = XformToShuffleWithZero(N); + if (Shuffle.getNode()) return Shuffle; // If the LHS and RHS are BUILD_VECTOR nodes, see if we can constant fold // this operation. if (LHS.getOpcode() == ISD::BUILD_VECTOR && RHS.getOpcode() == ISD::BUILD_VECTOR) { - SmallVector Ops; + SmallVector Ops; for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) { - SDOperand LHSOp = LHS.getOperand(i); - SDOperand RHSOp = RHS.getOperand(i); + SDValue LHSOp = LHS.getOperand(i); + SDValue RHSOp = RHS.getOperand(i); // If these two elements can't be folded, bail out. if ((LHSOp.getOpcode() != ISD::UNDEF && LHSOp.getOpcode() != ISD::Constant && @@ -4930,13 +5348,13 @@ SDOperand DAGCombiner::SimplifyVBinOp(SDNode *N) { if (N->getOpcode() == ISD::SDIV || N->getOpcode() == ISD::UDIV || N->getOpcode() == ISD::FDIV) { if ((RHSOp.getOpcode() == ISD::Constant && - cast(RHSOp.Val)->isNullValue()) || + cast(RHSOp.getNode())->isNullValue()) || (RHSOp.getOpcode() == ISD::ConstantFP && - cast(RHSOp.Val)->getValueAPF().isZero())) + cast(RHSOp.getNode())->getValueAPF().isZero())) break; } Ops.push_back(DAG.getNode(N->getOpcode(), EltType, LHSOp, RHSOp)); - AddToWorkList(Ops.back().Val); + AddToWorkList(Ops.back().getNode()); assert((Ops.back().getOpcode() == ISD::UNDEF || Ops.back().getOpcode() == ISD::Constant || Ops.back().getOpcode() == ISD::ConstantFP) && @@ -4944,36 +5362,36 @@ SDOperand DAGCombiner::SimplifyVBinOp(SDNode *N) { } if (Ops.size() == LHS.getNumOperands()) { - MVT::ValueType VT = LHS.getValueType(); + MVT VT = LHS.getValueType(); return DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size()); } } - return SDOperand(); + return SDValue(); } -SDOperand DAGCombiner::SimplifySelect(SDOperand N0, SDOperand N1, SDOperand N2){ +SDValue DAGCombiner::SimplifySelect(SDValue N0, SDValue N1, SDValue N2){ assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!"); - SDOperand SCC = SimplifySelectCC(N0.getOperand(0), N0.getOperand(1), N1, N2, + SDValue SCC = SimplifySelectCC(N0.getOperand(0), N0.getOperand(1), N1, N2, cast(N0.getOperand(2))->get()); // If we got a simplified select_cc node back from SimplifySelectCC, then // break it down into a new SETCC node, and a new SELECT node, and then return // the SELECT node, since we were called with a SELECT node. - if (SCC.Val) { + if (SCC.getNode()) { // Check to see if we got a select_cc back (to turn into setcc/select). // Otherwise, just return whatever node we got back, like fabs. if (SCC.getOpcode() == ISD::SELECT_CC) { - SDOperand SETCC = DAG.getNode(ISD::SETCC, N0.getValueType(), + SDValue SETCC = DAG.getNode(ISD::SETCC, N0.getValueType(), SCC.getOperand(0), SCC.getOperand(1), SCC.getOperand(4)); - AddToWorkList(SETCC.Val); + AddToWorkList(SETCC.getNode()); return DAG.getNode(ISD::SELECT, SCC.getValueType(), SCC.getOperand(2), SCC.getOperand(3), SETCC); } return SCC; } - return SDOperand(); + return SDValue(); } /// SimplifySelectOps - Given a SELECT or a SELECT_CC node, where LHS and RHS @@ -4983,8 +5401,8 @@ SDOperand DAGCombiner::SimplifySelect(SDOperand N0, SDOperand N1, SDOperand N2){ /// node) back to the top-level of the DAG combiner loop to avoid it being /// looked at. /// -bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDOperand LHS, - SDOperand RHS) { +bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS, + SDValue RHS) { // If this is a select from two identical things, try to pull the operation // through the select. @@ -4994,6 +5412,9 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDOperand LHS, // This triggers in things like "select bool X, 10.0, 123.0" after the FP // constants have been dropped into the constant pool. if (LHS.getOpcode() == ISD::LOAD && + // Do not let this transformation reduce the number of volatile loads. + !cast(LHS)->isVolatile() && + !cast(RHS)->isVolatile() && // Token chains must be identical. LHS.getOperand(0) == RHS.getOperand(0)) { LoadSDNode *LLD = cast(LHS); @@ -5004,12 +5425,12 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDOperand LHS, // FIXME: this conflates two src values, discarding one. This is not // the right thing to do, but nothing uses srcvalues now. When they do, // turn SrcValue into a list of locations. - SDOperand Addr; + SDValue Addr; if (TheSelect->getOpcode() == ISD::SELECT) { // Check that the condition doesn't reach either load. If so, folding // this will induce a cycle into the DAG. - if (!LLD->isPredecessorOf(TheSelect->getOperand(0).Val) && - !RLD->isPredecessorOf(TheSelect->getOperand(0).Val)) { + if (!LLD->isPredecessorOf(TheSelect->getOperand(0).getNode()) && + !RLD->isPredecessorOf(TheSelect->getOperand(0).getNode())) { Addr = DAG.getNode(ISD::SELECT, LLD->getBasePtr().getValueType(), TheSelect->getOperand(0), LLD->getBasePtr(), RLD->getBasePtr()); @@ -5017,10 +5438,10 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDOperand LHS, } else { // Check that the condition doesn't reach either load. If so, folding // this will induce a cycle into the DAG. - if (!LLD->isPredecessorOf(TheSelect->getOperand(0).Val) && - !RLD->isPredecessorOf(TheSelect->getOperand(0).Val) && - !LLD->isPredecessorOf(TheSelect->getOperand(1).Val) && - !RLD->isPredecessorOf(TheSelect->getOperand(1).Val)) { + if (!LLD->isPredecessorOf(TheSelect->getOperand(0).getNode()) && + !RLD->isPredecessorOf(TheSelect->getOperand(0).getNode()) && + !LLD->isPredecessorOf(TheSelect->getOperand(1).getNode()) && + !RLD->isPredecessorOf(TheSelect->getOperand(1).getNode())) { Addr = DAG.getNode(ISD::SELECT_CC, LLD->getBasePtr().getValueType(), TheSelect->getOperand(0), TheSelect->getOperand(1), @@ -5029,8 +5450,8 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDOperand LHS, } } - if (Addr.Val) { - SDOperand Load; + if (Addr.getNode()) { + SDValue Load; if (LLD->getExtensionType() == ISD::NON_EXTLOAD) Load = DAG.getLoad(TheSelect->getValueType(0), LLD->getChain(), Addr,LLD->getSrcValue(), @@ -5051,8 +5472,8 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDOperand LHS, // Users of the old loads now use the new load's chain. We know the // old-load value is dead now. - CombineTo(LHS.Val, Load.getValue(0), Load.getValue(1)); - CombineTo(RHS.Val, Load.getValue(0), Load.getValue(1)); + CombineTo(LHS.getNode(), Load.getValue(0), Load.getValue(1)); + CombineTo(RHS.getNode(), Load.getValue(0), Load.getValue(1)); return true; } } @@ -5062,25 +5483,26 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDOperand LHS, return false; } -SDOperand DAGCombiner::SimplifySelectCC(SDOperand N0, SDOperand N1, - SDOperand N2, SDOperand N3, - ISD::CondCode CC, bool NotExtCompare) { +SDValue DAGCombiner::SimplifySelectCC(SDValue N0, SDValue N1, + SDValue N2, SDValue N3, + ISD::CondCode CC, bool NotExtCompare) { - MVT::ValueType VT = N2.getValueType(); - ConstantSDNode *N1C = dyn_cast(N1.Val); - ConstantSDNode *N2C = dyn_cast(N2.Val); - ConstantSDNode *N3C = dyn_cast(N3.Val); + MVT VT = N2.getValueType(); + ConstantSDNode *N1C = dyn_cast(N1.getNode()); + ConstantSDNode *N2C = dyn_cast(N2.getNode()); + ConstantSDNode *N3C = dyn_cast(N3.getNode()); // Determine if the condition we're dealing with is constant - SDOperand SCC = SimplifySetCC(TLI.getSetCCResultType(N0), N0, N1, CC, false); - if (SCC.Val) AddToWorkList(SCC.Val); - ConstantSDNode *SCCC = dyn_cast_or_null(SCC.Val); + SDValue SCC = SimplifySetCC(TLI.getSetCCResultType(N0.getValueType()), + N0, N1, CC, false); + if (SCC.getNode()) AddToWorkList(SCC.getNode()); + ConstantSDNode *SCCC = dyn_cast_or_null(SCC.getNode()); // fold select_cc true, x, y -> x - if (SCCC && SCCC->getValue()) + if (SCCC && !SCCC->isNullValue()) return N2; // fold select_cc false, x, y -> y - if (SCCC && SCCC->getValue() == 0) + if (SCCC && SCCC->isNullValue()) return N3; // Check to see if we can simplify the select into an fabs node @@ -5104,56 +5526,57 @@ SDOperand DAGCombiner::SimplifySelectCC(SDOperand N0, SDOperand N1, // Check to see if we can perform the "gzip trick", transforming // select_cc setlt X, 0, A, 0 -> and (sra X, size(X)-1), A if (N1C && N3C && N3C->isNullValue() && CC == ISD::SETLT && - MVT::isInteger(N0.getValueType()) && - MVT::isInteger(N2.getValueType()) && - (N1C->isNullValue() || // (a < 0) ? b : 0 - (N1C->getValue() == 1 && N0 == N2))) { // (a < 1) ? a : 0 - MVT::ValueType XType = N0.getValueType(); - MVT::ValueType AType = N2.getValueType(); - if (XType >= AType) { + N0.getValueType().isInteger() && + N2.getValueType().isInteger() && + (N1C->isNullValue() || // (a < 0) ? b : 0 + (N1C->getAPIntValue() == 1 && N0 == N2))) { // (a < 1) ? a : 0 + MVT XType = N0.getValueType(); + MVT AType = N2.getValueType(); + if (XType.bitsGE(AType)) { // and (sra X, size(X)-1, A) -> "and (srl X, C2), A" iff A is a // single-bit constant. - if (N2C && ((N2C->getValue() & (N2C->getValue()-1)) == 0)) { - unsigned ShCtV = Log2_64(N2C->getValue()); - ShCtV = MVT::getSizeInBits(XType)-ShCtV-1; - SDOperand ShCt = DAG.getConstant(ShCtV, TLI.getShiftAmountTy()); - SDOperand Shift = DAG.getNode(ISD::SRL, XType, N0, ShCt); - AddToWorkList(Shift.Val); - if (XType > AType) { + if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue()-1)) == 0)) { + unsigned ShCtV = N2C->getAPIntValue().logBase2(); + ShCtV = XType.getSizeInBits()-ShCtV-1; + SDValue ShCt = DAG.getConstant(ShCtV, TLI.getShiftAmountTy()); + SDValue Shift = DAG.getNode(ISD::SRL, XType, N0, ShCt); + AddToWorkList(Shift.getNode()); + if (XType.bitsGT(AType)) { Shift = DAG.getNode(ISD::TRUNCATE, AType, Shift); - AddToWorkList(Shift.Val); + AddToWorkList(Shift.getNode()); } return DAG.getNode(ISD::AND, AType, Shift, N2); } - SDOperand Shift = DAG.getNode(ISD::SRA, XType, N0, - DAG.getConstant(MVT::getSizeInBits(XType)-1, + SDValue Shift = DAG.getNode(ISD::SRA, XType, N0, + DAG.getConstant(XType.getSizeInBits()-1, TLI.getShiftAmountTy())); - AddToWorkList(Shift.Val); - if (XType > AType) { + AddToWorkList(Shift.getNode()); + if (XType.bitsGT(AType)) { Shift = DAG.getNode(ISD::TRUNCATE, AType, Shift); - AddToWorkList(Shift.Val); + AddToWorkList(Shift.getNode()); } return DAG.getNode(ISD::AND, AType, Shift, N2); } } // fold select C, 16, 0 -> shl C, 4 - if (N2C && N3C && N3C->isNullValue() && isPowerOf2_64(N2C->getValue()) && - TLI.getSetCCResultContents() == TargetLowering::ZeroOrOneSetCCResult) { + if (N2C && N3C && N3C->isNullValue() && N2C->getAPIntValue().isPowerOf2() && + TLI.getBooleanContents() == TargetLowering::ZeroOrOneBooleanContent) { // If the caller doesn't want us to simplify this into a zext of a compare, // don't do it. - if (NotExtCompare && N2C->getValue() == 1) - return SDOperand(); + if (NotExtCompare && N2C->getAPIntValue() == 1) + return SDValue(); // Get a SetCC of the condition // FIXME: Should probably make sure that setcc is legal if we ever have a // target where it isn't. - SDOperand Temp, SCC; + SDValue Temp, SCC; // cast from setcc result type to select result type - if (AfterLegalize) { - SCC = DAG.getSetCC(TLI.getSetCCResultType(N0), N0, N1, CC); - if (N2.getValueType() < SCC.getValueType()) + if (LegalTypes) { + SCC = DAG.getSetCC(TLI.getSetCCResultType(N0.getValueType()), + N0, N1, CC); + if (N2.getValueType().bitsLT(SCC.getValueType())) Temp = DAG.getZeroExtendInReg(SCC, N2.getValueType()); else Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getValueType(), SCC); @@ -5161,24 +5584,25 @@ SDOperand DAGCombiner::SimplifySelectCC(SDOperand N0, SDOperand N1, SCC = DAG.getSetCC(MVT::i1, N0, N1, CC); Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getValueType(), SCC); } - AddToWorkList(SCC.Val); - AddToWorkList(Temp.Val); + AddToWorkList(SCC.getNode()); + AddToWorkList(Temp.getNode()); - if (N2C->getValue() == 1) + if (N2C->getAPIntValue() == 1) return Temp; // shl setcc result by log2 n2c return DAG.getNode(ISD::SHL, N2.getValueType(), Temp, - DAG.getConstant(Log2_64(N2C->getValue()), + DAG.getConstant(N2C->getAPIntValue().logBase2(), TLI.getShiftAmountTy())); } // Check to see if this is the equivalent of setcc // FIXME: Turn all of these into setcc if setcc if setcc is legal // otherwise, go ahead with the folds. - if (0 && N3C && N3C->isNullValue() && N2C && (N2C->getValue() == 1ULL)) { - MVT::ValueType XType = N0.getValueType(); - if (TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultType(N0))) { - SDOperand Res = DAG.getSetCC(TLI.getSetCCResultType(N0), N0, N1, CC); + if (0 && N3C && N3C->isNullValue() && N2C && (N2C->getAPIntValue() == 1ULL)) { + MVT XType = N0.getValueType(); + if (!LegalOperations || + TLI.isOperationLegal(ISD::SETCC, TLI.getSetCCResultType(XType))) { + SDValue Res = DAG.getSetCC(TLI.getSetCCResultType(XType), N0, N1, CC); if (Res.getValueType() != VT) Res = DAG.getNode(ISD::ZERO_EXTEND, VT, Res); return Res; @@ -5186,27 +5610,27 @@ SDOperand DAGCombiner::SimplifySelectCC(SDOperand N0, SDOperand N1, // seteq X, 0 -> srl (ctlz X, log2(size(X))) if (N1C && N1C->isNullValue() && CC == ISD::SETEQ && - TLI.isOperationLegal(ISD::CTLZ, XType)) { - SDOperand Ctlz = DAG.getNode(ISD::CTLZ, XType, N0); + (!LegalOperations || + TLI.isOperationLegal(ISD::CTLZ, XType))) { + SDValue Ctlz = DAG.getNode(ISD::CTLZ, XType, N0); return DAG.getNode(ISD::SRL, XType, Ctlz, - DAG.getConstant(Log2_32(MVT::getSizeInBits(XType)), + DAG.getConstant(Log2_32(XType.getSizeInBits()), TLI.getShiftAmountTy())); } // setgt X, 0 -> srl (and (-X, ~X), size(X)-1) if (N1C && N1C->isNullValue() && CC == ISD::SETGT) { - SDOperand NegN0 = DAG.getNode(ISD::SUB, XType, DAG.getConstant(0, XType), + SDValue NegN0 = DAG.getNode(ISD::SUB, XType, DAG.getConstant(0, XType), N0); - SDOperand NotN0 = DAG.getNode(ISD::XOR, XType, N0, - DAG.getConstant(~0ULL, XType)); + SDValue NotN0 = DAG.getNOT(N0, XType); return DAG.getNode(ISD::SRL, XType, DAG.getNode(ISD::AND, XType, NegN0, NotN0), - DAG.getConstant(MVT::getSizeInBits(XType)-1, + DAG.getConstant(XType.getSizeInBits()-1, TLI.getShiftAmountTy())); } // setgt X, -1 -> xor (srl (X, size(X)-1), 1) if (N1C && N1C->isAllOnesValue() && CC == ISD::SETGT) { - SDOperand Sign = DAG.getNode(ISD::SRL, XType, N0, - DAG.getConstant(MVT::getSizeInBits(XType)-1, + SDValue Sign = DAG.getNode(ISD::SRL, XType, N0, + DAG.getConstant(XType.getSizeInBits()-1, TLI.getShiftAmountTy())); return DAG.getNode(ISD::XOR, XType, Sign, DAG.getConstant(1, XType)); } @@ -5216,14 +5640,14 @@ SDOperand DAGCombiner::SimplifySelectCC(SDOperand N0, SDOperand N1, // Y = sra (X, size(X)-1); xor (add (X, Y), Y) if (N1C && N1C->isNullValue() && (CC == ISD::SETLT || CC == ISD::SETLE) && N0 == N3 && N2.getOpcode() == ISD::SUB && N0 == N2.getOperand(1) && - N2.getOperand(0) == N1 && MVT::isInteger(N0.getValueType())) { - MVT::ValueType XType = N0.getValueType(); - SDOperand Shift = DAG.getNode(ISD::SRA, XType, N0, - DAG.getConstant(MVT::getSizeInBits(XType)-1, + N2.getOperand(0) == N1 && N0.getValueType().isInteger()) { + MVT XType = N0.getValueType(); + SDValue Shift = DAG.getNode(ISD::SRA, XType, N0, + DAG.getConstant(XType.getSizeInBits()-1, TLI.getShiftAmountTy())); - SDOperand Add = DAG.getNode(ISD::ADD, XType, N0, Shift); - AddToWorkList(Shift.Val); - AddToWorkList(Add.Val); + SDValue Add = DAG.getNode(ISD::ADD, XType, N0, Shift); + AddToWorkList(Shift.getNode()); + AddToWorkList(Add.getNode()); return DAG.getNode(ISD::XOR, XType, Add, Shift); } // Check to see if this is an integer abs. select_cc setgt X, -1, X, -X -> @@ -5231,28 +5655,28 @@ SDOperand DAGCombiner::SimplifySelectCC(SDOperand N0, SDOperand N1, if (N1C && N1C->isAllOnesValue() && CC == ISD::SETGT && N0 == N2 && N3.getOpcode() == ISD::SUB && N0 == N3.getOperand(1)) { if (ConstantSDNode *SubC = dyn_cast(N3.getOperand(0))) { - MVT::ValueType XType = N0.getValueType(); - if (SubC->isNullValue() && MVT::isInteger(XType)) { - SDOperand Shift = DAG.getNode(ISD::SRA, XType, N0, - DAG.getConstant(MVT::getSizeInBits(XType)-1, + MVT XType = N0.getValueType(); + if (SubC->isNullValue() && XType.isInteger()) { + SDValue Shift = DAG.getNode(ISD::SRA, XType, N0, + DAG.getConstant(XType.getSizeInBits()-1, TLI.getShiftAmountTy())); - SDOperand Add = DAG.getNode(ISD::ADD, XType, N0, Shift); - AddToWorkList(Shift.Val); - AddToWorkList(Add.Val); + SDValue Add = DAG.getNode(ISD::ADD, XType, N0, Shift); + AddToWorkList(Shift.getNode()); + AddToWorkList(Add.getNode()); return DAG.getNode(ISD::XOR, XType, Add, Shift); } } } - return SDOperand(); + return SDValue(); } /// SimplifySetCC - This is a stub for TargetLowering::SimplifySetCC. -SDOperand DAGCombiner::SimplifySetCC(MVT::ValueType VT, SDOperand N0, - SDOperand N1, ISD::CondCode Cond, - bool foldBooleans) { +SDValue DAGCombiner::SimplifySetCC(MVT VT, SDValue N0, + SDValue N1, ISD::CondCode Cond, + bool foldBooleans) { TargetLowering::DAGCombinerInfo - DagCombineInfo(DAG, !AfterLegalize, false, this); + DagCombineInfo(DAG, Level == Unrestricted, false, this); return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo); } @@ -5260,9 +5684,9 @@ SDOperand DAGCombiner::SimplifySetCC(MVT::ValueType VT, SDOperand N0, /// return a DAG expression to select that will generate the same value by /// multiplying by a magic number. See: /// -SDOperand DAGCombiner::BuildSDIV(SDNode *N) { +SDValue DAGCombiner::BuildSDIV(SDNode *N) { std::vector Built; - SDOperand S = TLI.BuildSDIV(N, DAG, &Built); + SDValue S = TLI.BuildSDIV(N, DAG, &Built); for (std::vector::iterator ii = Built.begin(), ee = Built.end(); ii != ee; ++ii) @@ -5274,9 +5698,9 @@ SDOperand DAGCombiner::BuildSDIV(SDNode *N) { /// return a DAG expression to select that will generate the same value by /// multiplying by a magic number. See: /// -SDOperand DAGCombiner::BuildUDIV(SDNode *N) { +SDValue DAGCombiner::BuildUDIV(SDNode *N) { std::vector Built; - SDOperand S = TLI.BuildUDIV(N, DAG, &Built); + SDValue S = TLI.BuildUDIV(N, DAG, &Built); for (std::vector::iterator ii = Built.begin(), ee = Built.end(); ii != ee; ++ii) @@ -5286,7 +5710,7 @@ SDOperand DAGCombiner::BuildUDIV(SDNode *N) { /// FindBaseOffset - Return true if base is known not to alias with anything /// but itself. Provides base object and offset as results. -static bool FindBaseOffset(SDOperand Ptr, SDOperand &Base, int64_t &Offset) { +static bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset) { // Assume it is a primitive operation. Base = Ptr; Offset = 0; @@ -5294,7 +5718,7 @@ static bool FindBaseOffset(SDOperand Ptr, SDOperand &Base, int64_t &Offset) { if (Base.getOpcode() == ISD::ADD) { if (ConstantSDNode *C = dyn_cast(Base.getOperand(1))) { Base = Base.getOperand(0); - Offset += C->getValue(); + Offset += C->getZExtValue(); } } @@ -5306,16 +5730,16 @@ static bool FindBaseOffset(SDOperand Ptr, SDOperand &Base, int64_t &Offset) { /// isAlias - Return true if there is any possibility that the two addresses /// overlap. -bool DAGCombiner::isAlias(SDOperand Ptr1, int64_t Size1, +bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1, const Value *SrcValue1, int SrcValueOffset1, - SDOperand Ptr2, int64_t Size2, + SDValue Ptr2, int64_t Size2, const Value *SrcValue2, int SrcValueOffset2) { // If they are the same then they must be aliases. if (Ptr1 == Ptr2) return true; // Gather base node and offset information. - SDOperand Base1, Base2; + SDValue Base1, Base2; int64_t Offset1, Offset2; bool KnownBase1 = FindBaseOffset(Ptr1, Base1, Offset1); bool KnownBase2 = FindBaseOffset(Ptr2, Base2, Offset2); @@ -5347,17 +5771,17 @@ bool DAGCombiner::isAlias(SDOperand Ptr1, int64_t Size1, /// FindAliasInfo - Extracts the relevant alias information from the memory /// node. Returns true if the operand was a load. bool DAGCombiner::FindAliasInfo(SDNode *N, - SDOperand &Ptr, int64_t &Size, + SDValue &Ptr, int64_t &Size, const Value *&SrcValue, int &SrcValueOffset) { if (LoadSDNode *LD = dyn_cast(N)) { Ptr = LD->getBasePtr(); - Size = MVT::getSizeInBits(LD->getMemoryVT()) >> 3; + Size = LD->getMemoryVT().getSizeInBits() >> 3; SrcValue = LD->getSrcValue(); SrcValueOffset = LD->getSrcValueOffset(); return true; } else if (StoreSDNode *ST = dyn_cast(N)) { Ptr = ST->getBasePtr(); - Size = MVT::getSizeInBits(ST->getMemoryVT()) >> 3; + Size = ST->getMemoryVT().getSizeInBits() >> 3; SrcValue = ST->getSrcValue(); SrcValueOffset = ST->getSrcValueOffset(); } else { @@ -5369,13 +5793,13 @@ bool DAGCombiner::FindAliasInfo(SDNode *N, /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes, /// looking for aliasing nodes and adding them to the Aliases vector. -void DAGCombiner::GatherAllAliases(SDNode *N, SDOperand OriginalChain, - SmallVector &Aliases) { - SmallVector Chains; // List of chains to visit. +void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain, + SmallVector &Aliases) { + SmallVector Chains; // List of chains to visit. std::set Visited; // Visited node set. // Get alias information for node. - SDOperand Ptr; + SDValue Ptr; int64_t Size; const Value *SrcValue; int SrcValueOffset; @@ -5388,12 +5812,12 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDOperand OriginalChain, // aliases list. If not, then continue up the chain looking for the next // candidate. while (!Chains.empty()) { - SDOperand Chain = Chains.back(); + SDValue Chain = Chains.back(); Chains.pop_back(); // Don't bother if we've been before. - if (Visited.find(Chain.Val) != Visited.end()) continue; - Visited.insert(Chain.Val); + if (Visited.find(Chain.getNode()) != Visited.end()) continue; + Visited.insert(Chain.getNode()); switch (Chain.getOpcode()) { case ISD::EntryToken: @@ -5403,11 +5827,11 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDOperand OriginalChain, case ISD::LOAD: case ISD::STORE: { // Get alias information for Chain. - SDOperand OpPtr; + SDValue OpPtr; int64_t OpSize; const Value *OpSrcValue; int OpSrcValueOffset; - bool IsOpLoad = FindAliasInfo(Chain.Val, OpPtr, OpSize, + bool IsOpLoad = FindAliasInfo(Chain.getNode(), OpPtr, OpSize, OpSrcValue, OpSrcValueOffset); // If chain is alias then stop here. @@ -5419,7 +5843,7 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDOperand OriginalChain, // Look further up the chain. Chains.push_back(Chain.getOperand(0)); // Clean up old chain. - AddToWorkList(Chain.Val); + AddToWorkList(Chain.getNode()); } break; } @@ -5432,7 +5856,7 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDOperand OriginalChain, for (unsigned n = Chain.getNumOperands(); n;) Chains.push_back(Chain.getOperand(--n)); // Eliminate the token factor if we can. - AddToWorkList(Chain.Val); + AddToWorkList(Chain.getNode()); break; default: @@ -5445,8 +5869,8 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDOperand OriginalChain, /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes, looking /// for a better chain (aliasing node.) -SDOperand DAGCombiner::FindBetterChain(SDNode *N, SDOperand OldChain) { - SmallVector Aliases; // Ops for replacing token factor. +SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) { + SmallVector Aliases; // Ops for replacing token factor. // Accumulate all the aliases to this node. GatherAllAliases(N, OldChain, Aliases); @@ -5460,23 +5884,19 @@ SDOperand DAGCombiner::FindBetterChain(SDNode *N, SDOperand OldChain) { } // Construct a custom tailored token factor. - SDOperand NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, + SDValue NewChain = DAG.getNode(ISD::TokenFactor, MVT::Other, &Aliases[0], Aliases.size()); // Make sure the old chain gets cleaned up. - if (NewChain != OldChain) AddToWorkList(OldChain.Val); + if (NewChain != OldChain) AddToWorkList(OldChain.getNode()); return NewChain; } // SelectionDAG::Combine - This is the entry point for the file. // -void SelectionDAG::Combine(bool RunningAfterLegalize, AliasAnalysis &AA) { - if (!RunningAfterLegalize && ViewDAGCombine1) - viewGraph(); - if (RunningAfterLegalize && ViewDAGCombine2) - viewGraph(); +void SelectionDAG::Combine(CombineLevel Level, AliasAnalysis &AA, bool Fast) { /// run - This is the main entry point to this class. /// - DAGCombiner(*this, AA).Run(RunningAfterLegalize); + DAGCombiner(*this, AA, Fast).Run(Level); }