From: Sanjay Patel Date: Tue, 16 Jun 2015 16:25:43 +0000 (+0000) Subject: propagate IR-level fast-math-flags to DAG nodes, disabled by default X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=7ddee6601d49dc82ddf77910590a3f8acd12b5c5;p=oota-llvm.git propagate IR-level fast-math-flags to DAG nodes, disabled by default This is an updated version of the patch that was checked in at: http://reviews.llvm.org/rL237046 but subsequently reverted because it exposed a bug in the DAG Combiner: http://reviews.llvm.org/D9893 This time, there's an enablement flag ("EnableFMFInDAG") around the code in SelectionDAGBuilder where we copy the set of FP optimization flags from IR instructions to DAG nodes. So, in theory, there should be no functional change from this patch as-is, but it will allow testing with the added functionality to proceed via "-enable-fmf-dag" passed to llc. This patch adds the minimum plumbing necessary to use IR-level fast-math-flags (FMF) in the backend without actually using them for anything yet. This is a follow-on to: http://reviews.llvm.org/rL235997 Differential Revision: http://reviews.llvm.org/D10403 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@239828 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 78fdd040773..aa50dea2576 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -669,7 +669,7 @@ public: SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2, - bool nuw = false, bool nsw = false, bool exact = false); + const SDNodeFlags *Flags = nullptr); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2, SDValue N3); SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2, @@ -990,8 +990,7 @@ public: /// Get the specified node if it's already available, or else return NULL. SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTs, ArrayRef Ops, - bool nuw = false, bool nsw = false, - bool exact = false); + const SDNodeFlags *Flags = nullptr); /// Creates a SDDbgValue node. SDDbgValue *getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R, @@ -1253,8 +1252,8 @@ private: void allnodes_clear(); BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - SDValue N1, SDValue N2, bool nuw, bool nsw, - bool exact); + SDValue N1, SDValue N2, + const SDNodeFlags *Flags = nullptr); /// Look up the node specified by ID in CSEMap. If it exists, return it. If /// not, return the insertion token that will make insertion faster. This diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index daa19430192..69fe9ce1cc4 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1017,6 +1017,11 @@ static bool isBinOpWithFlags(unsigned Opcode) { case ISD::ADD: case ISD::SUB: case ISD::SHL: + case ISD::FADD: + case ISD::FDIV: + case ISD::FMUL: + case ISD::FREM: + case ISD::FSUB: return true; default: return false; @@ -1029,8 +1034,8 @@ class BinaryWithFlagsSDNode : public BinarySDNode { public: SDNodeFlags Flags; BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - SDValue X, SDValue Y) - : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags() {} + SDValue X, SDValue Y, const SDNodeFlags &NodeFlags) + : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags(NodeFlags) {} static bool classof(const SDNode *N) { return isBinOpWithFlags(N->getOpcode()); } diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index ab72941310d..95828fe374b 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -1463,12 +1463,9 @@ SDValue DAGCombiner::combine(SDNode *N) { if (isa(N0) || !isa(N1)) { SDValue Ops[] = {N1, N0}; SDNode *CSENode; - if (const BinaryWithFlagsSDNode *BinNode = - dyn_cast(N)) { + if (const auto *BinNode = dyn_cast(N)) { CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops, - BinNode->Flags.hasNoUnsignedWrap(), - BinNode->Flags.hasNoSignedWrap(), - BinNode->Flags.hasExact()); + &BinNode->Flags); } else { CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops); } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 20cec6ebad1..98eb8bf487f 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -400,19 +400,24 @@ static void AddNodeIDOperands(FoldingSetNodeID &ID, ID.AddInteger(Op.getResNo()); } } +/// Add logical or fast math flag values to FoldingSetNodeID value. +static void AddNodeIDFlags(FoldingSetNodeID &ID, unsigned Opcode, + const SDNodeFlags *Flags) { + if (!Flags || !isBinOpWithFlags(Opcode)) + return; -static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw, - bool exact) { - ID.AddBoolean(nuw); - ID.AddBoolean(nsw); - ID.AddBoolean(exact); + unsigned RawFlags = Flags->getRawFlags(); + // If no flags are set, do not alter the ID. We must match the ID of nodes + // that were created without explicitly specifying flags. This also saves time + // and allows a gradual increase in API usage of the optional optimization + // flags. + if (RawFlags != 0) + ID.AddInteger(RawFlags); } -/// AddBinaryNodeIDCustom - Add BinarySDNodes special infos -static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode, - bool nuw, bool nsw, bool exact) { - if (isBinOpWithFlags(Opcode)) - AddBinaryNodeIDCustom(ID, nuw, nsw, exact); +static void AddNodeIDFlags(FoldingSetNodeID &ID, const SDNode *N) { + if (auto *Node = dyn_cast(N)) + AddNodeIDFlags(ID, Node->getOpcode(), &Node->Flags); } static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC, @@ -507,20 +512,6 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { ID.AddInteger(ST->getPointerInfo().getAddrSpace()); break; } - case ISD::SDIV: - case ISD::UDIV: - case ISD::SRA: - case ISD::SRL: - case ISD::MUL: - case ISD::ADD: - case ISD::SUB: - case ISD::SHL: { - const BinaryWithFlagsSDNode *BinNode = cast(N); - AddBinaryNodeIDCustom( - ID, N->getOpcode(), BinNode->Flags.hasNoUnsignedWrap(), - BinNode->Flags.hasNoSignedWrap(), BinNode->Flags.hasExact()); - break; - } case ISD::ATOMIC_CMP_SWAP: case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: case ISD::ATOMIC_SWAP: @@ -564,6 +555,8 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { } } // end switch (N->getOpcode()) + AddNodeIDFlags(ID, N); + // Target specific memory nodes could also have address spaces to check. if (N->isTargetMemoryOpcode()) ID.AddInteger(cast(N)->getPointerInfo().getAddrSpace()); @@ -960,14 +953,16 @@ void SelectionDAG::allnodes_clear() { BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1, - SDValue N2, bool nuw, bool nsw, - bool exact) { + SDValue N2, + const SDNodeFlags *Flags) { if (isBinOpWithFlags(Opcode)) { + // If no flags were passed in, use a default flags object. + SDNodeFlags F; + if (Flags == nullptr) + Flags = &F; + BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode( - Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2); - FN->Flags.setNoUnsignedWrap(nuw); - FN->Flags.setNoSignedWrap(nsw); - FN->Flags.setExact(exact); + Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags); return FN; } @@ -3269,7 +3264,7 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT, } SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, - SDValue N2, bool nuw, bool nsw, bool exact) { + SDValue N2, const SDNodeFlags *Flags) { ConstantSDNode *N1C = dyn_cast(N1.getNode()); ConstantSDNode *N2C = dyn_cast(N2.getNode()); switch (Opcode) { @@ -3756,22 +3751,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, // Memoize this node if possible. BinarySDNode *N; SDVTList VTs = getVTList(VT); - const bool BinOpHasFlags = isBinOpWithFlags(Opcode); if (VT != MVT::Glue) { SDValue Ops[] = {N1, N2}; FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTs, Ops); - if (BinOpHasFlags) - AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact); + AddNodeIDFlags(ID, Opcode, Flags); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) return SDValue(E, 0); - N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact); + N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags); CSEMap.InsertNode(N, IP); } else { - N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact); + N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags); } InsertNode(N); @@ -6075,13 +6068,12 @@ SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT, /// getNodeIfExists - Get the specified node if it's already available, or /// else return NULL. SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, - ArrayRef Ops, bool nuw, bool nsw, - bool exact) { + ArrayRef Ops, + const SDNodeFlags *Flags) { if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTList, Ops); - if (isBinOpWithFlags(Opcode)) - AddBinaryNodeIDCustom(ID, nuw, nsw, exact); + AddNodeIDFlags(ID, Opcode, Flags); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, DebugLoc(), IP)) return E; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 4035820244a..d1f881412f2 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -78,6 +78,10 @@ LimitFPPrecision("limit-float-precision", cl::location(LimitFloatPrecision), cl::init(0)); +static cl::opt +EnableFMFInDAG("enable-fmf-dag", cl::init(false), cl::Hidden, + cl::desc("Enable fast-math-flags for DAG nodes")); + // Limit the width of DAG chains. This is important in general to prevent // prevent DAG-based analysis from blowing up. For example, alias analysis and // load clustering may not complete in reasonable time. It is difficult to @@ -2148,6 +2152,8 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) { bool nuw = false; bool nsw = false; bool exact = false; + FastMathFlags FMF; + if (const OverflowingBinaryOperator *OFBinOp = dyn_cast(&I)) { nuw = OFBinOp->hasNoUnsignedWrap(); @@ -2156,9 +2162,22 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned OpCode) { if (const PossiblyExactOperator *ExactOp = dyn_cast(&I)) exact = ExactOp->isExact(); - + if (const FPMathOperator *FPOp = dyn_cast(&I)) + FMF = FPOp->getFastMathFlags(); + + SDNodeFlags Flags; + Flags.setExact(exact); + Flags.setNoSignedWrap(nsw); + Flags.setNoUnsignedWrap(nuw); + if (EnableFMFInDAG) { + Flags.setAllowReciprocal(FMF.allowReciprocal()); + Flags.setNoInfs(FMF.noInfs()); + Flags.setNoNaNs(FMF.noNaNs()); + Flags.setNoSignedZeros(FMF.noSignedZeros()); + Flags.setUnsafeAlgebra(FMF.unsafeAlgebra()); + } SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(), - Op1, Op2, nuw, nsw, exact); + Op1, Op2, &Flags); setValue(&I, BinNodeValue); } @@ -2206,9 +2225,12 @@ void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { dyn_cast(&I)) exact = ExactOp->isExact(); } - + SDNodeFlags Flags; + Flags.setExact(exact); + Flags.setNoSignedWrap(nsw); + Flags.setNoUnsignedWrap(nuw); SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, - nuw, nsw, exact); + &Flags); setValue(&I, Res); } diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 9daf2a50ad8..c70c3a27040 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -2671,8 +2671,9 @@ SDValue TargetLowering::BuildExactSDIV(SDValue Op1, SDValue Op2, SDLoc dl, // TODO: For UDIV use SRL instead of SRA. SDValue Amt = DAG.getConstant(ShAmt, dl, getShiftAmountTy(Op1.getValueType())); - Op1 = DAG.getNode(ISD::SRA, dl, Op1.getValueType(), Op1, Amt, false, false, - true); + SDNodeFlags Flags; + Flags.setExact(true); + Op1 = DAG.getNode(ISD::SRA, dl, Op1.getValueType(), Op1, Amt, &Flags); d = d.ashr(ShAmt); }