R600/SI: Allow using f32 rcp / rsq when denormals not handled.

These are precise enough to use for OpenCL unless denormals
are handled.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@213107 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Target/R600/SIISelLowering.cpp b/lib/Target/R600/SIISelLowering.cpp
index b107ed4238a0e0611868f37c530aef36b39176ba..b3429b9748c2c85de012989f93fb9dcee207f382 100644 (file)
--- a/lib/Target/R600/SIISelLowering.cpp
+++ b/lib/Target/R600/SIISelLowering.cpp
@@ -936,16 +936,27 @@ SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   return DAG.getNode(ISD::BITCAST, DL, MVT::i64, Res);
 }
 
-static SDValue performUnsafeFDIV(SDValue Op, SelectionDAG &DAG) {
+// Catch division cases where we can use shortcuts with rcp and rsq
+// instructions.
+SDValue SITargetLowering::LowerFastFDIV(SDValue Op, SelectionDAG &DAG) const {
   SDLoc SL(Op);
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
   EVT VT = Op.getValueType();
+  bool Unsafe = DAG.getTarget().Options.UnsafeFPMath;
 
   if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(LHS)) {
-    if (CLHS->isExactlyValue(1.0)) {
+    if ((Unsafe || (VT == MVT::f32 && !Subtarget->hasFP32Denormals())) &&
+        CLHS->isExactlyValue(1.0)) {
+      // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
+      // the CI documentation has a worst case error of 1 ulp.
+      // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
+      // use it as long as we aren't trying to use denormals.
 
       // 1.0 / sqrt(x) -> rsq(x)
+      //
+      // XXX - Is UnsafeFPMath sufficient to do this for f64? The maximum ULP
+      // error seems really high at 2^29 ULP.
       if (RHS.getOpcode() == ISD::FSQRT)
         return DAG.getNode(AMDGPUISD::RSQ, SL, VT, RHS.getOperand(0));
 
@@ -954,15 +965,25 @@ static SDValue performUnsafeFDIV(SDValue Op, SelectionDAG &DAG) {
     }
   }
 
-  // Turn into multiply by the reciprocal
-  // x / y -> x * (1.0 / y)
-  SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
-  return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip);
+  if (Unsafe) {
+    // Turn into multiply by the reciprocal.
+    // x / y -> x * (1.0 / y)
+    SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS);
+    return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip);
+  }
+
+  return SDValue();
 }
 
 SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
-  if (DAG.getTarget().Options.UnsafeFPMath)
-    return performUnsafeFDIV(Op, DAG);
+  SDValue FastLowered = LowerFastFDIV(Op, DAG);
+  if (FastLowered.getNode())
+    return FastLowered;
+
+  // This uses v_rcp_f32 which does not handle denormals. Let this hit a
+  // selection error for now rather than do something incorrect.
+  if (Subtarget->hasFP32Denormals())
+    return SDValue();
 
   SDLoc SL(Op);
   SDValue LHS = Op.getOperand(0);

diff --git a/lib/Target/R600/SIISelLowering.h b/lib/Target/R600/SIISelLowering.h
index aa09d2c9f8e9bb9ccedcd916cbbeccd6da3cfd48..9e9a0b05f530f46e1fa47431dea0a93749379634 100644 (file)
--- a/lib/Target/R600/SIISelLowering.h
+++ b/lib/Target/R600/SIISelLowering.h
@@ -27,6 +27,7 @@ class SITargetLowering : public AMDGPUTargetLowering {
                                SelectionDAG &DAG) const;
   SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerFastFDIV(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFDIV32(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFDIV64(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFDIV(SDValue Op, SelectionDAG &DAG) const;

diff --git a/lib/Target/R600/SIInstructions.td b/lib/Target/R600/SIInstructions.td
index 1d2dd2f63a652dca78f177fa3186066e5e8e33a9..1a9be6c6157eaf74878e80661025e64bc60c7f09 100644 (file)
--- a/lib/Target/R600/SIInstructions.td
+++ b/lib/Target/R600/SIInstructions.td
@@ -1800,10 +1800,9 @@ def : Pat <
 // VOP1 Patterns
 //===----------------------------------------------------------------------===//
 
+let Predicates = [UnsafeFPMath] in {
 def : RcpPat<V_RCP_F64_e32, f64>;
 defm : RsqPat<V_RSQ_F64_e32, f64>;
-
-let Predicates = [UnsafeFPMath] in {
 defm : RsqPat<V_RSQ_F32_e32, f32>;
 }
 

diff --git a/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll b/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll
index 62a30a79a2d0ae549e6443cf6afd21926571f39d..b5dda0ce81f901a99d0e271de1fdb0df4fa81b61 100644 (file)
--- a/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll
+++ b/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll
@@ -20,7 +20,8 @@ define void @rcp_pat_f64(double addrspace(1)* %out, double %src) nounwind {
 }
 
 ; FUNC-LABEL: @rsq_rcp_pat_f64
-; SI: V_RSQ_F64_e32
+; SI-UNSAFE: V_RSQ_F64_e32
+; SI-SAFE-NOT: V_RSQ_F64_e32
 define void @rsq_rcp_pat_f64(double addrspace(1)* %out, double %src) nounwind {
   %sqrt = call double @llvm.sqrt.f64(double %src) nounwind readnone
   %rcp = call double @llvm.AMDGPU.rcp.f64(double %sqrt) nounwind readnone

diff --git a/test/CodeGen/R600/llvm.AMDGPU.rcp.ll b/test/CodeGen/R600/llvm.AMDGPU.rcp.ll
index 42910a99afde8a13f61e706e17328863c788c976..8d5d66e149ba450d184391aff56e21690a5d57da 100644 (file)
--- a/test/CodeGen/R600/llvm.AMDGPU.rcp.ll
+++ b/test/CodeGen/R600/llvm.AMDGPU.rcp.ll
@@ -1,5 +1,7 @@
-; RUN: llc -march=r600 -mcpu=SI -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
+
+; XUN: llc -march=r600 -mcpu=SI -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE-SPDENORM -check-prefix=SI -check-prefix=FUNC %s
 
 declare float @llvm.AMDGPU.rcp.f32(float) nounwind readnone
 declare double @llvm.AMDGPU.rcp.f64(double) nounwind readnone
@@ -25,15 +27,8 @@ define void @rcp_f64(double addrspace(1)* %out, double %src) nounwind {
 }
 
 ; FUNC-LABEL: @rcp_pat_f32
-; SI-UNSAFE-NOT: V_MUL_F32
-; SI-UNSAFE: V_RCP_F32_e32
-; SI-UNSAFE-NOT: V_MUL_F32
-
-; Check for surrounding multiplies the correct divide has.
-; SI-SAFE: V_MUL_F32
 ; SI-SAFE: V_RCP_F32_e32
-; SI-SAFE: V_MUL_F32
-
+; XSI-SAFE-SPDENORM-NOT: V_RCP_F32_e32
 define void @rcp_pat_f32(float addrspace(1)* %out, float %src) nounwind {
   %rcp = fdiv float 1.0, %src
   store float %rcp, float addrspace(1)* %out, align 4
@@ -60,7 +55,8 @@ define void @rsq_rcp_pat_f32(float addrspace(1)* %out, float %src) nounwind {
 }
 
 ; FUNC-LABEL: @rsq_rcp_pat_f64
-; SI: V_RSQ_F64_e32
+; SI-UNSAFE: V_RSQ_F64_e32
+; SI-SAFE-NOT: V_RSQ_F64_e32
 define void @rsq_rcp_pat_f64(double addrspace(1)* %out, double %src) nounwind {
   %sqrt = call double @llvm.sqrt.f64(double %src) nounwind readnone
   %rcp = call double @llvm.AMDGPU.rcp.f64(double %sqrt) nounwind readnone

diff --git a/test/CodeGen/R600/rsq.ll b/test/CodeGen/R600/rsq.ll
index 67177e9f68ba85c600cab35b06b948799c86afc0..3069f62724b7a670a61b7759b2ad43bfb1929f32 100644 (file)
--- a/test/CodeGen/R600/rsq.ll
+++ b/test/CodeGen/R600/rsq.ll
@@ -1,12 +1,11 @@
-; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s
-; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s
+; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s
+; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s
 
 declare float @llvm.sqrt.f32(float) nounwind readnone
 declare double @llvm.sqrt.f64(double) nounwind readnone
 
 ; SI-LABEL: @rsq_f32
-; SI-UNSAFE: V_RSQ_F32_e32
-; SI-SAFE: V_SQRT_F32
+; SI: V_RSQ_F32_e32
 ; SI: S_ENDPGM
 define void @rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
   %val = load float addrspace(1)* %in, align 4
@@ -17,7 +16,8 @@ define void @rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noali
 }
 
 ; SI-LABEL: @rsq_f64
-; SI: V_RSQ_F64_e32
+; SI-UNSAFE: V_RSQ_F64_e32
+; SI-SAFE: V_SQRT_F64_e32
 ; SI: S_ENDPGM
 define void @rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) nounwind {
   %val = load double addrspace(1)* %in, align 4