1 ; Make sure we still form mad even when unsafe math or fp-contract is allowed instead of fma.
3 ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=FUNC %s
4 ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=FUNC %s
5 ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=FUNC %s
7 ; Make sure we don't form mad with denormals
8 ; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=FUNC %s
9 ; RUN: llc -march=amdgcn -mcpu=verde -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
11 declare i32 @llvm.r600.read.tidig.x() #0
12 declare float @llvm.fabs.f32(float) #0
13 declare float @llvm.fma.f32(float, float, float) #0
14 declare float @llvm.fmuladd.f32(float, float, float) #0
16 ; (fadd (fmul x, y), z) -> (fma x, y, z)
17 ; FUNC-LABEL: {{^}}combine_to_mad_f32_0:
18 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
19 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
20 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
22 ; SI-STD: v_mac_f32_e32 [[C]], [[B]], [[A]]
24 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
26 ; SI-DENORM-SLOWFMAF-NOT: v_fma
27 ; SI-DENORM-SLOWFMAF-NOT: v_mad
29 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
30 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP]]
32 ; SI-DENORM: buffer_store_dword [[RESULT]]
33 ; SI-STD: buffer_store_dword [[C]]
34 define void @combine_to_mad_f32_0(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
35 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
36 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
37 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
38 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
39 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
41 %a = load float, float addrspace(1)* %gep.0
42 %b = load float, float addrspace(1)* %gep.1
43 %c = load float, float addrspace(1)* %gep.2
45 %mul = fmul float %a, %b
46 %fma = fadd float %mul, %c
47 store float %fma, float addrspace(1)* %gep.out
51 ; (fadd (fmul x, y), z) -> (fma x, y, z)
52 ; FUNC-LABEL: {{^}}combine_to_mad_f32_0_2use:
53 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
54 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
55 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
56 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
58 ; SI-STD-DAG: v_mac_f32_e32 [[C]], [[B]], [[A]]
59 ; SI-STD-DAG: v_mac_f32_e32 [[D]], [[B]], [[A]]
61 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], [[C]]
62 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], [[D]]
64 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
65 ; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT0:v[0-9]+]], [[C]], [[TMP]]
66 ; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT1:v[0-9]+]], [[D]], [[TMP]]
68 ; SI-DENORM-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
69 ; SI-DENORM-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
70 ; SI-STD-DAG: buffer_store_dword [[C]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
71 ; SI-STD-DAG: buffer_store_dword [[D]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
73 define void @combine_to_mad_f32_0_2use(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
74 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
75 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
76 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
77 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
78 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
79 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
80 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
82 %a = load float, float addrspace(1)* %gep.0
83 %b = load float, float addrspace(1)* %gep.1
84 %c = load float, float addrspace(1)* %gep.2
85 %d = load float, float addrspace(1)* %gep.3
87 %mul = fmul float %a, %b
88 %fma0 = fadd float %mul, %c
89 %fma1 = fadd float %mul, %d
91 store float %fma0, float addrspace(1)* %gep.out.0
92 store float %fma1, float addrspace(1)* %gep.out.1
96 ; (fadd x, (fmul y, z)) -> (fma y, z, x)
97 ; FUNC-LABEL: {{^}}combine_to_mad_f32_1:
98 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
99 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
100 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
102 ; SI-STD: v_mac_f32_e32 [[C]], [[B]], [[A]]
103 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
105 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
106 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[C]]
108 ; SI-DENORM: buffer_store_dword [[RESULT]]
109 ; SI-STD: buffer_store_dword [[C]]
110 define void @combine_to_mad_f32_1(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
111 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
112 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
113 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
114 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
115 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
117 %a = load float, float addrspace(1)* %gep.0
118 %b = load float, float addrspace(1)* %gep.1
119 %c = load float, float addrspace(1)* %gep.2
121 %mul = fmul float %a, %b
122 %fma = fadd float %c, %mul
123 store float %fma, float addrspace(1)* %gep.out
127 ; (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
128 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_0_f32:
129 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
130 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
131 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
133 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], -[[C]]
134 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], -[[C]]
136 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
137 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP]]
139 ; SI: buffer_store_dword [[RESULT]]
140 define void @combine_to_mad_fsub_0_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
141 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
142 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
143 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
144 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
145 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
147 %a = load float, float addrspace(1)* %gep.0
148 %b = load float, float addrspace(1)* %gep.1
149 %c = load float, float addrspace(1)* %gep.2
151 %mul = fmul float %a, %b
152 %fma = fsub float %mul, %c
153 store float %fma, float addrspace(1)* %gep.out
157 ; (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
158 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_0_f32_2use:
159 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
160 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
161 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
162 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
164 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], -[[C]]
165 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
167 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[A]], [[B]], -[[C]]
168 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
170 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
171 ; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT0:v[0-9]+]], [[C]], [[TMP]]
172 ; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[D]], [[TMP]]
174 ; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
175 ; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
177 define void @combine_to_mad_fsub_0_f32_2use(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
178 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
179 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
180 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
181 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
182 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
183 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
184 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
186 %a = load float, float addrspace(1)* %gep.0
187 %b = load float, float addrspace(1)* %gep.1
188 %c = load float, float addrspace(1)* %gep.2
189 %d = load float, float addrspace(1)* %gep.3
191 %mul = fmul float %a, %b
192 %fma0 = fsub float %mul, %c
193 %fma1 = fsub float %mul, %d
194 store float %fma0, float addrspace(1)* %gep.out.0
195 store float %fma1, float addrspace(1)* %gep.out.1
199 ; (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
200 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_1_f32:
201 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
202 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
203 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
205 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], [[C]]
206 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], [[C]]
208 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
209 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[C]]
211 ; SI: buffer_store_dword [[RESULT]]
212 define void @combine_to_mad_fsub_1_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
213 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
214 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
215 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
216 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
217 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
219 %a = load float, float addrspace(1)* %gep.0
220 %b = load float, float addrspace(1)* %gep.1
221 %c = load float, float addrspace(1)* %gep.2
223 %mul = fmul float %a, %b
224 %fma = fsub float %c, %mul
225 store float %fma, float addrspace(1)* %gep.out
229 ; (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
230 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_1_f32_2use:
231 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
232 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
233 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
235 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], [[C]]
236 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], [[D]]
238 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], [[C]]
239 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], [[D]]
241 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
242 ; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT0:v[0-9]+]], [[TMP]], [[C]]
243 ; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[TMP]], [[D]]
245 ; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
246 ; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
248 define void @combine_to_mad_fsub_1_f32_2use(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
249 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
250 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
251 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
252 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
253 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
254 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
255 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
257 %a = load float, float addrspace(1)* %gep.0
258 %b = load float, float addrspace(1)* %gep.1
259 %c = load float, float addrspace(1)* %gep.2
260 %d = load float, float addrspace(1)* %gep.3
262 %mul = fmul float %a, %b
263 %fma0 = fsub float %c, %mul
264 %fma1 = fsub float %d, %mul
265 store float %fma0, float addrspace(1)* %gep.out.0
266 store float %fma1, float addrspace(1)* %gep.out.1
270 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
271 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32:
272 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
273 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
274 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
276 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], -[[C]]
278 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[A]], [[B]], -[[C]]
280 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
281 ; SI-DENORM-SLOWFMAF: v_sub_f32_e64 [[RESULT:v[0-9]+]], -[[TMP]], [[C]]
283 ; SI: buffer_store_dword [[RESULT]]
284 define void @combine_to_mad_fsub_2_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
285 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
286 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
287 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
288 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
289 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
291 %a = load float, float addrspace(1)* %gep.0
292 %b = load float, float addrspace(1)* %gep.1
293 %c = load float, float addrspace(1)* %gep.2
295 %mul = fmul float %a, %b
296 %mul.neg = fsub float -0.0, %mul
297 %fma = fsub float %mul.neg, %c
299 store float %fma, float addrspace(1)* %gep.out
303 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
304 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32_2uses_neg:
305 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
306 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
307 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
309 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
310 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], -[[D]]
312 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
313 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], -[[D]]
315 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
316 ; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e64 [[RESULT0:v[0-9]+]], -[[TMP]], [[C]]
317 ; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e64 [[RESULT1:v[0-9]+]], -[[TMP]], [[D]]
319 ; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
320 ; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
322 define void @combine_to_mad_fsub_2_f32_2uses_neg(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
323 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
324 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
325 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
326 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
327 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
328 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
329 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
331 %a = load float, float addrspace(1)* %gep.0
332 %b = load float, float addrspace(1)* %gep.1
333 %c = load float, float addrspace(1)* %gep.2
334 %d = load float, float addrspace(1)* %gep.3
336 %mul = fmul float %a, %b
337 %mul.neg = fsub float -0.0, %mul
338 %fma0 = fsub float %mul.neg, %c
339 %fma1 = fsub float %mul.neg, %d
341 store float %fma0, float addrspace(1)* %gep.out.0
342 store float %fma1, float addrspace(1)* %gep.out.1
346 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
347 ; FUNC-LABEL: {{^}}combine_to_mad_fsub_2_f32_2uses_mul:
348 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
349 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
350 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
352 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
353 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
355 ; SI-DENORM-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
356 ; SI-DENORM-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
358 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[B]], [[A]]
359 ; SI-DENORM-SLOWFMAF-DAG: v_sub_f32_e64 [[RESULT0:v[0-9]+]], -[[TMP]], [[C]]
360 ; SI-DENORM-SLOWFMAF-DAG: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[D]], [[TMP]]
362 ; SI-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
363 ; SI-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
365 define void @combine_to_mad_fsub_2_f32_2uses_mul(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
366 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
367 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
368 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
369 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
370 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
371 %gep.out.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
372 %gep.out.1 = getelementptr float, float addrspace(1)* %gep.out.0, i32 1
374 %a = load float, float addrspace(1)* %gep.0
375 %b = load float, float addrspace(1)* %gep.1
376 %c = load float, float addrspace(1)* %gep.2
377 %d = load float, float addrspace(1)* %gep.3
379 %mul = fmul float %a, %b
380 %mul.neg = fsub float -0.0, %mul
381 %fma0 = fsub float %mul.neg, %c
382 %fma1 = fsub float %mul, %d
384 store float %fma0, float addrspace(1)* %gep.out.0
385 store float %fma1, float addrspace(1)* %gep.out.1
389 ; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
391 ; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_0_f32:
392 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
393 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
394 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
395 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
396 ; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
398 ; SI-STD: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
399 ; SI-STD: v_fma_f32 [[TMP1:v[0-9]+]], [[A]], [[B]], [[TMP0]]
400 ; SI-STD: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP1]]
402 ; SI-DENORM: v_fma_f32 [[TMP0:v[0-9]+]], [[D]], [[E]], -[[C]]
403 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[TMP0]]
405 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
406 ; SI-DENORM-SLOWFMAF: v_fma_f32 [[TMP1:v[0-9]+]], [[A]], [[B]], [[TMP0]]
407 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT1:v[0-9]+]], [[C]], [[TMP1]]
409 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
410 define void @aggressive_combine_to_mad_fsub_0_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
411 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
412 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
413 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
414 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
415 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
416 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
417 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
419 %x = load float, float addrspace(1)* %gep.0
420 %y = load float, float addrspace(1)* %gep.1
421 %z = load float, float addrspace(1)* %gep.2
422 %u = load float, float addrspace(1)* %gep.3
423 %v = load float, float addrspace(1)* %gep.4
425 %tmp0 = fmul float %u, %v
426 %tmp1 = call float @llvm.fma.f32(float %x, float %y, float %tmp0) #0
427 %tmp2 = fsub float %tmp1, %z
429 store float %tmp2, float addrspace(1)* %gep.out
433 ; fold (fsub x, (fma y, z, (fmul u, v)))
434 ; -> (fma (fneg y), z, (fma (fneg u), v, x))
436 ; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_1_f32:
437 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
438 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
439 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
440 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
441 ; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
443 ; SI-STD: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
444 ; SI-STD: v_fma_f32 [[TMP1:v[0-9]+]], [[B]], [[C]], [[TMP0]]
445 ; SI-STD: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[A]]
447 ; SI-DENORM: v_fma_f32 [[TMP0:v[0-9]+]], -[[D]], [[E]], [[A]]
448 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[B]], [[C]], [[TMP0]]
450 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
451 ; SI-DENORM-SLOWFMAF: v_fma_f32 [[TMP1:v[0-9]+]], [[B]], [[C]], [[TMP0]]
452 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[A]]
454 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
456 define void @aggressive_combine_to_mad_fsub_1_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
457 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
458 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
459 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
460 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
461 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
462 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
463 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
465 %x = load float, float addrspace(1)* %gep.0
466 %y = load float, float addrspace(1)* %gep.1
467 %z = load float, float addrspace(1)* %gep.2
468 %u = load float, float addrspace(1)* %gep.3
469 %v = load float, float addrspace(1)* %gep.4
471 %tmp0 = fmul float %u, %v
472 %tmp1 = call float @llvm.fma.f32(float %y, float %z, float %tmp0) #0
473 %tmp2 = fsub float %x, %tmp1
475 store float %tmp2, float addrspace(1)* %gep.out
479 ; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
481 ; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_2_f32:
482 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
483 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
484 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
485 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
486 ; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
488 ; SI-STD: v_mad_f32 [[TMP:v[0-9]+]], [[D]], [[E]], -[[C]]
489 ; SI-STD: v_mac_f32_e32 [[TMP]], [[B]], [[A]]
491 ; SI-DENORM: v_fma_f32 [[TMP:v[0-9]+]], [[D]], [[E]], -[[C]]
492 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[TMP]]
494 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
495 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP1:v[0-9]+]], [[B]], [[A]]
496 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[TMP2:v[0-9]+]], [[TMP0]], [[TMP1]]
497 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[C]], [[TMP2]]
499 ; SI-DENORM: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
500 ; SI-STD: buffer_store_dword [[TMP]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
502 define void @aggressive_combine_to_mad_fsub_2_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
503 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
504 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
505 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
506 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
507 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
508 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
509 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
511 %x = load float, float addrspace(1)* %gep.0
512 %y = load float, float addrspace(1)* %gep.1
513 %z = load float, float addrspace(1)* %gep.2
514 %u = load float, float addrspace(1)* %gep.3
515 %v = load float, float addrspace(1)* %gep.4
517 %tmp0 = fmul float %u, %v
518 %tmp1 = call float @llvm.fmuladd.f32(float %x, float %y, float %tmp0) #0
519 %tmp2 = fsub float %tmp1, %z
521 store float %tmp2, float addrspace(1)* %gep.out
525 ; fold (fsub x, (fmuladd y, z, (fmul u, v)))
526 ; -> (fmuladd (fneg y), z, (fmuladd (fneg u), v, x))
528 ; FUNC-LABEL: {{^}}aggressive_combine_to_mad_fsub_3_f32:
529 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
530 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
531 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
532 ; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
533 ; SI-DAG: buffer_load_dword [[E:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
535 ; SI-STD: v_mad_f32 [[TMP:v[0-9]+]], -[[D]], [[E]], [[A]]
536 ; SI-STD: v_mad_f32 [[RESULT:v[0-9]+]], -[[B]], [[C]], [[TMP]]
538 ; SI-DENORM: v_fma_f32 [[TMP:v[0-9]+]], -[[D]], [[E]], [[A]]
539 ; SI-DENORM: v_fma_f32 [[RESULT:v[0-9]+]], -[[B]], [[C]], [[TMP]]
541 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[E]], [[D]]
542 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP1:v[0-9]+]], [[C]], [[B]]
543 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[TMP2:v[0-9]+]], [[TMP0]], [[TMP1]]
544 ; SI-DENORM-SLOWFMAF: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP2]], [[A]]
546 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
548 define void @aggressive_combine_to_mad_fsub_3_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
549 %tid = tail call i32 @llvm.r600.read.tidig.x() #0
550 %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
551 %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
552 %gep.2 = getelementptr float, float addrspace(1)* %gep.0, i32 2
553 %gep.3 = getelementptr float, float addrspace(1)* %gep.0, i32 3
554 %gep.4 = getelementptr float, float addrspace(1)* %gep.0, i32 4
555 %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
557 %x = load float, float addrspace(1)* %gep.0
558 %y = load float, float addrspace(1)* %gep.1
559 %z = load float, float addrspace(1)* %gep.2
560 %u = load float, float addrspace(1)* %gep.3
561 %v = load float, float addrspace(1)* %gep.4
563 %tmp0 = fmul float %u, %v
564 %tmp1 = call float @llvm.fmuladd.f32(float %y, float %z, float %tmp0) #0
565 %tmp2 = fsub float %x, %tmp1
567 store float %tmp2, float addrspace(1)* %gep.out
571 attributes #0 = { nounwind readnone }
572 attributes #1 = { nounwind }