1 //===-- AMDILISelLowering.cpp - AMDIL DAG Lowering Implementation ---------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //==-----------------------------------------------------------------------===//
11 /// \brief TargetLowering functions borrowed from AMDIL.
13 //===----------------------------------------------------------------------===//
15 #include "AMDGPUISelLowering.h"
16 #include "AMDGPURegisterInfo.h"
17 #include "AMDGPUSubtarget.h"
18 #include "AMDILIntrinsicInfo.h"
19 #include "llvm/CodeGen/MachineFrameInfo.h"
20 #include "llvm/CodeGen/MachineRegisterInfo.h"
21 #include "llvm/CodeGen/PseudoSourceValue.h"
22 #include "llvm/CodeGen/SelectionDAG.h"
23 #include "llvm/CodeGen/SelectionDAGNodes.h"
24 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
25 #include "llvm/IR/CallingConv.h"
26 #include "llvm/IR/DerivedTypes.h"
27 #include "llvm/IR/Instructions.h"
28 #include "llvm/IR/Intrinsics.h"
29 #include "llvm/Support/raw_ostream.h"
30 #include "llvm/Target/TargetInstrInfo.h"
31 #include "llvm/Target/TargetOptions.h"
34 //===----------------------------------------------------------------------===//
35 // TargetLowering Implementation Help Functions End
36 //===----------------------------------------------------------------------===//
38 //===----------------------------------------------------------------------===//
39 // TargetLowering Class Implementation Begins
40 //===----------------------------------------------------------------------===//
41 void AMDGPUTargetLowering::InitAMDILLowering() {
42 static const int types[] = {
61 static const int IntTypes[] = {
68 static const int FloatTypes[] = {
73 static const int VectorTypes[] = {
85 const size_t NumTypes = array_lengthof(types);
86 const size_t NumFloatTypes = array_lengthof(FloatTypes);
87 const size_t NumIntTypes = array_lengthof(IntTypes);
88 const size_t NumVectorTypes = array_lengthof(VectorTypes);
90 const AMDGPUSubtarget &STM = getTargetMachine().getSubtarget<AMDGPUSubtarget>();
91 // These are the current register classes that are
94 for (unsigned int x = 0; x < NumTypes; ++x) {
95 MVT::SimpleValueType VT = (MVT::SimpleValueType)types[x];
97 setOperationAction(ISD::SUBE, VT, Expand);
98 setOperationAction(ISD::SUBC, VT, Expand);
99 setOperationAction(ISD::ADDE, VT, Expand);
100 setOperationAction(ISD::ADDC, VT, Expand);
101 setOperationAction(ISD::BRCOND, VT, Custom);
102 setOperationAction(ISD::BR_JT, VT, Expand);
103 setOperationAction(ISD::BRIND, VT, Expand);
104 // TODO: Implement custom UREM/SREM routines
105 setOperationAction(ISD::SREM, VT, Expand);
106 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
107 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
108 if (VT != MVT::i64 && VT != MVT::v2i64) {
109 setOperationAction(ISD::SDIV, VT, Custom);
112 for (unsigned int x = 0; x < NumFloatTypes; ++x) {
113 MVT::SimpleValueType VT = (MVT::SimpleValueType)FloatTypes[x];
115 // IL does not have these operations for floating point types
116 setOperationAction(ISD::FP_ROUND_INREG, VT, Expand);
117 setOperationAction(ISD::SETOLT, VT, Expand);
118 setOperationAction(ISD::SETOGE, VT, Expand);
119 setOperationAction(ISD::SETOGT, VT, Expand);
120 setOperationAction(ISD::SETOLE, VT, Expand);
121 setOperationAction(ISD::SETULT, VT, Expand);
122 setOperationAction(ISD::SETUGE, VT, Expand);
123 setOperationAction(ISD::SETUGT, VT, Expand);
124 setOperationAction(ISD::SETULE, VT, Expand);
127 for (unsigned int x = 0; x < NumIntTypes; ++x) {
128 MVT::SimpleValueType VT = (MVT::SimpleValueType)IntTypes[x];
130 // GPU also does not have divrem function for signed or unsigned
131 setOperationAction(ISD::SDIVREM, VT, Expand);
133 // GPU does not have [S|U]MUL_LOHI functions as a single instruction
134 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
135 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
137 setOperationAction(ISD::BSWAP, VT, Expand);
139 // GPU doesn't have any counting operators
140 setOperationAction(ISD::CTPOP, VT, Expand);
141 setOperationAction(ISD::CTTZ, VT, Expand);
142 setOperationAction(ISD::CTLZ, VT, Expand);
145 for (unsigned int ii = 0; ii < NumVectorTypes; ++ii) {
146 MVT::SimpleValueType VT = (MVT::SimpleValueType)VectorTypes[ii];
148 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
149 setOperationAction(ISD::SDIVREM, VT, Expand);
150 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
151 // setOperationAction(ISD::VSETCC, VT, Expand);
152 setOperationAction(ISD::SELECT_CC, VT, Expand);
155 setOperationAction(ISD::MULHU, MVT::i64, Expand);
156 setOperationAction(ISD::MULHU, MVT::v2i64, Expand);
157 setOperationAction(ISD::MULHS, MVT::i64, Expand);
158 setOperationAction(ISD::MULHS, MVT::v2i64, Expand);
159 setOperationAction(ISD::ADD, MVT::v2i64, Expand);
160 setOperationAction(ISD::SREM, MVT::v2i64, Expand);
161 setOperationAction(ISD::Constant , MVT::i64 , Legal);
162 setOperationAction(ISD::SDIV, MVT::v2i64, Expand);
163 setOperationAction(ISD::TRUNCATE, MVT::v2i64, Expand);
164 setOperationAction(ISD::SIGN_EXTEND, MVT::v2i64, Expand);
165 setOperationAction(ISD::ZERO_EXTEND, MVT::v2i64, Expand);
166 setOperationAction(ISD::ANY_EXTEND, MVT::v2i64, Expand);
167 if (STM.hasHWFP64()) {
168 // we support loading/storing v2f64 but not operations on the type
169 setOperationAction(ISD::FADD, MVT::v2f64, Expand);
170 setOperationAction(ISD::FSUB, MVT::v2f64, Expand);
171 setOperationAction(ISD::FMUL, MVT::v2f64, Expand);
172 setOperationAction(ISD::FP_ROUND_INREG, MVT::v2f64, Expand);
173 setOperationAction(ISD::FP_EXTEND, MVT::v2f64, Expand);
174 setOperationAction(ISD::ConstantFP , MVT::f64 , Legal);
175 // We want to expand vector conversions into their scalar
177 setOperationAction(ISD::TRUNCATE, MVT::v2f64, Expand);
178 setOperationAction(ISD::SIGN_EXTEND, MVT::v2f64, Expand);
179 setOperationAction(ISD::ZERO_EXTEND, MVT::v2f64, Expand);
180 setOperationAction(ISD::ANY_EXTEND, MVT::v2f64, Expand);
181 setOperationAction(ISD::FABS, MVT::f64, Expand);
182 setOperationAction(ISD::FABS, MVT::v2f64, Expand);
184 // TODO: Fix the UDIV24 algorithm so it works for these
185 // types correctly. This needs vector comparisons
186 // for this to work correctly.
187 setOperationAction(ISD::UDIV, MVT::v2i8, Expand);
188 setOperationAction(ISD::UDIV, MVT::v4i8, Expand);
189 setOperationAction(ISD::UDIV, MVT::v2i16, Expand);
190 setOperationAction(ISD::UDIV, MVT::v4i16, Expand);
191 setOperationAction(ISD::SUBC, MVT::Other, Expand);
192 setOperationAction(ISD::ADDE, MVT::Other, Expand);
193 setOperationAction(ISD::ADDC, MVT::Other, Expand);
194 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
195 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
196 setOperationAction(ISD::BRIND, MVT::Other, Expand);
199 // Use the default implementation.
200 setOperationAction(ISD::ConstantFP , MVT::f32 , Legal);
201 setOperationAction(ISD::Constant , MVT::i32 , Legal);
203 setSchedulingPreference(Sched::RegPressure);
204 setPow2DivIsCheap(false);
205 setSelectIsExpensive(true);
206 setJumpIsExpensive(true);
208 MaxStoresPerMemcpy = 4096;
209 MaxStoresPerMemmove = 4096;
210 MaxStoresPerMemset = 4096;
215 AMDGPUTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
216 const CallInst &I, unsigned Intrinsic) const {
220 // The backend supports 32 and 64 bit floating point immediates
222 AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
223 if (VT.getScalarType().getSimpleVT().SimpleTy == MVT::f32
224 || VT.getScalarType().getSimpleVT().SimpleTy == MVT::f64) {
232 AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
233 if (VT.getScalarType().getSimpleVT().SimpleTy == MVT::f32
234 || VT.getScalarType().getSimpleVT().SimpleTy == MVT::f64) {
242 // isMaskedValueZeroForTargetNode - Return true if 'Op & Mask' is known to
243 // be zero. Op is expected to be a target specific node. Used by DAG
246 //===----------------------------------------------------------------------===//
247 // Other Lowering Hooks
248 //===----------------------------------------------------------------------===//
251 AMDGPUTargetLowering::LowerSDIV(SDValue Op, SelectionDAG &DAG) const {
252 EVT OVT = Op.getValueType();
254 if (OVT.getScalarType() == MVT::i64) {
255 DST = LowerSDIV64(Op, DAG);
256 } else if (OVT.getScalarType() == MVT::i32) {
257 DST = LowerSDIV32(Op, DAG);
258 } else if (OVT.getScalarType() == MVT::i16
259 || OVT.getScalarType() == MVT::i8) {
260 DST = LowerSDIV24(Op, DAG);
262 DST = SDValue(Op.getNode(), 0);
268 AMDGPUTargetLowering::LowerSREM(SDValue Op, SelectionDAG &DAG) const {
269 EVT OVT = Op.getValueType();
271 if (OVT.getScalarType() == MVT::i64) {
272 DST = LowerSREM64(Op, DAG);
273 } else if (OVT.getScalarType() == MVT::i32) {
274 DST = LowerSREM32(Op, DAG);
275 } else if (OVT.getScalarType() == MVT::i16) {
276 DST = LowerSREM16(Op, DAG);
277 } else if (OVT.getScalarType() == MVT::i8) {
278 DST = LowerSREM8(Op, DAG);
280 DST = SDValue(Op.getNode(), 0);
286 AMDGPUTargetLowering::genIntType(uint32_t size, uint32_t numEle) const {
287 int iSize = (size * numEle);
288 int vEle = (iSize >> ((size == 64) ? 6 : 5));
294 return EVT(MVT::i64);
296 return EVT(MVT::getVectorVT(MVT::i64, vEle));
300 return EVT(MVT::i32);
302 return EVT(MVT::getVectorVT(MVT::i32, vEle));
308 AMDGPUTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
309 SDValue Chain = Op.getOperand(0);
310 SDValue Cond = Op.getOperand(1);
311 SDValue Jump = Op.getOperand(2);
313 Result = DAG.getNode(
314 AMDGPUISD::BRANCH_COND,
322 AMDGPUTargetLowering::LowerSDIV24(SDValue Op, SelectionDAG &DAG) const {
324 EVT OVT = Op.getValueType();
325 SDValue LHS = Op.getOperand(0);
326 SDValue RHS = Op.getOperand(1);
329 if (!OVT.isVector()) {
332 } else if (OVT.getVectorNumElements() == 2) {
335 } else if (OVT.getVectorNumElements() == 4) {
339 unsigned bitsize = OVT.getScalarType().getSizeInBits();
340 // char|short jq = ia ^ ib;
341 SDValue jq = DAG.getNode(ISD::XOR, DL, OVT, LHS, RHS);
343 // jq = jq >> (bitsize - 2)
344 jq = DAG.getNode(ISD::SRA, DL, OVT, jq, DAG.getConstant(bitsize - 2, OVT));
347 jq = DAG.getNode(ISD::OR, DL, OVT, jq, DAG.getConstant(1, OVT));
350 jq = DAG.getSExtOrTrunc(jq, DL, INTTY);
352 // int ia = (int)LHS;
353 SDValue ia = DAG.getSExtOrTrunc(LHS, DL, INTTY);
356 SDValue ib = DAG.getSExtOrTrunc(RHS, DL, INTTY);
358 // float fa = (float)ia;
359 SDValue fa = DAG.getNode(ISD::SINT_TO_FP, DL, FLTTY, ia);
361 // float fb = (float)ib;
362 SDValue fb = DAG.getNode(ISD::SINT_TO_FP, DL, FLTTY, ib);
364 // float fq = native_divide(fa, fb);
365 SDValue fq = DAG.getNode(AMDGPUISD::DIV_INF, DL, FLTTY, fa, fb);
368 fq = DAG.getNode(ISD::FTRUNC, DL, FLTTY, fq);
370 // float fqneg = -fq;
371 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FLTTY, fq);
373 // float fr = mad(fqneg, fb, fa);
374 SDValue fr = DAG.getNode(ISD::FADD, DL, FLTTY,
375 DAG.getNode(ISD::MUL, DL, FLTTY, fqneg, fb), fa);
378 SDValue iq = DAG.getNode(ISD::FP_TO_SINT, DL, INTTY, fq);
381 fr = DAG.getNode(ISD::FABS, DL, FLTTY, fr);
384 fb = DAG.getNode(ISD::FABS, DL, FLTTY, fb);
386 // int cv = fr >= fb;
388 if (INTTY == MVT::i32) {
389 cv = DAG.getSetCC(DL, INTTY, fr, fb, ISD::SETOGE);
391 cv = DAG.getSetCC(DL, INTTY, fr, fb, ISD::SETOGE);
393 // jq = (cv ? jq : 0);
394 jq = DAG.getNode(ISD::SELECT, DL, OVT, cv, jq,
395 DAG.getConstant(0, OVT));
397 iq = DAG.getSExtOrTrunc(iq, DL, OVT);
398 iq = DAG.getNode(ISD::ADD, DL, OVT, iq, jq);
403 AMDGPUTargetLowering::LowerSDIV32(SDValue Op, SelectionDAG &DAG) const {
405 EVT OVT = Op.getValueType();
406 SDValue LHS = Op.getOperand(0);
407 SDValue RHS = Op.getOperand(1);
408 // The LowerSDIV32 function generates equivalent to the following IL.
418 // ixor r10, r10, r11
429 SDValue r10 = DAG.getSelectCC(DL,
430 r0, DAG.getConstant(0, OVT),
431 DAG.getConstant(-1, MVT::i32),
432 DAG.getConstant(0, MVT::i32),
436 SDValue r11 = DAG.getSelectCC(DL,
437 r1, DAG.getConstant(0, OVT),
438 DAG.getConstant(-1, MVT::i32),
439 DAG.getConstant(0, MVT::i32),
443 r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10);
446 r1 = DAG.getNode(ISD::ADD, DL, OVT, r1, r11);
449 r0 = DAG.getNode(ISD::XOR, DL, OVT, r0, r10);
452 r1 = DAG.getNode(ISD::XOR, DL, OVT, r1, r11);
455 r0 = DAG.getNode(ISD::UDIV, DL, OVT, r0, r1);
457 // ixor r10, r10, r11
458 r10 = DAG.getNode(ISD::XOR, DL, OVT, r10, r11);
461 r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10);
464 SDValue DST = DAG.getNode(ISD::XOR, DL, OVT, r0, r10);
469 AMDGPUTargetLowering::LowerSDIV64(SDValue Op, SelectionDAG &DAG) const {
470 return SDValue(Op.getNode(), 0);
474 AMDGPUTargetLowering::LowerSREM8(SDValue Op, SelectionDAG &DAG) const {
476 EVT OVT = Op.getValueType();
477 MVT INTTY = MVT::i32;
478 if (OVT == MVT::v2i8) {
480 } else if (OVT == MVT::v4i8) {
483 SDValue LHS = DAG.getSExtOrTrunc(Op.getOperand(0), DL, INTTY);
484 SDValue RHS = DAG.getSExtOrTrunc(Op.getOperand(1), DL, INTTY);
485 LHS = DAG.getNode(ISD::SREM, DL, INTTY, LHS, RHS);
486 LHS = DAG.getSExtOrTrunc(LHS, DL, OVT);
491 AMDGPUTargetLowering::LowerSREM16(SDValue Op, SelectionDAG &DAG) const {
493 EVT OVT = Op.getValueType();
494 MVT INTTY = MVT::i32;
495 if (OVT == MVT::v2i16) {
497 } else if (OVT == MVT::v4i16) {
500 SDValue LHS = DAG.getSExtOrTrunc(Op.getOperand(0), DL, INTTY);
501 SDValue RHS = DAG.getSExtOrTrunc(Op.getOperand(1), DL, INTTY);
502 LHS = DAG.getNode(ISD::SREM, DL, INTTY, LHS, RHS);
503 LHS = DAG.getSExtOrTrunc(LHS, DL, OVT);
508 AMDGPUTargetLowering::LowerSREM32(SDValue Op, SelectionDAG &DAG) const {
510 EVT OVT = Op.getValueType();
511 SDValue LHS = Op.getOperand(0);
512 SDValue RHS = Op.getOperand(1);
513 // The LowerSREM32 function generates equivalent to the following IL.
535 SDValue r10 = DAG.getSetCC(DL, OVT, r0, DAG.getConstant(0, OVT), ISD::SETLT);
538 SDValue r11 = DAG.getSetCC(DL, OVT, r1, DAG.getConstant(0, OVT), ISD::SETLT);
541 r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10);
544 r1 = DAG.getNode(ISD::ADD, DL, OVT, r1, r11);
547 r0 = DAG.getNode(ISD::XOR, DL, OVT, r0, r10);
550 r1 = DAG.getNode(ISD::XOR, DL, OVT, r1, r11);
553 SDValue r20 = DAG.getNode(ISD::UREM, DL, OVT, r0, r1);
556 r20 = DAG.getNode(AMDGPUISD::UMUL, DL, OVT, r20, r1);
559 r0 = DAG.getNode(ISD::SUB, DL, OVT, r0, r20);
562 r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10);
565 SDValue DST = DAG.getNode(ISD::XOR, DL, OVT, r0, r10);
570 AMDGPUTargetLowering::LowerSREM64(SDValue Op, SelectionDAG &DAG) const {
571 return SDValue(Op.getNode(), 0);