1 //===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// \brief This is the parent TargetLowering class for hardware code gen
14 //===----------------------------------------------------------------------===//
16 #include "AMDGPUISelLowering.h"
18 #include "AMDGPUFrameLowering.h"
19 #include "AMDGPUIntrinsicInfo.h"
20 #include "AMDGPURegisterInfo.h"
21 #include "AMDGPUSubtarget.h"
22 #include "R600MachineFunctionInfo.h"
23 #include "SIMachineFunctionInfo.h"
24 #include "llvm/CodeGen/CallingConvLower.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineRegisterInfo.h"
27 #include "llvm/CodeGen/SelectionDAG.h"
28 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
29 #include "llvm/IR/DataLayout.h"
30 #include "llvm/IR/DiagnosticInfo.h"
31 #include "llvm/IR/DiagnosticPrinter.h"
37 /// Diagnostic information for unimplemented or unsupported feature reporting.
38 class DiagnosticInfoUnsupported : public DiagnosticInfo {
40 const Twine &Description;
45 static int getKindID() {
47 KindID = llvm::getNextAvailablePluginDiagnosticKind();
52 DiagnosticInfoUnsupported(const Function &Fn, const Twine &Desc,
53 DiagnosticSeverity Severity = DS_Error)
54 : DiagnosticInfo(getKindID(), Severity),
58 const Function &getFunction() const { return Fn; }
59 const Twine &getDescription() const { return Description; }
61 void print(DiagnosticPrinter &DP) const override {
62 DP << "unsupported " << getDescription() << " in " << Fn.getName();
65 static bool classof(const DiagnosticInfo *DI) {
66 return DI->getKind() == getKindID();
70 int DiagnosticInfoUnsupported::KindID = 0;
74 static bool allocateStack(unsigned ValNo, MVT ValVT, MVT LocVT,
75 CCValAssign::LocInfo LocInfo,
76 ISD::ArgFlagsTy ArgFlags, CCState &State) {
77 unsigned Offset = State.AllocateStack(ValVT.getStoreSize(),
78 ArgFlags.getOrigAlign());
79 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
84 #include "AMDGPUGenCallingConv.inc"
86 // Find a larger type to do a load / store of a vector with.
87 EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
88 unsigned StoreSize = VT.getStoreSizeInBits();
90 return EVT::getIntegerVT(Ctx, StoreSize);
92 assert(StoreSize % 32 == 0 && "Store size not a multiple of 32");
93 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32);
96 // Type for a vector that will be loaded to.
97 EVT AMDGPUTargetLowering::getEquivalentLoadRegType(LLVMContext &Ctx, EVT VT) {
98 unsigned StoreSize = VT.getStoreSizeInBits();
100 return EVT::getIntegerVT(Ctx, 32);
102 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32);
105 AMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM) :
106 TargetLowering(TM, new TargetLoweringObjectFileELF()) {
108 Subtarget = &TM.getSubtarget<AMDGPUSubtarget>();
110 setOperationAction(ISD::Constant, MVT::i32, Legal);
111 setOperationAction(ISD::Constant, MVT::i64, Legal);
112 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
113 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
115 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
116 setOperationAction(ISD::BRIND, MVT::Other, Expand);
118 // We need to custom lower some of the intrinsics
119 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
121 // Library functions. These default to Expand, but we have instructions
123 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
124 setOperationAction(ISD::FEXP2, MVT::f32, Legal);
125 setOperationAction(ISD::FPOW, MVT::f32, Legal);
126 setOperationAction(ISD::FLOG2, MVT::f32, Legal);
127 setOperationAction(ISD::FABS, MVT::f32, Legal);
128 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
129 setOperationAction(ISD::FRINT, MVT::f32, Legal);
130 setOperationAction(ISD::FROUND, MVT::f32, Legal);
131 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
133 setOperationAction(ISD::FREM, MVT::f32, Custom);
134 setOperationAction(ISD::FREM, MVT::f64, Custom);
136 // Lower floating point store/load to integer store/load to reduce the number
137 // of patterns in tablegen.
138 setOperationAction(ISD::STORE, MVT::f32, Promote);
139 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32);
141 setOperationAction(ISD::STORE, MVT::v2f32, Promote);
142 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32);
144 setOperationAction(ISD::STORE, MVT::i64, Promote);
145 AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32);
147 setOperationAction(ISD::STORE, MVT::v4f32, Promote);
148 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32);
150 setOperationAction(ISD::STORE, MVT::v8f32, Promote);
151 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32);
153 setOperationAction(ISD::STORE, MVT::v16f32, Promote);
154 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32);
156 setOperationAction(ISD::STORE, MVT::f64, Promote);
157 AddPromotedToType(ISD::STORE, MVT::f64, MVT::i64);
159 setOperationAction(ISD::STORE, MVT::v2f64, Promote);
160 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v2i64);
162 // Custom lowering of vector stores is required for local address space
164 setOperationAction(ISD::STORE, MVT::v4i32, Custom);
165 // XXX: Native v2i32 local address space stores are possible, but not
166 // currently implemented.
167 setOperationAction(ISD::STORE, MVT::v2i32, Custom);
169 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom);
170 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom);
171 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Custom);
173 // XXX: This can be change to Custom, once ExpandVectorStores can
174 // handle 64-bit stores.
175 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand);
177 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
178 setTruncStoreAction(MVT::i64, MVT::i8, Expand);
179 setTruncStoreAction(MVT::i64, MVT::i1, Expand);
180 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand);
181 setTruncStoreAction(MVT::v4i64, MVT::v4i1, Expand);
184 setOperationAction(ISD::LOAD, MVT::f32, Promote);
185 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32);
187 setOperationAction(ISD::LOAD, MVT::v2f32, Promote);
188 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32);
190 setOperationAction(ISD::LOAD, MVT::i64, Promote);
191 AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32);
193 setOperationAction(ISD::LOAD, MVT::v4f32, Promote);
194 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32);
196 setOperationAction(ISD::LOAD, MVT::v8f32, Promote);
197 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32);
199 setOperationAction(ISD::LOAD, MVT::v16f32, Promote);
200 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32);
202 setOperationAction(ISD::LOAD, MVT::f64, Promote);
203 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::i64);
205 setOperationAction(ISD::LOAD, MVT::v2f64, Promote);
206 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v2i64);
208 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom);
209 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom);
210 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i32, Custom);
211 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f32, Custom);
212 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom);
213 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom);
214 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f32, Custom);
215 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i32, Custom);
216 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f32, Custom);
217 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8i32, Custom);
219 setLoadExtAction(ISD::EXTLOAD, MVT::v2i8, Expand);
220 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i8, Expand);
221 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i8, Expand);
222 setLoadExtAction(ISD::EXTLOAD, MVT::v4i8, Expand);
223 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i8, Expand);
224 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i8, Expand);
225 setLoadExtAction(ISD::EXTLOAD, MVT::v2i16, Expand);
226 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i16, Expand);
227 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i16, Expand);
228 setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, Expand);
229 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, Expand);
230 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, Expand);
232 setOperationAction(ISD::BR_CC, MVT::i1, Expand);
234 if (Subtarget->getGeneration() < AMDGPUSubtarget::SEA_ISLANDS) {
235 setOperationAction(ISD::FCEIL, MVT::f64, Custom);
236 setOperationAction(ISD::FTRUNC, MVT::f64, Custom);
237 setOperationAction(ISD::FRINT, MVT::f64, Custom);
238 setOperationAction(ISD::FFLOOR, MVT::f64, Custom);
241 if (!Subtarget->hasBFI()) {
242 // fcopysign can be done in a single instruction with BFI.
243 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
244 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
247 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
249 setLoadExtAction(ISD::EXTLOAD, MVT::f16, Expand);
250 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
251 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
253 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
254 for (MVT VT : ScalarIntVTs) {
255 setOperationAction(ISD::SREM, VT, Expand);
256 setOperationAction(ISD::SDIV, VT, Expand);
258 // GPU does not have divrem function for signed or unsigned.
259 setOperationAction(ISD::SDIVREM, VT, Custom);
260 setOperationAction(ISD::UDIVREM, VT, Custom);
262 // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
263 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
264 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
266 setOperationAction(ISD::BSWAP, VT, Expand);
267 setOperationAction(ISD::CTTZ, VT, Expand);
268 setOperationAction(ISD::CTLZ, VT, Expand);
271 if (!Subtarget->hasBCNT(32))
272 setOperationAction(ISD::CTPOP, MVT::i32, Expand);
274 if (!Subtarget->hasBCNT(64))
275 setOperationAction(ISD::CTPOP, MVT::i64, Expand);
277 // The hardware supports 32-bit ROTR, but not ROTL.
278 setOperationAction(ISD::ROTL, MVT::i32, Expand);
279 setOperationAction(ISD::ROTL, MVT::i64, Expand);
280 setOperationAction(ISD::ROTR, MVT::i64, Expand);
282 setOperationAction(ISD::MUL, MVT::i64, Expand);
283 setOperationAction(ISD::MULHU, MVT::i64, Expand);
284 setOperationAction(ISD::MULHS, MVT::i64, Expand);
285 setOperationAction(ISD::UDIV, MVT::i32, Expand);
286 setOperationAction(ISD::UREM, MVT::i32, Expand);
287 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
288 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand);
290 if (!Subtarget->hasFFBH())
291 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand);
293 if (!Subtarget->hasFFBL())
294 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand);
296 static const MVT::SimpleValueType VectorIntTypes[] = {
297 MVT::v2i32, MVT::v4i32
300 for (MVT VT : VectorIntTypes) {
301 // Expand the following operations for the current type by default.
302 setOperationAction(ISD::ADD, VT, Expand);
303 setOperationAction(ISD::AND, VT, Expand);
304 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
305 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
306 setOperationAction(ISD::MUL, VT, Expand);
307 setOperationAction(ISD::OR, VT, Expand);
308 setOperationAction(ISD::SHL, VT, Expand);
309 setOperationAction(ISD::SRA, VT, Expand);
310 setOperationAction(ISD::SRL, VT, Expand);
311 setOperationAction(ISD::ROTL, VT, Expand);
312 setOperationAction(ISD::ROTR, VT, Expand);
313 setOperationAction(ISD::SUB, VT, Expand);
314 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
315 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
316 setOperationAction(ISD::SDIV, VT, Expand);
317 setOperationAction(ISD::UDIV, VT, Expand);
318 setOperationAction(ISD::SREM, VT, Expand);
319 setOperationAction(ISD::UREM, VT, Expand);
320 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
321 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
322 setOperationAction(ISD::SDIVREM, VT, Custom);
323 setOperationAction(ISD::UDIVREM, VT, Custom);
324 setOperationAction(ISD::ADDC, VT, Expand);
325 setOperationAction(ISD::SUBC, VT, Expand);
326 setOperationAction(ISD::ADDE, VT, Expand);
327 setOperationAction(ISD::SUBE, VT, Expand);
328 setOperationAction(ISD::SELECT, VT, Expand);
329 setOperationAction(ISD::VSELECT, VT, Expand);
330 setOperationAction(ISD::SELECT_CC, VT, Expand);
331 setOperationAction(ISD::XOR, VT, Expand);
332 setOperationAction(ISD::BSWAP, VT, Expand);
333 setOperationAction(ISD::CTPOP, VT, Expand);
334 setOperationAction(ISD::CTTZ, VT, Expand);
335 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand);
336 setOperationAction(ISD::CTLZ, VT, Expand);
337 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand);
338 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
341 static const MVT::SimpleValueType FloatVectorTypes[] = {
342 MVT::v2f32, MVT::v4f32
345 for (MVT VT : FloatVectorTypes) {
346 setOperationAction(ISD::FABS, VT, Expand);
347 setOperationAction(ISD::FADD, VT, Expand);
348 setOperationAction(ISD::FCEIL, VT, Expand);
349 setOperationAction(ISD::FCOS, VT, Expand);
350 setOperationAction(ISD::FDIV, VT, Expand);
351 setOperationAction(ISD::FEXP2, VT, Expand);
352 setOperationAction(ISD::FLOG2, VT, Expand);
353 setOperationAction(ISD::FREM, VT, Expand);
354 setOperationAction(ISD::FPOW, VT, Expand);
355 setOperationAction(ISD::FFLOOR, VT, Expand);
356 setOperationAction(ISD::FTRUNC, VT, Expand);
357 setOperationAction(ISD::FMUL, VT, Expand);
358 setOperationAction(ISD::FMA, VT, Expand);
359 setOperationAction(ISD::FRINT, VT, Expand);
360 setOperationAction(ISD::FNEARBYINT, VT, Expand);
361 setOperationAction(ISD::FSQRT, VT, Expand);
362 setOperationAction(ISD::FSIN, VT, Expand);
363 setOperationAction(ISD::FSUB, VT, Expand);
364 setOperationAction(ISD::FNEG, VT, Expand);
365 setOperationAction(ISD::SELECT, VT, Expand);
366 setOperationAction(ISD::VSELECT, VT, Expand);
367 setOperationAction(ISD::SELECT_CC, VT, Expand);
368 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
369 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
372 setOperationAction(ISD::FNEARBYINT, MVT::f32, Custom);
373 setOperationAction(ISD::FNEARBYINT, MVT::f64, Custom);
375 setTargetDAGCombine(ISD::MUL);
376 setTargetDAGCombine(ISD::SELECT_CC);
377 setTargetDAGCombine(ISD::STORE);
379 setSchedulingPreference(Sched::RegPressure);
380 setJumpIsExpensive(true);
382 // SI at least has hardware support for floating point exceptions, but no way
383 // of using or handling them is implemented. They are also optional in OpenCL
385 setHasFloatingPointExceptions(false);
387 setSelectIsExpensive(false);
388 PredictableSelectIsExpensive = false;
390 // There are no integer divide instructions, and these expand to a pretty
391 // large sequence of instructions.
392 setIntDivIsCheap(false);
393 setPow2SDivIsCheap(false);
395 // TODO: Investigate this when 64-bit divides are implemented.
396 addBypassSlowDiv(64, 32);
398 // FIXME: Need to really handle these.
399 MaxStoresPerMemcpy = 4096;
400 MaxStoresPerMemmove = 4096;
401 MaxStoresPerMemset = 4096;
404 //===----------------------------------------------------------------------===//
405 // Target Information
406 //===----------------------------------------------------------------------===//
408 MVT AMDGPUTargetLowering::getVectorIdxTy() const {
412 bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
416 // The backend supports 32 and 64 bit floating point immediates.
417 // FIXME: Why are we reporting vectors of FP immediates as legal?
418 bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
419 EVT ScalarVT = VT.getScalarType();
420 return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64);
423 // We don't want to shrink f64 / f32 constants.
424 bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
425 EVT ScalarVT = VT.getScalarType();
426 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
429 bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy,
431 if (LoadTy.getSizeInBits() != CastTy.getSizeInBits())
434 unsigned LScalarSize = LoadTy.getScalarType().getSizeInBits();
435 unsigned CastScalarSize = CastTy.getScalarType().getSizeInBits();
437 return ((LScalarSize <= CastScalarSize) ||
438 (CastScalarSize >= 32) ||
442 //===---------------------------------------------------------------------===//
444 //===---------------------------------------------------------------------===//
446 bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
447 assert(VT.isFloatingPoint());
448 return VT == MVT::f32 || VT == MVT::f64;
451 bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
452 assert(VT.isFloatingPoint());
453 return VT == MVT::f32 || VT == MVT::f64;
456 bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
457 // Truncate is just accessing a subregister.
458 return Dest.bitsLT(Source) && (Dest.getSizeInBits() % 32 == 0);
461 bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
462 // Truncate is just accessing a subregister.
463 return Dest->getPrimitiveSizeInBits() < Source->getPrimitiveSizeInBits() &&
464 (Dest->getPrimitiveSizeInBits() % 32 == 0);
467 bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
468 const DataLayout *DL = getDataLayout();
469 unsigned SrcSize = DL->getTypeSizeInBits(Src->getScalarType());
470 unsigned DestSize = DL->getTypeSizeInBits(Dest->getScalarType());
472 return SrcSize == 32 && DestSize == 64;
475 bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
476 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
477 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
478 // this will enable reducing 64-bit operations the 32-bit, which is always
480 return Src == MVT::i32 && Dest == MVT::i64;
483 bool AMDGPUTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
484 return isZExtFree(Val.getValueType(), VT2);
487 bool AMDGPUTargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const {
488 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
489 // limited number of native 64-bit operations. Shrinking an operation to fit
490 // in a single 32-bit register should always be helpful. As currently used,
491 // this is much less general than the name suggests, and is only used in
492 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
493 // not profitable, and may actually be harmful.
494 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
497 //===---------------------------------------------------------------------===//
498 // TargetLowering Callbacks
499 //===---------------------------------------------------------------------===//
501 void AMDGPUTargetLowering::AnalyzeFormalArguments(CCState &State,
502 const SmallVectorImpl<ISD::InputArg> &Ins) const {
504 State.AnalyzeFormalArguments(Ins, CC_AMDGPU);
507 SDValue AMDGPUTargetLowering::LowerReturn(
509 CallingConv::ID CallConv,
511 const SmallVectorImpl<ISD::OutputArg> &Outs,
512 const SmallVectorImpl<SDValue> &OutVals,
513 SDLoc DL, SelectionDAG &DAG) const {
514 return DAG.getNode(AMDGPUISD::RET_FLAG, DL, MVT::Other, Chain);
517 //===---------------------------------------------------------------------===//
518 // Target specific lowering
519 //===---------------------------------------------------------------------===//
521 SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
522 SmallVectorImpl<SDValue> &InVals) const {
523 SDValue Callee = CLI.Callee;
524 SelectionDAG &DAG = CLI.DAG;
526 const Function &Fn = *DAG.getMachineFunction().getFunction();
528 StringRef FuncName("<unknown>");
530 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Callee))
531 FuncName = G->getSymbol();
532 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
533 FuncName = G->getGlobal()->getName();
535 DiagnosticInfoUnsupported NoCalls(Fn, "call to function " + FuncName);
536 DAG.getContext()->diagnose(NoCalls);
540 SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
541 SelectionDAG &DAG) const {
542 switch (Op.getOpcode()) {
544 Op.getNode()->dump();
545 llvm_unreachable("Custom lowering code for this"
546 "instruction is not implemented yet!");
548 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
549 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
550 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
551 case ISD::FrameIndex: return LowerFrameIndex(Op, DAG);
552 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
553 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
554 case ISD::SDIVREM: return LowerSDIVREM(Op, DAG);
555 case ISD::FREM: return LowerFREM(Op, DAG);
556 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
557 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
558 case ISD::FRINT: return LowerFRINT(Op, DAG);
559 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
560 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
561 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
566 void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
567 SmallVectorImpl<SDValue> &Results,
568 SelectionDAG &DAG) const {
569 switch (N->getOpcode()) {
570 case ISD::SIGN_EXTEND_INREG:
571 // Different parts of legalization seem to interpret which type of
572 // sign_extend_inreg is the one to check for custom lowering. The extended
573 // from type is what really matters, but some places check for custom
574 // lowering of the result type. This results in trying to use
575 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
576 // nothing here and let the illegal result integer be handled normally.
579 SDNode *Node = LowerLOAD(SDValue(N, 0), DAG).getNode();
583 Results.push_back(SDValue(Node, 0));
584 Results.push_back(SDValue(Node, 1));
585 // XXX: LLVM seems not to replace Chain Value inside CustomWidenLowerNode
587 DAG.ReplaceAllUsesOfValueWith(SDValue(N,1), SDValue(Node, 1));
591 SDValue Lowered = LowerSTORE(SDValue(N, 0), DAG);
592 if (Lowered.getNode())
593 Results.push_back(Lowered);
601 // FIXME: This implements accesses to initialized globals in the constant
602 // address space by copying them to private and accessing that. It does not
603 // properly handle illegal types or vectors. The private vector loads are not
604 // scalarized, and the illegal scalars hit an assertion. This technique will not
605 // work well with large initializers, and this should eventually be
606 // removed. Initialized globals should be placed into a data section that the
607 // runtime will load into a buffer before the kernel is executed. Uses of the
608 // global need to be replaced with a pointer loaded from an implicit kernel
609 // argument into this buffer holding the copy of the data, which will remove the
610 // need for any of this.
611 SDValue AMDGPUTargetLowering::LowerConstantInitializer(const Constant* Init,
612 const GlobalValue *GV,
613 const SDValue &InitPtr,
615 SelectionDAG &DAG) const {
616 const DataLayout *TD = getTargetMachine().getSubtargetImpl()->getDataLayout();
618 Type *InitTy = Init->getType();
620 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Init)) {
621 EVT VT = EVT::getEVT(InitTy);
622 PointerType *PtrTy = PointerType::get(InitTy, AMDGPUAS::PRIVATE_ADDRESS);
623 return DAG.getStore(Chain, DL, DAG.getConstant(*CI, VT), InitPtr,
624 MachinePointerInfo(UndefValue::get(PtrTy)), false, false,
625 TD->getPrefTypeAlignment(InitTy));
628 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Init)) {
629 EVT VT = EVT::getEVT(CFP->getType());
630 PointerType *PtrTy = PointerType::get(CFP->getType(), 0);
631 return DAG.getStore(Chain, DL, DAG.getConstantFP(*CFP, VT), InitPtr,
632 MachinePointerInfo(UndefValue::get(PtrTy)), false, false,
633 TD->getPrefTypeAlignment(CFP->getType()));
636 if (StructType *ST = dyn_cast<StructType>(InitTy)) {
637 const StructLayout *SL = TD->getStructLayout(ST);
639 EVT PtrVT = InitPtr.getValueType();
640 SmallVector<SDValue, 8> Chains;
642 for (unsigned I = 0, N = ST->getNumElements(); I != N; ++I) {
643 SDValue Offset = DAG.getConstant(SL->getElementOffset(I), PtrVT);
644 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset);
646 Constant *Elt = Init->getAggregateElement(I);
647 Chains.push_back(LowerConstantInitializer(Elt, GV, Ptr, Chain, DAG));
650 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
653 if (SequentialType *SeqTy = dyn_cast<SequentialType>(InitTy)) {
654 EVT PtrVT = InitPtr.getValueType();
656 unsigned NumElements;
657 if (ArrayType *AT = dyn_cast<ArrayType>(SeqTy))
658 NumElements = AT->getNumElements();
659 else if (VectorType *VT = dyn_cast<VectorType>(SeqTy))
660 NumElements = VT->getNumElements();
662 llvm_unreachable("Unexpected type");
664 unsigned EltSize = TD->getTypeAllocSize(SeqTy->getElementType());
665 SmallVector<SDValue, 8> Chains;
666 for (unsigned i = 0; i < NumElements; ++i) {
667 SDValue Offset = DAG.getConstant(i * EltSize, PtrVT);
668 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset);
670 Constant *Elt = Init->getAggregateElement(i);
671 Chains.push_back(LowerConstantInitializer(Elt, GV, Ptr, Chain, DAG));
674 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
677 if (isa<UndefValue>(Init)) {
678 EVT VT = EVT::getEVT(InitTy);
679 PointerType *PtrTy = PointerType::get(InitTy, AMDGPUAS::PRIVATE_ADDRESS);
680 return DAG.getStore(Chain, DL, DAG.getUNDEF(VT), InitPtr,
681 MachinePointerInfo(UndefValue::get(PtrTy)), false, false,
682 TD->getPrefTypeAlignment(InitTy));
686 llvm_unreachable("Unhandled constant initializer");
689 SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
691 SelectionDAG &DAG) const {
693 const DataLayout *TD = getTargetMachine().getSubtargetImpl()->getDataLayout();
694 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op);
695 const GlobalValue *GV = G->getGlobal();
697 switch (G->getAddressSpace()) {
698 default: llvm_unreachable("Global Address lowering not implemented for this "
700 case AMDGPUAS::LOCAL_ADDRESS: {
701 // XXX: What does the value of G->getOffset() mean?
702 assert(G->getOffset() == 0 &&
703 "Do not know what to do with an non-zero offset");
706 if (MFI->LocalMemoryObjects.count(GV) == 0) {
707 uint64_t Size = TD->getTypeAllocSize(GV->getType()->getElementType());
708 Offset = MFI->LDSSize;
709 MFI->LocalMemoryObjects[GV] = Offset;
710 // XXX: Account for alignment?
711 MFI->LDSSize += Size;
713 Offset = MFI->LocalMemoryObjects[GV];
716 return DAG.getConstant(Offset, getPointerTy(AMDGPUAS::LOCAL_ADDRESS));
718 case AMDGPUAS::CONSTANT_ADDRESS: {
719 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
720 Type *EltType = GV->getType()->getElementType();
721 unsigned Size = TD->getTypeAllocSize(EltType);
722 unsigned Alignment = TD->getPrefTypeAlignment(EltType);
724 MVT PrivPtrVT = getPointerTy(AMDGPUAS::PRIVATE_ADDRESS);
725 MVT ConstPtrVT = getPointerTy(AMDGPUAS::CONSTANT_ADDRESS);
727 int FI = FrameInfo->CreateStackObject(Size, Alignment, false);
728 SDValue InitPtr = DAG.getFrameIndex(FI, PrivPtrVT);
730 const GlobalVariable *Var = cast<GlobalVariable>(GV);
731 if (!Var->hasInitializer()) {
732 // This has no use, but bugpoint will hit it.
733 return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op), ConstPtrVT);
736 const Constant *Init = Var->getInitializer();
737 SmallVector<SDNode*, 8> WorkList;
739 for (SDNode::use_iterator I = DAG.getEntryNode()->use_begin(),
740 E = DAG.getEntryNode()->use_end(); I != E; ++I) {
741 if (I->getOpcode() != AMDGPUISD::REGISTER_LOAD && I->getOpcode() != ISD::LOAD)
743 WorkList.push_back(*I);
745 SDValue Chain = LowerConstantInitializer(Init, GV, InitPtr, DAG.getEntryNode(), DAG);
746 for (SmallVector<SDNode*, 8>::iterator I = WorkList.begin(),
747 E = WorkList.end(); I != E; ++I) {
748 SmallVector<SDValue, 8> Ops;
749 Ops.push_back(Chain);
750 for (unsigned i = 1; i < (*I)->getNumOperands(); ++i) {
751 Ops.push_back((*I)->getOperand(i));
753 DAG.UpdateNodeOperands(*I, Ops);
755 return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op), ConstPtrVT);
760 SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
761 SelectionDAG &DAG) const {
762 SmallVector<SDValue, 8> Args;
763 SDValue A = Op.getOperand(0);
764 SDValue B = Op.getOperand(1);
766 DAG.ExtractVectorElements(A, Args);
767 DAG.ExtractVectorElements(B, Args);
769 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args);
772 SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
773 SelectionDAG &DAG) const {
775 SmallVector<SDValue, 8> Args;
776 unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
777 EVT VT = Op.getValueType();
778 DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
779 VT.getVectorNumElements());
781 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args);
784 SDValue AMDGPUTargetLowering::LowerFrameIndex(SDValue Op,
785 SelectionDAG &DAG) const {
787 MachineFunction &MF = DAG.getMachineFunction();
788 const AMDGPUFrameLowering *TFL = static_cast<const AMDGPUFrameLowering *>(
789 getTargetMachine().getSubtargetImpl()->getFrameLowering());
791 FrameIndexSDNode *FIN = cast<FrameIndexSDNode>(Op);
793 unsigned FrameIndex = FIN->getIndex();
794 unsigned Offset = TFL->getFrameIndexOffset(MF, FrameIndex);
795 return DAG.getConstant(Offset * 4 * TFL->getStackWidth(MF),
799 SDValue AMDGPUTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
800 SelectionDAG &DAG) const {
801 unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
803 EVT VT = Op.getValueType();
805 switch (IntrinsicID) {
807 case AMDGPUIntrinsic::AMDGPU_abs:
808 case AMDGPUIntrinsic::AMDIL_abs: // Legacy name.
809 return LowerIntrinsicIABS(Op, DAG);
810 case AMDGPUIntrinsic::AMDGPU_lrp:
811 return LowerIntrinsicLRP(Op, DAG);
812 case AMDGPUIntrinsic::AMDGPU_fract:
813 case AMDGPUIntrinsic::AMDIL_fraction: // Legacy name.
814 return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1));
816 case AMDGPUIntrinsic::AMDGPU_clamp:
817 case AMDGPUIntrinsic::AMDIL_clamp: // Legacy name.
818 return DAG.getNode(AMDGPUISD::CLAMP, DL, VT,
819 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
821 case Intrinsic::AMDGPU_div_scale: {
822 // 3rd parameter required to be a constant.
823 const ConstantSDNode *Param = dyn_cast<ConstantSDNode>(Op.getOperand(3));
825 return DAG.getUNDEF(VT);
827 // Translate to the operands expected by the machine instruction. The
828 // first parameter must be the same as the first instruction.
829 SDValue Numerator = Op.getOperand(1);
830 SDValue Denominator = Op.getOperand(2);
831 SDValue Src0 = Param->isAllOnesValue() ? Numerator : Denominator;
833 return DAG.getNode(AMDGPUISD::DIV_SCALE, DL, Op->getVTList(), Src0,
834 Denominator, Numerator);
837 case Intrinsic::AMDGPU_div_fmas:
838 return DAG.getNode(AMDGPUISD::DIV_FMAS, DL, VT,
839 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
841 case Intrinsic::AMDGPU_div_fixup:
842 return DAG.getNode(AMDGPUISD::DIV_FIXUP, DL, VT,
843 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
845 case Intrinsic::AMDGPU_trig_preop:
846 return DAG.getNode(AMDGPUISD::TRIG_PREOP, DL, VT,
847 Op.getOperand(1), Op.getOperand(2));
849 case Intrinsic::AMDGPU_rcp:
850 return DAG.getNode(AMDGPUISD::RCP, DL, VT, Op.getOperand(1));
852 case Intrinsic::AMDGPU_rsq:
853 return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
855 case AMDGPUIntrinsic::AMDGPU_legacy_rsq:
856 return DAG.getNode(AMDGPUISD::RSQ_LEGACY, DL, VT, Op.getOperand(1));
858 case Intrinsic::AMDGPU_rsq_clamped:
859 return DAG.getNode(AMDGPUISD::RSQ_CLAMPED, DL, VT, Op.getOperand(1));
861 case Intrinsic::AMDGPU_ldexp:
862 return DAG.getNode(AMDGPUISD::LDEXP, DL, VT, Op.getOperand(1),
865 case AMDGPUIntrinsic::AMDGPU_imax:
866 return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Op.getOperand(1),
868 case AMDGPUIntrinsic::AMDGPU_umax:
869 return DAG.getNode(AMDGPUISD::UMAX, DL, VT, Op.getOperand(1),
871 case AMDGPUIntrinsic::AMDGPU_imin:
872 return DAG.getNode(AMDGPUISD::SMIN, DL, VT, Op.getOperand(1),
874 case AMDGPUIntrinsic::AMDGPU_umin:
875 return DAG.getNode(AMDGPUISD::UMIN, DL, VT, Op.getOperand(1),
878 case AMDGPUIntrinsic::AMDGPU_umul24:
879 return DAG.getNode(AMDGPUISD::MUL_U24, DL, VT,
880 Op.getOperand(1), Op.getOperand(2));
882 case AMDGPUIntrinsic::AMDGPU_imul24:
883 return DAG.getNode(AMDGPUISD::MUL_I24, DL, VT,
884 Op.getOperand(1), Op.getOperand(2));
886 case AMDGPUIntrinsic::AMDGPU_umad24:
887 return DAG.getNode(AMDGPUISD::MAD_U24, DL, VT,
888 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
890 case AMDGPUIntrinsic::AMDGPU_imad24:
891 return DAG.getNode(AMDGPUISD::MAD_I24, DL, VT,
892 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
894 case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte0:
895 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, VT, Op.getOperand(1));
897 case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte1:
898 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE1, DL, VT, Op.getOperand(1));
900 case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte2:
901 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE2, DL, VT, Op.getOperand(1));
903 case AMDGPUIntrinsic::AMDGPU_cvt_f32_ubyte3:
904 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE3, DL, VT, Op.getOperand(1));
906 case AMDGPUIntrinsic::AMDGPU_bfe_i32:
907 return DAG.getNode(AMDGPUISD::BFE_I32, DL, VT,
912 case AMDGPUIntrinsic::AMDGPU_bfe_u32:
913 return DAG.getNode(AMDGPUISD::BFE_U32, DL, VT,
918 case AMDGPUIntrinsic::AMDGPU_bfi:
919 return DAG.getNode(AMDGPUISD::BFI, DL, VT,
924 case AMDGPUIntrinsic::AMDGPU_bfm:
925 return DAG.getNode(AMDGPUISD::BFM, DL, VT,
929 case AMDGPUIntrinsic::AMDGPU_brev:
930 return DAG.getNode(AMDGPUISD::BREV, DL, VT, Op.getOperand(1));
932 case AMDGPUIntrinsic::AMDIL_exp: // Legacy name.
933 return DAG.getNode(ISD::FEXP2, DL, VT, Op.getOperand(1));
935 case AMDGPUIntrinsic::AMDIL_round_nearest: // Legacy name.
936 return DAG.getNode(ISD::FRINT, DL, VT, Op.getOperand(1));
937 case AMDGPUIntrinsic::AMDGPU_trunc: // Legacy name.
938 return DAG.getNode(ISD::FTRUNC, DL, VT, Op.getOperand(1));
942 ///IABS(a) = SMAX(sub(0, a), a)
943 SDValue AMDGPUTargetLowering::LowerIntrinsicIABS(SDValue Op,
944 SelectionDAG &DAG) const {
946 EVT VT = Op.getValueType();
947 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT),
950 return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Neg, Op.getOperand(1));
953 /// Linear Interpolation
954 /// LRP(a, b, c) = muladd(a, b, (1 - a) * c)
955 SDValue AMDGPUTargetLowering::LowerIntrinsicLRP(SDValue Op,
956 SelectionDAG &DAG) const {
958 EVT VT = Op.getValueType();
959 SDValue OneSubA = DAG.getNode(ISD::FSUB, DL, VT,
960 DAG.getConstantFP(1.0f, MVT::f32),
962 SDValue OneSubAC = DAG.getNode(ISD::FMUL, DL, VT, OneSubA,
964 return DAG.getNode(ISD::FADD, DL, VT,
965 DAG.getNode(ISD::FMUL, DL, VT, Op.getOperand(1), Op.getOperand(2)),
969 /// \brief Generate Min/Max node
970 SDValue AMDGPUTargetLowering::CombineMinMax(SDNode *N,
971 SelectionDAG &DAG) const {
973 EVT VT = N->getValueType(0);
975 SDValue LHS = N->getOperand(0);
976 SDValue RHS = N->getOperand(1);
977 SDValue True = N->getOperand(2);
978 SDValue False = N->getOperand(3);
979 SDValue CC = N->getOperand(4);
981 if (VT != MVT::f32 ||
982 !((LHS == True && RHS == False) || (LHS == False && RHS == True))) {
986 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
1000 llvm_unreachable("Operation should already be optimised!");
1007 unsigned Opc = (LHS == True) ? AMDGPUISD::FMIN : AMDGPUISD::FMAX;
1008 return DAG.getNode(Opc, DL, VT, LHS, RHS);
1016 unsigned Opc = (LHS == True) ? AMDGPUISD::FMAX : AMDGPUISD::FMIN;
1017 return DAG.getNode(Opc, DL, VT, LHS, RHS);
1019 case ISD::SETCC_INVALID:
1020 llvm_unreachable("Invalid setcc condcode!");
1025 SDValue AMDGPUTargetLowering::ScalarizeVectorLoad(const SDValue Op,
1026 SelectionDAG &DAG) const {
1027 LoadSDNode *Load = cast<LoadSDNode>(Op);
1028 EVT MemVT = Load->getMemoryVT();
1029 EVT MemEltVT = MemVT.getVectorElementType();
1031 EVT LoadVT = Op.getValueType();
1032 EVT EltVT = LoadVT.getVectorElementType();
1033 EVT PtrVT = Load->getBasePtr().getValueType();
1035 unsigned NumElts = Load->getMemoryVT().getVectorNumElements();
1036 SmallVector<SDValue, 8> Loads;
1037 SmallVector<SDValue, 8> Chains;
1040 unsigned MemEltSize = MemEltVT.getStoreSize();
1041 MachinePointerInfo SrcValue(Load->getMemOperand()->getValue());
1043 for (unsigned i = 0; i < NumElts; ++i) {
1044 SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, Load->getBasePtr(),
1045 DAG.getConstant(i * MemEltSize, PtrVT));
1048 = DAG.getExtLoad(Load->getExtensionType(), SL, EltVT,
1049 Load->getChain(), Ptr,
1050 SrcValue.getWithOffset(i * MemEltSize),
1051 MemEltVT, Load->isVolatile(), Load->isNonTemporal(),
1052 Load->isInvariant(), Load->getAlignment());
1053 Loads.push_back(NewLoad.getValue(0));
1054 Chains.push_back(NewLoad.getValue(1));
1058 DAG.getNode(ISD::BUILD_VECTOR, SL, LoadVT, Loads),
1059 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains)
1062 return DAG.getMergeValues(Ops, SL);
1065 SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1066 SelectionDAG &DAG) const {
1067 EVT VT = Op.getValueType();
1069 // If this is a 2 element vector, we really want to scalarize and not create
1070 // weird 1 element vectors.
1071 if (VT.getVectorNumElements() == 2)
1072 return ScalarizeVectorLoad(Op, DAG);
1074 LoadSDNode *Load = cast<LoadSDNode>(Op);
1075 SDValue BasePtr = Load->getBasePtr();
1076 EVT PtrVT = BasePtr.getValueType();
1077 EVT MemVT = Load->getMemoryVT();
1079 MachinePointerInfo SrcValue(Load->getMemOperand()->getValue());
1082 EVT LoMemVT, HiMemVT;
1085 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
1086 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemVT);
1087 std::tie(Lo, Hi) = DAG.SplitVector(Op, SL, LoVT, HiVT);
1089 = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT,
1090 Load->getChain(), BasePtr,
1092 LoMemVT, Load->isVolatile(), Load->isNonTemporal(),
1093 Load->isInvariant(), Load->getAlignment());
1095 SDValue HiPtr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
1096 DAG.getConstant(LoMemVT.getStoreSize(), PtrVT));
1099 = DAG.getExtLoad(Load->getExtensionType(), SL, HiVT,
1100 Load->getChain(), HiPtr,
1101 SrcValue.getWithOffset(LoMemVT.getStoreSize()),
1102 HiMemVT, Load->isVolatile(), Load->isNonTemporal(),
1103 Load->isInvariant(), Load->getAlignment());
1106 DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad),
1107 DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
1108 LoLoad.getValue(1), HiLoad.getValue(1))
1111 return DAG.getMergeValues(Ops, SL);
1114 SDValue AMDGPUTargetLowering::MergeVectorStore(const SDValue &Op,
1115 SelectionDAG &DAG) const {
1116 StoreSDNode *Store = cast<StoreSDNode>(Op);
1117 EVT MemVT = Store->getMemoryVT();
1118 unsigned MemBits = MemVT.getSizeInBits();
1120 // Byte stores are really expensive, so if possible, try to pack 32-bit vector
1121 // truncating store into an i32 store.
1122 // XXX: We could also handle optimize other vector bitwidths.
1123 if (!MemVT.isVector() || MemBits > 32) {
1128 SDValue Value = Store->getValue();
1129 EVT VT = Value.getValueType();
1130 EVT ElemVT = VT.getVectorElementType();
1131 SDValue Ptr = Store->getBasePtr();
1132 EVT MemEltVT = MemVT.getVectorElementType();
1133 unsigned MemEltBits = MemEltVT.getSizeInBits();
1134 unsigned MemNumElements = MemVT.getVectorNumElements();
1135 unsigned PackedSize = MemVT.getStoreSizeInBits();
1136 SDValue Mask = DAG.getConstant((1 << MemEltBits) - 1, MVT::i32);
1138 assert(Value.getValueType().getScalarSizeInBits() >= 32);
1140 SDValue PackedValue;
1141 for (unsigned i = 0; i < MemNumElements; ++i) {
1142 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, Value,
1143 DAG.getConstant(i, MVT::i32));
1144 Elt = DAG.getZExtOrTrunc(Elt, DL, MVT::i32);
1145 Elt = DAG.getNode(ISD::AND, DL, MVT::i32, Elt, Mask); // getZeroExtendInReg
1147 SDValue Shift = DAG.getConstant(MemEltBits * i, MVT::i32);
1148 Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt, Shift);
1153 PackedValue = DAG.getNode(ISD::OR, DL, MVT::i32, PackedValue, Elt);
1157 if (PackedSize < 32) {
1158 EVT PackedVT = EVT::getIntegerVT(*DAG.getContext(), PackedSize);
1159 return DAG.getTruncStore(Store->getChain(), DL, PackedValue, Ptr,
1160 Store->getMemOperand()->getPointerInfo(),
1162 Store->isNonTemporal(), Store->isVolatile(),
1163 Store->getAlignment());
1166 return DAG.getStore(Store->getChain(), DL, PackedValue, Ptr,
1167 Store->getMemOperand()->getPointerInfo(),
1168 Store->isVolatile(), Store->isNonTemporal(),
1169 Store->getAlignment());
1172 SDValue AMDGPUTargetLowering::ScalarizeVectorStore(SDValue Op,
1173 SelectionDAG &DAG) const {
1174 StoreSDNode *Store = cast<StoreSDNode>(Op);
1175 EVT MemEltVT = Store->getMemoryVT().getVectorElementType();
1176 EVT EltVT = Store->getValue().getValueType().getVectorElementType();
1177 EVT PtrVT = Store->getBasePtr().getValueType();
1178 unsigned NumElts = Store->getMemoryVT().getVectorNumElements();
1181 SmallVector<SDValue, 8> Chains;
1183 unsigned EltSize = MemEltVT.getStoreSize();
1184 MachinePointerInfo SrcValue(Store->getMemOperand()->getValue());
1186 for (unsigned i = 0, e = NumElts; i != e; ++i) {
1187 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
1189 DAG.getConstant(i, MVT::i32));
1191 SDValue Offset = DAG.getConstant(i * MemEltVT.getStoreSize(), PtrVT);
1192 SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, Store->getBasePtr(), Offset);
1194 DAG.getTruncStore(Store->getChain(), SL, Val, Ptr,
1195 SrcValue.getWithOffset(i * EltSize),
1196 MemEltVT, Store->isNonTemporal(), Store->isVolatile(),
1197 Store->getAlignment());
1198 Chains.push_back(NewStore);
1201 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains);
1204 SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1205 SelectionDAG &DAG) const {
1206 StoreSDNode *Store = cast<StoreSDNode>(Op);
1207 SDValue Val = Store->getValue();
1208 EVT VT = Val.getValueType();
1210 // If this is a 2 element vector, we really want to scalarize and not create
1211 // weird 1 element vectors.
1212 if (VT.getVectorNumElements() == 2)
1213 return ScalarizeVectorStore(Op, DAG);
1215 EVT MemVT = Store->getMemoryVT();
1216 SDValue Chain = Store->getChain();
1217 SDValue BasePtr = Store->getBasePtr();
1221 EVT LoMemVT, HiMemVT;
1224 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
1225 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemVT);
1226 std::tie(Lo, Hi) = DAG.SplitVector(Val, SL, LoVT, HiVT);
1228 EVT PtrVT = BasePtr.getValueType();
1229 SDValue HiPtr = DAG.getNode(ISD::ADD, SL, PtrVT, BasePtr,
1230 DAG.getConstant(LoMemVT.getStoreSize(), PtrVT));
1232 MachinePointerInfo SrcValue(Store->getMemOperand()->getValue());
1234 = DAG.getTruncStore(Chain, SL, Lo,
1238 Store->isNonTemporal(),
1239 Store->isVolatile(),
1240 Store->getAlignment());
1242 = DAG.getTruncStore(Chain, SL, Hi,
1244 SrcValue.getWithOffset(LoMemVT.getStoreSize()),
1246 Store->isNonTemporal(),
1247 Store->isVolatile(),
1248 Store->getAlignment());
1250 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore);
1254 SDValue AMDGPUTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1256 LoadSDNode *Load = cast<LoadSDNode>(Op);
1257 ISD::LoadExtType ExtType = Load->getExtensionType();
1258 EVT VT = Op.getValueType();
1259 EVT MemVT = Load->getMemoryVT();
1261 if (ExtType != ISD::NON_EXTLOAD && !VT.isVector() && VT.getSizeInBits() > 32) {
1262 // We can do the extload to 32-bits, and then need to separately extend to
1265 SDValue ExtLoad32 = DAG.getExtLoad(ExtType, DL, MVT::i32,
1269 Load->getMemOperand());
1272 DAG.getNode(ISD::getExtForLoadExtType(ExtType), DL, VT, ExtLoad32),
1273 ExtLoad32.getValue(1)
1276 return DAG.getMergeValues(Ops, DL);
1279 if (ExtType == ISD::NON_EXTLOAD && VT.getSizeInBits() < 32) {
1280 assert(VT == MVT::i1 && "Only i1 non-extloads expected");
1281 // FIXME: Copied from PPC
1282 // First, load into 32 bits, then truncate to 1 bit.
1284 SDValue Chain = Load->getChain();
1285 SDValue BasePtr = Load->getBasePtr();
1286 MachineMemOperand *MMO = Load->getMemOperand();
1288 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain,
1289 BasePtr, MVT::i8, MMO);
1292 DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD),
1296 return DAG.getMergeValues(Ops, DL);
1299 if (Subtarget->getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS ||
1300 Load->getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS ||
1301 ExtType == ISD::NON_EXTLOAD || Load->getMemoryVT().bitsGE(MVT::i32))
1305 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Load->getBasePtr(),
1306 DAG.getConstant(2, MVT::i32));
1307 SDValue Ret = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op.getValueType(),
1308 Load->getChain(), Ptr,
1309 DAG.getTargetConstant(0, MVT::i32),
1311 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32,
1313 DAG.getConstant(0x3, MVT::i32));
1314 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1315 DAG.getConstant(3, MVT::i32));
1317 Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Ret, ShiftAmt);
1319 EVT MemEltVT = MemVT.getScalarType();
1320 if (ExtType == ISD::SEXTLOAD) {
1321 SDValue MemEltVTNode = DAG.getValueType(MemEltVT);
1324 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode),
1328 return DAG.getMergeValues(Ops, DL);
1332 DAG.getZeroExtendInReg(Ret, DL, MemEltVT),
1336 return DAG.getMergeValues(Ops, DL);
1339 SDValue AMDGPUTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1341 SDValue Result = AMDGPUTargetLowering::MergeVectorStore(Op, DAG);
1342 if (Result.getNode()) {
1346 StoreSDNode *Store = cast<StoreSDNode>(Op);
1347 SDValue Chain = Store->getChain();
1348 if ((Store->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1349 Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) &&
1350 Store->getValue().getValueType().isVector()) {
1351 return ScalarizeVectorStore(Op, DAG);
1354 EVT MemVT = Store->getMemoryVT();
1355 if (Store->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS &&
1356 MemVT.bitsLT(MVT::i32)) {
1358 if (Store->getMemoryVT() == MVT::i8) {
1360 } else if (Store->getMemoryVT() == MVT::i16) {
1363 SDValue BasePtr = Store->getBasePtr();
1364 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, BasePtr,
1365 DAG.getConstant(2, MVT::i32));
1366 SDValue Dst = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32,
1367 Chain, Ptr, DAG.getTargetConstant(0, MVT::i32));
1369 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, BasePtr,
1370 DAG.getConstant(0x3, MVT::i32));
1372 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1373 DAG.getConstant(3, MVT::i32));
1375 SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32,
1378 SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT);
1380 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32,
1381 MaskedValue, ShiftAmt);
1383 SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, DAG.getConstant(Mask, MVT::i32),
1385 DstMask = DAG.getNode(ISD::XOR, DL, MVT::i32, DstMask,
1386 DAG.getConstant(0xffffffff, MVT::i32));
1387 Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask);
1389 SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue);
1390 return DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other,
1391 Chain, Value, Ptr, DAG.getTargetConstant(0, MVT::i32));
1396 // This is a shortcut for integer division because we have fast i32<->f32
1397 // conversions, and fast f32 reciprocal instructions. The fractional part of a
1398 // float is enough to accurately represent up to a 24-bit integer.
1399 SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const {
1401 EVT VT = Op.getValueType();
1402 SDValue LHS = Op.getOperand(0);
1403 SDValue RHS = Op.getOperand(1);
1404 MVT IntVT = MVT::i32;
1405 MVT FltVT = MVT::f32;
1407 ISD::NodeType ToFp = sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
1408 ISD::NodeType ToInt = sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
1410 if (VT.isVector()) {
1411 unsigned NElts = VT.getVectorNumElements();
1412 IntVT = MVT::getVectorVT(MVT::i32, NElts);
1413 FltVT = MVT::getVectorVT(MVT::f32, NElts);
1416 unsigned BitSize = VT.getScalarType().getSizeInBits();
1418 SDValue jq = DAG.getConstant(1, IntVT);
1421 // char|short jq = ia ^ ib;
1422 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
1424 // jq = jq >> (bitsize - 2)
1425 jq = DAG.getNode(ISD::SRA, DL, VT, jq, DAG.getConstant(BitSize - 2, VT));
1428 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, VT));
1431 jq = DAG.getSExtOrTrunc(jq, DL, IntVT);
1434 // int ia = (int)LHS;
1436 DAG.getSExtOrTrunc(LHS, DL, IntVT) : DAG.getZExtOrTrunc(LHS, DL, IntVT);
1438 // int ib, (int)RHS;
1440 DAG.getSExtOrTrunc(RHS, DL, IntVT) : DAG.getZExtOrTrunc(RHS, DL, IntVT);
1442 // float fa = (float)ia;
1443 SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);
1445 // float fb = (float)ib;
1446 SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
1448 // float fq = native_divide(fa, fb);
1449 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
1450 fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb));
1453 fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq);
1455 // float fqneg = -fq;
1456 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq);
1458 // float fr = mad(fqneg, fb, fa);
1459 SDValue fr = DAG.getNode(ISD::FADD, DL, FltVT,
1460 DAG.getNode(ISD::FMUL, DL, FltVT, fqneg, fb), fa);
1462 // int iq = (int)fq;
1463 SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq);
1466 fr = DAG.getNode(ISD::FABS, DL, FltVT, fr);
1469 fb = DAG.getNode(ISD::FABS, DL, FltVT, fb);
1471 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), VT);
1473 // int cv = fr >= fb;
1474 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE);
1476 // jq = (cv ? jq : 0);
1477 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, VT));
1479 // dst = trunc/extend to legal type
1480 iq = sign ? DAG.getSExtOrTrunc(iq, DL, VT) : DAG.getZExtOrTrunc(iq, DL, VT);
1483 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq);
1485 // Rem needs compensation, it's easier to recompute it
1486 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
1487 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem);
1493 return DAG.getMergeValues(Res, DL);
1496 SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
1497 SelectionDAG &DAG) const {
1499 EVT VT = Op.getValueType();
1501 SDValue Num = Op.getOperand(0);
1502 SDValue Den = Op.getOperand(1);
1504 if (VT == MVT::i32) {
1505 if (DAG.MaskedValueIsZero(Op.getOperand(0), APInt(32, 0xff << 24)) &&
1506 DAG.MaskedValueIsZero(Op.getOperand(1), APInt(32, 0xff << 24))) {
1507 // TODO: We technically could do this for i64, but shouldn't that just be
1508 // handled by something generally reducing 64-bit division on 32-bit
1509 // values to 32-bit?
1510 return LowerDIVREM24(Op, DAG, false);
1514 // RCP = URECIP(Den) = 2^32 / Den + e
1515 // e is rounding error.
1516 SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den);
1518 // RCP_LO = umulo(RCP, Den) */
1519 SDValue RCP_LO = DAG.getNode(ISD::UMULO, DL, VT, RCP, Den);
1521 // RCP_HI = mulhu (RCP, Den) */
1522 SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den);
1524 // NEG_RCP_LO = -RCP_LO
1525 SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT),
1528 // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
1529 SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT),
1532 // Calculate the rounding error from the URECIP instruction
1533 // E = mulhu(ABS_RCP_LO, RCP)
1534 SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP);
1536 // RCP_A_E = RCP + E
1537 SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E);
1539 // RCP_S_E = RCP - E
1540 SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E);
1542 // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
1543 SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT),
1546 // Quotient = mulhu(Tmp0, Num)
1547 SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num);
1549 // Num_S_Remainder = Quotient * Den
1550 SDValue Num_S_Remainder = DAG.getNode(ISD::UMULO, DL, VT, Quotient, Den);
1552 // Remainder = Num - Num_S_Remainder
1553 SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder);
1555 // Remainder_GE_Den = (Remainder >= Den ? -1 : 0)
1556 SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den,
1557 DAG.getConstant(-1, VT),
1558 DAG.getConstant(0, VT),
1560 // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0)
1561 SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num,
1563 DAG.getConstant(-1, VT),
1564 DAG.getConstant(0, VT),
1566 // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
1567 SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den,
1570 // Calculate Division result:
1572 // Quotient_A_One = Quotient + 1
1573 SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient,
1574 DAG.getConstant(1, VT));
1576 // Quotient_S_One = Quotient - 1
1577 SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient,
1578 DAG.getConstant(1, VT));
1580 // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One)
1581 SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT),
1582 Quotient, Quotient_A_One, ISD::SETEQ);
1584 // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div)
1585 Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT),
1586 Quotient_S_One, Div, ISD::SETEQ);
1588 // Calculate Rem result:
1590 // Remainder_S_Den = Remainder - Den
1591 SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den);
1593 // Remainder_A_Den = Remainder + Den
1594 SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den);
1596 // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den)
1597 SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT),
1598 Remainder, Remainder_S_Den, ISD::SETEQ);
1600 // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
1601 Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT),
1602 Remainder_A_Den, Rem, ISD::SETEQ);
1607 return DAG.getMergeValues(Ops, DL);
1610 SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
1611 SelectionDAG &DAG) const {
1613 EVT VT = Op.getValueType();
1615 SDValue LHS = Op.getOperand(0);
1616 SDValue RHS = Op.getOperand(1);
1618 if (VT == MVT::i32) {
1619 if (DAG.ComputeNumSignBits(Op.getOperand(0)) > 8 &&
1620 DAG.ComputeNumSignBits(Op.getOperand(1)) > 8) {
1621 // TODO: We technically could do this for i64, but shouldn't that just be
1622 // handled by something generally reducing 64-bit division on 32-bit
1623 // values to 32-bit?
1624 return LowerDIVREM24(Op, DAG, true);
1628 SDValue Zero = DAG.getConstant(0, VT);
1629 SDValue NegOne = DAG.getConstant(-1, VT);
1631 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
1632 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
1633 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
1634 SDValue RSign = LHSign; // Remainder sign is the same as LHS
1636 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign);
1637 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign);
1639 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign);
1640 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign);
1642 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
1643 SDValue Rem = Div.getValue(1);
1645 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign);
1646 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign);
1648 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
1649 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);
1655 return DAG.getMergeValues(Res, DL);
1658 // (frem x, y) -> (fsub x, (fmul (ftrunc (fdiv x, y)), y))
1659 SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
1661 EVT VT = Op.getValueType();
1662 SDValue X = Op.getOperand(0);
1663 SDValue Y = Op.getOperand(1);
1665 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y);
1666 SDValue Floor = DAG.getNode(ISD::FTRUNC, SL, VT, Div);
1667 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Floor, Y);
1669 return DAG.getNode(ISD::FSUB, SL, VT, X, Mul);
1672 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
1674 SDValue Src = Op.getOperand(0);
1676 // result = trunc(src)
1677 // if (src > 0.0 && src != result)
1680 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
1682 const SDValue Zero = DAG.getConstantFP(0.0, MVT::f64);
1683 const SDValue One = DAG.getConstantFP(1.0, MVT::f64);
1685 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64);
1687 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
1688 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
1689 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
1691 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
1692 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
1695 SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
1697 SDValue Src = Op.getOperand(0);
1699 assert(Op.getValueType() == MVT::f64);
1701 const SDValue Zero = DAG.getConstant(0, MVT::i32);
1702 const SDValue One = DAG.getConstant(1, MVT::i32);
1704 SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
1706 // Extract the upper half, since this is where we will find the sign and
1708 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One);
1710 const unsigned FractBits = 52;
1711 const unsigned ExpBits = 11;
1713 // Extract the exponent.
1714 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_I32, SL, MVT::i32,
1716 DAG.getConstant(FractBits - 32, MVT::i32),
1717 DAG.getConstant(ExpBits, MVT::i32));
1718 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
1719 DAG.getConstant(1023, MVT::i32));
1721 // Extract the sign bit.
1722 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, MVT::i32);
1723 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
1725 // Extend back to to 64-bits.
1726 SDValue SignBit64 = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
1728 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64);
1730 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
1731 const SDValue FractMask
1732 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, MVT::i64);
1734 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
1735 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
1736 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
1738 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::i32);
1740 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, MVT::i32);
1742 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
1743 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
1745 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
1746 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
1748 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2);
1751 SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
1753 SDValue Src = Op.getOperand(0);
1755 assert(Op.getValueType() == MVT::f64);
1757 APFloat C1Val(APFloat::IEEEdouble, "0x1.0p+52");
1758 SDValue C1 = DAG.getConstantFP(C1Val, MVT::f64);
1759 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
1761 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
1762 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);
1764 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
1766 APFloat C2Val(APFloat::IEEEdouble, "0x1.fffffffffffffp+51");
1767 SDValue C2 = DAG.getConstantFP(C2Val, MVT::f64);
1769 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64);
1770 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
1772 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2);
1775 SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const {
1776 // FNEARBYINT and FRINT are the same, except in their handling of FP
1777 // exceptions. Those aren't really meaningful for us, and OpenCL only has
1778 // rint, so just treat them as equivalent.
1779 return DAG.getNode(ISD::FRINT, SDLoc(Op), Op.getValueType(), Op.getOperand(0));
1782 SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
1784 SDValue Src = Op.getOperand(0);
1786 // result = trunc(src);
1787 // if (src < 0.0 && src != result)
1790 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
1792 const SDValue Zero = DAG.getConstantFP(0.0, MVT::f64);
1793 const SDValue NegOne = DAG.getConstantFP(-1.0, MVT::f64);
1795 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64);
1797 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
1798 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
1799 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
1801 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
1802 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
1805 SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
1806 SelectionDAG &DAG) const {
1807 SDValue S0 = Op.getOperand(0);
1809 if (Op.getValueType() != MVT::f32 || S0.getValueType() != MVT::i64)
1812 // f32 uint_to_fp i64
1813 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0,
1814 DAG.getConstant(0, MVT::i32));
1815 SDValue FloatLo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Lo);
1816 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0,
1817 DAG.getConstant(1, MVT::i32));
1818 SDValue FloatHi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Hi);
1819 FloatHi = DAG.getNode(ISD::FMUL, DL, MVT::f32, FloatHi,
1820 DAG.getConstantFP(4294967296.0f, MVT::f32)); // 2^32
1821 return DAG.getNode(ISD::FADD, DL, MVT::f32, FloatLo, FloatHi);
1824 SDValue AMDGPUTargetLowering::ExpandSIGN_EXTEND_INREG(SDValue Op,
1826 SelectionDAG &DAG) const {
1827 MVT VT = Op.getSimpleValueType();
1829 SDValue Shift = DAG.getConstant(BitsDiff, VT);
1830 // Shift left by 'Shift' bits.
1831 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, Op.getOperand(0), Shift);
1832 // Signed shift Right by 'Shift' bits.
1833 return DAG.getNode(ISD::SRA, DL, VT, Shl, Shift);
1836 SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
1837 SelectionDAG &DAG) const {
1838 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1839 MVT VT = Op.getSimpleValueType();
1840 MVT ScalarVT = VT.getScalarType();
1845 SDValue Src = Op.getOperand(0);
1848 // TODO: Don't scalarize on Evergreen?
1849 unsigned NElts = VT.getVectorNumElements();
1850 SmallVector<SDValue, 8> Args;
1851 DAG.ExtractVectorElements(Src, Args, 0, NElts);
1853 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
1854 for (unsigned I = 0; I < NElts; ++I)
1855 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp);
1857 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Args);
1860 //===----------------------------------------------------------------------===//
1861 // Custom DAG optimizations
1862 //===----------------------------------------------------------------------===//
1864 static bool isU24(SDValue Op, SelectionDAG &DAG) {
1865 APInt KnownZero, KnownOne;
1866 EVT VT = Op.getValueType();
1867 DAG.computeKnownBits(Op, KnownZero, KnownOne);
1869 return (VT.getSizeInBits() - KnownZero.countLeadingOnes()) <= 24;
1872 static bool isI24(SDValue Op, SelectionDAG &DAG) {
1873 EVT VT = Op.getValueType();
1875 // In order for this to be a signed 24-bit value, bit 23, must
1877 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
1878 // as unsigned 24-bit values.
1879 (VT.getSizeInBits() - DAG.ComputeNumSignBits(Op)) < 24;
1882 static void simplifyI24(SDValue Op, TargetLowering::DAGCombinerInfo &DCI) {
1884 SelectionDAG &DAG = DCI.DAG;
1885 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1886 EVT VT = Op.getValueType();
1888 APInt Demanded = APInt::getLowBitsSet(VT.getSizeInBits(), 24);
1889 APInt KnownZero, KnownOne;
1890 TargetLowering::TargetLoweringOpt TLO(DAG, true, true);
1891 if (TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
1892 DCI.CommitTargetLoweringOpt(TLO);
1895 template <typename IntTy>
1896 static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0,
1897 uint32_t Offset, uint32_t Width) {
1898 if (Width + Offset < 32) {
1899 IntTy Result = (Src0 << (32 - Offset - Width)) >> (32 - Width);
1900 return DAG.getConstant(Result, MVT::i32);
1903 return DAG.getConstant(Src0 >> Offset, MVT::i32);
1906 static bool usesAllNormalStores(SDNode *LoadVal) {
1907 for (SDNode::use_iterator I = LoadVal->use_begin(); !I.atEnd(); ++I) {
1908 if (!ISD::isNormalStore(*I))
1915 // If we have a copy of an illegal type, replace it with a load / store of an
1916 // equivalently sized legal type. This avoids intermediate bit pack / unpack
1917 // instructions emitted when handling extloads and truncstores. Ideally we could
1918 // recognize the pack / unpack pattern to eliminate it.
1919 SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
1920 DAGCombinerInfo &DCI) const {
1921 if (!DCI.isBeforeLegalize())
1924 StoreSDNode *SN = cast<StoreSDNode>(N);
1925 SDValue Value = SN->getValue();
1926 EVT VT = Value.getValueType();
1928 if (isTypeLegal(VT) || SN->isVolatile() || !ISD::isNormalLoad(Value.getNode()))
1931 LoadSDNode *LoadVal = cast<LoadSDNode>(Value);
1932 if (LoadVal->isVolatile() || !usesAllNormalStores(LoadVal))
1935 EVT MemVT = LoadVal->getMemoryVT();
1938 SelectionDAG &DAG = DCI.DAG;
1939 EVT LoadVT = getEquivalentMemType(*DAG.getContext(), MemVT);
1941 SDValue NewLoad = DAG.getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD,
1943 LoadVal->getChain(),
1944 LoadVal->getBasePtr(),
1945 LoadVal->getOffset(),
1947 LoadVal->getMemOperand());
1949 SDValue CastLoad = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad.getValue(0));
1950 DCI.CombineTo(LoadVal, CastLoad, NewLoad.getValue(1), false);
1952 return DAG.getStore(SN->getChain(), SL, NewLoad,
1953 SN->getBasePtr(), SN->getMemOperand());
1956 SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
1957 DAGCombinerInfo &DCI) const {
1958 EVT VT = N->getValueType(0);
1960 if (VT.isVector() || VT.getSizeInBits() > 32)
1963 SelectionDAG &DAG = DCI.DAG;
1966 SDValue N0 = N->getOperand(0);
1967 SDValue N1 = N->getOperand(1);
1970 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
1971 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
1972 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
1973 Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1);
1974 } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
1975 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
1976 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
1977 Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1);
1982 // We need to use sext even for MUL_U24, because MUL_U24 is used
1983 // for signed multiply of 8 and 16-bit types.
1984 return DAG.getSExtOrTrunc(Mul, DL, VT);
1987 SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
1988 DAGCombinerInfo &DCI) const {
1989 SelectionDAG &DAG = DCI.DAG;
1992 switch(N->getOpcode()) {
1995 return performMulCombine(N, DCI);
1996 case AMDGPUISD::MUL_I24:
1997 case AMDGPUISD::MUL_U24: {
1998 SDValue N0 = N->getOperand(0);
1999 SDValue N1 = N->getOperand(1);
2000 simplifyI24(N0, DCI);
2001 simplifyI24(N1, DCI);
2004 case ISD::SELECT_CC: {
2005 return CombineMinMax(N, DAG);
2007 case AMDGPUISD::BFE_I32:
2008 case AMDGPUISD::BFE_U32: {
2009 assert(!N->getValueType(0).isVector() &&
2010 "Vector handling of BFE not implemented");
2011 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
2015 uint32_t WidthVal = Width->getZExtValue() & 0x1f;
2017 return DAG.getConstant(0, MVT::i32);
2019 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
2023 SDValue BitsFrom = N->getOperand(0);
2024 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
2026 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
2028 if (OffsetVal == 0) {
2029 // This is already sign / zero extended, so try to fold away extra BFEs.
2030 unsigned SignBits = Signed ? (32 - WidthVal + 1) : (32 - WidthVal);
2032 unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom);
2033 if (OpSignBits >= SignBits)
2036 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal);
2038 // This is a sign_extend_inreg. Replace it to take advantage of existing
2039 // DAG Combines. If not eliminated, we will match back to BFE during
2042 // TODO: The sext_inreg of extended types ends, although we can could
2043 // handle them in a single BFE.
2044 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom,
2045 DAG.getValueType(SmallVT));
2048 return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT);
2051 if (ConstantSDNode *Val = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
2053 return constantFoldBFE<int32_t>(DAG,
2054 Val->getSExtValue(),
2059 return constantFoldBFE<uint32_t>(DAG,
2060 Val->getZExtValue(),
2065 APInt Demanded = APInt::getBitsSet(32,
2067 OffsetVal + WidthVal);
2069 if ((OffsetVal + WidthVal) >= 32) {
2070 SDValue ShiftVal = DAG.getConstant(OffsetVal, MVT::i32);
2071 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
2072 BitsFrom, ShiftVal);
2075 APInt KnownZero, KnownOne;
2076 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2077 !DCI.isBeforeLegalizeOps());
2078 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2079 if (TLO.ShrinkDemandedConstant(BitsFrom, Demanded) ||
2080 TLI.SimplifyDemandedBits(BitsFrom, Demanded, KnownZero, KnownOne, TLO)) {
2081 DCI.CommitTargetLoweringOpt(TLO);
2088 return performStoreCombine(N, DCI);
2093 //===----------------------------------------------------------------------===//
2095 //===----------------------------------------------------------------------===//
2097 void AMDGPUTargetLowering::getOriginalFunctionArgs(
2100 const SmallVectorImpl<ISD::InputArg> &Ins,
2101 SmallVectorImpl<ISD::InputArg> &OrigIns) const {
2103 for (unsigned i = 0, e = Ins.size(); i < e; ++i) {
2104 if (Ins[i].ArgVT == Ins[i].VT) {
2105 OrigIns.push_back(Ins[i]);
2110 if (Ins[i].ArgVT.isVector() && !Ins[i].VT.isVector()) {
2111 // Vector has been split into scalars.
2112 VT = Ins[i].ArgVT.getVectorElementType();
2113 } else if (Ins[i].VT.isVector() && Ins[i].ArgVT.isVector() &&
2114 Ins[i].ArgVT.getVectorElementType() !=
2115 Ins[i].VT.getVectorElementType()) {
2116 // Vector elements have been promoted
2119 // Vector has been spilt into smaller vectors.
2123 ISD::InputArg Arg(Ins[i].Flags, VT, VT, Ins[i].Used,
2124 Ins[i].OrigArgIndex, Ins[i].PartOffset);
2125 OrigIns.push_back(Arg);
2129 bool AMDGPUTargetLowering::isHWTrueValue(SDValue Op) const {
2130 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) {
2131 return CFP->isExactlyValue(1.0);
2133 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
2134 return C->isAllOnesValue();
2139 bool AMDGPUTargetLowering::isHWFalseValue(SDValue Op) const {
2140 if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) {
2141 return CFP->getValueAPF().isZero();
2143 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
2144 return C->isNullValue();
2149 SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
2150 const TargetRegisterClass *RC,
2151 unsigned Reg, EVT VT) const {
2152 MachineFunction &MF = DAG.getMachineFunction();
2153 MachineRegisterInfo &MRI = MF.getRegInfo();
2154 unsigned VirtualRegister;
2155 if (!MRI.isLiveIn(Reg)) {
2156 VirtualRegister = MRI.createVirtualRegister(RC);
2157 MRI.addLiveIn(Reg, VirtualRegister);
2159 VirtualRegister = MRI.getLiveInVirtReg(Reg);
2161 return DAG.getRegister(VirtualRegister, VT);
2164 #define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node;
2166 const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
2168 default: return nullptr;
2170 NODE_NAME_CASE(CALL);
2171 NODE_NAME_CASE(UMUL);
2172 NODE_NAME_CASE(RET_FLAG);
2173 NODE_NAME_CASE(BRANCH_COND);
2176 NODE_NAME_CASE(DWORDADDR)
2177 NODE_NAME_CASE(FRACT)
2178 NODE_NAME_CASE(CLAMP)
2180 NODE_NAME_CASE(FMAX)
2181 NODE_NAME_CASE(SMAX)
2182 NODE_NAME_CASE(UMAX)
2183 NODE_NAME_CASE(FMIN)
2184 NODE_NAME_CASE(SMIN)
2185 NODE_NAME_CASE(UMIN)
2186 NODE_NAME_CASE(URECIP)
2187 NODE_NAME_CASE(DIV_SCALE)
2188 NODE_NAME_CASE(DIV_FMAS)
2189 NODE_NAME_CASE(DIV_FIXUP)
2190 NODE_NAME_CASE(TRIG_PREOP)
2193 NODE_NAME_CASE(RSQ_LEGACY)
2194 NODE_NAME_CASE(RSQ_CLAMPED)
2195 NODE_NAME_CASE(LDEXP)
2196 NODE_NAME_CASE(DOT4)
2197 NODE_NAME_CASE(BFE_U32)
2198 NODE_NAME_CASE(BFE_I32)
2201 NODE_NAME_CASE(BREV)
2202 NODE_NAME_CASE(MUL_U24)
2203 NODE_NAME_CASE(MUL_I24)
2204 NODE_NAME_CASE(MAD_U24)
2205 NODE_NAME_CASE(MAD_I24)
2206 NODE_NAME_CASE(EXPORT)
2207 NODE_NAME_CASE(CONST_ADDRESS)
2208 NODE_NAME_CASE(REGISTER_LOAD)
2209 NODE_NAME_CASE(REGISTER_STORE)
2210 NODE_NAME_CASE(LOAD_CONSTANT)
2211 NODE_NAME_CASE(LOAD_INPUT)
2212 NODE_NAME_CASE(SAMPLE)
2213 NODE_NAME_CASE(SAMPLEB)
2214 NODE_NAME_CASE(SAMPLED)
2215 NODE_NAME_CASE(SAMPLEL)
2216 NODE_NAME_CASE(CVT_F32_UBYTE0)
2217 NODE_NAME_CASE(CVT_F32_UBYTE1)
2218 NODE_NAME_CASE(CVT_F32_UBYTE2)
2219 NODE_NAME_CASE(CVT_F32_UBYTE3)
2220 NODE_NAME_CASE(BUILD_VERTICAL_VECTOR)
2221 NODE_NAME_CASE(CONST_DATA_PTR)
2222 NODE_NAME_CASE(STORE_MSKOR)
2223 NODE_NAME_CASE(TBUFFER_STORE_FORMAT)
2227 static void computeKnownBitsForMinMax(const SDValue Op0,
2231 const SelectionDAG &DAG,
2233 APInt Op0Zero, Op0One;
2234 APInt Op1Zero, Op1One;
2235 DAG.computeKnownBits(Op0, Op0Zero, Op0One, Depth);
2236 DAG.computeKnownBits(Op1, Op1Zero, Op1One, Depth);
2238 KnownZero = Op0Zero & Op1Zero;
2239 KnownOne = Op0One & Op1One;
2242 void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
2246 const SelectionDAG &DAG,
2247 unsigned Depth) const {
2249 KnownZero = KnownOne = APInt(KnownOne.getBitWidth(), 0); // Don't know anything.
2253 unsigned Opc = Op.getOpcode();
2258 case ISD::INTRINSIC_WO_CHAIN: {
2259 // FIXME: The intrinsic should just use the node.
2260 switch (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue()) {
2261 case AMDGPUIntrinsic::AMDGPU_imax:
2262 case AMDGPUIntrinsic::AMDGPU_umax:
2263 case AMDGPUIntrinsic::AMDGPU_imin:
2264 case AMDGPUIntrinsic::AMDGPU_umin:
2265 computeKnownBitsForMinMax(Op.getOperand(1), Op.getOperand(2),
2266 KnownZero, KnownOne, DAG, Depth);
2274 case AMDGPUISD::SMAX:
2275 case AMDGPUISD::UMAX:
2276 case AMDGPUISD::SMIN:
2277 case AMDGPUISD::UMIN:
2278 computeKnownBitsForMinMax(Op.getOperand(0), Op.getOperand(1),
2279 KnownZero, KnownOne, DAG, Depth);
2282 case AMDGPUISD::BFE_I32:
2283 case AMDGPUISD::BFE_U32: {
2284 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2288 unsigned BitWidth = 32;
2289 uint32_t Width = CWidth->getZExtValue() & 0x1f;
2291 KnownZero = APInt::getAllOnesValue(BitWidth);
2292 KnownOne = APInt::getNullValue(BitWidth);
2296 // FIXME: This could do a lot more. If offset is 0, should be the same as
2297 // sign_extend_inreg implementation, but that involves duplicating it.
2298 if (Opc == AMDGPUISD::BFE_I32)
2299 KnownOne = APInt::getHighBitsSet(BitWidth, BitWidth - Width);
2301 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - Width);
2308 unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
2310 const SelectionDAG &DAG,
2311 unsigned Depth) const {
2312 switch (Op.getOpcode()) {
2313 case AMDGPUISD::BFE_I32: {
2314 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2318 unsigned SignBits = 32 - Width->getZExtValue() + 1;
2319 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2320 if (!Offset || !Offset->isNullValue())
2323 // TODO: Could probably figure something out with non-0 offsets.
2324 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
2325 return std::max(SignBits, Op0SignBits);
2328 case AMDGPUISD::BFE_U32: {
2329 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2330 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;