1 //===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines an instruction selector for the NVPTX target.
12 //===----------------------------------------------------------------------===//
14 #include "NVPTXISelDAGToDAG.h"
15 #include "llvm/IR/GlobalValue.h"
16 #include "llvm/IR/Instructions.h"
17 #include "llvm/Support/CommandLine.h"
18 #include "llvm/Support/Debug.h"
19 #include "llvm/Support/ErrorHandling.h"
20 #include "llvm/Support/raw_ostream.h"
21 #include "llvm/Target/TargetIntrinsicInfo.h"
25 #define DEBUG_TYPE "nvptx-isel"
28 FMAContractLevel("nvptx-fma-level", cl::ZeroOrMore, cl::Hidden,
29 cl::desc("NVPTX Specific: FMA contraction (0: don't do it"
30 " 1: do it 2: do it aggressively"),
33 static cl::opt<int> UsePrecDivF32(
34 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
35 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
36 " IEEE Compliant F32 div.rnd if avaiable."),
40 UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
41 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
45 FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
46 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
50 /// createNVPTXISelDag - This pass converts a legalized DAG into a
51 /// NVPTX-specific DAG, ready for instruction scheduling.
52 FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
53 llvm::CodeGenOpt::Level OptLevel) {
54 return new NVPTXDAGToDAGISel(TM, OptLevel);
57 NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
58 CodeGenOpt::Level OptLevel)
59 : SelectionDAGISel(tm, OptLevel),
60 Subtarget(tm.getSubtarget<NVPTXSubtarget>()) {
62 doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1);
63 doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1);
65 (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2);
67 (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2);
69 allowFMA = (FMAContractLevel >= 1);
71 doMulWide = (OptLevel > 0);
74 int NVPTXDAGToDAGISel::getDivF32Level() const {
75 if (UsePrecDivF32.getNumOccurrences() > 0) {
76 // If nvptx-prec-div32=N is used on the command-line, always honor it
79 // Otherwise, use div.approx if fast math is enabled
80 if (TM.Options.UnsafeFPMath)
87 bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
88 if (UsePrecSqrtF32.getNumOccurrences() > 0) {
89 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
90 return UsePrecSqrtF32;
92 // Otherwise, use sqrt.approx if fast math is enabled
93 if (TM.Options.UnsafeFPMath)
100 bool NVPTXDAGToDAGISel::useF32FTZ() const {
101 if (FtzEnabled.getNumOccurrences() > 0) {
102 // If nvptx-f32ftz is used on the command-line, always honor it
105 const Function *F = MF->getFunction();
106 // Otherwise, check for an nvptx-f32ftz attribute on the function
107 if (F->hasFnAttribute("nvptx-f32ftz"))
108 return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex,
110 .getValueAsString() == "true");
116 /// Select - Select instructions not customized! Used for
117 /// expanded, promoted and normal instructions.
118 SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
120 if (N->isMachineOpcode()) {
122 return nullptr; // Already selected.
125 SDNode *ResNode = nullptr;
126 switch (N->getOpcode()) {
128 ResNode = SelectLoad(N);
131 ResNode = SelectStore(N);
133 case NVPTXISD::LoadV2:
134 case NVPTXISD::LoadV4:
135 ResNode = SelectLoadVector(N);
137 case NVPTXISD::LDGV2:
138 case NVPTXISD::LDGV4:
139 case NVPTXISD::LDUV2:
140 case NVPTXISD::LDUV4:
141 ResNode = SelectLDGLDUVector(N);
143 case NVPTXISD::StoreV2:
144 case NVPTXISD::StoreV4:
145 ResNode = SelectStoreVector(N);
147 case NVPTXISD::LoadParam:
148 case NVPTXISD::LoadParamV2:
149 case NVPTXISD::LoadParamV4:
150 ResNode = SelectLoadParam(N);
152 case NVPTXISD::StoreRetval:
153 case NVPTXISD::StoreRetvalV2:
154 case NVPTXISD::StoreRetvalV4:
155 ResNode = SelectStoreRetval(N);
157 case NVPTXISD::StoreParam:
158 case NVPTXISD::StoreParamV2:
159 case NVPTXISD::StoreParamV4:
160 case NVPTXISD::StoreParamS32:
161 case NVPTXISD::StoreParamU32:
162 ResNode = SelectStoreParam(N);
164 case ISD::INTRINSIC_WO_CHAIN:
165 ResNode = SelectIntrinsicNoChain(N);
167 case NVPTXISD::Tex1DFloatI32:
168 case NVPTXISD::Tex1DFloatFloat:
169 case NVPTXISD::Tex1DFloatFloatLevel:
170 case NVPTXISD::Tex1DFloatFloatGrad:
171 case NVPTXISD::Tex1DI32I32:
172 case NVPTXISD::Tex1DI32Float:
173 case NVPTXISD::Tex1DI32FloatLevel:
174 case NVPTXISD::Tex1DI32FloatGrad:
175 case NVPTXISD::Tex1DArrayFloatI32:
176 case NVPTXISD::Tex1DArrayFloatFloat:
177 case NVPTXISD::Tex1DArrayFloatFloatLevel:
178 case NVPTXISD::Tex1DArrayFloatFloatGrad:
179 case NVPTXISD::Tex1DArrayI32I32:
180 case NVPTXISD::Tex1DArrayI32Float:
181 case NVPTXISD::Tex1DArrayI32FloatLevel:
182 case NVPTXISD::Tex1DArrayI32FloatGrad:
183 case NVPTXISD::Tex2DFloatI32:
184 case NVPTXISD::Tex2DFloatFloat:
185 case NVPTXISD::Tex2DFloatFloatLevel:
186 case NVPTXISD::Tex2DFloatFloatGrad:
187 case NVPTXISD::Tex2DI32I32:
188 case NVPTXISD::Tex2DI32Float:
189 case NVPTXISD::Tex2DI32FloatLevel:
190 case NVPTXISD::Tex2DI32FloatGrad:
191 case NVPTXISD::Tex2DArrayFloatI32:
192 case NVPTXISD::Tex2DArrayFloatFloat:
193 case NVPTXISD::Tex2DArrayFloatFloatLevel:
194 case NVPTXISD::Tex2DArrayFloatFloatGrad:
195 case NVPTXISD::Tex2DArrayI32I32:
196 case NVPTXISD::Tex2DArrayI32Float:
197 case NVPTXISD::Tex2DArrayI32FloatLevel:
198 case NVPTXISD::Tex2DArrayI32FloatGrad:
199 case NVPTXISD::Tex3DFloatI32:
200 case NVPTXISD::Tex3DFloatFloat:
201 case NVPTXISD::Tex3DFloatFloatLevel:
202 case NVPTXISD::Tex3DFloatFloatGrad:
203 case NVPTXISD::Tex3DI32I32:
204 case NVPTXISD::Tex3DI32Float:
205 case NVPTXISD::Tex3DI32FloatLevel:
206 case NVPTXISD::Tex3DI32FloatGrad:
207 ResNode = SelectTextureIntrinsic(N);
209 case NVPTXISD::Suld1DI8Trap:
210 case NVPTXISD::Suld1DI16Trap:
211 case NVPTXISD::Suld1DI32Trap:
212 case NVPTXISD::Suld1DV2I8Trap:
213 case NVPTXISD::Suld1DV2I16Trap:
214 case NVPTXISD::Suld1DV2I32Trap:
215 case NVPTXISD::Suld1DV4I8Trap:
216 case NVPTXISD::Suld1DV4I16Trap:
217 case NVPTXISD::Suld1DV4I32Trap:
218 case NVPTXISD::Suld1DArrayI8Trap:
219 case NVPTXISD::Suld1DArrayI16Trap:
220 case NVPTXISD::Suld1DArrayI32Trap:
221 case NVPTXISD::Suld1DArrayV2I8Trap:
222 case NVPTXISD::Suld1DArrayV2I16Trap:
223 case NVPTXISD::Suld1DArrayV2I32Trap:
224 case NVPTXISD::Suld1DArrayV4I8Trap:
225 case NVPTXISD::Suld1DArrayV4I16Trap:
226 case NVPTXISD::Suld1DArrayV4I32Trap:
227 case NVPTXISD::Suld2DI8Trap:
228 case NVPTXISD::Suld2DI16Trap:
229 case NVPTXISD::Suld2DI32Trap:
230 case NVPTXISD::Suld2DV2I8Trap:
231 case NVPTXISD::Suld2DV2I16Trap:
232 case NVPTXISD::Suld2DV2I32Trap:
233 case NVPTXISD::Suld2DV4I8Trap:
234 case NVPTXISD::Suld2DV4I16Trap:
235 case NVPTXISD::Suld2DV4I32Trap:
236 case NVPTXISD::Suld2DArrayI8Trap:
237 case NVPTXISD::Suld2DArrayI16Trap:
238 case NVPTXISD::Suld2DArrayI32Trap:
239 case NVPTXISD::Suld2DArrayV2I8Trap:
240 case NVPTXISD::Suld2DArrayV2I16Trap:
241 case NVPTXISD::Suld2DArrayV2I32Trap:
242 case NVPTXISD::Suld2DArrayV4I8Trap:
243 case NVPTXISD::Suld2DArrayV4I16Trap:
244 case NVPTXISD::Suld2DArrayV4I32Trap:
245 case NVPTXISD::Suld3DI8Trap:
246 case NVPTXISD::Suld3DI16Trap:
247 case NVPTXISD::Suld3DI32Trap:
248 case NVPTXISD::Suld3DV2I8Trap:
249 case NVPTXISD::Suld3DV2I16Trap:
250 case NVPTXISD::Suld3DV2I32Trap:
251 case NVPTXISD::Suld3DV4I8Trap:
252 case NVPTXISD::Suld3DV4I16Trap:
253 case NVPTXISD::Suld3DV4I32Trap:
254 ResNode = SelectSurfaceIntrinsic(N);
256 case ISD::ADDRSPACECAST:
257 ResNode = SelectAddrSpaceCast(N);
264 return SelectCode(N);
267 static unsigned int getCodeAddrSpace(MemSDNode *N,
268 const NVPTXSubtarget &Subtarget) {
269 const Value *Src = N->getMemOperand()->getValue();
272 return NVPTX::PTXLdStInstCode::GENERIC;
274 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
275 switch (PT->getAddressSpace()) {
276 case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
277 case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
278 case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
279 case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
280 case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
281 case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
285 return NVPTX::PTXLdStInstCode::GENERIC;
288 SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
289 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
293 case Intrinsic::nvvm_texsurf_handle_internal:
294 return SelectTexSurfHandle(N);
298 SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
299 // Op 0 is the intrinsic ID
300 SDValue Wrapper = N->getOperand(1);
301 SDValue GlobalVal = Wrapper.getOperand(0);
302 return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
306 SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
307 SDValue Src = N->getOperand(0);
308 AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
309 unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
310 unsigned DstAddrSpace = CastN->getDestAddressSpace();
312 assert(SrcAddrSpace != DstAddrSpace &&
313 "addrspacecast must be between different address spaces");
315 if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
316 // Specific to generic
318 switch (SrcAddrSpace) {
319 default: report_fatal_error("Bad address space in addrspacecast");
320 case ADDRESS_SPACE_GLOBAL:
321 Opc = Subtarget.is64Bit() ? NVPTX::cvta_global_yes_64
322 : NVPTX::cvta_global_yes;
324 case ADDRESS_SPACE_SHARED:
325 Opc = Subtarget.is64Bit() ? NVPTX::cvta_shared_yes_64
326 : NVPTX::cvta_shared_yes;
328 case ADDRESS_SPACE_CONST:
329 Opc = Subtarget.is64Bit() ? NVPTX::cvta_const_yes_64
330 : NVPTX::cvta_const_yes;
332 case ADDRESS_SPACE_LOCAL:
333 Opc = Subtarget.is64Bit() ? NVPTX::cvta_local_yes_64
334 : NVPTX::cvta_local_yes;
337 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
339 // Generic to specific
340 if (SrcAddrSpace != 0)
341 report_fatal_error("Cannot cast between two non-generic address spaces");
343 switch (DstAddrSpace) {
344 default: report_fatal_error("Bad address space in addrspacecast");
345 case ADDRESS_SPACE_GLOBAL:
346 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_global_yes_64
347 : NVPTX::cvta_to_global_yes;
349 case ADDRESS_SPACE_SHARED:
350 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_shared_yes_64
351 : NVPTX::cvta_to_shared_yes;
353 case ADDRESS_SPACE_CONST:
354 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_const_yes_64
355 : NVPTX::cvta_to_const_yes;
357 case ADDRESS_SPACE_LOCAL:
358 Opc = Subtarget.is64Bit() ? NVPTX::cvta_to_local_yes_64
359 : NVPTX::cvta_to_local_yes;
362 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
366 SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
368 LoadSDNode *LD = cast<LoadSDNode>(N);
369 EVT LoadedVT = LD->getMemoryVT();
370 SDNode *NVPTXLD = nullptr;
372 // do not support pre/post inc/dec
376 if (!LoadedVT.isSimple())
379 // Address Space Setting
380 unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget);
383 // - .volatile is only availalble for .global and .shared
384 bool isVolatile = LD->isVolatile();
385 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
386 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
387 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
391 MVT SimpleVT = LoadedVT.getSimpleVT();
392 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
393 if (SimpleVT.isVector()) {
394 unsigned num = SimpleVT.getVectorNumElements();
396 vecType = NVPTX::PTXLdStInstCode::V2;
398 vecType = NVPTX::PTXLdStInstCode::V4;
403 // Type Setting: fromType + fromTypeWidth
405 // Sign : ISD::SEXTLOAD
406 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
408 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
409 MVT ScalarVT = SimpleVT.getScalarType();
410 // Read at least 8 bits (predicates are stored as 8-bit values)
411 unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
412 unsigned int fromType;
413 if ((LD->getExtensionType() == ISD::SEXTLOAD))
414 fromType = NVPTX::PTXLdStInstCode::Signed;
415 else if (ScalarVT.isFloatingPoint())
416 fromType = NVPTX::PTXLdStInstCode::Float;
418 fromType = NVPTX::PTXLdStInstCode::Unsigned;
420 // Create the machine instruction DAG
421 SDValue Chain = N->getOperand(0);
422 SDValue N1 = N->getOperand(1);
424 SDValue Offset, Base;
426 MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
428 if (SelectDirectAddr(N1, Addr)) {
431 Opcode = NVPTX::LD_i8_avar;
434 Opcode = NVPTX::LD_i16_avar;
437 Opcode = NVPTX::LD_i32_avar;
440 Opcode = NVPTX::LD_i64_avar;
443 Opcode = NVPTX::LD_f32_avar;
446 Opcode = NVPTX::LD_f64_avar;
451 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
452 getI32Imm(vecType), getI32Imm(fromType),
453 getI32Imm(fromTypeWidth), Addr, Chain };
454 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
455 } else if (Subtarget.is64Bit()
456 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
457 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
460 Opcode = NVPTX::LD_i8_asi;
463 Opcode = NVPTX::LD_i16_asi;
466 Opcode = NVPTX::LD_i32_asi;
469 Opcode = NVPTX::LD_i64_asi;
472 Opcode = NVPTX::LD_f32_asi;
475 Opcode = NVPTX::LD_f64_asi;
480 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
481 getI32Imm(vecType), getI32Imm(fromType),
482 getI32Imm(fromTypeWidth), Base, Offset, Chain };
483 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
484 } else if (Subtarget.is64Bit()
485 ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
486 : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
487 if (Subtarget.is64Bit()) {
490 Opcode = NVPTX::LD_i8_ari_64;
493 Opcode = NVPTX::LD_i16_ari_64;
496 Opcode = NVPTX::LD_i32_ari_64;
499 Opcode = NVPTX::LD_i64_ari_64;
502 Opcode = NVPTX::LD_f32_ari_64;
505 Opcode = NVPTX::LD_f64_ari_64;
513 Opcode = NVPTX::LD_i8_ari;
516 Opcode = NVPTX::LD_i16_ari;
519 Opcode = NVPTX::LD_i32_ari;
522 Opcode = NVPTX::LD_i64_ari;
525 Opcode = NVPTX::LD_f32_ari;
528 Opcode = NVPTX::LD_f64_ari;
534 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
535 getI32Imm(vecType), getI32Imm(fromType),
536 getI32Imm(fromTypeWidth), Base, Offset, Chain };
537 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
539 if (Subtarget.is64Bit()) {
542 Opcode = NVPTX::LD_i8_areg_64;
545 Opcode = NVPTX::LD_i16_areg_64;
548 Opcode = NVPTX::LD_i32_areg_64;
551 Opcode = NVPTX::LD_i64_areg_64;
554 Opcode = NVPTX::LD_f32_areg_64;
557 Opcode = NVPTX::LD_f64_areg_64;
565 Opcode = NVPTX::LD_i8_areg;
568 Opcode = NVPTX::LD_i16_areg;
571 Opcode = NVPTX::LD_i32_areg;
574 Opcode = NVPTX::LD_i64_areg;
577 Opcode = NVPTX::LD_f32_areg;
580 Opcode = NVPTX::LD_f64_areg;
586 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
587 getI32Imm(vecType), getI32Imm(fromType),
588 getI32Imm(fromTypeWidth), N1, Chain };
589 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
593 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
594 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
595 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
601 SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
603 SDValue Chain = N->getOperand(0);
604 SDValue Op1 = N->getOperand(1);
605 SDValue Addr, Offset, Base;
609 MemSDNode *MemSD = cast<MemSDNode>(N);
610 EVT LoadedVT = MemSD->getMemoryVT();
612 if (!LoadedVT.isSimple())
615 // Address Space Setting
616 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
619 // - .volatile is only availalble for .global and .shared
620 bool IsVolatile = MemSD->isVolatile();
621 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
622 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
623 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
627 MVT SimpleVT = LoadedVT.getSimpleVT();
629 // Type Setting: fromType + fromTypeWidth
631 // Sign : ISD::SEXTLOAD
632 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
634 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
635 MVT ScalarVT = SimpleVT.getScalarType();
636 // Read at least 8 bits (predicates are stored as 8-bit values)
637 unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
638 unsigned int FromType;
639 // The last operand holds the original LoadSDNode::getExtensionType() value
640 unsigned ExtensionType = cast<ConstantSDNode>(
641 N->getOperand(N->getNumOperands() - 1))->getZExtValue();
642 if (ExtensionType == ISD::SEXTLOAD)
643 FromType = NVPTX::PTXLdStInstCode::Signed;
644 else if (ScalarVT.isFloatingPoint())
645 FromType = NVPTX::PTXLdStInstCode::Float;
647 FromType = NVPTX::PTXLdStInstCode::Unsigned;
651 switch (N->getOpcode()) {
652 case NVPTXISD::LoadV2:
653 VecType = NVPTX::PTXLdStInstCode::V2;
655 case NVPTXISD::LoadV4:
656 VecType = NVPTX::PTXLdStInstCode::V4;
662 EVT EltVT = N->getValueType(0);
664 if (SelectDirectAddr(Op1, Addr)) {
665 switch (N->getOpcode()) {
668 case NVPTXISD::LoadV2:
669 switch (EltVT.getSimpleVT().SimpleTy) {
673 Opcode = NVPTX::LDV_i8_v2_avar;
676 Opcode = NVPTX::LDV_i16_v2_avar;
679 Opcode = NVPTX::LDV_i32_v2_avar;
682 Opcode = NVPTX::LDV_i64_v2_avar;
685 Opcode = NVPTX::LDV_f32_v2_avar;
688 Opcode = NVPTX::LDV_f64_v2_avar;
692 case NVPTXISD::LoadV4:
693 switch (EltVT.getSimpleVT().SimpleTy) {
697 Opcode = NVPTX::LDV_i8_v4_avar;
700 Opcode = NVPTX::LDV_i16_v4_avar;
703 Opcode = NVPTX::LDV_i32_v4_avar;
706 Opcode = NVPTX::LDV_f32_v4_avar;
712 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
713 getI32Imm(VecType), getI32Imm(FromType),
714 getI32Imm(FromTypeWidth), Addr, Chain };
715 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
716 } else if (Subtarget.is64Bit()
717 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
718 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
719 switch (N->getOpcode()) {
722 case NVPTXISD::LoadV2:
723 switch (EltVT.getSimpleVT().SimpleTy) {
727 Opcode = NVPTX::LDV_i8_v2_asi;
730 Opcode = NVPTX::LDV_i16_v2_asi;
733 Opcode = NVPTX::LDV_i32_v2_asi;
736 Opcode = NVPTX::LDV_i64_v2_asi;
739 Opcode = NVPTX::LDV_f32_v2_asi;
742 Opcode = NVPTX::LDV_f64_v2_asi;
746 case NVPTXISD::LoadV4:
747 switch (EltVT.getSimpleVT().SimpleTy) {
751 Opcode = NVPTX::LDV_i8_v4_asi;
754 Opcode = NVPTX::LDV_i16_v4_asi;
757 Opcode = NVPTX::LDV_i32_v4_asi;
760 Opcode = NVPTX::LDV_f32_v4_asi;
766 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
767 getI32Imm(VecType), getI32Imm(FromType),
768 getI32Imm(FromTypeWidth), Base, Offset, Chain };
769 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
770 } else if (Subtarget.is64Bit()
771 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
772 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
773 if (Subtarget.is64Bit()) {
774 switch (N->getOpcode()) {
777 case NVPTXISD::LoadV2:
778 switch (EltVT.getSimpleVT().SimpleTy) {
782 Opcode = NVPTX::LDV_i8_v2_ari_64;
785 Opcode = NVPTX::LDV_i16_v2_ari_64;
788 Opcode = NVPTX::LDV_i32_v2_ari_64;
791 Opcode = NVPTX::LDV_i64_v2_ari_64;
794 Opcode = NVPTX::LDV_f32_v2_ari_64;
797 Opcode = NVPTX::LDV_f64_v2_ari_64;
801 case NVPTXISD::LoadV4:
802 switch (EltVT.getSimpleVT().SimpleTy) {
806 Opcode = NVPTX::LDV_i8_v4_ari_64;
809 Opcode = NVPTX::LDV_i16_v4_ari_64;
812 Opcode = NVPTX::LDV_i32_v4_ari_64;
815 Opcode = NVPTX::LDV_f32_v4_ari_64;
821 switch (N->getOpcode()) {
824 case NVPTXISD::LoadV2:
825 switch (EltVT.getSimpleVT().SimpleTy) {
829 Opcode = NVPTX::LDV_i8_v2_ari;
832 Opcode = NVPTX::LDV_i16_v2_ari;
835 Opcode = NVPTX::LDV_i32_v2_ari;
838 Opcode = NVPTX::LDV_i64_v2_ari;
841 Opcode = NVPTX::LDV_f32_v2_ari;
844 Opcode = NVPTX::LDV_f64_v2_ari;
848 case NVPTXISD::LoadV4:
849 switch (EltVT.getSimpleVT().SimpleTy) {
853 Opcode = NVPTX::LDV_i8_v4_ari;
856 Opcode = NVPTX::LDV_i16_v4_ari;
859 Opcode = NVPTX::LDV_i32_v4_ari;
862 Opcode = NVPTX::LDV_f32_v4_ari;
869 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
870 getI32Imm(VecType), getI32Imm(FromType),
871 getI32Imm(FromTypeWidth), Base, Offset, Chain };
873 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
875 if (Subtarget.is64Bit()) {
876 switch (N->getOpcode()) {
879 case NVPTXISD::LoadV2:
880 switch (EltVT.getSimpleVT().SimpleTy) {
884 Opcode = NVPTX::LDV_i8_v2_areg_64;
887 Opcode = NVPTX::LDV_i16_v2_areg_64;
890 Opcode = NVPTX::LDV_i32_v2_areg_64;
893 Opcode = NVPTX::LDV_i64_v2_areg_64;
896 Opcode = NVPTX::LDV_f32_v2_areg_64;
899 Opcode = NVPTX::LDV_f64_v2_areg_64;
903 case NVPTXISD::LoadV4:
904 switch (EltVT.getSimpleVT().SimpleTy) {
908 Opcode = NVPTX::LDV_i8_v4_areg_64;
911 Opcode = NVPTX::LDV_i16_v4_areg_64;
914 Opcode = NVPTX::LDV_i32_v4_areg_64;
917 Opcode = NVPTX::LDV_f32_v4_areg_64;
923 switch (N->getOpcode()) {
926 case NVPTXISD::LoadV2:
927 switch (EltVT.getSimpleVT().SimpleTy) {
931 Opcode = NVPTX::LDV_i8_v2_areg;
934 Opcode = NVPTX::LDV_i16_v2_areg;
937 Opcode = NVPTX::LDV_i32_v2_areg;
940 Opcode = NVPTX::LDV_i64_v2_areg;
943 Opcode = NVPTX::LDV_f32_v2_areg;
946 Opcode = NVPTX::LDV_f64_v2_areg;
950 case NVPTXISD::LoadV4:
951 switch (EltVT.getSimpleVT().SimpleTy) {
955 Opcode = NVPTX::LDV_i8_v4_areg;
958 Opcode = NVPTX::LDV_i16_v4_areg;
961 Opcode = NVPTX::LDV_i32_v4_areg;
964 Opcode = NVPTX::LDV_f32_v4_areg;
971 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
972 getI32Imm(VecType), getI32Imm(FromType),
973 getI32Imm(FromTypeWidth), Op1, Chain };
974 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
977 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
978 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
979 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
984 SDNode *NVPTXDAGToDAGISel::SelectLDGLDUVector(SDNode *N) {
986 SDValue Chain = N->getOperand(0);
987 SDValue Op1 = N->getOperand(1);
991 MemSDNode *Mem = cast<MemSDNode>(N);
992 SDValue Base, Offset, Addr;
994 EVT EltVT = Mem->getMemoryVT().getVectorElementType();
996 if (SelectDirectAddr(Op1, Addr)) {
997 switch (N->getOpcode()) {
1000 case NVPTXISD::LDGV2:
1001 switch (EltVT.getSimpleVT().SimpleTy) {
1005 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
1008 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
1011 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
1014 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
1017 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
1020 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
1024 case NVPTXISD::LDUV2:
1025 switch (EltVT.getSimpleVT().SimpleTy) {
1029 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
1032 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
1035 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
1038 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
1041 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
1044 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
1048 case NVPTXISD::LDGV4:
1049 switch (EltVT.getSimpleVT().SimpleTy) {
1053 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
1056 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
1059 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
1062 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
1066 case NVPTXISD::LDUV4:
1067 switch (EltVT.getSimpleVT().SimpleTy) {
1071 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
1074 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
1077 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
1080 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
1086 SDValue Ops[] = { Addr, Chain };
1087 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
1088 } else if (Subtarget.is64Bit()
1089 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1090 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
1091 if (Subtarget.is64Bit()) {
1092 switch (N->getOpcode()) {
1095 case NVPTXISD::LDGV2:
1096 switch (EltVT.getSimpleVT().SimpleTy) {
1100 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
1103 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
1106 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
1109 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
1112 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
1115 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
1119 case NVPTXISD::LDUV2:
1120 switch (EltVT.getSimpleVT().SimpleTy) {
1124 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
1127 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
1130 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
1133 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
1136 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
1139 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
1143 case NVPTXISD::LDGV4:
1144 switch (EltVT.getSimpleVT().SimpleTy) {
1148 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
1151 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
1154 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
1157 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
1161 case NVPTXISD::LDUV4:
1162 switch (EltVT.getSimpleVT().SimpleTy) {
1166 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
1169 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
1172 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
1175 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
1181 switch (N->getOpcode()) {
1184 case NVPTXISD::LDGV2:
1185 switch (EltVT.getSimpleVT().SimpleTy) {
1189 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1192 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1195 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
1198 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
1201 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
1204 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
1208 case NVPTXISD::LDUV2:
1209 switch (EltVT.getSimpleVT().SimpleTy) {
1213 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1216 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1219 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
1222 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
1225 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
1228 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
1232 case NVPTXISD::LDGV4:
1233 switch (EltVT.getSimpleVT().SimpleTy) {
1237 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1240 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1243 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
1246 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
1250 case NVPTXISD::LDUV4:
1251 switch (EltVT.getSimpleVT().SimpleTy) {
1255 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1258 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1261 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
1264 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
1271 SDValue Ops[] = { Base, Offset, Chain };
1273 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
1275 if (Subtarget.is64Bit()) {
1276 switch (N->getOpcode()) {
1279 case NVPTXISD::LDGV2:
1280 switch (EltVT.getSimpleVT().SimpleTy) {
1284 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1287 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1290 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
1293 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
1296 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
1299 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
1303 case NVPTXISD::LDUV2:
1304 switch (EltVT.getSimpleVT().SimpleTy) {
1308 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1311 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1314 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
1317 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
1320 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
1323 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
1327 case NVPTXISD::LDGV4:
1328 switch (EltVT.getSimpleVT().SimpleTy) {
1332 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1335 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1338 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
1341 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
1345 case NVPTXISD::LDUV4:
1346 switch (EltVT.getSimpleVT().SimpleTy) {
1350 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1353 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1356 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
1359 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
1365 switch (N->getOpcode()) {
1368 case NVPTXISD::LDGV2:
1369 switch (EltVT.getSimpleVT().SimpleTy) {
1373 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1376 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
1379 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
1382 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
1385 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
1388 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
1392 case NVPTXISD::LDUV2:
1393 switch (EltVT.getSimpleVT().SimpleTy) {
1397 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
1400 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
1403 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
1406 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
1409 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
1412 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
1416 case NVPTXISD::LDGV4:
1417 switch (EltVT.getSimpleVT().SimpleTy) {
1421 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
1424 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
1427 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
1430 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
1434 case NVPTXISD::LDUV4:
1435 switch (EltVT.getSimpleVT().SimpleTy) {
1439 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
1442 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
1445 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
1448 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
1455 SDValue Ops[] = { Op1, Chain };
1456 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
1459 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1460 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1461 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1466 SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
1468 StoreSDNode *ST = cast<StoreSDNode>(N);
1469 EVT StoreVT = ST->getMemoryVT();
1470 SDNode *NVPTXST = nullptr;
1472 // do not support pre/post inc/dec
1473 if (ST->isIndexed())
1476 if (!StoreVT.isSimple())
1479 // Address Space Setting
1480 unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget);
1483 // - .volatile is only availalble for .global and .shared
1484 bool isVolatile = ST->isVolatile();
1485 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
1486 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
1487 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
1491 MVT SimpleVT = StoreVT.getSimpleVT();
1492 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
1493 if (SimpleVT.isVector()) {
1494 unsigned num = SimpleVT.getVectorNumElements();
1496 vecType = NVPTX::PTXLdStInstCode::V2;
1498 vecType = NVPTX::PTXLdStInstCode::V4;
1503 // Type Setting: toType + toTypeWidth
1504 // - for integer type, always use 'u'
1506 MVT ScalarVT = SimpleVT.getScalarType();
1507 unsigned toTypeWidth = ScalarVT.getSizeInBits();
1508 unsigned int toType;
1509 if (ScalarVT.isFloatingPoint())
1510 toType = NVPTX::PTXLdStInstCode::Float;
1512 toType = NVPTX::PTXLdStInstCode::Unsigned;
1514 // Create the machine instruction DAG
1515 SDValue Chain = N->getOperand(0);
1516 SDValue N1 = N->getOperand(1);
1517 SDValue N2 = N->getOperand(2);
1519 SDValue Offset, Base;
1521 MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
1523 if (SelectDirectAddr(N2, Addr)) {
1526 Opcode = NVPTX::ST_i8_avar;
1529 Opcode = NVPTX::ST_i16_avar;
1532 Opcode = NVPTX::ST_i32_avar;
1535 Opcode = NVPTX::ST_i64_avar;
1538 Opcode = NVPTX::ST_f32_avar;
1541 Opcode = NVPTX::ST_f64_avar;
1546 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1547 getI32Imm(vecType), getI32Imm(toType),
1548 getI32Imm(toTypeWidth), Addr, Chain };
1549 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1550 } else if (Subtarget.is64Bit()
1551 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
1552 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
1555 Opcode = NVPTX::ST_i8_asi;
1558 Opcode = NVPTX::ST_i16_asi;
1561 Opcode = NVPTX::ST_i32_asi;
1564 Opcode = NVPTX::ST_i64_asi;
1567 Opcode = NVPTX::ST_f32_asi;
1570 Opcode = NVPTX::ST_f64_asi;
1575 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1576 getI32Imm(vecType), getI32Imm(toType),
1577 getI32Imm(toTypeWidth), Base, Offset, Chain };
1578 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1579 } else if (Subtarget.is64Bit()
1580 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
1581 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
1582 if (Subtarget.is64Bit()) {
1585 Opcode = NVPTX::ST_i8_ari_64;
1588 Opcode = NVPTX::ST_i16_ari_64;
1591 Opcode = NVPTX::ST_i32_ari_64;
1594 Opcode = NVPTX::ST_i64_ari_64;
1597 Opcode = NVPTX::ST_f32_ari_64;
1600 Opcode = NVPTX::ST_f64_ari_64;
1608 Opcode = NVPTX::ST_i8_ari;
1611 Opcode = NVPTX::ST_i16_ari;
1614 Opcode = NVPTX::ST_i32_ari;
1617 Opcode = NVPTX::ST_i64_ari;
1620 Opcode = NVPTX::ST_f32_ari;
1623 Opcode = NVPTX::ST_f64_ari;
1629 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1630 getI32Imm(vecType), getI32Imm(toType),
1631 getI32Imm(toTypeWidth), Base, Offset, Chain };
1632 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1634 if (Subtarget.is64Bit()) {
1637 Opcode = NVPTX::ST_i8_areg_64;
1640 Opcode = NVPTX::ST_i16_areg_64;
1643 Opcode = NVPTX::ST_i32_areg_64;
1646 Opcode = NVPTX::ST_i64_areg_64;
1649 Opcode = NVPTX::ST_f32_areg_64;
1652 Opcode = NVPTX::ST_f64_areg_64;
1660 Opcode = NVPTX::ST_i8_areg;
1663 Opcode = NVPTX::ST_i16_areg;
1666 Opcode = NVPTX::ST_i32_areg;
1669 Opcode = NVPTX::ST_i64_areg;
1672 Opcode = NVPTX::ST_f32_areg;
1675 Opcode = NVPTX::ST_f64_areg;
1681 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
1682 getI32Imm(vecType), getI32Imm(toType),
1683 getI32Imm(toTypeWidth), N2, Chain };
1684 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
1688 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1689 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1690 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
1696 SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
1697 SDValue Chain = N->getOperand(0);
1698 SDValue Op1 = N->getOperand(1);
1699 SDValue Addr, Offset, Base;
1703 EVT EltVT = Op1.getValueType();
1704 MemSDNode *MemSD = cast<MemSDNode>(N);
1705 EVT StoreVT = MemSD->getMemoryVT();
1707 // Address Space Setting
1708 unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
1710 if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
1711 report_fatal_error("Cannot store to pointer that points to constant "
1716 // - .volatile is only availalble for .global and .shared
1717 bool IsVolatile = MemSD->isVolatile();
1718 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
1719 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
1720 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
1723 // Type Setting: toType + toTypeWidth
1724 // - for integer type, always use 'u'
1725 assert(StoreVT.isSimple() && "Store value is not simple");
1726 MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
1727 unsigned ToTypeWidth = ScalarVT.getSizeInBits();
1729 if (ScalarVT.isFloatingPoint())
1730 ToType = NVPTX::PTXLdStInstCode::Float;
1732 ToType = NVPTX::PTXLdStInstCode::Unsigned;
1734 SmallVector<SDValue, 12> StOps;
1738 switch (N->getOpcode()) {
1739 case NVPTXISD::StoreV2:
1740 VecType = NVPTX::PTXLdStInstCode::V2;
1741 StOps.push_back(N->getOperand(1));
1742 StOps.push_back(N->getOperand(2));
1743 N2 = N->getOperand(3);
1745 case NVPTXISD::StoreV4:
1746 VecType = NVPTX::PTXLdStInstCode::V4;
1747 StOps.push_back(N->getOperand(1));
1748 StOps.push_back(N->getOperand(2));
1749 StOps.push_back(N->getOperand(3));
1750 StOps.push_back(N->getOperand(4));
1751 N2 = N->getOperand(5);
1757 StOps.push_back(getI32Imm(IsVolatile));
1758 StOps.push_back(getI32Imm(CodeAddrSpace));
1759 StOps.push_back(getI32Imm(VecType));
1760 StOps.push_back(getI32Imm(ToType));
1761 StOps.push_back(getI32Imm(ToTypeWidth));
1763 if (SelectDirectAddr(N2, Addr)) {
1764 switch (N->getOpcode()) {
1767 case NVPTXISD::StoreV2:
1768 switch (EltVT.getSimpleVT().SimpleTy) {
1772 Opcode = NVPTX::STV_i8_v2_avar;
1775 Opcode = NVPTX::STV_i16_v2_avar;
1778 Opcode = NVPTX::STV_i32_v2_avar;
1781 Opcode = NVPTX::STV_i64_v2_avar;
1784 Opcode = NVPTX::STV_f32_v2_avar;
1787 Opcode = NVPTX::STV_f64_v2_avar;
1791 case NVPTXISD::StoreV4:
1792 switch (EltVT.getSimpleVT().SimpleTy) {
1796 Opcode = NVPTX::STV_i8_v4_avar;
1799 Opcode = NVPTX::STV_i16_v4_avar;
1802 Opcode = NVPTX::STV_i32_v4_avar;
1805 Opcode = NVPTX::STV_f32_v4_avar;
1810 StOps.push_back(Addr);
1811 } else if (Subtarget.is64Bit()
1812 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
1813 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
1814 switch (N->getOpcode()) {
1817 case NVPTXISD::StoreV2:
1818 switch (EltVT.getSimpleVT().SimpleTy) {
1822 Opcode = NVPTX::STV_i8_v2_asi;
1825 Opcode = NVPTX::STV_i16_v2_asi;
1828 Opcode = NVPTX::STV_i32_v2_asi;
1831 Opcode = NVPTX::STV_i64_v2_asi;
1834 Opcode = NVPTX::STV_f32_v2_asi;
1837 Opcode = NVPTX::STV_f64_v2_asi;
1841 case NVPTXISD::StoreV4:
1842 switch (EltVT.getSimpleVT().SimpleTy) {
1846 Opcode = NVPTX::STV_i8_v4_asi;
1849 Opcode = NVPTX::STV_i16_v4_asi;
1852 Opcode = NVPTX::STV_i32_v4_asi;
1855 Opcode = NVPTX::STV_f32_v4_asi;
1860 StOps.push_back(Base);
1861 StOps.push_back(Offset);
1862 } else if (Subtarget.is64Bit()
1863 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
1864 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
1865 if (Subtarget.is64Bit()) {
1866 switch (N->getOpcode()) {
1869 case NVPTXISD::StoreV2:
1870 switch (EltVT.getSimpleVT().SimpleTy) {
1874 Opcode = NVPTX::STV_i8_v2_ari_64;
1877 Opcode = NVPTX::STV_i16_v2_ari_64;
1880 Opcode = NVPTX::STV_i32_v2_ari_64;
1883 Opcode = NVPTX::STV_i64_v2_ari_64;
1886 Opcode = NVPTX::STV_f32_v2_ari_64;
1889 Opcode = NVPTX::STV_f64_v2_ari_64;
1893 case NVPTXISD::StoreV4:
1894 switch (EltVT.getSimpleVT().SimpleTy) {
1898 Opcode = NVPTX::STV_i8_v4_ari_64;
1901 Opcode = NVPTX::STV_i16_v4_ari_64;
1904 Opcode = NVPTX::STV_i32_v4_ari_64;
1907 Opcode = NVPTX::STV_f32_v4_ari_64;
1913 switch (N->getOpcode()) {
1916 case NVPTXISD::StoreV2:
1917 switch (EltVT.getSimpleVT().SimpleTy) {
1921 Opcode = NVPTX::STV_i8_v2_ari;
1924 Opcode = NVPTX::STV_i16_v2_ari;
1927 Opcode = NVPTX::STV_i32_v2_ari;
1930 Opcode = NVPTX::STV_i64_v2_ari;
1933 Opcode = NVPTX::STV_f32_v2_ari;
1936 Opcode = NVPTX::STV_f64_v2_ari;
1940 case NVPTXISD::StoreV4:
1941 switch (EltVT.getSimpleVT().SimpleTy) {
1945 Opcode = NVPTX::STV_i8_v4_ari;
1948 Opcode = NVPTX::STV_i16_v4_ari;
1951 Opcode = NVPTX::STV_i32_v4_ari;
1954 Opcode = NVPTX::STV_f32_v4_ari;
1960 StOps.push_back(Base);
1961 StOps.push_back(Offset);
1963 if (Subtarget.is64Bit()) {
1964 switch (N->getOpcode()) {
1967 case NVPTXISD::StoreV2:
1968 switch (EltVT.getSimpleVT().SimpleTy) {
1972 Opcode = NVPTX::STV_i8_v2_areg_64;
1975 Opcode = NVPTX::STV_i16_v2_areg_64;
1978 Opcode = NVPTX::STV_i32_v2_areg_64;
1981 Opcode = NVPTX::STV_i64_v2_areg_64;
1984 Opcode = NVPTX::STV_f32_v2_areg_64;
1987 Opcode = NVPTX::STV_f64_v2_areg_64;
1991 case NVPTXISD::StoreV4:
1992 switch (EltVT.getSimpleVT().SimpleTy) {
1996 Opcode = NVPTX::STV_i8_v4_areg_64;
1999 Opcode = NVPTX::STV_i16_v4_areg_64;
2002 Opcode = NVPTX::STV_i32_v4_areg_64;
2005 Opcode = NVPTX::STV_f32_v4_areg_64;
2011 switch (N->getOpcode()) {
2014 case NVPTXISD::StoreV2:
2015 switch (EltVT.getSimpleVT().SimpleTy) {
2019 Opcode = NVPTX::STV_i8_v2_areg;
2022 Opcode = NVPTX::STV_i16_v2_areg;
2025 Opcode = NVPTX::STV_i32_v2_areg;
2028 Opcode = NVPTX::STV_i64_v2_areg;
2031 Opcode = NVPTX::STV_f32_v2_areg;
2034 Opcode = NVPTX::STV_f64_v2_areg;
2038 case NVPTXISD::StoreV4:
2039 switch (EltVT.getSimpleVT().SimpleTy) {
2043 Opcode = NVPTX::STV_i8_v4_areg;
2046 Opcode = NVPTX::STV_i16_v4_areg;
2049 Opcode = NVPTX::STV_i32_v4_areg;
2052 Opcode = NVPTX::STV_f32_v4_areg;
2058 StOps.push_back(N2);
2061 StOps.push_back(Chain);
2063 ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
2065 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2066 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2067 cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2072 SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
2073 SDValue Chain = Node->getOperand(0);
2074 SDValue Offset = Node->getOperand(2);
2075 SDValue Flag = Node->getOperand(3);
2077 MemSDNode *Mem = cast<MemSDNode>(Node);
2080 switch (Node->getOpcode()) {
2083 case NVPTXISD::LoadParam:
2086 case NVPTXISD::LoadParamV2:
2089 case NVPTXISD::LoadParamV4:
2094 EVT EltVT = Node->getValueType(0);
2095 EVT MemVT = Mem->getMemoryVT();
2103 switch (MemVT.getSimpleVT().SimpleTy) {
2107 Opc = NVPTX::LoadParamMemI8;
2110 Opc = NVPTX::LoadParamMemI8;
2113 Opc = NVPTX::LoadParamMemI16;
2116 Opc = NVPTX::LoadParamMemI32;
2119 Opc = NVPTX::LoadParamMemI64;
2122 Opc = NVPTX::LoadParamMemF32;
2125 Opc = NVPTX::LoadParamMemF64;
2130 switch (MemVT.getSimpleVT().SimpleTy) {
2134 Opc = NVPTX::LoadParamMemV2I8;
2137 Opc = NVPTX::LoadParamMemV2I8;
2140 Opc = NVPTX::LoadParamMemV2I16;
2143 Opc = NVPTX::LoadParamMemV2I32;
2146 Opc = NVPTX::LoadParamMemV2I64;
2149 Opc = NVPTX::LoadParamMemV2F32;
2152 Opc = NVPTX::LoadParamMemV2F64;
2157 switch (MemVT.getSimpleVT().SimpleTy) {
2161 Opc = NVPTX::LoadParamMemV4I8;
2164 Opc = NVPTX::LoadParamMemV4I8;
2167 Opc = NVPTX::LoadParamMemV4I16;
2170 Opc = NVPTX::LoadParamMemV4I32;
2173 Opc = NVPTX::LoadParamMemV4F32;
2181 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
2182 } else if (VecSize == 2) {
2183 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
2185 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
2186 VTs = CurDAG->getVTList(EVTs);
2189 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2191 SmallVector<SDValue, 2> Ops;
2192 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2193 Ops.push_back(Chain);
2194 Ops.push_back(Flag);
2197 CurDAG->getMachineNode(Opc, DL, VTs, Ops);
2201 SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
2203 SDValue Chain = N->getOperand(0);
2204 SDValue Offset = N->getOperand(1);
2205 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2206 MemSDNode *Mem = cast<MemSDNode>(N);
2208 // How many elements do we have?
2209 unsigned NumElts = 1;
2210 switch (N->getOpcode()) {
2213 case NVPTXISD::StoreRetval:
2216 case NVPTXISD::StoreRetvalV2:
2219 case NVPTXISD::StoreRetvalV4:
2224 // Build vector of operands
2225 SmallVector<SDValue, 6> Ops;
2226 for (unsigned i = 0; i < NumElts; ++i)
2227 Ops.push_back(N->getOperand(i + 2));
2228 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2229 Ops.push_back(Chain);
2231 // Determine target opcode
2232 // If we have an i1, use an 8-bit store. The lowering code in
2233 // NVPTXISelLowering will have already emitted an upcast.
2234 unsigned Opcode = 0;
2239 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2243 Opcode = NVPTX::StoreRetvalI8;
2246 Opcode = NVPTX::StoreRetvalI8;
2249 Opcode = NVPTX::StoreRetvalI16;
2252 Opcode = NVPTX::StoreRetvalI32;
2255 Opcode = NVPTX::StoreRetvalI64;
2258 Opcode = NVPTX::StoreRetvalF32;
2261 Opcode = NVPTX::StoreRetvalF64;
2266 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2270 Opcode = NVPTX::StoreRetvalV2I8;
2273 Opcode = NVPTX::StoreRetvalV2I8;
2276 Opcode = NVPTX::StoreRetvalV2I16;
2279 Opcode = NVPTX::StoreRetvalV2I32;
2282 Opcode = NVPTX::StoreRetvalV2I64;
2285 Opcode = NVPTX::StoreRetvalV2F32;
2288 Opcode = NVPTX::StoreRetvalV2F64;
2293 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2297 Opcode = NVPTX::StoreRetvalV4I8;
2300 Opcode = NVPTX::StoreRetvalV4I8;
2303 Opcode = NVPTX::StoreRetvalV4I16;
2306 Opcode = NVPTX::StoreRetvalV4I32;
2309 Opcode = NVPTX::StoreRetvalV4F32;
2316 CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
2317 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2318 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2319 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2324 SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
2326 SDValue Chain = N->getOperand(0);
2327 SDValue Param = N->getOperand(1);
2328 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
2329 SDValue Offset = N->getOperand(2);
2330 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2331 MemSDNode *Mem = cast<MemSDNode>(N);
2332 SDValue Flag = N->getOperand(N->getNumOperands() - 1);
2334 // How many elements do we have?
2335 unsigned NumElts = 1;
2336 switch (N->getOpcode()) {
2339 case NVPTXISD::StoreParamU32:
2340 case NVPTXISD::StoreParamS32:
2341 case NVPTXISD::StoreParam:
2344 case NVPTXISD::StoreParamV2:
2347 case NVPTXISD::StoreParamV4:
2352 // Build vector of operands
2353 SmallVector<SDValue, 8> Ops;
2354 for (unsigned i = 0; i < NumElts; ++i)
2355 Ops.push_back(N->getOperand(i + 3));
2356 Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
2357 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2358 Ops.push_back(Chain);
2359 Ops.push_back(Flag);
2361 // Determine target opcode
2362 // If we have an i1, use an 8-bit store. The lowering code in
2363 // NVPTXISelLowering will have already emitted an upcast.
2364 unsigned Opcode = 0;
2365 switch (N->getOpcode()) {
2371 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2375 Opcode = NVPTX::StoreParamI8;
2378 Opcode = NVPTX::StoreParamI8;
2381 Opcode = NVPTX::StoreParamI16;
2384 Opcode = NVPTX::StoreParamI32;
2387 Opcode = NVPTX::StoreParamI64;
2390 Opcode = NVPTX::StoreParamF32;
2393 Opcode = NVPTX::StoreParamF64;
2398 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2402 Opcode = NVPTX::StoreParamV2I8;
2405 Opcode = NVPTX::StoreParamV2I8;
2408 Opcode = NVPTX::StoreParamV2I16;
2411 Opcode = NVPTX::StoreParamV2I32;
2414 Opcode = NVPTX::StoreParamV2I64;
2417 Opcode = NVPTX::StoreParamV2F32;
2420 Opcode = NVPTX::StoreParamV2F64;
2425 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2429 Opcode = NVPTX::StoreParamV4I8;
2432 Opcode = NVPTX::StoreParamV4I8;
2435 Opcode = NVPTX::StoreParamV4I16;
2438 Opcode = NVPTX::StoreParamV4I32;
2441 Opcode = NVPTX::StoreParamV4F32;
2447 // Special case: if we have a sign-extend/zero-extend node, insert the
2448 // conversion instruction first, and use that as the value operand to
2449 // the selected StoreParam node.
2450 case NVPTXISD::StoreParamU32: {
2451 Opcode = NVPTX::StoreParamI32;
2452 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2454 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
2455 MVT::i32, Ops[0], CvtNone);
2456 Ops[0] = SDValue(Cvt, 0);
2459 case NVPTXISD::StoreParamS32: {
2460 Opcode = NVPTX::StoreParamI32;
2461 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2463 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
2464 MVT::i32, Ops[0], CvtNone);
2465 Ops[0] = SDValue(Cvt, 0);
2470 SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
2472 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
2473 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2474 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2475 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2480 SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
2481 SDValue Chain = N->getOperand(0);
2482 SDValue TexRef = N->getOperand(1);
2483 SDValue SampRef = N->getOperand(2);
2484 SDNode *Ret = nullptr;
2486 SmallVector<SDValue, 8> Ops;
2488 switch (N->getOpcode()) {
2489 default: return nullptr;
2490 case NVPTXISD::Tex1DFloatI32:
2491 Opc = NVPTX::TEX_1D_F32_I32;
2493 case NVPTXISD::Tex1DFloatFloat:
2494 Opc = NVPTX::TEX_1D_F32_F32;
2496 case NVPTXISD::Tex1DFloatFloatLevel:
2497 Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
2499 case NVPTXISD::Tex1DFloatFloatGrad:
2500 Opc = NVPTX::TEX_1D_F32_F32_GRAD;
2502 case NVPTXISD::Tex1DI32I32:
2503 Opc = NVPTX::TEX_1D_I32_I32;
2505 case NVPTXISD::Tex1DI32Float:
2506 Opc = NVPTX::TEX_1D_I32_F32;
2508 case NVPTXISD::Tex1DI32FloatLevel:
2509 Opc = NVPTX::TEX_1D_I32_F32_LEVEL;
2511 case NVPTXISD::Tex1DI32FloatGrad:
2512 Opc = NVPTX::TEX_1D_I32_F32_GRAD;
2514 case NVPTXISD::Tex1DArrayFloatI32:
2515 Opc = NVPTX::TEX_1D_ARRAY_F32_I32;
2517 case NVPTXISD::Tex1DArrayFloatFloat:
2518 Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
2520 case NVPTXISD::Tex1DArrayFloatFloatLevel:
2521 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
2523 case NVPTXISD::Tex1DArrayFloatFloatGrad:
2524 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
2526 case NVPTXISD::Tex1DArrayI32I32:
2527 Opc = NVPTX::TEX_1D_ARRAY_I32_I32;
2529 case NVPTXISD::Tex1DArrayI32Float:
2530 Opc = NVPTX::TEX_1D_ARRAY_I32_F32;
2532 case NVPTXISD::Tex1DArrayI32FloatLevel:
2533 Opc = NVPTX::TEX_1D_ARRAY_I32_F32_LEVEL;
2535 case NVPTXISD::Tex1DArrayI32FloatGrad:
2536 Opc = NVPTX::TEX_1D_ARRAY_I32_F32_GRAD;
2538 case NVPTXISD::Tex2DFloatI32:
2539 Opc = NVPTX::TEX_2D_F32_I32;
2541 case NVPTXISD::Tex2DFloatFloat:
2542 Opc = NVPTX::TEX_2D_F32_F32;
2544 case NVPTXISD::Tex2DFloatFloatLevel:
2545 Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
2547 case NVPTXISD::Tex2DFloatFloatGrad:
2548 Opc = NVPTX::TEX_2D_F32_F32_GRAD;
2550 case NVPTXISD::Tex2DI32I32:
2551 Opc = NVPTX::TEX_2D_I32_I32;
2553 case NVPTXISD::Tex2DI32Float:
2554 Opc = NVPTX::TEX_2D_I32_F32;
2556 case NVPTXISD::Tex2DI32FloatLevel:
2557 Opc = NVPTX::TEX_2D_I32_F32_LEVEL;
2559 case NVPTXISD::Tex2DI32FloatGrad:
2560 Opc = NVPTX::TEX_2D_I32_F32_GRAD;
2562 case NVPTXISD::Tex2DArrayFloatI32:
2563 Opc = NVPTX::TEX_2D_ARRAY_F32_I32;
2565 case NVPTXISD::Tex2DArrayFloatFloat:
2566 Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
2568 case NVPTXISD::Tex2DArrayFloatFloatLevel:
2569 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
2571 case NVPTXISD::Tex2DArrayFloatFloatGrad:
2572 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
2574 case NVPTXISD::Tex2DArrayI32I32:
2575 Opc = NVPTX::TEX_2D_ARRAY_I32_I32;
2577 case NVPTXISD::Tex2DArrayI32Float:
2578 Opc = NVPTX::TEX_2D_ARRAY_I32_F32;
2580 case NVPTXISD::Tex2DArrayI32FloatLevel:
2581 Opc = NVPTX::TEX_2D_ARRAY_I32_F32_LEVEL;
2583 case NVPTXISD::Tex2DArrayI32FloatGrad:
2584 Opc = NVPTX::TEX_2D_ARRAY_I32_F32_GRAD;
2586 case NVPTXISD::Tex3DFloatI32:
2587 Opc = NVPTX::TEX_3D_F32_I32;
2589 case NVPTXISD::Tex3DFloatFloat:
2590 Opc = NVPTX::TEX_3D_F32_F32;
2592 case NVPTXISD::Tex3DFloatFloatLevel:
2593 Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
2595 case NVPTXISD::Tex3DFloatFloatGrad:
2596 Opc = NVPTX::TEX_3D_F32_F32_GRAD;
2598 case NVPTXISD::Tex3DI32I32:
2599 Opc = NVPTX::TEX_3D_I32_I32;
2601 case NVPTXISD::Tex3DI32Float:
2602 Opc = NVPTX::TEX_3D_I32_F32;
2604 case NVPTXISD::Tex3DI32FloatLevel:
2605 Opc = NVPTX::TEX_3D_I32_F32_LEVEL;
2607 case NVPTXISD::Tex3DI32FloatGrad:
2608 Opc = NVPTX::TEX_3D_I32_F32_GRAD;
2612 Ops.push_back(TexRef);
2613 Ops.push_back(SampRef);
2615 // Copy over indices
2616 for (unsigned i = 3; i < N->getNumOperands(); ++i) {
2617 Ops.push_back(N->getOperand(i));
2620 Ops.push_back(Chain);
2621 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
2625 SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
2626 SDValue Chain = N->getOperand(0);
2627 SDValue TexHandle = N->getOperand(1);
2628 SDNode *Ret = nullptr;
2630 SmallVector<SDValue, 8> Ops;
2631 switch (N->getOpcode()) {
2632 default: return nullptr;
2633 case NVPTXISD::Suld1DI8Trap:
2634 Opc = NVPTX::SULD_1D_I8_TRAP;
2635 Ops.push_back(TexHandle);
2636 Ops.push_back(N->getOperand(2));
2637 Ops.push_back(Chain);
2639 case NVPTXISD::Suld1DI16Trap:
2640 Opc = NVPTX::SULD_1D_I16_TRAP;
2641 Ops.push_back(TexHandle);
2642 Ops.push_back(N->getOperand(2));
2643 Ops.push_back(Chain);
2645 case NVPTXISD::Suld1DI32Trap:
2646 Opc = NVPTX::SULD_1D_I32_TRAP;
2647 Ops.push_back(TexHandle);
2648 Ops.push_back(N->getOperand(2));
2649 Ops.push_back(Chain);
2651 case NVPTXISD::Suld1DV2I8Trap:
2652 Opc = NVPTX::SULD_1D_V2I8_TRAP;
2653 Ops.push_back(TexHandle);
2654 Ops.push_back(N->getOperand(2));
2655 Ops.push_back(Chain);
2657 case NVPTXISD::Suld1DV2I16Trap:
2658 Opc = NVPTX::SULD_1D_V2I16_TRAP;
2659 Ops.push_back(TexHandle);
2660 Ops.push_back(N->getOperand(2));
2661 Ops.push_back(Chain);
2663 case NVPTXISD::Suld1DV2I32Trap:
2664 Opc = NVPTX::SULD_1D_V2I32_TRAP;
2665 Ops.push_back(TexHandle);
2666 Ops.push_back(N->getOperand(2));
2667 Ops.push_back(Chain);
2669 case NVPTXISD::Suld1DV4I8Trap:
2670 Opc = NVPTX::SULD_1D_V4I8_TRAP;
2671 Ops.push_back(TexHandle);
2672 Ops.push_back(N->getOperand(2));
2673 Ops.push_back(Chain);
2675 case NVPTXISD::Suld1DV4I16Trap:
2676 Opc = NVPTX::SULD_1D_V4I16_TRAP;
2677 Ops.push_back(TexHandle);
2678 Ops.push_back(N->getOperand(2));
2679 Ops.push_back(Chain);
2681 case NVPTXISD::Suld1DV4I32Trap:
2682 Opc = NVPTX::SULD_1D_V4I32_TRAP;
2683 Ops.push_back(TexHandle);
2684 Ops.push_back(N->getOperand(2));
2685 Ops.push_back(Chain);
2687 case NVPTXISD::Suld1DArrayI8Trap:
2688 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
2689 Ops.push_back(TexHandle);
2690 Ops.push_back(N->getOperand(2));
2691 Ops.push_back(N->getOperand(3));
2692 Ops.push_back(Chain);
2694 case NVPTXISD::Suld1DArrayI16Trap:
2695 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
2696 Ops.push_back(TexHandle);
2697 Ops.push_back(N->getOperand(2));
2698 Ops.push_back(N->getOperand(3));
2699 Ops.push_back(Chain);
2701 case NVPTXISD::Suld1DArrayI32Trap:
2702 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
2703 Ops.push_back(TexHandle);
2704 Ops.push_back(N->getOperand(2));
2705 Ops.push_back(N->getOperand(3));
2706 Ops.push_back(Chain);
2708 case NVPTXISD::Suld1DArrayV2I8Trap:
2709 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
2710 Ops.push_back(TexHandle);
2711 Ops.push_back(N->getOperand(2));
2712 Ops.push_back(N->getOperand(3));
2713 Ops.push_back(Chain);
2715 case NVPTXISD::Suld1DArrayV2I16Trap:
2716 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
2717 Ops.push_back(TexHandle);
2718 Ops.push_back(N->getOperand(2));
2719 Ops.push_back(N->getOperand(3));
2720 Ops.push_back(Chain);
2722 case NVPTXISD::Suld1DArrayV2I32Trap:
2723 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
2724 Ops.push_back(TexHandle);
2725 Ops.push_back(N->getOperand(2));
2726 Ops.push_back(N->getOperand(3));
2727 Ops.push_back(Chain);
2729 case NVPTXISD::Suld1DArrayV4I8Trap:
2730 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
2731 Ops.push_back(TexHandle);
2732 Ops.push_back(N->getOperand(2));
2733 Ops.push_back(N->getOperand(3));
2734 Ops.push_back(Chain);
2736 case NVPTXISD::Suld1DArrayV4I16Trap:
2737 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
2738 Ops.push_back(TexHandle);
2739 Ops.push_back(N->getOperand(2));
2740 Ops.push_back(N->getOperand(3));
2741 Ops.push_back(Chain);
2743 case NVPTXISD::Suld1DArrayV4I32Trap:
2744 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
2745 Ops.push_back(TexHandle);
2746 Ops.push_back(N->getOperand(2));
2747 Ops.push_back(N->getOperand(3));
2748 Ops.push_back(Chain);
2750 case NVPTXISD::Suld2DI8Trap:
2751 Opc = NVPTX::SULD_2D_I8_TRAP;
2752 Ops.push_back(TexHandle);
2753 Ops.push_back(N->getOperand(2));
2754 Ops.push_back(N->getOperand(3));
2755 Ops.push_back(Chain);
2757 case NVPTXISD::Suld2DI16Trap:
2758 Opc = NVPTX::SULD_2D_I16_TRAP;
2759 Ops.push_back(TexHandle);
2760 Ops.push_back(N->getOperand(2));
2761 Ops.push_back(N->getOperand(3));
2762 Ops.push_back(Chain);
2764 case NVPTXISD::Suld2DI32Trap:
2765 Opc = NVPTX::SULD_2D_I32_TRAP;
2766 Ops.push_back(TexHandle);
2767 Ops.push_back(N->getOperand(2));
2768 Ops.push_back(N->getOperand(3));
2769 Ops.push_back(Chain);
2771 case NVPTXISD::Suld2DV2I8Trap:
2772 Opc = NVPTX::SULD_2D_V2I8_TRAP;
2773 Ops.push_back(TexHandle);
2774 Ops.push_back(N->getOperand(2));
2775 Ops.push_back(N->getOperand(3));
2776 Ops.push_back(Chain);
2778 case NVPTXISD::Suld2DV2I16Trap:
2779 Opc = NVPTX::SULD_2D_V2I16_TRAP;
2780 Ops.push_back(TexHandle);
2781 Ops.push_back(N->getOperand(2));
2782 Ops.push_back(N->getOperand(3));
2783 Ops.push_back(Chain);
2785 case NVPTXISD::Suld2DV2I32Trap:
2786 Opc = NVPTX::SULD_2D_V2I32_TRAP;
2787 Ops.push_back(TexHandle);
2788 Ops.push_back(N->getOperand(2));
2789 Ops.push_back(N->getOperand(3));
2790 Ops.push_back(Chain);
2792 case NVPTXISD::Suld2DV4I8Trap:
2793 Opc = NVPTX::SULD_2D_V4I8_TRAP;
2794 Ops.push_back(TexHandle);
2795 Ops.push_back(N->getOperand(2));
2796 Ops.push_back(N->getOperand(3));
2797 Ops.push_back(Chain);
2799 case NVPTXISD::Suld2DV4I16Trap:
2800 Opc = NVPTX::SULD_2D_V4I16_TRAP;
2801 Ops.push_back(TexHandle);
2802 Ops.push_back(N->getOperand(2));
2803 Ops.push_back(N->getOperand(3));
2804 Ops.push_back(Chain);
2806 case NVPTXISD::Suld2DV4I32Trap:
2807 Opc = NVPTX::SULD_2D_V4I32_TRAP;
2808 Ops.push_back(TexHandle);
2809 Ops.push_back(N->getOperand(2));
2810 Ops.push_back(N->getOperand(3));
2811 Ops.push_back(Chain);
2813 case NVPTXISD::Suld2DArrayI8Trap:
2814 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
2815 Ops.push_back(TexHandle);
2816 Ops.push_back(N->getOperand(2));
2817 Ops.push_back(N->getOperand(3));
2818 Ops.push_back(N->getOperand(4));
2819 Ops.push_back(Chain);
2821 case NVPTXISD::Suld2DArrayI16Trap:
2822 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
2823 Ops.push_back(TexHandle);
2824 Ops.push_back(N->getOperand(2));
2825 Ops.push_back(N->getOperand(3));
2826 Ops.push_back(N->getOperand(4));
2827 Ops.push_back(Chain);
2829 case NVPTXISD::Suld2DArrayI32Trap:
2830 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
2831 Ops.push_back(TexHandle);
2832 Ops.push_back(N->getOperand(2));
2833 Ops.push_back(N->getOperand(3));
2834 Ops.push_back(N->getOperand(4));
2835 Ops.push_back(Chain);
2837 case NVPTXISD::Suld2DArrayV2I8Trap:
2838 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
2839 Ops.push_back(TexHandle);
2840 Ops.push_back(N->getOperand(2));
2841 Ops.push_back(N->getOperand(3));
2842 Ops.push_back(N->getOperand(4));
2843 Ops.push_back(Chain);
2845 case NVPTXISD::Suld2DArrayV2I16Trap:
2846 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
2847 Ops.push_back(TexHandle);
2848 Ops.push_back(N->getOperand(2));
2849 Ops.push_back(N->getOperand(3));
2850 Ops.push_back(N->getOperand(4));
2851 Ops.push_back(Chain);
2853 case NVPTXISD::Suld2DArrayV2I32Trap:
2854 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
2855 Ops.push_back(TexHandle);
2856 Ops.push_back(N->getOperand(2));
2857 Ops.push_back(N->getOperand(3));
2858 Ops.push_back(N->getOperand(4));
2859 Ops.push_back(Chain);
2861 case NVPTXISD::Suld2DArrayV4I8Trap:
2862 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
2863 Ops.push_back(TexHandle);
2864 Ops.push_back(N->getOperand(2));
2865 Ops.push_back(N->getOperand(3));
2866 Ops.push_back(N->getOperand(4));
2867 Ops.push_back(Chain);
2869 case NVPTXISD::Suld2DArrayV4I16Trap:
2870 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
2871 Ops.push_back(TexHandle);
2872 Ops.push_back(N->getOperand(2));
2873 Ops.push_back(N->getOperand(3));
2874 Ops.push_back(N->getOperand(4));
2875 Ops.push_back(Chain);
2877 case NVPTXISD::Suld2DArrayV4I32Trap:
2878 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
2879 Ops.push_back(TexHandle);
2880 Ops.push_back(N->getOperand(2));
2881 Ops.push_back(N->getOperand(3));
2882 Ops.push_back(N->getOperand(4));
2883 Ops.push_back(Chain);
2885 case NVPTXISD::Suld3DI8Trap:
2886 Opc = NVPTX::SULD_3D_I8_TRAP;
2887 Ops.push_back(TexHandle);
2888 Ops.push_back(N->getOperand(2));
2889 Ops.push_back(N->getOperand(3));
2890 Ops.push_back(N->getOperand(4));
2891 Ops.push_back(Chain);
2893 case NVPTXISD::Suld3DI16Trap:
2894 Opc = NVPTX::SULD_3D_I16_TRAP;
2895 Ops.push_back(TexHandle);
2896 Ops.push_back(N->getOperand(2));
2897 Ops.push_back(N->getOperand(3));
2898 Ops.push_back(N->getOperand(4));
2899 Ops.push_back(Chain);
2901 case NVPTXISD::Suld3DI32Trap:
2902 Opc = NVPTX::SULD_3D_I32_TRAP;
2903 Ops.push_back(TexHandle);
2904 Ops.push_back(N->getOperand(2));
2905 Ops.push_back(N->getOperand(3));
2906 Ops.push_back(N->getOperand(4));
2907 Ops.push_back(Chain);
2909 case NVPTXISD::Suld3DV2I8Trap:
2910 Opc = NVPTX::SULD_3D_V2I8_TRAP;
2911 Ops.push_back(TexHandle);
2912 Ops.push_back(N->getOperand(2));
2913 Ops.push_back(N->getOperand(3));
2914 Ops.push_back(N->getOperand(4));
2915 Ops.push_back(Chain);
2917 case NVPTXISD::Suld3DV2I16Trap:
2918 Opc = NVPTX::SULD_3D_V2I16_TRAP;
2919 Ops.push_back(TexHandle);
2920 Ops.push_back(N->getOperand(2));
2921 Ops.push_back(N->getOperand(3));
2922 Ops.push_back(N->getOperand(4));
2923 Ops.push_back(Chain);
2925 case NVPTXISD::Suld3DV2I32Trap:
2926 Opc = NVPTX::SULD_3D_V2I32_TRAP;
2927 Ops.push_back(TexHandle);
2928 Ops.push_back(N->getOperand(2));
2929 Ops.push_back(N->getOperand(3));
2930 Ops.push_back(N->getOperand(4));
2931 Ops.push_back(Chain);
2933 case NVPTXISD::Suld3DV4I8Trap:
2934 Opc = NVPTX::SULD_3D_V4I8_TRAP;
2935 Ops.push_back(TexHandle);
2936 Ops.push_back(N->getOperand(2));
2937 Ops.push_back(N->getOperand(3));
2938 Ops.push_back(N->getOperand(4));
2939 Ops.push_back(Chain);
2941 case NVPTXISD::Suld3DV4I16Trap:
2942 Opc = NVPTX::SULD_3D_V4I16_TRAP;
2943 Ops.push_back(TexHandle);
2944 Ops.push_back(N->getOperand(2));
2945 Ops.push_back(N->getOperand(3));
2946 Ops.push_back(N->getOperand(4));
2947 Ops.push_back(Chain);
2949 case NVPTXISD::Suld3DV4I32Trap:
2950 Opc = NVPTX::SULD_3D_V4I32_TRAP;
2951 Ops.push_back(TexHandle);
2952 Ops.push_back(N->getOperand(2));
2953 Ops.push_back(N->getOperand(3));
2954 Ops.push_back(N->getOperand(4));
2955 Ops.push_back(Chain);
2958 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
2962 // SelectDirectAddr - Match a direct address for DAG.
2963 // A direct address could be a globaladdress or externalsymbol.
2964 bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
2965 // Return true if TGA or ES.
2966 if (N.getOpcode() == ISD::TargetGlobalAddress ||
2967 N.getOpcode() == ISD::TargetExternalSymbol) {
2971 if (N.getOpcode() == NVPTXISD::Wrapper) {
2972 Address = N.getOperand(0);
2975 if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
2976 unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
2977 if (IID == Intrinsic::nvvm_ptr_gen_to_param)
2978 if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
2979 return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
2985 bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
2986 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
2987 if (Addr.getOpcode() == ISD::ADD) {
2988 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
2989 SDValue base = Addr.getOperand(0);
2990 if (SelectDirectAddr(base, Base)) {
2991 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
3000 bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
3001 SDValue &Base, SDValue &Offset) {
3002 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
3006 bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
3007 SDValue &Base, SDValue &Offset) {
3008 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
3012 bool NVPTXDAGToDAGISel::SelectADDRri_imp(
3013 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
3014 if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
3015 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
3016 Offset = CurDAG->getTargetConstant(0, mvt);
3019 if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
3020 Addr.getOpcode() == ISD::TargetGlobalAddress)
3021 return false; // direct calls.
3023 if (Addr.getOpcode() == ISD::ADD) {
3024 if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
3027 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
3028 if (FrameIndexSDNode *FIN =
3029 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
3030 // Constant offset from frame ref.
3031 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
3033 Base = Addr.getOperand(0);
3034 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
3042 bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
3043 SDValue &Base, SDValue &Offset) {
3044 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
3048 bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
3049 SDValue &Base, SDValue &Offset) {
3050 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
3053 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
3054 unsigned int spN) const {
3055 const Value *Src = nullptr;
3056 // Even though MemIntrinsicSDNode is a subclas of MemSDNode,
3057 // the classof() for MemSDNode does not include MemIntrinsicSDNode
3058 // (See SelectionDAGNodes.h). So we need to check for both.
3059 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
3060 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
3062 Src = mN->getMemOperand()->getValue();
3063 } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) {
3064 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
3066 Src = mN->getMemOperand()->getValue();
3070 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
3071 return (PT->getAddressSpace() == spN);
3075 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
3076 /// inline asm expressions.
3077 bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
3078 const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) {
3080 switch (ConstraintCode) {
3084 if (SelectDirectAddr(Op, Op0)) {
3085 OutOps.push_back(Op0);
3086 OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
3089 if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
3090 OutOps.push_back(Op0);
3091 OutOps.push_back(Op1);