1 //===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// \brief Custom DAG lowering for R600
13 //===----------------------------------------------------------------------===//
15 #include "R600ISelLowering.h"
16 #include "R600Defines.h"
17 #include "R600InstrInfo.h"
18 #include "R600MachineFunctionInfo.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 #include "llvm/CodeGen/MachineRegisterInfo.h"
21 #include "llvm/CodeGen/SelectionDAG.h"
22 #include "llvm/IR/Argument.h"
23 #include "llvm/IR/Function.h"
27 R600TargetLowering::R600TargetLowering(TargetMachine &TM) :
28 AMDGPUTargetLowering(TM),
29 TII(static_cast<const R600InstrInfo*>(TM.getInstrInfo())) {
30 setOperationAction(ISD::MUL, MVT::i64, Expand);
31 addRegisterClass(MVT::v4f32, &AMDGPU::R600_Reg128RegClass);
32 addRegisterClass(MVT::f32, &AMDGPU::R600_Reg32RegClass);
33 addRegisterClass(MVT::v4i32, &AMDGPU::R600_Reg128RegClass);
34 addRegisterClass(MVT::i32, &AMDGPU::R600_Reg32RegClass);
35 computeRegisterProperties();
37 setOperationAction(ISD::FADD, MVT::v4f32, Expand);
38 setOperationAction(ISD::FMUL, MVT::v4f32, Expand);
39 setOperationAction(ISD::FDIV, MVT::v4f32, Expand);
40 setOperationAction(ISD::FSUB, MVT::v4f32, Expand);
42 setOperationAction(ISD::ADD, MVT::v4i32, Expand);
43 setOperationAction(ISD::AND, MVT::v4i32, Expand);
44 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Expand);
45 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Expand);
46 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Expand);
47 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Expand);
48 setOperationAction(ISD::UDIV, MVT::v4i32, Expand);
49 setOperationAction(ISD::UREM, MVT::v4i32, Expand);
50 setOperationAction(ISD::SETCC, MVT::v4i32, Expand);
52 setOperationAction(ISD::BR_CC, MVT::i32, Custom);
53 setOperationAction(ISD::BR_CC, MVT::f32, Custom);
55 setOperationAction(ISD::FSUB, MVT::f32, Expand);
57 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
58 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
59 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i1, Custom);
60 setOperationAction(ISD::FPOW, MVT::f32, Custom);
62 setOperationAction(ISD::ROTL, MVT::i32, Custom);
64 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
65 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
67 setOperationAction(ISD::SETCC, MVT::i32, Custom);
68 setOperationAction(ISD::SETCC, MVT::f32, Custom);
69 setOperationAction(ISD::FP_TO_UINT, MVT::i1, Custom);
71 setOperationAction(ISD::SELECT, MVT::i32, Custom);
72 setOperationAction(ISD::SELECT, MVT::f32, Custom);
74 setOperationAction(ISD::STORE, MVT::i32, Custom);
75 setOperationAction(ISD::STORE, MVT::v4i32, Custom);
77 setOperationAction(ISD::LOAD, MVT::i32, Custom);
78 setOperationAction(ISD::LOAD, MVT::v4i32, Custom);
79 setTargetDAGCombine(ISD::FP_ROUND);
80 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
82 setSchedulingPreference(Sched::VLIW);
85 MachineBasicBlock * R600TargetLowering::EmitInstrWithCustomInserter(
86 MachineInstr * MI, MachineBasicBlock * BB) const {
87 MachineFunction * MF = BB->getParent();
88 MachineRegisterInfo &MRI = MF->getRegInfo();
89 MachineBasicBlock::iterator I = *MI;
91 switch (MI->getOpcode()) {
92 default: return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
93 case AMDGPU::SHADER_TYPE: break;
94 case AMDGPU::CLAMP_R600: {
95 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I,
97 MI->getOperand(0).getReg(),
98 MI->getOperand(1).getReg());
99 TII->addFlag(NewMI, 0, MO_FLAG_CLAMP);
103 case AMDGPU::FABS_R600: {
104 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I,
106 MI->getOperand(0).getReg(),
107 MI->getOperand(1).getReg());
108 TII->addFlag(NewMI, 0, MO_FLAG_ABS);
112 case AMDGPU::FNEG_R600: {
113 MachineInstr *NewMI = TII->buildDefaultInstruction(*BB, I,
115 MI->getOperand(0).getReg(),
116 MI->getOperand(1).getReg());
117 TII->addFlag(NewMI, 0, MO_FLAG_NEG);
121 case AMDGPU::MASK_WRITE: {
122 unsigned maskedRegister = MI->getOperand(0).getReg();
123 assert(TargetRegisterInfo::isVirtualRegister(maskedRegister));
124 MachineInstr * defInstr = MRI.getVRegDef(maskedRegister);
125 TII->addFlag(defInstr, 0, MO_FLAG_MASK);
129 case AMDGPU::MOV_IMM_F32:
130 TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(),
131 MI->getOperand(1).getFPImm()->getValueAPF()
132 .bitcastToAPInt().getZExtValue());
134 case AMDGPU::MOV_IMM_I32:
135 TII->buildMovImm(*BB, I, MI->getOperand(0).getReg(),
136 MI->getOperand(1).getImm());
140 case AMDGPU::RAT_WRITE_CACHELESS_32_eg:
141 case AMDGPU::RAT_WRITE_CACHELESS_128_eg: {
142 unsigned EOP = (llvm::next(I)->getOpcode() == AMDGPU::RETURN) ? 1 : 0;
144 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode()))
145 .addOperand(MI->getOperand(0))
146 .addOperand(MI->getOperand(1))
147 .addImm(EOP); // Set End of program bit
152 unsigned T0 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass);
153 unsigned T1 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass);
155 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_H), T0)
156 .addOperand(MI->getOperand(3))
157 .addOperand(MI->getOperand(4))
158 .addOperand(MI->getOperand(5))
159 .addOperand(MI->getOperand(6));
160 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_V), T1)
161 .addOperand(MI->getOperand(2))
162 .addOperand(MI->getOperand(4))
163 .addOperand(MI->getOperand(5))
164 .addOperand(MI->getOperand(6));
165 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SAMPLE_G))
166 .addOperand(MI->getOperand(0))
167 .addOperand(MI->getOperand(1))
168 .addOperand(MI->getOperand(4))
169 .addOperand(MI->getOperand(5))
170 .addOperand(MI->getOperand(6))
171 .addReg(T0, RegState::Implicit)
172 .addReg(T1, RegState::Implicit);
176 case AMDGPU::TXD_SHADOW: {
177 unsigned T0 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass);
178 unsigned T1 = MRI.createVirtualRegister(&AMDGPU::R600_Reg128RegClass);
180 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_H), T0)
181 .addOperand(MI->getOperand(3))
182 .addOperand(MI->getOperand(4))
183 .addOperand(MI->getOperand(5))
184 .addOperand(MI->getOperand(6));
185 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SET_GRADIENTS_V), T1)
186 .addOperand(MI->getOperand(2))
187 .addOperand(MI->getOperand(4))
188 .addOperand(MI->getOperand(5))
189 .addOperand(MI->getOperand(6));
190 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::TEX_SAMPLE_C_G))
191 .addOperand(MI->getOperand(0))
192 .addOperand(MI->getOperand(1))
193 .addOperand(MI->getOperand(4))
194 .addOperand(MI->getOperand(5))
195 .addOperand(MI->getOperand(6))
196 .addReg(T0, RegState::Implicit)
197 .addReg(T1, RegState::Implicit);
202 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP))
203 .addOperand(MI->getOperand(0))
207 case AMDGPU::BRANCH_COND_f32: {
208 MachineInstr *NewMI =
209 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::PRED_X),
210 AMDGPU::PREDICATE_BIT)
211 .addOperand(MI->getOperand(1))
212 .addImm(OPCODE_IS_NOT_ZERO)
214 TII->addFlag(NewMI, 0, MO_FLAG_PUSH);
215 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP))
216 .addOperand(MI->getOperand(0))
217 .addReg(AMDGPU::PREDICATE_BIT, RegState::Kill);
221 case AMDGPU::BRANCH_COND_i32: {
222 MachineInstr *NewMI =
223 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::PRED_X),
224 AMDGPU::PREDICATE_BIT)
225 .addOperand(MI->getOperand(1))
226 .addImm(OPCODE_IS_NOT_ZERO_INT)
228 TII->addFlag(NewMI, 0, MO_FLAG_PUSH);
229 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::JUMP))
230 .addOperand(MI->getOperand(0))
231 .addReg(AMDGPU::PREDICATE_BIT, RegState::Kill);
235 case AMDGPU::EG_ExportSwz:
236 case AMDGPU::R600_ExportSwz: {
237 // Instruction is left unmodified if its not the last one of its type
238 bool isLastInstructionOfItsType = true;
239 unsigned InstExportType = MI->getOperand(1).getImm();
240 for (MachineBasicBlock::iterator NextExportInst = llvm::next(I),
241 EndBlock = BB->end(); NextExportInst != EndBlock;
242 NextExportInst = llvm::next(NextExportInst)) {
243 if (NextExportInst->getOpcode() == AMDGPU::EG_ExportSwz ||
244 NextExportInst->getOpcode() == AMDGPU::R600_ExportSwz) {
245 unsigned CurrentInstExportType = NextExportInst->getOperand(1)
247 if (CurrentInstExportType == InstExportType) {
248 isLastInstructionOfItsType = false;
253 bool EOP = (llvm::next(I)->getOpcode() == AMDGPU::RETURN)? 1 : 0;
254 if (!EOP && !isLastInstructionOfItsType)
256 unsigned CfInst = (MI->getOpcode() == AMDGPU::EG_ExportSwz)? 84 : 40;
257 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(MI->getOpcode()))
258 .addOperand(MI->getOperand(0))
259 .addOperand(MI->getOperand(1))
260 .addOperand(MI->getOperand(2))
261 .addOperand(MI->getOperand(3))
262 .addOperand(MI->getOperand(4))
263 .addOperand(MI->getOperand(5))
264 .addOperand(MI->getOperand(6))
271 MI->eraseFromParent();
275 //===----------------------------------------------------------------------===//
276 // Custom DAG Lowering Operations
277 //===----------------------------------------------------------------------===//
279 using namespace llvm::Intrinsic;
280 using namespace llvm::AMDGPUIntrinsic;
283 InsertScalarToRegisterExport(SelectionDAG &DAG, DebugLoc DL, SDNode **ExportMap,
284 unsigned Slot, unsigned Channel, unsigned Inst, unsigned Type,
285 SDValue Scalar, SDValue Chain) {
286 if (!ExportMap[Slot]) {
287 SDValue Vector = DAG.getNode(ISD::INSERT_VECTOR_ELT,
289 DAG.getUNDEF(MVT::v4f32),
291 DAG.getConstant(Channel, MVT::i32));
293 unsigned Mask = 1 << Channel;
295 const SDValue Ops[] = {Chain, Vector, DAG.getConstant(Inst, MVT::i32),
296 DAG.getConstant(Type, MVT::i32), DAG.getConstant(Slot, MVT::i32),
297 DAG.getConstant(Mask, MVT::i32)};
299 SDValue Res = DAG.getNode(
304 ExportMap[Slot] = Res.getNode();
308 SDNode *ExportInstruction = (SDNode *) ExportMap[Slot] ;
309 SDValue PreviousVector = ExportInstruction->getOperand(1);
310 SDValue Vector = DAG.getNode(ISD::INSERT_VECTOR_ELT,
314 DAG.getConstant(Channel, MVT::i32));
316 unsigned Mask = dyn_cast<ConstantSDNode>(ExportInstruction->getOperand(5))
318 Mask |= (1 << Channel);
320 const SDValue Ops[] = {ExportInstruction->getOperand(0), Vector,
321 DAG.getConstant(Inst, MVT::i32),
322 DAG.getConstant(Type, MVT::i32),
323 DAG.getConstant(Slot, MVT::i32),
324 DAG.getConstant(Mask, MVT::i32)};
326 DAG.UpdateNodeOperands(ExportInstruction,
333 SDValue R600TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
334 switch (Op.getOpcode()) {
335 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
336 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
337 case ISD::ROTL: return LowerROTL(Op, DAG);
338 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
339 case ISD::SELECT: return LowerSELECT(Op, DAG);
340 case ISD::SETCC: return LowerSETCC(Op, DAG);
341 case ISD::STORE: return LowerSTORE(Op, DAG);
342 case ISD::LOAD: return LowerLOAD(Op, DAG);
343 case ISD::FPOW: return LowerFPOW(Op, DAG);
344 case ISD::INTRINSIC_VOID: {
345 SDValue Chain = Op.getOperand(0);
346 unsigned IntrinsicID =
347 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
348 switch (IntrinsicID) {
349 case AMDGPUIntrinsic::AMDGPU_store_output: {
350 MachineFunction &MF = DAG.getMachineFunction();
351 MachineRegisterInfo &MRI = MF.getRegInfo();
352 int64_t RegIndex = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
353 unsigned Reg = AMDGPU::R600_TReg32RegClass.getRegister(RegIndex);
354 if (!MRI.isLiveOut(Reg)) {
357 return DAG.getCopyToReg(Chain, Op.getDebugLoc(), Reg, Op.getOperand(2));
359 case AMDGPUIntrinsic::R600_store_pixel_color: {
360 MachineFunction &MF = DAG.getMachineFunction();
361 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
362 int64_t RegIndex = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
364 SDNode **OutputsMap = MFI->Outputs;
365 return InsertScalarToRegisterExport(DAG, Op.getDebugLoc(), OutputsMap,
366 RegIndex / 4, RegIndex % 4, 0, 0, Op.getOperand(2),
371 // default for switch(IntrinsicID)
374 // break out of case ISD::INTRINSIC_VOID in switch(Op.getOpcode())
377 case ISD::INTRINSIC_WO_CHAIN: {
378 unsigned IntrinsicID =
379 cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
380 EVT VT = Op.getValueType();
381 DebugLoc DL = Op.getDebugLoc();
382 switch(IntrinsicID) {
383 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
384 case AMDGPUIntrinsic::R600_load_input: {
385 int64_t RegIndex = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
386 unsigned Reg = AMDGPU::R600_TReg32RegClass.getRegister(RegIndex);
387 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass, Reg, VT);
390 case AMDGPUIntrinsic::R600_interp_input: {
391 int slot = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
392 int ijb = cast<ConstantSDNode>(Op.getOperand(2))->getSExtValue();
393 MachineSDNode *interp;
395 interp = DAG.getMachineNode(AMDGPU::INTERP_VEC_LOAD, DL,
396 MVT::v4f32, DAG.getTargetConstant(slot / 4 , MVT::i32));
397 return DAG.getTargetExtractSubreg(
398 TII->getRegisterInfo().getSubRegFromChannel(slot % 4),
399 DL, MVT::f32, SDValue(interp, 0));
403 interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_XY, DL,
404 MVT::f32, MVT::f32, DAG.getTargetConstant(slot / 4 , MVT::i32),
405 CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
406 AMDGPU::R600_TReg32RegClass.getRegister(2 * ijb + 1), MVT::f32),
407 CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
408 AMDGPU::R600_TReg32RegClass.getRegister(2 * ijb), MVT::f32));
410 interp = DAG.getMachineNode(AMDGPU::INTERP_PAIR_ZW, DL,
411 MVT::f32, MVT::f32, DAG.getTargetConstant(slot / 4 , MVT::i32),
412 CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
413 AMDGPU::R600_TReg32RegClass.getRegister(2 * ijb + 1), MVT::f32),
414 CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
415 AMDGPU::R600_TReg32RegClass.getRegister(2 * ijb), MVT::f32));
417 return SDValue(interp, slot % 2);
420 case r600_read_ngroups_x:
421 return LowerImplicitParameter(DAG, VT, DL, 0);
422 case r600_read_ngroups_y:
423 return LowerImplicitParameter(DAG, VT, DL, 1);
424 case r600_read_ngroups_z:
425 return LowerImplicitParameter(DAG, VT, DL, 2);
426 case r600_read_global_size_x:
427 return LowerImplicitParameter(DAG, VT, DL, 3);
428 case r600_read_global_size_y:
429 return LowerImplicitParameter(DAG, VT, DL, 4);
430 case r600_read_global_size_z:
431 return LowerImplicitParameter(DAG, VT, DL, 5);
432 case r600_read_local_size_x:
433 return LowerImplicitParameter(DAG, VT, DL, 6);
434 case r600_read_local_size_y:
435 return LowerImplicitParameter(DAG, VT, DL, 7);
436 case r600_read_local_size_z:
437 return LowerImplicitParameter(DAG, VT, DL, 8);
439 case r600_read_tgid_x:
440 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
442 case r600_read_tgid_y:
443 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
445 case r600_read_tgid_z:
446 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
448 case r600_read_tidig_x:
449 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
451 case r600_read_tidig_y:
452 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
454 case r600_read_tidig_z:
455 return CreateLiveInRegister(DAG, &AMDGPU::R600_TReg32RegClass,
458 // break out of case ISD::INTRINSIC_WO_CHAIN in switch(Op.getOpcode())
461 } // end switch(Op.getOpcode())
465 void R600TargetLowering::ReplaceNodeResults(SDNode *N,
466 SmallVectorImpl<SDValue> &Results,
467 SelectionDAG &DAG) const {
468 switch (N->getOpcode()) {
470 case ISD::FP_TO_UINT: Results.push_back(LowerFPTOUINT(N->getOperand(0), DAG));
473 SDNode *Node = LowerLOAD(SDValue(N, 0), DAG).getNode();
474 Results.push_back(SDValue(Node, 0));
475 Results.push_back(SDValue(Node, 1));
476 // XXX: LLVM seems not to replace Chain Value inside CustomWidenLowerNode
478 DAG.ReplaceAllUsesOfValueWith(SDValue(N,1), SDValue(Node, 1));
484 SDValue R600TargetLowering::LowerFPTOUINT(SDValue Op, SelectionDAG &DAG) const {
489 Op, DAG.getConstantFP(0.0f, MVT::f32),
490 DAG.getCondCode(ISD::SETNE)
494 SDValue R600TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
495 SDValue Chain = Op.getOperand(0);
496 SDValue CC = Op.getOperand(1);
497 SDValue LHS = Op.getOperand(2);
498 SDValue RHS = Op.getOperand(3);
499 SDValue JumpT = Op.getOperand(4);
503 if (LHS.getValueType() == MVT::i32) {
504 CmpValue = DAG.getNode(
509 DAG.getConstant(-1, MVT::i32),
510 DAG.getConstant(0, MVT::i32),
512 } else if (LHS.getValueType() == MVT::f32) {
513 CmpValue = DAG.getNode(
518 DAG.getConstantFP(1.0f, MVT::f32),
519 DAG.getConstantFP(0.0f, MVT::f32),
522 assert(0 && "Not valid type for br_cc");
524 Result = DAG.getNode(
525 AMDGPUISD::BRANCH_COND,
526 CmpValue.getDebugLoc(),
532 SDValue R600TargetLowering::LowerImplicitParameter(SelectionDAG &DAG, EVT VT,
534 unsigned DwordOffset) const {
535 unsigned ByteOffset = DwordOffset * 4;
536 PointerType * PtrType = PointerType::get(VT.getTypeForEVT(*DAG.getContext()),
537 AMDGPUAS::PARAM_I_ADDRESS);
539 // We shouldn't be using an offset wider than 16-bits for implicit parameters.
540 assert(isInt<16>(ByteOffset));
542 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
543 DAG.getConstant(ByteOffset, MVT::i32), // PTR
544 MachinePointerInfo(ConstantPointerNull::get(PtrType)),
545 false, false, false, 0);
548 SDValue R600TargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
549 DebugLoc DL = Op.getDebugLoc();
550 EVT VT = Op.getValueType();
552 return DAG.getNode(AMDGPUISD::BITALIGN, DL, VT,
555 DAG.getNode(ISD::SUB, DL, VT,
556 DAG.getConstant(32, MVT::i32),
560 bool R600TargetLowering::isZero(SDValue Op) const {
561 if(ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) {
562 return Cst->isNullValue();
563 } else if(ConstantFPSDNode *CstFP = dyn_cast<ConstantFPSDNode>(Op)){
564 return CstFP->isZero();
570 SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
571 DebugLoc DL = Op.getDebugLoc();
572 EVT VT = Op.getValueType();
574 SDValue LHS = Op.getOperand(0);
575 SDValue RHS = Op.getOperand(1);
576 SDValue True = Op.getOperand(2);
577 SDValue False = Op.getOperand(3);
578 SDValue CC = Op.getOperand(4);
581 // LHS and RHS are guaranteed to be the same value type
582 EVT CompareVT = LHS.getValueType();
584 // Check if we can lower this to a native operation.
586 // Try to lower to a CND* instruction:
587 // CND* instructions requires RHS to be zero. Some SELECT_CC nodes that
588 // can be lowered to CND* instructions can also be lowered to SET*
589 // instructions. CND* instructions are cheaper, because they dont't
590 // require additional instructions to convert their result to the correct
591 // value type, so this check should be first.
592 if (isZero(LHS) || isZero(RHS)) {
593 SDValue Cond = (isZero(LHS) ? RHS : LHS);
594 SDValue Zero = (isZero(LHS) ? LHS : RHS);
595 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
596 if (CompareVT != VT) {
597 // Bitcast True / False to the correct types. This will end up being
598 // a nop, but it allows us to define only a single pattern in the
599 // .TD files for each CND* instruction rather than having to have
600 // one pattern for integer True/False and one for fp True/False
601 True = DAG.getNode(ISD::BITCAST, DL, CompareVT, True);
602 False = DAG.getNode(ISD::BITCAST, DL, CompareVT, False);
605 CCOpcode = ISD::getSetCCSwappedOperands(CCOpcode);
618 CCOpcode = ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32);
626 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, CompareVT,
629 DAG.getCondCode(CCOpcode));
630 return DAG.getNode(ISD::BITCAST, DL, VT, SelectNode);
633 // Try to lower to a SET* instruction:
634 // We need all the operands of SELECT_CC to have the same value type, so if
635 // necessary we need to change True and False to be the same type as LHS and
636 // RHS, and then convert the result of the select_cc back to the correct type.
638 // Move hardware True/False values to the correct operand.
639 if (isHWTrueValue(False) && isHWFalseValue(True)) {
640 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
641 std::swap(False, True);
642 CC = DAG.getCondCode(ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32));
645 if (isHWTrueValue(True) && isHWFalseValue(False)) {
646 if (CompareVT != VT) {
647 if (VT == MVT::f32 && CompareVT == MVT::i32) {
648 SDValue Boolean = DAG.getNode(ISD::SELECT_CC, DL, CompareVT,
650 DAG.getConstant(-1, MVT::i32),
651 DAG.getConstant(0, MVT::i32),
653 // Convert integer values of true (-1) and false (0) to fp values of
654 // true (1.0f) and false (0.0f).
655 SDValue LSB = DAG.getNode(ISD::AND, DL, MVT::i32, Boolean,
656 DAG.getConstant(1, MVT::i32));
657 return DAG.getNode(ISD::UINT_TO_FP, DL, VT, LSB);
658 } else if (VT == MVT::i32 && CompareVT == MVT::f32) {
659 SDValue BoolAsFlt = DAG.getNode(ISD::SELECT_CC, DL, CompareVT,
661 DAG.getConstantFP(1.0f, MVT::f32),
662 DAG.getConstantFP(0.0f, MVT::f32),
664 // Convert fp values of true (1.0f) and false (0.0f) to integer values
665 // of true (-1) and false (0).
666 SDValue Neg = DAG.getNode(ISD::FNEG, DL, MVT::f32, BoolAsFlt);
667 return DAG.getNode(ISD::FP_TO_SINT, DL, VT, Neg);
669 // I don't think there will be any other type pairings.
670 assert(!"Unhandled operand type parings in SELECT_CC");
673 // This SELECT_CC is already legal.
674 return DAG.getNode(ISD::SELECT_CC, DL, VT, LHS, RHS, True, False, CC);
678 // Possible Min/Max pattern
679 SDValue MinMax = LowerMinMax(Op, DAG);
680 if (MinMax.getNode()) {
684 // If we make it this for it means we have no native instructions to handle
685 // this SELECT_CC, so we must lower it.
686 SDValue HWTrue, HWFalse;
688 if (CompareVT == MVT::f32) {
689 HWTrue = DAG.getConstantFP(1.0f, CompareVT);
690 HWFalse = DAG.getConstantFP(0.0f, CompareVT);
691 } else if (CompareVT == MVT::i32) {
692 HWTrue = DAG.getConstant(-1, CompareVT);
693 HWFalse = DAG.getConstant(0, CompareVT);
696 assert(!"Unhandled value type in LowerSELECT_CC");
699 // Lower this unsupported SELECT_CC into a combination of two supported
700 // SELECT_CC operations.
701 SDValue Cond = DAG.getNode(ISD::SELECT_CC, DL, CompareVT, LHS, RHS, HWTrue, HWFalse, CC);
703 return DAG.getNode(ISD::SELECT_CC, DL, VT,
706 DAG.getCondCode(ISD::SETNE));
709 SDValue R600TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
710 return DAG.getNode(ISD::SELECT_CC,
714 DAG.getConstant(0, MVT::i32),
717 DAG.getCondCode(ISD::SETNE));
720 SDValue R600TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
722 SDValue LHS = Op.getOperand(0);
723 SDValue RHS = Op.getOperand(1);
724 SDValue CC = Op.getOperand(2);
725 DebugLoc DL = Op.getDebugLoc();
726 assert(Op.getValueType() == MVT::i32);
727 if (LHS.getValueType() == MVT::i32) {
733 DAG.getConstant(-1, MVT::i32),
734 DAG.getConstant(0, MVT::i32),
736 } else if (LHS.getValueType() == MVT::f32) {
742 DAG.getConstantFP(1.0f, MVT::f32),
743 DAG.getConstantFP(0.0f, MVT::f32),
751 assert(0 && "Not valid type for set_cc");
757 DAG.getConstant(1, MVT::i32),
762 SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
763 DebugLoc DL = Op.getDebugLoc();
764 StoreSDNode *StoreNode = cast<StoreSDNode>(Op);
765 SDValue Chain = Op.getOperand(0);
766 SDValue Value = Op.getOperand(1);
767 SDValue Ptr = Op.getOperand(2);
769 if (StoreNode->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
770 Ptr->getOpcode() != AMDGPUISD::DWORDADDR) {
771 // Convert pointer from byte address to dword address.
772 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, Ptr.getValueType(),
773 DAG.getNode(ISD::SRL, DL, Ptr.getValueType(),
774 Ptr, DAG.getConstant(2, MVT::i32)));
776 if (StoreNode->isTruncatingStore() || StoreNode->isIndexed()) {
777 assert(!"Truncated and indexed stores not supported yet");
779 Chain = DAG.getStore(Chain, DL, Value, Ptr, StoreNode->getMemOperand());
786 // return (512 + (kc_bank << 12)
788 ConstantAddressBlock(unsigned AddressSpace) {
789 switch (AddressSpace) {
790 case AMDGPUAS::CONSTANT_BUFFER_0:
792 case AMDGPUAS::CONSTANT_BUFFER_1:
794 case AMDGPUAS::CONSTANT_BUFFER_2:
795 return 512 + 4096 * 2;
796 case AMDGPUAS::CONSTANT_BUFFER_3:
797 return 512 + 4096 * 3;
798 case AMDGPUAS::CONSTANT_BUFFER_4:
799 return 512 + 4096 * 4;
800 case AMDGPUAS::CONSTANT_BUFFER_5:
801 return 512 + 4096 * 5;
802 case AMDGPUAS::CONSTANT_BUFFER_6:
803 return 512 + 4096 * 6;
804 case AMDGPUAS::CONSTANT_BUFFER_7:
805 return 512 + 4096 * 7;
806 case AMDGPUAS::CONSTANT_BUFFER_8:
807 return 512 + 4096 * 8;
808 case AMDGPUAS::CONSTANT_BUFFER_9:
809 return 512 + 4096 * 9;
810 case AMDGPUAS::CONSTANT_BUFFER_10:
811 return 512 + 4096 * 10;
812 case AMDGPUAS::CONSTANT_BUFFER_11:
813 return 512 + 4096 * 11;
814 case AMDGPUAS::CONSTANT_BUFFER_12:
815 return 512 + 4096 * 12;
816 case AMDGPUAS::CONSTANT_BUFFER_13:
817 return 512 + 4096 * 13;
818 case AMDGPUAS::CONSTANT_BUFFER_14:
819 return 512 + 4096 * 14;
820 case AMDGPUAS::CONSTANT_BUFFER_15:
821 return 512 + 4096 * 15;
827 SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const
829 EVT VT = Op.getValueType();
830 DebugLoc DL = Op.getDebugLoc();
831 LoadSDNode *LoadNode = cast<LoadSDNode>(Op);
832 SDValue Chain = Op.getOperand(0);
833 SDValue Ptr = Op.getOperand(1);
836 int ConstantBlock = ConstantAddressBlock(LoadNode->getAddressSpace());
837 if (ConstantBlock > -1) {
839 if (dyn_cast<ConstantExpr>(LoadNode->getSrcValue()) ||
840 dyn_cast<Constant>(LoadNode->getSrcValue())) {
842 for (unsigned i = 0; i < 4; i++) {
843 // We want Const position encoded with the following formula :
844 // (((512 + (kc_bank << 12) + const_index) << 2) + chan)
845 // const_index is Ptr computed by llvm using an alignment of 16.
846 // Thus we add (((512 + (kc_bank << 12)) + chan ) * 4 here and
847 // then div by 4 at the ISel step
848 SDValue NewPtr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
849 DAG.getConstant(4 * i + ConstantBlock * 16, MVT::i32));
850 Slots[i] = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::i32, NewPtr);
852 Result = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v4i32, Slots, 4);
854 // non constant ptr cant be folded, keeps it as a v4f32 load
855 Result = DAG.getNode(AMDGPUISD::CONST_ADDRESS, DL, MVT::v4i32,
856 DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr, DAG.getConstant(4, MVT::i32))
860 if (!VT.isVector()) {
861 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Result,
862 DAG.getConstant(0, MVT::i32));
865 SDValue MergedValues[2] = {
869 return DAG.getMergeValues(MergedValues, 2, DL);
875 SDValue R600TargetLowering::LowerFPOW(SDValue Op,
876 SelectionDAG &DAG) const {
877 DebugLoc DL = Op.getDebugLoc();
878 EVT VT = Op.getValueType();
879 SDValue LogBase = DAG.getNode(ISD::FLOG2, DL, VT, Op.getOperand(0));
880 SDValue MulLogBase = DAG.getNode(ISD::FMUL, DL, VT, Op.getOperand(1), LogBase);
881 return DAG.getNode(ISD::FEXP2, DL, VT, MulLogBase);
884 /// XXX Only kernel functions are supported, so we can assume for now that
885 /// every function is a kernel function, but in the future we should use
886 /// separate calling conventions for kernel and non-kernel functions.
887 SDValue R600TargetLowering::LowerFormalArguments(
889 CallingConv::ID CallConv,
891 const SmallVectorImpl<ISD::InputArg> &Ins,
892 DebugLoc DL, SelectionDAG &DAG,
893 SmallVectorImpl<SDValue> &InVals) const {
894 unsigned ParamOffsetBytes = 36;
895 Function::const_arg_iterator FuncArg =
896 DAG.getMachineFunction().getFunction()->arg_begin();
897 for (unsigned i = 0, e = Ins.size(); i < e; ++i, ++FuncArg) {
899 Type *ArgType = FuncArg->getType();
900 unsigned ArgSizeInBits = ArgType->isPointerTy() ?
901 32 : ArgType->getPrimitiveSizeInBits();
902 unsigned ArgBytes = ArgSizeInBits >> 3;
904 if (ArgSizeInBits < VT.getSizeInBits()) {
905 assert(!ArgType->isFloatTy() &&
906 "Extending floating point arguments not supported yet");
907 ArgVT = MVT::getIntegerVT(ArgSizeInBits);
911 PointerType *PtrTy = PointerType::get(VT.getTypeForEVT(*DAG.getContext()),
912 AMDGPUAS::PARAM_I_ADDRESS);
913 SDValue Arg = DAG.getExtLoad(ISD::ZEXTLOAD, DL, VT, DAG.getRoot(),
914 DAG.getConstant(ParamOffsetBytes, MVT::i32),
915 MachinePointerInfo(new Argument(PtrTy)),
916 ArgVT, false, false, ArgBytes);
917 InVals.push_back(Arg);
918 ParamOffsetBytes += ArgBytes;
923 EVT R600TargetLowering::getSetCCResultType(EVT VT) const {
924 if (!VT.isVector()) return MVT::i32;
925 return VT.changeVectorElementTypeToInteger();
928 //===----------------------------------------------------------------------===//
929 // Custom DAG Optimizations
930 //===----------------------------------------------------------------------===//
932 SDValue R600TargetLowering::PerformDAGCombine(SDNode *N,
933 DAGCombinerInfo &DCI) const {
934 SelectionDAG &DAG = DCI.DAG;
936 switch (N->getOpcode()) {
937 // (f32 fp_round (f64 uint_to_fp a)) -> (f32 uint_to_fp a)
938 case ISD::FP_ROUND: {
939 SDValue Arg = N->getOperand(0);
940 if (Arg.getOpcode() == ISD::UINT_TO_FP && Arg.getValueType() == MVT::f64) {
941 return DAG.getNode(ISD::UINT_TO_FP, N->getDebugLoc(), N->getValueType(0),
946 // Extract_vec (Build_vector) generated by custom lowering
947 // also needs to be customly combined
948 case ISD::EXTRACT_VECTOR_ELT: {
949 SDValue Arg = N->getOperand(0);
950 if (Arg.getOpcode() == ISD::BUILD_VECTOR) {
951 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
952 unsigned Element = Const->getZExtValue();
953 return Arg->getOperand(Element);
956 if (Arg.getOpcode() == ISD::BITCAST &&
957 Arg.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
958 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
959 unsigned Element = Const->getZExtValue();
960 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), N->getVTList(),
961 Arg->getOperand(0).getOperand(Element));