1 //===-- PTXISelLowering.cpp - PTX DAG Lowering Implementation -------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the PTXTargetLowering class.
12 //===----------------------------------------------------------------------===//
15 #include "PTXISelLowering.h"
16 #include "PTXMachineFunctionInfo.h"
17 #include "PTXRegisterInfo.h"
18 #include "PTXSubtarget.h"
19 #include "llvm/Function.h"
20 #include "llvm/Support/ErrorHandling.h"
21 #include "llvm/CodeGen/CallingConvLower.h"
22 #include "llvm/CodeGen/MachineFunction.h"
23 #include "llvm/CodeGen/MachineFrameInfo.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/CodeGen/SelectionDAG.h"
26 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
27 #include "llvm/Support/Debug.h"
28 #include "llvm/Support/raw_ostream.h"
32 //===----------------------------------------------------------------------===//
33 // TargetLowering Implementation
34 //===----------------------------------------------------------------------===//
36 PTXTargetLowering::PTXTargetLowering(TargetMachine &TM)
37 : TargetLowering(TM, new TargetLoweringObjectFileELF()) {
38 // Set up the register classes.
39 addRegisterClass(MVT::i1, PTX::RegPredRegisterClass);
40 addRegisterClass(MVT::i16, PTX::RegI16RegisterClass);
41 addRegisterClass(MVT::i32, PTX::RegI32RegisterClass);
42 addRegisterClass(MVT::i64, PTX::RegI64RegisterClass);
43 addRegisterClass(MVT::f32, PTX::RegF32RegisterClass);
44 addRegisterClass(MVT::f64, PTX::RegF64RegisterClass);
46 setBooleanContents(ZeroOrOneBooleanContent);
47 setBooleanVectorContents(ZeroOrOneBooleanContent); // FIXME: Is this correct?
48 setMinFunctionAlignment(2);
50 // Let LLVM use loads/stores for all mem* operations
51 maxStoresPerMemcpy = 4096;
52 maxStoresPerMemmove = 4096;
53 maxStoresPerMemset = 4096;
55 ////////////////////////////////////
56 /////////// Expansion //////////////
57 ////////////////////////////////////
59 // (any/zero/sign) extload => load + (any/zero/sign) extend
61 setLoadExtAction(ISD::EXTLOAD, MVT::i16, Expand);
62 setLoadExtAction(ISD::ZEXTLOAD, MVT::i16, Expand);
63 setLoadExtAction(ISD::SEXTLOAD, MVT::i16, Expand);
65 // f32 extload => load + fextend
67 setLoadExtAction(ISD::EXTLOAD, MVT::f32, Expand);
69 // f64 truncstore => trunc + store
71 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
73 // sign_extend_inreg => sign_extend
75 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
79 setOperationAction(ISD::BR_CC, MVT::Other, Expand);
83 setOperationAction(ISD::SELECT_CC, MVT::Other, Expand);
84 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
85 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
87 ////////////////////////////////////
88 //////////// Legal /////////////////
89 ////////////////////////////////////
91 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
92 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
94 ////////////////////////////////////
95 //////////// Custom ////////////////
96 ////////////////////////////////////
98 // customise setcc to use bitwise logic if possible
100 setOperationAction(ISD::SETCC, MVT::i1, Custom);
102 // customize translation of memory addresses
104 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
105 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
107 // Compute derived properties from the register classes
108 computeRegisterProperties();
111 EVT PTXTargetLowering::getSetCCResultType(EVT VT) const {
115 SDValue PTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
116 switch (Op.getOpcode()) {
118 llvm_unreachable("Unimplemented operand");
120 return LowerSETCC(Op, DAG);
121 case ISD::GlobalAddress:
122 return LowerGlobalAddress(Op, DAG);
126 const char *PTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
129 llvm_unreachable("Unknown opcode");
130 case PTXISD::COPY_ADDRESS:
131 return "PTXISD::COPY_ADDRESS";
132 case PTXISD::LOAD_PARAM:
133 return "PTXISD::LOAD_PARAM";
134 case PTXISD::STORE_PARAM:
135 return "PTXISD::STORE_PARAM";
136 case PTXISD::READ_PARAM:
137 return "PTXISD::READ_PARAM";
138 case PTXISD::WRITE_PARAM:
139 return "PTXISD::WRITE_PARAM";
141 return "PTXISD::EXIT";
143 return "PTXISD::RET";
145 return "PTXISD::CALL";
149 //===----------------------------------------------------------------------===//
150 // Custom Lower Operation
151 //===----------------------------------------------------------------------===//
153 SDValue PTXTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
154 assert(Op.getValueType() == MVT::i1 && "SetCC type must be 1-bit integer");
155 SDValue Op0 = Op.getOperand(0);
156 SDValue Op1 = Op.getOperand(1);
157 SDValue Op2 = Op.getOperand(2);
158 DebugLoc dl = Op.getDebugLoc();
159 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
161 // Look for X == 0, X == 1, X != 0, or X != 1
162 // We can simplify these to bitwise logic
164 if (Op1.getOpcode() == ISD::Constant &&
165 (cast<ConstantSDNode>(Op1)->getZExtValue() == 1 ||
166 cast<ConstantSDNode>(Op1)->isNullValue()) &&
167 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
169 return DAG.getNode(ISD::AND, dl, MVT::i1, Op0, Op1);
172 return DAG.getNode(ISD::SETCC, dl, MVT::i1, Op0, Op1, Op2);
175 SDValue PTXTargetLowering::
176 LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
177 EVT PtrVT = getPointerTy();
178 DebugLoc dl = Op.getDebugLoc();
179 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
181 assert(PtrVT.isSimple() && "Pointer must be to primitive type.");
183 SDValue targetGlobal = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
184 SDValue movInstr = DAG.getNode(PTXISD::COPY_ADDRESS,
192 //===----------------------------------------------------------------------===//
193 // Calling Convention Implementation
194 //===----------------------------------------------------------------------===//
196 SDValue PTXTargetLowering::
197 LowerFormalArguments(SDValue Chain,
198 CallingConv::ID CallConv,
200 const SmallVectorImpl<ISD::InputArg> &Ins,
203 SmallVectorImpl<SDValue> &InVals) const {
204 if (isVarArg) llvm_unreachable("PTX does not support varargs");
206 MachineFunction &MF = DAG.getMachineFunction();
207 const PTXSubtarget& ST = getTargetMachine().getSubtarget<PTXSubtarget>();
208 PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
209 PTXParamManager &PM = MFI->getParamManager();
213 llvm_unreachable("Unsupported calling convention");
214 case CallingConv::PTX_Kernel:
215 MFI->setKernel(true);
217 case CallingConv::PTX_Device:
218 MFI->setKernel(false);
222 // We do one of two things here:
223 // IsKernel || SM >= 2.0 -> Use param space for arguments
224 // SM < 2.0 -> Use registers for arguments
225 if (MFI->isKernel() || ST.useParamSpaceForDeviceArgs()) {
226 // We just need to emit the proper LOAD_PARAM ISDs
227 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
228 assert((!MFI->isKernel() || Ins[i].VT != MVT::i1) &&
229 "Kernels cannot take pred operands");
231 unsigned ParamSize = Ins[i].VT.getStoreSizeInBits();
232 unsigned Param = PM.addArgumentParam(ParamSize);
233 const std::string &ParamName = PM.getParamName(Param);
234 SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
236 SDValue ArgValue = DAG.getNode(PTXISD::LOAD_PARAM, dl, Ins[i].VT, Chain,
238 InVals.push_back(ArgValue);
242 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
243 EVT RegVT = Ins[i].VT;
244 const TargetRegisterClass* TRC = getRegClassFor(RegVT);
247 // Determine which register class we need
248 if (RegVT == MVT::i1)
249 RegType = PTXRegisterType::Pred;
250 else if (RegVT == MVT::i16)
251 RegType = PTXRegisterType::B16;
252 else if (RegVT == MVT::i32)
253 RegType = PTXRegisterType::B32;
254 else if (RegVT == MVT::i64)
255 RegType = PTXRegisterType::B64;
256 else if (RegVT == MVT::f32)
257 RegType = PTXRegisterType::F32;
258 else if (RegVT == MVT::f64)
259 RegType = PTXRegisterType::F64;
261 llvm_unreachable("Unknown parameter type");
263 // Use a unique index in the instruction to prevent instruction folding.
264 // Yes, this is a hack.
265 SDValue Index = DAG.getTargetConstant(i, MVT::i32);
266 unsigned Reg = MF.getRegInfo().createVirtualRegister(TRC);
267 SDValue ArgValue = DAG.getNode(PTXISD::READ_PARAM, dl, RegVT, Chain,
270 InVals.push_back(ArgValue);
272 MFI->addRegister(Reg, RegType, PTXRegisterSpace::Argument);
279 SDValue PTXTargetLowering::
280 LowerReturn(SDValue Chain,
281 CallingConv::ID CallConv,
283 const SmallVectorImpl<ISD::OutputArg> &Outs,
284 const SmallVectorImpl<SDValue> &OutVals,
286 SelectionDAG &DAG) const {
287 if (isVarArg) llvm_unreachable("PTX does not support varargs");
291 llvm_unreachable("Unsupported calling convention.");
292 case CallingConv::PTX_Kernel:
293 assert(Outs.size() == 0 && "Kernel must return void.");
294 return DAG.getNode(PTXISD::EXIT, dl, MVT::Other, Chain);
295 case CallingConv::PTX_Device:
296 assert(Outs.size() <= 1 && "Can at most return one value.");
300 MachineFunction& MF = DAG.getMachineFunction();
301 PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
302 PTXParamManager &PM = MFI->getParamManager();
305 const PTXSubtarget& ST = getTargetMachine().getSubtarget<PTXSubtarget>();
307 if (ST.useParamSpaceForDeviceArgs()) {
308 assert(Outs.size() < 2 && "Device functions can return at most one value");
310 if (Outs.size() == 1) {
311 unsigned ParamSize = OutVals[0].getValueType().getSizeInBits();
312 unsigned Param = PM.addReturnParam(ParamSize);
313 const std::string &ParamName = PM.getParamName(Param);
314 SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
316 Chain = DAG.getNode(PTXISD::STORE_PARAM, dl, MVT::Other, Chain,
317 ParamValue, OutVals[0]);
320 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
321 EVT RegVT = Outs[i].VT;
322 const TargetRegisterClass* TRC;
325 // Determine which register class we need
326 if (RegVT == MVT::i1) {
327 TRC = PTX::RegPredRegisterClass;
328 RegType = PTXRegisterType::Pred;
330 else if (RegVT == MVT::i16) {
331 TRC = PTX::RegI16RegisterClass;
332 RegType = PTXRegisterType::B16;
334 else if (RegVT == MVT::i32) {
335 TRC = PTX::RegI32RegisterClass;
336 RegType = PTXRegisterType::B32;
338 else if (RegVT == MVT::i64) {
339 TRC = PTX::RegI64RegisterClass;
340 RegType = PTXRegisterType::B64;
342 else if (RegVT == MVT::f32) {
343 TRC = PTX::RegF32RegisterClass;
344 RegType = PTXRegisterType::F32;
346 else if (RegVT == MVT::f64) {
347 TRC = PTX::RegF64RegisterClass;
348 RegType = PTXRegisterType::F64;
351 llvm_unreachable("Unknown parameter type");
354 unsigned Reg = MF.getRegInfo().createVirtualRegister(TRC);
356 SDValue Copy = DAG.getCopyToReg(Chain, dl, Reg, OutVals[i]/*, Flag*/);
357 SDValue OutReg = DAG.getRegister(Reg, RegVT);
359 Chain = DAG.getNode(PTXISD::WRITE_PARAM, dl, MVT::Other, Copy, OutReg);
361 MFI->addRegister(Reg, RegType, PTXRegisterSpace::Return);
365 if (Flag.getNode() == 0) {
366 return DAG.getNode(PTXISD::RET, dl, MVT::Other, Chain);
369 return DAG.getNode(PTXISD::RET, dl, MVT::Other, Chain, Flag);
374 PTXTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
375 CallingConv::ID CallConv, bool isVarArg,
376 bool doesNotRet, bool &isTailCall,
377 const SmallVectorImpl<ISD::OutputArg> &Outs,
378 const SmallVectorImpl<SDValue> &OutVals,
379 const SmallVectorImpl<ISD::InputArg> &Ins,
380 DebugLoc dl, SelectionDAG &DAG,
381 SmallVectorImpl<SDValue> &InVals) const {
383 MachineFunction& MF = DAG.getMachineFunction();
384 PTXMachineFunctionInfo *PTXMFI = MF.getInfo<PTXMachineFunctionInfo>();
385 PTXParamManager &PM = PTXMFI->getParamManager();
386 MachineFrameInfo *MFI = MF.getFrameInfo();
388 assert(getTargetMachine().getSubtarget<PTXSubtarget>().callsAreHandled() &&
389 "Calls are not handled for the target device");
391 // Identify the callee function
392 const GlobalValue *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
393 const Function *function = cast<Function>(GV);
395 // allow non-device calls only for printf
396 bool isPrintf = function->getName() == "printf" || function->getName() == "puts";
398 assert((isPrintf || function->getCallingConv() == CallingConv::PTX_Device) &&
399 "PTX function calls must be to PTX device functions");
401 unsigned outSize = isPrintf ? 2 : Outs.size();
403 std::vector<SDValue> Ops;
404 // The layout of the ops will be [Chain, #Ins, Ins, Callee, #Outs, Outs]
405 Ops.resize(outSize + Ins.size() + 4);
409 // Identify the callee function
410 Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy());
411 Ops[Ins.size()+2] = Callee;
414 Ops[Ins.size()+3] = DAG.getTargetConstant(outSize, MVT::i32);
417 // first argument is the address of the global string variable in memory
418 unsigned Param0 = PM.addLocalParam(getPointerTy().getSizeInBits());
419 SDValue ParamValue0 = DAG.getTargetExternalSymbol(PM.getParamName(Param0).c_str(),
421 Chain = DAG.getNode(PTXISD::STORE_PARAM, dl, MVT::Other, Chain,
422 ParamValue0, OutVals[0]);
423 Ops[Ins.size()+4] = ParamValue0;
425 // alignment is the maximum size of all the arguments
426 unsigned alignment = 0;
427 for (unsigned i = 1; i < OutVals.size(); ++i) {
428 alignment = std::max(alignment,
429 OutVals[i].getValueType().getSizeInBits());
432 // size is the alignment multiplied by the number of arguments
433 unsigned size = alignment * (OutVals.size() - 1);
435 // second argument is the address of the stack object (unless no arguments)
436 unsigned Param1 = PM.addLocalParam(getPointerTy().getSizeInBits());
437 SDValue ParamValue1 = DAG.getTargetExternalSymbol(PM.getParamName(Param1).c_str(),
439 Ops[Ins.size()+5] = ParamValue1;
443 // create a local stack object to store the arguments
444 unsigned StackObject = MFI->CreateStackObject(size / 8, alignment / 8, false);
445 SDValue FrameIndex = DAG.getFrameIndex(StackObject, getPointerTy());
447 // store each of the arguments to the stack in turn
448 for (unsigned int i = 1; i != OutVals.size(); i++) {
449 SDValue FrameAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), FrameIndex, DAG.getTargetConstant((i - 1) * 8, getPointerTy()));
450 Chain = DAG.getStore(Chain, dl, OutVals[i], FrameAddr,
451 MachinePointerInfo(),
455 // copy the address of the local frame index to get the address in non-local space
456 SDValue genericAddr = DAG.getNode(PTXISD::COPY_ADDRESS, dl, getPointerTy(), FrameIndex);
458 // store this address in the second argument
459 Chain = DAG.getNode(PTXISD::STORE_PARAM, dl, MVT::Other, Chain, ParamValue1, genericAddr);
464 // Generate STORE_PARAM nodes for each function argument. In PTX, function
465 // arguments are explicitly stored into .param variables and passed as
466 // arguments. There is no register/stack-based calling convention in PTX.
467 for (unsigned i = 0; i != OutVals.size(); ++i) {
468 unsigned Size = OutVals[i].getValueType().getSizeInBits();
469 unsigned Param = PM.addLocalParam(Size);
470 const std::string &ParamName = PM.getParamName(Param);
471 SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
473 Chain = DAG.getNode(PTXISD::STORE_PARAM, dl, MVT::Other, Chain,
474 ParamValue, OutVals[i]);
475 Ops[i+Ins.size()+4] = ParamValue;
479 std::vector<SDValue> InParams;
481 // Generate list of .param variables to hold the return value(s).
482 Ops[1] = DAG.getTargetConstant(Ins.size(), MVT::i32);
483 for (unsigned i = 0; i < Ins.size(); ++i) {
484 unsigned Size = Ins[i].VT.getStoreSizeInBits();
485 unsigned Param = PM.addLocalParam(Size);
486 const std::string &ParamName = PM.getParamName(Param);
487 SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
489 Ops[i+2] = ParamValue;
490 InParams.push_back(ParamValue);
495 // Create the CALL node.
496 Chain = DAG.getNode(PTXISD::CALL, dl, MVT::Other, &Ops[0], Ops.size());
498 // Create the LOAD_PARAM nodes that retrieve the function return value(s).
499 for (unsigned i = 0; i < Ins.size(); ++i) {
500 SDValue Load = DAG.getNode(PTXISD::LOAD_PARAM, dl, Ins[i].VT, Chain,
502 InVals.push_back(Load);
508 unsigned PTXTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT) {
509 // All arguments consist of one "register," regardless of the type.