- RegsToPass[i].second.getValueType()));
-
- if (InFlag.Val)
- Ops.push_back(InFlag);
-
- // FIXME: Do not generate X86ISD::TAILCALL for now.
- Chain = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
- NodeTys, &Ops[0], Ops.size());
- InFlag = Chain.getValue(1);
-
- // Returns a flag for retval copy to use.
- NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
- Ops.clear();
- Ops.push_back(Chain);
- Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
- Ops.push_back(DAG.getConstant(0, getPointerTy()));
- Ops.push_back(InFlag);
- Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, &Ops[0], Ops.size());
- InFlag = Chain.getValue(1);
-
- // Handle result values, copying them out of physregs into vregs that we
- // return.
- return SDOperand(LowerCallResult(Chain, InFlag, Op.Val, CC, DAG), Op.ResNo);
-}
-
-//===----------------------------------------------------------------------===//
-// Fast & FastCall Calling Convention implementation
-//===----------------------------------------------------------------------===//
-//
-// The X86 'fast' calling convention passes up to two integer arguments in
-// registers (an appropriate portion of EAX/EDX), passes arguments in C order,
-// and requires that the callee pop its arguments off the stack (allowing proper
-// tail calls), and has the same return value conventions as C calling convs.
-//
-// This calling convention always arranges for the callee pop value to be 8n+4
-// bytes, which is needed for tail recursion elimination and stack alignment
-// reasons.
-//
-// Note that this can be enhanced in the future to pass fp vals in registers
-// (when we have a global fp allocator) and do other tricks.
-//
-//===----------------------------------------------------------------------===//
-// The X86 'fastcall' calling convention passes up to two integer arguments in
-// registers (an appropriate portion of ECX/EDX), passes arguments in C order,
-// and requires that the callee pop its arguments off the stack (allowing proper
-// tail calls), and has the same return value conventions as C calling convs.
-//
-// This calling convention always arranges for the callee pop value to be 8n+4
-// bytes, which is needed for tail recursion elimination and stack alignment
-// reasons.
-SDOperand
-X86TargetLowering::LowerFastCCArguments(SDOperand Op, SelectionDAG &DAG,
- bool isFastCall) {
- unsigned NumArgs = Op.Val->getNumValues()-1;
- MachineFunction &MF = DAG.getMachineFunction();
- MachineFrameInfo *MFI = MF.getFrameInfo();
- SDOperand Root = Op.getOperand(0);
- SmallVector<SDOperand, 8> ArgValues;
-
- // Add DAG nodes to load the arguments... On entry to a function the stack
- // frame looks like this:
- //
- // [ESP] -- return address
- // [ESP + 4] -- first nonreg argument (leftmost lexically)
- // [ESP + 8] -- second nonreg argument, if 1st argument is <= 4 bytes in size
- // ...
- unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot
-
- // Keep track of the number of integer regs passed so far. This can be either
- // 0 (neither EAX/ECX or EDX used), 1 (EAX/ECX is used) or 2 (EAX/ECX and EDX
- // are both used).
- unsigned NumIntRegs = 0;
- unsigned NumXMMRegs = 0; // XMM regs used for parameter passing.
-
- static const unsigned XMMArgRegs[] = {
- X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3
- };
-
- static const unsigned GPRArgRegs[][2][2] = {
- {{ X86::AL, X86::DL }, { X86::CL, X86::DL }},
- {{ X86::AX, X86::DX }, { X86::CX, X86::DX }},
- {{ X86::EAX, X86::EDX }, { X86::ECX, X86::EDX }}
- };
-
- static const TargetRegisterClass* GPRClasses[3] = {
- X86::GR8RegisterClass, X86::GR16RegisterClass, X86::GR32RegisterClass
- };
-
- unsigned GPRInd = (isFastCall ? 1 : 0);
- for (unsigned i = 0; i < NumArgs; ++i) {
- MVT::ValueType ObjectVT = Op.getValue(i).getValueType();
- unsigned ArgIncrement = 4;
- unsigned ObjSize = 0;
- unsigned ObjXMMRegs = 0;
- unsigned ObjIntRegs = 0;
- unsigned Reg = 0;
- SDOperand ArgValue;
-
- HowToPassCallArgument(ObjectVT,
- true, // Use as much registers as possible
- NumIntRegs, NumXMMRegs,
- (isFastCall ? 2 : FASTCC_NUM_INT_ARGS_INREGS),
- ObjSize, ObjIntRegs, ObjXMMRegs);
-
- if (ObjSize > 4)
- ArgIncrement = ObjSize;
-
- if (ObjIntRegs || ObjXMMRegs) {
- switch (ObjectVT) {
- default: assert(0 && "Unhandled argument type!");
- case MVT::i8:
- case MVT::i16:
- case MVT::i32: {
- unsigned RegToUse = GPRArgRegs[ObjectVT-MVT::i8][GPRInd][NumIntRegs];
- Reg = AddLiveIn(MF, RegToUse, GPRClasses[ObjectVT-MVT::i8]);
- ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT);
- break;
- }
- case MVT::v16i8:
- case MVT::v8i16:
- case MVT::v4i32:
- case MVT::v2i64:
- case MVT::v4f32:
- case MVT::v2f64: {
- assert(!isFastCall && "Unhandled argument type!");
- Reg = AddLiveIn(MF, XMMArgRegs[NumXMMRegs], X86::VR128RegisterClass);
- ArgValue = DAG.getCopyFromReg(Root, Reg, ObjectVT);
- break;
- }
- }
- NumIntRegs += ObjIntRegs;
- NumXMMRegs += ObjXMMRegs;
- }
- if (ObjSize) {
- // XMM arguments have to be aligned on 16-byte boundary.
- if (ObjSize == 16)
- ArgOffset = ((ArgOffset + 15) / 16) * 16;
- // Create the SelectionDAG nodes corresponding to a load from this
- // parameter.
- int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
- SDOperand FIN = DAG.getFrameIndex(FI, getPointerTy());
- ArgValue = DAG.getLoad(Op.Val->getValueType(i), Root, FIN, NULL, 0);
-
- ArgOffset += ArgIncrement; // Move on to the next argument.
- }
-
- ArgValues.push_back(ArgValue);
- }
-
- ArgValues.push_back(Root);
-
- // Make sure the instruction takes 8n+4 bytes to make sure the start of the
- // arguments and the arguments after the retaddr has been pushed are aligned.
- if ((ArgOffset & 7) == 0)
- ArgOffset += 4;
-
- VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs.
- RegSaveFrameIndex = 0xAAAAAAA; // X86-64 only.
- ReturnAddrIndex = 0; // No return address slot generated yet.
- BytesToPopOnReturn = ArgOffset; // Callee pops all stack arguments.
- BytesCallerReserves = 0;
-
- MF.getInfo<X86FunctionInfo>()->setBytesToPopOnReturn(BytesToPopOnReturn);
-
- // Finally, inform the code generator which regs we return values in.
- switch (getValueType(MF.getFunction()->getReturnType())) {
- default: assert(0 && "Unknown type!");
- case MVT::isVoid: break;
- case MVT::i1:
- case MVT::i8:
- case MVT::i16:
- case MVT::i32:
- MF.addLiveOut(X86::EAX);
- break;
- case MVT::i64:
- MF.addLiveOut(X86::EAX);
- MF.addLiveOut(X86::EDX);
- break;
- case MVT::f32:
- case MVT::f64:
- MF.addLiveOut(X86::ST0);
- break;
- case MVT::v16i8:
- case MVT::v8i16:
- case MVT::v4i32:
- case MVT::v2i64:
- case MVT::v4f32:
- case MVT::v2f64:
- assert(!isFastCall && "Unknown result type");
- MF.addLiveOut(X86::XMM0);
- break;
- }
-
- // Return the new list of results.
- return DAG.getNode(ISD::MERGE_VALUES, Op.Val->getVTList(),
- &ArgValues[0], ArgValues.size()).getValue(Op.ResNo);
-}
-
-SDOperand X86TargetLowering::LowerFastCCCallTo(SDOperand Op, SelectionDAG &DAG,
- unsigned CC) {
- SDOperand Chain = Op.getOperand(0);
- bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0;
- SDOperand Callee = Op.getOperand(4);
- unsigned NumOps = (Op.getNumOperands() - 5) / 2;
-
- // Count how many bytes are to be pushed on the stack.
- unsigned NumBytes = 0;
-
- // Keep track of the number of integer regs passed so far. This can be either
- // 0 (neither EAX/ECX or EDX used), 1 (EAX/ECX is used) or 2 (EAX/ECX and EDX
- // are both used).
- unsigned NumIntRegs = 0;
- unsigned NumXMMRegs = 0; // XMM regs used for parameter passing.
-
- static const unsigned GPRArgRegs[][2][2] = {
- {{ X86::AL, X86::DL }, { X86::CL, X86::DL }},
- {{ X86::AX, X86::DX }, { X86::CX, X86::DX }},
- {{ X86::EAX, X86::EDX }, { X86::ECX, X86::EDX }}
- };
- static const unsigned XMMArgRegs[] = {
- X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3
- };
-
- bool isFastCall = CC == CallingConv::X86_FastCall;
- unsigned GPRInd = isFastCall ? 1 : 0;
- for (unsigned i = 0; i != NumOps; ++i) {
- SDOperand Arg = Op.getOperand(5+2*i);
-
- switch (Arg.getValueType()) {
- default: assert(0 && "Unknown value type!");
- case MVT::i8:
- case MVT::i16:
- case MVT::i32: {
- unsigned MaxNumIntRegs = (isFastCall ? 2 : FASTCC_NUM_INT_ARGS_INREGS);
- if (NumIntRegs < MaxNumIntRegs) {
- ++NumIntRegs;
- break;
- }
- } // Fall through
- case MVT::f32:
- NumBytes += 4;
- break;
- case MVT::f64:
- NumBytes += 8;
- break;
- case MVT::v16i8:
- case MVT::v8i16:
- case MVT::v4i32:
- case MVT::v2i64:
- case MVT::v4f32:
- case MVT::v2f64:
- assert(!isFastCall && "Unknown value type!");
- if (NumXMMRegs < 4)
- NumXMMRegs++;
- else {
- // XMM arguments have to be aligned on 16-byte boundary.
- NumBytes = ((NumBytes + 15) / 16) * 16;
- NumBytes += 16;
- }
- break;
- }
- }
-
- // Make sure the instruction takes 8n+4 bytes to make sure the start of the
- // arguments and the arguments after the retaddr has been pushed are aligned.
- if ((NumBytes & 7) == 0)
- NumBytes += 4;
-
- Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy()));
-
- // Arguments go on the stack in reverse order, as specified by the ABI.
- unsigned ArgOffset = 0;
- NumIntRegs = 0;
- SmallVector<std::pair<unsigned, SDOperand>, 8> RegsToPass;
- SmallVector<SDOperand, 8> MemOpChains;
- SDOperand StackPtr = DAG.getRegister(X86StackPtr, getPointerTy());
- for (unsigned i = 0; i != NumOps; ++i) {
- SDOperand Arg = Op.getOperand(5+2*i);
-
- switch (Arg.getValueType()) {
- default: assert(0 && "Unexpected ValueType for argument!");
- case MVT::i8:
- case MVT::i16:
- case MVT::i32: {
- unsigned MaxNumIntRegs = (isFastCall ? 2 : FASTCC_NUM_INT_ARGS_INREGS);
- if (NumIntRegs < MaxNumIntRegs) {
- unsigned RegToUse =
- GPRArgRegs[Arg.getValueType()-MVT::i8][GPRInd][NumIntRegs];
- RegsToPass.push_back(std::make_pair(RegToUse, Arg));
- ++NumIntRegs;
- break;
- }
- } // Fall through
- case MVT::f32: {
- SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
- PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
- MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
- ArgOffset += 4;
- break;
- }
- case MVT::f64: {
- SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
- PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
- MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
- ArgOffset += 8;
- break;
- }
- case MVT::v16i8:
- case MVT::v8i16:
- case MVT::v4i32:
- case MVT::v2i64:
- case MVT::v4f32:
- case MVT::v2f64:
- assert(!isFastCall && "Unexpected ValueType for argument!");
- if (NumXMMRegs < 4) {
- RegsToPass.push_back(std::make_pair(XMMArgRegs[NumXMMRegs], Arg));
- NumXMMRegs++;
- } else {
- // XMM arguments have to be aligned on 16-byte boundary.
- ArgOffset = ((ArgOffset + 15) / 16) * 16;
- SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
- PtrOff = DAG.getNode(ISD::ADD, getPointerTy(), StackPtr, PtrOff);
- MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
- ArgOffset += 16;
- }
- break;
- }
- }
-
- if (!MemOpChains.empty())
- Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
- &MemOpChains[0], MemOpChains.size());
-
- // Build a sequence of copy-to-reg nodes chained together with token chain
- // and flag operands which copy the outgoing args into registers.
- SDOperand InFlag;
- for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
- Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
- InFlag);
- InFlag = Chain.getValue(1);
- }
-
- // If the callee is a GlobalAddress node (quite common, every direct call is)
- // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
- if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
- // We should use extra load for direct calls to dllimported functions in
- // non-JIT mode.
- if (!Subtarget->GVRequiresExtraLoad(G->getGlobal(),
- getTargetMachine(), true))
- Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
- } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
- Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
-
- // ELF / PIC requires GOT in the EBX register before function calls via PLT
- // GOT pointer.
- if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
- Subtarget->isPICStyleGOT()) {
- Chain = DAG.getCopyToReg(Chain, X86::EBX,
- DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()),
- InFlag);
- InFlag = Chain.getValue(1);
- }
-
- // Returns a chain & a flag for retval copy to use.
- SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
- SmallVector<SDOperand, 8> Ops;
- Ops.push_back(Chain);
- Ops.push_back(Callee);
-
- // Add argument registers to the end of the list so that they are known live
- // into the call.
- for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
- Ops.push_back(DAG.getRegister(RegsToPass[i].first,
- RegsToPass[i].second.getValueType()));
-
- // Add an implicit use GOT pointer in EBX.
- if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
- Subtarget->isPICStyleGOT())
- Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy()));