1 //===-- X86FastISel.cpp - X86 FastISel implementation ---------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the X86-specific support for the FastISel class. Much
11 // of the target-specific code is generated by tablegen in the file
12 // X86GenFastISel.inc, which is #included here.
14 //===----------------------------------------------------------------------===//
17 #include "X86CallingConv.h"
18 #include "X86InstrBuilder.h"
19 #include "X86InstrInfo.h"
20 #include "X86MachineFunctionInfo.h"
21 #include "X86RegisterInfo.h"
22 #include "X86Subtarget.h"
23 #include "X86TargetMachine.h"
24 #include "llvm/Analysis/BranchProbabilityInfo.h"
25 #include "llvm/CodeGen/Analysis.h"
26 #include "llvm/CodeGen/FastISel.h"
27 #include "llvm/CodeGen/FunctionLoweringInfo.h"
28 #include "llvm/CodeGen/MachineConstantPool.h"
29 #include "llvm/CodeGen/MachineFrameInfo.h"
30 #include "llvm/CodeGen/MachineRegisterInfo.h"
31 #include "llvm/IR/CallSite.h"
32 #include "llvm/IR/CallingConv.h"
33 #include "llvm/IR/DerivedTypes.h"
34 #include "llvm/IR/GetElementPtrTypeIterator.h"
35 #include "llvm/IR/GlobalAlias.h"
36 #include "llvm/IR/GlobalVariable.h"
37 #include "llvm/IR/Instructions.h"
38 #include "llvm/IR/IntrinsicInst.h"
39 #include "llvm/IR/Operator.h"
40 #include "llvm/MC/MCAsmInfo.h"
41 #include "llvm/MC/MCSymbol.h"
42 #include "llvm/Support/ErrorHandling.h"
43 #include "llvm/Target/TargetOptions.h"
48 class X86FastISel final : public FastISel {
49 /// Subtarget - Keep a pointer to the X86Subtarget around so that we can
50 /// make the right decision when generating code for different targets.
51 const X86Subtarget *Subtarget;
53 /// X86ScalarSSEf32, X86ScalarSSEf64 - Select between SSE or x87
54 /// floating point ops.
55 /// When SSE is available, use it for f32 operations.
56 /// When SSE2 is available, use it for f64 operations.
61 explicit X86FastISel(FunctionLoweringInfo &funcInfo,
62 const TargetLibraryInfo *libInfo)
63 : FastISel(funcInfo, libInfo) {
64 Subtarget = &funcInfo.MF->getSubtarget<X86Subtarget>();
65 X86ScalarSSEf64 = Subtarget->hasSSE2();
66 X86ScalarSSEf32 = Subtarget->hasSSE1();
69 bool fastSelectInstruction(const Instruction *I) override;
71 /// \brief The specified machine instr operand is a vreg, and that
72 /// vreg is being provided by the specified load instruction. If possible,
73 /// try to fold the load as an operand to the instruction, returning true if
75 bool tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
76 const LoadInst *LI) override;
78 bool fastLowerArguments() override;
79 bool fastLowerCall(CallLoweringInfo &CLI) override;
80 bool fastLowerIntrinsicCall(const IntrinsicInst *II) override;
82 #include "X86GenFastISel.inc"
85 bool X86FastEmitCompare(const Value *LHS, const Value *RHS, EVT VT, DebugLoc DL);
87 bool X86FastEmitLoad(EVT VT, X86AddressMode &AM, MachineMemOperand *MMO,
88 unsigned &ResultReg, unsigned Alignment = 1);
90 bool X86FastEmitStore(EVT VT, const Value *Val, X86AddressMode &AM,
91 MachineMemOperand *MMO = nullptr, bool Aligned = false);
92 bool X86FastEmitStore(EVT VT, unsigned ValReg, bool ValIsKill,
94 MachineMemOperand *MMO = nullptr, bool Aligned = false);
96 bool X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT, unsigned Src, EVT SrcVT,
99 bool X86SelectAddress(const Value *V, X86AddressMode &AM);
100 bool X86SelectCallAddress(const Value *V, X86AddressMode &AM);
102 bool X86SelectLoad(const Instruction *I);
104 bool X86SelectStore(const Instruction *I);
106 bool X86SelectRet(const Instruction *I);
108 bool X86SelectCmp(const Instruction *I);
110 bool X86SelectZExt(const Instruction *I);
112 bool X86SelectBranch(const Instruction *I);
114 bool X86SelectShift(const Instruction *I);
116 bool X86SelectDivRem(const Instruction *I);
118 bool X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I);
120 bool X86FastEmitSSESelect(MVT RetVT, const Instruction *I);
122 bool X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I);
124 bool X86SelectSelect(const Instruction *I);
126 bool X86SelectTrunc(const Instruction *I);
128 bool X86SelectFPExtOrFPTrunc(const Instruction *I, unsigned Opc,
129 const TargetRegisterClass *RC);
131 bool X86SelectFPExt(const Instruction *I);
132 bool X86SelectFPTrunc(const Instruction *I);
133 bool X86SelectSIToFP(const Instruction *I);
135 const X86InstrInfo *getInstrInfo() const {
136 return Subtarget->getInstrInfo();
138 const X86TargetMachine *getTargetMachine() const {
139 return static_cast<const X86TargetMachine *>(&TM);
142 bool handleConstantAddresses(const Value *V, X86AddressMode &AM);
144 unsigned X86MaterializeInt(const ConstantInt *CI, MVT VT);
145 unsigned X86MaterializeFP(const ConstantFP *CFP, MVT VT);
146 unsigned X86MaterializeGV(const GlobalValue *GV, MVT VT);
147 unsigned fastMaterializeConstant(const Constant *C) override;
149 unsigned fastMaterializeAlloca(const AllocaInst *C) override;
151 unsigned fastMaterializeFloatZero(const ConstantFP *CF) override;
153 /// isScalarFPTypeInSSEReg - Return true if the specified scalar FP type is
154 /// computed in an SSE register, not on the X87 floating point stack.
155 bool isScalarFPTypeInSSEReg(EVT VT) const {
156 return (VT == MVT::f64 && X86ScalarSSEf64) || // f64 is when SSE2
157 (VT == MVT::f32 && X86ScalarSSEf32); // f32 is when SSE1
160 bool isTypeLegal(Type *Ty, MVT &VT, bool AllowI1 = false);
162 bool IsMemcpySmall(uint64_t Len);
164 bool TryEmitSmallMemcpy(X86AddressMode DestAM,
165 X86AddressMode SrcAM, uint64_t Len);
167 bool foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
170 const MachineInstrBuilder &addFullAddress(const MachineInstrBuilder &MIB,
174 } // end anonymous namespace.
176 static std::pair<X86::CondCode, bool>
177 getX86ConditionCode(CmpInst::Predicate Predicate) {
178 X86::CondCode CC = X86::COND_INVALID;
179 bool NeedSwap = false;
182 // Floating-point Predicates
183 case CmpInst::FCMP_UEQ: CC = X86::COND_E; break;
184 case CmpInst::FCMP_OLT: NeedSwap = true; // fall-through
185 case CmpInst::FCMP_OGT: CC = X86::COND_A; break;
186 case CmpInst::FCMP_OLE: NeedSwap = true; // fall-through
187 case CmpInst::FCMP_OGE: CC = X86::COND_AE; break;
188 case CmpInst::FCMP_UGT: NeedSwap = true; // fall-through
189 case CmpInst::FCMP_ULT: CC = X86::COND_B; break;
190 case CmpInst::FCMP_UGE: NeedSwap = true; // fall-through
191 case CmpInst::FCMP_ULE: CC = X86::COND_BE; break;
192 case CmpInst::FCMP_ONE: CC = X86::COND_NE; break;
193 case CmpInst::FCMP_UNO: CC = X86::COND_P; break;
194 case CmpInst::FCMP_ORD: CC = X86::COND_NP; break;
195 case CmpInst::FCMP_OEQ: // fall-through
196 case CmpInst::FCMP_UNE: CC = X86::COND_INVALID; break;
198 // Integer Predicates
199 case CmpInst::ICMP_EQ: CC = X86::COND_E; break;
200 case CmpInst::ICMP_NE: CC = X86::COND_NE; break;
201 case CmpInst::ICMP_UGT: CC = X86::COND_A; break;
202 case CmpInst::ICMP_UGE: CC = X86::COND_AE; break;
203 case CmpInst::ICMP_ULT: CC = X86::COND_B; break;
204 case CmpInst::ICMP_ULE: CC = X86::COND_BE; break;
205 case CmpInst::ICMP_SGT: CC = X86::COND_G; break;
206 case CmpInst::ICMP_SGE: CC = X86::COND_GE; break;
207 case CmpInst::ICMP_SLT: CC = X86::COND_L; break;
208 case CmpInst::ICMP_SLE: CC = X86::COND_LE; break;
211 return std::make_pair(CC, NeedSwap);
214 static std::pair<unsigned, bool>
215 getX86SSEConditionCode(CmpInst::Predicate Predicate) {
217 bool NeedSwap = false;
219 // SSE Condition code mapping:
229 default: llvm_unreachable("Unexpected predicate");
230 case CmpInst::FCMP_OEQ: CC = 0; break;
231 case CmpInst::FCMP_OGT: NeedSwap = true; // fall-through
232 case CmpInst::FCMP_OLT: CC = 1; break;
233 case CmpInst::FCMP_OGE: NeedSwap = true; // fall-through
234 case CmpInst::FCMP_OLE: CC = 2; break;
235 case CmpInst::FCMP_UNO: CC = 3; break;
236 case CmpInst::FCMP_UNE: CC = 4; break;
237 case CmpInst::FCMP_ULE: NeedSwap = true; // fall-through
238 case CmpInst::FCMP_UGE: CC = 5; break;
239 case CmpInst::FCMP_ULT: NeedSwap = true; // fall-through
240 case CmpInst::FCMP_UGT: CC = 6; break;
241 case CmpInst::FCMP_ORD: CC = 7; break;
242 case CmpInst::FCMP_UEQ:
243 case CmpInst::FCMP_ONE: CC = 8; break;
246 return std::make_pair(CC, NeedSwap);
249 /// \brief Adds a complex addressing mode to the given machine instr builder.
250 /// Note, this will constrain the index register. If its not possible to
251 /// constrain the given index register, then a new one will be created. The
252 /// IndexReg field of the addressing mode will be updated to match in this case.
253 const MachineInstrBuilder &
254 X86FastISel::addFullAddress(const MachineInstrBuilder &MIB,
255 X86AddressMode &AM) {
256 // First constrain the index register. It needs to be a GR64_NOSP.
257 AM.IndexReg = constrainOperandRegClass(MIB->getDesc(), AM.IndexReg,
258 MIB->getNumOperands() +
260 return ::addFullAddress(MIB, AM);
263 /// \brief Check if it is possible to fold the condition from the XALU intrinsic
264 /// into the user. The condition code will only be updated on success.
265 bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
267 if (!isa<ExtractValueInst>(Cond))
270 const auto *EV = cast<ExtractValueInst>(Cond);
271 if (!isa<IntrinsicInst>(EV->getAggregateOperand()))
274 const auto *II = cast<IntrinsicInst>(EV->getAggregateOperand());
276 const Function *Callee = II->getCalledFunction();
278 cast<StructType>(Callee->getReturnType())->getTypeAtIndex(0U);
279 if (!isTypeLegal(RetTy, RetVT))
282 if (RetVT != MVT::i32 && RetVT != MVT::i64)
286 switch (II->getIntrinsicID()) {
287 default: return false;
288 case Intrinsic::sadd_with_overflow:
289 case Intrinsic::ssub_with_overflow:
290 case Intrinsic::smul_with_overflow:
291 case Intrinsic::umul_with_overflow: TmpCC = X86::COND_O; break;
292 case Intrinsic::uadd_with_overflow:
293 case Intrinsic::usub_with_overflow: TmpCC = X86::COND_B; break;
296 // Check if both instructions are in the same basic block.
297 if (II->getParent() != I->getParent())
300 // Make sure nothing is in the way
301 BasicBlock::const_iterator Start(I);
302 BasicBlock::const_iterator End(II);
303 for (auto Itr = std::prev(Start); Itr != End; --Itr) {
304 // We only expect extractvalue instructions between the intrinsic and the
305 // instruction to be selected.
306 if (!isa<ExtractValueInst>(Itr))
309 // Check that the extractvalue operand comes from the intrinsic.
310 const auto *EVI = cast<ExtractValueInst>(Itr);
311 if (EVI->getAggregateOperand() != II)
319 bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
320 EVT evt = TLI.getValueType(DL, Ty, /*HandleUnknown=*/true);
321 if (evt == MVT::Other || !evt.isSimple())
322 // Unhandled type. Halt "fast" selection and bail.
325 VT = evt.getSimpleVT();
326 // For now, require SSE/SSE2 for performing floating-point operations,
327 // since x87 requires additional work.
328 if (VT == MVT::f64 && !X86ScalarSSEf64)
330 if (VT == MVT::f32 && !X86ScalarSSEf32)
332 // Similarly, no f80 support yet.
335 // We only handle legal types. For example, on x86-32 the instruction
336 // selector contains all of the 64-bit instructions from x86-64,
337 // under the assumption that i64 won't be used if the target doesn't
339 return (AllowI1 && VT == MVT::i1) || TLI.isTypeLegal(VT);
342 #include "X86GenCallingConv.inc"
344 /// X86FastEmitLoad - Emit a machine instruction to load a value of type VT.
345 /// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.
346 /// Return true and the result register by reference if it is possible.
347 bool X86FastISel::X86FastEmitLoad(EVT VT, X86AddressMode &AM,
348 MachineMemOperand *MMO, unsigned &ResultReg,
349 unsigned Alignment) {
350 // Get opcode and regclass of the output for the given load instruction.
352 const TargetRegisterClass *RC = nullptr;
353 switch (VT.getSimpleVT().SimpleTy) {
354 default: return false;
358 RC = &X86::GR8RegClass;
362 RC = &X86::GR16RegClass;
366 RC = &X86::GR32RegClass;
369 // Must be in x86-64 mode.
371 RC = &X86::GR64RegClass;
374 if (X86ScalarSSEf32) {
375 Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm;
376 RC = &X86::FR32RegClass;
379 RC = &X86::RFP32RegClass;
383 if (X86ScalarSSEf64) {
384 Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm;
385 RC = &X86::FR64RegClass;
388 RC = &X86::RFP64RegClass;
392 // No f80 support yet.
396 Opc = Subtarget->hasAVX() ? X86::VMOVAPSrm : X86::MOVAPSrm;
398 Opc = Subtarget->hasAVX() ? X86::VMOVUPSrm : X86::MOVUPSrm;
399 RC = &X86::VR128RegClass;
403 Opc = Subtarget->hasAVX() ? X86::VMOVAPDrm : X86::MOVAPDrm;
405 Opc = Subtarget->hasAVX() ? X86::VMOVUPDrm : X86::MOVUPDrm;
406 RC = &X86::VR128RegClass;
413 Opc = Subtarget->hasAVX() ? X86::VMOVDQArm : X86::MOVDQArm;
415 Opc = Subtarget->hasAVX() ? X86::VMOVDQUrm : X86::MOVDQUrm;
416 RC = &X86::VR128RegClass;
420 ResultReg = createResultReg(RC);
421 MachineInstrBuilder MIB =
422 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg);
423 addFullAddress(MIB, AM);
425 MIB->addMemOperand(*FuncInfo.MF, MMO);
429 /// X86FastEmitStore - Emit a machine instruction to store a value Val of
430 /// type VT. The address is either pre-computed, consisted of a base ptr, Ptr
431 /// and a displacement offset, or a GlobalAddress,
432 /// i.e. V. Return true if it is possible.
433 bool X86FastISel::X86FastEmitStore(EVT VT, unsigned ValReg, bool ValIsKill,
435 MachineMemOperand *MMO, bool Aligned) {
436 bool HasSSE2 = Subtarget->hasSSE2();
437 bool HasSSE4A = Subtarget->hasSSE4A();
438 bool HasAVX = Subtarget->hasAVX();
439 bool IsNonTemporal = MMO && MMO->isNonTemporal();
441 // Get opcode and regclass of the output for the given store instruction.
443 switch (VT.getSimpleVT().SimpleTy) {
444 case MVT::f80: // No f80 support yet.
445 default: return false;
447 // Mask out all but lowest bit.
448 unsigned AndResult = createResultReg(&X86::GR8RegClass);
449 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
450 TII.get(X86::AND8ri), AndResult)
451 .addReg(ValReg, getKillRegState(ValIsKill)).addImm(1);
454 // FALLTHROUGH, handling i1 as i8.
455 case MVT::i8: Opc = X86::MOV8mr; break;
456 case MVT::i16: Opc = X86::MOV16mr; break;
458 Opc = (IsNonTemporal && HasSSE2) ? X86::MOVNTImr : X86::MOV32mr;
461 // Must be in x86-64 mode.
462 Opc = (IsNonTemporal && HasSSE2) ? X86::MOVNTI_64mr : X86::MOV64mr;
465 if (X86ScalarSSEf32) {
466 if (IsNonTemporal && HasSSE4A)
469 Opc = HasAVX ? X86::VMOVSSmr : X86::MOVSSmr;
474 if (X86ScalarSSEf32) {
475 if (IsNonTemporal && HasSSE4A)
478 Opc = HasAVX ? X86::VMOVSDmr : X86::MOVSDmr;
485 Opc = HasAVX ? X86::VMOVNTPSmr : X86::MOVNTPSmr;
487 Opc = HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr;
489 Opc = HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr;
494 Opc = HasAVX ? X86::VMOVNTPDmr : X86::MOVNTPDmr;
496 Opc = HasAVX ? X86::VMOVAPDmr : X86::MOVAPDmr;
498 Opc = HasAVX ? X86::VMOVUPDmr : X86::MOVUPDmr;
506 Opc = HasAVX ? X86::VMOVNTDQmr : X86::MOVNTDQmr;
508 Opc = HasAVX ? X86::VMOVDQAmr : X86::MOVDQAmr;
510 Opc = Subtarget->hasAVX() ? X86::VMOVDQUmr : X86::MOVDQUmr;
514 MachineInstrBuilder MIB =
515 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc));
516 addFullAddress(MIB, AM).addReg(ValReg, getKillRegState(ValIsKill));
518 MIB->addMemOperand(*FuncInfo.MF, MMO);
523 bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val,
525 MachineMemOperand *MMO, bool Aligned) {
526 // Handle 'null' like i32/i64 0.
527 if (isa<ConstantPointerNull>(Val))
528 Val = Constant::getNullValue(DL.getIntPtrType(Val->getContext()));
530 // If this is a store of a simple constant, fold the constant into the store.
531 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val)) {
534 switch (VT.getSimpleVT().SimpleTy) {
536 case MVT::i1: Signed = false; // FALLTHROUGH to handle as i8.
537 case MVT::i8: Opc = X86::MOV8mi; break;
538 case MVT::i16: Opc = X86::MOV16mi; break;
539 case MVT::i32: Opc = X86::MOV32mi; break;
541 // Must be a 32-bit sign extended value.
542 if (isInt<32>(CI->getSExtValue()))
543 Opc = X86::MOV64mi32;
548 MachineInstrBuilder MIB =
549 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc));
550 addFullAddress(MIB, AM).addImm(Signed ? (uint64_t) CI->getSExtValue()
551 : CI->getZExtValue());
553 MIB->addMemOperand(*FuncInfo.MF, MMO);
558 unsigned ValReg = getRegForValue(Val);
562 bool ValKill = hasTrivialKill(Val);
563 return X86FastEmitStore(VT, ValReg, ValKill, AM, MMO, Aligned);
566 /// X86FastEmitExtend - Emit a machine instruction to extend a value Src of
567 /// type SrcVT to type DstVT using the specified extension opcode Opc (e.g.
568 /// ISD::SIGN_EXTEND).
569 bool X86FastISel::X86FastEmitExtend(ISD::NodeType Opc, EVT DstVT,
570 unsigned Src, EVT SrcVT,
571 unsigned &ResultReg) {
572 unsigned RR = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Opc,
573 Src, /*TODO: Kill=*/false);
581 bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
582 // Handle constant address.
583 if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
584 // Can't handle alternate code models yet.
585 if (TM.getCodeModel() != CodeModel::Small)
588 // Can't handle TLS yet.
589 if (GV->isThreadLocal())
592 // RIP-relative addresses can't have additional register operands, so if
593 // we've already folded stuff into the addressing mode, just force the
594 // global value into its own register, which we can use as the basereg.
595 if (!Subtarget->isPICStyleRIPRel() ||
596 (AM.Base.Reg == 0 && AM.IndexReg == 0)) {
597 // Okay, we've committed to selecting this global. Set up the address.
600 // Allow the subtarget to classify the global.
601 unsigned char GVFlags = Subtarget->ClassifyGlobalReference(GV, TM);
603 // If this reference is relative to the pic base, set it now.
604 if (isGlobalRelativeToPICBase(GVFlags)) {
605 // FIXME: How do we know Base.Reg is free??
606 AM.Base.Reg = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
609 // Unless the ABI requires an extra load, return a direct reference to
611 if (!isGlobalStubReference(GVFlags)) {
612 if (Subtarget->isPICStyleRIPRel()) {
613 // Use rip-relative addressing if we can. Above we verified that the
614 // base and index registers are unused.
615 assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
616 AM.Base.Reg = X86::RIP;
618 AM.GVOpFlags = GVFlags;
622 // Ok, we need to do a load from a stub. If we've already loaded from
623 // this stub, reuse the loaded pointer, otherwise emit the load now.
624 DenseMap<const Value *, unsigned>::iterator I = LocalValueMap.find(V);
626 if (I != LocalValueMap.end() && I->second != 0) {
629 // Issue load from stub.
631 const TargetRegisterClass *RC = nullptr;
632 X86AddressMode StubAM;
633 StubAM.Base.Reg = AM.Base.Reg;
635 StubAM.GVOpFlags = GVFlags;
637 // Prepare for inserting code in the local-value area.
638 SavePoint SaveInsertPt = enterLocalValueArea();
640 if (TLI.getPointerTy(DL) == MVT::i64) {
642 RC = &X86::GR64RegClass;
644 if (Subtarget->isPICStyleRIPRel())
645 StubAM.Base.Reg = X86::RIP;
648 RC = &X86::GR32RegClass;
651 LoadReg = createResultReg(RC);
652 MachineInstrBuilder LoadMI =
653 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), LoadReg);
654 addFullAddress(LoadMI, StubAM);
656 // Ok, back to normal mode.
657 leaveLocalValueArea(SaveInsertPt);
659 // Prevent loading GV stub multiple times in same MBB.
660 LocalValueMap[V] = LoadReg;
663 // Now construct the final address. Note that the Disp, Scale,
664 // and Index values may already be set here.
665 AM.Base.Reg = LoadReg;
671 // If all else fails, try to materialize the value in a register.
672 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
673 if (AM.Base.Reg == 0) {
674 AM.Base.Reg = getRegForValue(V);
675 return AM.Base.Reg != 0;
677 if (AM.IndexReg == 0) {
678 assert(AM.Scale == 1 && "Scale with no index!");
679 AM.IndexReg = getRegForValue(V);
680 return AM.IndexReg != 0;
687 /// X86SelectAddress - Attempt to fill in an address from the given value.
689 bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
690 SmallVector<const Value *, 32> GEPs;
692 const User *U = nullptr;
693 unsigned Opcode = Instruction::UserOp1;
694 if (const Instruction *I = dyn_cast<Instruction>(V)) {
695 // Don't walk into other basic blocks; it's possible we haven't
696 // visited them yet, so the instructions may not yet be assigned
697 // virtual registers.
698 if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(V)) ||
699 FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) {
700 Opcode = I->getOpcode();
703 } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(V)) {
704 Opcode = C->getOpcode();
708 if (PointerType *Ty = dyn_cast<PointerType>(V->getType()))
709 if (Ty->getAddressSpace() > 255)
710 // Fast instruction selection doesn't support the special
716 case Instruction::BitCast:
717 // Look past bitcasts.
718 return X86SelectAddress(U->getOperand(0), AM);
720 case Instruction::IntToPtr:
721 // Look past no-op inttoptrs.
722 if (TLI.getValueType(DL, U->getOperand(0)->getType()) ==
723 TLI.getPointerTy(DL))
724 return X86SelectAddress(U->getOperand(0), AM);
727 case Instruction::PtrToInt:
728 // Look past no-op ptrtoints.
729 if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
730 return X86SelectAddress(U->getOperand(0), AM);
733 case Instruction::Alloca: {
734 // Do static allocas.
735 const AllocaInst *A = cast<AllocaInst>(V);
736 DenseMap<const AllocaInst *, int>::iterator SI =
737 FuncInfo.StaticAllocaMap.find(A);
738 if (SI != FuncInfo.StaticAllocaMap.end()) {
739 AM.BaseType = X86AddressMode::FrameIndexBase;
740 AM.Base.FrameIndex = SI->second;
746 case Instruction::Add: {
747 // Adds of constants are common and easy enough.
748 if (const ConstantInt *CI = dyn_cast<ConstantInt>(U->getOperand(1))) {
749 uint64_t Disp = (int32_t)AM.Disp + (uint64_t)CI->getSExtValue();
750 // They have to fit in the 32-bit signed displacement field though.
751 if (isInt<32>(Disp)) {
752 AM.Disp = (uint32_t)Disp;
753 return X86SelectAddress(U->getOperand(0), AM);
759 case Instruction::GetElementPtr: {
760 X86AddressMode SavedAM = AM;
762 // Pattern-match simple GEPs.
763 uint64_t Disp = (int32_t)AM.Disp;
764 unsigned IndexReg = AM.IndexReg;
765 unsigned Scale = AM.Scale;
766 gep_type_iterator GTI = gep_type_begin(U);
767 // Iterate through the indices, folding what we can. Constants can be
768 // folded, and one dynamic index can be handled, if the scale is supported.
769 for (User::const_op_iterator i = U->op_begin() + 1, e = U->op_end();
770 i != e; ++i, ++GTI) {
771 const Value *Op = *i;
772 if (StructType *STy = dyn_cast<StructType>(*GTI)) {
773 const StructLayout *SL = DL.getStructLayout(STy);
774 Disp += SL->getElementOffset(cast<ConstantInt>(Op)->getZExtValue());
778 // A array/variable index is always of the form i*S where S is the
779 // constant scale size. See if we can push the scale into immediates.
780 uint64_t S = DL.getTypeAllocSize(GTI.getIndexedType());
782 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
783 // Constant-offset addressing.
784 Disp += CI->getSExtValue() * S;
787 if (canFoldAddIntoGEP(U, Op)) {
788 // A compatible add with a constant operand. Fold the constant.
790 cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1));
791 Disp += CI->getSExtValue() * S;
792 // Iterate on the other operand.
793 Op = cast<AddOperator>(Op)->getOperand(0);
797 (!AM.GV || !Subtarget->isPICStyleRIPRel()) &&
798 (S == 1 || S == 2 || S == 4 || S == 8)) {
799 // Scaled-index addressing.
801 IndexReg = getRegForGEPIndex(Op).first;
807 goto unsupported_gep;
811 // Check for displacement overflow.
812 if (!isInt<32>(Disp))
815 AM.IndexReg = IndexReg;
817 AM.Disp = (uint32_t)Disp;
820 if (const GetElementPtrInst *GEP =
821 dyn_cast<GetElementPtrInst>(U->getOperand(0))) {
822 // Ok, the GEP indices were covered by constant-offset and scaled-index
823 // addressing. Update the address state and move on to examining the base.
826 } else if (X86SelectAddress(U->getOperand(0), AM)) {
830 // If we couldn't merge the gep value into this addr mode, revert back to
831 // our address and just match the value instead of completely failing.
834 for (SmallVectorImpl<const Value *>::reverse_iterator
835 I = GEPs.rbegin(), E = GEPs.rend(); I != E; ++I)
836 if (handleConstantAddresses(*I, AM))
841 // Ok, the GEP indices weren't all covered.
846 return handleConstantAddresses(V, AM);
849 /// X86SelectCallAddress - Attempt to fill in an address from the given value.
851 bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) {
852 const User *U = nullptr;
853 unsigned Opcode = Instruction::UserOp1;
854 const Instruction *I = dyn_cast<Instruction>(V);
855 // Record if the value is defined in the same basic block.
857 // This information is crucial to know whether or not folding an
859 // Indeed, FastISel generates or reuses a virtual register for all
860 // operands of all instructions it selects. Obviously, the definition and
861 // its uses must use the same virtual register otherwise the produced
862 // code is incorrect.
863 // Before instruction selection, FunctionLoweringInfo::set sets the virtual
864 // registers for values that are alive across basic blocks. This ensures
865 // that the values are consistently set between across basic block, even
866 // if different instruction selection mechanisms are used (e.g., a mix of
867 // SDISel and FastISel).
868 // For values local to a basic block, the instruction selection process
869 // generates these virtual registers with whatever method is appropriate
870 // for its needs. In particular, FastISel and SDISel do not share the way
871 // local virtual registers are set.
872 // Therefore, this is impossible (or at least unsafe) to share values
873 // between basic blocks unless they use the same instruction selection
874 // method, which is not guarantee for X86.
875 // Moreover, things like hasOneUse could not be used accurately, if we
876 // allow to reference values across basic blocks whereas they are not
877 // alive across basic blocks initially.
880 Opcode = I->getOpcode();
882 InMBB = I->getParent() == FuncInfo.MBB->getBasicBlock();
883 } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(V)) {
884 Opcode = C->getOpcode();
890 case Instruction::BitCast:
891 // Look past bitcasts if its operand is in the same BB.
893 return X86SelectCallAddress(U->getOperand(0), AM);
896 case Instruction::IntToPtr:
897 // Look past no-op inttoptrs if its operand is in the same BB.
899 TLI.getValueType(DL, U->getOperand(0)->getType()) ==
900 TLI.getPointerTy(DL))
901 return X86SelectCallAddress(U->getOperand(0), AM);
904 case Instruction::PtrToInt:
905 // Look past no-op ptrtoints if its operand is in the same BB.
906 if (InMBB && TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
907 return X86SelectCallAddress(U->getOperand(0), AM);
911 // Handle constant address.
912 if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
913 // Can't handle alternate code models yet.
914 if (TM.getCodeModel() != CodeModel::Small)
917 // RIP-relative addresses can't have additional register operands.
918 if (Subtarget->isPICStyleRIPRel() &&
919 (AM.Base.Reg != 0 || AM.IndexReg != 0))
922 // Can't handle DLL Import.
923 if (GV->hasDLLImportStorageClass())
927 if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV))
928 if (GVar->isThreadLocal())
931 // Okay, we've committed to selecting this global. Set up the basic address.
934 // No ABI requires an extra load for anything other than DLLImport, which
935 // we rejected above. Return a direct reference to the global.
936 if (Subtarget->isPICStyleRIPRel()) {
937 // Use rip-relative addressing if we can. Above we verified that the
938 // base and index registers are unused.
939 assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
940 AM.Base.Reg = X86::RIP;
941 } else if (Subtarget->isPICStyleStubPIC()) {
942 AM.GVOpFlags = X86II::MO_PIC_BASE_OFFSET;
943 } else if (Subtarget->isPICStyleGOT()) {
944 AM.GVOpFlags = X86II::MO_GOTOFF;
950 // If all else fails, try to materialize the value in a register.
951 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
952 if (AM.Base.Reg == 0) {
953 AM.Base.Reg = getRegForValue(V);
954 return AM.Base.Reg != 0;
956 if (AM.IndexReg == 0) {
957 assert(AM.Scale == 1 && "Scale with no index!");
958 AM.IndexReg = getRegForValue(V);
959 return AM.IndexReg != 0;
967 /// X86SelectStore - Select and emit code to implement store instructions.
968 bool X86FastISel::X86SelectStore(const Instruction *I) {
969 // Atomic stores need special handling.
970 const StoreInst *S = cast<StoreInst>(I);
975 const Value *Val = S->getValueOperand();
976 const Value *Ptr = S->getPointerOperand();
979 if (!isTypeLegal(Val->getType(), VT, /*AllowI1=*/true))
982 unsigned Alignment = S->getAlignment();
983 unsigned ABIAlignment = DL.getABITypeAlignment(Val->getType());
984 if (Alignment == 0) // Ensure that codegen never sees alignment 0
985 Alignment = ABIAlignment;
986 bool Aligned = Alignment >= ABIAlignment;
989 if (!X86SelectAddress(Ptr, AM))
992 return X86FastEmitStore(VT, Val, AM, createMachineMemOperandFor(I), Aligned);
995 /// X86SelectRet - Select and emit code to implement ret instructions.
996 bool X86FastISel::X86SelectRet(const Instruction *I) {
997 const ReturnInst *Ret = cast<ReturnInst>(I);
998 const Function &F = *I->getParent()->getParent();
999 const X86MachineFunctionInfo *X86MFInfo =
1000 FuncInfo.MF->getInfo<X86MachineFunctionInfo>();
1002 if (!FuncInfo.CanLowerReturn)
1005 if (TLI.supportSplitCSR(FuncInfo.MF))
1008 CallingConv::ID CC = F.getCallingConv();
1009 if (CC != CallingConv::C &&
1010 CC != CallingConv::Fast &&
1011 CC != CallingConv::X86_FastCall &&
1012 CC != CallingConv::X86_64_SysV)
1015 if (Subtarget->isCallingConvWin64(CC))
1018 // Don't handle popping bytes on return for now.
1019 if (X86MFInfo->getBytesToPopOnReturn() != 0)
1022 // fastcc with -tailcallopt is intended to provide a guaranteed
1023 // tail call optimization. Fastisel doesn't know how to do that.
1024 if (CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt)
1027 // Let SDISel handle vararg functions.
1031 // Build a list of return value registers.
1032 SmallVector<unsigned, 4> RetRegs;
1034 if (Ret->getNumOperands() > 0) {
1035 SmallVector<ISD::OutputArg, 4> Outs;
1036 GetReturnInfo(F.getReturnType(), F.getAttributes(), Outs, TLI, DL);
1038 // Analyze operands of the call, assigning locations to each operand.
1039 SmallVector<CCValAssign, 16> ValLocs;
1040 CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, ValLocs, I->getContext());
1041 CCInfo.AnalyzeReturn(Outs, RetCC_X86);
1043 const Value *RV = Ret->getOperand(0);
1044 unsigned Reg = getRegForValue(RV);
1048 // Only handle a single return value for now.
1049 if (ValLocs.size() != 1)
1052 CCValAssign &VA = ValLocs[0];
1054 // Don't bother handling odd stuff for now.
1055 if (VA.getLocInfo() != CCValAssign::Full)
1057 // Only handle register returns for now.
1061 // The calling-convention tables for x87 returns don't tell
1063 if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
1066 unsigned SrcReg = Reg + VA.getValNo();
1067 EVT SrcVT = TLI.getValueType(DL, RV->getType());
1068 EVT DstVT = VA.getValVT();
1069 // Special handling for extended integers.
1070 if (SrcVT != DstVT) {
1071 if (SrcVT != MVT::i1 && SrcVT != MVT::i8 && SrcVT != MVT::i16)
1074 if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt())
1077 assert(DstVT == MVT::i32 && "X86 should always ext to i32");
1079 if (SrcVT == MVT::i1) {
1080 if (Outs[0].Flags.isSExt())
1082 SrcReg = fastEmitZExtFromI1(MVT::i8, SrcReg, /*TODO: Kill=*/false);
1085 unsigned Op = Outs[0].Flags.isZExt() ? ISD::ZERO_EXTEND :
1087 SrcReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), Op,
1088 SrcReg, /*TODO: Kill=*/false);
1092 unsigned DstReg = VA.getLocReg();
1093 const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
1094 // Avoid a cross-class copy. This is very unlikely.
1095 if (!SrcRC->contains(DstReg))
1097 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1098 TII.get(TargetOpcode::COPY), DstReg).addReg(SrcReg);
1100 // Add register to return instruction.
1101 RetRegs.push_back(VA.getLocReg());
1104 // All x86 ABIs require that for returning structs by value we copy
1105 // the sret argument into %rax/%eax (depending on ABI) for the return.
1106 // We saved the argument into a virtual register in the entry block,
1107 // so now we copy the value out and into %rax/%eax.
1108 if (F.hasStructRetAttr()) {
1109 unsigned Reg = X86MFInfo->getSRetReturnReg();
1111 "SRetReturnReg should have been set in LowerFormalArguments()!");
1112 unsigned RetReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
1113 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1114 TII.get(TargetOpcode::COPY), RetReg).addReg(Reg);
1115 RetRegs.push_back(RetReg);
1118 // Now emit the RET.
1119 MachineInstrBuilder MIB =
1120 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1121 TII.get(Subtarget->is64Bit() ? X86::RETQ : X86::RETL));
1122 for (unsigned i = 0, e = RetRegs.size(); i != e; ++i)
1123 MIB.addReg(RetRegs[i], RegState::Implicit);
1127 /// X86SelectLoad - Select and emit code to implement load instructions.
1129 bool X86FastISel::X86SelectLoad(const Instruction *I) {
1130 const LoadInst *LI = cast<LoadInst>(I);
1132 // Atomic loads need special handling.
1137 if (!isTypeLegal(LI->getType(), VT, /*AllowI1=*/true))
1140 const Value *Ptr = LI->getPointerOperand();
1143 if (!X86SelectAddress(Ptr, AM))
1146 unsigned Alignment = LI->getAlignment();
1147 unsigned ABIAlignment = DL.getABITypeAlignment(LI->getType());
1148 if (Alignment == 0) // Ensure that codegen never sees alignment 0
1149 Alignment = ABIAlignment;
1151 unsigned ResultReg = 0;
1152 if (!X86FastEmitLoad(VT, AM, createMachineMemOperandFor(LI), ResultReg,
1156 updateValueMap(I, ResultReg);
1160 static unsigned X86ChooseCmpOpcode(EVT VT, const X86Subtarget *Subtarget) {
1161 bool HasAVX = Subtarget->hasAVX();
1162 bool X86ScalarSSEf32 = Subtarget->hasSSE1();
1163 bool X86ScalarSSEf64 = Subtarget->hasSSE2();
1165 switch (VT.getSimpleVT().SimpleTy) {
1167 case MVT::i8: return X86::CMP8rr;
1168 case MVT::i16: return X86::CMP16rr;
1169 case MVT::i32: return X86::CMP32rr;
1170 case MVT::i64: return X86::CMP64rr;
1172 return X86ScalarSSEf32 ? (HasAVX ? X86::VUCOMISSrr : X86::UCOMISSrr) : 0;
1174 return X86ScalarSSEf64 ? (HasAVX ? X86::VUCOMISDrr : X86::UCOMISDrr) : 0;
1178 /// If we have a comparison with RHS as the RHS of the comparison, return an
1179 /// opcode that works for the compare (e.g. CMP32ri) otherwise return 0.
1180 static unsigned X86ChooseCmpImmediateOpcode(EVT VT, const ConstantInt *RHSC) {
1181 int64_t Val = RHSC->getSExtValue();
1182 switch (VT.getSimpleVT().SimpleTy) {
1183 // Otherwise, we can't fold the immediate into this comparison.
1190 return X86::CMP16ri8;
1191 return X86::CMP16ri;
1194 return X86::CMP32ri8;
1195 return X86::CMP32ri;
1198 return X86::CMP64ri8;
1199 // 64-bit comparisons are only valid if the immediate fits in a 32-bit sext
1202 return X86::CMP64ri32;
1207 bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1,
1208 EVT VT, DebugLoc CurDbgLoc) {
1209 unsigned Op0Reg = getRegForValue(Op0);
1210 if (Op0Reg == 0) return false;
1212 // Handle 'null' like i32/i64 0.
1213 if (isa<ConstantPointerNull>(Op1))
1214 Op1 = Constant::getNullValue(DL.getIntPtrType(Op0->getContext()));
1216 // We have two options: compare with register or immediate. If the RHS of
1217 // the compare is an immediate that we can fold into this compare, use
1218 // CMPri, otherwise use CMPrr.
1219 if (const ConstantInt *Op1C = dyn_cast<ConstantInt>(Op1)) {
1220 if (unsigned CompareImmOpc = X86ChooseCmpImmediateOpcode(VT, Op1C)) {
1221 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurDbgLoc, TII.get(CompareImmOpc))
1223 .addImm(Op1C->getSExtValue());
1228 unsigned CompareOpc = X86ChooseCmpOpcode(VT, Subtarget);
1229 if (CompareOpc == 0) return false;
1231 unsigned Op1Reg = getRegForValue(Op1);
1232 if (Op1Reg == 0) return false;
1233 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurDbgLoc, TII.get(CompareOpc))
1240 bool X86FastISel::X86SelectCmp(const Instruction *I) {
1241 const CmpInst *CI = cast<CmpInst>(I);
1244 if (!isTypeLegal(I->getOperand(0)->getType(), VT))
1247 // Try to optimize or fold the cmp.
1248 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
1249 unsigned ResultReg = 0;
1250 switch (Predicate) {
1252 case CmpInst::FCMP_FALSE: {
1253 ResultReg = createResultReg(&X86::GR32RegClass);
1254 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV32r0),
1256 ResultReg = fastEmitInst_extractsubreg(MVT::i8, ResultReg, /*Kill=*/true,
1262 case CmpInst::FCMP_TRUE: {
1263 ResultReg = createResultReg(&X86::GR8RegClass);
1264 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV8ri),
1265 ResultReg).addImm(1);
1271 updateValueMap(I, ResultReg);
1275 const Value *LHS = CI->getOperand(0);
1276 const Value *RHS = CI->getOperand(1);
1278 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
1279 // We don't have to materialize a zero constant for this case and can just use
1280 // %x again on the RHS.
1281 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
1282 const auto *RHSC = dyn_cast<ConstantFP>(RHS);
1283 if (RHSC && RHSC->isNullValue())
1287 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1288 static unsigned SETFOpcTable[2][3] = {
1289 { X86::SETEr, X86::SETNPr, X86::AND8rr },
1290 { X86::SETNEr, X86::SETPr, X86::OR8rr }
1292 unsigned *SETFOpc = nullptr;
1293 switch (Predicate) {
1295 case CmpInst::FCMP_OEQ: SETFOpc = &SETFOpcTable[0][0]; break;
1296 case CmpInst::FCMP_UNE: SETFOpc = &SETFOpcTable[1][0]; break;
1299 ResultReg = createResultReg(&X86::GR8RegClass);
1301 if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc()))
1304 unsigned FlagReg1 = createResultReg(&X86::GR8RegClass);
1305 unsigned FlagReg2 = createResultReg(&X86::GR8RegClass);
1306 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[0]),
1308 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[1]),
1310 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[2]),
1311 ResultReg).addReg(FlagReg1).addReg(FlagReg2);
1312 updateValueMap(I, ResultReg);
1318 std::tie(CC, SwapArgs) = getX86ConditionCode(Predicate);
1319 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1320 unsigned Opc = X86::getSETFromCond(CC);
1323 std::swap(LHS, RHS);
1325 // Emit a compare of LHS/RHS.
1326 if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc()))
1329 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg);
1330 updateValueMap(I, ResultReg);
1334 bool X86FastISel::X86SelectZExt(const Instruction *I) {
1335 EVT DstVT = TLI.getValueType(DL, I->getType());
1336 if (!TLI.isTypeLegal(DstVT))
1339 unsigned ResultReg = getRegForValue(I->getOperand(0));
1343 // Handle zero-extension from i1 to i8, which is common.
1344 MVT SrcVT = TLI.getSimpleValueType(DL, I->getOperand(0)->getType());
1345 if (SrcVT.SimpleTy == MVT::i1) {
1346 // Set the high bits to zero.
1347 ResultReg = fastEmitZExtFromI1(MVT::i8, ResultReg, /*TODO: Kill=*/false);
1354 if (DstVT == MVT::i64) {
1355 // Handle extension to 64-bits via sub-register shenanigans.
1358 switch (SrcVT.SimpleTy) {
1359 case MVT::i8: MovInst = X86::MOVZX32rr8; break;
1360 case MVT::i16: MovInst = X86::MOVZX32rr16; break;
1361 case MVT::i32: MovInst = X86::MOV32rr; break;
1362 default: llvm_unreachable("Unexpected zext to i64 source type");
1365 unsigned Result32 = createResultReg(&X86::GR32RegClass);
1366 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(MovInst), Result32)
1369 ResultReg = createResultReg(&X86::GR64RegClass);
1370 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::SUBREG_TO_REG),
1372 .addImm(0).addReg(Result32).addImm(X86::sub_32bit);
1373 } else if (DstVT != MVT::i8) {
1374 ResultReg = fastEmit_r(MVT::i8, DstVT.getSimpleVT(), ISD::ZERO_EXTEND,
1375 ResultReg, /*Kill=*/true);
1380 updateValueMap(I, ResultReg);
1384 bool X86FastISel::X86SelectBranch(const Instruction *I) {
1385 // Unconditional branches are selected by tablegen-generated code.
1386 // Handle a conditional branch.
1387 const BranchInst *BI = cast<BranchInst>(I);
1388 MachineBasicBlock *TrueMBB = FuncInfo.MBBMap[BI->getSuccessor(0)];
1389 MachineBasicBlock *FalseMBB = FuncInfo.MBBMap[BI->getSuccessor(1)];
1391 // Fold the common case of a conditional branch with a comparison
1392 // in the same block (values defined on other blocks may not have
1393 // initialized registers).
1395 if (const CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition())) {
1396 if (CI->hasOneUse() && CI->getParent() == I->getParent()) {
1397 EVT VT = TLI.getValueType(DL, CI->getOperand(0)->getType());
1399 // Try to optimize or fold the cmp.
1400 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
1401 switch (Predicate) {
1403 case CmpInst::FCMP_FALSE: fastEmitBranch(FalseMBB, DbgLoc); return true;
1404 case CmpInst::FCMP_TRUE: fastEmitBranch(TrueMBB, DbgLoc); return true;
1407 const Value *CmpLHS = CI->getOperand(0);
1408 const Value *CmpRHS = CI->getOperand(1);
1410 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x,
1412 // We don't have to materialize a zero constant for this case and can just
1413 // use %x again on the RHS.
1414 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
1415 const auto *CmpRHSC = dyn_cast<ConstantFP>(CmpRHS);
1416 if (CmpRHSC && CmpRHSC->isNullValue())
1420 // Try to take advantage of fallthrough opportunities.
1421 if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {
1422 std::swap(TrueMBB, FalseMBB);
1423 Predicate = CmpInst::getInversePredicate(Predicate);
1426 // FCMP_OEQ and FCMP_UNE cannot be expressed with a single flag/condition
1427 // code check. Instead two branch instructions are required to check all
1428 // the flags. First we change the predicate to a supported condition code,
1429 // which will be the first branch. Later one we will emit the second
1431 bool NeedExtraBranch = false;
1432 switch (Predicate) {
1434 case CmpInst::FCMP_OEQ:
1435 std::swap(TrueMBB, FalseMBB); // fall-through
1436 case CmpInst::FCMP_UNE:
1437 NeedExtraBranch = true;
1438 Predicate = CmpInst::FCMP_ONE;
1444 std::tie(CC, SwapArgs) = getX86ConditionCode(Predicate);
1445 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1447 BranchOpc = X86::GetCondBranchFromCond(CC);
1449 std::swap(CmpLHS, CmpRHS);
1451 // Emit a compare of the LHS and RHS, setting the flags.
1452 if (!X86FastEmitCompare(CmpLHS, CmpRHS, VT, CI->getDebugLoc()))
1455 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(BranchOpc))
1458 // X86 requires a second branch to handle UNE (and OEQ, which is mapped
1460 if (NeedExtraBranch) {
1461 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::JP_1))
1465 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1468 } else if (TruncInst *TI = dyn_cast<TruncInst>(BI->getCondition())) {
1469 // Handle things like "%cond = trunc i32 %X to i1 / br i1 %cond", which
1470 // typically happen for _Bool and C++ bools.
1472 if (TI->hasOneUse() && TI->getParent() == I->getParent() &&
1473 isTypeLegal(TI->getOperand(0)->getType(), SourceVT)) {
1474 unsigned TestOpc = 0;
1475 switch (SourceVT.SimpleTy) {
1477 case MVT::i8: TestOpc = X86::TEST8ri; break;
1478 case MVT::i16: TestOpc = X86::TEST16ri; break;
1479 case MVT::i32: TestOpc = X86::TEST32ri; break;
1480 case MVT::i64: TestOpc = X86::TEST64ri32; break;
1483 unsigned OpReg = getRegForValue(TI->getOperand(0));
1484 if (OpReg == 0) return false;
1485 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TestOpc))
1486 .addReg(OpReg).addImm(1);
1488 unsigned JmpOpc = X86::JNE_1;
1489 if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {
1490 std::swap(TrueMBB, FalseMBB);
1494 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(JmpOpc))
1497 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1501 } else if (foldX86XALUIntrinsic(CC, BI, BI->getCondition())) {
1502 // Fake request the condition, otherwise the intrinsic might be completely
1504 unsigned TmpReg = getRegForValue(BI->getCondition());
1508 unsigned BranchOpc = X86::GetCondBranchFromCond(CC);
1510 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(BranchOpc))
1512 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1516 // Otherwise do a clumsy setcc and re-test it.
1517 // Note that i1 essentially gets ANY_EXTEND'ed to i8 where it isn't used
1518 // in an explicit cast, so make sure to handle that correctly.
1519 unsigned OpReg = getRegForValue(BI->getCondition());
1520 if (OpReg == 0) return false;
1522 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri))
1523 .addReg(OpReg).addImm(1);
1524 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::JNE_1))
1526 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1530 bool X86FastISel::X86SelectShift(const Instruction *I) {
1531 unsigned CReg = 0, OpReg = 0;
1532 const TargetRegisterClass *RC = nullptr;
1533 if (I->getType()->isIntegerTy(8)) {
1535 RC = &X86::GR8RegClass;
1536 switch (I->getOpcode()) {
1537 case Instruction::LShr: OpReg = X86::SHR8rCL; break;
1538 case Instruction::AShr: OpReg = X86::SAR8rCL; break;
1539 case Instruction::Shl: OpReg = X86::SHL8rCL; break;
1540 default: return false;
1542 } else if (I->getType()->isIntegerTy(16)) {
1544 RC = &X86::GR16RegClass;
1545 switch (I->getOpcode()) {
1546 case Instruction::LShr: OpReg = X86::SHR16rCL; break;
1547 case Instruction::AShr: OpReg = X86::SAR16rCL; break;
1548 case Instruction::Shl: OpReg = X86::SHL16rCL; break;
1549 default: return false;
1551 } else if (I->getType()->isIntegerTy(32)) {
1553 RC = &X86::GR32RegClass;
1554 switch (I->getOpcode()) {
1555 case Instruction::LShr: OpReg = X86::SHR32rCL; break;
1556 case Instruction::AShr: OpReg = X86::SAR32rCL; break;
1557 case Instruction::Shl: OpReg = X86::SHL32rCL; break;
1558 default: return false;
1560 } else if (I->getType()->isIntegerTy(64)) {
1562 RC = &X86::GR64RegClass;
1563 switch (I->getOpcode()) {
1564 case Instruction::LShr: OpReg = X86::SHR64rCL; break;
1565 case Instruction::AShr: OpReg = X86::SAR64rCL; break;
1566 case Instruction::Shl: OpReg = X86::SHL64rCL; break;
1567 default: return false;
1574 if (!isTypeLegal(I->getType(), VT))
1577 unsigned Op0Reg = getRegForValue(I->getOperand(0));
1578 if (Op0Reg == 0) return false;
1580 unsigned Op1Reg = getRegForValue(I->getOperand(1));
1581 if (Op1Reg == 0) return false;
1582 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
1583 CReg).addReg(Op1Reg);
1585 // The shift instruction uses X86::CL. If we defined a super-register
1586 // of X86::CL, emit a subreg KILL to precisely describe what we're doing here.
1587 if (CReg != X86::CL)
1588 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1589 TII.get(TargetOpcode::KILL), X86::CL)
1590 .addReg(CReg, RegState::Kill);
1592 unsigned ResultReg = createResultReg(RC);
1593 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(OpReg), ResultReg)
1595 updateValueMap(I, ResultReg);
1599 bool X86FastISel::X86SelectDivRem(const Instruction *I) {
1600 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1601 const static unsigned NumOps = 4; // SDiv, SRem, UDiv, URem
1602 const static bool S = true; // IsSigned
1603 const static bool U = false; // !IsSigned
1604 const static unsigned Copy = TargetOpcode::COPY;
1605 // For the X86 DIV/IDIV instruction, in most cases the dividend
1606 // (numerator) must be in a specific register pair highreg:lowreg,
1607 // producing the quotient in lowreg and the remainder in highreg.
1608 // For most data types, to set up the instruction, the dividend is
1609 // copied into lowreg, and lowreg is sign-extended or zero-extended
1610 // into highreg. The exception is i8, where the dividend is defined
1611 // as a single register rather than a register pair, and we
1612 // therefore directly sign-extend or zero-extend the dividend into
1613 // lowreg, instead of copying, and ignore the highreg.
1614 const static struct DivRemEntry {
1615 // The following portion depends only on the data type.
1616 const TargetRegisterClass *RC;
1617 unsigned LowInReg; // low part of the register pair
1618 unsigned HighInReg; // high part of the register pair
1619 // The following portion depends on both the data type and the operation.
1620 struct DivRemResult {
1621 unsigned OpDivRem; // The specific DIV/IDIV opcode to use.
1622 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1623 // highreg, or copying a zero into highreg.
1624 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1625 // zero/sign-extending into lowreg for i8.
1626 unsigned DivRemResultReg; // Register containing the desired result.
1627 bool IsOpSigned; // Whether to use signed or unsigned form.
1628 } ResultTable[NumOps];
1629 } OpTable[NumTypes] = {
1630 { &X86::GR8RegClass, X86::AX, 0, {
1631 { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S }, // SDiv
1632 { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S }, // SRem
1633 { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U }, // UDiv
1634 { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U }, // URem
1637 { &X86::GR16RegClass, X86::AX, X86::DX, {
1638 { X86::IDIV16r, X86::CWD, Copy, X86::AX, S }, // SDiv
1639 { X86::IDIV16r, X86::CWD, Copy, X86::DX, S }, // SRem
1640 { X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U }, // UDiv
1641 { X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U }, // URem
1644 { &X86::GR32RegClass, X86::EAX, X86::EDX, {
1645 { X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S }, // SDiv
1646 { X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S }, // SRem
1647 { X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U }, // UDiv
1648 { X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U }, // URem
1651 { &X86::GR64RegClass, X86::RAX, X86::RDX, {
1652 { X86::IDIV64r, X86::CQO, Copy, X86::RAX, S }, // SDiv
1653 { X86::IDIV64r, X86::CQO, Copy, X86::RDX, S }, // SRem
1654 { X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U }, // UDiv
1655 { X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U }, // URem
1661 if (!isTypeLegal(I->getType(), VT))
1664 unsigned TypeIndex, OpIndex;
1665 switch (VT.SimpleTy) {
1666 default: return false;
1667 case MVT::i8: TypeIndex = 0; break;
1668 case MVT::i16: TypeIndex = 1; break;
1669 case MVT::i32: TypeIndex = 2; break;
1670 case MVT::i64: TypeIndex = 3;
1671 if (!Subtarget->is64Bit())
1676 switch (I->getOpcode()) {
1677 default: llvm_unreachable("Unexpected div/rem opcode");
1678 case Instruction::SDiv: OpIndex = 0; break;
1679 case Instruction::SRem: OpIndex = 1; break;
1680 case Instruction::UDiv: OpIndex = 2; break;
1681 case Instruction::URem: OpIndex = 3; break;
1684 const DivRemEntry &TypeEntry = OpTable[TypeIndex];
1685 const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1686 unsigned Op0Reg = getRegForValue(I->getOperand(0));
1689 unsigned Op1Reg = getRegForValue(I->getOperand(1));
1693 // Move op0 into low-order input register.
1694 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1695 TII.get(OpEntry.OpCopy), TypeEntry.LowInReg).addReg(Op0Reg);
1696 // Zero-extend or sign-extend into high-order input register.
1697 if (OpEntry.OpSignExtend) {
1698 if (OpEntry.IsOpSigned)
1699 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1700 TII.get(OpEntry.OpSignExtend));
1702 unsigned Zero32 = createResultReg(&X86::GR32RegClass);
1703 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1704 TII.get(X86::MOV32r0), Zero32);
1706 // Copy the zero into the appropriate sub/super/identical physical
1707 // register. Unfortunately the operations needed are not uniform enough
1708 // to fit neatly into the table above.
1709 if (VT.SimpleTy == MVT::i16) {
1710 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1711 TII.get(Copy), TypeEntry.HighInReg)
1712 .addReg(Zero32, 0, X86::sub_16bit);
1713 } else if (VT.SimpleTy == MVT::i32) {
1714 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1715 TII.get(Copy), TypeEntry.HighInReg)
1717 } else if (VT.SimpleTy == MVT::i64) {
1718 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1719 TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
1720 .addImm(0).addReg(Zero32).addImm(X86::sub_32bit);
1724 // Generate the DIV/IDIV instruction.
1725 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1726 TII.get(OpEntry.OpDivRem)).addReg(Op1Reg);
1727 // For i8 remainder, we can't reference AH directly, as we'll end
1728 // up with bogus copies like %R9B = COPY %AH. Reference AX
1729 // instead to prevent AH references in a REX instruction.
1731 // The current assumption of the fast register allocator is that isel
1732 // won't generate explicit references to the GPR8_NOREX registers. If
1733 // the allocator and/or the backend get enhanced to be more robust in
1734 // that regard, this can be, and should be, removed.
1735 unsigned ResultReg = 0;
1736 if ((I->getOpcode() == Instruction::SRem ||
1737 I->getOpcode() == Instruction::URem) &&
1738 OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) {
1739 unsigned SourceSuperReg = createResultReg(&X86::GR16RegClass);
1740 unsigned ResultSuperReg = createResultReg(&X86::GR16RegClass);
1741 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1742 TII.get(Copy), SourceSuperReg).addReg(X86::AX);
1744 // Shift AX right by 8 bits instead of using AH.
1745 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::SHR16ri),
1746 ResultSuperReg).addReg(SourceSuperReg).addImm(8);
1748 // Now reference the 8-bit subreg of the result.
1749 ResultReg = fastEmitInst_extractsubreg(MVT::i8, ResultSuperReg,
1750 /*Kill=*/true, X86::sub_8bit);
1752 // Copy the result out of the physreg if we haven't already.
1754 ResultReg = createResultReg(TypeEntry.RC);
1755 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Copy), ResultReg)
1756 .addReg(OpEntry.DivRemResultReg);
1758 updateValueMap(I, ResultReg);
1763 /// \brief Emit a conditional move instruction (if the are supported) to lower
1765 bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {
1766 // Check if the subtarget supports these instructions.
1767 if (!Subtarget->hasCMov())
1770 // FIXME: Add support for i8.
1771 if (RetVT < MVT::i16 || RetVT > MVT::i64)
1774 const Value *Cond = I->getOperand(0);
1775 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
1776 bool NeedTest = true;
1777 X86::CondCode CC = X86::COND_NE;
1779 // Optimize conditions coming from a compare if both instructions are in the
1780 // same basic block (values defined in other basic blocks may not have
1781 // initialized registers).
1782 const auto *CI = dyn_cast<CmpInst>(Cond);
1783 if (CI && (CI->getParent() == I->getParent())) {
1784 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
1786 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1787 static unsigned SETFOpcTable[2][3] = {
1788 { X86::SETNPr, X86::SETEr , X86::TEST8rr },
1789 { X86::SETPr, X86::SETNEr, X86::OR8rr }
1791 unsigned *SETFOpc = nullptr;
1792 switch (Predicate) {
1794 case CmpInst::FCMP_OEQ:
1795 SETFOpc = &SETFOpcTable[0][0];
1796 Predicate = CmpInst::ICMP_NE;
1798 case CmpInst::FCMP_UNE:
1799 SETFOpc = &SETFOpcTable[1][0];
1800 Predicate = CmpInst::ICMP_NE;
1805 std::tie(CC, NeedSwap) = getX86ConditionCode(Predicate);
1806 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1808 const Value *CmpLHS = CI->getOperand(0);
1809 const Value *CmpRHS = CI->getOperand(1);
1811 std::swap(CmpLHS, CmpRHS);
1813 EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType());
1814 // Emit a compare of the LHS and RHS, setting the flags.
1815 if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))
1819 unsigned FlagReg1 = createResultReg(&X86::GR8RegClass);
1820 unsigned FlagReg2 = createResultReg(&X86::GR8RegClass);
1821 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[0]),
1823 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(SETFOpc[1]),
1825 auto const &II = TII.get(SETFOpc[2]);
1826 if (II.getNumDefs()) {
1827 unsigned TmpReg = createResultReg(&X86::GR8RegClass);
1828 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, TmpReg)
1829 .addReg(FlagReg2).addReg(FlagReg1);
1831 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1832 .addReg(FlagReg2).addReg(FlagReg1);
1836 } else if (foldX86XALUIntrinsic(CC, I, Cond)) {
1837 // Fake request the condition, otherwise the intrinsic might be completely
1839 unsigned TmpReg = getRegForValue(Cond);
1847 // Selects operate on i1, however, CondReg is 8 bits width and may contain
1848 // garbage. Indeed, only the less significant bit is supposed to be
1849 // accurate. If we read more than the lsb, we may see non-zero values
1850 // whereas lsb is zero. Therefore, we have to truncate Op0Reg to i1 for
1851 // the select. This is achieved by performing TEST against 1.
1852 unsigned CondReg = getRegForValue(Cond);
1855 bool CondIsKill = hasTrivialKill(Cond);
1857 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri))
1858 .addReg(CondReg, getKillRegState(CondIsKill)).addImm(1);
1861 const Value *LHS = I->getOperand(1);
1862 const Value *RHS = I->getOperand(2);
1864 unsigned RHSReg = getRegForValue(RHS);
1865 bool RHSIsKill = hasTrivialKill(RHS);
1867 unsigned LHSReg = getRegForValue(LHS);
1868 bool LHSIsKill = hasTrivialKill(LHS);
1870 if (!LHSReg || !RHSReg)
1873 unsigned Opc = X86::getCMovFromCond(CC, RC->getSize());
1874 unsigned ResultReg = fastEmitInst_rr(Opc, RC, RHSReg, RHSIsKill,
1876 updateValueMap(I, ResultReg);
1880 /// \brief Emit SSE or AVX instructions to lower the select.
1882 /// Try to use SSE1/SSE2 instructions to simulate a select without branches.
1883 /// This lowers fp selects into a CMP/AND/ANDN/OR sequence when the necessary
1884 /// SSE instructions are available. If AVX is available, try to use a VBLENDV.
1885 bool X86FastISel::X86FastEmitSSESelect(MVT RetVT, const Instruction *I) {
1886 // Optimize conditions coming from a compare if both instructions are in the
1887 // same basic block (values defined in other basic blocks may not have
1888 // initialized registers).
1889 const auto *CI = dyn_cast<FCmpInst>(I->getOperand(0));
1890 if (!CI || (CI->getParent() != I->getParent()))
1893 if (I->getType() != CI->getOperand(0)->getType() ||
1894 !((Subtarget->hasSSE1() && RetVT == MVT::f32) ||
1895 (Subtarget->hasSSE2() && RetVT == MVT::f64)))
1898 const Value *CmpLHS = CI->getOperand(0);
1899 const Value *CmpRHS = CI->getOperand(1);
1900 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
1902 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
1903 // We don't have to materialize a zero constant for this case and can just use
1904 // %x again on the RHS.
1905 if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
1906 const auto *CmpRHSC = dyn_cast<ConstantFP>(CmpRHS);
1907 if (CmpRHSC && CmpRHSC->isNullValue())
1913 std::tie(CC, NeedSwap) = getX86SSEConditionCode(Predicate);
1918 std::swap(CmpLHS, CmpRHS);
1920 // Choose the SSE instruction sequence based on data type (float or double).
1921 static unsigned OpcTable[2][4] = {
1922 { X86::CMPSSrr, X86::FsANDPSrr, X86::FsANDNPSrr, X86::FsORPSrr },
1923 { X86::CMPSDrr, X86::FsANDPDrr, X86::FsANDNPDrr, X86::FsORPDrr }
1926 unsigned *Opc = nullptr;
1927 switch (RetVT.SimpleTy) {
1928 default: return false;
1929 case MVT::f32: Opc = &OpcTable[0][0]; break;
1930 case MVT::f64: Opc = &OpcTable[1][0]; break;
1933 const Value *LHS = I->getOperand(1);
1934 const Value *RHS = I->getOperand(2);
1936 unsigned LHSReg = getRegForValue(LHS);
1937 bool LHSIsKill = hasTrivialKill(LHS);
1939 unsigned RHSReg = getRegForValue(RHS);
1940 bool RHSIsKill = hasTrivialKill(RHS);
1942 unsigned CmpLHSReg = getRegForValue(CmpLHS);
1943 bool CmpLHSIsKill = hasTrivialKill(CmpLHS);
1945 unsigned CmpRHSReg = getRegForValue(CmpRHS);
1946 bool CmpRHSIsKill = hasTrivialKill(CmpRHS);
1948 if (!LHSReg || !RHSReg || !CmpLHS || !CmpRHS)
1951 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
1954 if (Subtarget->hasAVX()) {
1955 const TargetRegisterClass *FR32 = &X86::FR32RegClass;
1956 const TargetRegisterClass *VR128 = &X86::VR128RegClass;
1958 // If we have AVX, create 1 blendv instead of 3 logic instructions.
1959 // Blendv was introduced with SSE 4.1, but the 2 register form implicitly
1960 // uses XMM0 as the selection register. That may need just as many
1961 // instructions as the AND/ANDN/OR sequence due to register moves, so
1963 unsigned CmpOpcode =
1964 (RetVT.SimpleTy == MVT::f32) ? X86::VCMPSSrr : X86::VCMPSDrr;
1965 unsigned BlendOpcode =
1966 (RetVT.SimpleTy == MVT::f32) ? X86::VBLENDVPSrr : X86::VBLENDVPDrr;
1968 unsigned CmpReg = fastEmitInst_rri(CmpOpcode, FR32, CmpLHSReg, CmpLHSIsKill,
1969 CmpRHSReg, CmpRHSIsKill, CC);
1970 unsigned VBlendReg = fastEmitInst_rrr(BlendOpcode, VR128, RHSReg, RHSIsKill,
1971 LHSReg, LHSIsKill, CmpReg, true);
1972 ResultReg = createResultReg(RC);
1973 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1974 TII.get(TargetOpcode::COPY), ResultReg).addReg(VBlendReg);
1976 unsigned CmpReg = fastEmitInst_rri(Opc[0], RC, CmpLHSReg, CmpLHSIsKill,
1977 CmpRHSReg, CmpRHSIsKill, CC);
1978 unsigned AndReg = fastEmitInst_rr(Opc[1], RC, CmpReg, /*IsKill=*/false,
1980 unsigned AndNReg = fastEmitInst_rr(Opc[2], RC, CmpReg, /*IsKill=*/true,
1982 ResultReg = fastEmitInst_rr(Opc[3], RC, AndNReg, /*IsKill=*/true,
1983 AndReg, /*IsKill=*/true);
1985 updateValueMap(I, ResultReg);
1989 bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I) {
1990 // These are pseudo CMOV instructions and will be later expanded into control-
1993 switch (RetVT.SimpleTy) {
1994 default: return false;
1995 case MVT::i8: Opc = X86::CMOV_GR8; break;
1996 case MVT::i16: Opc = X86::CMOV_GR16; break;
1997 case MVT::i32: Opc = X86::CMOV_GR32; break;
1998 case MVT::f32: Opc = X86::CMOV_FR32; break;
1999 case MVT::f64: Opc = X86::CMOV_FR64; break;
2002 const Value *Cond = I->getOperand(0);
2003 X86::CondCode CC = X86::COND_NE;
2005 // Optimize conditions coming from a compare if both instructions are in the
2006 // same basic block (values defined in other basic blocks may not have
2007 // initialized registers).
2008 const auto *CI = dyn_cast<CmpInst>(Cond);
2009 if (CI && (CI->getParent() == I->getParent())) {
2011 std::tie(CC, NeedSwap) = getX86ConditionCode(CI->getPredicate());
2012 if (CC > X86::LAST_VALID_COND)
2015 const Value *CmpLHS = CI->getOperand(0);
2016 const Value *CmpRHS = CI->getOperand(1);
2019 std::swap(CmpLHS, CmpRHS);
2021 EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType());
2022 if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))
2025 unsigned CondReg = getRegForValue(Cond);
2028 bool CondIsKill = hasTrivialKill(Cond);
2029 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TEST8ri))
2030 .addReg(CondReg, getKillRegState(CondIsKill)).addImm(1);
2033 const Value *LHS = I->getOperand(1);
2034 const Value *RHS = I->getOperand(2);
2036 unsigned LHSReg = getRegForValue(LHS);
2037 bool LHSIsKill = hasTrivialKill(LHS);
2039 unsigned RHSReg = getRegForValue(RHS);
2040 bool RHSIsKill = hasTrivialKill(RHS);
2042 if (!LHSReg || !RHSReg)
2045 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
2047 unsigned ResultReg =
2048 fastEmitInst_rri(Opc, RC, RHSReg, RHSIsKill, LHSReg, LHSIsKill, CC);
2049 updateValueMap(I, ResultReg);
2053 bool X86FastISel::X86SelectSelect(const Instruction *I) {
2055 if (!isTypeLegal(I->getType(), RetVT))
2058 // Check if we can fold the select.
2059 if (const auto *CI = dyn_cast<CmpInst>(I->getOperand(0))) {
2060 CmpInst::Predicate Predicate = optimizeCmpPredicate(CI);
2061 const Value *Opnd = nullptr;
2062 switch (Predicate) {
2064 case CmpInst::FCMP_FALSE: Opnd = I->getOperand(2); break;
2065 case CmpInst::FCMP_TRUE: Opnd = I->getOperand(1); break;
2067 // No need for a select anymore - this is an unconditional move.
2069 unsigned OpReg = getRegForValue(Opnd);
2072 bool OpIsKill = hasTrivialKill(Opnd);
2073 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
2074 unsigned ResultReg = createResultReg(RC);
2075 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2076 TII.get(TargetOpcode::COPY), ResultReg)
2077 .addReg(OpReg, getKillRegState(OpIsKill));
2078 updateValueMap(I, ResultReg);
2083 // First try to use real conditional move instructions.
2084 if (X86FastEmitCMoveSelect(RetVT, I))
2087 // Try to use a sequence of SSE instructions to simulate a conditional move.
2088 if (X86FastEmitSSESelect(RetVT, I))
2091 // Fall-back to pseudo conditional move instructions, which will be later
2092 // converted to control-flow.
2093 if (X86FastEmitPseudoSelect(RetVT, I))
2099 bool X86FastISel::X86SelectSIToFP(const Instruction *I) {
2100 // The target-independent selection algorithm in FastISel already knows how
2101 // to select a SINT_TO_FP if the target is SSE but not AVX.
2102 // Early exit if the subtarget doesn't have AVX.
2103 if (!Subtarget->hasAVX())
2106 if (!I->getOperand(0)->getType()->isIntegerTy(32))
2109 // Select integer to float/double conversion.
2110 unsigned OpReg = getRegForValue(I->getOperand(0));
2114 const TargetRegisterClass *RC = nullptr;
2117 if (I->getType()->isDoubleTy()) {
2118 // sitofp int -> double
2119 Opcode = X86::VCVTSI2SDrr;
2120 RC = &X86::FR64RegClass;
2121 } else if (I->getType()->isFloatTy()) {
2122 // sitofp int -> float
2123 Opcode = X86::VCVTSI2SSrr;
2124 RC = &X86::FR32RegClass;
2128 unsigned ImplicitDefReg = createResultReg(RC);
2129 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2130 TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
2131 unsigned ResultReg =
2132 fastEmitInst_rr(Opcode, RC, ImplicitDefReg, true, OpReg, false);
2133 updateValueMap(I, ResultReg);
2137 // Helper method used by X86SelectFPExt and X86SelectFPTrunc.
2138 bool X86FastISel::X86SelectFPExtOrFPTrunc(const Instruction *I,
2140 const TargetRegisterClass *RC) {
2141 assert((I->getOpcode() == Instruction::FPExt ||
2142 I->getOpcode() == Instruction::FPTrunc) &&
2143 "Instruction must be an FPExt or FPTrunc!");
2145 unsigned OpReg = getRegForValue(I->getOperand(0));
2149 unsigned ResultReg = createResultReg(RC);
2150 MachineInstrBuilder MIB;
2151 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpc),
2153 if (Subtarget->hasAVX())
2156 updateValueMap(I, ResultReg);
2160 bool X86FastISel::X86SelectFPExt(const Instruction *I) {
2161 if (X86ScalarSSEf64 && I->getType()->isDoubleTy() &&
2162 I->getOperand(0)->getType()->isFloatTy()) {
2163 // fpext from float to double.
2164 unsigned Opc = Subtarget->hasAVX() ? X86::VCVTSS2SDrr : X86::CVTSS2SDrr;
2165 return X86SelectFPExtOrFPTrunc(I, Opc, &X86::FR64RegClass);
2171 bool X86FastISel::X86SelectFPTrunc(const Instruction *I) {
2172 if (X86ScalarSSEf64 && I->getType()->isFloatTy() &&
2173 I->getOperand(0)->getType()->isDoubleTy()) {
2174 // fptrunc from double to float.
2175 unsigned Opc = Subtarget->hasAVX() ? X86::VCVTSD2SSrr : X86::CVTSD2SSrr;
2176 return X86SelectFPExtOrFPTrunc(I, Opc, &X86::FR32RegClass);
2182 bool X86FastISel::X86SelectTrunc(const Instruction *I) {
2183 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
2184 EVT DstVT = TLI.getValueType(DL, I->getType());
2186 // This code only handles truncation to byte.
2187 if (DstVT != MVT::i8 && DstVT != MVT::i1)
2189 if (!TLI.isTypeLegal(SrcVT))
2192 unsigned InputReg = getRegForValue(I->getOperand(0));
2194 // Unhandled operand. Halt "fast" selection and bail.
2197 if (SrcVT == MVT::i8) {
2198 // Truncate from i8 to i1; no code needed.
2199 updateValueMap(I, InputReg);
2203 bool KillInputReg = false;
2204 if (!Subtarget->is64Bit()) {
2205 // If we're on x86-32; we can't extract an i8 from a general register.
2206 // First issue a copy to GR16_ABCD or GR32_ABCD.
2207 const TargetRegisterClass *CopyRC =
2208 (SrcVT == MVT::i16) ? &X86::GR16_ABCDRegClass : &X86::GR32_ABCDRegClass;
2209 unsigned CopyReg = createResultReg(CopyRC);
2210 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2211 TII.get(TargetOpcode::COPY), CopyReg).addReg(InputReg);
2213 KillInputReg = true;
2216 // Issue an extract_subreg.
2217 unsigned ResultReg = fastEmitInst_extractsubreg(MVT::i8,
2218 InputReg, KillInputReg,
2223 updateValueMap(I, ResultReg);
2227 bool X86FastISel::IsMemcpySmall(uint64_t Len) {
2228 return Len <= (Subtarget->is64Bit() ? 32 : 16);
2231 bool X86FastISel::TryEmitSmallMemcpy(X86AddressMode DestAM,
2232 X86AddressMode SrcAM, uint64_t Len) {
2234 // Make sure we don't bloat code by inlining very large memcpy's.
2235 if (!IsMemcpySmall(Len))
2238 bool i64Legal = Subtarget->is64Bit();
2240 // We don't care about alignment here since we just emit integer accesses.
2243 if (Len >= 8 && i64Legal)
2253 bool RV = X86FastEmitLoad(VT, SrcAM, nullptr, Reg);
2254 RV &= X86FastEmitStore(VT, Reg, /*Kill=*/true, DestAM);
2255 assert(RV && "Failed to emit load or store??");
2257 unsigned Size = VT.getSizeInBits()/8;
2259 DestAM.Disp += Size;
2266 bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
2267 // FIXME: Handle more intrinsics.
2268 switch (II->getIntrinsicID()) {
2269 default: return false;
2270 case Intrinsic::convert_from_fp16:
2271 case Intrinsic::convert_to_fp16: {
2272 if (Subtarget->useSoftFloat() || !Subtarget->hasF16C())
2275 const Value *Op = II->getArgOperand(0);
2276 unsigned InputReg = getRegForValue(Op);
2280 // F16C only allows converting from float to half and from half to float.
2281 bool IsFloatToHalf = II->getIntrinsicID() == Intrinsic::convert_to_fp16;
2282 if (IsFloatToHalf) {
2283 if (!Op->getType()->isFloatTy())
2286 if (!II->getType()->isFloatTy())
2290 unsigned ResultReg = 0;
2291 const TargetRegisterClass *RC = TLI.getRegClassFor(MVT::v8i16);
2292 if (IsFloatToHalf) {
2293 // 'InputReg' is implicitly promoted from register class FR32 to
2294 // register class VR128 by method 'constrainOperandRegClass' which is
2295 // directly called by 'fastEmitInst_ri'.
2296 // Instruction VCVTPS2PHrr takes an extra immediate operand which is
2297 // used to provide rounding control.
2298 InputReg = fastEmitInst_ri(X86::VCVTPS2PHrr, RC, InputReg, false, 0);
2300 // Move the lower 32-bits of ResultReg to another register of class GR32.
2301 ResultReg = createResultReg(&X86::GR32RegClass);
2302 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2303 TII.get(X86::VMOVPDI2DIrr), ResultReg)
2304 .addReg(InputReg, RegState::Kill);
2306 // The result value is in the lower 16-bits of ResultReg.
2307 unsigned RegIdx = X86::sub_16bit;
2308 ResultReg = fastEmitInst_extractsubreg(MVT::i16, ResultReg, true, RegIdx);
2310 assert(Op->getType()->isIntegerTy(16) && "Expected a 16-bit integer!");
2311 // Explicitly sign-extend the input to 32-bit.
2312 InputReg = fastEmit_r(MVT::i16, MVT::i32, ISD::SIGN_EXTEND, InputReg,
2315 // The following SCALAR_TO_VECTOR will be expanded into a VMOVDI2PDIrr.
2316 InputReg = fastEmit_r(MVT::i32, MVT::v4i32, ISD::SCALAR_TO_VECTOR,
2317 InputReg, /*Kill=*/true);
2319 InputReg = fastEmitInst_r(X86::VCVTPH2PSrr, RC, InputReg, /*Kill=*/true);
2321 // The result value is in the lower 32-bits of ResultReg.
2322 // Emit an explicit copy from register class VR128 to register class FR32.
2323 ResultReg = createResultReg(&X86::FR32RegClass);
2324 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2325 TII.get(TargetOpcode::COPY), ResultReg)
2326 .addReg(InputReg, RegState::Kill);
2329 updateValueMap(II, ResultReg);
2332 case Intrinsic::frameaddress: {
2333 MachineFunction *MF = FuncInfo.MF;
2334 if (MF->getTarget().getMCAsmInfo()->usesWindowsCFI())
2337 Type *RetTy = II->getCalledFunction()->getReturnType();
2340 if (!isTypeLegal(RetTy, VT))
2344 const TargetRegisterClass *RC = nullptr;
2346 switch (VT.SimpleTy) {
2347 default: llvm_unreachable("Invalid result type for frameaddress.");
2348 case MVT::i32: Opc = X86::MOV32rm; RC = &X86::GR32RegClass; break;
2349 case MVT::i64: Opc = X86::MOV64rm; RC = &X86::GR64RegClass; break;
2352 // This needs to be set before we call getPtrSizedFrameRegister, otherwise
2353 // we get the wrong frame register.
2354 MachineFrameInfo *MFI = MF->getFrameInfo();
2355 MFI->setFrameAddressIsTaken(true);
2357 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
2358 unsigned FrameReg = RegInfo->getPtrSizedFrameRegister(*MF);
2359 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
2360 (FrameReg == X86::EBP && VT == MVT::i32)) &&
2361 "Invalid Frame Register!");
2363 // Always make a copy of the frame register to to a vreg first, so that we
2364 // never directly reference the frame register (the TwoAddressInstruction-
2365 // Pass doesn't like that).
2366 unsigned SrcReg = createResultReg(RC);
2367 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2368 TII.get(TargetOpcode::COPY), SrcReg).addReg(FrameReg);
2370 // Now recursively load from the frame address.
2371 // movq (%rbp), %rax
2372 // movq (%rax), %rax
2373 // movq (%rax), %rax
2376 unsigned Depth = cast<ConstantInt>(II->getOperand(0))->getZExtValue();
2378 DestReg = createResultReg(RC);
2379 addDirectMem(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2380 TII.get(Opc), DestReg), SrcReg);
2384 updateValueMap(II, SrcReg);
2387 case Intrinsic::memcpy: {
2388 const MemCpyInst *MCI = cast<MemCpyInst>(II);
2389 // Don't handle volatile or variable length memcpys.
2390 if (MCI->isVolatile())
2393 if (isa<ConstantInt>(MCI->getLength())) {
2394 // Small memcpy's are common enough that we want to do them
2395 // without a call if possible.
2396 uint64_t Len = cast<ConstantInt>(MCI->getLength())->getZExtValue();
2397 if (IsMemcpySmall(Len)) {
2398 X86AddressMode DestAM, SrcAM;
2399 if (!X86SelectAddress(MCI->getRawDest(), DestAM) ||
2400 !X86SelectAddress(MCI->getRawSource(), SrcAM))
2402 TryEmitSmallMemcpy(DestAM, SrcAM, Len);
2407 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2408 if (!MCI->getLength()->getType()->isIntegerTy(SizeWidth))
2411 if (MCI->getSourceAddressSpace() > 255 || MCI->getDestAddressSpace() > 255)
2414 return lowerCallTo(II, "memcpy", II->getNumArgOperands() - 2);
2416 case Intrinsic::memset: {
2417 const MemSetInst *MSI = cast<MemSetInst>(II);
2419 if (MSI->isVolatile())
2422 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2423 if (!MSI->getLength()->getType()->isIntegerTy(SizeWidth))
2426 if (MSI->getDestAddressSpace() > 255)
2429 return lowerCallTo(II, "memset", II->getNumArgOperands() - 2);
2431 case Intrinsic::stackprotector: {
2432 // Emit code to store the stack guard onto the stack.
2433 EVT PtrTy = TLI.getPointerTy(DL);
2435 const Value *Op1 = II->getArgOperand(0); // The guard's value.
2436 const AllocaInst *Slot = cast<AllocaInst>(II->getArgOperand(1));
2438 MFI.setStackProtectorIndex(FuncInfo.StaticAllocaMap[Slot]);
2440 // Grab the frame index.
2442 if (!X86SelectAddress(Slot, AM)) return false;
2443 if (!X86FastEmitStore(PtrTy, Op1, AM)) return false;
2446 case Intrinsic::dbg_declare: {
2447 const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
2449 assert(DI->getAddress() && "Null address should be checked earlier!");
2450 if (!X86SelectAddress(DI->getAddress(), AM))
2452 const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
2453 // FIXME may need to add RegState::Debug to any registers produced,
2454 // although ESP/EBP should be the only ones at the moment.
2455 assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) &&
2456 "Expected inlined-at fields to agree");
2457 addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II), AM)
2459 .addMetadata(DI->getVariable())
2460 .addMetadata(DI->getExpression());
2463 case Intrinsic::trap: {
2464 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::TRAP));
2467 case Intrinsic::sqrt: {
2468 if (!Subtarget->hasSSE1())
2471 Type *RetTy = II->getCalledFunction()->getReturnType();
2474 if (!isTypeLegal(RetTy, VT))
2477 // Unfortunately we can't use fastEmit_r, because the AVX version of FSQRT
2478 // is not generated by FastISel yet.
2479 // FIXME: Update this code once tablegen can handle it.
2480 static const unsigned SqrtOpc[2][2] = {
2481 {X86::SQRTSSr, X86::VSQRTSSr},
2482 {X86::SQRTSDr, X86::VSQRTSDr}
2484 bool HasAVX = Subtarget->hasAVX();
2486 const TargetRegisterClass *RC;
2487 switch (VT.SimpleTy) {
2488 default: return false;
2489 case MVT::f32: Opc = SqrtOpc[0][HasAVX]; RC = &X86::FR32RegClass; break;
2490 case MVT::f64: Opc = SqrtOpc[1][HasAVX]; RC = &X86::FR64RegClass; break;
2493 const Value *SrcVal = II->getArgOperand(0);
2494 unsigned SrcReg = getRegForValue(SrcVal);
2499 unsigned ImplicitDefReg = 0;
2501 ImplicitDefReg = createResultReg(RC);
2502 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2503 TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
2506 unsigned ResultReg = createResultReg(RC);
2507 MachineInstrBuilder MIB;
2508 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc),
2512 MIB.addReg(ImplicitDefReg);
2516 updateValueMap(II, ResultReg);
2519 case Intrinsic::sadd_with_overflow:
2520 case Intrinsic::uadd_with_overflow:
2521 case Intrinsic::ssub_with_overflow:
2522 case Intrinsic::usub_with_overflow:
2523 case Intrinsic::smul_with_overflow:
2524 case Intrinsic::umul_with_overflow: {
2525 // This implements the basic lowering of the xalu with overflow intrinsics
2526 // into add/sub/mul followed by either seto or setb.
2527 const Function *Callee = II->getCalledFunction();
2528 auto *Ty = cast<StructType>(Callee->getReturnType());
2529 Type *RetTy = Ty->getTypeAtIndex(0U);
2530 Type *CondTy = Ty->getTypeAtIndex(1);
2533 if (!isTypeLegal(RetTy, VT))
2536 if (VT < MVT::i8 || VT > MVT::i64)
2539 const Value *LHS = II->getArgOperand(0);
2540 const Value *RHS = II->getArgOperand(1);
2542 // Canonicalize immediate to the RHS.
2543 if (isa<ConstantInt>(LHS) && !isa<ConstantInt>(RHS) &&
2544 isCommutativeIntrinsic(II))
2545 std::swap(LHS, RHS);
2547 bool UseIncDec = false;
2548 if (isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isOne())
2551 unsigned BaseOpc, CondOpc;
2552 switch (II->getIntrinsicID()) {
2553 default: llvm_unreachable("Unexpected intrinsic!");
2554 case Intrinsic::sadd_with_overflow:
2555 BaseOpc = UseIncDec ? unsigned(X86ISD::INC) : unsigned(ISD::ADD);
2556 CondOpc = X86::SETOr;
2558 case Intrinsic::uadd_with_overflow:
2559 BaseOpc = ISD::ADD; CondOpc = X86::SETBr; break;
2560 case Intrinsic::ssub_with_overflow:
2561 BaseOpc = UseIncDec ? unsigned(X86ISD::DEC) : unsigned(ISD::SUB);
2562 CondOpc = X86::SETOr;
2564 case Intrinsic::usub_with_overflow:
2565 BaseOpc = ISD::SUB; CondOpc = X86::SETBr; break;
2566 case Intrinsic::smul_with_overflow:
2567 BaseOpc = X86ISD::SMUL; CondOpc = X86::SETOr; break;
2568 case Intrinsic::umul_with_overflow:
2569 BaseOpc = X86ISD::UMUL; CondOpc = X86::SETOr; break;
2572 unsigned LHSReg = getRegForValue(LHS);
2575 bool LHSIsKill = hasTrivialKill(LHS);
2577 unsigned ResultReg = 0;
2578 // Check if we have an immediate version.
2579 if (const auto *CI = dyn_cast<ConstantInt>(RHS)) {
2580 static const unsigned Opc[2][4] = {
2581 { X86::INC8r, X86::INC16r, X86::INC32r, X86::INC64r },
2582 { X86::DEC8r, X86::DEC16r, X86::DEC32r, X86::DEC64r }
2585 if (BaseOpc == X86ISD::INC || BaseOpc == X86ISD::DEC) {
2586 ResultReg = createResultReg(TLI.getRegClassFor(VT));
2587 bool IsDec = BaseOpc == X86ISD::DEC;
2588 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2589 TII.get(Opc[IsDec][VT.SimpleTy-MVT::i8]), ResultReg)
2590 .addReg(LHSReg, getKillRegState(LHSIsKill));
2592 ResultReg = fastEmit_ri(VT, VT, BaseOpc, LHSReg, LHSIsKill,
2593 CI->getZExtValue());
2599 RHSReg = getRegForValue(RHS);
2602 RHSIsKill = hasTrivialKill(RHS);
2603 ResultReg = fastEmit_rr(VT, VT, BaseOpc, LHSReg, LHSIsKill, RHSReg,
2607 // FastISel doesn't have a pattern for all X86::MUL*r and X86::IMUL*r. Emit
2609 if (BaseOpc == X86ISD::UMUL && !ResultReg) {
2610 static const unsigned MULOpc[] =
2611 { X86::MUL8r, X86::MUL16r, X86::MUL32r, X86::MUL64r };
2612 static const unsigned Reg[] = { X86::AL, X86::AX, X86::EAX, X86::RAX };
2613 // First copy the first operand into RAX, which is an implicit input to
2614 // the X86::MUL*r instruction.
2615 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2616 TII.get(TargetOpcode::COPY), Reg[VT.SimpleTy-MVT::i8])
2617 .addReg(LHSReg, getKillRegState(LHSIsKill));
2618 ResultReg = fastEmitInst_r(MULOpc[VT.SimpleTy-MVT::i8],
2619 TLI.getRegClassFor(VT), RHSReg, RHSIsKill);
2620 } else if (BaseOpc == X86ISD::SMUL && !ResultReg) {
2621 static const unsigned MULOpc[] =
2622 { X86::IMUL8r, X86::IMUL16rr, X86::IMUL32rr, X86::IMUL64rr };
2623 if (VT == MVT::i8) {
2624 // Copy the first operand into AL, which is an implicit input to the
2625 // X86::IMUL8r instruction.
2626 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2627 TII.get(TargetOpcode::COPY), X86::AL)
2628 .addReg(LHSReg, getKillRegState(LHSIsKill));
2629 ResultReg = fastEmitInst_r(MULOpc[0], TLI.getRegClassFor(VT), RHSReg,
2632 ResultReg = fastEmitInst_rr(MULOpc[VT.SimpleTy-MVT::i8],
2633 TLI.getRegClassFor(VT), LHSReg, LHSIsKill,
2640 unsigned ResultReg2 = FuncInfo.CreateRegs(CondTy);
2641 assert((ResultReg+1) == ResultReg2 && "Nonconsecutive result registers.");
2642 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CondOpc),
2645 updateValueMap(II, ResultReg, 2);
2648 case Intrinsic::x86_sse_cvttss2si:
2649 case Intrinsic::x86_sse_cvttss2si64:
2650 case Intrinsic::x86_sse2_cvttsd2si:
2651 case Intrinsic::x86_sse2_cvttsd2si64: {
2653 switch (II->getIntrinsicID()) {
2654 default: llvm_unreachable("Unexpected intrinsic.");
2655 case Intrinsic::x86_sse_cvttss2si:
2656 case Intrinsic::x86_sse_cvttss2si64:
2657 if (!Subtarget->hasSSE1())
2659 IsInputDouble = false;
2661 case Intrinsic::x86_sse2_cvttsd2si:
2662 case Intrinsic::x86_sse2_cvttsd2si64:
2663 if (!Subtarget->hasSSE2())
2665 IsInputDouble = true;
2669 Type *RetTy = II->getCalledFunction()->getReturnType();
2671 if (!isTypeLegal(RetTy, VT))
2674 static const unsigned CvtOpc[2][2][2] = {
2675 { { X86::CVTTSS2SIrr, X86::VCVTTSS2SIrr },
2676 { X86::CVTTSS2SI64rr, X86::VCVTTSS2SI64rr } },
2677 { { X86::CVTTSD2SIrr, X86::VCVTTSD2SIrr },
2678 { X86::CVTTSD2SI64rr, X86::VCVTTSD2SI64rr } }
2680 bool HasAVX = Subtarget->hasAVX();
2682 switch (VT.SimpleTy) {
2683 default: llvm_unreachable("Unexpected result type.");
2684 case MVT::i32: Opc = CvtOpc[IsInputDouble][0][HasAVX]; break;
2685 case MVT::i64: Opc = CvtOpc[IsInputDouble][1][HasAVX]; break;
2688 // Check if we can fold insertelement instructions into the convert.
2689 const Value *Op = II->getArgOperand(0);
2690 while (auto *IE = dyn_cast<InsertElementInst>(Op)) {
2691 const Value *Index = IE->getOperand(2);
2692 if (!isa<ConstantInt>(Index))
2694 unsigned Idx = cast<ConstantInt>(Index)->getZExtValue();
2697 Op = IE->getOperand(1);
2700 Op = IE->getOperand(0);
2703 unsigned Reg = getRegForValue(Op);
2707 unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
2708 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
2711 updateValueMap(II, ResultReg);
2717 bool X86FastISel::fastLowerArguments() {
2718 if (!FuncInfo.CanLowerReturn)
2721 const Function *F = FuncInfo.Fn;
2725 CallingConv::ID CC = F->getCallingConv();
2726 if (CC != CallingConv::C)
2729 if (Subtarget->isCallingConvWin64(CC))
2732 if (!Subtarget->is64Bit())
2735 // Only handle simple cases. i.e. Up to 6 i32/i64 scalar arguments.
2736 unsigned GPRCnt = 0;
2737 unsigned FPRCnt = 0;
2739 for (auto const &Arg : F->args()) {
2740 // The first argument is at index 1.
2742 if (F->getAttributes().hasAttribute(Idx, Attribute::ByVal) ||
2743 F->getAttributes().hasAttribute(Idx, Attribute::InReg) ||
2744 F->getAttributes().hasAttribute(Idx, Attribute::StructRet) ||
2745 F->getAttributes().hasAttribute(Idx, Attribute::Nest))
2748 Type *ArgTy = Arg.getType();
2749 if (ArgTy->isStructTy() || ArgTy->isArrayTy() || ArgTy->isVectorTy())
2752 EVT ArgVT = TLI.getValueType(DL, ArgTy);
2753 if (!ArgVT.isSimple()) return false;
2754 switch (ArgVT.getSimpleVT().SimpleTy) {
2755 default: return false;
2762 if (!Subtarget->hasSSE1())
2775 static const MCPhysReg GPR32ArgRegs[] = {
2776 X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D
2778 static const MCPhysReg GPR64ArgRegs[] = {
2779 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8 , X86::R9
2781 static const MCPhysReg XMMArgRegs[] = {
2782 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
2783 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
2786 unsigned GPRIdx = 0;
2787 unsigned FPRIdx = 0;
2788 for (auto const &Arg : F->args()) {
2789 MVT VT = TLI.getSimpleValueType(DL, Arg.getType());
2790 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
2792 switch (VT.SimpleTy) {
2793 default: llvm_unreachable("Unexpected value type.");
2794 case MVT::i32: SrcReg = GPR32ArgRegs[GPRIdx++]; break;
2795 case MVT::i64: SrcReg = GPR64ArgRegs[GPRIdx++]; break;
2796 case MVT::f32: // fall-through
2797 case MVT::f64: SrcReg = XMMArgRegs[FPRIdx++]; break;
2799 unsigned DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
2800 // FIXME: Unfortunately it's necessary to emit a copy from the livein copy.
2801 // Without this, EmitLiveInCopies may eliminate the livein if its only
2802 // use is a bitcast (which isn't turned into an instruction).
2803 unsigned ResultReg = createResultReg(RC);
2804 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2805 TII.get(TargetOpcode::COPY), ResultReg)
2806 .addReg(DstReg, getKillRegState(true));
2807 updateValueMap(&Arg, ResultReg);
2812 static unsigned computeBytesPoppedByCallee(const X86Subtarget *Subtarget,
2814 ImmutableCallSite *CS) {
2815 if (Subtarget->is64Bit())
2817 if (Subtarget->getTargetTriple().isOSMSVCRT())
2819 if (CC == CallingConv::Fast || CC == CallingConv::GHC ||
2820 CC == CallingConv::HiPE)
2824 if (CS->arg_empty() || !CS->paramHasAttr(1, Attribute::StructRet) ||
2825 CS->paramHasAttr(1, Attribute::InReg) || Subtarget->isTargetMCU())
2831 bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
2832 auto &OutVals = CLI.OutVals;
2833 auto &OutFlags = CLI.OutFlags;
2834 auto &OutRegs = CLI.OutRegs;
2835 auto &Ins = CLI.Ins;
2836 auto &InRegs = CLI.InRegs;
2837 CallingConv::ID CC = CLI.CallConv;
2838 bool &IsTailCall = CLI.IsTailCall;
2839 bool IsVarArg = CLI.IsVarArg;
2840 const Value *Callee = CLI.Callee;
2841 MCSymbol *Symbol = CLI.Symbol;
2843 bool Is64Bit = Subtarget->is64Bit();
2844 bool IsWin64 = Subtarget->isCallingConvWin64(CC);
2846 // Handle only C, fastcc, and webkit_js calling conventions for now.
2848 default: return false;
2849 case CallingConv::C:
2850 case CallingConv::Fast:
2851 case CallingConv::WebKit_JS:
2852 case CallingConv::X86_FastCall:
2853 case CallingConv::X86_64_Win64:
2854 case CallingConv::X86_64_SysV:
2858 // Allow SelectionDAG isel to handle tail calls.
2862 // fastcc with -tailcallopt is intended to provide a guaranteed
2863 // tail call optimization. Fastisel doesn't know how to do that.
2864 if (CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt)
2867 // Don't know how to handle Win64 varargs yet. Nothing special needed for
2868 // x86-32. Special handling for x86-64 is implemented.
2869 if (IsVarArg && IsWin64)
2872 // Don't know about inalloca yet.
2873 if (CLI.CS && CLI.CS->hasInAllocaArgument())
2876 // Fast-isel doesn't know about callee-pop yet.
2877 if (X86::isCalleePop(CC, Subtarget->is64Bit(), IsVarArg,
2878 TM.Options.GuaranteedTailCallOpt))
2881 SmallVector<MVT, 16> OutVTs;
2882 SmallVector<unsigned, 16> ArgRegs;
2884 // If this is a constant i1/i8/i16 argument, promote to i32 to avoid an extra
2885 // instruction. This is safe because it is common to all FastISel supported
2886 // calling conventions on x86.
2887 for (int i = 0, e = OutVals.size(); i != e; ++i) {
2888 Value *&Val = OutVals[i];
2889 ISD::ArgFlagsTy Flags = OutFlags[i];
2890 if (auto *CI = dyn_cast<ConstantInt>(Val)) {
2891 if (CI->getBitWidth() < 32) {
2893 Val = ConstantExpr::getSExt(CI, Type::getInt32Ty(CI->getContext()));
2895 Val = ConstantExpr::getZExt(CI, Type::getInt32Ty(CI->getContext()));
2899 // Passing bools around ends up doing a trunc to i1 and passing it.
2900 // Codegen this as an argument + "and 1".
2902 auto *TI = dyn_cast<TruncInst>(Val);
2904 if (TI && TI->getType()->isIntegerTy(1) && CLI.CS &&
2905 (TI->getParent() == CLI.CS->getInstruction()->getParent()) &&
2907 Value *PrevVal = TI->getOperand(0);
2908 ResultReg = getRegForValue(PrevVal);
2913 if (!isTypeLegal(PrevVal->getType(), VT))
2917 fastEmit_ri(VT, VT, ISD::AND, ResultReg, hasTrivialKill(PrevVal), 1);
2919 if (!isTypeLegal(Val->getType(), VT))
2921 ResultReg = getRegForValue(Val);
2927 ArgRegs.push_back(ResultReg);
2928 OutVTs.push_back(VT);
2931 // Analyze operands of the call, assigning locations to each operand.
2932 SmallVector<CCValAssign, 16> ArgLocs;
2933 CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, ArgLocs, CLI.RetTy->getContext());
2935 // Allocate shadow area for Win64
2937 CCInfo.AllocateStack(32, 8);
2939 CCInfo.AnalyzeCallOperands(OutVTs, OutFlags, CC_X86);
2941 // Get a count of how many bytes are to be pushed on the stack.
2942 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
2944 // Issue CALLSEQ_START
2945 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
2946 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown))
2947 .addImm(NumBytes).addImm(0);
2949 // Walk the register/memloc assignments, inserting copies/loads.
2950 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
2951 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2952 CCValAssign const &VA = ArgLocs[i];
2953 const Value *ArgVal = OutVals[VA.getValNo()];
2954 MVT ArgVT = OutVTs[VA.getValNo()];
2956 if (ArgVT == MVT::x86mmx)
2959 unsigned ArgReg = ArgRegs[VA.getValNo()];
2961 // Promote the value if needed.
2962 switch (VA.getLocInfo()) {
2963 case CCValAssign::Full: break;
2964 case CCValAssign::SExt: {
2965 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
2966 "Unexpected extend");
2967 bool Emitted = X86FastEmitExtend(ISD::SIGN_EXTEND, VA.getLocVT(), ArgReg,
2969 assert(Emitted && "Failed to emit a sext!"); (void)Emitted;
2970 ArgVT = VA.getLocVT();
2973 case CCValAssign::ZExt: {
2974 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
2975 "Unexpected extend");
2976 bool Emitted = X86FastEmitExtend(ISD::ZERO_EXTEND, VA.getLocVT(), ArgReg,
2978 assert(Emitted && "Failed to emit a zext!"); (void)Emitted;
2979 ArgVT = VA.getLocVT();
2982 case CCValAssign::AExt: {
2983 assert(VA.getLocVT().isInteger() && !VA.getLocVT().isVector() &&
2984 "Unexpected extend");
2985 bool Emitted = X86FastEmitExtend(ISD::ANY_EXTEND, VA.getLocVT(), ArgReg,
2988 Emitted = X86FastEmitExtend(ISD::ZERO_EXTEND, VA.getLocVT(), ArgReg,
2991 Emitted = X86FastEmitExtend(ISD::SIGN_EXTEND, VA.getLocVT(), ArgReg,
2994 assert(Emitted && "Failed to emit a aext!"); (void)Emitted;
2995 ArgVT = VA.getLocVT();
2998 case CCValAssign::BCvt: {
2999 ArgReg = fastEmit_r(ArgVT, VA.getLocVT(), ISD::BITCAST, ArgReg,
3000 /*TODO: Kill=*/false);
3001 assert(ArgReg && "Failed to emit a bitcast!");
3002 ArgVT = VA.getLocVT();
3005 case CCValAssign::VExt:
3006 // VExt has not been implemented, so this should be impossible to reach
3007 // for now. However, fallback to Selection DAG isel once implemented.
3009 case CCValAssign::AExtUpper:
3010 case CCValAssign::SExtUpper:
3011 case CCValAssign::ZExtUpper:
3012 case CCValAssign::FPExt:
3013 llvm_unreachable("Unexpected loc info!");
3014 case CCValAssign::Indirect:
3015 // FIXME: Indirect doesn't need extending, but fast-isel doesn't fully
3020 if (VA.isRegLoc()) {
3021 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3022 TII.get(TargetOpcode::COPY), VA.getLocReg()).addReg(ArgReg);
3023 OutRegs.push_back(VA.getLocReg());
3025 assert(VA.isMemLoc());
3027 // Don't emit stores for undef values.
3028 if (isa<UndefValue>(ArgVal))
3031 unsigned LocMemOffset = VA.getLocMemOffset();
3033 AM.Base.Reg = RegInfo->getStackRegister();
3034 AM.Disp = LocMemOffset;
3035 ISD::ArgFlagsTy Flags = OutFlags[VA.getValNo()];
3036 unsigned Alignment = DL.getABITypeAlignment(ArgVal->getType());
3037 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
3038 MachinePointerInfo::getStack(*FuncInfo.MF, LocMemOffset),
3039 MachineMemOperand::MOStore, ArgVT.getStoreSize(), Alignment);
3040 if (Flags.isByVal()) {
3041 X86AddressMode SrcAM;
3042 SrcAM.Base.Reg = ArgReg;
3043 if (!TryEmitSmallMemcpy(AM, SrcAM, Flags.getByValSize()))
3045 } else if (isa<ConstantInt>(ArgVal) || isa<ConstantPointerNull>(ArgVal)) {
3046 // If this is a really simple value, emit this with the Value* version
3047 // of X86FastEmitStore. If it isn't simple, we don't want to do this,
3048 // as it can cause us to reevaluate the argument.
3049 if (!X86FastEmitStore(ArgVT, ArgVal, AM, MMO))
3052 bool ValIsKill = hasTrivialKill(ArgVal);
3053 if (!X86FastEmitStore(ArgVT, ArgReg, ValIsKill, AM, MMO))
3059 // ELF / PIC requires GOT in the EBX register before function calls via PLT
3061 if (Subtarget->isPICStyleGOT()) {
3062 unsigned Base = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3063 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3064 TII.get(TargetOpcode::COPY), X86::EBX).addReg(Base);
3067 if (Is64Bit && IsVarArg && !IsWin64) {
3068 // From AMD64 ABI document:
3069 // For calls that may call functions that use varargs or stdargs
3070 // (prototype-less calls or calls to functions containing ellipsis (...) in
3071 // the declaration) %al is used as hidden argument to specify the number
3072 // of SSE registers used. The contents of %al do not need to match exactly
3073 // the number of registers, but must be an ubound on the number of SSE
3074 // registers used and is in the range 0 - 8 inclusive.
3076 // Count the number of XMM registers allocated.
3077 static const MCPhysReg XMMArgRegs[] = {
3078 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3079 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3081 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs);
3082 assert((Subtarget->hasSSE1() || !NumXMMRegs)
3083 && "SSE registers cannot be used when SSE is disabled");
3084 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV8ri),
3085 X86::AL).addImm(NumXMMRegs);
3088 // Materialize callee address in a register. FIXME: GV address can be
3089 // handled with a CALLpcrel32 instead.
3090 X86AddressMode CalleeAM;
3091 if (!X86SelectCallAddress(Callee, CalleeAM))
3094 unsigned CalleeOp = 0;
3095 const GlobalValue *GV = nullptr;
3096 if (CalleeAM.GV != nullptr) {
3098 } else if (CalleeAM.Base.Reg != 0) {
3099 CalleeOp = CalleeAM.Base.Reg;
3104 MachineInstrBuilder MIB;
3106 // Register-indirect call.
3107 unsigned CallOpc = Is64Bit ? X86::CALL64r : X86::CALL32r;
3108 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc))
3112 assert(GV && "Not a direct call");
3113 unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
3115 // See if we need any target-specific flags on the GV operand.
3116 unsigned char OpFlags = 0;
3118 // On ELF targets, in both X86-64 and X86-32 mode, direct calls to
3119 // external symbols most go through the PLT in PIC mode. If the symbol
3120 // has hidden or protected visibility, or if it is static or local, then
3121 // we don't need to use the PLT - we can directly call it.
3122 if (Subtarget->isTargetELF() &&
3123 TM.getRelocationModel() == Reloc::PIC_ &&
3124 GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) {
3125 OpFlags = X86II::MO_PLT;
3126 } else if (Subtarget->isPICStyleStubAny() &&
3127 !GV->isStrongDefinitionForLinker() &&
3128 (!Subtarget->getTargetTriple().isMacOSX() ||
3129 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) {
3130 // PC-relative references to external symbols should go through $stub,
3131 // unless we're building with the leopard linker or later, which
3132 // automatically synthesizes these stubs.
3133 OpFlags = X86II::MO_DARWIN_STUB;
3136 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc));
3138 MIB.addSym(Symbol, OpFlags);
3140 MIB.addGlobalAddress(GV, 0, OpFlags);
3143 // Add a register mask operand representing the call-preserved registers.
3144 // Proper defs for return values will be added by setPhysRegsDeadExcept().
3145 MIB.addRegMask(TRI.getCallPreservedMask(*FuncInfo.MF, CC));
3147 // Add an implicit use GOT pointer in EBX.
3148 if (Subtarget->isPICStyleGOT())
3149 MIB.addReg(X86::EBX, RegState::Implicit);
3151 if (Is64Bit && IsVarArg && !IsWin64)
3152 MIB.addReg(X86::AL, RegState::Implicit);
3154 // Add implicit physical register uses to the call.
3155 for (auto Reg : OutRegs)
3156 MIB.addReg(Reg, RegState::Implicit);
3158 // Issue CALLSEQ_END
3159 unsigned NumBytesForCalleeToPop =
3160 computeBytesPoppedByCallee(Subtarget, CC, CLI.CS);
3161 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
3162 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
3163 .addImm(NumBytes).addImm(NumBytesForCalleeToPop);
3165 // Now handle call return values.
3166 SmallVector<CCValAssign, 16> RVLocs;
3167 CCState CCRetInfo(CC, IsVarArg, *FuncInfo.MF, RVLocs,
3168 CLI.RetTy->getContext());
3169 CCRetInfo.AnalyzeCallResult(Ins, RetCC_X86);
3171 // Copy all of the result registers out of their specified physreg.
3172 unsigned ResultReg = FuncInfo.CreateRegs(CLI.RetTy);
3173 for (unsigned i = 0; i != RVLocs.size(); ++i) {
3174 CCValAssign &VA = RVLocs[i];
3175 EVT CopyVT = VA.getValVT();
3176 unsigned CopyReg = ResultReg + i;
3178 // If this is x86-64, and we disabled SSE, we can't return FP values
3179 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
3180 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {
3181 report_fatal_error("SSE register return with SSE disabled");
3184 // If we prefer to use the value in xmm registers, copy it out as f80 and
3185 // use a truncate to move it from fp stack reg to xmm reg.
3186 if ((VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1) &&
3187 isScalarFPTypeInSSEReg(VA.getValVT())) {
3189 CopyReg = createResultReg(&X86::RFP80RegClass);
3192 // Copy out the result.
3193 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3194 TII.get(TargetOpcode::COPY), CopyReg).addReg(VA.getLocReg());
3195 InRegs.push_back(VA.getLocReg());
3197 // Round the f80 to the right size, which also moves it to the appropriate
3198 // xmm register. This is accomplished by storing the f80 value in memory
3199 // and then loading it back.
3200 if (CopyVT != VA.getValVT()) {
3201 EVT ResVT = VA.getValVT();
3202 unsigned Opc = ResVT == MVT::f32 ? X86::ST_Fp80m32 : X86::ST_Fp80m64;
3203 unsigned MemSize = ResVT.getSizeInBits()/8;
3204 int FI = MFI.CreateStackObject(MemSize, MemSize, false);
3205 addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3208 Opc = ResVT == MVT::f32 ? X86::MOVSSrm : X86::MOVSDrm;
3209 addFrameReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3210 TII.get(Opc), ResultReg + i), FI);
3214 CLI.ResultReg = ResultReg;
3215 CLI.NumResultRegs = RVLocs.size();
3222 X86FastISel::fastSelectInstruction(const Instruction *I) {
3223 switch (I->getOpcode()) {
3225 case Instruction::Load:
3226 return X86SelectLoad(I);
3227 case Instruction::Store:
3228 return X86SelectStore(I);
3229 case Instruction::Ret:
3230 return X86SelectRet(I);
3231 case Instruction::ICmp:
3232 case Instruction::FCmp:
3233 return X86SelectCmp(I);
3234 case Instruction::ZExt:
3235 return X86SelectZExt(I);
3236 case Instruction::Br:
3237 return X86SelectBranch(I);
3238 case Instruction::LShr:
3239 case Instruction::AShr:
3240 case Instruction::Shl:
3241 return X86SelectShift(I);
3242 case Instruction::SDiv:
3243 case Instruction::UDiv:
3244 case Instruction::SRem:
3245 case Instruction::URem:
3246 return X86SelectDivRem(I);
3247 case Instruction::Select:
3248 return X86SelectSelect(I);
3249 case Instruction::Trunc:
3250 return X86SelectTrunc(I);
3251 case Instruction::FPExt:
3252 return X86SelectFPExt(I);
3253 case Instruction::FPTrunc:
3254 return X86SelectFPTrunc(I);
3255 case Instruction::SIToFP:
3256 return X86SelectSIToFP(I);
3257 case Instruction::IntToPtr: // Deliberate fall-through.
3258 case Instruction::PtrToInt: {
3259 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
3260 EVT DstVT = TLI.getValueType(DL, I->getType());
3261 if (DstVT.bitsGT(SrcVT))
3262 return X86SelectZExt(I);
3263 if (DstVT.bitsLT(SrcVT))
3264 return X86SelectTrunc(I);
3265 unsigned Reg = getRegForValue(I->getOperand(0));
3266 if (Reg == 0) return false;
3267 updateValueMap(I, Reg);
3270 case Instruction::BitCast: {
3271 // Select SSE2/AVX bitcasts between 128/256 bit vector types.
3272 if (!Subtarget->hasSSE2())
3275 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
3276 EVT DstVT = TLI.getValueType(DL, I->getType());
3278 if (!SrcVT.isSimple() || !DstVT.isSimple())
3281 if (!SrcVT.is128BitVector() &&
3282 !(Subtarget->hasAVX() && SrcVT.is256BitVector()))
3285 unsigned Reg = getRegForValue(I->getOperand(0));
3289 // No instruction is needed for conversion. Reuse the register used by
3290 // the fist operand.
3291 updateValueMap(I, Reg);
3299 unsigned X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {
3303 uint64_t Imm = CI->getZExtValue();
3305 unsigned SrcReg = fastEmitInst_(X86::MOV32r0, &X86::GR32RegClass);
3306 switch (VT.SimpleTy) {
3307 default: llvm_unreachable("Unexpected value type");
3310 return fastEmitInst_extractsubreg(MVT::i8, SrcReg, /*Kill=*/true,
3313 return fastEmitInst_extractsubreg(MVT::i16, SrcReg, /*Kill=*/true,
3318 unsigned ResultReg = createResultReg(&X86::GR64RegClass);
3319 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3320 TII.get(TargetOpcode::SUBREG_TO_REG), ResultReg)
3321 .addImm(0).addReg(SrcReg).addImm(X86::sub_32bit);
3328 switch (VT.SimpleTy) {
3329 default: llvm_unreachable("Unexpected value type");
3330 case MVT::i1: VT = MVT::i8; // fall-through
3331 case MVT::i8: Opc = X86::MOV8ri; break;
3332 case MVT::i16: Opc = X86::MOV16ri; break;
3333 case MVT::i32: Opc = X86::MOV32ri; break;
3335 if (isUInt<32>(Imm))
3337 else if (isInt<32>(Imm))
3338 Opc = X86::MOV64ri32;
3344 if (VT == MVT::i64 && Opc == X86::MOV32ri) {
3345 unsigned SrcReg = fastEmitInst_i(Opc, &X86::GR32RegClass, Imm);
3346 unsigned ResultReg = createResultReg(&X86::GR64RegClass);
3347 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3348 TII.get(TargetOpcode::SUBREG_TO_REG), ResultReg)
3349 .addImm(0).addReg(SrcReg).addImm(X86::sub_32bit);
3352 return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
3355 unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
3356 if (CFP->isNullValue())
3357 return fastMaterializeFloatZero(CFP);
3359 // Can't handle alternate code models yet.
3360 CodeModel::Model CM = TM.getCodeModel();
3361 if (CM != CodeModel::Small && CM != CodeModel::Large)
3364 // Get opcode and regclass of the output for the given load instruction.
3366 const TargetRegisterClass *RC = nullptr;
3367 switch (VT.SimpleTy) {
3370 if (X86ScalarSSEf32) {
3371 Opc = Subtarget->hasAVX() ? X86::VMOVSSrm : X86::MOVSSrm;
3372 RC = &X86::FR32RegClass;
3374 Opc = X86::LD_Fp32m;
3375 RC = &X86::RFP32RegClass;
3379 if (X86ScalarSSEf64) {
3380 Opc = Subtarget->hasAVX() ? X86::VMOVSDrm : X86::MOVSDrm;
3381 RC = &X86::FR64RegClass;
3383 Opc = X86::LD_Fp64m;
3384 RC = &X86::RFP64RegClass;
3388 // No f80 support yet.
3392 // MachineConstantPool wants an explicit alignment.
3393 unsigned Align = DL.getPrefTypeAlignment(CFP->getType());
3395 // Alignment of vector types. FIXME!
3396 Align = DL.getTypeAllocSize(CFP->getType());
3399 // x86-32 PIC requires a PIC base register for constant pools.
3400 unsigned PICBase = 0;
3401 unsigned char OpFlag = 0;
3402 if (Subtarget->isPICStyleStubPIC()) { // Not dynamic-no-pic
3403 OpFlag = X86II::MO_PIC_BASE_OFFSET;
3404 PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3405 } else if (Subtarget->isPICStyleGOT()) {
3406 OpFlag = X86II::MO_GOTOFF;
3407 PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3408 } else if (Subtarget->isPICStyleRIPRel() &&
3409 TM.getCodeModel() == CodeModel::Small) {
3413 // Create the load from the constant pool.
3414 unsigned CPI = MCP.getConstantPoolIndex(CFP, Align);
3415 unsigned ResultReg = createResultReg(RC);
3417 if (CM == CodeModel::Large) {
3418 unsigned AddrReg = createResultReg(&X86::GR64RegClass);
3419 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV64ri),
3421 .addConstantPoolIndex(CPI, 0, OpFlag);
3422 MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3423 TII.get(Opc), ResultReg);
3424 addDirectMem(MIB, AddrReg);
3425 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
3426 MachinePointerInfo::getConstantPool(*FuncInfo.MF),
3427 MachineMemOperand::MOLoad, DL.getPointerSize(), Align);
3428 MIB->addMemOperand(*FuncInfo.MF, MMO);
3432 addConstantPoolReference(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3433 TII.get(Opc), ResultReg),
3434 CPI, PICBase, OpFlag);
3438 unsigned X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
3439 // Can't handle alternate code models yet.
3440 if (TM.getCodeModel() != CodeModel::Small)
3443 // Materialize addresses with LEA/MOV instructions.
3445 if (X86SelectAddress(GV, AM)) {
3446 // If the expression is just a basereg, then we're done, otherwise we need
3448 if (AM.BaseType == X86AddressMode::RegBase &&
3449 AM.IndexReg == 0 && AM.Disp == 0 && AM.GV == nullptr)
3452 unsigned ResultReg = createResultReg(TLI.getRegClassFor(VT));
3453 if (TM.getRelocationModel() == Reloc::Static &&
3454 TLI.getPointerTy(DL) == MVT::i64) {
3455 // The displacement code could be more than 32 bits away so we need to use
3456 // an instruction with a 64 bit immediate
3457 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(X86::MOV64ri),
3459 .addGlobalAddress(GV);
3462 TLI.getPointerTy(DL) == MVT::i32
3463 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3465 addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3466 TII.get(Opc), ResultReg), AM);
3473 unsigned X86FastISel::fastMaterializeConstant(const Constant *C) {
3474 EVT CEVT = TLI.getValueType(DL, C->getType(), true);
3476 // Only handle simple types.
3477 if (!CEVT.isSimple())
3479 MVT VT = CEVT.getSimpleVT();
3481 if (const auto *CI = dyn_cast<ConstantInt>(C))
3482 return X86MaterializeInt(CI, VT);
3483 else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
3484 return X86MaterializeFP(CFP, VT);
3485 else if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
3486 return X86MaterializeGV(GV, VT);
3491 unsigned X86FastISel::fastMaterializeAlloca(const AllocaInst *C) {
3492 // Fail on dynamic allocas. At this point, getRegForValue has already
3493 // checked its CSE maps, so if we're here trying to handle a dynamic
3494 // alloca, we're not going to succeed. X86SelectAddress has a
3495 // check for dynamic allocas, because it's called directly from
3496 // various places, but targetMaterializeAlloca also needs a check
3497 // in order to avoid recursion between getRegForValue,
3498 // X86SelectAddrss, and targetMaterializeAlloca.
3499 if (!FuncInfo.StaticAllocaMap.count(C))
3501 assert(C->isStaticAlloca() && "dynamic alloca in the static alloca map?");
3504 if (!X86SelectAddress(C, AM))
3507 TLI.getPointerTy(DL) == MVT::i32
3508 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3510 const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
3511 unsigned ResultReg = createResultReg(RC);
3512 addFullAddress(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
3513 TII.get(Opc), ResultReg), AM);
3517 unsigned X86FastISel::fastMaterializeFloatZero(const ConstantFP *CF) {
3519 if (!isTypeLegal(CF->getType(), VT))
3522 // Get opcode and regclass for the given zero.
3524 const TargetRegisterClass *RC = nullptr;
3525 switch (VT.SimpleTy) {
3528 if (X86ScalarSSEf32) {
3529 Opc = X86::FsFLD0SS;
3530 RC = &X86::FR32RegClass;
3532 Opc = X86::LD_Fp032;
3533 RC = &X86::RFP32RegClass;
3537 if (X86ScalarSSEf64) {
3538 Opc = X86::FsFLD0SD;
3539 RC = &X86::FR64RegClass;
3541 Opc = X86::LD_Fp064;
3542 RC = &X86::RFP64RegClass;
3546 // No f80 support yet.
3550 unsigned ResultReg = createResultReg(RC);
3551 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg);
3556 bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
3557 const LoadInst *LI) {
3558 const Value *Ptr = LI->getPointerOperand();
3560 if (!X86SelectAddress(Ptr, AM))
3563 const X86InstrInfo &XII = (const X86InstrInfo &)TII;
3565 unsigned Size = DL.getTypeAllocSize(LI->getType());
3566 unsigned Alignment = LI->getAlignment();
3568 if (Alignment == 0) // Ensure that codegen never sees alignment 0
3569 Alignment = DL.getABITypeAlignment(LI->getType());
3571 SmallVector<MachineOperand, 8> AddrOps;
3572 AM.getFullAddress(AddrOps);
3574 MachineInstr *Result = XII.foldMemoryOperandImpl(
3575 *FuncInfo.MF, MI, OpNo, AddrOps, FuncInfo.InsertPt, Size, Alignment,
3576 /*AllowCommute=*/true);
3580 // The index register could be in the wrong register class. Unfortunately,
3581 // foldMemoryOperandImpl could have commuted the instruction so its not enough
3582 // to just look at OpNo + the offset to the index reg. We actually need to
3583 // scan the instruction to find the index reg and see if its the correct reg
3585 unsigned OperandNo = 0;
3586 for (MachineInstr::mop_iterator I = Result->operands_begin(),
3587 E = Result->operands_end(); I != E; ++I, ++OperandNo) {
3588 MachineOperand &MO = *I;
3589 if (!MO.isReg() || MO.isDef() || MO.getReg() != AM.IndexReg)
3591 // Found the index reg, now try to rewrite it.
3592 unsigned IndexReg = constrainOperandRegClass(Result->getDesc(),
3593 MO.getReg(), OperandNo);
3594 if (IndexReg == MO.getReg())
3596 MO.setReg(IndexReg);
3599 Result->addMemOperand(*FuncInfo.MF, createMachineMemOperandFor(LI));
3600 MI->eraseFromParent();
3606 FastISel *X86::createFastISel(FunctionLoweringInfo &funcInfo,
3607 const TargetLibraryInfo *libInfo) {
3608 return new X86FastISel(funcInfo, libInfo);