1 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines an instruction selector for the ARM target.
12 //===----------------------------------------------------------------------===//
14 #define DEBUG_TYPE "arm-isel"
16 #include "ARMBaseInstrInfo.h"
17 #include "ARMTargetMachine.h"
18 #include "MCTargetDesc/ARMAddressingModes.h"
19 #include "llvm/CallingConv.h"
20 #include "llvm/Constants.h"
21 #include "llvm/DerivedTypes.h"
22 #include "llvm/Function.h"
23 #include "llvm/Intrinsics.h"
24 #include "llvm/LLVMContext.h"
25 #include "llvm/CodeGen/MachineFrameInfo.h"
26 #include "llvm/CodeGen/MachineFunction.h"
27 #include "llvm/CodeGen/MachineInstrBuilder.h"
28 #include "llvm/CodeGen/SelectionDAG.h"
29 #include "llvm/CodeGen/SelectionDAGISel.h"
30 #include "llvm/Target/TargetLowering.h"
31 #include "llvm/Target/TargetOptions.h"
32 #include "llvm/Support/CommandLine.h"
33 #include "llvm/Support/Compiler.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/raw_ostream.h"
41 DisableShifterOp("disable-shifter-op", cl::Hidden,
42 cl::desc("Disable isel of shifter-op"),
46 CheckVMLxHazard("check-vmlx-hazard", cl::Hidden,
47 cl::desc("Check fp vmla / vmls hazard at isel time"),
50 //===--------------------------------------------------------------------===//
51 /// ARMDAGToDAGISel - ARM specific code to select ARM machine
52 /// instructions for SelectionDAG operations.
57 AM2_BASE, // Simple AM2 (+-imm12)
58 AM2_SHOP // Shifter-op AM2
61 class ARMDAGToDAGISel : public SelectionDAGISel {
62 ARMBaseTargetMachine &TM;
63 const ARMBaseInstrInfo *TII;
65 /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
66 /// make the right decision when generating code for different targets.
67 const ARMSubtarget *Subtarget;
70 explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
71 CodeGenOpt::Level OptLevel)
72 : SelectionDAGISel(tm, OptLevel), TM(tm),
73 TII(static_cast<const ARMBaseInstrInfo*>(TM.getInstrInfo())),
74 Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
77 virtual const char *getPassName() const {
78 return "ARM Instruction Selection";
81 /// getI32Imm - Return a target constant of type i32 with the specified
83 inline SDValue getI32Imm(unsigned Imm) {
84 return CurDAG->getTargetConstant(Imm, MVT::i32);
87 SDNode *Select(SDNode *N);
90 bool hasNoVMLxHazardUse(SDNode *N) const;
91 bool isShifterOpProfitable(const SDValue &Shift,
92 ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
93 bool SelectRegShifterOperand(SDValue N, SDValue &A,
94 SDValue &B, SDValue &C,
95 bool CheckProfitability = true);
96 bool SelectImmShifterOperand(SDValue N, SDValue &A,
97 SDValue &B, bool CheckProfitability = true);
98 bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
99 SDValue &B, SDValue &C) {
100 // Don't apply the profitability check
101 return SelectRegShifterOperand(N, A, B, C, false);
103 bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
105 // Don't apply the profitability check
106 return SelectImmShifterOperand(N, A, B, false);
109 bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
110 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
112 AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
113 SDValue &Offset, SDValue &Opc);
114 bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
116 return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
119 bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
121 return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
124 bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
126 SelectAddrMode2Worker(N, Base, Offset, Opc);
127 // return SelectAddrMode2ShOp(N, Base, Offset, Opc);
128 // This always matches one way or another.
132 bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
133 SDValue &Offset, SDValue &Opc);
134 bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
135 SDValue &Offset, SDValue &Opc);
136 bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
137 SDValue &Offset, SDValue &Opc);
138 bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
139 bool SelectAddrMode3(SDValue N, SDValue &Base,
140 SDValue &Offset, SDValue &Opc);
141 bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
142 SDValue &Offset, SDValue &Opc);
143 bool SelectAddrMode5(SDValue N, SDValue &Base,
145 bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
146 bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
148 bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
150 // Thumb Addressing Modes:
151 bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
152 bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset,
154 bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset);
155 bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset);
156 bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset);
157 bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
159 bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
161 bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
163 bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
165 bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
167 // Thumb 2 Addressing Modes:
168 bool SelectT2ShifterOperandReg(SDValue N,
169 SDValue &BaseReg, SDValue &Opc);
170 bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
171 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
173 bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
175 bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
176 SDValue &OffReg, SDValue &ShImm);
178 inline bool is_so_imm(unsigned Imm) const {
179 return ARM_AM::getSOImmVal(Imm) != -1;
182 inline bool is_so_imm_not(unsigned Imm) const {
183 return ARM_AM::getSOImmVal(~Imm) != -1;
186 inline bool is_t2_so_imm(unsigned Imm) const {
187 return ARM_AM::getT2SOImmVal(Imm) != -1;
190 inline bool is_t2_so_imm_not(unsigned Imm) const {
191 return ARM_AM::getT2SOImmVal(~Imm) != -1;
194 // Include the pieces autogenerated from the target description.
195 #include "ARMGenDAGISel.inc"
198 /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
200 SDNode *SelectARMIndexedLoad(SDNode *N);
201 SDNode *SelectT2IndexedLoad(SDNode *N);
203 /// SelectVLD - Select NEON load intrinsics. NumVecs should be
204 /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for
205 /// loads of D registers and even subregs and odd subregs of Q registers.
206 /// For NumVecs <= 2, QOpcodes1 is not used.
207 SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
208 const uint16_t *DOpcodes,
209 const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
211 /// SelectVST - Select NEON store intrinsics. NumVecs should
212 /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for
213 /// stores of D registers and even subregs and odd subregs of Q registers.
214 /// For NumVecs <= 2, QOpcodes1 is not used.
215 SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
216 const uint16_t *DOpcodes,
217 const uint16_t *QOpcodes0, const uint16_t *QOpcodes1);
219 /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should
220 /// be 2, 3 or 4. The opcode arrays specify the instructions used for
221 /// load/store of D registers and Q registers.
222 SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad,
223 bool isUpdating, unsigned NumVecs,
224 const uint16_t *DOpcodes, const uint16_t *QOpcodes);
226 /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs
227 /// should be 2, 3 or 4. The opcode array specifies the instructions used
228 /// for loading D registers. (Q registers are not supported.)
229 SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
230 const uint16_t *Opcodes);
232 /// SelectVTBL - Select NEON VTBL and VTBX intrinsics. NumVecs should be 2,
233 /// 3 or 4. These are custom-selected so that a REG_SEQUENCE can be
234 /// generated to force the table registers to be consecutive.
235 SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
237 /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
238 SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
240 /// SelectCMOVOp - Select CMOV instructions for ARM.
241 SDNode *SelectCMOVOp(SDNode *N);
242 SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
243 ARMCC::CondCodes CCVal, SDValue CCR,
245 SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
246 ARMCC::CondCodes CCVal, SDValue CCR,
248 SDNode *SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
249 ARMCC::CondCodes CCVal, SDValue CCR,
251 SDNode *SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
252 ARMCC::CondCodes CCVal, SDValue CCR,
255 // Select special operations if node forms integer ABS pattern
256 SDNode *SelectABSOp(SDNode *N);
258 SDNode *SelectConcatVector(SDNode *N);
260 SDNode *SelectAtomic64(SDNode *Node, unsigned Opc);
262 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
263 /// inline asm expressions.
264 virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
266 std::vector<SDValue> &OutOps);
268 // Form pairs of consecutive S, D, or Q registers.
269 SDNode *PairSRegs(EVT VT, SDValue V0, SDValue V1);
270 SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
271 SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
273 // Form sequences of 4 consecutive S, D, or Q registers.
274 SDNode *QuadSRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
275 SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
276 SDNode *QuadQRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
278 // Get the alignment operand for a NEON VLD or VST instruction.
279 SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector);
283 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant
284 /// operand. If so Imm will receive the 32-bit value.
285 static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
286 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
287 Imm = cast<ConstantSDNode>(N)->getZExtValue();
293 // isInt32Immediate - This method tests to see if a constant operand.
294 // If so Imm will receive the 32 bit value.
295 static bool isInt32Immediate(SDValue N, unsigned &Imm) {
296 return isInt32Immediate(N.getNode(), Imm);
299 // isOpcWithIntImmediate - This method tests to see if the node is a specific
300 // opcode and that it has a immediate integer right operand.
301 // If so Imm will receive the 32 bit value.
302 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
303 return N->getOpcode() == Opc &&
304 isInt32Immediate(N->getOperand(1).getNode(), Imm);
307 /// \brief Check whether a particular node is a constant value representable as
308 /// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
310 /// \param ScaledConstant [out] - On success, the pre-scaled constant value.
311 static bool isScaledConstantInRange(SDValue Node, int Scale,
312 int RangeMin, int RangeMax,
313 int &ScaledConstant) {
314 assert(Scale > 0 && "Invalid scale!");
316 // Check that this is a constant.
317 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
321 ScaledConstant = (int) C->getZExtValue();
322 if ((ScaledConstant % Scale) != 0)
325 ScaledConstant /= Scale;
326 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
329 /// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
330 /// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
331 /// least on current ARM implementations) which should be avoidded.
332 bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
333 if (OptLevel == CodeGenOpt::None)
336 if (!CheckVMLxHazard)
339 if (!Subtarget->isCortexA8() && !Subtarget->isLikeA9() &&
340 !Subtarget->isSwift())
346 SDNode *Use = *N->use_begin();
347 if (Use->getOpcode() == ISD::CopyToReg)
349 if (Use->isMachineOpcode()) {
350 const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
353 unsigned Opcode = MCID.getOpcode();
354 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
356 // vmlx feeding into another vmlx. We actually want to unfold
357 // the use later in the MLxExpansion pass. e.g.
359 // vmla (stall 8 cycles)
364 // This adds up to about 18 - 19 cycles.
367 // vmul (stall 4 cycles)
368 // vadd adds up to about 14 cycles.
369 return TII->isFpMLxInstruction(Opcode);
375 bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
376 ARM_AM::ShiftOpc ShOpcVal,
378 if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
380 if (Shift.hasOneUse())
383 return ShOpcVal == ARM_AM::lsl &&
384 (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
387 bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
390 bool CheckProfitability) {
391 if (DisableShifterOp)
394 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
396 // Don't match base register only case. That is matched to a separate
397 // lower complexity pattern with explicit register operand.
398 if (ShOpcVal == ARM_AM::no_shift) return false;
400 BaseReg = N.getOperand(0);
401 unsigned ShImmVal = 0;
402 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
403 if (!RHS) return false;
404 ShImmVal = RHS->getZExtValue() & 31;
405 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
410 bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
414 bool CheckProfitability) {
415 if (DisableShifterOp)
418 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
420 // Don't match base register only case. That is matched to a separate
421 // lower complexity pattern with explicit register operand.
422 if (ShOpcVal == ARM_AM::no_shift) return false;
424 BaseReg = N.getOperand(0);
425 unsigned ShImmVal = 0;
426 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
427 if (RHS) return false;
429 ShReg = N.getOperand(1);
430 if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
432 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
438 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
441 // Match simple R + imm12 operands.
444 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
445 !CurDAG->isBaseWithConstantOffset(N)) {
446 if (N.getOpcode() == ISD::FrameIndex) {
447 // Match frame index.
448 int FI = cast<FrameIndexSDNode>(N)->getIndex();
449 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
450 OffImm = CurDAG->getTargetConstant(0, MVT::i32);
454 if (N.getOpcode() == ARMISD::Wrapper &&
455 !(Subtarget->useMovt() &&
456 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
457 Base = N.getOperand(0);
460 OffImm = CurDAG->getTargetConstant(0, MVT::i32);
464 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
465 int RHSC = (int)RHS->getZExtValue();
466 if (N.getOpcode() == ISD::SUB)
469 if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
470 Base = N.getOperand(0);
471 if (Base.getOpcode() == ISD::FrameIndex) {
472 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
473 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
475 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
482 OffImm = CurDAG->getTargetConstant(0, MVT::i32);
488 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
490 if (N.getOpcode() == ISD::MUL &&
491 ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
492 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
493 // X * [3,5,9] -> X + X * [2,4,8] etc.
494 int RHSC = (int)RHS->getZExtValue();
497 ARM_AM::AddrOpc AddSub = ARM_AM::add;
499 AddSub = ARM_AM::sub;
502 if (isPowerOf2_32(RHSC)) {
503 unsigned ShAmt = Log2_32(RHSC);
504 Base = Offset = N.getOperand(0);
505 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
514 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
515 // ISD::OR that is equivalent to an ISD::ADD.
516 !CurDAG->isBaseWithConstantOffset(N))
519 // Leave simple R +/- imm12 operands for LDRi12
520 if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
522 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
523 -0x1000+1, 0x1000, RHSC)) // 12 bits.
527 // Otherwise this is R +/- [possibly shifted] R.
528 ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
529 ARM_AM::ShiftOpc ShOpcVal =
530 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
533 Base = N.getOperand(0);
534 Offset = N.getOperand(1);
536 if (ShOpcVal != ARM_AM::no_shift) {
537 // Check to see if the RHS of the shift is a constant, if not, we can't fold
539 if (ConstantSDNode *Sh =
540 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
541 ShAmt = Sh->getZExtValue();
542 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
543 Offset = N.getOperand(1).getOperand(0);
546 ShOpcVal = ARM_AM::no_shift;
549 ShOpcVal = ARM_AM::no_shift;
553 // Try matching (R shl C) + (R).
554 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
555 !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
556 N.getOperand(0).hasOneUse())) {
557 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
558 if (ShOpcVal != ARM_AM::no_shift) {
559 // Check to see if the RHS of the shift is a constant, if not, we can't
561 if (ConstantSDNode *Sh =
562 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
563 ShAmt = Sh->getZExtValue();
564 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
565 Offset = N.getOperand(0).getOperand(0);
566 Base = N.getOperand(1);
569 ShOpcVal = ARM_AM::no_shift;
572 ShOpcVal = ARM_AM::no_shift;
577 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
585 AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
589 if (N.getOpcode() == ISD::MUL &&
590 (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) {
591 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
592 // X * [3,5,9] -> X + X * [2,4,8] etc.
593 int RHSC = (int)RHS->getZExtValue();
596 ARM_AM::AddrOpc AddSub = ARM_AM::add;
598 AddSub = ARM_AM::sub;
601 if (isPowerOf2_32(RHSC)) {
602 unsigned ShAmt = Log2_32(RHSC);
603 Base = Offset = N.getOperand(0);
604 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
613 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
614 // ISD::OR that is equivalent to an ADD.
615 !CurDAG->isBaseWithConstantOffset(N)) {
617 if (N.getOpcode() == ISD::FrameIndex) {
618 int FI = cast<FrameIndexSDNode>(N)->getIndex();
619 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
620 } else if (N.getOpcode() == ARMISD::Wrapper &&
621 !(Subtarget->useMovt() &&
622 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
623 Base = N.getOperand(0);
625 Offset = CurDAG->getRegister(0, MVT::i32);
626 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
632 // Match simple R +/- imm12 operands.
633 if (N.getOpcode() != ISD::SUB) {
635 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
636 -0x1000+1, 0x1000, RHSC)) { // 12 bits.
637 Base = N.getOperand(0);
638 if (Base.getOpcode() == ISD::FrameIndex) {
639 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
640 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
642 Offset = CurDAG->getRegister(0, MVT::i32);
644 ARM_AM::AddrOpc AddSub = ARM_AM::add;
646 AddSub = ARM_AM::sub;
649 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
656 if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) {
657 // Compute R +/- (R << N) and reuse it.
659 Offset = CurDAG->getRegister(0, MVT::i32);
660 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
666 // Otherwise this is R +/- [possibly shifted] R.
667 ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub;
668 ARM_AM::ShiftOpc ShOpcVal =
669 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
672 Base = N.getOperand(0);
673 Offset = N.getOperand(1);
675 if (ShOpcVal != ARM_AM::no_shift) {
676 // Check to see if the RHS of the shift is a constant, if not, we can't fold
678 if (ConstantSDNode *Sh =
679 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
680 ShAmt = Sh->getZExtValue();
681 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
682 Offset = N.getOperand(1).getOperand(0);
685 ShOpcVal = ARM_AM::no_shift;
688 ShOpcVal = ARM_AM::no_shift;
692 // Try matching (R shl C) + (R).
693 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
694 !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
695 N.getOperand(0).hasOneUse())) {
696 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
697 if (ShOpcVal != ARM_AM::no_shift) {
698 // Check to see if the RHS of the shift is a constant, if not, we can't
700 if (ConstantSDNode *Sh =
701 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
702 ShAmt = Sh->getZExtValue();
703 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
704 Offset = N.getOperand(0).getOperand(0);
705 Base = N.getOperand(1);
708 ShOpcVal = ARM_AM::no_shift;
711 ShOpcVal = ARM_AM::no_shift;
716 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
721 bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
722 SDValue &Offset, SDValue &Opc) {
723 unsigned Opcode = Op->getOpcode();
724 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
725 ? cast<LoadSDNode>(Op)->getAddressingMode()
726 : cast<StoreSDNode>(Op)->getAddressingMode();
727 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
728 ? ARM_AM::add : ARM_AM::sub;
730 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
734 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
736 if (ShOpcVal != ARM_AM::no_shift) {
737 // Check to see if the RHS of the shift is a constant, if not, we can't fold
739 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
740 ShAmt = Sh->getZExtValue();
741 if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
742 Offset = N.getOperand(0);
745 ShOpcVal = ARM_AM::no_shift;
748 ShOpcVal = ARM_AM::no_shift;
752 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
757 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
758 SDValue &Offset, SDValue &Opc) {
759 unsigned Opcode = Op->getOpcode();
760 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
761 ? cast<LoadSDNode>(Op)->getAddressingMode()
762 : cast<StoreSDNode>(Op)->getAddressingMode();
763 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
764 ? ARM_AM::add : ARM_AM::sub;
766 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
767 if (AddSub == ARM_AM::sub) Val *= -1;
768 Offset = CurDAG->getRegister(0, MVT::i32);
769 Opc = CurDAG->getTargetConstant(Val, MVT::i32);
777 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
778 SDValue &Offset, SDValue &Opc) {
779 unsigned Opcode = Op->getOpcode();
780 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
781 ? cast<LoadSDNode>(Op)->getAddressingMode()
782 : cast<StoreSDNode>(Op)->getAddressingMode();
783 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
784 ? ARM_AM::add : ARM_AM::sub;
786 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
787 Offset = CurDAG->getRegister(0, MVT::i32);
788 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
797 bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
802 bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
803 SDValue &Base, SDValue &Offset,
805 if (N.getOpcode() == ISD::SUB) {
806 // X - C is canonicalize to X + -C, no need to handle it here.
807 Base = N.getOperand(0);
808 Offset = N.getOperand(1);
809 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
813 if (!CurDAG->isBaseWithConstantOffset(N)) {
815 if (N.getOpcode() == ISD::FrameIndex) {
816 int FI = cast<FrameIndexSDNode>(N)->getIndex();
817 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
819 Offset = CurDAG->getRegister(0, MVT::i32);
820 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
824 // If the RHS is +/- imm8, fold into addr mode.
826 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
827 -256 + 1, 256, RHSC)) { // 8 bits.
828 Base = N.getOperand(0);
829 if (Base.getOpcode() == ISD::FrameIndex) {
830 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
831 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
833 Offset = CurDAG->getRegister(0, MVT::i32);
835 ARM_AM::AddrOpc AddSub = ARM_AM::add;
837 AddSub = ARM_AM::sub;
840 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
844 Base = N.getOperand(0);
845 Offset = N.getOperand(1);
846 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
850 bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
851 SDValue &Offset, SDValue &Opc) {
852 unsigned Opcode = Op->getOpcode();
853 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
854 ? cast<LoadSDNode>(Op)->getAddressingMode()
855 : cast<StoreSDNode>(Op)->getAddressingMode();
856 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
857 ? ARM_AM::add : ARM_AM::sub;
859 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
860 Offset = CurDAG->getRegister(0, MVT::i32);
861 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
866 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
870 bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
871 SDValue &Base, SDValue &Offset) {
872 if (!CurDAG->isBaseWithConstantOffset(N)) {
874 if (N.getOpcode() == ISD::FrameIndex) {
875 int FI = cast<FrameIndexSDNode>(N)->getIndex();
876 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
877 } else if (N.getOpcode() == ARMISD::Wrapper &&
878 !(Subtarget->useMovt() &&
879 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
880 Base = N.getOperand(0);
882 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
887 // If the RHS is +/- imm8, fold into addr mode.
889 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4,
890 -256 + 1, 256, RHSC)) {
891 Base = N.getOperand(0);
892 if (Base.getOpcode() == ISD::FrameIndex) {
893 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
894 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
897 ARM_AM::AddrOpc AddSub = ARM_AM::add;
899 AddSub = ARM_AM::sub;
902 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
908 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
913 bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
917 unsigned Alignment = 0;
918 if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) {
919 // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
920 // The maximum alignment is equal to the memory size being referenced.
921 unsigned LSNAlign = LSN->getAlignment();
922 unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8;
923 if (LSNAlign >= MemSize && MemSize > 1)
926 // All other uses of addrmode6 are for intrinsics. For now just record
927 // the raw alignment value; it will be refined later based on the legal
928 // alignment operands for the intrinsic.
929 Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment();
932 Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
936 bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
938 LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
939 ISD::MemIndexedMode AM = LdSt->getAddressingMode();
940 if (AM != ISD::POST_INC)
943 if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
944 if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
945 Offset = CurDAG->getRegister(0, MVT::i32);
950 bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
951 SDValue &Offset, SDValue &Label) {
952 if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
953 Offset = N.getOperand(0);
954 SDValue N1 = N.getOperand(1);
955 Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
964 //===----------------------------------------------------------------------===//
965 // Thumb Addressing Modes
966 //===----------------------------------------------------------------------===//
968 bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
969 SDValue &Base, SDValue &Offset){
970 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
971 ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
972 if (!NC || !NC->isNullValue())
979 Base = N.getOperand(0);
980 Offset = N.getOperand(1);
985 ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base,
986 SDValue &Offset, unsigned Scale) {
988 SDValue TmpBase, TmpOffImm;
989 if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
990 return false; // We want to select tLDRspi / tSTRspi instead.
992 if (N.getOpcode() == ARMISD::Wrapper &&
993 N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
994 return false; // We want to select tLDRpci instead.
997 if (!CurDAG->isBaseWithConstantOffset(N))
1000 // Thumb does not have [sp, r] address mode.
1001 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1002 RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1003 if ((LHSR && LHSR->getReg() == ARM::SP) ||
1004 (RHSR && RHSR->getReg() == ARM::SP))
1007 // FIXME: Why do we explicitly check for a match here and then return false?
1008 // Presumably to allow something else to match, but shouldn't this be
1011 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC))
1014 Base = N.getOperand(0);
1015 Offset = N.getOperand(1);
1020 ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N,
1023 return SelectThumbAddrModeRI(N, Base, Offset, 1);
1027 ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N,
1030 return SelectThumbAddrModeRI(N, Base, Offset, 2);
1034 ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N,
1037 return SelectThumbAddrModeRI(N, Base, Offset, 4);
1041 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1042 SDValue &Base, SDValue &OffImm) {
1044 SDValue TmpBase, TmpOffImm;
1045 if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
1046 return false; // We want to select tLDRspi / tSTRspi instead.
1048 if (N.getOpcode() == ARMISD::Wrapper &&
1049 N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
1050 return false; // We want to select tLDRpci instead.
1053 if (!CurDAG->isBaseWithConstantOffset(N)) {
1054 if (N.getOpcode() == ARMISD::Wrapper &&
1055 !(Subtarget->useMovt() &&
1056 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1057 Base = N.getOperand(0);
1062 OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1066 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1067 RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
1068 if ((LHSR && LHSR->getReg() == ARM::SP) ||
1069 (RHSR && RHSR->getReg() == ARM::SP)) {
1070 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0));
1071 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1072 unsigned LHSC = LHS ? LHS->getZExtValue() : 0;
1073 unsigned RHSC = RHS ? RHS->getZExtValue() : 0;
1075 // Thumb does not have [sp, #imm5] address mode for non-zero imm5.
1076 if (LHSC != 0 || RHSC != 0) return false;
1079 OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1083 // If the RHS is + imm5 * scale, fold into addr mode.
1085 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1086 Base = N.getOperand(0);
1087 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1091 Base = N.getOperand(0);
1092 OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1097 ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1099 return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1103 ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1105 return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1109 ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1111 return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1114 bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1115 SDValue &Base, SDValue &OffImm) {
1116 if (N.getOpcode() == ISD::FrameIndex) {
1117 int FI = cast<FrameIndexSDNode>(N)->getIndex();
1118 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1119 OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1123 if (!CurDAG->isBaseWithConstantOffset(N))
1126 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1127 if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1128 (LHSR && LHSR->getReg() == ARM::SP)) {
1129 // If the RHS is + imm8 * scale, fold into addr mode.
1131 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1132 Base = N.getOperand(0);
1133 if (Base.getOpcode() == ISD::FrameIndex) {
1134 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1135 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1137 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1146 //===----------------------------------------------------------------------===//
1147 // Thumb 2 Addressing Modes
1148 //===----------------------------------------------------------------------===//
1151 bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
1153 if (DisableShifterOp)
1156 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
1158 // Don't match base register only case. That is matched to a separate
1159 // lower complexity pattern with explicit register operand.
1160 if (ShOpcVal == ARM_AM::no_shift) return false;
1162 BaseReg = N.getOperand(0);
1163 unsigned ShImmVal = 0;
1164 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1165 ShImmVal = RHS->getZExtValue() & 31;
1166 Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
1173 bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1174 SDValue &Base, SDValue &OffImm) {
1175 // Match simple R + imm12 operands.
1178 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1179 !CurDAG->isBaseWithConstantOffset(N)) {
1180 if (N.getOpcode() == ISD::FrameIndex) {
1181 // Match frame index.
1182 int FI = cast<FrameIndexSDNode>(N)->getIndex();
1183 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1184 OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1188 if (N.getOpcode() == ARMISD::Wrapper &&
1189 !(Subtarget->useMovt() &&
1190 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1191 Base = N.getOperand(0);
1192 if (Base.getOpcode() == ISD::TargetConstantPool)
1193 return false; // We want to select t2LDRpci instead.
1196 OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1200 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1201 if (SelectT2AddrModeImm8(N, Base, OffImm))
1202 // Let t2LDRi8 handle (R - imm8).
1205 int RHSC = (int)RHS->getZExtValue();
1206 if (N.getOpcode() == ISD::SUB)
1209 if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1210 Base = N.getOperand(0);
1211 if (Base.getOpcode() == ISD::FrameIndex) {
1212 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1213 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1215 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1222 OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1226 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1227 SDValue &Base, SDValue &OffImm) {
1228 // Match simple R - imm8 operands.
1229 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1230 !CurDAG->isBaseWithConstantOffset(N))
1233 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1234 int RHSC = (int)RHS->getSExtValue();
1235 if (N.getOpcode() == ISD::SUB)
1238 if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1239 Base = N.getOperand(0);
1240 if (Base.getOpcode() == ISD::FrameIndex) {
1241 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1242 Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1244 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1252 bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1254 unsigned Opcode = Op->getOpcode();
1255 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1256 ? cast<LoadSDNode>(Op)->getAddressingMode()
1257 : cast<StoreSDNode>(Op)->getAddressingMode();
1259 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1260 OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1261 ? CurDAG->getTargetConstant(RHSC, MVT::i32)
1262 : CurDAG->getTargetConstant(-RHSC, MVT::i32);
1269 bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1271 SDValue &OffReg, SDValue &ShImm) {
1272 // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1273 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1276 // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1277 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1278 int RHSC = (int)RHS->getZExtValue();
1279 if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1281 else if (RHSC < 0 && RHSC >= -255) // 8 bits
1285 // Look for (R + R) or (R + (R << [1,2,3])).
1287 Base = N.getOperand(0);
1288 OffReg = N.getOperand(1);
1290 // Swap if it is ((R << c) + R).
1291 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
1292 if (ShOpcVal != ARM_AM::lsl) {
1293 ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1294 if (ShOpcVal == ARM_AM::lsl)
1295 std::swap(Base, OffReg);
1298 if (ShOpcVal == ARM_AM::lsl) {
1299 // Check to see if the RHS of the shift is a constant, if not, we can't fold
1301 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1302 ShAmt = Sh->getZExtValue();
1303 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1304 OffReg = OffReg.getOperand(0);
1307 ShOpcVal = ARM_AM::no_shift;
1310 ShOpcVal = ARM_AM::no_shift;
1314 ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
1319 //===--------------------------------------------------------------------===//
1321 /// getAL - Returns a ARMCC::AL immediate node.
1322 static inline SDValue getAL(SelectionDAG *CurDAG) {
1323 return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
1326 SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
1327 LoadSDNode *LD = cast<LoadSDNode>(N);
1328 ISD::MemIndexedMode AM = LD->getAddressingMode();
1329 if (AM == ISD::UNINDEXED)
1332 EVT LoadedVT = LD->getMemoryVT();
1333 SDValue Offset, AMOpc;
1334 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1335 unsigned Opcode = 0;
1337 if (LoadedVT == MVT::i32 && isPre &&
1338 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1339 Opcode = ARM::LDR_PRE_IMM;
1341 } else if (LoadedVT == MVT::i32 && !isPre &&
1342 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1343 Opcode = ARM::LDR_POST_IMM;
1345 } else if (LoadedVT == MVT::i32 &&
1346 SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1347 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1350 } else if (LoadedVT == MVT::i16 &&
1351 SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1353 Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1354 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1355 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1356 } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1357 if (LD->getExtensionType() == ISD::SEXTLOAD) {
1358 if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1360 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1364 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1366 Opcode = ARM::LDRB_PRE_IMM;
1367 } else if (!isPre &&
1368 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1370 Opcode = ARM::LDRB_POST_IMM;
1371 } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1373 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1379 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1380 SDValue Chain = LD->getChain();
1381 SDValue Base = LD->getBasePtr();
1382 SDValue Ops[]= { Base, AMOpc, getAL(CurDAG),
1383 CurDAG->getRegister(0, MVT::i32), Chain };
1384 return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32,
1385 MVT::i32, MVT::Other, Ops, 5);
1387 SDValue Chain = LD->getChain();
1388 SDValue Base = LD->getBasePtr();
1389 SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
1390 CurDAG->getRegister(0, MVT::i32), Chain };
1391 return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32,
1392 MVT::i32, MVT::Other, Ops, 6);
1399 SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
1400 LoadSDNode *LD = cast<LoadSDNode>(N);
1401 ISD::MemIndexedMode AM = LD->getAddressingMode();
1402 if (AM == ISD::UNINDEXED)
1405 EVT LoadedVT = LD->getMemoryVT();
1406 bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1408 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1409 unsigned Opcode = 0;
1411 if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1412 switch (LoadedVT.getSimpleVT().SimpleTy) {
1414 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1418 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1420 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1425 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1427 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1436 SDValue Chain = LD->getChain();
1437 SDValue Base = LD->getBasePtr();
1438 SDValue Ops[]= { Base, Offset, getAL(CurDAG),
1439 CurDAG->getRegister(0, MVT::i32), Chain };
1440 return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
1441 MVT::Other, Ops, 5);
1447 /// PairSRegs - Form a D register from a pair of S registers.
1449 SDNode *ARMDAGToDAGISel::PairSRegs(EVT VT, SDValue V0, SDValue V1) {
1450 DebugLoc dl = V0.getNode()->getDebugLoc();
1452 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32);
1453 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1454 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1455 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1456 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1459 /// PairDRegs - Form a quad register from a pair of D registers.
1461 SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
1462 DebugLoc dl = V0.getNode()->getDebugLoc();
1463 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32);
1464 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1465 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1466 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1467 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1470 /// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
1472 SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
1473 DebugLoc dl = V0.getNode()->getDebugLoc();
1474 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1475 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1476 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1477 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1478 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 5);
1481 /// QuadSRegs - Form 4 consecutive S registers.
1483 SDNode *ARMDAGToDAGISel::QuadSRegs(EVT VT, SDValue V0, SDValue V1,
1484 SDValue V2, SDValue V3) {
1485 DebugLoc dl = V0.getNode()->getDebugLoc();
1487 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32);
1488 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1489 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1490 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
1491 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
1492 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1493 V2, SubReg2, V3, SubReg3 };
1494 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1497 /// QuadDRegs - Form 4 consecutive D registers.
1499 SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
1500 SDValue V2, SDValue V3) {
1501 DebugLoc dl = V0.getNode()->getDebugLoc();
1502 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32);
1503 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1504 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1505 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
1506 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
1507 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1508 V2, SubReg2, V3, SubReg3 };
1509 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1512 /// QuadQRegs - Form 4 consecutive Q registers.
1514 SDNode *ARMDAGToDAGISel::QuadQRegs(EVT VT, SDValue V0, SDValue V1,
1515 SDValue V2, SDValue V3) {
1516 DebugLoc dl = V0.getNode()->getDebugLoc();
1517 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32);
1518 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1519 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1520 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1521 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1522 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1523 V2, SubReg2, V3, SubReg3 };
1524 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 9);
1527 /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1528 /// of a NEON VLD or VST instruction. The supported values depend on the
1529 /// number of registers being loaded.
1530 SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
1531 bool is64BitVector) {
1532 unsigned NumRegs = NumVecs;
1533 if (!is64BitVector && NumVecs < 3)
1536 unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1537 if (Alignment >= 32 && NumRegs == 4)
1539 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1541 else if (Alignment >= 8)
1546 return CurDAG->getTargetConstant(Alignment, MVT::i32);
1549 // Get the register stride update opcode of a VLD/VST instruction that
1550 // is otherwise equivalent to the given fixed stride updating instruction.
1551 static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1554 case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1555 case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1556 case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1557 case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1558 case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1559 case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1560 case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1561 case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1563 case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1564 case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1565 case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1566 case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1567 case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1568 case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1569 case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1570 case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1571 case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1572 case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1574 case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1575 case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1576 case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1577 case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1578 case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1579 case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1581 case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1582 case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1583 case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1584 case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1585 case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1586 case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1588 case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1589 case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1590 case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1592 return Opc; // If not one we handle, return it unchanged.
1595 SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1596 const uint16_t *DOpcodes,
1597 const uint16_t *QOpcodes0,
1598 const uint16_t *QOpcodes1) {
1599 assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1600 DebugLoc dl = N->getDebugLoc();
1602 SDValue MemAddr, Align;
1603 unsigned AddrOpIdx = isUpdating ? 1 : 2;
1604 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1607 SDValue Chain = N->getOperand(0);
1608 EVT VT = N->getValueType(0);
1609 bool is64BitVector = VT.is64BitVector();
1610 Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1612 unsigned OpcodeIndex;
1613 switch (VT.getSimpleVT().SimpleTy) {
1614 default: llvm_unreachable("unhandled vld type");
1615 // Double-register operations:
1616 case MVT::v8i8: OpcodeIndex = 0; break;
1617 case MVT::v4i16: OpcodeIndex = 1; break;
1619 case MVT::v2i32: OpcodeIndex = 2; break;
1620 case MVT::v1i64: OpcodeIndex = 3; break;
1621 // Quad-register operations:
1622 case MVT::v16i8: OpcodeIndex = 0; break;
1623 case MVT::v8i16: OpcodeIndex = 1; break;
1625 case MVT::v4i32: OpcodeIndex = 2; break;
1626 case MVT::v2i64: OpcodeIndex = 3;
1627 assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1635 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1638 ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1640 std::vector<EVT> ResTys;
1641 ResTys.push_back(ResTy);
1643 ResTys.push_back(MVT::i32);
1644 ResTys.push_back(MVT::Other);
1646 SDValue Pred = getAL(CurDAG);
1647 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1649 SmallVector<SDValue, 7> Ops;
1651 // Double registers and VLD1/VLD2 quad registers are directly supported.
1652 if (is64BitVector || NumVecs <= 2) {
1653 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1654 QOpcodes0[OpcodeIndex]);
1655 Ops.push_back(MemAddr);
1656 Ops.push_back(Align);
1658 SDValue Inc = N->getOperand(AddrOpIdx + 1);
1659 // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0
1660 // case entirely when the rest are updated to that form, too.
1661 if ((NumVecs == 1 || NumVecs == 2) && !isa<ConstantSDNode>(Inc.getNode()))
1662 Opc = getVLDSTRegisterUpdateOpcode(Opc);
1663 // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1664 // check for that explicitly too. Horribly hacky, but temporary.
1665 if ((NumVecs != 1 && NumVecs != 2 && Opc != ARM::VLD1q64wb_fixed) ||
1666 !isa<ConstantSDNode>(Inc.getNode()))
1667 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1669 Ops.push_back(Pred);
1670 Ops.push_back(Reg0);
1671 Ops.push_back(Chain);
1672 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1675 // Otherwise, quad registers are loaded with two separate instructions,
1676 // where one loads the even registers and the other loads the odd registers.
1677 EVT AddrTy = MemAddr.getValueType();
1679 // Load the even subregs. This is always an updating load, so that it
1680 // provides the address to the second load for the odd subregs.
1682 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1683 const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1684 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1685 ResTy, AddrTy, MVT::Other, OpsA, 7);
1686 Chain = SDValue(VLdA, 2);
1688 // Load the odd subregs.
1689 Ops.push_back(SDValue(VLdA, 1));
1690 Ops.push_back(Align);
1692 SDValue Inc = N->getOperand(AddrOpIdx + 1);
1693 assert(isa<ConstantSDNode>(Inc.getNode()) &&
1694 "only constant post-increment update allowed for VLD3/4");
1696 Ops.push_back(Reg0);
1698 Ops.push_back(SDValue(VLdA, 0));
1699 Ops.push_back(Pred);
1700 Ops.push_back(Reg0);
1701 Ops.push_back(Chain);
1702 VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
1703 Ops.data(), Ops.size());
1706 // Transfer memoperands.
1707 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1708 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1709 cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
1714 // Extract out the subregisters.
1715 SDValue SuperReg = SDValue(VLd, 0);
1716 assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1717 ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1718 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1719 for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1720 ReplaceUses(SDValue(N, Vec),
1721 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1722 ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1724 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1728 SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1729 const uint16_t *DOpcodes,
1730 const uint16_t *QOpcodes0,
1731 const uint16_t *QOpcodes1) {
1732 assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1733 DebugLoc dl = N->getDebugLoc();
1735 SDValue MemAddr, Align;
1736 unsigned AddrOpIdx = isUpdating ? 1 : 2;
1737 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1738 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1741 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1742 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1744 SDValue Chain = N->getOperand(0);
1745 EVT VT = N->getOperand(Vec0Idx).getValueType();
1746 bool is64BitVector = VT.is64BitVector();
1747 Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1749 unsigned OpcodeIndex;
1750 switch (VT.getSimpleVT().SimpleTy) {
1751 default: llvm_unreachable("unhandled vst type");
1752 // Double-register operations:
1753 case MVT::v8i8: OpcodeIndex = 0; break;
1754 case MVT::v4i16: OpcodeIndex = 1; break;
1756 case MVT::v2i32: OpcodeIndex = 2; break;
1757 case MVT::v1i64: OpcodeIndex = 3; break;
1758 // Quad-register operations:
1759 case MVT::v16i8: OpcodeIndex = 0; break;
1760 case MVT::v8i16: OpcodeIndex = 1; break;
1762 case MVT::v4i32: OpcodeIndex = 2; break;
1763 case MVT::v2i64: OpcodeIndex = 3;
1764 assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1768 std::vector<EVT> ResTys;
1770 ResTys.push_back(MVT::i32);
1771 ResTys.push_back(MVT::Other);
1773 SDValue Pred = getAL(CurDAG);
1774 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1775 SmallVector<SDValue, 7> Ops;
1777 // Double registers and VST1/VST2 quad registers are directly supported.
1778 if (is64BitVector || NumVecs <= 2) {
1781 SrcReg = N->getOperand(Vec0Idx);
1782 } else if (is64BitVector) {
1783 // Form a REG_SEQUENCE to force register allocation.
1784 SDValue V0 = N->getOperand(Vec0Idx + 0);
1785 SDValue V1 = N->getOperand(Vec0Idx + 1);
1787 SrcReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1789 SDValue V2 = N->getOperand(Vec0Idx + 2);
1790 // If it's a vst3, form a quad D-register and leave the last part as
1792 SDValue V3 = (NumVecs == 3)
1793 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1794 : N->getOperand(Vec0Idx + 3);
1795 SrcReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1798 // Form a QQ register.
1799 SDValue Q0 = N->getOperand(Vec0Idx);
1800 SDValue Q1 = N->getOperand(Vec0Idx + 1);
1801 SrcReg = SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0);
1804 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1805 QOpcodes0[OpcodeIndex]);
1806 Ops.push_back(MemAddr);
1807 Ops.push_back(Align);
1809 SDValue Inc = N->getOperand(AddrOpIdx + 1);
1810 // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0
1811 // case entirely when the rest are updated to that form, too.
1812 if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode()))
1813 Opc = getVLDSTRegisterUpdateOpcode(Opc);
1814 // We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so
1815 // check for that explicitly too. Horribly hacky, but temporary.
1816 if ((NumVecs > 2 && Opc != ARM::VST1q64wb_fixed) ||
1817 !isa<ConstantSDNode>(Inc.getNode()))
1818 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1820 Ops.push_back(SrcReg);
1821 Ops.push_back(Pred);
1822 Ops.push_back(Reg0);
1823 Ops.push_back(Chain);
1825 CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
1827 // Transfer memoperands.
1828 cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
1833 // Otherwise, quad registers are stored with two separate instructions,
1834 // where one stores the even registers and the other stores the odd registers.
1836 // Form the QQQQ REG_SEQUENCE.
1837 SDValue V0 = N->getOperand(Vec0Idx + 0);
1838 SDValue V1 = N->getOperand(Vec0Idx + 1);
1839 SDValue V2 = N->getOperand(Vec0Idx + 2);
1840 SDValue V3 = (NumVecs == 3)
1841 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1842 : N->getOperand(Vec0Idx + 3);
1843 SDValue RegSeq = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1845 // Store the even D registers. This is always an updating store, so that it
1846 // provides the address to the second store for the odd subregs.
1847 const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
1848 SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1849 MemAddr.getValueType(),
1850 MVT::Other, OpsA, 7);
1851 cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1);
1852 Chain = SDValue(VStA, 1);
1854 // Store the odd D registers.
1855 Ops.push_back(SDValue(VStA, 0));
1856 Ops.push_back(Align);
1858 SDValue Inc = N->getOperand(AddrOpIdx + 1);
1859 assert(isa<ConstantSDNode>(Inc.getNode()) &&
1860 "only constant post-increment update allowed for VST3/4");
1862 Ops.push_back(Reg0);
1864 Ops.push_back(RegSeq);
1865 Ops.push_back(Pred);
1866 Ops.push_back(Reg0);
1867 Ops.push_back(Chain);
1868 SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
1869 Ops.data(), Ops.size());
1870 cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1);
1874 SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1875 bool isUpdating, unsigned NumVecs,
1876 const uint16_t *DOpcodes,
1877 const uint16_t *QOpcodes) {
1878 assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1879 DebugLoc dl = N->getDebugLoc();
1881 SDValue MemAddr, Align;
1882 unsigned AddrOpIdx = isUpdating ? 1 : 2;
1883 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1884 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1887 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1888 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1890 SDValue Chain = N->getOperand(0);
1892 cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
1893 EVT VT = N->getOperand(Vec0Idx).getValueType();
1894 bool is64BitVector = VT.is64BitVector();
1896 unsigned Alignment = 0;
1898 Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1899 unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
1900 if (Alignment > NumBytes)
1901 Alignment = NumBytes;
1902 if (Alignment < 8 && Alignment < NumBytes)
1904 // Alignment must be a power of two; make sure of that.
1905 Alignment = (Alignment & -Alignment);
1909 Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1911 unsigned OpcodeIndex;
1912 switch (VT.getSimpleVT().SimpleTy) {
1913 default: llvm_unreachable("unhandled vld/vst lane type");
1914 // Double-register operations:
1915 case MVT::v8i8: OpcodeIndex = 0; break;
1916 case MVT::v4i16: OpcodeIndex = 1; break;
1918 case MVT::v2i32: OpcodeIndex = 2; break;
1919 // Quad-register operations:
1920 case MVT::v8i16: OpcodeIndex = 0; break;
1922 case MVT::v4i32: OpcodeIndex = 1; break;
1925 std::vector<EVT> ResTys;
1927 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1930 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
1931 MVT::i64, ResTyElts));
1934 ResTys.push_back(MVT::i32);
1935 ResTys.push_back(MVT::Other);
1937 SDValue Pred = getAL(CurDAG);
1938 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1940 SmallVector<SDValue, 8> Ops;
1941 Ops.push_back(MemAddr);
1942 Ops.push_back(Align);
1944 SDValue Inc = N->getOperand(AddrOpIdx + 1);
1945 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc);
1949 SDValue V0 = N->getOperand(Vec0Idx + 0);
1950 SDValue V1 = N->getOperand(Vec0Idx + 1);
1953 SuperReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1955 SuperReg = SDValue(PairQRegs(MVT::v4i64, V0, V1), 0);
1957 SDValue V2 = N->getOperand(Vec0Idx + 2);
1958 SDValue V3 = (NumVecs == 3)
1959 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1960 : N->getOperand(Vec0Idx + 3);
1962 SuperReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1964 SuperReg = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1966 Ops.push_back(SuperReg);
1967 Ops.push_back(getI32Imm(Lane));
1968 Ops.push_back(Pred);
1969 Ops.push_back(Reg0);
1970 Ops.push_back(Chain);
1972 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1973 QOpcodes[OpcodeIndex]);
1974 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys,
1975 Ops.data(), Ops.size());
1976 cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1);
1980 // Extract the subregisters.
1981 SuperReg = SDValue(VLdLn, 0);
1982 assert(ARM::dsub_7 == ARM::dsub_0+7 &&
1983 ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1984 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
1985 for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1986 ReplaceUses(SDValue(N, Vec),
1987 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1988 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
1990 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
1994 SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating,
1996 const uint16_t *Opcodes) {
1997 assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
1998 DebugLoc dl = N->getDebugLoc();
2000 SDValue MemAddr, Align;
2001 if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
2004 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2005 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2007 SDValue Chain = N->getOperand(0);
2008 EVT VT = N->getValueType(0);
2010 unsigned Alignment = 0;
2012 Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2013 unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
2014 if (Alignment > NumBytes)
2015 Alignment = NumBytes;
2016 if (Alignment < 8 && Alignment < NumBytes)
2018 // Alignment must be a power of two; make sure of that.
2019 Alignment = (Alignment & -Alignment);
2023 Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
2025 unsigned OpcodeIndex;
2026 switch (VT.getSimpleVT().SimpleTy) {
2027 default: llvm_unreachable("unhandled vld-dup type");
2028 case MVT::v8i8: OpcodeIndex = 0; break;
2029 case MVT::v4i16: OpcodeIndex = 1; break;
2031 case MVT::v2i32: OpcodeIndex = 2; break;
2034 SDValue Pred = getAL(CurDAG);
2035 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2037 unsigned Opc = Opcodes[OpcodeIndex];
2038 SmallVector<SDValue, 6> Ops;
2039 Ops.push_back(MemAddr);
2040 Ops.push_back(Align);
2042 // fixed-stride update instructions don't have an explicit writeback
2043 // operand. It's implicit in the opcode itself.
2044 SDValue Inc = N->getOperand(2);
2045 if (!isa<ConstantSDNode>(Inc.getNode()))
2047 // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2048 else if (NumVecs > 2)
2049 Ops.push_back(Reg0);
2051 Ops.push_back(Pred);
2052 Ops.push_back(Reg0);
2053 Ops.push_back(Chain);
2055 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2056 std::vector<EVT> ResTys;
2057 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
2059 ResTys.push_back(MVT::i32);
2060 ResTys.push_back(MVT::Other);
2062 CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), Ops.size());
2063 cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
2064 SuperReg = SDValue(VLdDup, 0);
2066 // Extract the subregisters.
2067 assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
2068 unsigned SubIdx = ARM::dsub_0;
2069 for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2070 ReplaceUses(SDValue(N, Vec),
2071 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2072 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2074 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2078 SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
2080 assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
2081 DebugLoc dl = N->getDebugLoc();
2082 EVT VT = N->getValueType(0);
2083 unsigned FirstTblReg = IsExt ? 2 : 1;
2085 // Form a REG_SEQUENCE to force register allocation.
2087 SDValue V0 = N->getOperand(FirstTblReg + 0);
2088 SDValue V1 = N->getOperand(FirstTblReg + 1);
2090 RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
2092 SDValue V2 = N->getOperand(FirstTblReg + 2);
2093 // If it's a vtbl3, form a quad D-register and leave the last part as
2095 SDValue V3 = (NumVecs == 3)
2096 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2097 : N->getOperand(FirstTblReg + 3);
2098 RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
2101 SmallVector<SDValue, 6> Ops;
2103 Ops.push_back(N->getOperand(1));
2104 Ops.push_back(RegSeq);
2105 Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
2106 Ops.push_back(getAL(CurDAG)); // predicate
2107 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
2108 return CurDAG->getMachineNode(Opc, dl, VT, Ops.data(), Ops.size());
2111 SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
2113 if (!Subtarget->hasV6T2Ops())
2116 unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2117 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2120 // For unsigned extracts, check for a shift right and mask
2121 unsigned And_imm = 0;
2122 if (N->getOpcode() == ISD::AND) {
2123 if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2125 // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2126 if (And_imm & (And_imm + 1))
2129 unsigned Srl_imm = 0;
2130 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
2132 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2134 // Note: The width operand is encoded as width-1.
2135 unsigned Width = CountTrailingOnes_32(And_imm) - 1;
2136 unsigned LSB = Srl_imm;
2137 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2138 SDValue Ops[] = { N->getOperand(0).getOperand(0),
2139 CurDAG->getTargetConstant(LSB, MVT::i32),
2140 CurDAG->getTargetConstant(Width, MVT::i32),
2141 getAL(CurDAG), Reg0 };
2142 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2148 // Otherwise, we're looking for a shift of a shift
2149 unsigned Shl_imm = 0;
2150 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2151 assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2152 unsigned Srl_imm = 0;
2153 if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2154 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2155 // Note: The width operand is encoded as width-1.
2156 unsigned Width = 32 - Srl_imm - 1;
2157 int LSB = Srl_imm - Shl_imm;
2160 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2161 SDValue Ops[] = { N->getOperand(0).getOperand(0),
2162 CurDAG->getTargetConstant(LSB, MVT::i32),
2163 CurDAG->getTargetConstant(Width, MVT::i32),
2164 getAL(CurDAG), Reg0 };
2165 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2171 SDNode *ARMDAGToDAGISel::
2172 SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2173 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2176 if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
2177 unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
2178 unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
2181 case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
2182 case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
2183 case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
2184 case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
2186 llvm_unreachable("Unknown so_reg opcode!");
2189 CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
2190 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2191 SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
2192 return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
2197 SDNode *ARMDAGToDAGISel::
2198 SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2199 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2203 if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) {
2204 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2205 SDValue Ops[] = { FalseVal, CPTmp0, CPTmp2, CC, CCR, InFlag };
2206 return CurDAG->SelectNodeTo(N, ARM::MOVCCsi, MVT::i32, Ops, 6);
2209 if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
2210 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2211 SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
2212 return CurDAG->SelectNodeTo(N, ARM::MOVCCsr, MVT::i32, Ops, 7);
2217 SDNode *ARMDAGToDAGISel::
2218 SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2219 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2220 ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2225 unsigned TrueImm = T->getZExtValue();
2226 if (is_t2_so_imm(TrueImm)) {
2227 Opc = ARM::t2MOVCCi;
2228 } else if (TrueImm <= 0xffff) {
2229 Opc = ARM::t2MOVCCi16;
2230 } else if (is_t2_so_imm_not(TrueImm)) {
2232 Opc = ARM::t2MVNCCi;
2233 } else if (TrueVal.getNode()->hasOneUse() && Subtarget->hasV6T2Ops()) {
2235 Opc = ARM::t2MOVCCi32imm;
2239 SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2240 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2241 SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2242 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2248 SDNode *ARMDAGToDAGISel::
2249 SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2250 ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2251 ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2256 unsigned TrueImm = T->getZExtValue();
2257 bool isSoImm = is_so_imm(TrueImm);
2260 } else if (Subtarget->hasV6T2Ops() && TrueImm <= 0xffff) {
2261 Opc = ARM::MOVCCi16;
2262 } else if (is_so_imm_not(TrueImm)) {
2265 } else if (TrueVal.getNode()->hasOneUse() &&
2266 (Subtarget->hasV6T2Ops() || ARM_AM::isSOImmTwoPartVal(TrueImm))) {
2268 Opc = ARM::MOVCCi32imm;
2272 SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2273 SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2274 SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2275 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2281 SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
2282 EVT VT = N->getValueType(0);
2283 SDValue FalseVal = N->getOperand(0);
2284 SDValue TrueVal = N->getOperand(1);
2285 SDValue CC = N->getOperand(2);
2286 SDValue CCR = N->getOperand(3);
2287 SDValue InFlag = N->getOperand(4);
2288 assert(CC.getOpcode() == ISD::Constant);
2289 assert(CCR.getOpcode() == ISD::Register);
2290 ARMCC::CondCodes CCVal =
2291 (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
2293 if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
2294 // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2295 // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2296 // Pattern complexity = 18 cost = 1 size = 0
2297 if (Subtarget->isThumb()) {
2298 SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
2299 CCVal, CCR, InFlag);
2301 Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
2302 ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2306 SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
2307 CCVal, CCR, InFlag);
2309 Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
2310 ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2315 // Pattern: (ARMcmov:i32 GPR:i32:$false,
2316 // (imm:i32)<<P:Pred_so_imm>>:$true,
2318 // Emits: (MOVCCi:i32 GPR:i32:$false,
2319 // (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
2320 // Pattern complexity = 10 cost = 1 size = 0
2321 if (Subtarget->isThumb()) {
2322 SDNode *Res = SelectT2CMOVImmOp(N, FalseVal, TrueVal,
2323 CCVal, CCR, InFlag);
2325 Res = SelectT2CMOVImmOp(N, TrueVal, FalseVal,
2326 ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2330 SDNode *Res = SelectARMCMOVImmOp(N, FalseVal, TrueVal,
2331 CCVal, CCR, InFlag);
2333 Res = SelectARMCMOVImmOp(N, TrueVal, FalseVal,
2334 ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2340 // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2341 // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2342 // Pattern complexity = 6 cost = 1 size = 0
2344 // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2345 // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2346 // Pattern complexity = 6 cost = 11 size = 0
2348 // Also VMOVScc and VMOVDcc.
2349 SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
2350 SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
2352 switch (VT.getSimpleVT().SimpleTy) {
2353 default: llvm_unreachable("Illegal conditional move type!");
2355 Opc = Subtarget->isThumb()
2356 ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
2366 return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
2369 /// Target-specific DAG combining for ISD::XOR.
2370 /// Target-independent combining lowers SELECT_CC nodes of the form
2371 /// select_cc setg[ge] X, 0, X, -X
2372 /// select_cc setgt X, -1, X, -X
2373 /// select_cc setl[te] X, 0, -X, X
2374 /// select_cc setlt X, 1, -X, X
2375 /// which represent Integer ABS into:
2376 /// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2377 /// ARM instruction selection detects the latter and matches it to
2378 /// ARM::ABS or ARM::t2ABS machine node.
2379 SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){
2380 SDValue XORSrc0 = N->getOperand(0);
2381 SDValue XORSrc1 = N->getOperand(1);
2382 EVT VT = N->getValueType(0);
2384 if (Subtarget->isThumb1Only())
2387 if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
2390 SDValue ADDSrc0 = XORSrc0.getOperand(0);
2391 SDValue ADDSrc1 = XORSrc0.getOperand(1);
2392 SDValue SRASrc0 = XORSrc1.getOperand(0);
2393 SDValue SRASrc1 = XORSrc1.getOperand(1);
2394 ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1);
2395 EVT XType = SRASrc0.getValueType();
2396 unsigned Size = XType.getSizeInBits() - 1;
2398 if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2399 XType.isInteger() && SRAConstant != NULL &&
2400 Size == SRAConstant->getZExtValue()) {
2401 unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
2402 return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2408 SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
2409 // The only time a CONCAT_VECTORS operation can have legal types is when
2410 // two 64-bit vectors are concatenated to a 128-bit vector.
2411 EVT VT = N->getValueType(0);
2412 if (!VT.is128BitVector() || N->getNumOperands() != 2)
2413 llvm_unreachable("unexpected CONCAT_VECTORS");
2414 return PairDRegs(VT, N->getOperand(0), N->getOperand(1));
2417 SDNode *ARMDAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) {
2418 SmallVector<SDValue, 6> Ops;
2419 Ops.push_back(Node->getOperand(1)); // Ptr
2420 Ops.push_back(Node->getOperand(2)); // Low part of Val1
2421 Ops.push_back(Node->getOperand(3)); // High part of Val1
2422 if (Opc == ARM::ATOMCMPXCHG6432) {
2423 Ops.push_back(Node->getOperand(4)); // Low part of Val2
2424 Ops.push_back(Node->getOperand(5)); // High part of Val2
2426 Ops.push_back(Node->getOperand(0)); // Chain
2427 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2428 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand();
2429 SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(),
2430 MVT::i32, MVT::i32, MVT::Other,
2431 Ops.data() ,Ops.size());
2432 cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1);
2436 SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
2437 DebugLoc dl = N->getDebugLoc();
2439 if (N->isMachineOpcode())
2440 return NULL; // Already selected.
2442 switch (N->getOpcode()) {
2445 // Select special operations if XOR node forms integer ABS pattern
2446 SDNode *ResNode = SelectABSOp(N);
2449 // Other cases are autogenerated.
2452 case ISD::Constant: {
2453 unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2455 if (Subtarget->hasThumb2())
2456 // Thumb2-aware targets have the MOVT instruction, so all immediates can
2457 // be done with MOV + MOVT, at worst.
2460 if (Subtarget->isThumb()) {
2461 UseCP = (Val > 255 && // MOV
2462 ~Val > 255 && // MOV + MVN
2463 !ARM_AM::isThumbImmShiftedVal(Val)); // MOV + LSL
2465 UseCP = (ARM_AM::getSOImmVal(Val) == -1 && // MOV
2466 ARM_AM::getSOImmVal(~Val) == -1 && // MVN
2467 !ARM_AM::isSOImmTwoPartVal(Val)); // two instrs.
2472 CurDAG->getTargetConstantPool(ConstantInt::get(
2473 Type::getInt32Ty(*CurDAG->getContext()), Val),
2474 TLI.getPointerTy());
2477 if (Subtarget->isThumb1Only()) {
2478 SDValue Pred = getAL(CurDAG);
2479 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2480 SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
2481 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2486 CurDAG->getTargetConstant(0, MVT::i32),
2488 CurDAG->getRegister(0, MVT::i32),
2489 CurDAG->getEntryNode()
2491 ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2494 ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
2498 // Other cases are autogenerated.
2501 case ISD::FrameIndex: {
2502 // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2503 int FI = cast<FrameIndexSDNode>(N)->getIndex();
2504 SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
2505 if (Subtarget->isThumb1Only()) {
2506 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2507 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2508 return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, Ops, 4);
2510 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2511 ARM::t2ADDri : ARM::ADDri);
2512 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2513 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2514 CurDAG->getRegister(0, MVT::i32) };
2515 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2519 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2523 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
2527 if (Subtarget->isThumb1Only())
2529 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2530 unsigned RHSV = C->getZExtValue();
2532 if (isPowerOf2_32(RHSV-1)) { // 2^n+1?
2533 unsigned ShImm = Log2_32(RHSV-1);
2536 SDValue V = N->getOperand(0);
2537 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2538 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2539 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2540 if (Subtarget->isThumb()) {
2541 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2542 return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
2544 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2545 return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops, 7);
2548 if (isPowerOf2_32(RHSV+1)) { // 2^n-1?
2549 unsigned ShImm = Log2_32(RHSV+1);
2552 SDValue V = N->getOperand(0);
2553 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2554 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2555 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2556 if (Subtarget->isThumb()) {
2557 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2558 return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6);
2560 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2561 return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops, 7);
2567 // Check for unsigned bitfield extract
2568 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2571 // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2572 // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2573 // are entirely contributed by c2 and lower 16-bits are entirely contributed
2574 // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2575 // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2576 EVT VT = N->getValueType(0);
2579 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2581 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2584 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2585 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2588 if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2589 SDValue N2 = N0.getOperand(1);
2590 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
2593 unsigned N1CVal = N1C->getZExtValue();
2594 unsigned N2CVal = N2C->getZExtValue();
2595 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2596 (N1CVal & 0xffffU) == 0xffffU &&
2597 (N2CVal & 0xffffU) == 0x0U) {
2598 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2600 SDValue Ops[] = { N0.getOperand(0), Imm16,
2601 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2602 return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
2607 case ARMISD::VMOVRRD:
2608 return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
2609 N->getOperand(0), getAL(CurDAG),
2610 CurDAG->getRegister(0, MVT::i32));
2611 case ISD::UMUL_LOHI: {
2612 if (Subtarget->isThumb1Only())
2614 if (Subtarget->isThumb()) {
2615 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2616 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2617 CurDAG->getRegister(0, MVT::i32) };
2618 return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32,Ops,4);
2620 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2621 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2622 CurDAG->getRegister(0, MVT::i32) };
2623 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2624 ARM::UMULL : ARM::UMULLv5,
2625 dl, MVT::i32, MVT::i32, Ops, 5);
2628 case ISD::SMUL_LOHI: {
2629 if (Subtarget->isThumb1Only())
2631 if (Subtarget->isThumb()) {
2632 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2633 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2634 return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32,Ops,4);
2636 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2637 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2638 CurDAG->getRegister(0, MVT::i32) };
2639 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2640 ARM::SMULL : ARM::SMULLv5,
2641 dl, MVT::i32, MVT::i32, Ops, 5);
2644 case ARMISD::UMLAL:{
2645 if (Subtarget->isThumb()) {
2646 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2647 N->getOperand(3), getAL(CurDAG),
2648 CurDAG->getRegister(0, MVT::i32)};
2649 return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops, 6);
2651 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2652 N->getOperand(3), getAL(CurDAG),
2653 CurDAG->getRegister(0, MVT::i32),
2654 CurDAG->getRegister(0, MVT::i32) };
2655 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2656 ARM::UMLAL : ARM::UMLALv5,
2657 dl, MVT::i32, MVT::i32, Ops, 7);
2660 case ARMISD::SMLAL:{
2661 if (Subtarget->isThumb()) {
2662 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2663 N->getOperand(3), getAL(CurDAG),
2664 CurDAG->getRegister(0, MVT::i32)};
2665 return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops, 6);
2667 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2668 N->getOperand(3), getAL(CurDAG),
2669 CurDAG->getRegister(0, MVT::i32),
2670 CurDAG->getRegister(0, MVT::i32) };
2671 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2672 ARM::SMLAL : ARM::SMLALv5,
2673 dl, MVT::i32, MVT::i32, Ops, 7);
2677 SDNode *ResNode = 0;
2678 if (Subtarget->isThumb() && Subtarget->hasThumb2())
2679 ResNode = SelectT2IndexedLoad(N);
2681 ResNode = SelectARMIndexedLoad(N);
2684 // Other cases are autogenerated.
2687 case ARMISD::BRCOND: {
2688 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2689 // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2690 // Pattern complexity = 6 cost = 1 size = 0
2692 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2693 // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2694 // Pattern complexity = 6 cost = 1 size = 0
2696 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2697 // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2698 // Pattern complexity = 6 cost = 1 size = 0
2700 unsigned Opc = Subtarget->isThumb() ?
2701 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2702 SDValue Chain = N->getOperand(0);
2703 SDValue N1 = N->getOperand(1);
2704 SDValue N2 = N->getOperand(2);
2705 SDValue N3 = N->getOperand(3);
2706 SDValue InFlag = N->getOperand(4);
2707 assert(N1.getOpcode() == ISD::BasicBlock);
2708 assert(N2.getOpcode() == ISD::Constant);
2709 assert(N3.getOpcode() == ISD::Register);
2711 SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2712 cast<ConstantSDNode>(N2)->getZExtValue()),
2714 SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2715 SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2717 Chain = SDValue(ResNode, 0);
2718 if (N->getNumValues() == 2) {
2719 InFlag = SDValue(ResNode, 1);
2720 ReplaceUses(SDValue(N, 1), InFlag);
2722 ReplaceUses(SDValue(N, 0),
2723 SDValue(Chain.getNode(), Chain.getResNo()));
2727 return SelectCMOVOp(N);
2728 case ARMISD::VZIP: {
2730 EVT VT = N->getValueType(0);
2731 switch (VT.getSimpleVT().SimpleTy) {
2732 default: return NULL;
2733 case MVT::v8i8: Opc = ARM::VZIPd8; break;
2734 case MVT::v4i16: Opc = ARM::VZIPd16; break;
2736 // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2737 case MVT::v2i32: Opc = ARM::VTRNd32; break;
2738 case MVT::v16i8: Opc = ARM::VZIPq8; break;
2739 case MVT::v8i16: Opc = ARM::VZIPq16; break;
2741 case MVT::v4i32: Opc = ARM::VZIPq32; break;
2743 SDValue Pred = getAL(CurDAG);
2744 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2745 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2746 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2748 case ARMISD::VUZP: {
2750 EVT VT = N->getValueType(0);
2751 switch (VT.getSimpleVT().SimpleTy) {
2752 default: return NULL;
2753 case MVT::v8i8: Opc = ARM::VUZPd8; break;
2754 case MVT::v4i16: Opc = ARM::VUZPd16; break;
2756 // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2757 case MVT::v2i32: Opc = ARM::VTRNd32; break;
2758 case MVT::v16i8: Opc = ARM::VUZPq8; break;
2759 case MVT::v8i16: Opc = ARM::VUZPq16; break;
2761 case MVT::v4i32: Opc = ARM::VUZPq32; break;
2763 SDValue Pred = getAL(CurDAG);
2764 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2765 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2766 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2768 case ARMISD::VTRN: {
2770 EVT VT = N->getValueType(0);
2771 switch (VT.getSimpleVT().SimpleTy) {
2772 default: return NULL;
2773 case MVT::v8i8: Opc = ARM::VTRNd8; break;
2774 case MVT::v4i16: Opc = ARM::VTRNd16; break;
2776 case MVT::v2i32: Opc = ARM::VTRNd32; break;
2777 case MVT::v16i8: Opc = ARM::VTRNq8; break;
2778 case MVT::v8i16: Opc = ARM::VTRNq16; break;
2780 case MVT::v4i32: Opc = ARM::VTRNq32; break;
2782 SDValue Pred = getAL(CurDAG);
2783 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2784 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2785 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2787 case ARMISD::BUILD_VECTOR: {
2788 EVT VecVT = N->getValueType(0);
2789 EVT EltVT = VecVT.getVectorElementType();
2790 unsigned NumElts = VecVT.getVectorNumElements();
2791 if (EltVT == MVT::f64) {
2792 assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
2793 return PairDRegs(VecVT, N->getOperand(0), N->getOperand(1));
2795 assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
2797 return PairSRegs(VecVT, N->getOperand(0), N->getOperand(1));
2798 assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
2799 return QuadSRegs(VecVT, N->getOperand(0), N->getOperand(1),
2800 N->getOperand(2), N->getOperand(3));
2803 case ARMISD::VLD2DUP: {
2804 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
2806 return SelectVLDDup(N, false, 2, Opcodes);
2809 case ARMISD::VLD3DUP: {
2810 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
2811 ARM::VLD3DUPd16Pseudo,
2812 ARM::VLD3DUPd32Pseudo };
2813 return SelectVLDDup(N, false, 3, Opcodes);
2816 case ARMISD::VLD4DUP: {
2817 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
2818 ARM::VLD4DUPd16Pseudo,
2819 ARM::VLD4DUPd32Pseudo };
2820 return SelectVLDDup(N, false, 4, Opcodes);
2823 case ARMISD::VLD2DUP_UPD: {
2824 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
2825 ARM::VLD2DUPd16wb_fixed,
2826 ARM::VLD2DUPd32wb_fixed };
2827 return SelectVLDDup(N, true, 2, Opcodes);
2830 case ARMISD::VLD3DUP_UPD: {
2831 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
2832 ARM::VLD3DUPd16Pseudo_UPD,
2833 ARM::VLD3DUPd32Pseudo_UPD };
2834 return SelectVLDDup(N, true, 3, Opcodes);
2837 case ARMISD::VLD4DUP_UPD: {
2838 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
2839 ARM::VLD4DUPd16Pseudo_UPD,
2840 ARM::VLD4DUPd32Pseudo_UPD };
2841 return SelectVLDDup(N, true, 4, Opcodes);
2844 case ARMISD::VLD1_UPD: {
2845 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
2846 ARM::VLD1d16wb_fixed,
2847 ARM::VLD1d32wb_fixed,
2848 ARM::VLD1d64wb_fixed };
2849 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
2850 ARM::VLD1q16wb_fixed,
2851 ARM::VLD1q32wb_fixed,
2852 ARM::VLD1q64wb_fixed };
2853 return SelectVLD(N, true, 1, DOpcodes, QOpcodes, 0);
2856 case ARMISD::VLD2_UPD: {
2857 static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
2858 ARM::VLD2d16wb_fixed,
2859 ARM::VLD2d32wb_fixed,
2860 ARM::VLD1q64wb_fixed};
2861 static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
2862 ARM::VLD2q16PseudoWB_fixed,
2863 ARM::VLD2q32PseudoWB_fixed };
2864 return SelectVLD(N, true, 2, DOpcodes, QOpcodes, 0);
2867 case ARMISD::VLD3_UPD: {
2868 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
2869 ARM::VLD3d16Pseudo_UPD,
2870 ARM::VLD3d32Pseudo_UPD,
2871 ARM::VLD1q64wb_fixed};
2872 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
2873 ARM::VLD3q16Pseudo_UPD,
2874 ARM::VLD3q32Pseudo_UPD };
2875 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
2876 ARM::VLD3q16oddPseudo_UPD,
2877 ARM::VLD3q32oddPseudo_UPD };
2878 return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2881 case ARMISD::VLD4_UPD: {
2882 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
2883 ARM::VLD4d16Pseudo_UPD,
2884 ARM::VLD4d32Pseudo_UPD,
2885 ARM::VLD1q64wb_fixed};
2886 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2887 ARM::VLD4q16Pseudo_UPD,
2888 ARM::VLD4q32Pseudo_UPD };
2889 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2890 ARM::VLD4q16oddPseudo_UPD,
2891 ARM::VLD4q32oddPseudo_UPD };
2892 return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2895 case ARMISD::VLD2LN_UPD: {
2896 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
2897 ARM::VLD2LNd16Pseudo_UPD,
2898 ARM::VLD2LNd32Pseudo_UPD };
2899 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
2900 ARM::VLD2LNq32Pseudo_UPD };
2901 return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
2904 case ARMISD::VLD3LN_UPD: {
2905 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
2906 ARM::VLD3LNd16Pseudo_UPD,
2907 ARM::VLD3LNd32Pseudo_UPD };
2908 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
2909 ARM::VLD3LNq32Pseudo_UPD };
2910 return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
2913 case ARMISD::VLD4LN_UPD: {
2914 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
2915 ARM::VLD4LNd16Pseudo_UPD,
2916 ARM::VLD4LNd32Pseudo_UPD };
2917 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
2918 ARM::VLD4LNq32Pseudo_UPD };
2919 return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
2922 case ARMISD::VST1_UPD: {
2923 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
2924 ARM::VST1d16wb_fixed,
2925 ARM::VST1d32wb_fixed,
2926 ARM::VST1d64wb_fixed };
2927 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
2928 ARM::VST1q16wb_fixed,
2929 ARM::VST1q32wb_fixed,
2930 ARM::VST1q64wb_fixed };
2931 return SelectVST(N, true, 1, DOpcodes, QOpcodes, 0);
2934 case ARMISD::VST2_UPD: {
2935 static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
2936 ARM::VST2d16wb_fixed,
2937 ARM::VST2d32wb_fixed,
2938 ARM::VST1q64wb_fixed};
2939 static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
2940 ARM::VST2q16PseudoWB_fixed,
2941 ARM::VST2q32PseudoWB_fixed };
2942 return SelectVST(N, true, 2, DOpcodes, QOpcodes, 0);
2945 case ARMISD::VST3_UPD: {
2946 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
2947 ARM::VST3d16Pseudo_UPD,
2948 ARM::VST3d32Pseudo_UPD,
2949 ARM::VST1d64TPseudoWB_fixed};
2950 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2951 ARM::VST3q16Pseudo_UPD,
2952 ARM::VST3q32Pseudo_UPD };
2953 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
2954 ARM::VST3q16oddPseudo_UPD,
2955 ARM::VST3q32oddPseudo_UPD };
2956 return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2959 case ARMISD::VST4_UPD: {
2960 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
2961 ARM::VST4d16Pseudo_UPD,
2962 ARM::VST4d32Pseudo_UPD,
2963 ARM::VST1d64QPseudoWB_fixed};
2964 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2965 ARM::VST4q16Pseudo_UPD,
2966 ARM::VST4q32Pseudo_UPD };
2967 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
2968 ARM::VST4q16oddPseudo_UPD,
2969 ARM::VST4q32oddPseudo_UPD };
2970 return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2973 case ARMISD::VST2LN_UPD: {
2974 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
2975 ARM::VST2LNd16Pseudo_UPD,
2976 ARM::VST2LNd32Pseudo_UPD };
2977 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
2978 ARM::VST2LNq32Pseudo_UPD };
2979 return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
2982 case ARMISD::VST3LN_UPD: {
2983 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
2984 ARM::VST3LNd16Pseudo_UPD,
2985 ARM::VST3LNd32Pseudo_UPD };
2986 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
2987 ARM::VST3LNq32Pseudo_UPD };
2988 return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
2991 case ARMISD::VST4LN_UPD: {
2992 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
2993 ARM::VST4LNd16Pseudo_UPD,
2994 ARM::VST4LNd32Pseudo_UPD };
2995 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
2996 ARM::VST4LNq32Pseudo_UPD };
2997 return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
3000 case ISD::INTRINSIC_VOID:
3001 case ISD::INTRINSIC_W_CHAIN: {
3002 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
3007 case Intrinsic::arm_ldrexd: {
3008 SDValue MemAddr = N->getOperand(2);
3009 DebugLoc dl = N->getDebugLoc();
3010 SDValue Chain = N->getOperand(0);
3012 unsigned NewOpc = ARM::LDREXD;
3013 if (Subtarget->isThumb() && Subtarget->hasThumb2())
3014 NewOpc = ARM::t2LDREXD;
3016 // arm_ldrexd returns a i64 value in {i32, i32}
3017 std::vector<EVT> ResTys;
3018 ResTys.push_back(MVT::i32);
3019 ResTys.push_back(MVT::i32);
3020 ResTys.push_back(MVT::Other);
3022 // place arguments in the right order
3023 SmallVector<SDValue, 7> Ops;
3024 Ops.push_back(MemAddr);
3025 Ops.push_back(getAL(CurDAG));
3026 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3027 Ops.push_back(Chain);
3028 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(),
3030 // Transfer memoperands.
3031 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3032 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3033 cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
3035 // Until there's support for specifing explicit register constraints
3036 // like the use of even/odd register pair, hardcode ldrexd to always
3037 // use the pair [R0, R1] to hold the load result.
3038 Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R0,
3039 SDValue(Ld, 0), SDValue(0,0));
3040 Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R1,
3041 SDValue(Ld, 1), Chain.getValue(1));
3044 SDValue Glue = Chain.getValue(1);
3045 if (!SDValue(N, 0).use_empty()) {
3046 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3047 ARM::R0, MVT::i32, Glue);
3048 Glue = Result.getValue(2);
3049 ReplaceUses(SDValue(N, 0), Result);
3051 if (!SDValue(N, 1).use_empty()) {
3052 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3053 ARM::R1, MVT::i32, Glue);
3054 Glue = Result.getValue(2);
3055 ReplaceUses(SDValue(N, 1), Result);
3058 ReplaceUses(SDValue(N, 2), SDValue(Ld, 2));
3062 case Intrinsic::arm_strexd: {
3063 DebugLoc dl = N->getDebugLoc();
3064 SDValue Chain = N->getOperand(0);
3065 SDValue Val0 = N->getOperand(2);
3066 SDValue Val1 = N->getOperand(3);
3067 SDValue MemAddr = N->getOperand(4);
3069 // Until there's support for specifing explicit register constraints
3070 // like the use of even/odd register pair, hardcode strexd to always
3071 // use the pair [R2, R3] to hold the i64 (i32, i32) value to be stored.
3072 Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ARM::R2, Val0,
3074 Chain = CurDAG->getCopyToReg(Chain, dl, ARM::R3, Val1, Chain.getValue(1));
3076 SDValue Glue = Chain.getValue(1);
3077 Val0 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3078 ARM::R2, MVT::i32, Glue);
3079 Glue = Val0.getValue(1);
3080 Val1 = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
3081 ARM::R3, MVT::i32, Glue);
3083 // Store exclusive double return a i32 value which is the return status
3084 // of the issued store.
3085 std::vector<EVT> ResTys;
3086 ResTys.push_back(MVT::i32);
3087 ResTys.push_back(MVT::Other);
3089 // place arguments in the right order
3090 SmallVector<SDValue, 7> Ops;
3091 Ops.push_back(Val0);
3092 Ops.push_back(Val1);
3093 Ops.push_back(MemAddr);
3094 Ops.push_back(getAL(CurDAG));
3095 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3096 Ops.push_back(Chain);
3098 unsigned NewOpc = ARM::STREXD;
3099 if (Subtarget->isThumb() && Subtarget->hasThumb2())
3100 NewOpc = ARM::t2STREXD;
3102 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops.data(),
3104 // Transfer memoperands.
3105 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3106 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3107 cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
3112 case Intrinsic::arm_neon_vld1: {
3113 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3114 ARM::VLD1d32, ARM::VLD1d64 };
3115 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3116 ARM::VLD1q32, ARM::VLD1q64};
3117 return SelectVLD(N, false, 1, DOpcodes, QOpcodes, 0);
3120 case Intrinsic::arm_neon_vld2: {
3121 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3122 ARM::VLD2d32, ARM::VLD1q64 };
3123 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3124 ARM::VLD2q32Pseudo };
3125 return SelectVLD(N, false, 2, DOpcodes, QOpcodes, 0);
3128 case Intrinsic::arm_neon_vld3: {
3129 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3132 ARM::VLD1d64TPseudo };
3133 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3134 ARM::VLD3q16Pseudo_UPD,
3135 ARM::VLD3q32Pseudo_UPD };
3136 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3137 ARM::VLD3q16oddPseudo,
3138 ARM::VLD3q32oddPseudo };
3139 return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3142 case Intrinsic::arm_neon_vld4: {
3143 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3146 ARM::VLD1d64QPseudo };
3147 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3148 ARM::VLD4q16Pseudo_UPD,
3149 ARM::VLD4q32Pseudo_UPD };
3150 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3151 ARM::VLD4q16oddPseudo,
3152 ARM::VLD4q32oddPseudo };
3153 return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3156 case Intrinsic::arm_neon_vld2lane: {
3157 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3158 ARM::VLD2LNd16Pseudo,
3159 ARM::VLD2LNd32Pseudo };
3160 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3161 ARM::VLD2LNq32Pseudo };
3162 return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3165 case Intrinsic::arm_neon_vld3lane: {
3166 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3167 ARM::VLD3LNd16Pseudo,
3168 ARM::VLD3LNd32Pseudo };
3169 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3170 ARM::VLD3LNq32Pseudo };
3171 return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3174 case Intrinsic::arm_neon_vld4lane: {
3175 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3176 ARM::VLD4LNd16Pseudo,
3177 ARM::VLD4LNd32Pseudo };
3178 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3179 ARM::VLD4LNq32Pseudo };
3180 return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3183 case Intrinsic::arm_neon_vst1: {
3184 static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3185 ARM::VST1d32, ARM::VST1d64 };
3186 static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3187 ARM::VST1q32, ARM::VST1q64 };
3188 return SelectVST(N, false, 1, DOpcodes, QOpcodes, 0);
3191 case Intrinsic::arm_neon_vst2: {
3192 static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3193 ARM::VST2d32, ARM::VST1q64 };
3194 static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3195 ARM::VST2q32Pseudo };
3196 return SelectVST(N, false, 2, DOpcodes, QOpcodes, 0);
3199 case Intrinsic::arm_neon_vst3: {
3200 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3203 ARM::VST1d64TPseudo };
3204 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3205 ARM::VST3q16Pseudo_UPD,
3206 ARM::VST3q32Pseudo_UPD };
3207 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3208 ARM::VST3q16oddPseudo,
3209 ARM::VST3q32oddPseudo };
3210 return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3213 case Intrinsic::arm_neon_vst4: {
3214 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3217 ARM::VST1d64QPseudo };
3218 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3219 ARM::VST4q16Pseudo_UPD,
3220 ARM::VST4q32Pseudo_UPD };
3221 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3222 ARM::VST4q16oddPseudo,
3223 ARM::VST4q32oddPseudo };
3224 return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3227 case Intrinsic::arm_neon_vst2lane: {
3228 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3229 ARM::VST2LNd16Pseudo,
3230 ARM::VST2LNd32Pseudo };
3231 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3232 ARM::VST2LNq32Pseudo };
3233 return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3236 case Intrinsic::arm_neon_vst3lane: {
3237 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3238 ARM::VST3LNd16Pseudo,
3239 ARM::VST3LNd32Pseudo };
3240 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3241 ARM::VST3LNq32Pseudo };
3242 return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3245 case Intrinsic::arm_neon_vst4lane: {
3246 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3247 ARM::VST4LNd16Pseudo,
3248 ARM::VST4LNd32Pseudo };
3249 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3250 ARM::VST4LNq32Pseudo };
3251 return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3257 case ISD::INTRINSIC_WO_CHAIN: {
3258 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
3263 case Intrinsic::arm_neon_vtbl2:
3264 return SelectVTBL(N, false, 2, ARM::VTBL2);
3265 case Intrinsic::arm_neon_vtbl3:
3266 return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
3267 case Intrinsic::arm_neon_vtbl4:
3268 return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
3270 case Intrinsic::arm_neon_vtbx2:
3271 return SelectVTBL(N, true, 2, ARM::VTBX2);
3272 case Intrinsic::arm_neon_vtbx3:
3273 return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
3274 case Intrinsic::arm_neon_vtbx4:
3275 return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
3280 case ARMISD::VTBL1: {
3281 DebugLoc dl = N->getDebugLoc();
3282 EVT VT = N->getValueType(0);
3283 SmallVector<SDValue, 6> Ops;
3285 Ops.push_back(N->getOperand(0));
3286 Ops.push_back(N->getOperand(1));
3287 Ops.push_back(getAL(CurDAG)); // Predicate
3288 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3289 return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops.data(), Ops.size());
3291 case ARMISD::VTBL2: {
3292 DebugLoc dl = N->getDebugLoc();
3293 EVT VT = N->getValueType(0);
3295 // Form a REG_SEQUENCE to force register allocation.
3296 SDValue V0 = N->getOperand(0);
3297 SDValue V1 = N->getOperand(1);
3298 SDValue RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
3300 SmallVector<SDValue, 6> Ops;
3301 Ops.push_back(RegSeq);
3302 Ops.push_back(N->getOperand(2));
3303 Ops.push_back(getAL(CurDAG)); // Predicate
3304 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register
3305 return CurDAG->getMachineNode(ARM::VTBL2, dl, VT,
3306 Ops.data(), Ops.size());
3309 case ISD::CONCAT_VECTORS:
3310 return SelectConcatVector(N);
3312 case ARMISD::ATOMOR64_DAG:
3313 return SelectAtomic64(N, ARM::ATOMOR6432);
3314 case ARMISD::ATOMXOR64_DAG:
3315 return SelectAtomic64(N, ARM::ATOMXOR6432);
3316 case ARMISD::ATOMADD64_DAG:
3317 return SelectAtomic64(N, ARM::ATOMADD6432);
3318 case ARMISD::ATOMSUB64_DAG:
3319 return SelectAtomic64(N, ARM::ATOMSUB6432);
3320 case ARMISD::ATOMNAND64_DAG:
3321 return SelectAtomic64(N, ARM::ATOMNAND6432);
3322 case ARMISD::ATOMAND64_DAG:
3323 return SelectAtomic64(N, ARM::ATOMAND6432);
3324 case ARMISD::ATOMSWAP64_DAG:
3325 return SelectAtomic64(N, ARM::ATOMSWAP6432);
3326 case ARMISD::ATOMCMPXCHG64_DAG:
3327 return SelectAtomic64(N, ARM::ATOMCMPXCHG6432);
3330 return SelectCode(N);
3333 bool ARMDAGToDAGISel::
3334 SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
3335 std::vector<SDValue> &OutOps) {
3336 assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
3337 // Require the address to be in a register. That is safe for all ARM
3338 // variants and it is hard to do anything much smarter without knowing
3339 // how the operand is used.
3340 OutOps.push_back(Op);
3344 /// createARMISelDag - This pass converts a legalized DAG into a
3345 /// ARM-specific DAG, ready for instruction scheduling.
3347 FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
3348 CodeGenOpt::Level OptLevel) {
3349 return new ARMDAGToDAGISel(TM, OptLevel);