From: Evan Cheng Date: Tue, 4 Dec 2012 22:41:50 +0000 (+0000) Subject: ARM custom lower ctpop for vector types. Patch by Pete Couperus. X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=c8e7045c8a3b3fb9ee8f4a7d4d4a52a46b3d420a;p=oota-llvm.git ARM custom lower ctpop for vector types. Patch by Pete Couperus. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@169325 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp index 998823e3441..443ce63719a 100644 --- a/lib/Target/ARM/ARMISelLowering.cpp +++ b/lib/Target/ARM/ARMISelLowering.cpp @@ -546,6 +546,14 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM) setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand); setOperationAction(ISD::FP_EXTEND, MVT::v2f64, Expand); + // NEON does not have single instruction CTPOP for vectors with element + // types wider than 8-bits. However, custom lowering can leverage the + // v8i8/v16i8 vcnt instruction. + setOperationAction(ISD::CTPOP, MVT::v2i32, Custom); + setOperationAction(ISD::CTPOP, MVT::v4i32, Custom); + setOperationAction(ISD::CTPOP, MVT::v4i16, Custom); + setOperationAction(ISD::CTPOP, MVT::v8i16, Custom); + setTargetDAGCombine(ISD::INTRINSIC_VOID); setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); @@ -3554,6 +3562,114 @@ static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ISD::CTLZ, dl, VT, rbit); } +/// getCTPOP16BitCounts - Returns a v8i8/v16i8 vector containing the bit-count +/// for each 16-bit element from operand, repeated. The basic idea is to +/// leverage vcnt to get the 8-bit counts, gather and add the results. +/// +/// Trace for v4i16: +/// input = [v0 v1 v2 v3 ] (vi 16-bit element) +/// cast: N0 = [w0 w1 w2 w3 w4 w5 w6 w7] (v0 = [w0 w1], wi 8-bit element) +/// vcnt: N1 = [b0 b1 b2 b3 b4 b5 b6 b7] (bi = bit-count of 8-bit element wi) +/// vrev: N2 = [b1 b0 b3 b2 b5 b4 b7 b6] +/// [b0 b1 b2 b3 b4 b5 b6 b7] +/// +[b1 b0 b3 b2 b5 b4 b7 b6] +/// N3=N1+N2 = [k0 k0 k1 k1 k2 k2 k3 k3] (k0 = b0+b1 = bit-count of 16-bit v0, +/// vuzp: = [k0 k1 k2 k3 k0 k1 k2 k3] each ki is 8-bits) +static SDValue getCTPOP16BitCounts(SDNode *N, SelectionDAG &DAG) { + EVT VT = N->getValueType(0); + DebugLoc DL = N->getDebugLoc(); + + EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8; + SDValue N0 = DAG.getNode(ISD::BITCAST, DL, VT8Bit, N->getOperand(0)); + SDValue N1 = DAG.getNode(ISD::CTPOP, DL, VT8Bit, N0); + SDValue N2 = DAG.getNode(ARMISD::VREV16, DL, VT8Bit, N1); + SDValue N3 = DAG.getNode(ISD::ADD, DL, VT8Bit, N1, N2); + return DAG.getNode(ARMISD::VUZP, DL, VT8Bit, N3, N3); +} + +/// lowerCTPOP16BitElements - Returns a v4i16/v8i16 vector containing the +/// bit-count for each 16-bit element from the operand. We need slightly +/// different sequencing for v4i16 and v8i16 to stay within NEON's available +/// 64/128-bit registers. +/// +/// Trace for v4i16: +/// input = [v0 v1 v2 v3 ] (vi 16-bit element) +/// v8i8: BitCounts = [k0 k1 k2 k3 k0 k1 k2 k3 ] (ki is the bit-count of vi) +/// v8i16:Extended = [k0 k1 k2 k3 k0 k1 k2 k3 ] +/// v4i16:Extracted = [k0 k1 k2 k3 ] +static SDValue lowerCTPOP16BitElements(SDNode *N, SelectionDAG &DAG) { + EVT VT = N->getValueType(0); + DebugLoc DL = N->getDebugLoc(); + + SDValue BitCounts = getCTPOP16BitCounts(N, DAG); + if (VT.is64BitVector()) { + SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, BitCounts); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, Extended, + DAG.getIntPtrConstant(0)); + } else { + SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, + BitCounts, DAG.getIntPtrConstant(0)); + return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, Extracted); + } +} + +/// lowerCTPOP32BitElements - Returns a v2i32/v4i32 vector containing the +/// bit-count for each 32-bit element from the operand. The idea here is +/// to split the vector into 16-bit elements, leverage the 16-bit count +/// routine, and then combine the results. +/// +/// Trace for v2i32 (v4i32 similar with Extracted/Extended exchanged): +/// input = [v0 v1 ] (vi: 32-bit elements) +/// Bitcast = [w0 w1 w2 w3 ] (wi: 16-bit elements, v0 = [w0 w1]) +/// Counts16 = [k0 k1 k2 k3 ] (ki: 16-bit elements, bit-count of wi) +/// vrev: N0 = [k1 k0 k3 k2 ] +/// [k0 k1 k2 k3 ] +/// N1 =+[k1 k0 k3 k2 ] +/// [k0 k2 k1 k3 ] +/// N2 =+[k1 k3 k0 k2 ] +/// [k0 k2 k1 k3 ] +/// Extended =+[k1 k3 k0 k2 ] +/// [k0 k2 ] +/// Extracted=+[k1 k3 ] +/// +static SDValue lowerCTPOP32BitElements(SDNode *N, SelectionDAG &DAG) { + EVT VT = N->getValueType(0); + DebugLoc DL = N->getDebugLoc(); + + EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16; + + SDValue Bitcast = DAG.getNode(ISD::BITCAST, DL, VT16Bit, N->getOperand(0)); + SDValue Counts16 = lowerCTPOP16BitElements(Bitcast.getNode(), DAG); + SDValue N0 = DAG.getNode(ARMISD::VREV32, DL, VT16Bit, Counts16); + SDValue N1 = DAG.getNode(ISD::ADD, DL, VT16Bit, Counts16, N0); + SDValue N2 = DAG.getNode(ARMISD::VUZP, DL, VT16Bit, N1, N1); + + if (VT.is64BitVector()) { + SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, N2); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i32, Extended, + DAG.getIntPtrConstant(0)); + } else { + SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, N2, + DAG.getIntPtrConstant(0)); + return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, Extracted); + } +} + +static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG, + const ARMSubtarget *ST) { + EVT VT = N->getValueType(0); + + assert(ST->hasNEON() && "Custom ctpop lowering requires NEON."); + assert((VT == MVT::v2i32 || VT == MVT::v4i32) || + (VT == MVT::v4i16 || VT == MVT::v8i16) && + "Unexpected type for custom ctpop lowering"); + + if (VT.getVectorElementType() == MVT::i32) + return lowerCTPOP32BitElements(N, DAG); + else + return lowerCTPOP16BitElements(N, DAG); +} + static SDValue LowerShift(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *ST) { EVT VT = N->getValueType(0); @@ -5411,6 +5527,7 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::SRL_PARTS: case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG); case ISD::CTTZ: return LowerCTTZ(Op.getNode(), DAG, Subtarget); + case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget); case ISD::SETCC: return LowerVSETCC(Op, DAG); case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget); case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget); diff --git a/test/CodeGen/ARM/popcnt.ll b/test/CodeGen/ARM/popcnt.ll new file mode 100644 index 00000000000..0b9c9467c20 --- /dev/null +++ b/test/CodeGen/ARM/popcnt.ll @@ -0,0 +1,191 @@ +; RUN: llc < %s -march=arm -mattr=+neon | FileCheck %s +; Implement ctpop with vcnt + +define <8 x i8> @vcnt8(<8 x i8>* %A) nounwind { +;CHECK: vcnt8: +;CHECK: vcnt.8 {{d[0-9]+}}, {{d[0-9]+}} + %tmp1 = load <8 x i8>* %A + %tmp2 = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> %tmp1) + ret <8 x i8> %tmp2 +} + +define <16 x i8> @vcntQ8(<16 x i8>* %A) nounwind { +;CHECK: vcntQ8: +;CHECK: vcnt.8 {{q[0-9]+}}, {{q[0-9]+}} + %tmp1 = load <16 x i8>* %A + %tmp2 = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %tmp1) + ret <16 x i8> %tmp2 +} + +define <4 x i16> @vcnt16(<4 x i16>* %A) nounwind { +; CHECK: vcnt16: +; CHECK: vcnt.8 {{d[0-9]+}}, {{d[0-9]+}} +; CHECK: vrev16.8 {{d[0-9]+}}, {{d[0-9]+}} +; CHECK: vadd.i8 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}} +; CHECK: vuzp.8 {{d[0-9]+}}, {{d[0-9]+}} +; CHECK: vmovl.u8 {{q[0-9]+}}, {{d[0-9]+}} + %tmp1 = load <4 x i16>* %A + %tmp2 = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %tmp1) + ret <4 x i16> %tmp2 +} + +define <8 x i16> @vcntQ16(<8 x i16>* %A) nounwind { +; CHECK: vcntQ16: +; CHECK: vcnt.8 {{q[0-9]+}}, {{q[0-9]+}} +; CHECK: vrev16.8 {{q[0-9]+}}, {{q[0-9]+}} +; CHECK: vadd.i8 {{q[0-9]+}}, {{q[0-9]+}}, {{q[0-9]+}} +; CHECK: vuzp.8 {{q[0-9]+}}, {{q[0-9]+}} +; CHECK: vmovl.u8 {{q[0-9]+}}, {{d[0-9]+}} + %tmp1 = load <8 x i16>* %A + %tmp2 = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %tmp1) + ret <8 x i16> %tmp2 +} + +define <2 x i32> @vcnt32(<2 x i32>* %A) nounwind { +; CHECK: vcnt32: +; CHECK: vcnt.8 {{d[0-9]+}}, {{d[0-9]+}} +; CHECK: vrev16.8 {{d[0-9]+}}, {{d[0-9]+}} +; CHECK: vadd.i8 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}} +; CHECK: vuzp.8 {{d[0-9]+}}, {{d[0-9]+}} +; CHECK: vmovl.u8 {{q[0-9]+}}, {{d[0-9]+}} +; CHECK: vrev32.16 {{d[0-9]+}}, {{d[0-9]+}} +; CHECK: vuzp.16 {{d[0-9]+}}, {{d[0-9]+}} +; CHECK: vmovl.u16 {{q[0-9]+}}, {{d[0-9]+}} + %tmp1 = load <2 x i32>* %A + %tmp2 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %tmp1) + ret <2 x i32> %tmp2 +} + +define <4 x i32> @vcntQ32(<4 x i32>* %A) nounwind { +; CHECK: vcntQ32: +; CHECK: vcnt.8 {{q[0-9]+}}, {{q[0-9]+}} +; CHECK: vrev16.8 {{q[0-9]+}}, {{q[0-9]+}} +; CHECK: vadd.i8 {{q[0-9]+}}, {{q[0-9]+}}, {{q[0-9]+}} +; CHECK: vuzp.8 {{q[0-9]+}}, {{q[0-9]+}} +; CHECK: vmovl.u8 {{q[0-9]+}}, {{d[0-9]+}} +; CHECK: vrev32.16 {{q[0-9]+}}, {{q[0-9]+}} +; CHECK: vuzp.16 {{q[0-9]+}}, {{q[0-9]+}} +; CHECK: vmovl.u16 {{q[0-9]+}}, {{d[0-9]+}} + %tmp1 = load <4 x i32>* %A + %tmp2 = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %tmp1) + ret <4 x i32> %tmp2 +} + +declare <8 x i8> @llvm.ctpop.v8i8(<8 x i8>) nounwind readnone +declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>) nounwind readnone +declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>) nounwind readnone +declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>) nounwind readnone +declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>) nounwind readnone +declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>) nounwind readnone + +define <8 x i8> @vclz8(<8 x i8>* %A) nounwind { +;CHECK: vclz8: +;CHECK: vclz.i8 {{d[0-9]+}}, {{d[0-9]+}} + %tmp1 = load <8 x i8>* %A + %tmp2 = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %tmp1, i1 0) + ret <8 x i8> %tmp2 +} + +define <4 x i16> @vclz16(<4 x i16>* %A) nounwind { +;CHECK: vclz16: +;CHECK: vclz.i16 {{d[0-9]+}}, {{d[0-9]+}} + %tmp1 = load <4 x i16>* %A + %tmp2 = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %tmp1, i1 0) + ret <4 x i16> %tmp2 +} + +define <2 x i32> @vclz32(<2 x i32>* %A) nounwind { +;CHECK: vclz32: +;CHECK: vclz.i32 {{d[0-9]+}}, {{d[0-9]+}} + %tmp1 = load <2 x i32>* %A + %tmp2 = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %tmp1, i1 0) + ret <2 x i32> %tmp2 +} + +define <16 x i8> @vclzQ8(<16 x i8>* %A) nounwind { +;CHECK: vclzQ8: +;CHECK: vclz.i8 {{q[0-9]+}}, {{q[0-9]+}} + %tmp1 = load <16 x i8>* %A + %tmp2 = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %tmp1, i1 0) + ret <16 x i8> %tmp2 +} + +define <8 x i16> @vclzQ16(<8 x i16>* %A) nounwind { +;CHECK: vclzQ16: +;CHECK: vclz.i16 {{q[0-9]+}}, {{q[0-9]+}} + %tmp1 = load <8 x i16>* %A + %tmp2 = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %tmp1, i1 0) + ret <8 x i16> %tmp2 +} + +define <4 x i32> @vclzQ32(<4 x i32>* %A) nounwind { +;CHECK: vclzQ32: +;CHECK: vclz.i32 {{q[0-9]+}}, {{q[0-9]+}} + %tmp1 = load <4 x i32>* %A + %tmp2 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %tmp1, i1 0) + ret <4 x i32> %tmp2 +} + +declare <8 x i8> @llvm.ctlz.v8i8(<8 x i8>, i1) nounwind readnone +declare <4 x i16> @llvm.ctlz.v4i16(<4 x i16>, i1) nounwind readnone +declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1) nounwind readnone + +declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>, i1) nounwind readnone +declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>, i1) nounwind readnone +declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1) nounwind readnone + +define <8 x i8> @vclss8(<8 x i8>* %A) nounwind { +;CHECK: vclss8: +;CHECK: vcls.s8 + %tmp1 = load <8 x i8>* %A + %tmp2 = call <8 x i8> @llvm.arm.neon.vcls.v8i8(<8 x i8> %tmp1) + ret <8 x i8> %tmp2 +} + +define <4 x i16> @vclss16(<4 x i16>* %A) nounwind { +;CHECK: vclss16: +;CHECK: vcls.s16 + %tmp1 = load <4 x i16>* %A + %tmp2 = call <4 x i16> @llvm.arm.neon.vcls.v4i16(<4 x i16> %tmp1) + ret <4 x i16> %tmp2 +} + +define <2 x i32> @vclss32(<2 x i32>* %A) nounwind { +;CHECK: vclss32: +;CHECK: vcls.s32 + %tmp1 = load <2 x i32>* %A + %tmp2 = call <2 x i32> @llvm.arm.neon.vcls.v2i32(<2 x i32> %tmp1) + ret <2 x i32> %tmp2 +} + +define <16 x i8> @vclsQs8(<16 x i8>* %A) nounwind { +;CHECK: vclsQs8: +;CHECK: vcls.s8 + %tmp1 = load <16 x i8>* %A + %tmp2 = call <16 x i8> @llvm.arm.neon.vcls.v16i8(<16 x i8> %tmp1) + ret <16 x i8> %tmp2 +} + +define <8 x i16> @vclsQs16(<8 x i16>* %A) nounwind { +;CHECK: vclsQs16: +;CHECK: vcls.s16 + %tmp1 = load <8 x i16>* %A + %tmp2 = call <8 x i16> @llvm.arm.neon.vcls.v8i16(<8 x i16> %tmp1) + ret <8 x i16> %tmp2 +} + +define <4 x i32> @vclsQs32(<4 x i32>* %A) nounwind { +;CHECK: vclsQs32: +;CHECK: vcls.s32 + %tmp1 = load <4 x i32>* %A + %tmp2 = call <4 x i32> @llvm.arm.neon.vcls.v4i32(<4 x i32> %tmp1) + ret <4 x i32> %tmp2 +} + +declare <8 x i8> @llvm.arm.neon.vcls.v8i8(<8 x i8>) nounwind readnone +declare <4 x i16> @llvm.arm.neon.vcls.v4i16(<4 x i16>) nounwind readnone +declare <2 x i32> @llvm.arm.neon.vcls.v2i32(<2 x i32>) nounwind readnone + +declare <16 x i8> @llvm.arm.neon.vcls.v16i8(<16 x i8>) nounwind readnone +declare <8 x i16> @llvm.arm.neon.vcls.v8i16(<8 x i16>) nounwind readnone +declare <4 x i32> @llvm.arm.neon.vcls.v4i32(<4 x i32>) nounwind readnone