From 62c939d7d5572e57963a5f26fb6fe802e13dc0bf Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 3 Dec 2008 05:21:24 +0000 Subject: [PATCH] Mark x86's V_SET0 and V_SETALLONES with isSimpleLoad, and teach X86's foldMemoryOperand how to "fold" them, by converting them into constant-pool loads. When they aren't folded, they use xorps/cmpeqd, but for example when register pressure is high, they may now be folded as memory operands, which reduces register pressure. Also, mark V_SET0 isAsCheapAsAMove so that two-address-elimination will remat it instead of copying zeros around (V_SETALLONES was already marked). git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@60461 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/Target/Target.td | 2 +- include/llvm/Target/TargetInstrDesc.h | 13 ++-- lib/Target/X86/X86InstrInfo.cpp | 34 +++++++- lib/Target/X86/X86InstrSSE.td | 8 +- test/CodeGen/X86/fold-pcmpeqd-0.ll | 107 ++++++++++++++++++++++++++ test/CodeGen/X86/fold-pcmpeqd-1.ll | 11 +++ utils/TableGen/CodeGenDAGPatterns.cpp | 9 --- 7 files changed, 164 insertions(+), 20 deletions(-) create mode 100644 test/CodeGen/X86/fold-pcmpeqd-0.ll create mode 100644 test/CodeGen/X86/fold-pcmpeqd-1.ll diff --git a/include/llvm/Target/Target.td b/include/llvm/Target/Target.td index 91e44f23a52..c3daf09accc 100644 --- a/include/llvm/Target/Target.td +++ b/include/llvm/Target/Target.td @@ -189,7 +189,7 @@ class Instruction { bit isIndirectBranch = 0; // Is this instruction an indirect branch? bit isBarrier = 0; // Can control flow fall through this instruction? bit isCall = 0; // Is this instruction a call instruction? - bit isSimpleLoad = 0; // Is this just a load instruction? + bit isSimpleLoad = 0; // Can this be folded as a memory operand? bit mayLoad = 0; // Is it possible for this inst to read memory? bit mayStore = 0; // Is it possible for this inst to write memory? bit isTwoAddress = 0; // Is this a two address instruction? diff --git a/include/llvm/Target/TargetInstrDesc.h b/include/llvm/Target/TargetInstrDesc.h index 6d9a9bca28b..70893e313b8 100644 --- a/include/llvm/Target/TargetInstrDesc.h +++ b/include/llvm/Target/TargetInstrDesc.h @@ -301,11 +301,14 @@ public: return Flags & (1 << TID::DelaySlot); } - /// isSimpleLoad - Return true for instructions that are simple loads from - /// memory. This should only be set on instructions that load a value from - /// memory and return it in their only virtual register definition. - /// Instructions that return a value loaded from memory and then modified in - /// some way should not return true for this. + /// isSimpleLoad - Return true for instructions that can be folded as + /// memory operands in other instructions. The most common use for this + /// is instructions that are simple loads from memory that don't modify + /// the loaded value in any way, but it can also be used for instructions + /// that can be expressed as constant-pool loads, such as V_SETALLONES + /// on x86, to allow them to be folded when it is beneficial. + /// This should only be set on instructions that return a value in their + /// only virtual register definition. bool isSimpleLoad() const { return Flags & (1 << TID::SimpleLoad); } diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp index 4b2a20d1809..a43c5ddad3f 100644 --- a/lib/Target/X86/X86InstrInfo.cpp +++ b/lib/Target/X86/X86InstrInfo.cpp @@ -19,6 +19,7 @@ #include "X86Subtarget.h" #include "X86TargetMachine.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -2127,9 +2128,36 @@ MachineInstr* X86InstrInfo::foldMemoryOperand(MachineFunction &MF, return NULL; SmallVector MOs; - unsigned NumOps = LoadMI->getDesc().getNumOperands(); - for (unsigned i = NumOps - 4; i != NumOps; ++i) - MOs.push_back(LoadMI->getOperand(i)); + if (LoadMI->getOpcode() == X86::V_SET0 || + LoadMI->getOpcode() == X86::V_SETALLONES) { + // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure. + // Create a constant-pool entry and operands to load from it. + + // x86-32 PIC requires a PIC base register for constant pools. + unsigned PICBase = 0; + if (TM.getRelocationModel() == Reloc::PIC_ && + !TM.getSubtarget().is64Bit()) + PICBase = TM.getInstrInfo()->getGlobalBaseReg(&MF); + + // Create a v4i32 constant-pool entry. + MachineConstantPool &MCP = *MF.getConstantPool(); + const VectorType *Ty = VectorType::get(Type::Int32Ty, 4); + Constant *C = LoadMI->getOpcode() == X86::V_SET0 ? + ConstantVector::getNullValue(Ty) : + ConstantVector::getAllOnesValue(Ty); + unsigned CPI = MCP.getConstantPoolIndex(C, /*AlignmentLog2=*/4); + + // Create operands to load from the constant pool entry. + MOs.push_back(MachineOperand::CreateReg(PICBase, false)); + MOs.push_back(MachineOperand::CreateImm(1)); + MOs.push_back(MachineOperand::CreateReg(0, false)); + MOs.push_back(MachineOperand::CreateCPI(CPI, 0)); + } else { + // Folding a normal load. Just copy the load's address operands. + unsigned NumOps = LoadMI->getDesc().getNumOperands(); + for (unsigned i = NumOps - 4; i != NumOps; ++i) + MOs.push_back(LoadMI->getOperand(i)); + } return foldMemoryOperand(MF, MI, Ops[0], MOs); } diff --git a/lib/Target/X86/X86InstrSSE.td b/lib/Target/X86/X86InstrSSE.td index 426bddcea09..58e923d6df5 100644 --- a/lib/Target/X86/X86InstrSSE.td +++ b/lib/Target/X86/X86InstrSSE.td @@ -987,7 +987,9 @@ def STMXCSR : PSI<0xAE, MRM3m, (outs), (ins i32mem:$dst), "stmxcsr\t$dst", [(int_x86_sse_stmxcsr addr:$dst)]>; // Alias instructions that map zero vector to pxor / xorp* for sse. -let isReMaterializable = 1 in +// We set isSimpleLoad because this can be converted to a constant-pool +// load of an all-zeros value if folding it would be beneficial. +let isReMaterializable = 1, isAsCheapAsAMove = 1, isSimpleLoad = 1 in def V_SET0 : PSI<0x57, MRMInitReg, (outs VR128:$dst), (ins), "xorps\t$dst, $dst", [(set VR128:$dst, (v4i32 immAllZerosV))]>; @@ -2253,7 +2255,9 @@ def : Pat<(membarrier (i8 imm:$ll), (i8 imm:$ls), (i8 imm:$sl), (i8 imm:$ss), (i8 1)), (MFENCE)>; // Alias instructions that map zero vector to pxor / xorp* for sse. -let isReMaterializable = 1, isAsCheapAsAMove = 1 in +// We set isSimpleLoad because this can be converted to a constant-pool +// load of an all-ones value if folding it would be beneficial. +let isReMaterializable = 1, isAsCheapAsAMove = 1, isSimpleLoad = 1 in def V_SETALLONES : PDI<0x76, MRMInitReg, (outs VR128:$dst), (ins), "pcmpeqd\t$dst, $dst", [(set VR128:$dst, (v4i32 immAllOnesV))]>; diff --git a/test/CodeGen/X86/fold-pcmpeqd-0.ll b/test/CodeGen/X86/fold-pcmpeqd-0.ll new file mode 100644 index 00000000000..79fb792bcf6 --- /dev/null +++ b/test/CodeGen/X86/fold-pcmpeqd-0.ll @@ -0,0 +1,107 @@ +; RUN: llvm-as < %s | llc | not grep pcmpeqd +; RUN: llvm-as < %s | llc -march=x86-64 | grep pcmpeqd | count 1 + +; On x86-64, this testcase shouldn't need to spill the -1 value, +; so it should just use pcmpeqd to materialize an all-ones vector. +; On x86-32, there aren't enough registers, so an all-ones +; constant pool should be created so it can be folded. + +target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32" +target triple = "i686-apple-cl.1.0" + %struct.__ImageExecInfo = type <{ <4 x i32>, <4 x float>, <2 x i64>, i8*, i8*, i8*, i32, i32, i32, i32, i32 }> + %struct._cl_image_format_t = type <{ i32, i32, i32 }> + %struct._image2d_t = type <{ i8*, %struct._cl_image_format_t, i32, i32, i32, i32, i32, i32 }> + +define void @program_1(%struct._image2d_t* %dest, %struct._image2d_t* %t0, <4 x float> %p0, <4 x float> %p1, <4 x float> %p4, <4 x float> %p5, <4 x float> %p6) nounwind { +entry: + %tmp3.i = load i32* null ; [#uses=1] + %cmp = icmp sgt i32 %tmp3.i, 200 ; [#uses=1] + br i1 %cmp, label %forcond, label %ifthen + +ifthen: ; preds = %entry + ret void + +forcond: ; preds = %entry + %tmp3.i536 = load i32* null ; [#uses=1] + %cmp12 = icmp slt i32 0, %tmp3.i536 ; [#uses=1] + br i1 %cmp12, label %forbody, label %afterfor + +forbody: ; preds = %forcond + %bitcast204.i313 = bitcast <4 x i32> zeroinitializer to <4 x float> ; <<4 x float>> [#uses=1] + %mul233 = mul <4 x float> %bitcast204.i313, zeroinitializer ; <<4 x float>> [#uses=1] + %mul257 = mul <4 x float> %mul233, zeroinitializer ; <<4 x float>> [#uses=1] + %mul275 = mul <4 x float> %mul257, zeroinitializer ; <<4 x float>> [#uses=1] + %tmp51 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %mul275, <4 x float> zeroinitializer) nounwind ; <<4 x float>> [#uses=1] + %bitcast198.i182 = bitcast <4 x float> zeroinitializer to <4 x i32> ; <<4 x i32>> [#uses=0] + %bitcast204.i185 = bitcast <4 x i32> zeroinitializer to <4 x float> ; <<4 x float>> [#uses=1] + %tmp69 = call <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float> zeroinitializer) nounwind ; <<4 x i32>> [#uses=1] + %tmp70 = call <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32> %tmp69) nounwind ; <<4 x float>> [#uses=1] + %sub140.i78 = sub <4 x float> zeroinitializer, %tmp70 ; <<4 x float>> [#uses=2] + %mul166.i86 = mul <4 x float> zeroinitializer, %sub140.i78 ; <<4 x float>> [#uses=1] + %add167.i87 = add <4 x float> %mul166.i86, < float 0x3FE62ACB60000000, float 0x3FE62ACB60000000, float 0x3FE62ACB60000000, float 0x3FE62ACB60000000 > ; <<4 x float>> [#uses=1] + %mul171.i88 = mul <4 x float> %add167.i87, %sub140.i78 ; <<4 x float>> [#uses=1] + %add172.i89 = add <4 x float> %mul171.i88, < float 0x3FF0000A40000000, float 0x3FF0000A40000000, float 0x3FF0000A40000000, float 0x3FF0000A40000000 > ; <<4 x float>> [#uses=1] + %bitcast176.i90 = bitcast <4 x float> %add172.i89 to <4 x i32> ; <<4 x i32>> [#uses=1] + %andnps178.i92 = and <4 x i32> %bitcast176.i90, zeroinitializer ; <<4 x i32>> [#uses=1] + %bitcast179.i93 = bitcast <4 x i32> %andnps178.i92 to <4 x float> ; <<4 x float>> [#uses=1] + %mul186.i96 = mul <4 x float> %bitcast179.i93, zeroinitializer ; <<4 x float>> [#uses=1] + %bitcast190.i98 = bitcast <4 x float> %mul186.i96 to <4 x i32> ; <<4 x i32>> [#uses=1] + %andnps192.i100 = and <4 x i32> %bitcast190.i98, zeroinitializer ; <<4 x i32>> [#uses=1] + %xorps.i102 = xor <4 x i32> zeroinitializer, < i32 -1, i32 -1, i32 -1, i32 -1 > ; <<4 x i32>> [#uses=1] + %orps203.i103 = or <4 x i32> %andnps192.i100, %xorps.i102 ; <<4 x i32>> [#uses=1] + %bitcast204.i104 = bitcast <4 x i32> %orps203.i103 to <4 x float> ; <<4 x float>> [#uses=1] + %cmple.i = call <4 x float> @llvm.x86.sse.cmp.ps(<4 x float> zeroinitializer, <4 x float> %tmp51, i8 2) nounwind ; <<4 x float>> [#uses=1] + %tmp80 = call <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32> zeroinitializer) nounwind ; <<4 x float>> [#uses=1] + %sub140.i = sub <4 x float> zeroinitializer, %tmp80 ; <<4 x float>> [#uses=1] + %bitcast148.i = bitcast <4 x float> zeroinitializer to <4 x i32> ; <<4 x i32>> [#uses=1] + %andnps150.i = and <4 x i32> %bitcast148.i, < i32 -2139095041, i32 -2139095041, i32 -2139095041, i32 -2139095041 > ; <<4 x i32>> [#uses=0] + %mul171.i = mul <4 x float> zeroinitializer, %sub140.i ; <<4 x float>> [#uses=1] + %add172.i = add <4 x float> %mul171.i, < float 0x3FF0000A40000000, float 0x3FF0000A40000000, float 0x3FF0000A40000000, float 0x3FF0000A40000000 > ; <<4 x float>> [#uses=1] + %bitcast176.i = bitcast <4 x float> %add172.i to <4 x i32> ; <<4 x i32>> [#uses=1] + %andnps178.i = and <4 x i32> %bitcast176.i, zeroinitializer ; <<4 x i32>> [#uses=1] + %bitcast179.i = bitcast <4 x i32> %andnps178.i to <4 x float> ; <<4 x float>> [#uses=1] + %mul186.i = mul <4 x float> %bitcast179.i, zeroinitializer ; <<4 x float>> [#uses=1] + %bitcast189.i = bitcast <4 x float> zeroinitializer to <4 x i32> ; <<4 x i32>> [#uses=0] + %bitcast190.i = bitcast <4 x float> %mul186.i to <4 x i32> ; <<4 x i32>> [#uses=1] + %andnps192.i = and <4 x i32> %bitcast190.i, zeroinitializer ; <<4 x i32>> [#uses=1] + %bitcast198.i = bitcast <4 x float> %cmple.i to <4 x i32> ; <<4 x i32>> [#uses=1] + %xorps.i = xor <4 x i32> %bitcast198.i, < i32 -1, i32 -1, i32 -1, i32 -1 > ; <<4 x i32>> [#uses=1] + %orps203.i = or <4 x i32> %andnps192.i, %xorps.i ; <<4 x i32>> [#uses=1] + %bitcast204.i = bitcast <4 x i32> %orps203.i to <4 x float> ; <<4 x float>> [#uses=1] + %mul307 = mul <4 x float> %bitcast204.i185, zeroinitializer ; <<4 x float>> [#uses=1] + %mul310 = mul <4 x float> %bitcast204.i104, zeroinitializer ; <<4 x float>> [#uses=2] + %mul313 = mul <4 x float> %bitcast204.i, zeroinitializer ; <<4 x float>> [#uses=1] + %tmp82 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %mul307, <4 x float> zeroinitializer) nounwind ; <<4 x float>> [#uses=1] + %bitcast11.i15 = bitcast <4 x float> %tmp82 to <4 x i32> ; <<4 x i32>> [#uses=1] + %andnps.i17 = and <4 x i32> %bitcast11.i15, zeroinitializer ; <<4 x i32>> [#uses=1] + %orps.i18 = or <4 x i32> %andnps.i17, zeroinitializer ; <<4 x i32>> [#uses=1] + %bitcast17.i19 = bitcast <4 x i32> %orps.i18 to <4 x float> ; <<4 x float>> [#uses=1] + %tmp83 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %mul310, <4 x float> zeroinitializer) nounwind ; <<4 x float>> [#uses=1] + %bitcast.i3 = bitcast <4 x float> %mul310 to <4 x i32> ; <<4 x i32>> [#uses=1] + %bitcast6.i4 = bitcast <4 x float> zeroinitializer to <4 x i32> ; <<4 x i32>> [#uses=2] + %andps.i5 = and <4 x i32> %bitcast.i3, %bitcast6.i4 ; <<4 x i32>> [#uses=1] + %bitcast11.i6 = bitcast <4 x float> %tmp83 to <4 x i32> ; <<4 x i32>> [#uses=1] + %not.i7 = xor <4 x i32> %bitcast6.i4, < i32 -1, i32 -1, i32 -1, i32 -1 > ; <<4 x i32>> [#uses=1] + %andnps.i8 = and <4 x i32> %bitcast11.i6, %not.i7 ; <<4 x i32>> [#uses=1] + %orps.i9 = or <4 x i32> %andnps.i8, %andps.i5 ; <<4 x i32>> [#uses=1] + %bitcast17.i10 = bitcast <4 x i32> %orps.i9 to <4 x float> ; <<4 x float>> [#uses=1] + %bitcast.i = bitcast <4 x float> %mul313 to <4 x i32> ; <<4 x i32>> [#uses=1] + %andps.i = and <4 x i32> %bitcast.i, zeroinitializer ; <<4 x i32>> [#uses=1] + %orps.i = or <4 x i32> zeroinitializer, %andps.i ; <<4 x i32>> [#uses=1] + %bitcast17.i = bitcast <4 x i32> %orps.i to <4 x float> ; <<4 x float>> [#uses=1] + call void null(<4 x float> %bitcast17.i19, <4 x float> %bitcast17.i10, <4 x float> %bitcast17.i, <4 x float> zeroinitializer, %struct.__ImageExecInfo* null, <4 x i32> zeroinitializer) nounwind + unreachable + +afterfor: ; preds = %forcond + ret void +} + +declare <4 x float> @llvm.x86.sse.cmp.ps(<4 x float>, <4 x float>, i8) nounwind readnone + +declare <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32>) nounwind readnone + +declare <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float>) nounwind readnone + +declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>) nounwind readnone + +declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>) nounwind readnone diff --git a/test/CodeGen/X86/fold-pcmpeqd-1.ll b/test/CodeGen/X86/fold-pcmpeqd-1.ll new file mode 100644 index 00000000000..e2141ebf685 --- /dev/null +++ b/test/CodeGen/X86/fold-pcmpeqd-1.ll @@ -0,0 +1,11 @@ +; RUN: llvm-as < %s | llc -march=x86 -mattr=+sse2 > %t +; RUN: grep pcmpeqd %t | count 1 +; RUN: grep xor %t | count 1 +; RUN: not grep LCP %t + +define <2 x double> @foo() nounwind { + ret <2 x double> bitcast (<2 x i64> to <2 x double>) +} +define <2 x double> @bar() nounwind { + ret <2 x double> bitcast (<2 x i64> to <2 x double>) +} diff --git a/utils/TableGen/CodeGenDAGPatterns.cpp b/utils/TableGen/CodeGenDAGPatterns.cpp index b3ed27b8d89..859a8ccac8d 100644 --- a/utils/TableGen/CodeGenDAGPatterns.cpp +++ b/utils/TableGen/CodeGenDAGPatterns.cpp @@ -1759,15 +1759,6 @@ static void InferFromPattern(const CodeGenInstruction &Inst, MayLoad = true; } - // Sanity-check the isSimpleLoad flag. - if (Inst.isSimpleLoad) { - if (!MayLoad) - fprintf(stderr, - "Warning: mayLoad flag not set or inferred for instruction '%s'" - " which has isSimpleLoad set.\n", - Inst.TheDef->getName().c_str()); - } - if (Inst.neverHasSideEffects) { if (HadPattern) fprintf(stderr, "Warning: neverHasSideEffects set on instruction '%s' " -- 2.34.1