From: Andrea Di Biagio Date: Fri, 9 Oct 2015 10:53:41 +0000 (+0000) Subject: [MemCpyOpt] Fix wrong merging adjacent nontemporal stores into memset calls. X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=12b807e83a2dcbc0fb8adf62657f0837096df0f0;p=oota-llvm.git [MemCpyOpt] Fix wrong merging adjacent nontemporal stores into memset calls. Pass MemCpyOpt doesn't check if a store instruction is nontemporal. As a consequence, adjacent nontemporal stores are always merged into a memset call. Example: ;;; define void @foo(<4 x float>* nocapture %p) { entry: store <4 x float> zeroinitializer, <4 x float>* %p, align 16, !nontemporal !0 %p1 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 1 store <4 x float> zeroinitializer, <4 x float>* %p1, align 16, !nontemporal !0 ret void } !0 = !{i32 1} ;;; In this example, the two nontemporal stores are combined to a memset of zero which does not preserve the nontemporal hint. Later on the backend (tested on a x86-64 corei7) expands that memset call into a sequence of two normal 16-byte aligned vector stores. opt -memcpyopt example.ll -S -o - | llc -mcpu=corei7 -o - Before: xorps %xmm0, %xmm0 movaps %xmm0, 16(%rdi) movaps %xmm0, (%rdi) With this patch, we no longer merge nontemporal stores into calls to memset. In this example, llc correctly expands the two stores into two movntps: xorps %xmm0, %xmm0 movntps %xmm0, 16(%rdi) movntps %xmm0, (%rdi) In theory, we could extend the usage of !nontemporal metadata to memcpy/memset calls. However a change like that would only have the effect of forcing the backend to expand !nontemporal memsets back to sequences of store instructions. A memset library call would not have exactly the same semantic of a builtin !nontemporal memset call. So, SelectionDAG will have to conservatively expand it back to a sequence of !nontemporal stores (effectively undoing the merging). Differential Revision: http://reviews.llvm.org/D13519 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@249820 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/lib/Transforms/Scalar/MemCpyOptimizer.cpp index 4039c6f3534..64f8aafcda0 100644 --- a/lib/Transforms/Scalar/MemCpyOptimizer.cpp +++ b/lib/Transforms/Scalar/MemCpyOptimizer.cpp @@ -488,6 +488,16 @@ Instruction *MemCpyOpt::tryMergingIntoMemset(Instruction *StartInst, bool MemCpyOpt::processStore(StoreInst *SI, BasicBlock::iterator &BBI) { if (!SI->isSimple()) return false; + + // Avoid merging nontemporal stores since the resulting + // memcpy/memset would not be able to preserve the nontemporal hint. + // In theory we could teach how to propagate the !nontemporal metadata to + // memset calls. However, that change would force the backend to + // conservatively expand !nontemporal memset calls back to sequences of + // store instructions (effectively undoing the merging). + if (SI->getMetadata(LLVMContext::MD_nontemporal)) + return false; + const DataLayout &DL = SI->getModule()->getDataLayout(); // Detect cases where we're performing call slot forwarding, but diff --git a/test/Transforms/MemCpyOpt/nontemporal.ll b/test/Transforms/MemCpyOpt/nontemporal.ll new file mode 100644 index 00000000000..d9dafcc7b81 --- /dev/null +++ b/test/Transforms/MemCpyOpt/nontemporal.ll @@ -0,0 +1,49 @@ +; RUN: opt < %s -memcpyopt -S | FileCheck %s + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" + +; Verify that we don't combine nontemporal stores into memset calls. + +define void @nontemporal_stores_1(<4 x float>* nocapture %dst) { +; CHECK-LABEL: @nontemporal_stores_1 +; CHECK: store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0 +; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr1, align 16, !nontemporal !0 +; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr2, align 16, !nontemporal !0 +; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr3, align 16, !nontemporal !0 +; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr4, align 16, !nontemporal !0 +; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr5, align 16, !nontemporal !0 +; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr6, align 16, !nontemporal !0 +; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr7, align 16, !nontemporal !0 +; CHECK-NEXT: ret void +entry: + store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0 + %ptr1 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 1 + store <4 x float> zeroinitializer, <4 x float>* %ptr1, align 16, !nontemporal !0 + %ptr2 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 2 + store <4 x float> zeroinitializer, <4 x float>* %ptr2, align 16, !nontemporal !0 + %ptr3 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 3 + store <4 x float> zeroinitializer, <4 x float>* %ptr3, align 16, !nontemporal !0 + %ptr4 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 4 + store <4 x float> zeroinitializer, <4 x float>* %ptr4, align 16, !nontemporal !0 + %ptr5 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 5 + store <4 x float> zeroinitializer, <4 x float>* %ptr5, align 16, !nontemporal !0 + %ptr6 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 6 + store <4 x float> zeroinitializer, <4 x float>* %ptr6, align 16, !nontemporal !0 + %ptr7 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 7 + store <4 x float> zeroinitializer, <4 x float>* %ptr7, align 16, !nontemporal !0 + ret void +} + +define void @nontemporal_stores_2(<4 x float>* nocapture %dst) { +; CHECK-LABEL: @nontemporal_stores_2 +; CHECK: store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0 +; CHECK: store <4 x float> zeroinitializer, <4 x float>* %ptr1, align 16, !nontemporal !0 +; CHECK-NEXT: ret void +entry: + store <4 x float> zeroinitializer, <4 x float>* %dst, align 16, !nontemporal !0 + %ptr1 = getelementptr inbounds <4 x float>, <4 x float>* %dst, i64 1 + store <4 x float> zeroinitializer, <4 x float>* %ptr1, align 16, !nontemporal !0 + ret void +} + +!0 = !{i32 1}