1 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This pass lowers atomic intrinsics to non-atomic form for use in a known
11 // non-preemptible environment.
13 //===----------------------------------------------------------------------===//
15 #define DEBUG_TYPE "loweratomic"
16 #include "llvm/Transforms/Scalar.h"
17 #include "llvm/Function.h"
18 #include "llvm/IntrinsicInst.h"
19 #include "llvm/Pass.h"
20 #include "llvm/Support/IRBuilder.h"
23 static bool LowerAtomicIntrinsic(IntrinsicInst *II) {
24 IRBuilder<> Builder(II->getParent(), II);
25 unsigned IID = II->getIntrinsicID();
27 case Intrinsic::memory_barrier:
30 case Intrinsic::atomic_load_add:
31 case Intrinsic::atomic_load_sub:
32 case Intrinsic::atomic_load_and:
33 case Intrinsic::atomic_load_nand:
34 case Intrinsic::atomic_load_or:
35 case Intrinsic::atomic_load_xor:
36 case Intrinsic::atomic_load_max:
37 case Intrinsic::atomic_load_min:
38 case Intrinsic::atomic_load_umax:
39 case Intrinsic::atomic_load_umin: {
40 Value *Ptr = II->getArgOperand(0), *Delta = II->getArgOperand(1);
42 LoadInst *Orig = Builder.CreateLoad(Ptr);
45 default: assert(0 && "Unrecognized atomic modify operation");
46 case Intrinsic::atomic_load_add:
47 Res = Builder.CreateAdd(Orig, Delta);
49 case Intrinsic::atomic_load_sub:
50 Res = Builder.CreateSub(Orig, Delta);
52 case Intrinsic::atomic_load_and:
53 Res = Builder.CreateAnd(Orig, Delta);
55 case Intrinsic::atomic_load_nand:
56 Res = Builder.CreateNot(Builder.CreateAnd(Orig, Delta));
58 case Intrinsic::atomic_load_or:
59 Res = Builder.CreateOr(Orig, Delta);
61 case Intrinsic::atomic_load_xor:
62 Res = Builder.CreateXor(Orig, Delta);
64 case Intrinsic::atomic_load_max:
65 Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Delta),
68 case Intrinsic::atomic_load_min:
69 Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Delta),
72 case Intrinsic::atomic_load_umax:
73 Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Delta),
76 case Intrinsic::atomic_load_umin:
77 Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Delta),
81 Builder.CreateStore(Res, Ptr);
83 II->replaceAllUsesWith(Orig);
87 case Intrinsic::atomic_swap: {
88 Value *Ptr = II->getArgOperand(0), *Val = II->getArgOperand(1);
89 LoadInst *Orig = Builder.CreateLoad(Ptr);
90 Builder.CreateStore(Val, Ptr);
91 II->replaceAllUsesWith(Orig);
95 case Intrinsic::atomic_cmp_swap: {
96 Value *Ptr = II->getArgOperand(0), *Cmp = II->getArgOperand(1);
97 Value *Val = II->getArgOperand(2);
99 LoadInst *Orig = Builder.CreateLoad(Ptr);
100 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
101 Value *Res = Builder.CreateSelect(Equal, Val, Orig);
102 Builder.CreateStore(Res, Ptr);
103 II->replaceAllUsesWith(Orig);
111 assert(II->use_empty() &&
112 "Lowering should have eliminated any uses of the intrinsic call!");
113 II->eraseFromParent();
118 bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
119 IRBuilder<> Builder(CXI->getParent(), CXI);
120 Value *Ptr = CXI->getPointerOperand();
121 Value *Cmp = CXI->getCompareOperand();
122 Value *Val = CXI->getNewValOperand();
124 LoadInst *Orig = Builder.CreateLoad(Ptr);
125 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
126 Value *Res = Builder.CreateSelect(Equal, Val, Orig);
127 Builder.CreateStore(Res, Ptr);
129 CXI->replaceAllUsesWith(Orig);
130 CXI->eraseFromParent();
134 bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
135 IRBuilder<> Builder(RMWI->getParent(), RMWI);
136 Value *Ptr = RMWI->getPointerOperand();
137 Value *Val = RMWI->getValOperand();
139 LoadInst *Orig = Builder.CreateLoad(Ptr);
142 switch (RMWI->getOperation()) {
143 default: llvm_unreachable("Unexpected RMW operation");
144 case AtomicRMWInst::Xchg:
147 case AtomicRMWInst::Add:
148 Res = Builder.CreateAdd(Orig, Val);
150 case AtomicRMWInst::Sub:
151 Res = Builder.CreateSub(Orig, Val);
153 case AtomicRMWInst::And:
154 Res = Builder.CreateAnd(Orig, Val);
156 case AtomicRMWInst::Nand:
157 Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
159 case AtomicRMWInst::Or:
160 Res = Builder.CreateOr(Orig, Val);
162 case AtomicRMWInst::Xor:
163 Res = Builder.CreateXor(Orig, Val);
165 case AtomicRMWInst::Max:
166 Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
169 case AtomicRMWInst::Min:
170 Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
173 case AtomicRMWInst::UMax:
174 Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
177 case AtomicRMWInst::UMin:
178 Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
182 Builder.CreateStore(Res, Ptr);
183 RMWI->replaceAllUsesWith(Orig);
184 RMWI->eraseFromParent();
188 static bool LowerFenceInst(FenceInst *FI) {
189 FI->eraseFromParent();
193 static bool LowerLoadInst(LoadInst *LI) {
194 LI->setAtomic(NotAtomic);
198 static bool LowerStoreInst(StoreInst *SI) {
199 SI->setAtomic(NotAtomic);
204 struct LowerAtomic : public BasicBlockPass {
206 LowerAtomic() : BasicBlockPass(ID) {
207 initializeLowerAtomicPass(*PassRegistry::getPassRegistry());
209 bool runOnBasicBlock(BasicBlock &BB) {
210 bool Changed = false;
211 for (BasicBlock::iterator DI = BB.begin(), DE = BB.end(); DI != DE; ) {
212 Instruction *Inst = DI++;
213 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst))
214 Changed |= LowerAtomicIntrinsic(II);
215 else if (FenceInst *FI = dyn_cast<FenceInst>(Inst))
216 Changed |= LowerFenceInst(FI);
217 else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(Inst))
218 Changed |= LowerAtomicCmpXchgInst(CXI);
219 else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst))
220 Changed |= LowerAtomicRMWInst(RMWI);
221 else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
224 } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
234 char LowerAtomic::ID = 0;
235 INITIALIZE_PASS(LowerAtomic, "loweratomic",
236 "Lower atomic intrinsics to non-atomic form",
239 Pass *llvm::createLowerAtomicPass() { return new LowerAtomic(); }