//
//===----------------------------------------------------------------------===//
+#include "TaintRelaxedAtomicsUtils.h"
+#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/CodeGen/AtomicExpandUtils.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/NoFolder.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetLowering.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetSubtargetInfo.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
using namespace llvm;
bool isIdempotentRMW(AtomicRMWInst *AI);
bool simplifyIdempotentRMW(AtomicRMWInst *AI);
};
+
+
+ // If 'LI' is a relaxed load, and it is immediately followed by a
+// atomic read-modify-write that has acq_rel parameter, we don't have to do
+// anything since the rmw serves as a natural barrier.
+void MarkRelaxedLoadBeforeAcqrelRMW(LoadInst* LI) {
+ auto* BB = LI->getParent();
+ auto BBI = LI->getIterator();
+ for (BBI++; BBI != BB->end(); BBI++) {
+ Instruction* CurInst = &*BBI;
+ if (!CurInst) {
+ return;
+ }
+ if (!CurInst->isAtomic()) {
+ continue;
+ }
+ auto* RMW = dyn_cast<AtomicRMWInst>(CurInst);
+ if (!RMW) {
+ return;
+ }
+ if (RMW->getOrdering() == AcquireRelease ||
+ RMW->getOrdering() == SequentiallyConsistent) {
+ LI->setHasSubsequentAcqlRMW(true);
+ }
+ }
+}
+
}
char AtomicExpand::ID = 0;
TLI = TM->getSubtargetImpl(F)->getTargetLowering();
SmallVector<Instruction *, 1> AtomicInsts;
+ SmallVector<LoadInst*, 1> MonotonicLoadInsts;
// Changing control-flow while iterating through it is a bad idea, so gather a
// list of all atomic instructions before we start.
if (I->isAtomic()) {
switch (I->getOpcode()) {
case Instruction::AtomicCmpXchg: {
+ // XXX-comment: AtomicCmpXchg in AArch64 will be translated to a
+ // conditional branch that contains the value of the load anyway, so
+ // we don't need to do anything.
+ /*
auto* CmpXchg = dyn_cast<AtomicCmpXchgInst>(&*I);
auto SuccOrdering = CmpXchg->getSuccessOrdering();
if (SuccOrdering == Monotonic) {
} else if (SuccOrdering == Release) {
CmpXchg->setSuccessOrdering(AcquireRelease);
}
+ */
break;
}
case Instruction::AtomicRMW: {
+ // XXX-comment: Similar to AtomicCmpXchg. These instructions in
+ // AArch64 will be translated to a loop whose condition depends on the
+ // store status, which further depends on the load value.
+ /*
auto* RMW = dyn_cast<AtomicRMWInst>(&*I);
if (RMW->getOrdering() == Monotonic) {
RMW->setOrdering(Acquire);
}
+ */
break;
}
case Instruction::Load: {
auto* LI = dyn_cast<LoadInst>(&*I);
if (LI->getOrdering() == Monotonic) {
+ /*
+ DEBUG(dbgs() << "Transforming relaxed loads to acquire loads: "
+ << *LI << '\n');
LI->setOrdering(Acquire);
+ */
+// MonotonicLoadInsts.push_back(LI);
+ MarkRelaxedLoadBeforeAcqrelRMW(LI);
}
break;
}
if (TLI->getInsertFencesForAtomic()) {
if (LI && isAtLeastAcquire(LI->getOrdering())) {
FenceOrdering = LI->getOrdering();
- LI->setOrdering(Monotonic);
+// AddFakeConditionalBranch(
IsStore = false;
IsLoad = true;
} else if (SI && isAtLeastRelease(SI->getOrdering())) {
MadeChange |= expandAtomicCmpXchg(CASI);
}
}
+
return MadeChange;
}
Builder.SetInsertPoint(LoopBB);
Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
+ // XXX-update: For relaxed RMWs (i.e., fetch_* operations), we still need to
+ // taint the load part. However, we only need to taint those whose results are
+ // not immediately used by a conditional branch or a store address.
+ Value* StoreAddr = Addr;
+ auto* LoadedPartInst = dyn_cast<Instruction>(Loaded);
+ assert(LoadedPartInst && "Load part of RMW should be an instruction!");
+ if (MemOpOrder != Acquire && MemOpOrder != AcquireRelease &&
+ MemOpOrder != SequentiallyConsistent) {
+ // Also check whether the result is used immediately. If so, taint the
+ // address of the upcoming store-exclusive.
+ if (NeedExtraConstraints(I)) {
+ StoreAddr = taintRMWStoreAddressWithLoadPart(Builder, Addr, LoadedPartInst);
+ }
+ }
+
Value *NewVal = PerformOp(Builder, Loaded);
Value *StoreSuccess =
- TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);
+ TLI->emitStoreConditional(Builder, NewVal, StoreAddr, MemOpOrder);
Value *TryAgain = Builder.CreateICmpNE(
StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain");
Builder.CreateCondBr(TryAgain, LoopBB, ExitBB);