#include "llvm/ADT/SmallString.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/CaptureTracking.h"
+#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/EscapeEnumerator.h"
-// #include "llvm/Transforms/Utils/ModuleUtils.h"
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
#include <vector>
using namespace llvm;
+#define CDS_DEBUG
#define DEBUG_TYPE "CDS"
#include <llvm/IR/DebugLoc.h>
-Value *getPosition( Instruction * I, IRBuilder <> IRB, bool print = false)
+static inline Value *getPosition( Instruction * I, IRBuilder <> IRB, bool print = false)
{
const DebugLoc & debug_location = I->getDebugLoc ();
std::string position_string;
return IRB.CreateGlobalStringPtr (position_string);
}
+static inline bool checkSignature(Function * func, Value * args[]) {
+ FunctionType * FType = func->getFunctionType();
+ for (unsigned i = 0 ; i < FType->getNumParams(); i++) {
+ if (FType->getParamType(i) != args[i]->getType()) {
+#ifdef CDS_DEBUG
+ errs() << "expects: " << *FType->getParamType(i)
+ << "\tbut receives: " << *args[i]->getType() << "\n";
+#endif
+ return false;
+ }
+ }
+
+ return true;
+}
+
STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
STATISTIC(NumOmittedReadsBeforeWrite,
switch (order) {
case AtomicOrdering::Monotonic:
return (int)AtomicOrderingCABI::relaxed;
- // case AtomicOrdering::Consume: // not specified yet
- // return AtomicOrderingCABI::consume;
+ //case AtomicOrdering::Consume: // not specified yet
+ // return AtomicOrderingCABI::consume;
case AtomicOrdering::Acquire:
return (int)AtomicOrderingCABI::acquire;
case AtomicOrdering::Release:
if (isa<Function>(FuncOrBitcast))
return cast<Function>(FuncOrBitcast);
FuncOrBitcast->print(errs());
- errs() << '\n';
+ errs() << "\n";
std::string Err;
raw_string_ostream Stream(Err);
Stream << "CDSPass interface function redefined: " << *FuncOrBitcast;
bool isAtomicCall(Instruction *I);
bool instrumentAtomic(Instruction *I, const DataLayout &DL);
bool instrumentAtomicCall(CallInst *CI, const DataLayout &DL);
+ bool shouldInstrumentBeforeAtomics(Instruction *I);
void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local,
SmallVectorImpl<Instruction *> &All,
const DataLayout &DL);
bool addrPointsToConstantData(Value *Addr);
int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL);
+ bool instrumentLoops(Function &F);
Function * CDSFuncEntry;
Function * CDSFuncExit;
SmallString<32> AtomicStoreName("cds_atomic_store" + BitSizeStr);
CDSLoad[i] = checkCDSPassInterfaceFunction(
- M.getOrInsertFunction(LoadName, Attr, VoidTy, PtrTy));
+ M.getOrInsertFunction(LoadName, Attr, VoidTy, Int8PtrTy));
CDSStore[i] = checkCDSPassInterfaceFunction(
- M.getOrInsertFunction(StoreName, Attr, VoidTy, PtrTy));
+ M.getOrInsertFunction(StoreName, Attr, VoidTy, Int8PtrTy));
CDSVolatileLoad[i] = checkCDSPassInterfaceFunction(
M.getOrInsertFunction(VolatileLoadName,
Attr, Ty, PtrTy, Int8PtrTy));
return false;
}
+bool CDSPass::shouldInstrumentBeforeAtomics(Instruction * Inst) {
+ if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
+ AtomicOrdering ordering = LI->getOrdering();
+ if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
+ return true;
+ } else if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
+ AtomicOrdering ordering = SI->getOrdering();
+ if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
+ return true;
+ } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst)) {
+ AtomicOrdering ordering = RMWI->getOrdering();
+ if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
+ return true;
+ } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(Inst)) {
+ AtomicOrdering ordering = CASI->getSuccessOrdering();
+ if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
+ return true;
+ } else if (FenceInst *FI = dyn_cast<FenceInst>(Inst)) {
+ AtomicOrdering ordering = FI->getOrdering();
+ if ( isAtLeastOrStrongerThan(ordering, AtomicOrdering::Acquire) )
+ return true;
+ }
+
+ return false;
+}
+
void CDSPass::chooseInstructionsToInstrument(
SmallVectorImpl<Instruction *> &Local, SmallVectorImpl<Instruction *> &All,
const DataLayout &DL) {
}*/
bool CDSPass::runOnFunction(Function &F) {
- if (F.getName() == "main") {
- F.setName("user_main");
- errs() << "main replaced by user_main\n";
- }
-
initializeCallbacks( *F.getParent() );
SmallVector<Instruction*, 8> AllLoadsAndStores;
SmallVector<Instruction*, 8> LocalLoadsAndStores;
bool HasVolatile = false;
const DataLayout &DL = F.getParent()->getDataLayout();
+ // instrumentLoops(F);
+
for (auto &BB : F) {
for (auto &Inst : BB) {
- if ( (&Inst)->isAtomic() || isAtomicCall(&Inst) ) {
+ if ( (&Inst)->isAtomic() ) {
AtomicAccesses.push_back(&Inst);
HasAtomic = true;
+
+ if (shouldInstrumentBeforeAtomics(&Inst)) {
+ chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores,
+ DL);
+ }
+ } else if (isAtomicCall(&Inst) ) {
+ AtomicAccesses.push_back(&Inst);
+ HasAtomic = true;
+ chooseInstructionsToInstrument(LocalLoadsAndStores, AllLoadsAndStores,
+ DL);
} else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) {
LoadInst *LI = dyn_cast<LoadInst>(&Inst);
StoreInst *SI = dyn_cast<StoreInst>(&Inst);
Res |= instrumentMemIntrinsic(Inst);
}
- // Only instrument functions that contain atomics or volatiles
+ // Instrument function entry and exit for functions containing atomics or volatiles
if (Res && ( HasAtomic || HasVolatile) ) {
IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
/* Unused for now
// As such they cannot have regular uses like an instrumentation function and
// it makes no sense to track them as memory.
if (Addr->isSwiftError())
- return false;
+ return false;
int Idx = getMemoryAccessFuncIndex(Addr, DL);
if (Idx < 0)
return false;
-// not supported by CDS yet
-/* if (IsWrite && isVtableAccess(I)) {
- LLVM_DEBUG(dbgs() << " VPTR : " << *I << "\n");
- Value *StoredValue = cast<StoreInst>(I)->getValueOperand();
- // StoredValue may be a vector type if we are storing several vptrs at once.
- // In this case, just take the first element of the vector since this is
- // enough to find vptr races.
- if (isa<VectorType>(StoredValue->getType()))
- StoredValue = IRB.CreateExtractElement(
- StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0));
- if (StoredValue->getType()->isIntegerTy())
- StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy());
- // Call TsanVptrUpdate.
- IRB.CreateCall(TsanVptrUpdate,
- {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
- IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())});
- NumInstrumentedVtableWrites++;
- return true;
- }
-
- if (!IsWrite && isVtableAccess(I)) {
- IRB.CreateCall(TsanVptrLoad,
- IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()));
- NumInstrumentedVtableReads++;
- return true;
- }
-*/
-
- Value *OnAccessFunc = nullptr;
- OnAccessFunc = IsWrite ? CDSStore[Idx] : CDSLoad[Idx];
-
- Type *ArgType = IRB.CreatePointerCast(Addr, Addr->getType())->getType();
+ if (IsWrite && isVtableAccess(I)) {
+ /* TODO
+ LLVM_DEBUG(dbgs() << " VPTR : " << *I << "\n");
+ Value *StoredValue = cast<StoreInst>(I)->getValueOperand();
+ // StoredValue may be a vector type if we are storing several vptrs at once.
+ // In this case, just take the first element of the vector since this is
+ // enough to find vptr races.
+ if (isa<VectorType>(StoredValue->getType()))
+ StoredValue = IRB.CreateExtractElement(
+ StoredValue, ConstantInt::get(IRB.getInt32Ty(), 0));
+ if (StoredValue->getType()->isIntegerTy())
+ StoredValue = IRB.CreateIntToPtr(StoredValue, IRB.getInt8PtrTy());
+ // Call TsanVptrUpdate.
+ IRB.CreateCall(TsanVptrUpdate,
+ {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
+ IRB.CreatePointerCast(StoredValue, IRB.getInt8PtrTy())});
+ NumInstrumentedVtableWrites++;
+ */
+ return true;
+ }
- if ( ArgType != Int8PtrTy && ArgType != Int16PtrTy &&
- ArgType != Int32PtrTy && ArgType != Int64PtrTy ) {
- // if other types of load or stores are passed in
- return false;
+ if (!IsWrite && isVtableAccess(I)) {
+ /* TODO
+ IRB.CreateCall(TsanVptrLoad,
+ IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()));
+ NumInstrumentedVtableReads++;
+ */
+ return true;
}
- IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, Addr->getType()));
+ // TODO: unaligned reads and writes
+ Value *OnAccessFunc = nullptr;
+ OnAccessFunc = IsWrite ? CDSStore[Idx] : CDSLoad[Idx];
+ IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()));
if (IsWrite) NumInstrumentedWrites++;
else NumInstrumentedReads++;
return true;
Value *position = getPosition(I, IRB);
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
- assert( LI->isVolatile() );
Value *Addr = LI->getPointerOperand();
int Idx=getMemoryAccessFuncIndex(Addr, DL);
if (Idx < 0)
return false;
+ const unsigned ByteSize = 1U << Idx;
+ const unsigned BitSize = ByteSize * 8;
+ Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize);
+ Type *PtrTy = Ty->getPointerTo();
+ Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy), position};
- Value *args[] = {Addr, position};
- Instruction* funcInst = CallInst::Create(CDSVolatileLoad[Idx], args);
- ReplaceInstWithInst(LI, funcInst);
+ Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType();
+ Value *C = IRB.CreateCall(CDSVolatileLoad[Idx], Args);
+ Value *Cast = IRB.CreateBitOrPointerCast(C, OrigTy);
+ I->replaceAllUsesWith(Cast);
} else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
- assert( SI->isVolatile() );
Value *Addr = SI->getPointerOperand();
int Idx=getMemoryAccessFuncIndex(Addr, DL);
if (Idx < 0)
return false;
-
- Value *val = SI->getValueOperand();
- Value *args[] = {Addr, val, position};
- Instruction* funcInst = CallInst::Create(CDSVolatileStore[Idx], args);
- ReplaceInstWithInst(SI, funcInst);
+ const unsigned ByteSize = 1U << Idx;
+ const unsigned BitSize = ByteSize * 8;
+ Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize);
+ Type *PtrTy = Ty->getPointerTo();
+ Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy),
+ IRB.CreateBitOrPointerCast(SI->getValueOperand(), Ty),
+ position};
+ CallInst *C = CallInst::Create(CDSVolatileStore[Idx], Args);
+ ReplaceInstWithInst(I, C);
} else {
return false;
}
}
Value *position = getPosition(I, IRB);
-
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
Value *Addr = LI->getPointerOperand();
int Idx=getMemoryAccessFuncIndex(Addr, DL);
int atomic_order_index = getAtomicOrderIndex(LI->getOrdering());
Value *order = ConstantInt::get(OrdTy, atomic_order_index);
- Value *args[] = {Addr, order, position};
- Instruction* funcInst = CallInst::Create(CDSAtomicLoad[Idx], args);
+ Value *Args[] = {Addr, order, position};
+ Instruction* funcInst = CallInst::Create(CDSAtomicLoad[Idx], Args);
ReplaceInstWithInst(LI, funcInst);
} else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
Value *Addr = SI->getPointerOperand();
int atomic_order_index = getAtomicOrderIndex(SI->getOrdering());
Value *val = SI->getValueOperand();
Value *order = ConstantInt::get(OrdTy, atomic_order_index);
- Value *args[] = {Addr, val, order, position};
- Instruction* funcInst = CallInst::Create(CDSAtomicStore[Idx], args);
+ Value *Args[] = {Addr, val, order, position};
+ Instruction* funcInst = CallInst::Create(CDSAtomicStore[Idx], Args);
ReplaceInstWithInst(SI, funcInst);
} else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) {
Value *Addr = RMWI->getPointerOperand();
int atomic_order_index = getAtomicOrderIndex(RMWI->getOrdering());
Value *val = RMWI->getValOperand();
Value *order = ConstantInt::get(OrdTy, atomic_order_index);
- Value *args[] = {Addr, val, order, position};
- Instruction* funcInst = CallInst::Create(CDSAtomicRMW[RMWI->getOperation()][Idx], args);
+ Value *Args[] = {Addr, val, order, position};
+ Instruction* funcInst = CallInst::Create(CDSAtomicRMW[RMWI->getOperation()][Idx], Args);
ReplaceInstWithInst(RMWI, funcInst);
} else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) {
IRBuilder<> IRB(CASI);
Value *args[] = {ptr, val, position};
+ if (!checkSignature(CDSAtomicInit[Idx], args))
+ return false;
+
Instruction* funcInst = CallInst::Create(CDSAtomicInit[Idx], args);
ReplaceInstWithInst(CI, funcInst);
-
return true;
}
order = ConstantInt::get(OrdTy,
(int) AtomicOrderingCABI::seq_cst);
Value *args[] = {ptr, order, position};
-
+
+ if (!checkSignature(CDSAtomicLoad[Idx], args))
+ return false;
+
Instruction* funcInst = CallInst::Create(CDSAtomicLoad[Idx], args);
ReplaceInstWithInst(CI, funcInst);
Value *order = IRB.CreateBitOrPointerCast(parameters[1], OrdTy);
Value *args[] = {ptr, order, position};
+ // Without this check, gdax does not compile :(
if (!CI->getType()->isPointerTy()) {
return false;
}
+ if (!checkSignature(CDSAtomicLoad[Idx], args))
+ return false;
+
CallInst *funcInst = IRB.CreateCall(CDSAtomicLoad[Idx], args);
Value *RetVal = IRB.CreateIntToPtr(funcInst, CI->getType());
order = ConstantInt::get(OrdTy,
(int) AtomicOrderingCABI::seq_cst);
Value *args[] = {ptr, val, order, position};
-
+
+ if (!checkSignature(CDSAtomicStore[Idx], args))
+ return false;
+
Instruction* funcInst = CallInst::Create(CDSAtomicStore[Idx], args);
ReplaceInstWithInst(CI, funcInst);
return true;
} else if (funName.contains("atomic") &&
funName.contains("store") ) {
- // does this version of call always have an atomic order as an argument?
- Value *OrigVal = parameters[1];
+ // Does this version of call always have an atomic order as an argument?
+ if (parameters.size() < 3)
+ return false;
+ Value *OrigVal = parameters[1];
Value *ptr = IRB.CreatePointerCast(OrigPtr, PtrTy);
+
Value *val;
if (OrigVal->getType()->isPtrOrPtrVectorTy())
val = IRB.CreatePointerCast(OrigVal, Ty);
Value *order = IRB.CreateBitOrPointerCast(parameters[2], OrdTy);
Value *args[] = {ptr, val, order, position};
+ if (!checkSignature(CDSAtomicStore[Idx], args))
+ return false;
+
Instruction* funcInst = CallInst::Create(CDSAtomicStore[Idx], args);
ReplaceInstWithInst(CI, funcInst);
if (funName.contains("atomic_fetch_") ||
funName.contains("atomic_exchange")) {
- /* TODO: implement stricter function name checking */
- if (funName.contains("non"))
- return false;
-
bool isExplicit = funName.contains("_explicit");
Value *OrigVal = parameters[1];
order = ConstantInt::get(OrdTy,
(int) AtomicOrderingCABI::seq_cst);
Value *args[] = {ptr, val, order, position};
-
+
+ if (!checkSignature(CDSAtomicRMW[op][Idx], args))
+ return false;
+
Instruction* funcInst = CallInst::Create(CDSAtomicRMW[op][Idx], args);
ReplaceInstWithInst(CI, funcInst);
} else if (funName.contains("exchange") &&
!funName.contains("compare_exchange") ) {
if (CI->getType()->isPointerTy()) {
- // Can not deal with this now
+ /**
+ * TODO: instrument the following case
+ * mcs-lock.h
+ * std::atomic<struct T *> m_tail;
+ * struct T * me;
+ * struct T * pred = m_tail.exchange(me, memory_order_*);
+ */
errs() << "atomic exchange captured. Not implemented yet. ";
errs() << "See source file :";
getPosition(CI, IRB, true);
Value *order = IRB.CreateBitOrPointerCast(parameters[2], OrdTy);
Value *args[] = {ptr, val, order, position};
+
int op = AtomicRMWInst::Xchg;
-
+
+ if (!checkSignature(CDSAtomicRMW[op][Idx], args))
+ return false;
+
Instruction* funcInst = CallInst::Create(CDSAtomicRMW[op][Idx], args);
ReplaceInstWithInst(CI, funcInst);
+
+ return true;
}
/* atomic_compare_exchange_*;
Value *args[] = {Addr, CmpOperand, NewOperand,
order_succ, order_fail, position};
-
+
+ if (!checkSignature(CDSAtomicCAS_V2[Idx], args))
+ return false;
+
Instruction* funcInst = CallInst::Create(CDSAtomicCAS_V2[Idx], args);
ReplaceInstWithInst(CI, funcInst);
Value *args[] = {Addr, CmpOperand, NewOperand,
order_succ, order_fail, position};
+
+ if (!checkSignature(CDSAtomicCAS_V2[Idx], args))
+ return false;
+
Instruction* funcInst = CallInst::Create(CDSAtomicCAS_V2[Idx], args);
ReplaceInstWithInst(CI, funcInst);
return Idx;
}
+bool CDSPass::instrumentLoops(Function &F)
+{
+ DominatorTree DT(F);
+ LoopInfo LI(DT);
+
+ SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
+ bool instrumented = false;
+
+ // Do a post-order traversal of the loops so that counter updates can be
+ // iteratively hoisted outside the loop nest.
+ for (auto *Loop : llvm::reverse(Loops)) {
+ bool instrument_loop = false;
+
+ // Iterator over loop blocks and search for atomics and volatiles
+ Loop::block_iterator it;
+ for (it = Loop->block_begin(); it != Loop->block_end(); it++) {
+ BasicBlock * block = *it;
+ for (auto &Inst : *block) {
+ if ( (&Inst)->isAtomic() ) {
+ instrument_loop = true;
+ break;
+ } else if (isAtomicCall(&Inst)) {
+ instrument_loop = true;
+ break;
+ } else if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) {
+ LoadInst *LI = dyn_cast<LoadInst>(&Inst);
+ StoreInst *SI = dyn_cast<StoreInst>(&Inst);
+ bool isVolatile = ( LI ? LI->isVolatile() : SI->isVolatile() );
+
+ if (isVolatile) {
+ instrument_loop = true;
+ break;
+ }
+ }
+ }
+
+ if (instrument_loop)
+ break;
+ }
+
+ if (instrument_loop) {
+ // TODO: what to instrument?
+ errs() << "Function: " << F.getName() << "\n";
+ BasicBlock * header = Loop->getHeader();
+ header->dump();
+
+ instrumented = true;
+ }
+ }
+
+ return instrumented;
+}
char CDSPass::ID = 0;
projects / c11llvm.git / blobdiff