From: weiyu Date: Thu, 11 Jul 2019 23:48:50 +0000 (-0700) Subject: Merge branch 'new_fuzzer' into branch-weiyu X-Git-Url: http://demsky.eecs.uci.edu/git/?a=commitdiff_plain;h=7db53a974507e129e6b32980c97f9759d1390711;hp=fdfcb2e993567ebdc484639dd18dfc4aadda2609;p=c11tester.git Merge branch 'new_fuzzer' into branch-weiyu --- diff --git a/Makefile b/Makefile index 05afc05c..e5fc14f7 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ OBJECTS := libthreads.o schedule.o model.o threads.o librace.o action.o \ datarace.o impatomic.o cmodelint.o \ snapshot.o malloc.o mymemory.o common.o mutex.o conditionvariable.o \ context.o execution.o libannotate.o plugins.o pthread.o futex.o fuzzer.o \ - sleeps.o history.o printf.o + sleeps.o history.o funcnode.o funcinst.o printf.o CPPFLAGS += -Iinclude -I. LDFLAGS := -ldl -lrt -rdynamic @@ -36,6 +36,9 @@ README.html: README.md malloc.o: malloc.c $(CC) -fPIC -c malloc.c -DMSPACES -DONLY_MSPACES -DHAVE_MMAP=1 $(CPPFLAGS) -Wno-unused-variable +printf.o: printf.c + $(CC) -fPIC -c printf.c $(CPPFLAGS) + futex.o: futex.cc $(CXX) -fPIC -c futex.cc -std=c++11 $(CPPFLAGS) diff --git a/classlist.h b/classlist.h index e6a5e4ec..d3d9d3d2 100644 --- a/classlist.h +++ b/classlist.h @@ -1,5 +1,6 @@ #ifndef CLASSLIST_H #define CLASSLIST_H +#include #include "stl-model.h" class ClockVector; @@ -13,10 +14,14 @@ class Scheduler; class Thread; class TraceAnalysis; class Fuzzer; +class FuncNode; +class FuncInst; struct model_snapshot_members; struct bug_message; typedef SnapList action_list_t; +typedef SnapList func_id_list_t; +typedef SnapList func_inst_list_t; extern volatile int forklock; #endif diff --git a/cmodelint.cc b/cmodelint.cc index 98f7f89d..c42ffe41 100644 --- a/cmodelint.cc +++ b/cmodelint.cc @@ -369,10 +369,16 @@ void cds_func_entry(const char * funcName) { ModelHistory *history = model->get_history(); if ( !history->getFuncMap()->contains(funcName) ) { + /* add func id to func map */ func_id = history->get_func_counter(); history->incr_func_counter(); - history->getFuncMap()->put(funcName, func_id); + + /* add func id to reverse func map */ + ModelVector * func_map_rev = history->getFuncMapRev(); + if ( func_map_rev->size() <= func_id ) + func_map_rev->resize( func_id + 1 ); + func_map_rev->at(func_id) = funcName; } else { func_id = history->getFuncMap()->get(funcName); } diff --git a/execution.cc b/execution.cc index c9fb39f1..c5c40a91 100644 --- a/execution.cc +++ b/execution.cc @@ -13,6 +13,7 @@ #include "datarace.h" #include "threads-model.h" #include "bugmessage.h" +#include "history.h" #include "fuzzer.h" #define INITIAL_THREAD_ID 0 @@ -62,8 +63,10 @@ ModelExecution::ModelExecution(ModelChecker *m, Scheduler *scheduler) : thrd_last_action(1), thrd_last_fence_release(), priv(new struct model_snapshot_members ()), - mo_graph(new CycleGraph()), - fuzzer(new Fuzzer()) + mo_graph(new CycleGraph()), + fuzzer(new Fuzzer()), + thrd_func_list(), + thrd_func_inst_lists() { /* Initialize a model-checker thread, for special ModelActions */ model_thread = new Thread(get_next_id()); @@ -323,8 +326,9 @@ bool ModelExecution::process_mutex(ModelAction *curr) } //otherwise fall into the lock case case ATOMIC_LOCK: { - if (curr->get_cv()->getClock(state->alloc_tid) <= state->alloc_clock) - assert_bug("Lock access before initialization"); + //TODO: FIND SOME BETTER WAY TO CHECK LOCK INITIALIZED OR NOT + //if (curr->get_cv()->getClock(state->alloc_tid) <= state->alloc_clock) + // assert_bug("Lock access before initialization"); state->locked = get_thread(curr); ModelAction *unlock = get_last_unlock(curr); //synchronize with the previous unlock statement @@ -385,10 +389,7 @@ bool ModelExecution::process_mutex(ModelAction *curr) */ void ModelExecution::process_write(ModelAction *curr) { - w_modification_order(curr); - - get_thread(curr)->set_return_value(VALUE_NONE); } @@ -1537,6 +1538,9 @@ Thread * ModelExecution::take_step(ModelAction *curr) curr = check_current_action(curr); ASSERT(curr); + // model_print("poitner loc: %p, thread: %d, type: %d, order: %d, position: %s\n", curr, curr->get_tid(), curr->get_type(), curr->get_mo(), curr->get_position() ); + model->get_history()->add_func_atomic( curr, curr_thrd->get_id() ); + if (curr_thrd->is_blocked() || curr_thrd->is_complete()) scheduler->remove_thread(curr_thrd); diff --git a/execution.h b/execution.h index f8634ffa..ced21e4c 100644 --- a/execution.h +++ b/execution.h @@ -86,6 +86,9 @@ public: HashTable * getMutexMap() {return &mutex_map;} ModelAction * check_current_action(ModelAction *curr); + SnapVector * get_thrd_func_list() { return &thrd_func_list; } + SnapVector< SnapList *> * get_thrd_func_inst_lists() { return &thrd_func_inst_lists; } + SNAPSHOTALLOC private: int get_execution_number() const; @@ -188,6 +191,22 @@ private: Fuzzer * fuzzer; Thread * action_select_next_thread(const ModelAction *curr) const; + + /* thrd_func_list stores a list of function ids for each thread. + * Each element in thrd_func_list stores the functions that + * thread i has entered and yet to exit from + * + * This data structure is handled by ModelHistory + */ + SnapVector< func_id_list_t * > thrd_func_list; + + /* Keeps track of atomic actions that thread i has performed in some + * function. Index of SnapVector is thread id. SnapList simulates + * the call stack. + * + * This data structure is handled by ModelHistory + */ + SnapVector< SnapList< func_inst_list_t *> *> thrd_func_inst_lists; }; #endif /* __EXECUTION_H__ */ diff --git a/funcinst.cc b/funcinst.cc new file mode 100644 index 00000000..61e3d922 --- /dev/null +++ b/funcinst.cc @@ -0,0 +1,62 @@ +#include "funcinst.h" + +FuncInst::FuncInst(ModelAction *act, FuncNode *func_node) : + collisions() +{ + ASSERT(act); + ASSERT(func_node); + this->position = act->get_position(); + this->location = act->get_location(); + this->type = act->get_type(); + this->func_node = func_node; +} + +/* @param other Preceding FuncInst in the same execution trace + * Add other to predecessors if it has been added + * + * @return false: other is already in predecessors + * true : other is added to precedessors + */ +bool FuncInst::add_pred(FuncInst * other) +{ + func_inst_list_mt::iterator it; + for (it = predecessors.begin(); it != predecessors.end(); it++) { + FuncInst * inst = *it; + if (inst == other) + return false; + } + + predecessors.push_back(other); + return true; +} + +bool FuncInst::add_succ(FuncInst * other) +{ + func_inst_list_mt::iterator it; + for (it = successors.begin(); it != successors.end(); it++) { + FuncInst * inst = *it; + if ( inst == other ) + return false; + } + + successors.push_back(other); + return true; +} + +FuncInst * FuncInst::search_in_collision(ModelAction *act) +{ + action_type type = act->get_type(); + + func_inst_list_mt::iterator it; + for (it = collisions.begin(); it != collisions.end(); it++) { + FuncInst * inst = *it; + if ( inst->get_type() == type ) + return inst; + } + return NULL; +} + +bool FuncInst::is_read() const +{ + return type == ATOMIC_READ || type == ATOMIC_RMWR || type == ATOMIC_RMWRCAS; +} diff --git a/funcinst.h b/funcinst.h new file mode 100644 index 00000000..85b9c20e --- /dev/null +++ b/funcinst.h @@ -0,0 +1,45 @@ +#include "action.h" +#include "hashtable.h" + +class ModelAction; + +typedef ModelList func_inst_list_mt; + +class FuncInst { +public: + FuncInst(ModelAction *act, FuncNode *func_node); + ~FuncInst(); + + //ModelAction * get_action() const { return action; } + const char * get_position() const { return position; } + void * get_location() const { return location; } + action_type get_type() const { return type; } + FuncNode * get_func_node() const { return func_node; } + + bool add_pred(FuncInst * other); + bool add_succ(FuncInst * other); + + FuncInst * search_in_collision(ModelAction *act); + + func_inst_list_mt * get_collisions() { return &collisions; } + func_inst_list_mt * get_preds() { return &predecessors; } + func_inst_list_mt * get_succs() { return &successors; } + + bool is_read() const; + + MEMALLOC +private: + //ModelAction * const action; + const char * position; + void * location; + action_type type; + FuncNode * func_node; + + /* collisions store a list of FuncInsts with the same position + * but different action types. For example, CAS is broken down + * as three different atomic operations in cmodelint.cc */ + func_inst_list_mt collisions; + + func_inst_list_mt predecessors; + func_inst_list_mt successors; +}; diff --git a/funcnode.cc b/funcnode.cc new file mode 100644 index 00000000..ce91815f --- /dev/null +++ b/funcnode.cc @@ -0,0 +1,100 @@ +#include "funcnode.h" + +FuncNode::FuncNode() : + func_inst_map(), + inst_list(), + entry_insts() +{} + +/* Check whether FuncInst with the same type, position, and location + * as act has been added to func_inst_map or not. If so, return it; + * if not, add it and return it. + * + * @return FuncInst with the same type, position, and location as act */ +FuncInst * FuncNode::get_or_add_action(ModelAction *act) +{ + ASSERT(act); + const char * position = act->get_position(); + + /* Actions THREAD_CREATE, THREAD_START, THREAD_YIELD, THREAD_JOIN, + * THREAD_FINISH, PTHREAD_CREATE, PTHREAD_JOIN, + * ATOMIC_LOCK, ATOMIC_TRYLOCK, and ATOMIC_UNLOCK are not tagged with their + * source line numbers + */ + if (position == NULL) + return NULL; + + if ( func_inst_map.contains(position) ) { + FuncInst * inst = func_inst_map.get(position); + + if (inst->get_type() != act->get_type() ) { + // model_print("action with a different type occurs at line number %s\n", position); + FuncInst * func_inst = inst->search_in_collision(act); + + if (func_inst != NULL) { + // return the FuncInst found in the collision list + return func_inst; + } + + func_inst = new FuncInst(act, this); + inst->get_collisions()->push_back(func_inst); + inst_list.push_back(func_inst); // delete? + if (func_inst->is_read()) + group_reads_by_loc(func_inst); + + return func_inst; + } + + return inst; + } + + FuncInst * func_inst = new FuncInst(act, this); + func_inst_map.put(position, func_inst); + inst_list.push_back(func_inst); + + if (func_inst->is_read()) + group_reads_by_loc(func_inst); + + return func_inst; +} + +void FuncNode::add_entry_inst(FuncInst * inst) +{ + if (inst == NULL) + return; + + func_inst_list_mt::iterator it; + for (it = entry_insts.begin(); it != entry_insts.end(); it++) { + if (inst == *it) + return; + } + + entry_insts.push_back(inst); +} + +/* group atomic read actions by memory location */ +void FuncNode::group_reads_by_loc(FuncInst * inst) +{ + ASSERT(inst); + if ( !inst->is_read() ) + return; + + void * location = inst->get_location(); + + func_inst_list_mt * reads; + if ( !reads_by_loc.contains(location) ) { + reads = new func_inst_list_mt(); + reads->push_back(inst); + reads_by_loc.put(location, reads); + return; + } + + reads = reads_by_loc.get(location); + func_inst_list_mt::iterator it; + for (it = reads->begin(); it != reads->end(); it++) { + if (inst == *it) + return; + } + + reads->push_back(inst); +} diff --git a/funcnode.h b/funcnode.h new file mode 100644 index 00000000..51f19f0c --- /dev/null +++ b/funcnode.h @@ -0,0 +1,49 @@ +#include "action.h" +#include "funcinst.h" +#include "hashtable.h" + +class ModelAction; + +typedef ModelList func_inst_list_mt; + +class FuncNode { +public: + FuncNode(); + ~FuncNode(); + + uint32_t get_func_id() { return func_id; } + const char * get_func_name() { return func_name; } + void set_func_id(uint32_t id) { func_id = id; } + void set_func_name(const char * name) { func_name = name; } + + FuncInst * get_or_add_action(ModelAction *act); + + HashTable * getFuncInstMap() { return &func_inst_map; } + func_inst_list_mt * get_inst_list() { return &inst_list; } + func_inst_list_mt * get_entry_insts() { return &entry_insts; } + void add_entry_inst(FuncInst * inst); + + void group_reads_by_loc(FuncInst * inst); + + MEMALLOC +private: + uint32_t func_id; + const char * func_name; + + /* Use source line number as the key of hashtable, to check if + * atomic operation with this line number has been added or not + * + * To do: cds_atomic_compare_exchange contains three atomic operations + * that are feeded with the same source line number by llvm pass + */ + HashTable func_inst_map; + + /* list of all atomic actions in this function */ + func_inst_list_mt inst_list; + + /* possible entry atomic actions in this function */ + func_inst_list_mt entry_insts; + + /* group atomic read actions by memory location */ + HashTable reads_by_loc; +}; diff --git a/hashtable.h b/hashtable.h index a16a24c9..20fa56e7 100644 --- a/hashtable.h +++ b/hashtable.h @@ -8,6 +8,7 @@ #include #include #include +#include "stl-model.h" #include "mymemory.h" #include "common.h" @@ -56,6 +57,8 @@ public: HashTable(unsigned int initialcapacity = 1024, double factor = 0.5) { // Allocate space for the hash table table = (struct hashlistnode<_Key, _Val> *)_calloc(initialcapacity, sizeof(struct hashlistnode<_Key, _Val>)); + keyset = (_Key *)_calloc(initialcapacity, sizeof(_Key)); + keyset_end = keyset; loadfactor = factor; capacity = initialcapacity; capacitymask = initialcapacity - 1; @@ -67,6 +70,7 @@ public: /** @brief Hash table destructor */ ~HashTable() { _free(table); + _free(keyset); } /** Override: new operator */ @@ -92,6 +96,7 @@ public: /** @brief Reset the table to its initial state. */ void reset() { memset(table, 0, capacity * sizeof(struct hashlistnode<_Key, _Val>)); + memset(keyset, 0, capacity * sizeof(_Key)); size = 0; } @@ -122,6 +127,8 @@ public: search->key = key; search->val = val; + *keyset_end = key; + keyset_end++; size++; } @@ -176,6 +183,9 @@ public: void resize(unsigned int newsize) { struct hashlistnode<_Key, _Val> *oldtable = table; struct hashlistnode<_Key, _Val> *newtable; + _Key *oldkeyset = keyset; + _Key *newkeyset; + unsigned int oldcapacity = capacity; if ((newtable = (struct hashlistnode<_Key, _Val> *)_calloc(newsize, sizeof(struct hashlistnode<_Key, _Val>))) == NULL) { @@ -183,7 +193,15 @@ public: exit(EXIT_FAILURE); } + if ((newkeyset = (_Key *)_calloc(newsize, sizeof(_Key))) == NULL ) { + model_print("calloc error %s %d\n", __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + table = newtable; // Update the global hashtable upon resize() + keyset = newkeyset; + keyset_end = newkeyset; + capacity = newsize; capacitymask = newsize - 1; @@ -207,11 +225,25 @@ public: search->val = bin->val; } + memcpy(keyset, oldkeyset, size * sizeof(_Key)); // copy keyset + keyset_end = keyset + size; // pointer arithmetic + _free(oldtable); // Free the memory of the old hash table + _free(oldkeyset); + } + + _Key * getKeySet() { + return keyset; + } + + unsigned int getSize() { + return size; } private: struct hashlistnode<_Key, _Val> *table; + _Key *keyset; + _Key *keyset_end; unsigned int capacity; unsigned int size; unsigned int capacitymask; diff --git a/history.cc b/history.cc index e0560bad..d81c50fb 100644 --- a/history.cc +++ b/history.cc @@ -1,36 +1,173 @@ #include #include "history.h" #include "action.h" +#include "funcnode.h" +#include "common.h" +#include "model.h" +#include "execution.h" + + +/** @brief Constructor */ ModelHistory::ModelHistory() : - func_id(1), /* function id starts with 1 */ + func_counter(0), /* function id starts with 0 */ func_map(), - func_history(), - work_list() + func_map_rev(), + func_atomics() {} void ModelHistory::enter_function(const uint32_t func_id, thread_id_t tid) { - if ( !work_list.contains(tid) ) { - // This thread has not been pushed to work_list - SnapList * func_list = new SnapList(); - func_list->push_back(func_id); - work_list.put(tid, func_list); - } else { - SnapList * func_list = work_list.get(tid); - func_list->push_back(func_id); + //model_print("thread %d entering func %d\n", tid, func_id); + uint32_t id = id_to_int(tid); + SnapVector * thrd_func_list = model->get_execution()->get_thrd_func_list(); + SnapVector< SnapList *> * + thrd_func_inst_lists = model->get_execution()->get_thrd_func_inst_lists(); + + if ( thrd_func_list->size() <= id ) { + thrd_func_list->resize( id + 1 ); + thrd_func_inst_lists->resize( id + 1 ); + } + + func_id_list_t * func_list = thrd_func_list->at(id); + SnapList * func_inst_lists = thrd_func_inst_lists->at(id); + + if (func_list == NULL) { + func_list = new func_id_list_t(); + thrd_func_list->at(id) = func_list; + } + + if (func_inst_lists == NULL) { + func_inst_lists = new SnapList< func_inst_list_t *>(); + thrd_func_inst_lists->at(id) = func_inst_lists; } + + func_list->push_back(func_id); + func_inst_lists->push_back( new func_inst_list_t() ); } void ModelHistory::exit_function(const uint32_t func_id, thread_id_t tid) { - SnapList * func_list = work_list.get(tid); + uint32_t id = id_to_int(tid); + SnapVector * thrd_func_list = model->get_execution()->get_thrd_func_list(); + SnapVector< SnapList *> * + thrd_func_inst_lists = model->get_execution()->get_thrd_func_inst_lists(); + + func_id_list_t * func_list = thrd_func_list->at(id); + SnapList * func_inst_lists = thrd_func_inst_lists->at(id); + uint32_t last_func_id = func_list->back(); if (last_func_id == func_id) { func_list->pop_back(); + + func_inst_list_t * curr_inst_list = func_inst_lists->back(); + link_insts(curr_inst_list); + + func_inst_lists->pop_back(); } else { model_print("trying to exit with a wrong function id\n"); model_print("--- last_func: %d, func_id: %d\n", last_func_id, func_id); } + //model_print("thread %d exiting func %d\n", tid, func_id); +} + +void ModelHistory::add_func_atomic(ModelAction *act, thread_id_t tid) +{ + /* return if thread i has not entered any function or has exited + from all functions */ + SnapVector * thrd_func_list = model->get_execution()->get_thrd_func_list(); + SnapVector< SnapList *> * + thrd_func_inst_lists = model->get_execution()->get_thrd_func_inst_lists(); + + uint32_t id = id_to_int(tid); + if ( thrd_func_list->size() <= id ) + return; + else if (thrd_func_list->at(id) == NULL) + return; + + /* get the function id that thread i is currently in */ + func_id_list_t * func_list = thrd_func_list->at(id); + SnapList * func_inst_lists = thrd_func_inst_lists->at(id); + + uint32_t func_id = func_list->back(); + + if ( func_atomics.size() <= func_id ) + func_atomics.resize( func_id + 1 ); + + FuncNode * func_node = func_atomics[func_id]; + if (func_node == NULL) { + const char * func_name = func_map_rev[func_id]; + func_node = new FuncNode(); + func_node->set_func_id(func_id); + func_node->set_func_name(func_name); + + func_atomics[func_id] = func_node; + } + + FuncInst * inst = func_node->get_or_add_action(act); + if (inst != NULL) { + func_inst_list_t * curr_inst_list = func_inst_lists->back(); + + ASSERT(curr_inst_list != NULL); + curr_inst_list->push_back(inst); + } +} + +/* Link FuncInsts in a list - add one FuncInst to another's predecessors and successors */ +void ModelHistory::link_insts(func_inst_list_t * inst_list) +{ + if (inst_list == NULL) + return; + + func_inst_list_t::iterator it = inst_list->begin(); + func_inst_list_t::iterator prev; + + /* add the first instruction to the list of entry insts */ + FuncInst * entry_inst = *it; + FuncNode * func_node = entry_inst->get_func_node(); + func_node->add_entry_inst(entry_inst); + + it++; + while (it != inst_list->end()) { + prev = it; + prev--; + + FuncInst * prev_inst = *prev; + FuncInst * curr_inst = *it; + + prev_inst->add_succ(curr_inst); + curr_inst->add_pred(prev_inst); + + it++; + } +} + +void ModelHistory::print() +{ + for (uint32_t i = 0; i < func_atomics.size(); i++ ) { + FuncNode * funcNode = func_atomics[i]; + if (funcNode == NULL) + continue; + + func_inst_list_mt * entry_insts = funcNode->get_entry_insts(); + + model_print("function %s has entry actions\n", funcNode->get_func_name()); + func_inst_list_mt::iterator it; + for (it = entry_insts->begin(); it != entry_insts->end(); it++) { + FuncInst *inst = *it; + model_print("type: %d, at: %s\n", inst->get_type(), inst->get_position()); + } + +/* + func_inst_list_mt * inst_list = funcNode->get_inst_list(); + + model_print("function %s has following actions\n", funcNode->get_func_name()); + func_inst_list_mt::iterator it; + for (it = inst_list->begin(); it != inst_list->end(); it++) { + FuncInst *inst = *it; + model_print("type: %d, at: %s\n", inst->get_type(), inst->get_position()); + } +*/ + } } diff --git a/history.h b/history.h index 179439ce..b6707f9f 100644 --- a/history.h +++ b/history.h @@ -1,39 +1,37 @@ #include "stl-model.h" #include "common.h" #include "hashtable.h" -#include "modeltypes.h" - -/* forward declaration */ -class ModelAction; - -typedef ModelList action_mlist_t; +#include "threads-model.h" class ModelHistory { public: ModelHistory(); + ~ModelHistory(); void enter_function(const uint32_t func_id, thread_id_t tid); void exit_function(const uint32_t func_id, thread_id_t tid); - uint32_t get_func_counter() { return func_id; } - void incr_func_counter() { func_id++; } + uint32_t get_func_counter() { return func_counter; } + void incr_func_counter() { func_counter++; } + + void add_func_atomic(ModelAction *act, thread_id_t tid); - HashTable * getFuncMap() { return &func_map; } - HashTable * getFuncHistory() { return &func_history; } + HashTable * getFuncMap() { return &func_map; } + ModelVector * getFuncMapRev() { return &func_map_rev; } + ModelVector * getFuncAtomics() { return &func_atomics; } + + void link_insts(func_inst_list_t * inst_list); void print(); + MEMALLOC private: - uint32_t func_id; - - /* map function names to integer ids */ - HashTable func_map; + uint32_t func_counter; - HashTable func_history; + /* map function names to integer ids */ + HashTable func_map; + /* map integer ids to function names */ + ModelVector func_map_rev; - /* work_list stores a list of function ids for each thread - * SnapList is intended to be used as a stack storing - * the functions that thread i has entered and yet to exit from - */ - HashTable *, uintptr_t, 4> work_list; + ModelVector func_atomics; };